Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(credentials): delete-path follow-ups from #14215 review (SKY-13271) (#7946)
Suchintand025fb210a7f
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 4396f0c27..520641fa1 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -1320,8 +1320,8 @@ class AgentFunction:organization_id: str,item_id: str,vault_type: CredentialVaultType,- ) -> None:- return+ ) -> bool:+ return Falseasync def process_registered_credential_item(self,diff --git a/skyvern/forge/sdk/api/real_azure.py b/skyvern/forge/sdk/api/real_azure.pyindex 120fd1a48..b3cddd559 100644--- a/skyvern/forge/sdk/api/real_azure.py+++ b/skyvern/forge/sdk/api/real_azure.py@@ -5,7 +5,7 @@ from mimetypes import add_type, guess_typefrom typing import IO, Selfimport structlog-from azure.core.exceptions import HttpResponseError, ResourceNotFoundError+from azure.core.exceptions import ResourceNotFoundErrorfrom azure.identity.aio import ClientSecretCredential, DefaultAzureCredentialfrom azure.keyvault.secrets.aio import SecretClientfrom azure.storage.blob import BlobSasPermissions, ContentSettings, StandardBlobTier, generate_blob_sas@@ -47,9 +47,11 @@ class RealAsyncAzureVaultClient(AsyncAzureVaultClient):try:secret = await secret_client.get_secret(secret_name)return secret.value- except Exception as e:- LOG.exception("Failed to get secret from Azure Key Vault.", secret_name=secret_name, error=e)+ except ResourceNotFoundError:return None+ except Exception:+ LOG.exception("Failed to get secret from Azure Key Vault.", secret_name=secret_name)+ raisefinally:await secret_client.close()@@ -70,34 +72,12 @@ class RealAsyncAzureVaultClient(AsyncAzureVaultClient):# enabled and readable; fail the write so it retries instead of silently orphaning them.if enumerated_any:raise- except HttpResponseError as e:- # An identity permitted to set but not list secret versions raises a 401/403 here, and- # disabling priors is best-effort, so proceed rather than failing every write. Re-raise any- # other status, or a failure after enumerating any versions (a first page of only- # current/already-disabled versions leaves `previous_versions` empty yet still means the- # error is partial), so the write fails and retries instead of silently leaving the- # un-enumerated prior versions enabled and readable.- if enumerated_any or e.status_code not in (401, 403):- raise- LOG.warning(- "Could not enumerate prior Azure secret versions to disable; proceeding with the write",- secret_name=secret_name,- exc_info=True,- )secret = await secret_client.set_secret(secret_name, secret_value)# Azure Key Vault leaves every prior version enabled and readable by explicit version id; disable# the ones that predate this write so a rotated/detached secret value can't be read back.for version in previous_versions:- try:- await secret_client.update_secret_properties(secret_name, version, enabled=False)- except Exception:- LOG.warning(- "Failed to disable prior Azure secret version",- secret_name=secret_name,- version=version,- exc_info=True,- )+ await secret_client.update_secret_properties(secret_name, version, enabled=False)return secret.nameexcept Exception as e:LOG.exception("Failed to create secret from Azure Key Vault.", secret_name=secret_name, error=e)diff --git a/skyvern/forge/sdk/core/aiohttp_helper.py b/skyvern/forge/sdk/core/aiohttp_helper.pyindex 8d1a96b75..7d9d434a7 100644--- a/skyvern/forge/sdk/core/aiohttp_helper.py+++ b/skyvern/forge/sdk/core/aiohttp_helper.py@@ -362,6 +362,7 @@ async def aiohttp_delete() -> dict[str, Any]:async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout)) as session:count = 0+ last_exception: Exception | None = Nonewhile count <= retry:try:async with session.delete(@@ -376,8 +377,11 @@ async def aiohttp_delete(raise HttpException(response.status, url)LOG.error(f"Failed to delete data from {url}", status_code=response.status)return {}- except Exception:+ except Exception as exc:+ last_exception = excif retry_timeout > 0:await asyncio.sleep(retry_timeout)count += 1+ if last_exception is not None:+ raise last_exceptionraise Exception(f"Failed to delete data from {url}")diff --git a/skyvern/forge/sdk/services/credential/azure_credential_vault_service.py b/skyvern/forge/sdk/services/credential/azure_credential_vault_service.pyindex 41e827493..2eda4dea6 100644--- a/skyvern/forge/sdk/services/credential/azure_credential_vault_service.py+++ b/skyvern/forge/sdk/services/credential/azure_credential_vault_service.py@@ -117,16 +117,38 @@ class AzureCredentialVaultService(CredentialVaultService):self,credential: Credential,) -> None:- # Empty the secret value before deleting the DB row: a transient vault failure then aborts the whole- # delete with the row (and its item_id) intact for a retry, instead of orphaning a still-readable- # secret with no DB pointer left to scrub it. The slow full-vault delete is deferred to- # post_delete_credential_item so customers do not have to wait.- await self._client.create_or_update_secret(- vault_name=self._vault_name,+ previous_secret_value = await self._client.get_secret(secret_name=credential.item_id,- secret_value="",+ vault_name=self._vault_name,)- await app.DATABASE.credentials.delete_credential(credential.credential_id, credential.organization_id)+ if previous_secret_value is None:+ # Vault item already gone (prior partial delete); nothing to scrub, and the row must stay deletable.+ await app.DATABASE.credentials.delete_credential(credential.credential_id, credential.organization_id)+ return++ try:+ await self._client.create_or_update_secret(+ vault_name=self._vault_name,+ secret_name=credential.item_id,+ secret_value="",+ )+ await app.DATABASE.credentials.delete_credential(credential.credential_id, credential.organization_id)+ except BaseException:+ try:+ await self._client.create_or_update_secret(+ vault_name=self._vault_name,+ secret_name=credential.item_id,+ secret_value=previous_secret_value,+ )+ except BaseException:+ LOG.error(+ "Failed to restore Azure vault secret after credential deletion failed",+ credential_id=credential.credential_id,+ item_id=credential.item_id,+ organization_id=credential.organization_id,+ exc_info=True,+ )+ raiseasync def post_delete_credential_item(self, item_id: str, _organization_id: str | None = None) -> bool:"""diff --git a/skyvern/forge/sdk/services/credential/bitwarden_credential_service.py b/skyvern/forge/sdk/services/credential/bitwarden_credential_service.pyindex 62f654ead..3ff901646 100644--- a/skyvern/forge/sdk/services/credential/bitwarden_credential_service.py+++ b/skyvern/forge/sdk/services/credential/bitwarden_credential_service.py@@ -1,6 +1,7 @@import structlogfrom fastapi import HTTPException+from skyvern.exceptions import HttpExceptionfrom skyvern.forge import appfrom skyvern.forge.sdk.schemas.credentials import (CreateCredentialRequest,@@ -110,7 +111,47 @@ class BitwardenCredentialVaultService(CredentialVaultService):raise HTTPException(status_code=404, detail="Credential account not found. It might have been deleted.")await app.DATABASE.credentials.delete_credential(credential.credential_id, credential.organization_id)- await BitwardenService.delete_credential_item(credential.item_id)+ await self._run_task_to_completion(+ self._delete_credential_item_or_enqueue_cleanup(credential),+ cancellation_failure_message="Bitwarden vault-item cleanup failed while the delete was being cancelled",+ log_context={+ "credential_id": credential.credential_id,+ "item_id": credential.item_id,+ "organization_id": credential.organization_id,+ },+ )++ async def _delete_credential_item_or_enqueue_cleanup(self, credential: Credential) -> None:+ try:+ await BitwardenService.delete_credential_item(credential.item_id)+ except BaseException as exc:+ if isinstance(exc, HttpException) and exc.status_code == 404:+ return+ try:+ cleanup_enqueued = await self._enqueue_orphaned_vault_item_cleanup(+ organization_id=credential.organization_id,+ item_id=credential.item_id,+ vault_type=CredentialVaultType.BITWARDEN,+ )+ except Exception:+ if isinstance(exc, Exception):+ raise+ # Preserve provider cancellation even when its fallback enqueue also fails.+ else:+ log = LOG.warning if cleanup_enqueued else LOG.error+ message = (+ "Bitwarden vault-item delete failed after DB row deletion; enqueued durable cleanup"+ if cleanup_enqueued+ else "Bitwarden vault-item delete failed after DB row deletion; durable cleanup unavailable"+ )+ log(+ message,+ organization_id=credential.organization_id,+ item_id=credential.item_id,+ error_type=type(exc).__name__,+ )+ if not isinstance(exc, Exception):+ raiseasync def post_delete_credential_item(self, item_id: str, organization_id: str | None = None) -> bool:try:diff --git a/skyvern/forge/sdk/services/credential/credential_vault_service.py b/skyvern/forge/sdk/services/credential/credential_vault_service.pyindex da9694770..0bd23ac55 100644--- a/skyvern/forge/sdk/services/credential/credential_vault_service.py+++ b/skyvern/forge/sdk/services/credential/credential_vault_service.py@@ -49,6 +49,66 @@ class CredentialVaultService(ABC):async def get_credential_item(self, db_credential: Credential) -> CredentialItem:"""Retrieve the full credential data from the vault."""+ async def _enqueue_orphaned_vault_item_cleanup(+ self,+ *,+ organization_id: str,+ item_id: str,+ vault_type: CredentialVaultType,+ ) -> bool:+ try:+ return bool(+ await app.AGENT_FUNCTION.on_credential_item_orphaned(+ organization_id=organization_id,+ item_id=item_id,+ vault_type=vault_type,+ )+ )+ except Exception:+ LOG.error(+ "Durable vault-item cleanup enqueue failed; item requires manual reconciliation",+ organization_id=organization_id,+ item_id=item_id,+ vault_type=vault_type,+ exc_info=True,+ )+ raise++ async def _run_task_to_completion(+ self,+ awaitable: Awaitable[None],+ *,+ suppress_cancellation: bool = False,+ cancellation_failure_message: str | None = None,+ log_context: dict[str, object] | None = None,+ ) -> None:+ task = asyncio.ensure_future(awaitable)+ cancellation: asyncio.CancelledError | None = None+ while not task.done():+ try:+ await asyncio.shield(task)+ except asyncio.CancelledError as exc:+ cancellation = cancellation or exc+ except Exception:+ if cancellation is None:+ raise+ break++ if cancellation is None:+ task.result()+ return++ try:+ task.result()+ except asyncio.CancelledError:+ pass+ except Exception:+ if cancellation_failure_message:+ LOG.error(cancellation_failure_message, **(log_context or {}), exc_info=True)++ if not suppress_cancellation:+ raise cancellation+async def _reclaim_orphaned_vault_item(self,*,@@ -76,31 +136,17 @@ class CredentialVaultService(ABC):vault_type=vault_type,exc_info=True,)- # The caller invokes this inside `except BaseException: ... ; raise`, so a durable-enqueue- # failure must not escape _run(): it would replace the in-flight exception (e.g. the- # write-lease cancellation the caller is re-raising) with an unrelated one. Log and swallow so- # the caller's bare `raise` always re-raises the original.try:- await app.AGENT_FUNCTION.on_credential_item_orphaned(+ await self._enqueue_orphaned_vault_item_cleanup(organization_id=organization_id,item_id=item_id,vault_type=vault_type,)except Exception:- LOG.error(- "Durable vault-item cleanup enqueue failed; item may stay orphaned until the reaper reclaims it",- organization_id=organization_id,- item_id=item_id,- vault_type=vault_type,- exc_info=True,- )+ # The original failure or cancellation must win over a failed fallback enqueue.+ pass- reclaim = asyncio.ensure_future(_run())- while not reclaim.done():- try:- await asyncio.shield(reclaim)- except asyncio.CancelledError:- continue+ await self._run_task_to_completion(_run(), suppress_cancellation=True)async def _preserve_omitted_credit_card_fields(self,diff --git a/tests/unit/test_aiohttp_helper.py b/tests/unit/test_aiohttp_helper.pyindex 54d74c4d9..d231c18d7 100644--- a/tests/unit/test_aiohttp_helper.py+++ b/tests/unit/test_aiohttp_helper.py@@ -8,7 +8,7 @@ import aiohttpimport pytestfrom skyvern.exceptions import BlockedHost, HttpException-from skyvern.forge.sdk.core.aiohttp_helper import SSRFGuardedResolver, aiohttp_request+from skyvern.forge.sdk.core.aiohttp_helper import SSRFGuardedResolver, aiohttp_delete, aiohttp_requestfrom skyvern.utils.url_validators import MAX_SAFE_REDIRECTS, validate_fetch_url@@ -20,6 +20,30 @@ def public_dns(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr("skyvern.utils.url_validators.socket.getaddrinfo", resolves_public)+async def test_aiohttp_delete_propagates_http_exception_after_retries() -> None:+ url = "https://example.com/object/item/item_test"+ mock_response = AsyncMock()+ mock_response.status = 404+ mock_response.__aenter__ = AsyncMock(return_value=mock_response)+ mock_response.__aexit__ = AsyncMock(return_value=None)++ mock_session = MagicMock()+ mock_session.__aenter__ = AsyncMock(return_value=mock_session)+ mock_session.__aexit__ = AsyncMock(return_value=None)+ mock_session.delete = MagicMock(return_value=mock_response)++ with (+ patch("skyvern.forge.sdk.core.aiohttp_helper.aiohttp.ClientSession", return_value=mock_session),+ pytest.raises(HttpException) as exc_info,+ ):+ await aiohttp_delete(url, retry=1)++ assert exc_info.value.status_code == 404+ assert exc_info.value.url == url+ assert mock_session.delete.call_count == 2++@pytest.mark.asyncioasync def test_ssrf_guarded_resolver_blocks_dns_rebind_after_validation(monkeypatch: pytest.MonkeyPatch) -> None:dns_answers = iter(diff --git a/tests/unit/test_azure_credential_vault_service.py b/tests/unit/test_azure_credential_vault_service.pyindex 41d0c67e3..8b95da84c 100644--- a/tests/unit/test_azure_credential_vault_service.py+++ b/tests/unit/test_azure_credential_vault_service.py@@ -1,10 +1,11 @@from types import SimpleNamespace-from unittest.mock import AsyncMock+from unittest.mock import AsyncMock, MagicMockimport pytestfrom skyvern.forge import appfrom skyvern.forge.sdk.schemas.credentials import PasswordCredential, TotpType+from skyvern.forge.sdk.services.credential import azure_credential_vault_service as service_modulefrom skyvern.forge.sdk.services.credential.azure_credential_vault_service import AzureCredentialVaultService_PASSWORD_METADATA = {"tenant": "north"}@@ -43,6 +44,10 @@ async def test_azure_password_data_image_round_trips_metadata_and_loads_legacy()async def test_azure_delete_scrubs_vault_before_removing_db_row(monkeypatch: pytest.MonkeyPatch) -> None:calls: list[str] = []+ async def read_secret(**_kwargs: object) -> str:+ calls.append("vault_read")+ return "stored-value"+async def scrub(**_kwargs: object) -> str:calls.append("vault_scrub")return "secret-id"@@ -51,6 +56,7 @@ async def test_azure_delete_scrubs_vault_before_removing_db_row(monkeypatch: pytcalls.append("db_delete")client = AsyncMock()+ client.get_secret = AsyncMock(side_effect=read_secret)client.create_or_update_secret = AsyncMock(side_effect=scrub)service = AzureCredentialVaultService(client=client, vault_name="vault")monkeypatch.setattr(@@ -60,13 +66,15 @@ async def test_azure_delete_scrubs_vault_before_removing_db_row(monkeypatch: pytawait service.delete_credential(SimpleNamespace(item_id="item_1", credential_id="cred_1", organization_id="org_1"))# Scrub the still-readable secret before dropping the DB row so a vault failure cannot orphan it.- assert calls == ["vault_scrub", "db_delete"]+ assert calls == ["vault_read", "vault_scrub", "db_delete"]+ client.get_secret.assert_awaited_once_with(secret_name="item_1", vault_name="vault")client.create_or_update_secret.assert_awaited_once_with(vault_name="vault", secret_name="item_1", secret_value="")@pytest.mark.asyncioasync def test_azure_delete_leaves_db_row_when_vault_scrub_fails(monkeypatch: pytest.MonkeyPatch) -> None:client = AsyncMock()+ client.get_secret = AsyncMock(return_value="stored-value")client.create_or_update_secret = AsyncMock(side_effect=RuntimeError("vault down"))service = AzureCredentialVaultService(client=client, vault_name="vault")db_delete = AsyncMock()@@ -80,3 +88,121 @@ async def test_azure_delete_leaves_db_row_when_vault_scrub_fails(monkeypatch: py# A transient vault failure must abort the delete with the DB row (and its item_id) intact so a retry# can still scrub the secret, rather than orphaning a still-readable secret with no DB pointer.db_delete.assert_not_awaited()+++async def test_azure_delete_restores_vault_value_when_scrub_partially_fails(monkeypatch: pytest.MonkeyPatch) -> None:+ stored_value = "stored-value"++ async def scrub_or_restore(*, secret_value: str, **_kwargs: object) -> str:+ nonlocal stored_value+ stored_value = secret_value+ if secret_value == "":+ raise RuntimeError("vault connection dropped after scrub")+ return "item_1"++ client = AsyncMock()+ client.get_secret = AsyncMock(return_value=stored_value)+ client.create_or_update_secret = AsyncMock(side_effect=scrub_or_restore)+ service = AzureCredentialVaultService(client=client, vault_name="vault")+ db_delete = AsyncMock()+ monkeypatch.setattr(app.DATABASE, "credentials", SimpleNamespace(delete_credential=db_delete))++ with pytest.raises(RuntimeError, match="vault connection dropped after scrub"):+ await service.delete_credential(+ SimpleNamespace(item_id="item_1", credential_id="cred_1", organization_id="org_1")+ )++ assert stored_value == "stored-value"+ assert [call.kwargs["secret_value"] for call in client.create_or_update_secret.await_args_list] == [+ "",+ "stored-value",+ ]+ db_delete.assert_not_awaited()+++async def test_azure_delete_leaves_db_row_when_vault_read_fails(monkeypatch: pytest.MonkeyPatch) -> None:+ client = AsyncMock()+ client.get_secret = AsyncMock(side_effect=RuntimeError("vault read unavailable"))+ client.create_or_update_secret = AsyncMock()+ service = AzureCredentialVaultService(client=client, vault_name="vault")+ db_delete = AsyncMock()+ monkeypatch.setattr(app.DATABASE, "credentials", SimpleNamespace(delete_credential=db_delete))++ with pytest.raises(RuntimeError, match="vault read unavailable"):+ await service.delete_credential(+ SimpleNamespace(item_id="item_1", credential_id="cred_1", organization_id="org_1")+ )++ client.create_or_update_secret.assert_not_awaited()+ db_delete.assert_not_awaited()+++async def test_azure_delete_removes_db_row_when_vault_item_already_gone(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ client = AsyncMock()+ client.get_secret = AsyncMock(return_value=None)+ service = AzureCredentialVaultService(client=client, vault_name="vault")+ db_delete = AsyncMock()+ monkeypatch.setattr(app.DATABASE, "credentials", SimpleNamespace(delete_credential=db_delete))++ await service.delete_credential(SimpleNamespace(item_id="item_1", credential_id="cred_1", organization_id="org_1"))++ # A row whose vault item vanished (prior partial delete) must stay deletable: no scrub, straight DB delete.+ client.create_or_update_secret.assert_not_awaited()+ db_delete.assert_awaited_once_with("cred_1", "org_1")+++async def test_azure_delete_restores_vault_value_when_db_delete_fails(monkeypatch: pytest.MonkeyPatch) -> None:+ client = AsyncMock()+ client.get_secret = AsyncMock(return_value="stored-value")+ client.create_or_update_secret = AsyncMock(return_value="item_1")+ service = AzureCredentialVaultService(client=client, vault_name="vault")+ monkeypatch.setattr(+ app.DATABASE,+ "credentials",+ SimpleNamespace(delete_credential=AsyncMock(side_effect=RuntimeError("database unavailable"))),+ )++ with pytest.raises(RuntimeError, match="database unavailable"):+ await service.delete_credential(+ SimpleNamespace(item_id="item_1", credential_id="cred_1", organization_id="org_1")+ )++ assert [call.kwargs["secret_value"] for call in client.create_or_update_secret.await_args_list] == [+ "",+ "stored-value",+ ]+++async def test_azure_delete_reraises_db_error_when_vault_restore_fails(monkeypatch: pytest.MonkeyPatch) -> None:+ client = AsyncMock()+ client.get_secret = AsyncMock(return_value="stored-value")+ client.create_or_update_secret = AsyncMock(side_effect=["item_1", RuntimeError("restore failed")])+ service = AzureCredentialVaultService(client=client, vault_name="vault")+ monkeypatch.setattr(+ app.DATABASE,+ "credentials",+ SimpleNamespace(delete_credential=AsyncMock(side_effect=RuntimeError("database unavailable"))),+ )+ log_error = MagicMock()+ monkeypatch.setattr(service_module.LOG, "error", log_error)++ with pytest.raises(RuntimeError, match="database unavailable"):+ await service.delete_credential(+ SimpleNamespace(item_id="item_1", credential_id="cred_1", organization_id="org_1")+ )++ assert [call.kwargs["secret_value"] for call in client.create_or_update_secret.await_args_list] == [+ "",+ "stored-value",+ ]+ log_error.assert_called_once()+ assert log_error.call_args.kwargs["credential_id"] == "cred_1"+ assert log_error.call_args.kwargs["item_id"] == "item_1"+ assert log_error.call_args.kwargs["exc_info"] is Truediff --git a/tests/unit/test_bitwarden_credential_vault_service.py b/tests/unit/test_bitwarden_credential_vault_service.pyindex 3dfed77a2..263287770 100644--- a/tests/unit/test_bitwarden_credential_vault_service.py+++ b/tests/unit/test_bitwarden_credential_vault_service.py@@ -1,10 +1,11 @@import asynciofrom datetime import datetimefrom types import SimpleNamespace-from unittest.mock import AsyncMock+from unittest.mock import AsyncMock, MagicMockimport pytest+from skyvern.exceptions import HttpExceptionfrom skyvern.forge import appfrom skyvern.forge.sdk.schemas.credentials import (CreateCredentialRequest,@@ -13,6 +14,7 @@ from skyvern.forge.sdk.schemas.credentials import (CredentialVaultType,NonEmptyPasswordCredential,)+from skyvern.forge.sdk.services import bitwarden as bitwarden_modulefrom skyvern.forge.sdk.services.credential import bitwarden_credential_service as service_modulefrom skyvern.forge.sdk.services.credential.bitwarden_credential_service import BitwardenCredentialVaultService@@ -77,6 +79,298 @@ def _prepare_update(return service, delete_item+async def test_delete_credential_removes_db_row_before_vault_item(monkeypatch: pytest.MonkeyPatch) -> None:+ calls: list[str] = []+ monkeypatch.setattr(+ app.DATABASE.credentials,+ "get_organization_bitwarden_collection",+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),+ )+ monkeypatch.setattr(+ service_module.BitwardenService,+ "delete_credential_item",+ AsyncMock(side_effect=lambda *_: calls.append("vault_delete")),+ )+ monkeypatch.setattr(+ app.DATABASE.credentials,+ "delete_credential",+ AsyncMock(side_effect=lambda *_args, **_kwargs: calls.append("db_delete")),+ )++ await BitwardenCredentialVaultService().delete_credential(_credential())++ assert calls == ["db_delete", "vault_delete"]+++async def test_delete_credential_enqueues_cleanup_when_vault_delete_fails(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(+ app.DATABASE.credentials,+ "get_organization_bitwarden_collection",+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),+ )+ monkeypatch.setattr(+ service_module.BitwardenService,+ "delete_credential_item",+ AsyncMock(side_effect=RuntimeError("vault unavailable")),+ )+ db_delete = AsyncMock()+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", db_delete)+ orphaned = AsyncMock()+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)++ await BitwardenCredentialVaultService().delete_credential(_credential())++ db_delete.assert_awaited_once_with("cred_test", "org_test")+ orphaned.assert_awaited_once_with(+ organization_id="org_test",+ item_id="item_old",+ vault_type=CredentialVaultType.BITWARDEN,+ )+++async def test_delete_credential_surfaces_durable_cleanup_enqueue_failure(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(+ app.DATABASE.credentials,+ "get_organization_bitwarden_collection",+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),+ )+ monkeypatch.setattr(+ service_module.BitwardenService,+ "delete_credential_item",+ AsyncMock(side_effect=RuntimeError("vault unavailable")),+ )+ db_delete = AsyncMock()+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", db_delete)+ orphaned = AsyncMock(side_effect=RuntimeError("cleanup enqueue unavailable"))+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)++ with pytest.raises(RuntimeError, match="cleanup enqueue unavailable"):+ await BitwardenCredentialVaultService().delete_credential(_credential())++ db_delete.assert_awaited_once_with("cred_test", "org_test")+ orphaned.assert_awaited_once_with(+ organization_id="org_test",+ item_id="item_old",+ vault_type=CredentialVaultType.BITWARDEN,+ )+++async def test_delete_credential_enqueues_cleanup_when_vault_delete_is_cancelled(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(+ app.DATABASE.credentials,+ "get_organization_bitwarden_collection",+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),+ )+ monkeypatch.setattr(+ service_module.BitwardenService,+ "delete_credential_item",+ AsyncMock(side_effect=asyncio.CancelledError()),+ )+ db_delete = AsyncMock()+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", db_delete)+ orphaned = AsyncMock()+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)++ with pytest.raises(asyncio.CancelledError):+ await BitwardenCredentialVaultService().delete_credential(_credential())++ db_delete.assert_awaited_once_with("cred_test", "org_test")+ orphaned.assert_awaited_once_with(+ organization_id="org_test",+ item_id="item_old",+ vault_type=CredentialVaultType.BITWARDEN,+ )+++async def test_delete_credential_vault_cancellation_wins_over_cleanup_enqueue_failure(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(+ app.DATABASE.credentials,+ "get_organization_bitwarden_collection",+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),+ )+ monkeypatch.setattr(+ service_module.BitwardenService,+ "delete_credential_item",+ AsyncMock(side_effect=asyncio.CancelledError()),+ )+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", AsyncMock())+ orphaned = AsyncMock(side_effect=RuntimeError("cleanup enqueue unavailable"))+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)++ with pytest.raises(asyncio.CancelledError):+ await BitwardenCredentialVaultService().delete_credential(_credential())++ orphaned.assert_awaited_once_with(+ organization_id="org_test",+ item_id="item_old",+ vault_type=CredentialVaultType.BITWARDEN,+ )+++async def test_delete_credential_reports_when_durable_cleanup_is_unavailable(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(+ app.DATABASE.credentials,+ "get_organization_bitwarden_collection",+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),+ )+ monkeypatch.setattr(+ service_module.BitwardenService,+ "delete_credential_item",+ AsyncMock(side_effect=RuntimeError("vault unavailable")),+ )+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", AsyncMock())+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", AsyncMock(return_value=False))+ log_error = MagicMock()+ monkeypatch.setattr(service_module.LOG, "error", log_error)++ await BitwardenCredentialVaultService().delete_credential(_credential())++ log_error.assert_called_once_with(+ "Bitwarden vault-item delete failed after DB row deletion; durable cleanup unavailable",+ organization_id="org_test",+ item_id="item_old",+ error_type="RuntimeError",+ )+++async def test_delete_credential_finishes_vault_delete_before_propagating_caller_cancellation(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ delete_started = asyncio.Event()+ release_delete = asyncio.Event()+ delete_finished = asyncio.Event()++ async def delete_item(_item_id: str) -> None:+ delete_started.set()+ await release_delete.wait()+ delete_finished.set()++ monkeypatch.setattr(+ app.DATABASE.credentials,+ "get_organization_bitwarden_collection",+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),+ )+ monkeypatch.setattr(service_module.BitwardenService, "delete_credential_item", AsyncMock(side_effect=delete_item))+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", AsyncMock())+ orphaned = AsyncMock()+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)++ delete_task = asyncio.create_task(BitwardenCredentialVaultService().delete_credential(_credential()))+ await delete_started.wait()+ delete_task.cancel()+ await asyncio.sleep(0)+ release_delete.set()++ with pytest.raises(asyncio.CancelledError):+ await delete_task++ assert delete_finished.is_set()+ orphaned.assert_not_awaited()+++async def test_delete_credential_cancellation_wins_over_cleanup_enqueue_failure(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ delete_started = asyncio.Event()+ release_delete = asyncio.Event()++ async def delete_item(_item_id: str) -> None:+ delete_started.set()+ await release_delete.wait()+ raise RuntimeError("vault unavailable")++ monkeypatch.setattr(+ app.DATABASE.credentials,+ "get_organization_bitwarden_collection",+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),+ )+ monkeypatch.setattr(service_module.BitwardenService, "delete_credential_item", AsyncMock(side_effect=delete_item))+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", AsyncMock())+ orphaned = AsyncMock(side_effect=RuntimeError("cleanup enqueue unavailable"))+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)++ delete_task = asyncio.create_task(BitwardenCredentialVaultService().delete_credential(_credential()))+ await delete_started.wait()+ delete_task.cancel()+ await asyncio.sleep(0)+ release_delete.set()++ # Both the vault delete and the cleanup enqueue fail while the caller is cancelling: the+ # cancellation must still be what propagates, with the cleanup failure only logged.+ with pytest.raises(asyncio.CancelledError):+ await delete_task++ orphaned.assert_awaited_once()+++async def test_delete_credential_does_not_touch_vault_when_db_delete_fails(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(+ app.DATABASE.credentials,+ "get_organization_bitwarden_collection",+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),+ )+ delete_item = AsyncMock()+ monkeypatch.setattr(service_module.BitwardenService, "delete_credential_item", delete_item)+ monkeypatch.setattr(+ app.DATABASE.credentials,+ "delete_credential",+ AsyncMock(side_effect=RuntimeError("database unavailable")),+ )+ orphaned = AsyncMock()+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)++ with pytest.raises(RuntimeError, match="database unavailable"):+ await BitwardenCredentialVaultService().delete_credential(_credential())++ delete_item.assert_not_awaited()+ orphaned.assert_not_awaited()+++async def test_delete_credential_removes_db_row_when_vault_item_already_gone(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ item_url = f"{bitwarden_module.BITWARDEN_SERVER_BASE_URL}/object/item/item_old"+ monkeypatch.setattr(+ app.DATABASE.credentials,+ "get_organization_bitwarden_collection",+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),+ )+ monkeypatch.setattr(+ bitwarden_module.BitwardenService,+ "_get_skyvern_auth_secrets",+ AsyncMock(return_value=("master-password", "client-id", "client-secret", "admin-password")),+ )+ monkeypatch.setattr(bitwarden_module.BitwardenService, "_unlock_using_server", AsyncMock())+ aiohttp_delete = AsyncMock(side_effect=HttpException(404, item_url))+ monkeypatch.setattr(bitwarden_module, "aiohttp_delete", aiohttp_delete)+ db_delete = AsyncMock()+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", db_delete)+ orphaned = AsyncMock()+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)++ await BitwardenCredentialVaultService().delete_credential(_credential())++ db_delete.assert_awaited_once_with("cred_test", "org_test")+ aiohttp_delete.assert_awaited_once_with(item_url, timeout=120)+ orphaned.assert_not_awaited()++@pytest.mark.asyncioasync def test_post_delete_credential_item_returns_provider_failure_status(monkeypatch: pytest.MonkeyPatch) -> None:delete_item = AsyncMock(side_effect=RuntimeError("vault unavailable"))diff --git a/tests/unit/test_real_azure.py b/tests/unit/test_real_azure.pyindex ed1b14d0c..6d95cb3eb 100644--- a/tests/unit/test_real_azure.py+++ b/tests/unit/test_real_azure.py@@ -35,11 +35,14 @@ def _http_error(status_code: int) -> HttpResponseError:class _UnauthorizedVersionIterator:+ def __init__(self, status_code: int) -> None:+ self._status_code = status_code+def __aiter__(self) -> "_UnauthorizedVersionIterator":return selfasync def __anext__(self) -> object:- raise _http_error(403) # caller lacks secrets/list permission+ raise _http_error(self._status_code)class _NonAuthErrorVersionIterator:@@ -91,6 +94,32 @@ def _vault_client(monkeypatch: pytest.MonkeyPatch, secret_client: MagicMock) ->return client+async def test_get_secret_returns_none_when_secret_is_not_found(monkeypatch: pytest.MonkeyPatch) -> None:+ secret_client = MagicMock()+ secret_client.get_secret = AsyncMock(side_effect=ResourceNotFoundError("secret not found"))+ secret_client.close = AsyncMock()+ client = _vault_client(monkeypatch, secret_client)++ result = await client.get_secret("my-secret", "vault")++ assert result is None+ secret_client.close.assert_awaited_once()+++async def test_get_secret_reraises_transient_read_failure(monkeypatch: pytest.MonkeyPatch) -> None:+ secret_client = MagicMock()+ secret_client.get_secret = AsyncMock(side_effect=RuntimeError("vault unavailable"))+ secret_client.close = AsyncMock()+ client = _vault_client(monkeypatch, secret_client)++ with pytest.raises(RuntimeError, match="vault unavailable"):+ await client.get_secret("my-secret", "vault")++ secret_client.close.assert_awaited_once()++@pytest.mark.asyncioasync def test_create_or_update_secret_disables_only_preexisting_enabled_versions(monkeypatch: pytest.MonkeyPatch,@@ -143,19 +172,20 @@ async def test_create_or_update_secret_snapshots_versions_before_writing_new_one@pytest.mark.asyncio-async def test_create_or_update_secret_survives_version_disable_failure(+async def test_create_or_update_secret_fails_when_enumerated_version_cannot_be_disabled(monkeypatch: pytest.MonkeyPatch,) -> None:secret_client = _secret_client(_AsyncVersionIterator([SimpleNamespace(version="v-old", enabled=True)]),- disable_error=RuntimeError("disable failed"),+ disable_error=_http_error(403),)client = _vault_client(monkeypatch, secret_client)- result = await client.create_or_update_secret("my-secret", "new-value", "vault")+ with pytest.raises(HttpResponseError, match="status 403"):+ await client.create_or_update_secret("my-secret", "new-value", "vault")- assert result == "my-secret"secret_client.set_secret.assert_awaited_once()+ secret_client.update_secret_properties.assert_awaited_once_with("my-secret", "v-old", enabled=False)secret_client.close.assert_awaited_once()@@ -174,16 +204,19 @@ async def test_create_or_update_secret_handles_missing_prior_versions(@pytest.mark.asyncio-async def test_create_or_update_secret_writes_when_version_listing_is_unauthorized([email protected]("status_code", [401, 403])+async def test_create_or_update_secret_fails_when_version_listing_is_unauthorized(monkeypatch: pytest.MonkeyPatch,+ status_code: int,) -> None:- secret_client = _secret_client(_UnauthorizedVersionIterator())+ secret_client = _secret_client(_UnauthorizedVersionIterator(status_code))client = _vault_client(monkeypatch, secret_client)- result = await client.create_or_update_secret("my-secret", "new-value", "vault")+ with pytest.raises(HttpResponseError) as exc_info:+ await client.create_or_update_secret("my-secret", "new-value", "vault")- assert result == "my-secret"- secret_client.set_secret.assert_awaited_once_with("my-secret", "new-value")+ assert exc_info.value.status_code == status_code+ secret_client.set_secret.assert_not_awaited()secret_client.update_secret_properties.assert_not_awaited()secret_client.close.assert_awaited_once()