Skip to content

Automate browser based workflows with AI

fix(credentials): delete-path follow-ups from #14215 review (SKY-13271) (#7946)

Suchintand025fb210a7f
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 4396f0c27..520641fa1 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -1320,8 +1320,8 @@ class AgentFunction:
organization_id: str,
item_id: str,
vault_type: CredentialVaultType,
- ) -> None:
- return
+ ) -> bool:
+ return False
async def process_registered_credential_item(
self,
diff --git a/skyvern/forge/sdk/api/real_azure.py b/skyvern/forge/sdk/api/real_azure.py
index 120fd1a48..b3cddd559 100644
--- a/skyvern/forge/sdk/api/real_azure.py
+++ b/skyvern/forge/sdk/api/real_azure.py
@@ -5,7 +5,7 @@ from mimetypes import add_type, guess_type
from typing import IO, Self
import structlog
-from azure.core.exceptions import HttpResponseError, ResourceNotFoundError
+from azure.core.exceptions import ResourceNotFoundError
from azure.identity.aio import ClientSecretCredential, DefaultAzureCredential
from azure.keyvault.secrets.aio import SecretClient
from azure.storage.blob import BlobSasPermissions, ContentSettings, StandardBlobTier, generate_blob_sas
@@ -47,9 +47,11 @@ class RealAsyncAzureVaultClient(AsyncAzureVaultClient):
try:
secret = await secret_client.get_secret(secret_name)
return secret.value
- except Exception as e:
- LOG.exception("Failed to get secret from Azure Key Vault.", secret_name=secret_name, error=e)
+ except ResourceNotFoundError:
return None
+ except Exception:
+ LOG.exception("Failed to get secret from Azure Key Vault.", secret_name=secret_name)
+ raise
finally:
await secret_client.close()
@@ -70,34 +72,12 @@ class RealAsyncAzureVaultClient(AsyncAzureVaultClient):
# enabled and readable; fail the write so it retries instead of silently orphaning them.
if enumerated_any:
raise
- except HttpResponseError as e:
- # An identity permitted to set but not list secret versions raises a 401/403 here, and
- # disabling priors is best-effort, so proceed rather than failing every write. Re-raise any
- # other status, or a failure after enumerating any versions (a first page of only
- # current/already-disabled versions leaves `previous_versions` empty yet still means the
- # error is partial), so the write fails and retries instead of silently leaving the
- # un-enumerated prior versions enabled and readable.
- if enumerated_any or e.status_code not in (401, 403):
- raise
- LOG.warning(
- "Could not enumerate prior Azure secret versions to disable; proceeding with the write",
- secret_name=secret_name,
- exc_info=True,
- )
secret = await secret_client.set_secret(secret_name, secret_value)
# Azure Key Vault leaves every prior version enabled and readable by explicit version id; disable
# the ones that predate this write so a rotated/detached secret value can't be read back.
for version in previous_versions:
- try:
- await secret_client.update_secret_properties(secret_name, version, enabled=False)
- except Exception:
- LOG.warning(
- "Failed to disable prior Azure secret version",
- secret_name=secret_name,
- version=version,
- exc_info=True,
- )
+ await secret_client.update_secret_properties(secret_name, version, enabled=False)
return secret.name
except Exception as e:
LOG.exception("Failed to create secret from Azure Key Vault.", secret_name=secret_name, error=e)
diff --git a/skyvern/forge/sdk/core/aiohttp_helper.py b/skyvern/forge/sdk/core/aiohttp_helper.py
index 8d1a96b75..7d9d434a7 100644
--- a/skyvern/forge/sdk/core/aiohttp_helper.py
+++ b/skyvern/forge/sdk/core/aiohttp_helper.py
@@ -362,6 +362,7 @@ async def aiohttp_delete(
) -> dict[str, Any]:
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout)) as session:
count = 0
+ last_exception: Exception | None = None
while count <= retry:
try:
async with session.delete(
@@ -376,8 +377,11 @@ async def aiohttp_delete(
raise HttpException(response.status, url)
LOG.error(f"Failed to delete data from {url}", status_code=response.status)
return {}
- except Exception:
+ except Exception as exc:
+ last_exception = exc
if retry_timeout > 0:
await asyncio.sleep(retry_timeout)
count += 1
+ if last_exception is not None:
+ raise last_exception
raise Exception(f"Failed to delete data from {url}")
diff --git a/skyvern/forge/sdk/services/credential/azure_credential_vault_service.py b/skyvern/forge/sdk/services/credential/azure_credential_vault_service.py
index 41e827493..2eda4dea6 100644
--- a/skyvern/forge/sdk/services/credential/azure_credential_vault_service.py
+++ b/skyvern/forge/sdk/services/credential/azure_credential_vault_service.py
@@ -117,16 +117,38 @@ class AzureCredentialVaultService(CredentialVaultService):
self,
credential: Credential,
) -> None:
- # Empty the secret value before deleting the DB row: a transient vault failure then aborts the whole
- # delete with the row (and its item_id) intact for a retry, instead of orphaning a still-readable
- # secret with no DB pointer left to scrub it. The slow full-vault delete is deferred to
- # post_delete_credential_item so customers do not have to wait.
- await self._client.create_or_update_secret(
- vault_name=self._vault_name,
+ previous_secret_value = await self._client.get_secret(
secret_name=credential.item_id,
- secret_value="",
+ vault_name=self._vault_name,
)
- await app.DATABASE.credentials.delete_credential(credential.credential_id, credential.organization_id)
+ if previous_secret_value is None:
+ # Vault item already gone (prior partial delete); nothing to scrub, and the row must stay deletable.
+ await app.DATABASE.credentials.delete_credential(credential.credential_id, credential.organization_id)
+ return
+
+ try:
+ await self._client.create_or_update_secret(
+ vault_name=self._vault_name,
+ secret_name=credential.item_id,
+ secret_value="",
+ )
+ await app.DATABASE.credentials.delete_credential(credential.credential_id, credential.organization_id)
+ except BaseException:
+ try:
+ await self._client.create_or_update_secret(
+ vault_name=self._vault_name,
+ secret_name=credential.item_id,
+ secret_value=previous_secret_value,
+ )
+ except BaseException:
+ LOG.error(
+ "Failed to restore Azure vault secret after credential deletion failed",
+ credential_id=credential.credential_id,
+ item_id=credential.item_id,
+ organization_id=credential.organization_id,
+ exc_info=True,
+ )
+ raise
async def post_delete_credential_item(self, item_id: str, _organization_id: str | None = None) -> bool:
"""
diff --git a/skyvern/forge/sdk/services/credential/bitwarden_credential_service.py b/skyvern/forge/sdk/services/credential/bitwarden_credential_service.py
index 62f654ead..3ff901646 100644
--- a/skyvern/forge/sdk/services/credential/bitwarden_credential_service.py
+++ b/skyvern/forge/sdk/services/credential/bitwarden_credential_service.py
@@ -1,6 +1,7 @@
import structlog
from fastapi import HTTPException
+from skyvern.exceptions import HttpException
from skyvern.forge import app
from skyvern.forge.sdk.schemas.credentials import (
CreateCredentialRequest,
@@ -110,7 +111,47 @@ class BitwardenCredentialVaultService(CredentialVaultService):
raise HTTPException(status_code=404, detail="Credential account not found. It might have been deleted.")
await app.DATABASE.credentials.delete_credential(credential.credential_id, credential.organization_id)
- await BitwardenService.delete_credential_item(credential.item_id)
+ await self._run_task_to_completion(
+ self._delete_credential_item_or_enqueue_cleanup(credential),
+ cancellation_failure_message="Bitwarden vault-item cleanup failed while the delete was being cancelled",
+ log_context={
+ "credential_id": credential.credential_id,
+ "item_id": credential.item_id,
+ "organization_id": credential.organization_id,
+ },
+ )
+
+ async def _delete_credential_item_or_enqueue_cleanup(self, credential: Credential) -> None:
+ try:
+ await BitwardenService.delete_credential_item(credential.item_id)
+ except BaseException as exc:
+ if isinstance(exc, HttpException) and exc.status_code == 404:
+ return
+ try:
+ cleanup_enqueued = await self._enqueue_orphaned_vault_item_cleanup(
+ organization_id=credential.organization_id,
+ item_id=credential.item_id,
+ vault_type=CredentialVaultType.BITWARDEN,
+ )
+ except Exception:
+ if isinstance(exc, Exception):
+ raise
+ # Preserve provider cancellation even when its fallback enqueue also fails.
+ else:
+ log = LOG.warning if cleanup_enqueued else LOG.error
+ message = (
+ "Bitwarden vault-item delete failed after DB row deletion; enqueued durable cleanup"
+ if cleanup_enqueued
+ else "Bitwarden vault-item delete failed after DB row deletion; durable cleanup unavailable"
+ )
+ log(
+ message,
+ organization_id=credential.organization_id,
+ item_id=credential.item_id,
+ error_type=type(exc).__name__,
+ )
+ if not isinstance(exc, Exception):
+ raise
async def post_delete_credential_item(self, item_id: str, organization_id: str | None = None) -> bool:
try:
diff --git a/skyvern/forge/sdk/services/credential/credential_vault_service.py b/skyvern/forge/sdk/services/credential/credential_vault_service.py
index da9694770..0bd23ac55 100644
--- a/skyvern/forge/sdk/services/credential/credential_vault_service.py
+++ b/skyvern/forge/sdk/services/credential/credential_vault_service.py
@@ -49,6 +49,66 @@ class CredentialVaultService(ABC):
async def get_credential_item(self, db_credential: Credential) -> CredentialItem:
"""Retrieve the full credential data from the vault."""
+ async def _enqueue_orphaned_vault_item_cleanup(
+ self,
+ *,
+ organization_id: str,
+ item_id: str,
+ vault_type: CredentialVaultType,
+ ) -> bool:
+ try:
+ return bool(
+ await app.AGENT_FUNCTION.on_credential_item_orphaned(
+ organization_id=organization_id,
+ item_id=item_id,
+ vault_type=vault_type,
+ )
+ )
+ except Exception:
+ LOG.error(
+ "Durable vault-item cleanup enqueue failed; item requires manual reconciliation",
+ organization_id=organization_id,
+ item_id=item_id,
+ vault_type=vault_type,
+ exc_info=True,
+ )
+ raise
+
+ async def _run_task_to_completion(
+ self,
+ awaitable: Awaitable[None],
+ *,
+ suppress_cancellation: bool = False,
+ cancellation_failure_message: str | None = None,
+ log_context: dict[str, object] | None = None,
+ ) -> None:
+ task = asyncio.ensure_future(awaitable)
+ cancellation: asyncio.CancelledError | None = None
+ while not task.done():
+ try:
+ await asyncio.shield(task)
+ except asyncio.CancelledError as exc:
+ cancellation = cancellation or exc
+ except Exception:
+ if cancellation is None:
+ raise
+ break
+
+ if cancellation is None:
+ task.result()
+ return
+
+ try:
+ task.result()
+ except asyncio.CancelledError:
+ pass
+ except Exception:
+ if cancellation_failure_message:
+ LOG.error(cancellation_failure_message, **(log_context or {}), exc_info=True)
+
+ if not suppress_cancellation:
+ raise cancellation
+
async def _reclaim_orphaned_vault_item(
self,
*,
@@ -76,31 +136,17 @@ class CredentialVaultService(ABC):
vault_type=vault_type,
exc_info=True,
)
- # The caller invokes this inside `except BaseException: ... ; raise`, so a durable-enqueue
- # failure must not escape _run(): it would replace the in-flight exception (e.g. the
- # write-lease cancellation the caller is re-raising) with an unrelated one. Log and swallow so
- # the caller's bare `raise` always re-raises the original.
try:
- await app.AGENT_FUNCTION.on_credential_item_orphaned(
+ await self._enqueue_orphaned_vault_item_cleanup(
organization_id=organization_id,
item_id=item_id,
vault_type=vault_type,
)
except Exception:
- LOG.error(
- "Durable vault-item cleanup enqueue failed; item may stay orphaned until the reaper reclaims it",
- organization_id=organization_id,
- item_id=item_id,
- vault_type=vault_type,
- exc_info=True,
- )
+ # The original failure or cancellation must win over a failed fallback enqueue.
+ pass
- reclaim = asyncio.ensure_future(_run())
- while not reclaim.done():
- try:
- await asyncio.shield(reclaim)
- except asyncio.CancelledError:
- continue
+ await self._run_task_to_completion(_run(), suppress_cancellation=True)
async def _preserve_omitted_credit_card_fields(
self,
diff --git a/tests/unit/test_aiohttp_helper.py b/tests/unit/test_aiohttp_helper.py
index 54d74c4d9..d231c18d7 100644
--- a/tests/unit/test_aiohttp_helper.py
+++ b/tests/unit/test_aiohttp_helper.py
@@ -8,7 +8,7 @@ import aiohttp
import pytest
from skyvern.exceptions import BlockedHost, HttpException
-from skyvern.forge.sdk.core.aiohttp_helper import SSRFGuardedResolver, aiohttp_request
+from skyvern.forge.sdk.core.aiohttp_helper import SSRFGuardedResolver, aiohttp_delete, aiohttp_request
from skyvern.utils.url_validators import MAX_SAFE_REDIRECTS, validate_fetch_url
@@ -20,6 +20,30 @@ def public_dns(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr("skyvern.utils.url_validators.socket.getaddrinfo", resolves_public)
+async def test_aiohttp_delete_propagates_http_exception_after_retries() -> None:
+ url = "https://example.com/object/item/item_test"
+ mock_response = AsyncMock()
+ mock_response.status = 404
+ mock_response.__aenter__ = AsyncMock(return_value=mock_response)
+ mock_response.__aexit__ = AsyncMock(return_value=None)
+
+ mock_session = MagicMock()
+ mock_session.__aenter__ = AsyncMock(return_value=mock_session)
+ mock_session.__aexit__ = AsyncMock(return_value=None)
+ mock_session.delete = MagicMock(return_value=mock_response)
+
+ with (
+ patch("skyvern.forge.sdk.core.aiohttp_helper.aiohttp.ClientSession", return_value=mock_session),
+ pytest.raises(HttpException) as exc_info,
+ ):
+ await aiohttp_delete(url, retry=1)
+
+ assert exc_info.value.status_code == 404
+ assert exc_info.value.url == url
+ assert mock_session.delete.call_count == 2
+
+
@pytest.mark.asyncio
async def test_ssrf_guarded_resolver_blocks_dns_rebind_after_validation(monkeypatch: pytest.MonkeyPatch) -> None:
dns_answers = iter(
diff --git a/tests/unit/test_azure_credential_vault_service.py b/tests/unit/test_azure_credential_vault_service.py
index 41d0c67e3..8b95da84c 100644
--- a/tests/unit/test_azure_credential_vault_service.py
+++ b/tests/unit/test_azure_credential_vault_service.py
@@ -1,10 +1,11 @@
from types import SimpleNamespace
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, MagicMock
import pytest
from skyvern.forge import app
from skyvern.forge.sdk.schemas.credentials import PasswordCredential, TotpType
+from skyvern.forge.sdk.services.credential import azure_credential_vault_service as service_module
from skyvern.forge.sdk.services.credential.azure_credential_vault_service import AzureCredentialVaultService
_PASSWORD_METADATA = {"tenant": "north"}
@@ -43,6 +44,10 @@ async def test_azure_password_data_image_round_trips_metadata_and_loads_legacy()
async def test_azure_delete_scrubs_vault_before_removing_db_row(monkeypatch: pytest.MonkeyPatch) -> None:
calls: list[str] = []
+ async def read_secret(**_kwargs: object) -> str:
+ calls.append("vault_read")
+ return "stored-value"
+
async def scrub(**_kwargs: object) -> str:
calls.append("vault_scrub")
return "secret-id"
@@ -51,6 +56,7 @@ async def test_azure_delete_scrubs_vault_before_removing_db_row(monkeypatch: pyt
calls.append("db_delete")
client = AsyncMock()
+ client.get_secret = AsyncMock(side_effect=read_secret)
client.create_or_update_secret = AsyncMock(side_effect=scrub)
service = AzureCredentialVaultService(client=client, vault_name="vault")
monkeypatch.setattr(
@@ -60,13 +66,15 @@ async def test_azure_delete_scrubs_vault_before_removing_db_row(monkeypatch: pyt
await service.delete_credential(SimpleNamespace(item_id="item_1", credential_id="cred_1", organization_id="org_1"))
# Scrub the still-readable secret before dropping the DB row so a vault failure cannot orphan it.
- assert calls == ["vault_scrub", "db_delete"]
+ assert calls == ["vault_read", "vault_scrub", "db_delete"]
+ client.get_secret.assert_awaited_once_with(secret_name="item_1", vault_name="vault")
client.create_or_update_secret.assert_awaited_once_with(vault_name="vault", secret_name="item_1", secret_value="")
@pytest.mark.asyncio
async def test_azure_delete_leaves_db_row_when_vault_scrub_fails(monkeypatch: pytest.MonkeyPatch) -> None:
client = AsyncMock()
+ client.get_secret = AsyncMock(return_value="stored-value")
client.create_or_update_secret = AsyncMock(side_effect=RuntimeError("vault down"))
service = AzureCredentialVaultService(client=client, vault_name="vault")
db_delete = AsyncMock()
@@ -80,3 +88,121 @@ async def test_azure_delete_leaves_db_row_when_vault_scrub_fails(monkeypatch: py
# A transient vault failure must abort the delete with the DB row (and its item_id) intact so a retry
# can still scrub the secret, rather than orphaning a still-readable secret with no DB pointer.
db_delete.assert_not_awaited()
+
+
+async def test_azure_delete_restores_vault_value_when_scrub_partially_fails(monkeypatch: pytest.MonkeyPatch) -> None:
+ stored_value = "stored-value"
+
+ async def scrub_or_restore(*, secret_value: str, **_kwargs: object) -> str:
+ nonlocal stored_value
+ stored_value = secret_value
+ if secret_value == "":
+ raise RuntimeError("vault connection dropped after scrub")
+ return "item_1"
+
+ client = AsyncMock()
+ client.get_secret = AsyncMock(return_value=stored_value)
+ client.create_or_update_secret = AsyncMock(side_effect=scrub_or_restore)
+ service = AzureCredentialVaultService(client=client, vault_name="vault")
+ db_delete = AsyncMock()
+ monkeypatch.setattr(app.DATABASE, "credentials", SimpleNamespace(delete_credential=db_delete))
+
+ with pytest.raises(RuntimeError, match="vault connection dropped after scrub"):
+ await service.delete_credential(
+ SimpleNamespace(item_id="item_1", credential_id="cred_1", organization_id="org_1")
+ )
+
+ assert stored_value == "stored-value"
+ assert [call.kwargs["secret_value"] for call in client.create_or_update_secret.await_args_list] == [
+ "",
+ "stored-value",
+ ]
+ db_delete.assert_not_awaited()
+
+
+async def test_azure_delete_leaves_db_row_when_vault_read_fails(monkeypatch: pytest.MonkeyPatch) -> None:
+ client = AsyncMock()
+ client.get_secret = AsyncMock(side_effect=RuntimeError("vault read unavailable"))
+ client.create_or_update_secret = AsyncMock()
+ service = AzureCredentialVaultService(client=client, vault_name="vault")
+ db_delete = AsyncMock()
+ monkeypatch.setattr(app.DATABASE, "credentials", SimpleNamespace(delete_credential=db_delete))
+
+ with pytest.raises(RuntimeError, match="vault read unavailable"):
+ await service.delete_credential(
+ SimpleNamespace(item_id="item_1", credential_id="cred_1", organization_id="org_1")
+ )
+
+ client.create_or_update_secret.assert_not_awaited()
+ db_delete.assert_not_awaited()
+
+
+async def test_azure_delete_removes_db_row_when_vault_item_already_gone(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ client = AsyncMock()
+ client.get_secret = AsyncMock(return_value=None)
+ service = AzureCredentialVaultService(client=client, vault_name="vault")
+ db_delete = AsyncMock()
+ monkeypatch.setattr(app.DATABASE, "credentials", SimpleNamespace(delete_credential=db_delete))
+
+ await service.delete_credential(SimpleNamespace(item_id="item_1", credential_id="cred_1", organization_id="org_1"))
+
+ # A row whose vault item vanished (prior partial delete) must stay deletable: no scrub, straight DB delete.
+ client.create_or_update_secret.assert_not_awaited()
+ db_delete.assert_awaited_once_with("cred_1", "org_1")
+
+
+async def test_azure_delete_restores_vault_value_when_db_delete_fails(monkeypatch: pytest.MonkeyPatch) -> None:
+ client = AsyncMock()
+ client.get_secret = AsyncMock(return_value="stored-value")
+ client.create_or_update_secret = AsyncMock(return_value="item_1")
+ service = AzureCredentialVaultService(client=client, vault_name="vault")
+ monkeypatch.setattr(
+ app.DATABASE,
+ "credentials",
+ SimpleNamespace(delete_credential=AsyncMock(side_effect=RuntimeError("database unavailable"))),
+ )
+
+ with pytest.raises(RuntimeError, match="database unavailable"):
+ await service.delete_credential(
+ SimpleNamespace(item_id="item_1", credential_id="cred_1", organization_id="org_1")
+ )
+
+ assert [call.kwargs["secret_value"] for call in client.create_or_update_secret.await_args_list] == [
+ "",
+ "stored-value",
+ ]
+
+
+async def test_azure_delete_reraises_db_error_when_vault_restore_fails(monkeypatch: pytest.MonkeyPatch) -> None:
+ client = AsyncMock()
+ client.get_secret = AsyncMock(return_value="stored-value")
+ client.create_or_update_secret = AsyncMock(side_effect=["item_1", RuntimeError("restore failed")])
+ service = AzureCredentialVaultService(client=client, vault_name="vault")
+ monkeypatch.setattr(
+ app.DATABASE,
+ "credentials",
+ SimpleNamespace(delete_credential=AsyncMock(side_effect=RuntimeError("database unavailable"))),
+ )
+ log_error = MagicMock()
+ monkeypatch.setattr(service_module.LOG, "error", log_error)
+
+ with pytest.raises(RuntimeError, match="database unavailable"):
+ await service.delete_credential(
+ SimpleNamespace(item_id="item_1", credential_id="cred_1", organization_id="org_1")
+ )
+
+ assert [call.kwargs["secret_value"] for call in client.create_or_update_secret.await_args_list] == [
+ "",
+ "stored-value",
+ ]
+ log_error.assert_called_once()
+ assert log_error.call_args.kwargs["credential_id"] == "cred_1"
+ assert log_error.call_args.kwargs["item_id"] == "item_1"
+ assert log_error.call_args.kwargs["exc_info"] is True
diff --git a/tests/unit/test_bitwarden_credential_vault_service.py b/tests/unit/test_bitwarden_credential_vault_service.py
index 3dfed77a2..263287770 100644
--- a/tests/unit/test_bitwarden_credential_vault_service.py
+++ b/tests/unit/test_bitwarden_credential_vault_service.py
@@ -1,10 +1,11 @@
import asyncio
from datetime import datetime
from types import SimpleNamespace
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, MagicMock
import pytest
+from skyvern.exceptions import HttpException
from skyvern.forge import app
from skyvern.forge.sdk.schemas.credentials import (
CreateCredentialRequest,
@@ -13,6 +14,7 @@ from skyvern.forge.sdk.schemas.credentials import (
CredentialVaultType,
NonEmptyPasswordCredential,
)
+from skyvern.forge.sdk.services import bitwarden as bitwarden_module
from skyvern.forge.sdk.services.credential import bitwarden_credential_service as service_module
from skyvern.forge.sdk.services.credential.bitwarden_credential_service import BitwardenCredentialVaultService
@@ -77,6 +79,298 @@ def _prepare_update(
return service, delete_item
+async def test_delete_credential_removes_db_row_before_vault_item(monkeypatch: pytest.MonkeyPatch) -> None:
+ calls: list[str] = []
+ monkeypatch.setattr(
+ app.DATABASE.credentials,
+ "get_organization_bitwarden_collection",
+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),
+ )
+ monkeypatch.setattr(
+ service_module.BitwardenService,
+ "delete_credential_item",
+ AsyncMock(side_effect=lambda *_: calls.append("vault_delete")),
+ )
+ monkeypatch.setattr(
+ app.DATABASE.credentials,
+ "delete_credential",
+ AsyncMock(side_effect=lambda *_args, **_kwargs: calls.append("db_delete")),
+ )
+
+ await BitwardenCredentialVaultService().delete_credential(_credential())
+
+ assert calls == ["db_delete", "vault_delete"]
+
+
+async def test_delete_credential_enqueues_cleanup_when_vault_delete_fails(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(
+ app.DATABASE.credentials,
+ "get_organization_bitwarden_collection",
+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),
+ )
+ monkeypatch.setattr(
+ service_module.BitwardenService,
+ "delete_credential_item",
+ AsyncMock(side_effect=RuntimeError("vault unavailable")),
+ )
+ db_delete = AsyncMock()
+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", db_delete)
+ orphaned = AsyncMock()
+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)
+
+ await BitwardenCredentialVaultService().delete_credential(_credential())
+
+ db_delete.assert_awaited_once_with("cred_test", "org_test")
+ orphaned.assert_awaited_once_with(
+ organization_id="org_test",
+ item_id="item_old",
+ vault_type=CredentialVaultType.BITWARDEN,
+ )
+
+
+async def test_delete_credential_surfaces_durable_cleanup_enqueue_failure(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(
+ app.DATABASE.credentials,
+ "get_organization_bitwarden_collection",
+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),
+ )
+ monkeypatch.setattr(
+ service_module.BitwardenService,
+ "delete_credential_item",
+ AsyncMock(side_effect=RuntimeError("vault unavailable")),
+ )
+ db_delete = AsyncMock()
+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", db_delete)
+ orphaned = AsyncMock(side_effect=RuntimeError("cleanup enqueue unavailable"))
+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)
+
+ with pytest.raises(RuntimeError, match="cleanup enqueue unavailable"):
+ await BitwardenCredentialVaultService().delete_credential(_credential())
+
+ db_delete.assert_awaited_once_with("cred_test", "org_test")
+ orphaned.assert_awaited_once_with(
+ organization_id="org_test",
+ item_id="item_old",
+ vault_type=CredentialVaultType.BITWARDEN,
+ )
+
+
+async def test_delete_credential_enqueues_cleanup_when_vault_delete_is_cancelled(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(
+ app.DATABASE.credentials,
+ "get_organization_bitwarden_collection",
+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),
+ )
+ monkeypatch.setattr(
+ service_module.BitwardenService,
+ "delete_credential_item",
+ AsyncMock(side_effect=asyncio.CancelledError()),
+ )
+ db_delete = AsyncMock()
+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", db_delete)
+ orphaned = AsyncMock()
+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)
+
+ with pytest.raises(asyncio.CancelledError):
+ await BitwardenCredentialVaultService().delete_credential(_credential())
+
+ db_delete.assert_awaited_once_with("cred_test", "org_test")
+ orphaned.assert_awaited_once_with(
+ organization_id="org_test",
+ item_id="item_old",
+ vault_type=CredentialVaultType.BITWARDEN,
+ )
+
+
+async def test_delete_credential_vault_cancellation_wins_over_cleanup_enqueue_failure(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(
+ app.DATABASE.credentials,
+ "get_organization_bitwarden_collection",
+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),
+ )
+ monkeypatch.setattr(
+ service_module.BitwardenService,
+ "delete_credential_item",
+ AsyncMock(side_effect=asyncio.CancelledError()),
+ )
+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", AsyncMock())
+ orphaned = AsyncMock(side_effect=RuntimeError("cleanup enqueue unavailable"))
+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)
+
+ with pytest.raises(asyncio.CancelledError):
+ await BitwardenCredentialVaultService().delete_credential(_credential())
+
+ orphaned.assert_awaited_once_with(
+ organization_id="org_test",
+ item_id="item_old",
+ vault_type=CredentialVaultType.BITWARDEN,
+ )
+
+
+async def test_delete_credential_reports_when_durable_cleanup_is_unavailable(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(
+ app.DATABASE.credentials,
+ "get_organization_bitwarden_collection",
+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),
+ )
+ monkeypatch.setattr(
+ service_module.BitwardenService,
+ "delete_credential_item",
+ AsyncMock(side_effect=RuntimeError("vault unavailable")),
+ )
+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", AsyncMock())
+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", AsyncMock(return_value=False))
+ log_error = MagicMock()
+ monkeypatch.setattr(service_module.LOG, "error", log_error)
+
+ await BitwardenCredentialVaultService().delete_credential(_credential())
+
+ log_error.assert_called_once_with(
+ "Bitwarden vault-item delete failed after DB row deletion; durable cleanup unavailable",
+ organization_id="org_test",
+ item_id="item_old",
+ error_type="RuntimeError",
+ )
+
+
+async def test_delete_credential_finishes_vault_delete_before_propagating_caller_cancellation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ delete_started = asyncio.Event()
+ release_delete = asyncio.Event()
+ delete_finished = asyncio.Event()
+
+ async def delete_item(_item_id: str) -> None:
+ delete_started.set()
+ await release_delete.wait()
+ delete_finished.set()
+
+ monkeypatch.setattr(
+ app.DATABASE.credentials,
+ "get_organization_bitwarden_collection",
+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),
+ )
+ monkeypatch.setattr(service_module.BitwardenService, "delete_credential_item", AsyncMock(side_effect=delete_item))
+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", AsyncMock())
+ orphaned = AsyncMock()
+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)
+
+ delete_task = asyncio.create_task(BitwardenCredentialVaultService().delete_credential(_credential()))
+ await delete_started.wait()
+ delete_task.cancel()
+ await asyncio.sleep(0)
+ release_delete.set()
+
+ with pytest.raises(asyncio.CancelledError):
+ await delete_task
+
+ assert delete_finished.is_set()
+ orphaned.assert_not_awaited()
+
+
+async def test_delete_credential_cancellation_wins_over_cleanup_enqueue_failure(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ delete_started = asyncio.Event()
+ release_delete = asyncio.Event()
+
+ async def delete_item(_item_id: str) -> None:
+ delete_started.set()
+ await release_delete.wait()
+ raise RuntimeError("vault unavailable")
+
+ monkeypatch.setattr(
+ app.DATABASE.credentials,
+ "get_organization_bitwarden_collection",
+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),
+ )
+ monkeypatch.setattr(service_module.BitwardenService, "delete_credential_item", AsyncMock(side_effect=delete_item))
+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", AsyncMock())
+ orphaned = AsyncMock(side_effect=RuntimeError("cleanup enqueue unavailable"))
+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)
+
+ delete_task = asyncio.create_task(BitwardenCredentialVaultService().delete_credential(_credential()))
+ await delete_started.wait()
+ delete_task.cancel()
+ await asyncio.sleep(0)
+ release_delete.set()
+
+ # Both the vault delete and the cleanup enqueue fail while the caller is cancelling: the
+ # cancellation must still be what propagates, with the cleanup failure only logged.
+ with pytest.raises(asyncio.CancelledError):
+ await delete_task
+
+ orphaned.assert_awaited_once()
+
+
+async def test_delete_credential_does_not_touch_vault_when_db_delete_fails(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(
+ app.DATABASE.credentials,
+ "get_organization_bitwarden_collection",
+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),
+ )
+ delete_item = AsyncMock()
+ monkeypatch.setattr(service_module.BitwardenService, "delete_credential_item", delete_item)
+ monkeypatch.setattr(
+ app.DATABASE.credentials,
+ "delete_credential",
+ AsyncMock(side_effect=RuntimeError("database unavailable")),
+ )
+ orphaned = AsyncMock()
+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)
+
+ with pytest.raises(RuntimeError, match="database unavailable"):
+ await BitwardenCredentialVaultService().delete_credential(_credential())
+
+ delete_item.assert_not_awaited()
+ orphaned.assert_not_awaited()
+
+
+async def test_delete_credential_removes_db_row_when_vault_item_already_gone(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ item_url = f"{bitwarden_module.BITWARDEN_SERVER_BASE_URL}/object/item/item_old"
+ monkeypatch.setattr(
+ app.DATABASE.credentials,
+ "get_organization_bitwarden_collection",
+ AsyncMock(return_value=SimpleNamespace(collection_id="collection_test")),
+ )
+ monkeypatch.setattr(
+ bitwarden_module.BitwardenService,
+ "_get_skyvern_auth_secrets",
+ AsyncMock(return_value=("master-password", "client-id", "client-secret", "admin-password")),
+ )
+ monkeypatch.setattr(bitwarden_module.BitwardenService, "_unlock_using_server", AsyncMock())
+ aiohttp_delete = AsyncMock(side_effect=HttpException(404, item_url))
+ monkeypatch.setattr(bitwarden_module, "aiohttp_delete", aiohttp_delete)
+ db_delete = AsyncMock()
+ monkeypatch.setattr(app.DATABASE.credentials, "delete_credential", db_delete)
+ orphaned = AsyncMock()
+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_credential_item_orphaned", orphaned)
+
+ await BitwardenCredentialVaultService().delete_credential(_credential())
+
+ db_delete.assert_awaited_once_with("cred_test", "org_test")
+ aiohttp_delete.assert_awaited_once_with(item_url, timeout=120)
+ orphaned.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_post_delete_credential_item_returns_provider_failure_status(monkeypatch: pytest.MonkeyPatch) -> None:
delete_item = AsyncMock(side_effect=RuntimeError("vault unavailable"))
diff --git a/tests/unit/test_real_azure.py b/tests/unit/test_real_azure.py
index ed1b14d0c..6d95cb3eb 100644
--- a/tests/unit/test_real_azure.py
+++ b/tests/unit/test_real_azure.py
@@ -35,11 +35,14 @@ def _http_error(status_code: int) -> HttpResponseError:
class _UnauthorizedVersionIterator:
+ def __init__(self, status_code: int) -> None:
+ self._status_code = status_code
+
def __aiter__(self) -> "_UnauthorizedVersionIterator":
return self
async def __anext__(self) -> object:
- raise _http_error(403) # caller lacks secrets/list permission
+ raise _http_error(self._status_code)
class _NonAuthErrorVersionIterator:
@@ -91,6 +94,32 @@ def _vault_client(monkeypatch: pytest.MonkeyPatch, secret_client: MagicMock) ->
return client
+async def test_get_secret_returns_none_when_secret_is_not_found(monkeypatch: pytest.MonkeyPatch) -> None:
+ secret_client = MagicMock()
+ secret_client.get_secret = AsyncMock(side_effect=ResourceNotFoundError("secret not found"))
+ secret_client.close = AsyncMock()
+ client = _vault_client(monkeypatch, secret_client)
+
+ result = await client.get_secret("my-secret", "vault")
+
+ assert result is None
+ secret_client.close.assert_awaited_once()
+
+
+async def test_get_secret_reraises_transient_read_failure(monkeypatch: pytest.MonkeyPatch) -> None:
+ secret_client = MagicMock()
+ secret_client.get_secret = AsyncMock(side_effect=RuntimeError("vault unavailable"))
+ secret_client.close = AsyncMock()
+ client = _vault_client(monkeypatch, secret_client)
+
+ with pytest.raises(RuntimeError, match="vault unavailable"):
+ await client.get_secret("my-secret", "vault")
+
+ secret_client.close.assert_awaited_once()
+
+
@pytest.mark.asyncio
async def test_create_or_update_secret_disables_only_preexisting_enabled_versions(
monkeypatch: pytest.MonkeyPatch,
@@ -143,19 +172,20 @@ async def test_create_or_update_secret_snapshots_versions_before_writing_new_one
@pytest.mark.asyncio
-async def test_create_or_update_secret_survives_version_disable_failure(
+async def test_create_or_update_secret_fails_when_enumerated_version_cannot_be_disabled(
monkeypatch: pytest.MonkeyPatch,
) -> None:
secret_client = _secret_client(
_AsyncVersionIterator([SimpleNamespace(version="v-old", enabled=True)]),
- disable_error=RuntimeError("disable failed"),
+ disable_error=_http_error(403),
)
client = _vault_client(monkeypatch, secret_client)
- result = await client.create_or_update_secret("my-secret", "new-value", "vault")
+ with pytest.raises(HttpResponseError, match="status 403"):
+ await client.create_or_update_secret("my-secret", "new-value", "vault")
- assert result == "my-secret"
secret_client.set_secret.assert_awaited_once()
+ secret_client.update_secret_properties.assert_awaited_once_with("my-secret", "v-old", enabled=False)
secret_client.close.assert_awaited_once()
@@ -174,16 +204,19 @@ async def test_create_or_update_secret_handles_missing_prior_versions(
@pytest.mark.asyncio
-async def test_create_or_update_secret_writes_when_version_listing_is_unauthorized(
[email protected]("status_code", [401, 403])
+async def test_create_or_update_secret_fails_when_version_listing_is_unauthorized(
monkeypatch: pytest.MonkeyPatch,
+ status_code: int,
) -> None:
- secret_client = _secret_client(_UnauthorizedVersionIterator())
+ secret_client = _secret_client(_UnauthorizedVersionIterator(status_code))
client = _vault_client(monkeypatch, secret_client)
- result = await client.create_or_update_secret("my-secret", "new-value", "vault")
+ with pytest.raises(HttpResponseError) as exc_info:
+ await client.create_or_update_secret("my-secret", "new-value", "vault")
- assert result == "my-secret"
- secret_client.set_secret.assert_awaited_once_with("my-secret", "new-value")
+ assert exc_info.value.status_code == status_code
+ secret_client.set_secret.assert_not_awaited()
secret_client.update_secret_properties.assert_not_awaited()
secret_client.close.assert_awaited_once()