Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Stabilize request principal attribution (#8751)
Aaron Perez643d9d049b40
diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsxindex ba0ddaa43..c7f9b0de6 100644--- a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx@@ -96,7 +96,7 @@ const exampleCases = [hint: "finditparts.com",label: "Add a product to cart",prompt:- 'Go to https://www.finditparts.com first. Search for the product "W01-377-8537", add it to cart and then navigate to the cart page. Your goal is COMPLETE when you\'re on the cart page and the specified product is in the cart. Extract all product quantity information from the cart page. Do not attempt to checkout.',+ 'Go to https://www.finditparts.com first. Search for the product "W01-377-8537", add it to cart and then navigate to the cart page. Extract all product quantity information from the cart page. Do not attempt to checkout.',icon: <CartIcon className="size-6" />,},{diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 5e9710461..eccfa6970 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -74,7 +74,10 @@ from skyvern.forge.sdk.services import (sftp_service,)from skyvern.forge.sdk.services.credentials import AuthenticatorTotpParseResult-from skyvern.forge.sdk.services.request_principal import BearerIdentity+from skyvern.forge.sdk.services.request_principal import (+ BearerIdentityResolution,+ BearerIdentityStatus,+)from skyvern.forge.sdk.trace import tracedfrom skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, BlockTypeVarfrom skyvern.forge.sdk.workflow.retry_policy import WORKFLOW_WEBHOOK_HTTP_TIMEOUT_SECONDS@@ -2972,9 +2975,9 @@ class AgentFunction:"""Return whether the user belongs to the organization, or None when membership cannot be determined."""return None- async def resolve_bearer_identity(self, bearer_token: str, organization_id: str) -> BearerIdentity | None:- """Return the user the bearer proves for this organization, or None without an identity provider or proof."""- return None+ async def resolve_bearer_identity(self, bearer_token: str, organization_id: str) -> BearerIdentityResolution:+ """Return a verified bearer identity or why it could not be resolved."""+ return BearerIdentityResolution(None, BearerIdentityStatus.identity_provider_unconfigured)async def on_workflow_saved(self,diff --git a/skyvern/forge/request_logging.py b/skyvern/forge/request_logging.pyindex 38fd05bad..39821c948 100644--- a/skyvern/forge/request_logging.py+++ b/skyvern/forge/request_logging.py@@ -1,12 +1,13 @@from __future__ import annotations+import asyncioimport jsonimport reimport sysimport timeimport typingfrom contextvars import ContextVar-from dataclasses import dataclass+from dataclasses import dataclass, fieldfrom functools import partialimport structlog@@ -92,11 +93,19 @@ class _RequestIdentity:organization_name: str | None = Noneorg_age: int | None = Noneprincipal: RequestPrincipal | None = None+ principal_resolution_key: tuple[str, bytes | None] | None = None+ principal_resolution_conflict: bool = False+ bearer_identity_status: str | None = None+ principal_resolution_lock: asyncio.Lock = field(default_factory=asyncio.Lock)_request_identity: ContextVar[_RequestIdentity | None] = ContextVar("raw_request_identity", default=None)+def get_request_principal_state() -> _RequestIdentity | None:+ return _request_identity.get()++def set_request_organization(organization_id: str | None,organization_name: str | None = None,@@ -112,6 +121,9 @@ def set_request_organization(if identity is None:returnif organization_id:+ if identity.principal is not None and organization_id != identity.principal.organization_id:+ identity.principal_resolution_conflict = True+ returnidentity.organization_id = organization_idif organization_name:identity.organization_name = organization_name@@ -119,17 +131,27 @@ def set_request_organization(identity.org_age = org_age-def set_request_principal(principal: RequestPrincipal) -> None:+def set_request_principal(+ principal: RequestPrincipal,+ resolution_key: tuple[str, bytes | None],+ bearer_identity_status: str,+) -> None:identity = _request_identity.get()- if identity is not None:+ if identity is None:+ return+ if identity.principal is None:identity.principal = principal+ identity.principal_resolution_key = resolution_key+ identity.bearer_identity_status = bearer_identity_status+ elif identity.principal_resolution_key != resolution_key:+ identity.principal_resolution_conflict = True-def _identity_log_fields() -> dict[str, str | int | None]:+def _identity_log_fields() -> dict[str, str | int | bool | None]:identity = _request_identity.get()if identity is None:return {}- fields: dict[str, str | int | None] = {}+ fields: dict[str, str | int | bool | None] = {}if identity.organization_id:fields["organization_id"] = identity.organization_idif identity.organization_name:@@ -142,6 +164,8 @@ def _identity_log_fields() -> dict[str, str | int | None]:fields["user_id"] = identity.principal.user_idfields["org_role"] = identity.principal.org_rolefields["org_role_claim"] = identity.principal.org_role_claim+ fields["bearer_identity_status"] = identity.bearer_identity_status+ fields["principal_resolution_conflict"] = identity.principal_resolution_conflictreturn fields@@ -317,11 +341,11 @@ def log_raw_request_exception(status_code: int) -> None:async def log_raw_request_middleware(request: Request, call_next: Callable[[Request], Awaitable[Response]]) -> Response:+ _request_identity.set(_RequestIdentity())if not settings.LOG_RAW_API_REQUESTS:return await call_next(request)start_time = time.monotonic()- _request_identity.set(_RequestIdentity())try:body_bytes = await request.body()except ClientDisconnect:diff --git a/skyvern/forge/sdk/services/request_principal.py b/skyvern/forge/sdk/services/request_principal.pyindex 5e45510aa..561bc6e25 100644--- a/skyvern/forge/sdk/services/request_principal.py+++ b/skyvern/forge/sdk/services/request_principal.py@@ -1,4 +1,5 @@import asyncio+import hashlibfrom contextvars import ContextVarfrom dataclasses import dataclassfrom enum import StrEnum@@ -7,13 +8,12 @@ import structlogfrom fastapi import HTTPExceptionfrom skyvern.forge import app-from skyvern.forge.request_logging import set_request_principal+from skyvern.forge.request_logging import get_request_principal_state, set_request_principalfrom skyvern.forge.sdk.schemas.organizations import OrganizationAuthTokenTypeLOG = structlog.get_logger()-# ponytail: a failing identity lookup is retried by every request that carries a bearer, each bounded by-# this timeout. Cache the failure briefly if an identity-provider outage ever shows up as request latency.+# ponytail: a failed identity lookup can hold the request for up to this bound.BEARER_IDENTITY_TIMEOUT_SECONDS = 2.0@@ -26,6 +26,16 @@ class AuthKind(StrEnum):ui_session_and_bearer = "ui_session_and_bearer"+class BearerIdentityStatus(StrEnum):+ verified = "verified"+ no_bearer = "no_bearer"+ invalid_bearer = "invalid_bearer"+ organization_mismatch = "organization_mismatch"+ identity_provider_unconfigured = "identity_provider_unconfigured"+ lookup_failed = "lookup_failed"+ lookup_timed_out = "lookup_timed_out"++@dataclass(frozen=True)class BearerIdentity:user_id: str@@ -34,6 +44,12 @@ class BearerIdentity:org_role_claim: str | None = None+@dataclass(frozen=True)+class BearerIdentityResolution:+ identity: BearerIdentity | None+ status: BearerIdentityStatus++@dataclass(frozen=True)class RequestPrincipal:organization_id: str@@ -43,29 +59,38 @@ class RequestPrincipal:org_role_claim: str | None = None-_ResolutionKey = tuple[str, OrganizationAuthTokenType | None, str | None]-_request_principal: ContextVar[tuple[_ResolutionKey, RequestPrincipal] | None] = ContextVar(+_ResolutionKey = tuple[str, str | None, bytes | None]+_request_principal: ContextVar[tuple[_ResolutionKey, RequestPrincipal, BearerIdentityStatus] | None] = ContextVar("request_principal", default=None)def get_request_principal() -> RequestPrincipal | None:+ request_state = get_request_principal_state()+ if request_state is not None:+ return request_state.principalresolved = _request_principal.get()return resolved[1] if resolved else None-async def _resolve_bearer_identity(bearer_token: str, organization_id: str) -> BearerIdentity | None:+async def _resolve_bearer_identity(bearer_token: str, organization_id: str) -> BearerIdentityResolution:try:- identity = await asyncio.wait_for(+ resolution = await asyncio.wait_for(app.AGENT_FUNCTION.resolve_bearer_identity(bearer_token, organization_id),timeout=BEARER_IDENTITY_TIMEOUT_SECONDS,)+ except TimeoutError:+ return BearerIdentityResolution(None, BearerIdentityStatus.lookup_timed_out)except HTTPException:- return None- except Exception:+ return BearerIdentityResolution(None, BearerIdentityStatus.invalid_bearer)+ except Exception: # noqa: BLE001 - identity attribution must never reject an authenticated request.LOG.warning("Failed to resolve the bearer identity for the request principal", exc_info=True)- return None- return identity if isinstance(identity, BearerIdentity) else None+ return BearerIdentityResolution(None, BearerIdentityStatus.lookup_failed)+ return (+ resolution+ if isinstance(resolution, BearerIdentityResolution)+ else BearerIdentityResolution(None, BearerIdentityStatus.lookup_failed)+ )def _auth_kind(api_key_type: OrganizationAuthTokenType | None, has_user: bool) -> AuthKind:@@ -82,17 +107,47 @@ async def resolve_request_principal(api_key_type: OrganizationAuthTokenType | None = None,bearer_token: str | None = None,) -> RequestPrincipal:- """Record who the authenticated request acts as. Runs after authentication and never rejects.+ """Record the caller's identity after auth without changing the request outcome."""+ bearer_hash = hashlib.sha256(bearer_token.encode()).digest() if bearer_token else None+ resolution_key = (organization_id, api_key_type.value if api_key_type is not None else None, bearer_hash)+ identity_key = (organization_id, bearer_hash)+ request_state = get_request_principal_state()+ if request_state is not None:+ async with request_state.principal_resolution_lock:+ if request_state.principal is not None:+ set_request_principal(+ request_state.principal,+ identity_key,+ request_state.bearer_identity_status or BearerIdentityStatus.no_bearer.value,+ )+ return request_state.principal++ identity_result = (+ await _resolve_bearer_identity(bearer_token, organization_id)+ if bearer_token+ else BearerIdentityResolution(None, BearerIdentityStatus.no_bearer)+ )+ identity = identity_result.identity+ principal = RequestPrincipal(+ organization_id=organization_id,+ auth_kind=_auth_kind(api_key_type, identity is not None),+ user_id=identity.user_id if identity else None,+ org_role=identity.org_role if identity else None,+ org_role_claim=identity.org_role_claim if identity else None,+ )+ set_request_principal(principal, identity_key, identity_result.status.value)+ return principal- ``api_key_type`` is the key that authorized the request, or None when the bearer did. The user- and role only ever come from a bearer the deployment's identity provider vouches for.- """- resolution_key = (organization_id, api_key_type, bearer_token)resolved = _request_principal.get()if resolved is not None and resolved[0] == resolution_key:return resolved[1]- identity = await _resolve_bearer_identity(bearer_token, organization_id) if bearer_token else None+ identity_result = (+ await _resolve_bearer_identity(bearer_token, organization_id)+ if bearer_token+ else BearerIdentityResolution(None, BearerIdentityStatus.no_bearer)+ )+ identity = identity_result.identityprincipal = RequestPrincipal(organization_id=organization_id,auth_kind=_auth_kind(api_key_type, identity is not None),@@ -100,6 +155,5 @@ async def resolve_request_principal(org_role=identity.org_role if identity else None,org_role_claim=identity.org_role_claim if identity else None,)- _request_principal.set((resolution_key, principal))- set_request_principal(principal)+ _request_principal.set((resolution_key, principal, identity_result.status))return principaldiff --git a/tests/unit/test_org_auth_service.py b/tests/unit/test_org_auth_service.pyindex c03484be5..04b8095a5 100644--- a/tests/unit/test_org_auth_service.py+++ b/tests/unit/test_org_auth_service.py@@ -4,8 +4,10 @@ from dataclasses import asdictfrom datetime import UTC, datetime, timedeltafrom time import time as current_timefrom types import SimpleNamespace+from typing import Annotatedfrom unittest.mock import MagicMock+import httpximport jwtimport pytestfrom fastapi import Depends, FastAPI, HTTPException, WebSocket@@ -27,7 +29,12 @@ from skyvern.forge.sdk.services.org_auth_service import (_get_api_key_debug_fields,_normalize_api_key_with_flags,)-from skyvern.forge.sdk.services.request_principal import BearerIdentity, get_request_principal+from skyvern.forge.sdk.services.request_principal import (+ BearerIdentity,+ BearerIdentityResolution,+ BearerIdentityStatus,+ get_request_principal,+)def test_authenticated_org_age_is_cached_for_lines_that_only_name_the_org() -> None:@@ -1115,14 +1122,16 @@ class _StubIdentityProvider(AgentFunction):def __init__(self) -> None:self.resolved: list[tuple[str, str]] = []- async def resolve_bearer_identity(self, bearer_token: str, organization_id: str) -> BearerIdentity | None:+ async def resolve_bearer_identity(self, bearer_token: str, organization_id: str) -> BearerIdentityResolution:self.resolved.append((bearer_token, organization_id))if bearer_token == "stale-bearer":raise HTTPException(status_code=403, detail="Auth token is expired")if bearer_token == "broken-bearer":raise RuntimeError("identity provider lookup failed")identity = {"admin-bearer": _ADMIN, "member-bearer": _MEMBER, "roleless-bearer": _ROLELESS}.get(bearer_token)- return BearerIdentity(**identity) if identity else None+ if identity:+ return BearerIdentityResolution(BearerIdentity(**identity), BearerIdentityStatus.verified)+ return BearerIdentityResolution(None, BearerIdentityStatus.organization_mismatch)def _principal_app() -> FastAPI:@@ -1137,6 +1146,12 @@ def _principal_app() -> FastAPI:async def org(_: Organization = Depends(org_auth_service.get_current_org)) -> dict[str, str | None] | None:return principal()+ @fastapi_app.post("/stable-response")+ async def stable_response(+ _: Annotated[Organization, Depends(org_auth_service.get_current_org)],+ ) -> dict[str, str]:+ return {"result": "unchanged"}+@fastapi_app.post("/credential")async def credential(_: Organization = Depends(org_auth_service.get_current_org_for_credential_routes),@@ -1168,6 +1183,13 @@ def _principal_app() -> FastAPI:) -> dict[str, str | None] | None:return principal()+ @fastapi_app.post("/credential-and-user")+ async def credential_and_user(+ current_org: Annotated[Organization, Depends(org_auth_service.get_current_org_for_credential_routes)],+ user_id: Annotated[str | None, Depends(org_auth_service.get_current_user_id_or_none)],+ ) -> dict[str, object]:+ return {"organization_id": current_org.organization_id, "user_id": user_id, "principal": principal()}+@fastapi_app.post("/anonymous")async def anonymous() -> dict[str, str | None] | None:return principal()@@ -1360,6 +1382,154 @@ async def test_two_auth_dependencies_on_one_route_resolve_the_bearer_once(assert identity_provider.resolved == [("admin-bearer", organization_id)]+ ("bearer_organization_id", "expected_conflict"),+ [("org-bearer-authority", True), ("org-ui-session", False)],+ ids=["different-orgs", "same-org"],+)+async def test_credential_route_principal_conflict_tracks_organization_or_bearer(+ monkeypatch: pytest.MonkeyPatch,+ raw_request_log: MagicMock,+ bearer_organization_id: str,+ expected_conflict: bool,+) -> None:+ headers, _, identity_provider = await _principal_credentials(monkeypatch, "ui_session", "Bearer admin-bearer")+ bearer_organization = _make_org(bearer_organization_id)++ async def authenticate(_token: str, _attribution_header: str | None = None) -> Organization:+ return bearer_organization++ async def authenticate_user(_token: str) -> str:+ return "user_admin"++ monkeypatch.setattr(org_auth_service.app, "authentication_function", authenticate)+ monkeypatch.setattr(org_auth_service.app, "authenticate_user_function", authenticate_user)++ with TestClient(_principal_app()) as client:+ response = client.post("/credential-and-user", headers=headers)++ expected_principal = {+ "organization_id": bearer_organization.organization_id,+ "auth_kind": "bearer",+ **_ADMIN,+ }+ assert response.status_code == 200+ assert response.json() == {+ "organization_id": bearer_organization.organization_id,+ "user_id": None,+ "principal": expected_principal,+ }+ assert identity_provider.resolved == [("admin-bearer", bearer_organization.organization_id)]+ log = raw_request_log.info.call_args.kwargs+ assert log["organization_id"] == bearer_organization.organization_id+ assert log["user_id"] == "user_admin"+ assert log["principal_resolution_conflict"] is expected_conflict+++async def test_credential_route_principal_is_stable_when_raw_request_logging_is_disabled(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ headers, _, identity_provider = await _principal_credentials(monkeypatch, "ui_session", "Bearer admin-bearer")+ bearer_organization = _make_org("org-ui-session")++ async def authenticate(_token: str, _attribution_header: str | None = None) -> Organization:+ return bearer_organization++ async def authenticate_user(_token: str) -> str:+ return "user_admin"++ monkeypatch.setattr(org_auth_service.app, "authentication_function", authenticate)+ monkeypatch.setattr(org_auth_service.app, "authenticate_user_function", authenticate_user)+ monkeypatch.setattr(request_logging.settings, "LOG_RAW_API_REQUESTS", False)++ with TestClient(_principal_app()) as client:+ response = client.post("/credential-and-user", headers=headers)++ assert response.status_code == 200+ assert response.json() == {+ "organization_id": bearer_organization.organization_id,+ "user_id": None,+ "principal": {+ "organization_id": bearer_organization.organization_id,+ "auth_kind": "bearer",+ **_ADMIN,+ },+ }+ assert identity_provider.resolved == [("admin-bearer", bearer_organization.organization_id)]+++async def test_concurrent_principal_resolutions_keep_one_canonical_result(+ monkeypatch: pytest.MonkeyPatch,+ raw_request_log: MagicMock,+) -> None:+ identity_provider = _StubIdentityProvider()+ first_lookup_started = asyncio.Event()+ release_first_lookup = asyncio.Event()++ async def resolve_identity(bearer_token: str, organization_id: str) -> BearerIdentityResolution:+ identity_provider.resolved.append((bearer_token, organization_id))+ if bearer_token == "admin-bearer":+ first_lookup_started.set()+ await release_first_lookup.wait()+ return BearerIdentityResolution(BearerIdentity(**_ADMIN), BearerIdentityStatus.verified)+ return BearerIdentityResolution(BearerIdentity(**_MEMBER), BearerIdentityStatus.verified)++ monkeypatch.setattr(org_auth_service.app, "AGENT_FUNCTION", identity_provider)+ monkeypatch.setattr(identity_provider, "resolve_bearer_identity", resolve_identity)+ fastapi_app = _principal_app()++ @fastapi_app.post("/concurrent-principals")+ async def concurrent_principals() -> list[dict[str, object]]:+ first_task = asyncio.create_task(+ request_principal.resolve_request_principal("org-a", bearer_token="admin-bearer")+ )+ await asyncio.wait_for(first_lookup_started.wait(), timeout=1)+ second_task = asyncio.create_task(+ request_principal.resolve_request_principal("org-a", bearer_token="member-bearer")+ )+ await asyncio.sleep(0)+ release_first_lookup.set()+ first, second = await asyncio.wait_for(asyncio.gather(first_task, second_task), timeout=1)+ return [asdict(first), asdict(second)]++ async with httpx.AsyncClient(transport=httpx.ASGITransport(app=fastapi_app), base_url="http://test") as client:+ response = await client.post("/concurrent-principals")++ expected = {+ "organization_id": "org-a",+ "auth_kind": "bearer",+ **_ADMIN,+ }+ assert response.status_code == 200+ assert response.json() == [expected, expected]+ assert identity_provider.resolved == [("admin-bearer", "org-a")]+ assert raw_request_log.info.call_args.kwargs["user_id"] == "user_admin"+ assert raw_request_log.info.call_args.kwargs["principal_resolution_conflict"] is True+++async def test_fallback_principal_is_recomputed_when_its_resolution_key_changes(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ identity_provider = _StubIdentityProvider()+ monkeypatch.setattr(org_auth_service.app, "AGENT_FUNCTION", identity_provider)+ context_token = request_principal._request_principal.set(None)+ try:+ first = await request_principal.resolve_request_principal("org-a", bearer_token="admin-bearer")+ second = await request_principal.resolve_request_principal("org-b", bearer_token="member-bearer")+ finally:+ request_principal._request_principal.reset(context_token)++ assert first.organization_id == "org-a"+ assert first.user_id == "user_admin"+ assert second.organization_id == "org-b"+ assert second.user_id == "user_member"+ assert identity_provider.resolved == [("admin-bearer", "org-a"), ("member-bearer", "org-b")]++@pytest.mark.asyncioasync def test_caller_context_with_a_bearer_alone_resolves_the_identity_once(monkeypatch: pytest.MonkeyPatch,@@ -1384,9 +1554,9 @@ async def test_a_hanging_identity_lookup_cannot_hold_up_an_authenticated_requestmonkeypatch, "api", "Bearer admin-bearer")- async def hang(bearer_token: str, organization_id: str) -> BearerIdentity | None:+ async def hang(bearer_token: str, organization_id: str) -> BearerIdentityResolution:await asyncio.Event().wait()- return None+ return BearerIdentityResolution(None, BearerIdentityStatus.lookup_failed)monkeypatch.setattr(identity_provider, "resolve_bearer_identity", hang)monkeypatch.setattr(request_principal, "BEARER_IDENTITY_TIMEOUT_SECONDS", 0.05)@@ -1396,6 +1566,58 @@ async def test_a_hanging_identity_lookup_cannot_hold_up_an_authenticated_requestassert response.status_code == 200assert response.json() == {"organization_id": organization_id, "auth_kind": "api_key", **_NO_USER}+ assert raw_request_log.info.call_args.kwargs["bearer_identity_status"] == "lookup_timed_out"+++ ("authorization", "identity_provider", "expected_reason"),+ [+ (None, "configured", "no_bearer"),+ ("Bearer stale-bearer", "configured", "invalid_bearer"),+ ("Bearer bearer-of-another-organization", "configured", "organization_mismatch"),+ ("Bearer admin-bearer", "unconfigured", "identity_provider_unconfigured"),+ ("Bearer broken-bearer", "configured", "lookup_failed"),+ ],+ ids=["no-bearer", "stale-bearer", "foreign-organization", "no-identity-provider", "lookup-failed"],+)+async def test_raw_request_records_why_a_bearer_user_is_absent(+ monkeypatch: pytest.MonkeyPatch,+ raw_request_log: MagicMock,+ authorization: str | None,+ identity_provider: str,+ expected_reason: str,+) -> None:+ headers, _, _ = await _principal_credentials(monkeypatch, "api", authorization)+ if identity_provider == "unconfigured":+ monkeypatch.setattr(org_auth_service.app, "AGENT_FUNCTION", AgentFunction())++ with TestClient(_principal_app()) as client:+ response = client.post("/org", headers=headers)++ assert response.status_code == 200+ assert response.json()["user_id"] is None+ assert response.json()["org_role"] is None+ assert raw_request_log.info.call_args.kwargs["bearer_identity_status"] == expected_reason+++async def test_unexpected_identity_lookup_failure_is_logged_without_rejecting_request(+ monkeypatch: pytest.MonkeyPatch,+ raw_request_log: MagicMock,+) -> None:+ lookup_log = MagicMock()+ monkeypatch.setattr(request_principal, "LOG", lookup_log, raising=False)+ headers, organization_id, _ = await _principal_credentials(monkeypatch, "api", "Bearer broken-bearer")++ with TestClient(_principal_app()) as client:+ response = client.post("/org", headers=headers)++ assert response.status_code == 200+ assert response.json() == {"organization_id": organization_id, "auth_kind": "api_key", **_NO_USER}+ assert raw_request_log.info.call_args.kwargs["bearer_identity_status"] == "lookup_failed"+ lookup_log.warning.assert_called_once()+ assert lookup_log.warning.call_args.kwargs["exc_info"] is True@pytest.mark.asyncio