Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9804): distinguish missing-TOTP-source from delivery timeout in MFA terminate paths (#5987)
pedrohsdb57193ef453ee
diff --git a/skyvern/errors/errors.py b/skyvern/errors/errors.pyindex d1c83e8ab..f281365b2 100644--- a/skyvern/errors/errors.py+++ b/skyvern/errors/errors.py@@ -74,3 +74,12 @@ class TimeoutGetTOTPVerificationCodeError(SkyvernDefinedError):class TOTPExpiredError(SkyvernDefinedError):error_code: str = "OTP_EXPIRED"reasoning: str = "TOTP verification code has expired during multi-field input sequence."+++class MissingTOTPSourceError(SkyvernDefinedError):+ error_code: str = "MISSING_TOTP_SOURCE"+ reasoning: str = (+ "MFA was required but no TOTP source is configured. Configure one of: a totp_verification_url or "+ "totp_identifier on the login block (or run override), or attach a credential parameter whose "+ "credential stores a TOTP secret."+ )diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex f3d823f4c..8ec227237 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -55,6 +55,7 @@ from skyvern.exceptions import (NoTOTPVerificationCodeFound,PDFEmbedBase64DecodeError,ScrapingFailed,+ SkyvernException,StepTerminationError,StepUnableToExecuteError,TaskAlreadyCanceled,@@ -298,6 +299,22 @@ def _schedule_summary_shadow_check_for_hit()+def _build_totp_timeout_reasoning(task: Task) -> str:+ # Mirror poll_otp_value's URL-then-identifier precedence so we only report the+ # source that was actually queried. URL goes through strip_query_params to keep+ # query-string tokens out of the customer-visible failure_reason.+ if not (task.totp_verification_url or task.totp_identifier):+ raise SkyvernException(+ "_build_totp_timeout_reasoning called with no TOTP source; "+ "NoTOTPVerificationCodeFound should only fire when poll_otp_value was invoked"+ )+ if task.totp_verification_url:+ polled = f"totp_verification_url={strip_query_params(task.totp_verification_url)}"+ else:+ polled = f"totp_identifier={task.totp_identifier}"+ return f"No TOTP verification code found. Going to terminate. Polled source: {polled}."++class ForgeAgent:def __init__(self) -> None:self.async_operation_pool = AsyncOperationPool()@@ -1444,6 +1461,20 @@ class ForgeAgent:if context:context.pop_totp_code(task.task_id)except NoTOTPVerificationCodeFound:+ # Surface only the source poll_otp_value actually queried so+ # the failure_reason tells the customer which delivery endpoint+ # went silent.+ timeout_reasoning = _build_totp_timeout_reasoning(task)+ LOG.warning(+ "TOTP polling timed out — terminating task",+ task_id=task.task_id,+ workflow_run_id=task.workflow_run_id,+ totp_verification_url=strip_query_params(task.totp_verification_url)+ if task.totp_verification_url+ else None,+ totp_identifier=task.totp_identifier,+ organization_id=task.organization_id,+ )actions = [TerminateAction(organization_id=task.organization_id,@@ -1452,8 +1483,8 @@ class ForgeAgent:step_id=step.step_id,step_order=step.order,action_order=0,- reasoning="No TOTP verification code found. Going to terminate.",- intention="No TOTP verification code found. Going to terminate.",+ reasoning=timeout_reasoning,+ intention=timeout_reasoning,errors=[TimeoutGetTOTPVerificationCodeError().to_user_defined_error()],)]diff --git a/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py b/skyvern/forge/sdk/routes/streaming/channels/exfiltration.pyindex 233fdc9f7..7e8da57da 100644--- a/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py+++ b/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py@@ -11,11 +11,13 @@ Channel data:"""import asyncio+import contextlibimport dataclassesimport enumimport jsonimport timeimport typing as t+import weakrefimport structlogfrom playwright.async_api import CDPSession, ConsoleMessage, Page@@ -46,37 +48,237 @@ class ExfiltratedEvent:OnExfiltrationEvent = t.Callable[[list[ExfiltratedEvent]], None]+class PageConsoleCapture:+ console_listener: t.Callable[[ConsoleMessage], object]+ cdp_session: CDPSession | None = None++class ExfiltrationChannel(CdpChannel):"""ExfiltrationChannel."""+ BINDING_NAME: t.ClassVar[str] = "__skyvern_exfiltrate_event"+ CONSOLE_DEDUP_TTL_SECONDS: t.ClassVar[float] = 5.0+ REFRESH_INTERVAL_SECONDS: t.ClassVar[float] = 1.0+ _active_binding_channels: t.ClassVar[weakref.WeakKeyDictionary[Page, "ExfiltrationChannel"]] = (+ weakref.WeakKeyDictionary()+ )+ _binding_registered_pages: t.ClassVar[weakref.WeakSet[Page]] = weakref.WeakSet()+def __init__(self, *, on_event: OnExfiltrationEvent, vnc_channel: VncChannel) -> None:self.cdp_session: CDPSession | None = Noneself.on_event = on_event+ self._page_console_captures: weakref.WeakKeyDictionary[Page, PageConsoleCapture] = weakref.WeakKeyDictionary()+ self._recent_console_event_fingerprints: dict[str, float] = {}+ self._pending_event_tasks: set[asyncio.Task[None]] = set()+ self._refresh_task: asyncio.Task | None = Nonesuper().__init__(vnc_channel=vnc_channel)- def _handle_console_event(self, msg: ConsoleMessage) -> None:- """Parse console messages for exfiltrated event data."""- text = msg.text- if text.startswith("[EXFIL]"):+ def _track_event_task(self, coro: t.Coroutine[t.Any, t.Any, None]) -> None:+ task = asyncio.create_task(coro)+ self._pending_event_tasks.add(task)+ task.add_done_callback(self._on_event_task_done)++ def _on_event_task_done(self, task: asyncio.Task[None]) -> None:+ self._pending_event_tasks.discard(task)+ if task.cancelled():+ return++ try:+ task.result()+ except Exception:+ LOG.exception(f"{self.class_name} async exfiltration event task failed")++ def _parse_exfil_payload(self, payload: object) -> dict[str, t.Any] | None:+ if isinstance(payload, str):try:- event_data = json.loads(text[7:]) # Strip '[EXFIL]' prefix-- messages = [- ExfiltratedEvent(- kind="exfiltrated-event",- event_name="user_interaction",- params=event_data,- source=ExfiltratedEventSource.CONSOLE,- timestamp=time.time(),- ),- ]-- self.on_event(messages)- except Exception:- LOG.exception(f"{self.class_name} Failed to parse exfiltrated event", text=text)+ payload = json.loads(payload)+ except json.JSONDecodeError:+ return None++ if isinstance(payload, dict):+ return t.cast(dict[str, t.Any], payload)++ return None++ def _parse_exfil_text(self, text: str) -> dict[str, t.Any] | None:+ if not text.startswith("[EXFIL]"):+ return None++ return self._parse_exfil_payload(text[7:].strip())++ def _parse_exfil_args(self, args: list[object]) -> dict[str, t.Any] | None:+ if len(args) < 2 or args[0] != "[EXFIL]":+ return None++ return self._parse_exfil_payload(args[1])++ def _extract_cdp_remote_object_value(self, arg: object) -> object:+ if not isinstance(arg, dict):+ return arg++ if "value" in arg:+ return arg["value"]++ if arg.get("type") == "string" and "description" in arg:+ return arg["description"]++ preview = arg.get("preview")+ if isinstance(preview, dict):+ properties = preview.get("properties")+ if isinstance(properties, list):+ materialized: dict[str, object] = {}+ for prop in properties:+ if not isinstance(prop, dict):+ continue+ name = prop.get("name")+ if not isinstance(name, str):+ continue+ if "value" in prop:+ materialized[name] = prop["value"]+ elif isinstance(prop.get("valuePreview"), dict) and "value" in prop["valuePreview"]:+ materialized[name] = prop["valuePreview"]["value"]+ if materialized:+ return materialized++ return arg++ def _prune_console_dedup_cache(self, now: float) -> None:+ expired = [+ fingerprint+ for fingerprint, emitted_at in self._recent_console_event_fingerprints.items()+ if now - emitted_at > self.CONSOLE_DEDUP_TTL_SECONDS+ ]+ for fingerprint in expired:+ self._recent_console_event_fingerprints.pop(fingerprint, None)++ def _should_emit_console_event(self, event_data: dict[str, t.Any]) -> bool:+ try:+ fingerprint = json.dumps(event_data, sort_keys=True, separators=(",", ":"))+ except TypeError:+ fingerprint = json.dumps(event_data, sort_keys=True, separators=(",", ":"), default=str)++ now = time.monotonic()+ self._prune_console_dedup_cache(now)+ previous = self._recent_console_event_fingerprints.get(fingerprint)+ if previous is not None and now - previous <= self.CONSOLE_DEDUP_TTL_SECONDS:+ return False++ self._recent_console_event_fingerprints[fingerprint] = now+ return True++ def _emit_console_event(self, event_data: dict[str, t.Any]) -> None:+ if not self._should_emit_console_event(event_data):+ return++ self.on_event(+ [+ ExfiltratedEvent(+ kind="exfiltrated-event",+ event_name="user_interaction",+ params=event_data,+ source=ExfiltratedEventSource.CONSOLE,+ timestamp=time.time(),+ )+ ]+ )++ def _handle_binding_event(self, source: dict[str, t.Any], payload: object) -> None:+ page = source.get("page") if isinstance(source, dict) else None+ active_channel = self._active_binding_channels.get(page, self) if page else self+ event_data = active_channel._parse_exfil_payload(payload)+ if event_data is None:+ return++ active_channel._emit_console_event(event_data)++ async def _handle_console_event_async(self, msg: ConsoleMessage) -> None:+ """Parse Playwright console messages for exfiltrated event data."""+ event_data: dict[str, t.Any] | None = None+ try:+ args = []+ for arg in msg.args[:2]:+ args.append(await arg.json_value())+ event_data = self._parse_exfil_args(args)+ except Exception:+ LOG.debug(f"{self.class_name} Failed to inspect console args for EXFIL event", exc_info=True)++ text = msg.text+ if event_data is None:+ event_data = self._parse_exfil_text(text)++ if event_data is None:+ return++ self._emit_console_event(event_data)++ def _handle_console_event(self, msg: ConsoleMessage) -> None:+ self._track_event_task(self._handle_console_event_async(msg))++ async def _handle_runtime_console_event_async(self, params: dict[str, t.Any]) -> None:+ raw_args = params.get("args")+ if not isinstance(raw_args, list):+ return++ event_data = self._parse_exfil_args([self._extract_cdp_remote_object_value(arg) for arg in raw_args[:2]])+ if event_data is None:+ return++ self._emit_console_event(event_data)++ async def _attach_page_cdp_console_capture(self, page: Page) -> CDPSession | None:+ cdp_session = await page.context.new_cdp_session(page)+ await cdp_session.send("Runtime.enable")+ cdp_session.on(+ "Runtime.consoleAPICalled",+ lambda params: self._track_event_task(self._handle_runtime_console_event_async(params)),+ )+ return cdp_session++ async def _ensure_binding(self, page: Page) -> None:+ self._active_binding_channels[page] = self++ if page in self._binding_registered_pages:+ return++ try:+ await page.expose_binding(self.BINDING_NAME, self._handle_binding_event)+ self._binding_registered_pages.add(page)+ except Exception:+ LOG.debug(f"{self.class_name} failed to expose exfiltration binding", page_url=page.url, exc_info=True)++ async def _install_exfiltration_script(self, page: Page, *, add_init_script: bool) -> None:+ binding_script = f"window.__skyvern_exfiltration_binding_name = {json.dumps(self.BINDING_NAME)};"++ if add_init_script:+ await page.add_init_script(binding_script)+ await page.add_init_script(self.js("exfiltrate"))++ await page.evaluate(binding_script)+ await page.evaluate(self.js("exfiltrate"))++ async def _refresh_exfiltration_loop(self) -> None:+ while True:+ await asyncio.sleep(self.REFRESH_INTERVAL_SECONDS)++ browser_context = self.browser_context+ if not browser_context:+ continue++ for page in list(browser_context.pages):+ if page.url.startswith("devtools:"):+ continue+ try:+ await self.exfiltrate(page)+ except Exception:+ LOG.debug(+ f"{self.class_name} failed to refresh exfiltration on page",+ url=page.url,+ exc_info=True,+ )def _handle_cdp_event(self, event_name: str, params: dict) -> None:LOG.debug(f"{self.class_name} cdp event captured: {event_name}", params=params)@@ -136,12 +338,29 @@ class ExfiltrationChannel(CdpChannel):if page.url.startswith("devtools:"):return self+ existing_capture = self._page_console_captures.get(page)+ if existing_capture:+ self._active_binding_channels[page] = self+ await self._install_exfiltration_script(page, add_init_script=False)+ return self+LOG.info(f"{self.class_name} setting up exfiltration on new page.", url=page.url)- page.on("console", self._handle_console_event)+ await self._ensure_binding(page)- await page.add_init_script(self.js("exfiltrate"))- await page.evaluate(self.js("exfiltrate"))+ def console_listener(msg: ConsoleMessage) -> None:+ self._handle_console_event(msg)++ page.on("console", console_listener)++ await self._install_exfiltration_script(page, add_init_script=True)++ capture = PageConsoleCapture(console_listener=console_listener)+ self._page_console_captures[page] = capture+ try:+ capture.cdp_session = await self._attach_page_cdp_console_capture(page)+ except Exception:+ LOG.debug(f"{self.class_name} failed to attach page CDP EXFIL listener", page_url=page.url, exc_info=True)LOG.info(f"{self.class_name} setup complete on page.", url=page.url)@@ -267,30 +486,70 @@ class ExfiltrationChannel(CdpChannel):self.enable_decoration()+ if self._refresh_task is None or self._refresh_task.done():+ self._refresh_task = asyncio.create_task(self._refresh_exfiltration_loop())+return selfasync def stop(self) -> t.Self:LOG.info(f"{self.class_name} stopping.")- if not self.cdp_session:- return self+ if self._refresh_task:+ self._refresh_task.cancel()+ with contextlib.suppress(asyncio.CancelledError):+ await self._refresh_task+ self._refresh_task = None- try:- await self.cdp_session.detach()- except Exception:- pass+ pending_event_tasks = list(self._pending_event_tasks)+ for task in pending_event_tasks:+ task.cancel()+ if pending_event_tasks:+ await asyncio.gather(*pending_event_tasks, return_exceptions=True)+ self._pending_event_tasks.clear()++ if self.cdp_session:+ try:+ await self.cdp_session.detach()+ except Exception:+ passself.cdp_session = None- pages = self.browser_context.pages if self.browser_context else []+ captures = list(self._page_console_captures.items())+ self._page_console_captures.clear()+ pages = [page for page, _ in captures]- for page in pages:+ if self.browser_context:+ for page in self.browser_context.pages:+ if all(existing is not page for existing in pages):+ pages.append(page)++ for page, capture in captures:try:- page.remove_listener("console", self._handle_console_event)+ page.remove_listener("console", capture.console_listener)except KeyError:- pass # listener not found+ pass++ if capture.cdp_session:+ try:+ await capture.cdp_session.detach()+ except Exception:+ pass++ for page in pages:+ if self._active_binding_channels.get(page) is self:+ self._active_binding_channels.pop(page, None)+ self._binding_registered_pages.discard(page)++ try:+ await page.evaluate("window.__skyvern_exfiltration_binding_name = null;")+ except Exception:+ LOG.debug(f"{self.class_name} failed to clear exfiltration binding name", url=page.url, exc_info=True)+await self.undecorate(page)+ self._recent_console_event_fingerprints.clear()+LOG.info(f"{self.class_name} stopped.")return selfdiff --git a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.jsindex 4df444276..747acfb91 100644--- a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js+++ b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js@@ -211,6 +211,16 @@},};+ const bindingName = window.__skyvern_exfiltration_binding_name;+ const binding =+ typeof bindingName === "string" ? window[bindingName] : null;++ if (typeof binding === "function") {+ Promise.resolve(binding(eventData)).catch((err) => {+ console.log("[SYS] exfiltration: binding transport failed.", err);+ });+ }+console.log("[EXFIL]", JSON.stringify(eventData));},true,diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.pyindex 36864b6e4..27bbaf1ea 100644--- a/skyvern/webeye/actions/parse_actions.py+++ b/skyvern/webeye/actions/parse_actions.py@@ -8,6 +8,7 @@ from opentelemetry import trace as otel_tracefrom pydantic import ValidationErrorfrom skyvern.constants import EXTRACT_ACTION_SCROLL_AMOUNT, SCROLL_AMOUNT_MULTIPLIER+from skyvern.errors.errors import GetTOTPVerificationCodeError, MissingTOTPSourceErrorfrom skyvern.exceptions import FailedToGetTOTPVerificationCode, NoTOTPVerificationCodeFound, UnsupportedActionTypefrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine@@ -22,6 +23,7 @@ from skyvern.services.otp_service import (try_generate_totp_from_credential,)from skyvern.utils.image_resizer import Resolution, scale_coordinates+from skyvern.utils.url_validators import strip_query_paramsfrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.actions import (Action,@@ -55,6 +57,29 @@ from skyvern.webeye.scraper.scraped_page import ScrapedPageLOG = structlog.get_logger()+def _has_credential_totp_candidate(workflow_run_id: str | None) -> bool:+ # Mirrors try_generate_totp_from_credential selection: active-with-TOTP, or+ # exactly one TOTP-bearing candidate when no active is set (multi-no-active -> False).+ if not workflow_run_id:+ return False+ workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(workflow_run_id)+ if not workflow_run_context:+ return False++ current_context = skyvern_context.current()+ active_credential_key = current_context.active_credential_parameter_key if current_context else None+ if active_credential_key:+ value = workflow_run_context.values.get(active_credential_key)+ return isinstance(value, dict) and isinstance(value.get("totp"), str)++ candidate_keys = [+ key+ for key, value in workflow_run_context.values.items()+ if isinstance(value, dict) and isinstance(value.get("totp"), str)+ ]+ return len(candidate_keys) == 1++def parse_action(action: Dict[str, Any],scraped_page: ScrapedPage,@@ -916,6 +941,7 @@ async def generate_cua_fallback_actions(action = TerminateAction(reasoning=reasoning,intention=reasoning,+ errors=[GetTOTPVerificationCodeError(reason=reasoning_suffix).to_user_defined_error()],)except FailedToGetTOTPVerificationCode as e:reasoning_suffix = f"Failed to get magic link. Reason: {e.reason}"@@ -923,11 +949,27 @@ async def generate_cua_fallback_actions(action = TerminateAction(reasoning=reasoning,intention=reasoning,+ errors=[GetTOTPVerificationCodeError(reason=reasoning_suffix).to_user_defined_error()],)else:+ missing_source_error = MissingTOTPSourceError()+ LOG.warning(+ "Magic-link action requested but no TOTP source configured",+ task_id=task.task_id,+ workflow_run_id=task.workflow_run_id,+ totp_verification_url=strip_query_params(task.totp_verification_url)+ if task.totp_verification_url+ else None,+ totp_identifier=task.totp_identifier,+ organization_id=task.organization_id,+ )+ reasoning = (+ f"{reasoning}. {missing_source_error.reasoning}" if reasoning else missing_source_error.reasoning+ )action = TerminateAction(reasoning=reasoning,intention=reasoning,+ errors=[missing_source_error.to_user_defined_error()],)elif skyvern_action_type == "get_verification_code":@@ -943,7 +985,9 @@ async def generate_cua_fallback_actions(task_id=task.task_id,organization_id=task.organization_id,workflow_run_id=task.workflow_run_id,- totp_verification_url=task.totp_verification_url,+ totp_verification_url=strip_query_params(task.totp_verification_url)+ if task.totp_verification_url+ else None,totp_identifier=task.totp_identifier,)try:@@ -971,11 +1015,73 @@ async def generate_cua_fallback_actions(intention=reasoning,)else:- # Terminate the task since OTP is necessary but wasn't found/generated- action = TerminateAction(- reasoning=reasoning,- intention=reasoning,+ # Three-way classification: missing config, configured-but-empty (any of+ # URL / identifier / credential), or wrong-type otp_value. Each gets a+ # distinct customer-visible signal so webhook consumers can branch.+ has_credential = _has_credential_totp_candidate(task.workflow_run_id)+ no_source_configured = (+ otp_value is None and not task.totp_verification_url and not task.totp_identifier and not has_credential)+ configured_but_empty = otp_value is None and not no_source_configured+ if no_source_configured:+ missing_source_error = MissingTOTPSourceError()+ LOG.warning(+ "Verification-code action requested but no TOTP source configured",+ task_id=task.task_id,+ workflow_run_id=task.workflow_run_id,+ totp_verification_url=strip_query_params(task.totp_verification_url)+ if task.totp_verification_url+ else None,+ totp_identifier=task.totp_identifier,+ organization_id=task.organization_id,+ )+ reasoning = (+ f"{reasoning}. {missing_source_error.reasoning}" if reasoning else missing_source_error.reasoning+ )+ action = TerminateAction(+ reasoning=reasoning,+ intention=reasoning,+ errors=[missing_source_error.to_user_defined_error()],+ )+ elif configured_but_empty:+ # Source configured (URL, identifier, or credential) but no code obtained.+ # Could be silent poll_otp_value None-return, broken delivery, or+ # credential generation failure. Attach OTP_ERROR so programmatic+ # consumers can branch — distinguishes from the wrong-type case below.+ if task.totp_verification_url:+ polled_source = f"totp_verification_url={strip_query_params(task.totp_verification_url)}"+ elif task.totp_identifier:+ polled_source = f"totp_identifier={task.totp_identifier}"+ else:+ polled_source = "credential parameter with TOTP"+ LOG.warning(+ "TOTP source configured but produced no code",+ task_id=task.task_id,+ workflow_run_id=task.workflow_run_id,+ totp_verification_url=strip_query_params(task.totp_verification_url)+ if task.totp_verification_url+ else None,+ totp_identifier=task.totp_identifier,+ has_credential_totp=has_credential,+ organization_id=task.organization_id,+ )+ reasoning_suffix = (+ f"Configured TOTP source ({polled_source}) produced no code — "+ "check delivery endpoint or contact Skyvern support if the issue persists."+ )+ reasoning = f"{reasoning}. {reasoning_suffix}" if reasoning else reasoning_suffix+ action = TerminateAction(+ reasoning=reasoning,+ intention=reasoning,+ errors=[GetTOTPVerificationCodeError(reason=reasoning_suffix).to_user_defined_error()],+ )+ else:+ # otp_value is non-None but wrong type (e.g. magic link returned for+ # a TOTP request). Source exists, wrong shape. No new error code.+ action = TerminateAction(+ reasoning=reasoning,+ intention=reasoning,+ )action.organization_id = task.organization_idaction.workflow_run_id = task.workflow_run_iddiff --git a/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py b/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.pynew file mode 100644index 000000000..66daedba4--- /dev/null+++ b/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py@@ -0,0 +1,268 @@+"""Tests for browser exfiltration channel helpers."""++from __future__ import annotations++import asyncio+import gc+import json+import typing as t+import weakref+from unittest.mock import AsyncMock, MagicMock++import pytest++from skyvern.forge.sdk.routes.streaming.channels.exfiltration import (+ ExfiltratedEventSource,+ ExfiltrationChannel,+ PageConsoleCapture,+)+++def _make_vnc_channel() -> MagicMock:+ browser_session = MagicMock()+ browser_session.browser_address = "http://localhost:9222"+ browser_session.persistent_browser_session_id = "pbs_123"++ vnc_channel = MagicMock()+ vnc_channel.browser_session = browser_session+ vnc_channel.identity = {+ "client_id": "client-1",+ "browser_session_id": browser_session.persistent_browser_session_id,+ }++ return vnc_channel+++def _make_event_data() -> dict[str, object]:+ return {+ "type": "click",+ "timestamp": 1234,+ "url": "https://example.com",+ "target": {+ "tagName": "BUTTON",+ "id": "submit",+ "skyId": "sky-1",+ "text": ["Submit"],+ },+ }+++def _make_page(url: str = "https://example.com") -> MagicMock:+ page = MagicMock()+ page.url = url+ page.context = MagicMock()+ page.on = MagicMock()+ page.remove_listener = MagicMock()+ page.add_init_script = AsyncMock()+ page.evaluate = AsyncMock()+ page.expose_binding = AsyncMock()+ return page+++def _make_channel(on_event: MagicMock | None = None) -> tuple[ExfiltrationChannel, MagicMock]:+ event_callback = on_event or MagicMock()+ return ExfiltrationChannel(on_event=event_callback, vnc_channel=_make_vnc_channel()), event_callback++[email protected](autouse=True)+def restore_exfiltration_channel_class_state() -> t.Iterator[None]:+ active_binding_channels = weakref.WeakKeyDictionary(ExfiltrationChannel._active_binding_channels)+ binding_registered_pages = weakref.WeakSet(ExfiltrationChannel._binding_registered_pages)++ yield++ ExfiltrationChannel._active_binding_channels = active_binding_channels+ ExfiltrationChannel._binding_registered_pages = binding_registered_pages+++class TestExfiltrationChannelEvents:+ def test_binding_event_emits_user_interaction(self) -> None:+ channel, on_event = _make_channel()+ page = _make_page()+ event_data = _make_event_data()+ ExfiltrationChannel._active_binding_channels[page] = channel++ channel._handle_binding_event({"page": page}, event_data)++ on_event.assert_called_once()+ emitted = on_event.call_args.args[0]+ assert len(emitted) == 1+ assert emitted[0].source == ExfiltratedEventSource.CONSOLE+ assert emitted[0].event_name == "user_interaction"+ assert emitted[0].params == event_data++ def test_page_tracking_releases_collected_pages(self) -> None:+ channel, _ = _make_channel()+ page = _make_page()+ page_ref = weakref.ref(page)++ ExfiltrationChannel._active_binding_channels[page] = channel+ ExfiltrationChannel._binding_registered_pages.add(page)+ channel._page_console_captures[page] = PageConsoleCapture(console_listener=MagicMock())++ del page+ gc.collect()++ assert page_ref() is None+ assert not ExfiltrationChannel._active_binding_channels+ assert not ExfiltrationChannel._binding_registered_pages+ assert not channel._page_console_captures++ @pytest.mark.asyncio+ async def test_playwright_console_text_payload_emits_user_interaction(self) -> None:+ channel, on_event = _make_channel()+ event_data = _make_event_data()++ message = MagicMock()+ message.args = []+ message.text = f"[EXFIL] {json.dumps(event_data)}"++ await channel._handle_console_event_async(message)++ on_event.assert_called_once()+ assert on_event.call_args.args[0][0].params == event_data++ @pytest.mark.asyncio+ async def test_playwright_console_listener_tracks_event_task(self) -> None:+ channel, _ = _make_channel()+ started = asyncio.Event()+ release = asyncio.Event()++ async def handle_console_event(_: object) -> None:+ started.set()+ await release.wait()++ channel._handle_console_event_async = handle_console_event # type: ignore[method-assign]++ channel._handle_console_event(MagicMock())+ await started.wait()++ tasks = list(channel._pending_event_tasks)+ assert len(tasks) == 1++ release.set()+ await asyncio.gather(*tasks)+ assert not channel._pending_event_tasks++ @pytest.mark.asyncio+ async def test_playwright_console_args_payload_emits_when_text_format_differs(self) -> None:+ channel, on_event = _make_channel()+ event_data = _make_event_data()++ marker = MagicMock()+ marker.json_value = AsyncMock(return_value="[EXFIL]")+ payload = MagicMock()+ payload.json_value = AsyncMock(return_value=json.dumps(event_data))+ message = MagicMock()+ message.args = [marker, payload]+ message.text = "[EXFIL] JSHandle@object"++ await channel._handle_console_event_async(message)++ on_event.assert_called_once()+ assert on_event.call_args.args[0][0].params == event_data++ @pytest.mark.asyncio+ async def test_duplicate_binding_console_and_runtime_events_emit_once(self) -> None:+ channel, on_event = _make_channel()+ page = _make_page()+ event_data = _make_event_data()+ ExfiltrationChannel._active_binding_channels[page] = channel++ message = MagicMock()+ message.args = []+ message.text = f"[EXFIL] {json.dumps(event_data)}"++ channel._handle_binding_event({"page": page}, event_data)+ await channel._handle_console_event_async(message)+ await channel._handle_runtime_console_event_async(+ {+ "args": [+ {"type": "string", "value": "[EXFIL]"},+ {"type": "string", "value": json.dumps(event_data)},+ ]+ }+ )++ on_event.assert_called_once()++ @pytest.mark.asyncio+ async def test_exfiltrate_registers_binding_console_script_and_cdp_fallback(self) -> None:+ channel, _ = _make_channel()+ page = _make_page()+ cdp_session = MagicMock()+ cdp_session.send = AsyncMock()+ cdp_session.on = MagicMock()+ page.context.new_cdp_session = AsyncMock(return_value=cdp_session)++ result = await channel.exfiltrate(page)++ assert result is channel+ page.expose_binding.assert_awaited_once_with(channel.BINDING_NAME, channel._handle_binding_event)+ page.on.assert_called_once()+ assert page.add_init_script.await_count == 2+ assert page.evaluate.await_count == 2+ assert channel._page_console_captures[page].cdp_session is cdp_session+ cdp_session.send.assert_awaited_once_with("Runtime.enable")+ cdp_session.on.assert_called_once()++ @pytest.mark.asyncio+ async def test_page_cdp_console_callback_tracks_event_task(self) -> None:+ channel, on_event = _make_channel()+ page = _make_page()+ event_data = _make_event_data()+ cdp_session = MagicMock()+ cdp_session.send = AsyncMock()+ cdp_session.on = MagicMock()+ page.context.new_cdp_session = AsyncMock(return_value=cdp_session)++ await channel._attach_page_cdp_console_capture(page)++ event_name, callback = cdp_session.on.call_args.args+ assert event_name == "Runtime.consoleAPICalled"++ callback(+ {+ "args": [+ {"type": "string", "value": "[EXFIL]"},+ {"type": "string", "value": json.dumps(event_data)},+ ]+ }+ )+ tasks = list(channel._pending_event_tasks)+ assert len(tasks) == 1++ await asyncio.gather(*tasks)+ assert not channel._pending_event_tasks+ on_event.assert_called_once()++ @pytest.mark.asyncio+ async def test_exfiltrate_keeps_binding_and_console_when_cdp_fallback_fails(self) -> None:+ channel, _ = _make_channel()+ page = _make_page()+ page.context.new_cdp_session = AsyncMock(side_effect=RuntimeError("boom"))++ result = await channel.exfiltrate(page)++ assert result is channel+ page.expose_binding.assert_awaited_once_with(channel.BINDING_NAME, channel._handle_binding_event)+ page.on.assert_called_once()+ assert channel._page_console_captures[page].cdp_session is None++ @pytest.mark.asyncio+ async def test_exfiltrate_rearms_existing_page_without_duplicate_listeners(self) -> None:+ channel, _ = _make_channel()+ page = _make_page()+ page.context.new_cdp_session = AsyncMock(side_effect=RuntimeError("boom"))++ await channel.exfiltrate(page)+ page.on.reset_mock()+ page.add_init_script.reset_mock()+ page.evaluate.reset_mock()++ result = await channel.exfiltrate(page)++ assert result is channel+ page.on.assert_not_called()+ page.add_init_script.assert_not_awaited()+ assert page.evaluate.await_count == 2diff --git a/tests/unit/test_missing_totp_source.py b/tests/unit/test_missing_totp_source.pynew file mode 100644index 000000000..86ca7ef12--- /dev/null+++ b/tests/unit/test_missing_totp_source.py@@ -0,0 +1,337 @@+from __future__ import annotations++from unittest.mock import AsyncMock, MagicMock++import pytest++from skyvern.errors.errors import (+ MissingTOTPSourceError,+ TimeoutGetTOTPVerificationCodeError,+)+from skyvern.exceptions import SkyvernException+from skyvern.forge.agent import _build_totp_timeout_reasoning+from skyvern.forge.sdk.schemas.totp_codes import OTPType+from skyvern.webeye.actions.action_types import ActionType+from skyvern.webeye.actions.actions import TerminateAction, VerificationCodeAction+from skyvern.webeye.actions.parse_actions import generate_cua_fallback_actions+++def _make_task(+ *,+ totp_verification_url: str | None = None,+ totp_identifier: str | None = None,+ organization_id: str | None = "o_test",+ workflow_run_id: str | None = "wr_test",+) -> MagicMock:+ task = MagicMock()+ task.task_id = "tsk_test"+ task.organization_id = organization_id+ task.workflow_run_id = workflow_run_id+ task.workflow_permanent_id = "wpid_test"+ task.totp_verification_url = totp_verification_url+ task.totp_identifier = totp_identifier+ task.navigation_payload = None+ task.navigation_goal = "log in"+ return task+++def _make_step() -> MagicMock:+ step = MagicMock()+ step.step_id = "stp_test"+ step.order = 0+ return step+++def _patch_cua_fallback_common(monkeypatch: pytest.MonkeyPatch, action_type: str) -> None:+ """Patch the LLM handler and credential helpers common to most tests."""++ async def _fake_llm(*args, **kwargs): # type: ignore[no-untyped-def]+ return {"action": action_type}++ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.app.LLM_API_HANDLER", AsyncMock(side_effect=_fake_llm))+ monkeypatch.setattr(+ "skyvern.webeye.actions.parse_actions.extract_totp_from_navigation_inputs",+ lambda *_: None,+ )+ monkeypatch.setattr(+ "skyvern.webeye.actions.parse_actions.try_generate_totp_from_credential",+ lambda *_: None,+ )+++# ---------- get_verification_code branch ----------+++async def test_get_verification_code_with_no_source_emits_missing_totp_source(monkeypatch: pytest.MonkeyPatch) -> None:+ task = _make_task()+ step = _make_step()+ _patch_cua_fallback_common(monkeypatch, "get_verification_code")+ monkeypatch.setattr(+ "skyvern.webeye.actions.parse_actions._has_credential_totp_candidate",+ lambda *_: False,+ )++ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)++ assert len(actions) == 1+ action = actions[0]+ assert isinstance(action, TerminateAction)+ assert action.action_type == ActionType.TERMINATE+ assert action.errors+ assert action.errors[0].error_code == MissingTOTPSourceError().error_code+ assert "no TOTP source is configured" in action.reasoning+++async def test_get_verification_code_with_url_configured_emits_configured_but_empty_reasoning(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # URL set, poll returns None (silent delivery or org-token-missing) → OTP_ERROR+ # so programmatic consumers can distinguish from wrong-type case.+ task = _make_task(totp_verification_url="https://example.invalid/totp?secret=abc123")+ step = _make_step()+ _patch_cua_fallback_common(monkeypatch, "get_verification_code")+ monkeypatch.setattr(+ "skyvern.webeye.actions.parse_actions._has_credential_totp_candidate",+ lambda *_: False,+ )++ async def _no_poll(*args, **kwargs): # type: ignore[no-untyped-def]+ return None++ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.poll_otp_value", _no_poll)++ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)++ assert len(actions) == 1+ action = actions[0]+ assert isinstance(action, TerminateAction)+ assert all(err.error_code != MissingTOTPSourceError().error_code for err in action.errors)+ assert action.errors+ assert action.errors[0].error_code == "OTP_ERROR"+ # Reasoning surfaces the polled URL, with the secret-bearing query string stripped.+ assert "Configured TOTP source" in action.reasoning+ assert "totp_verification_url=https://example.invalid/totp" in action.reasoning+ assert "secret=abc123" not in action.reasoning+++async def test_get_verification_code_with_only_credential_configured_but_failing_emits_otp_error(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # When the ONLY configured source is a credential and generation fails (returns+ # None), the customer must get OTP_ERROR with a credential-specific reasoning —+ # not silently fall into the wrong-type case with no errors.+ task = _make_task()+ step = _make_step()+ _patch_cua_fallback_common(monkeypatch, "get_verification_code")+ monkeypatch.setattr(+ "skyvern.webeye.actions.parse_actions._has_credential_totp_candidate",+ lambda *_: True,+ )++ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)++ assert len(actions) == 1+ action = actions[0]+ assert isinstance(action, TerminateAction)+ assert action.errors+ assert action.errors[0].error_code == "OTP_ERROR"+ assert "credential parameter with TOTP" in action.reasoning+++async def test_get_verification_code_with_credential_totp_returns_verification(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Guards against the diagnostic branches swallowing valid runs.+ task = _make_task()+ step = _make_step()+ _patch_cua_fallback_common(monkeypatch, "get_verification_code")++ fake_otp_value = MagicMock()+ fake_otp_value.value = "123456"+ fake_otp_value.get_otp_type.return_value = OTPType.TOTP++ monkeypatch.setattr(+ "skyvern.webeye.actions.parse_actions.try_generate_totp_from_credential",+ lambda *_: fake_otp_value,+ )++ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)++ assert len(actions) == 1+ action = actions[0]+ assert isinstance(action, VerificationCodeAction)+ assert action.verification_code == "123456"+++async def test_get_verification_code_with_multiple_unselected_credentials_emits_missing_source(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Multi-no-active: try_generate_totp_from_credential returns None per+ # otp_service.py's selection logic, and the helper reports no usable source.+ # MISSING_TOTP_SOURCE must fire — not OTP_ERROR — because nothing was actually+ # configured for THIS run.+ task = _make_task()+ step = _make_step()+ _patch_cua_fallback_common(monkeypatch, "get_verification_code")+ monkeypatch.setattr(+ "skyvern.webeye.actions.parse_actions._has_credential_totp_candidate",+ lambda *_: False, # mirroring multi-no-active selection result+ )++ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)++ assert len(actions) == 1+ action = actions[0]+ assert isinstance(action, TerminateAction)+ assert action.errors+ assert action.errors[0].error_code == MissingTOTPSourceError().error_code+++# ---------- get_magic_link branch ----------+++async def test_get_magic_link_with_no_source_emits_missing_totp_source(monkeypatch: pytest.MonkeyPatch) -> None:+ task = _make_task()+ step = _make_step()+ _patch_cua_fallback_common(monkeypatch, "get_magic_link")++ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)++ assert len(actions) == 1+ action = actions[0]+ assert isinstance(action, TerminateAction)+ assert action.errors+ assert action.errors[0].error_code == MissingTOTPSourceError().error_code+++async def test_get_magic_link_with_credential_candidate_still_emits_missing_source(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Magic-link cannot be satisfied by credential TOTP (credentials emit only+ # OTPType.TOTP). The classifier must not consult the credential probe here.+ task = _make_task()+ step = _make_step()+ _patch_cua_fallback_common(monkeypatch, "get_magic_link")++ # Even if _has_credential_totp_candidate would return True, the magic-link+ # branch must NOT consult it. Patch to True to prove the branch ignores it.+ monkeypatch.setattr(+ "skyvern.webeye.actions.parse_actions._has_credential_totp_candidate",+ lambda *_: True,+ )++ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)++ assert len(actions) == 1+ action = actions[0]+ assert isinstance(action, TerminateAction)+ assert action.errors+ assert action.errors[0].error_code == MissingTOTPSourceError().error_code+++async def test_get_magic_link_with_url_but_poll_returns_none_emits_otp_error(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Symmetry with get_verification_code: configured-but-empty must attach+ # OTP_ERROR so webhook consumers can branch failures the same way across+ # both OTP-type paths.+ task = _make_task(totp_verification_url="https://example.invalid/magic?secret=abc")+ step = _make_step()+ _patch_cua_fallback_common(monkeypatch, "get_magic_link")++ async def _no_poll(*args, **kwargs): # type: ignore[no-untyped-def]+ return None++ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.poll_otp_value", _no_poll)++ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)++ assert len(actions) == 1+ action = actions[0]+ assert isinstance(action, TerminateAction)+ assert action.errors+ assert action.errors[0].error_code == "OTP_ERROR"+ assert "No magic link found" in action.reasoning+++async def test_get_magic_link_with_failed_delivery_emits_otp_error(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # FailedToGetTOTPVerificationCode -> broken delivery endpoint, not silent+ # timeout. Must attach OTP_ERROR with the underlying reason surfaced.+ from skyvern.exceptions import FailedToGetTOTPVerificationCode++ task = _make_task(totp_verification_url="https://example.invalid/magic")+ step = _make_step()+ _patch_cua_fallback_common(monkeypatch, "get_magic_link")++ async def _fail_poll(*args, **kwargs): # type: ignore[no-untyped-def]+ raise FailedToGetTOTPVerificationCode(reason="upstream 502")++ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.poll_otp_value", _fail_poll)++ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)++ assert len(actions) == 1+ action = actions[0]+ assert isinstance(action, TerminateAction)+ assert action.errors+ assert action.errors[0].error_code == "OTP_ERROR"+ assert "upstream 502" in action.reasoning+++# ---------- agent.py _build_totp_timeout_reasoning helper ----------+++def test_non_cua_timeout_reasoning_uses_url_when_present() -> None:+ # URL precedence mirrors poll_otp_value's if/elif. Query string stripped to+ # avoid leaking auth tokens into webhook payloads.+ task = _make_task(+ totp_verification_url="https://example.invalid/totp?token=secret",+ totp_identifier="[email protected]",+ )+ reasoning = _build_totp_timeout_reasoning(task)+ assert "totp_verification_url=https://example.invalid/totp" in reasoning+ assert "token=secret" not in reasoning+ # Identifier is suppressed — only the URL was actually polled.+ assert "totp_identifier=" not in reasoning+++def test_non_cua_timeout_reasoning_uses_identifier_when_url_absent() -> None:+ task = _make_task(totp_identifier="[email protected]")+ reasoning = _build_totp_timeout_reasoning(task)+ assert "[email protected]" in reasoning+ assert "totp_verification_url=" not in reasoning+++def test_non_cua_timeout_reasoning_raises_when_called_without_source() -> None:+ # SkyvernException (not assert) so the guard survives python -O.+ task = _make_task(totp_verification_url=None, totp_identifier=None)+ with pytest.raises(SkyvernException):+ _build_totp_timeout_reasoning(task)+++# ---------- wire-format pins ----------+++def test_missing_totp_source_error_to_user_defined_error_shape() -> None:+ # Pins the wire format. Customers may have alerts keyed on the exact string.+ err = MissingTOTPSourceError().to_user_defined_error()+ assert err.error_code == "MISSING_TOTP_SOURCE"+ assert err.confidence_float == 1.0+ assert "totp_verification_url" in err.reasoning or "totp_identifier" in err.reasoning+++def test_timeout_error_unchanged_for_backcompat() -> None:+ # OTP_TIMEOUT wire string must remain stable — customers may key alerting on it.+ err = TimeoutGetTOTPVerificationCodeError().to_user_defined_error()+ assert err.error_code == "OTP_TIMEOUT"