Skip to content

Automate browser based workflows with AI

fix(SKY-9804): distinguish missing-TOTP-source from delivery timeout in MFA terminate paths (#5987)

pedrohsdb57193ef453ee
diff --git a/skyvern/errors/errors.py b/skyvern/errors/errors.py
index d1c83e8ab..f281365b2 100644
--- a/skyvern/errors/errors.py
+++ b/skyvern/errors/errors.py
@@ -74,3 +74,12 @@ class TimeoutGetTOTPVerificationCodeError(SkyvernDefinedError):
class TOTPExpiredError(SkyvernDefinedError):
error_code: str = "OTP_EXPIRED"
reasoning: str = "TOTP verification code has expired during multi-field input sequence."
+
+
+class MissingTOTPSourceError(SkyvernDefinedError):
+ error_code: str = "MISSING_TOTP_SOURCE"
+ reasoning: str = (
+ "MFA was required but no TOTP source is configured. Configure one of: a totp_verification_url or "
+ "totp_identifier on the login block (or run override), or attach a credential parameter whose "
+ "credential stores a TOTP secret."
+ )
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index f3d823f4c..8ec227237 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -55,6 +55,7 @@ from skyvern.exceptions import (
NoTOTPVerificationCodeFound,
PDFEmbedBase64DecodeError,
ScrapingFailed,
+ SkyvernException,
StepTerminationError,
StepUnableToExecuteError,
TaskAlreadyCanceled,
@@ -298,6 +299,22 @@ def _schedule_summary_shadow_check_for_hit(
)
+def _build_totp_timeout_reasoning(task: Task) -> str:
+ # Mirror poll_otp_value's URL-then-identifier precedence so we only report the
+ # source that was actually queried. URL goes through strip_query_params to keep
+ # query-string tokens out of the customer-visible failure_reason.
+ if not (task.totp_verification_url or task.totp_identifier):
+ raise SkyvernException(
+ "_build_totp_timeout_reasoning called with no TOTP source; "
+ "NoTOTPVerificationCodeFound should only fire when poll_otp_value was invoked"
+ )
+ if task.totp_verification_url:
+ polled = f"totp_verification_url={strip_query_params(task.totp_verification_url)}"
+ else:
+ polled = f"totp_identifier={task.totp_identifier}"
+ return f"No TOTP verification code found. Going to terminate. Polled source: {polled}."
+
+
class ForgeAgent:
def __init__(self) -> None:
self.async_operation_pool = AsyncOperationPool()
@@ -1444,6 +1461,20 @@ class ForgeAgent:
if context:
context.pop_totp_code(task.task_id)
except NoTOTPVerificationCodeFound:
+ # Surface only the source poll_otp_value actually queried so
+ # the failure_reason tells the customer which delivery endpoint
+ # went silent.
+ timeout_reasoning = _build_totp_timeout_reasoning(task)
+ LOG.warning(
+ "TOTP polling timed out — terminating task",
+ task_id=task.task_id,
+ workflow_run_id=task.workflow_run_id,
+ totp_verification_url=strip_query_params(task.totp_verification_url)
+ if task.totp_verification_url
+ else None,
+ totp_identifier=task.totp_identifier,
+ organization_id=task.organization_id,
+ )
actions = [
TerminateAction(
organization_id=task.organization_id,
@@ -1452,8 +1483,8 @@ class ForgeAgent:
step_id=step.step_id,
step_order=step.order,
action_order=0,
- reasoning="No TOTP verification code found. Going to terminate.",
- intention="No TOTP verification code found. Going to terminate.",
+ reasoning=timeout_reasoning,
+ intention=timeout_reasoning,
errors=[TimeoutGetTOTPVerificationCodeError().to_user_defined_error()],
)
]
diff --git a/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py b/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py
index 233fdc9f7..7e8da57da 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/exfiltration.py
@@ -11,11 +11,13 @@ Channel data:
"""
import asyncio
+import contextlib
import dataclasses
import enum
import json
import time
import typing as t
+import weakref
import structlog
from playwright.async_api import CDPSession, ConsoleMessage, Page
@@ -46,37 +48,237 @@ class ExfiltratedEvent:
OnExfiltrationEvent = t.Callable[[list[ExfiltratedEvent]], None]
+class PageConsoleCapture:
+ console_listener: t.Callable[[ConsoleMessage], object]
+ cdp_session: CDPSession | None = None
+
+
class ExfiltrationChannel(CdpChannel):
"""
ExfiltrationChannel.
"""
+ BINDING_NAME: t.ClassVar[str] = "__skyvern_exfiltrate_event"
+ CONSOLE_DEDUP_TTL_SECONDS: t.ClassVar[float] = 5.0
+ REFRESH_INTERVAL_SECONDS: t.ClassVar[float] = 1.0
+ _active_binding_channels: t.ClassVar[weakref.WeakKeyDictionary[Page, "ExfiltrationChannel"]] = (
+ weakref.WeakKeyDictionary()
+ )
+ _binding_registered_pages: t.ClassVar[weakref.WeakSet[Page]] = weakref.WeakSet()
+
def __init__(self, *, on_event: OnExfiltrationEvent, vnc_channel: VncChannel) -> None:
self.cdp_session: CDPSession | None = None
self.on_event = on_event
+ self._page_console_captures: weakref.WeakKeyDictionary[Page, PageConsoleCapture] = weakref.WeakKeyDictionary()
+ self._recent_console_event_fingerprints: dict[str, float] = {}
+ self._pending_event_tasks: set[asyncio.Task[None]] = set()
+ self._refresh_task: asyncio.Task | None = None
super().__init__(vnc_channel=vnc_channel)
- def _handle_console_event(self, msg: ConsoleMessage) -> None:
- """Parse console messages for exfiltrated event data."""
- text = msg.text
- if text.startswith("[EXFIL]"):
+ def _track_event_task(self, coro: t.Coroutine[t.Any, t.Any, None]) -> None:
+ task = asyncio.create_task(coro)
+ self._pending_event_tasks.add(task)
+ task.add_done_callback(self._on_event_task_done)
+
+ def _on_event_task_done(self, task: asyncio.Task[None]) -> None:
+ self._pending_event_tasks.discard(task)
+ if task.cancelled():
+ return
+
+ try:
+ task.result()
+ except Exception:
+ LOG.exception(f"{self.class_name} async exfiltration event task failed")
+
+ def _parse_exfil_payload(self, payload: object) -> dict[str, t.Any] | None:
+ if isinstance(payload, str):
try:
- event_data = json.loads(text[7:]) # Strip '[EXFIL]' prefix
-
- messages = [
- ExfiltratedEvent(
- kind="exfiltrated-event",
- event_name="user_interaction",
- params=event_data,
- source=ExfiltratedEventSource.CONSOLE,
- timestamp=time.time(),
- ),
- ]
-
- self.on_event(messages)
- except Exception:
- LOG.exception(f"{self.class_name} Failed to parse exfiltrated event", text=text)
+ payload = json.loads(payload)
+ except json.JSONDecodeError:
+ return None
+
+ if isinstance(payload, dict):
+ return t.cast(dict[str, t.Any], payload)
+
+ return None
+
+ def _parse_exfil_text(self, text: str) -> dict[str, t.Any] | None:
+ if not text.startswith("[EXFIL]"):
+ return None
+
+ return self._parse_exfil_payload(text[7:].strip())
+
+ def _parse_exfil_args(self, args: list[object]) -> dict[str, t.Any] | None:
+ if len(args) < 2 or args[0] != "[EXFIL]":
+ return None
+
+ return self._parse_exfil_payload(args[1])
+
+ def _extract_cdp_remote_object_value(self, arg: object) -> object:
+ if not isinstance(arg, dict):
+ return arg
+
+ if "value" in arg:
+ return arg["value"]
+
+ if arg.get("type") == "string" and "description" in arg:
+ return arg["description"]
+
+ preview = arg.get("preview")
+ if isinstance(preview, dict):
+ properties = preview.get("properties")
+ if isinstance(properties, list):
+ materialized: dict[str, object] = {}
+ for prop in properties:
+ if not isinstance(prop, dict):
+ continue
+ name = prop.get("name")
+ if not isinstance(name, str):
+ continue
+ if "value" in prop:
+ materialized[name] = prop["value"]
+ elif isinstance(prop.get("valuePreview"), dict) and "value" in prop["valuePreview"]:
+ materialized[name] = prop["valuePreview"]["value"]
+ if materialized:
+ return materialized
+
+ return arg
+
+ def _prune_console_dedup_cache(self, now: float) -> None:
+ expired = [
+ fingerprint
+ for fingerprint, emitted_at in self._recent_console_event_fingerprints.items()
+ if now - emitted_at > self.CONSOLE_DEDUP_TTL_SECONDS
+ ]
+ for fingerprint in expired:
+ self._recent_console_event_fingerprints.pop(fingerprint, None)
+
+ def _should_emit_console_event(self, event_data: dict[str, t.Any]) -> bool:
+ try:
+ fingerprint = json.dumps(event_data, sort_keys=True, separators=(",", ":"))
+ except TypeError:
+ fingerprint = json.dumps(event_data, sort_keys=True, separators=(",", ":"), default=str)
+
+ now = time.monotonic()
+ self._prune_console_dedup_cache(now)
+ previous = self._recent_console_event_fingerprints.get(fingerprint)
+ if previous is not None and now - previous <= self.CONSOLE_DEDUP_TTL_SECONDS:
+ return False
+
+ self._recent_console_event_fingerprints[fingerprint] = now
+ return True
+
+ def _emit_console_event(self, event_data: dict[str, t.Any]) -> None:
+ if not self._should_emit_console_event(event_data):
+ return
+
+ self.on_event(
+ [
+ ExfiltratedEvent(
+ kind="exfiltrated-event",
+ event_name="user_interaction",
+ params=event_data,
+ source=ExfiltratedEventSource.CONSOLE,
+ timestamp=time.time(),
+ )
+ ]
+ )
+
+ def _handle_binding_event(self, source: dict[str, t.Any], payload: object) -> None:
+ page = source.get("page") if isinstance(source, dict) else None
+ active_channel = self._active_binding_channels.get(page, self) if page else self
+ event_data = active_channel._parse_exfil_payload(payload)
+ if event_data is None:
+ return
+
+ active_channel._emit_console_event(event_data)
+
+ async def _handle_console_event_async(self, msg: ConsoleMessage) -> None:
+ """Parse Playwright console messages for exfiltrated event data."""
+ event_data: dict[str, t.Any] | None = None
+ try:
+ args = []
+ for arg in msg.args[:2]:
+ args.append(await arg.json_value())
+ event_data = self._parse_exfil_args(args)
+ except Exception:
+ LOG.debug(f"{self.class_name} Failed to inspect console args for EXFIL event", exc_info=True)
+
+ text = msg.text
+ if event_data is None:
+ event_data = self._parse_exfil_text(text)
+
+ if event_data is None:
+ return
+
+ self._emit_console_event(event_data)
+
+ def _handle_console_event(self, msg: ConsoleMessage) -> None:
+ self._track_event_task(self._handle_console_event_async(msg))
+
+ async def _handle_runtime_console_event_async(self, params: dict[str, t.Any]) -> None:
+ raw_args = params.get("args")
+ if not isinstance(raw_args, list):
+ return
+
+ event_data = self._parse_exfil_args([self._extract_cdp_remote_object_value(arg) for arg in raw_args[:2]])
+ if event_data is None:
+ return
+
+ self._emit_console_event(event_data)
+
+ async def _attach_page_cdp_console_capture(self, page: Page) -> CDPSession | None:
+ cdp_session = await page.context.new_cdp_session(page)
+ await cdp_session.send("Runtime.enable")
+ cdp_session.on(
+ "Runtime.consoleAPICalled",
+ lambda params: self._track_event_task(self._handle_runtime_console_event_async(params)),
+ )
+ return cdp_session
+
+ async def _ensure_binding(self, page: Page) -> None:
+ self._active_binding_channels[page] = self
+
+ if page in self._binding_registered_pages:
+ return
+
+ try:
+ await page.expose_binding(self.BINDING_NAME, self._handle_binding_event)
+ self._binding_registered_pages.add(page)
+ except Exception:
+ LOG.debug(f"{self.class_name} failed to expose exfiltration binding", page_url=page.url, exc_info=True)
+
+ async def _install_exfiltration_script(self, page: Page, *, add_init_script: bool) -> None:
+ binding_script = f"window.__skyvern_exfiltration_binding_name = {json.dumps(self.BINDING_NAME)};"
+
+ if add_init_script:
+ await page.add_init_script(binding_script)
+ await page.add_init_script(self.js("exfiltrate"))
+
+ await page.evaluate(binding_script)
+ await page.evaluate(self.js("exfiltrate"))
+
+ async def _refresh_exfiltration_loop(self) -> None:
+ while True:
+ await asyncio.sleep(self.REFRESH_INTERVAL_SECONDS)
+
+ browser_context = self.browser_context
+ if not browser_context:
+ continue
+
+ for page in list(browser_context.pages):
+ if page.url.startswith("devtools:"):
+ continue
+ try:
+ await self.exfiltrate(page)
+ except Exception:
+ LOG.debug(
+ f"{self.class_name} failed to refresh exfiltration on page",
+ url=page.url,
+ exc_info=True,
+ )
def _handle_cdp_event(self, event_name: str, params: dict) -> None:
LOG.debug(f"{self.class_name} cdp event captured: {event_name}", params=params)
@@ -136,12 +338,29 @@ class ExfiltrationChannel(CdpChannel):
if page.url.startswith("devtools:"):
return self
+ existing_capture = self._page_console_captures.get(page)
+ if existing_capture:
+ self._active_binding_channels[page] = self
+ await self._install_exfiltration_script(page, add_init_script=False)
+ return self
+
LOG.info(f"{self.class_name} setting up exfiltration on new page.", url=page.url)
- page.on("console", self._handle_console_event)
+ await self._ensure_binding(page)
- await page.add_init_script(self.js("exfiltrate"))
- await page.evaluate(self.js("exfiltrate"))
+ def console_listener(msg: ConsoleMessage) -> None:
+ self._handle_console_event(msg)
+
+ page.on("console", console_listener)
+
+ await self._install_exfiltration_script(page, add_init_script=True)
+
+ capture = PageConsoleCapture(console_listener=console_listener)
+ self._page_console_captures[page] = capture
+ try:
+ capture.cdp_session = await self._attach_page_cdp_console_capture(page)
+ except Exception:
+ LOG.debug(f"{self.class_name} failed to attach page CDP EXFIL listener", page_url=page.url, exc_info=True)
LOG.info(f"{self.class_name} setup complete on page.", url=page.url)
@@ -267,30 +486,70 @@ class ExfiltrationChannel(CdpChannel):
self.enable_decoration()
+ if self._refresh_task is None or self._refresh_task.done():
+ self._refresh_task = asyncio.create_task(self._refresh_exfiltration_loop())
+
return self
async def stop(self) -> t.Self:
LOG.info(f"{self.class_name} stopping.")
- if not self.cdp_session:
- return self
+ if self._refresh_task:
+ self._refresh_task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await self._refresh_task
+ self._refresh_task = None
- try:
- await self.cdp_session.detach()
- except Exception:
- pass
+ pending_event_tasks = list(self._pending_event_tasks)
+ for task in pending_event_tasks:
+ task.cancel()
+ if pending_event_tasks:
+ await asyncio.gather(*pending_event_tasks, return_exceptions=True)
+ self._pending_event_tasks.clear()
+
+ if self.cdp_session:
+ try:
+ await self.cdp_session.detach()
+ except Exception:
+ pass
self.cdp_session = None
- pages = self.browser_context.pages if self.browser_context else []
+ captures = list(self._page_console_captures.items())
+ self._page_console_captures.clear()
+ pages = [page for page, _ in captures]
- for page in pages:
+ if self.browser_context:
+ for page in self.browser_context.pages:
+ if all(existing is not page for existing in pages):
+ pages.append(page)
+
+ for page, capture in captures:
try:
- page.remove_listener("console", self._handle_console_event)
+ page.remove_listener("console", capture.console_listener)
except KeyError:
- pass # listener not found
+ pass
+
+ if capture.cdp_session:
+ try:
+ await capture.cdp_session.detach()
+ except Exception:
+ pass
+
+ for page in pages:
+ if self._active_binding_channels.get(page) is self:
+ self._active_binding_channels.pop(page, None)
+ self._binding_registered_pages.discard(page)
+
+ try:
+ await page.evaluate("window.__skyvern_exfiltration_binding_name = null;")
+ except Exception:
+ LOG.debug(f"{self.class_name} failed to clear exfiltration binding name", url=page.url, exc_info=True)
+
await self.undecorate(page)
+ self._recent_console_event_fingerprints.clear()
+
LOG.info(f"{self.class_name} stopped.")
return self
diff --git a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js
index 4df444276..747acfb91 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js
+++ b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js
@@ -211,6 +211,16 @@
},
};
+ const bindingName = window.__skyvern_exfiltration_binding_name;
+ const binding =
+ typeof bindingName === "string" ? window[bindingName] : null;
+
+ if (typeof binding === "function") {
+ Promise.resolve(binding(eventData)).catch((err) => {
+ console.log("[SYS] exfiltration: binding transport failed.", err);
+ });
+ }
+
console.log("[EXFIL]", JSON.stringify(eventData));
},
true,
diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.py
index 36864b6e4..27bbaf1ea 100644
--- a/skyvern/webeye/actions/parse_actions.py
+++ b/skyvern/webeye/actions/parse_actions.py
@@ -8,6 +8,7 @@ from opentelemetry import trace as otel_trace
from pydantic import ValidationError
from skyvern.constants import EXTRACT_ACTION_SCROLL_AMOUNT, SCROLL_AMOUNT_MULTIPLIER
+from skyvern.errors.errors import GetTOTPVerificationCodeError, MissingTOTPSourceError
from skyvern.exceptions import FailedToGetTOTPVerificationCode, NoTOTPVerificationCodeFound, UnsupportedActionType
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
@@ -22,6 +23,7 @@ from skyvern.services.otp_service import (
try_generate_totp_from_credential,
)
from skyvern.utils.image_resizer import Resolution, scale_coordinates
+from skyvern.utils.url_validators import strip_query_params
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import (
Action,
@@ -55,6 +57,29 @@ from skyvern.webeye.scraper.scraped_page import ScrapedPage
LOG = structlog.get_logger()
+def _has_credential_totp_candidate(workflow_run_id: str | None) -> bool:
+ # Mirrors try_generate_totp_from_credential selection: active-with-TOTP, or
+ # exactly one TOTP-bearing candidate when no active is set (multi-no-active -> False).
+ if not workflow_run_id:
+ return False
+ workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(workflow_run_id)
+ if not workflow_run_context:
+ return False
+
+ current_context = skyvern_context.current()
+ active_credential_key = current_context.active_credential_parameter_key if current_context else None
+ if active_credential_key:
+ value = workflow_run_context.values.get(active_credential_key)
+ return isinstance(value, dict) and isinstance(value.get("totp"), str)
+
+ candidate_keys = [
+ key
+ for key, value in workflow_run_context.values.items()
+ if isinstance(value, dict) and isinstance(value.get("totp"), str)
+ ]
+ return len(candidate_keys) == 1
+
+
def parse_action(
action: Dict[str, Any],
scraped_page: ScrapedPage,
@@ -916,6 +941,7 @@ async def generate_cua_fallback_actions(
action = TerminateAction(
reasoning=reasoning,
intention=reasoning,
+ errors=[GetTOTPVerificationCodeError(reason=reasoning_suffix).to_user_defined_error()],
)
except FailedToGetTOTPVerificationCode as e:
reasoning_suffix = f"Failed to get magic link. Reason: {e.reason}"
@@ -923,11 +949,27 @@ async def generate_cua_fallback_actions(
action = TerminateAction(
reasoning=reasoning,
intention=reasoning,
+ errors=[GetTOTPVerificationCodeError(reason=reasoning_suffix).to_user_defined_error()],
)
else:
+ missing_source_error = MissingTOTPSourceError()
+ LOG.warning(
+ "Magic-link action requested but no TOTP source configured",
+ task_id=task.task_id,
+ workflow_run_id=task.workflow_run_id,
+ totp_verification_url=strip_query_params(task.totp_verification_url)
+ if task.totp_verification_url
+ else None,
+ totp_identifier=task.totp_identifier,
+ organization_id=task.organization_id,
+ )
+ reasoning = (
+ f"{reasoning}. {missing_source_error.reasoning}" if reasoning else missing_source_error.reasoning
+ )
action = TerminateAction(
reasoning=reasoning,
intention=reasoning,
+ errors=[missing_source_error.to_user_defined_error()],
)
elif skyvern_action_type == "get_verification_code":
@@ -943,7 +985,9 @@ async def generate_cua_fallback_actions(
task_id=task.task_id,
organization_id=task.organization_id,
workflow_run_id=task.workflow_run_id,
- totp_verification_url=task.totp_verification_url,
+ totp_verification_url=strip_query_params(task.totp_verification_url)
+ if task.totp_verification_url
+ else None,
totp_identifier=task.totp_identifier,
)
try:
@@ -971,11 +1015,73 @@ async def generate_cua_fallback_actions(
intention=reasoning,
)
else:
- # Terminate the task since OTP is necessary but wasn't found/generated
- action = TerminateAction(
- reasoning=reasoning,
- intention=reasoning,
+ # Three-way classification: missing config, configured-but-empty (any of
+ # URL / identifier / credential), or wrong-type otp_value. Each gets a
+ # distinct customer-visible signal so webhook consumers can branch.
+ has_credential = _has_credential_totp_candidate(task.workflow_run_id)
+ no_source_configured = (
+ otp_value is None and not task.totp_verification_url and not task.totp_identifier and not has_credential
)
+ configured_but_empty = otp_value is None and not no_source_configured
+ if no_source_configured:
+ missing_source_error = MissingTOTPSourceError()
+ LOG.warning(
+ "Verification-code action requested but no TOTP source configured",
+ task_id=task.task_id,
+ workflow_run_id=task.workflow_run_id,
+ totp_verification_url=strip_query_params(task.totp_verification_url)
+ if task.totp_verification_url
+ else None,
+ totp_identifier=task.totp_identifier,
+ organization_id=task.organization_id,
+ )
+ reasoning = (
+ f"{reasoning}. {missing_source_error.reasoning}" if reasoning else missing_source_error.reasoning
+ )
+ action = TerminateAction(
+ reasoning=reasoning,
+ intention=reasoning,
+ errors=[missing_source_error.to_user_defined_error()],
+ )
+ elif configured_but_empty:
+ # Source configured (URL, identifier, or credential) but no code obtained.
+ # Could be silent poll_otp_value None-return, broken delivery, or
+ # credential generation failure. Attach OTP_ERROR so programmatic
+ # consumers can branch — distinguishes from the wrong-type case below.
+ if task.totp_verification_url:
+ polled_source = f"totp_verification_url={strip_query_params(task.totp_verification_url)}"
+ elif task.totp_identifier:
+ polled_source = f"totp_identifier={task.totp_identifier}"
+ else:
+ polled_source = "credential parameter with TOTP"
+ LOG.warning(
+ "TOTP source configured but produced no code",
+ task_id=task.task_id,
+ workflow_run_id=task.workflow_run_id,
+ totp_verification_url=strip_query_params(task.totp_verification_url)
+ if task.totp_verification_url
+ else None,
+ totp_identifier=task.totp_identifier,
+ has_credential_totp=has_credential,
+ organization_id=task.organization_id,
+ )
+ reasoning_suffix = (
+ f"Configured TOTP source ({polled_source}) produced no code — "
+ "check delivery endpoint or contact Skyvern support if the issue persists."
+ )
+ reasoning = f"{reasoning}. {reasoning_suffix}" if reasoning else reasoning_suffix
+ action = TerminateAction(
+ reasoning=reasoning,
+ intention=reasoning,
+ errors=[GetTOTPVerificationCodeError(reason=reasoning_suffix).to_user_defined_error()],
+ )
+ else:
+ # otp_value is non-None but wrong type (e.g. magic link returned for
+ # a TOTP request). Source exists, wrong shape. No new error code.
+ action = TerminateAction(
+ reasoning=reasoning,
+ intention=reasoning,
+ )
action.organization_id = task.organization_id
action.workflow_run_id = task.workflow_run_id
diff --git a/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py b/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py
new file mode 100644
index 000000000..66daedba4
--- /dev/null
+++ b/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py
@@ -0,0 +1,268 @@
+"""Tests for browser exfiltration channel helpers."""
+
+from __future__ import annotations
+
+import asyncio
+import gc
+import json
+import typing as t
+import weakref
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from skyvern.forge.sdk.routes.streaming.channels.exfiltration import (
+ ExfiltratedEventSource,
+ ExfiltrationChannel,
+ PageConsoleCapture,
+)
+
+
+def _make_vnc_channel() -> MagicMock:
+ browser_session = MagicMock()
+ browser_session.browser_address = "http://localhost:9222"
+ browser_session.persistent_browser_session_id = "pbs_123"
+
+ vnc_channel = MagicMock()
+ vnc_channel.browser_session = browser_session
+ vnc_channel.identity = {
+ "client_id": "client-1",
+ "browser_session_id": browser_session.persistent_browser_session_id,
+ }
+
+ return vnc_channel
+
+
+def _make_event_data() -> dict[str, object]:
+ return {
+ "type": "click",
+ "timestamp": 1234,
+ "url": "https://example.com",
+ "target": {
+ "tagName": "BUTTON",
+ "id": "submit",
+ "skyId": "sky-1",
+ "text": ["Submit"],
+ },
+ }
+
+
+def _make_page(url: str = "https://example.com") -> MagicMock:
+ page = MagicMock()
+ page.url = url
+ page.context = MagicMock()
+ page.on = MagicMock()
+ page.remove_listener = MagicMock()
+ page.add_init_script = AsyncMock()
+ page.evaluate = AsyncMock()
+ page.expose_binding = AsyncMock()
+ return page
+
+
+def _make_channel(on_event: MagicMock | None = None) -> tuple[ExfiltrationChannel, MagicMock]:
+ event_callback = on_event or MagicMock()
+ return ExfiltrationChannel(on_event=event_callback, vnc_channel=_make_vnc_channel()), event_callback
+
+
[email protected](autouse=True)
+def restore_exfiltration_channel_class_state() -> t.Iterator[None]:
+ active_binding_channels = weakref.WeakKeyDictionary(ExfiltrationChannel._active_binding_channels)
+ binding_registered_pages = weakref.WeakSet(ExfiltrationChannel._binding_registered_pages)
+
+ yield
+
+ ExfiltrationChannel._active_binding_channels = active_binding_channels
+ ExfiltrationChannel._binding_registered_pages = binding_registered_pages
+
+
+class TestExfiltrationChannelEvents:
+ def test_binding_event_emits_user_interaction(self) -> None:
+ channel, on_event = _make_channel()
+ page = _make_page()
+ event_data = _make_event_data()
+ ExfiltrationChannel._active_binding_channels[page] = channel
+
+ channel._handle_binding_event({"page": page}, event_data)
+
+ on_event.assert_called_once()
+ emitted = on_event.call_args.args[0]
+ assert len(emitted) == 1
+ assert emitted[0].source == ExfiltratedEventSource.CONSOLE
+ assert emitted[0].event_name == "user_interaction"
+ assert emitted[0].params == event_data
+
+ def test_page_tracking_releases_collected_pages(self) -> None:
+ channel, _ = _make_channel()
+ page = _make_page()
+ page_ref = weakref.ref(page)
+
+ ExfiltrationChannel._active_binding_channels[page] = channel
+ ExfiltrationChannel._binding_registered_pages.add(page)
+ channel._page_console_captures[page] = PageConsoleCapture(console_listener=MagicMock())
+
+ del page
+ gc.collect()
+
+ assert page_ref() is None
+ assert not ExfiltrationChannel._active_binding_channels
+ assert not ExfiltrationChannel._binding_registered_pages
+ assert not channel._page_console_captures
+
+ @pytest.mark.asyncio
+ async def test_playwright_console_text_payload_emits_user_interaction(self) -> None:
+ channel, on_event = _make_channel()
+ event_data = _make_event_data()
+
+ message = MagicMock()
+ message.args = []
+ message.text = f"[EXFIL] {json.dumps(event_data)}"
+
+ await channel._handle_console_event_async(message)
+
+ on_event.assert_called_once()
+ assert on_event.call_args.args[0][0].params == event_data
+
+ @pytest.mark.asyncio
+ async def test_playwright_console_listener_tracks_event_task(self) -> None:
+ channel, _ = _make_channel()
+ started = asyncio.Event()
+ release = asyncio.Event()
+
+ async def handle_console_event(_: object) -> None:
+ started.set()
+ await release.wait()
+
+ channel._handle_console_event_async = handle_console_event # type: ignore[method-assign]
+
+ channel._handle_console_event(MagicMock())
+ await started.wait()
+
+ tasks = list(channel._pending_event_tasks)
+ assert len(tasks) == 1
+
+ release.set()
+ await asyncio.gather(*tasks)
+ assert not channel._pending_event_tasks
+
+ @pytest.mark.asyncio
+ async def test_playwright_console_args_payload_emits_when_text_format_differs(self) -> None:
+ channel, on_event = _make_channel()
+ event_data = _make_event_data()
+
+ marker = MagicMock()
+ marker.json_value = AsyncMock(return_value="[EXFIL]")
+ payload = MagicMock()
+ payload.json_value = AsyncMock(return_value=json.dumps(event_data))
+ message = MagicMock()
+ message.args = [marker, payload]
+ message.text = "[EXFIL] JSHandle@object"
+
+ await channel._handle_console_event_async(message)
+
+ on_event.assert_called_once()
+ assert on_event.call_args.args[0][0].params == event_data
+
+ @pytest.mark.asyncio
+ async def test_duplicate_binding_console_and_runtime_events_emit_once(self) -> None:
+ channel, on_event = _make_channel()
+ page = _make_page()
+ event_data = _make_event_data()
+ ExfiltrationChannel._active_binding_channels[page] = channel
+
+ message = MagicMock()
+ message.args = []
+ message.text = f"[EXFIL] {json.dumps(event_data)}"
+
+ channel._handle_binding_event({"page": page}, event_data)
+ await channel._handle_console_event_async(message)
+ await channel._handle_runtime_console_event_async(
+ {
+ "args": [
+ {"type": "string", "value": "[EXFIL]"},
+ {"type": "string", "value": json.dumps(event_data)},
+ ]
+ }
+ )
+
+ on_event.assert_called_once()
+
+ @pytest.mark.asyncio
+ async def test_exfiltrate_registers_binding_console_script_and_cdp_fallback(self) -> None:
+ channel, _ = _make_channel()
+ page = _make_page()
+ cdp_session = MagicMock()
+ cdp_session.send = AsyncMock()
+ cdp_session.on = MagicMock()
+ page.context.new_cdp_session = AsyncMock(return_value=cdp_session)
+
+ result = await channel.exfiltrate(page)
+
+ assert result is channel
+ page.expose_binding.assert_awaited_once_with(channel.BINDING_NAME, channel._handle_binding_event)
+ page.on.assert_called_once()
+ assert page.add_init_script.await_count == 2
+ assert page.evaluate.await_count == 2
+ assert channel._page_console_captures[page].cdp_session is cdp_session
+ cdp_session.send.assert_awaited_once_with("Runtime.enable")
+ cdp_session.on.assert_called_once()
+
+ @pytest.mark.asyncio
+ async def test_page_cdp_console_callback_tracks_event_task(self) -> None:
+ channel, on_event = _make_channel()
+ page = _make_page()
+ event_data = _make_event_data()
+ cdp_session = MagicMock()
+ cdp_session.send = AsyncMock()
+ cdp_session.on = MagicMock()
+ page.context.new_cdp_session = AsyncMock(return_value=cdp_session)
+
+ await channel._attach_page_cdp_console_capture(page)
+
+ event_name, callback = cdp_session.on.call_args.args
+ assert event_name == "Runtime.consoleAPICalled"
+
+ callback(
+ {
+ "args": [
+ {"type": "string", "value": "[EXFIL]"},
+ {"type": "string", "value": json.dumps(event_data)},
+ ]
+ }
+ )
+ tasks = list(channel._pending_event_tasks)
+ assert len(tasks) == 1
+
+ await asyncio.gather(*tasks)
+ assert not channel._pending_event_tasks
+ on_event.assert_called_once()
+
+ @pytest.mark.asyncio
+ async def test_exfiltrate_keeps_binding_and_console_when_cdp_fallback_fails(self) -> None:
+ channel, _ = _make_channel()
+ page = _make_page()
+ page.context.new_cdp_session = AsyncMock(side_effect=RuntimeError("boom"))
+
+ result = await channel.exfiltrate(page)
+
+ assert result is channel
+ page.expose_binding.assert_awaited_once_with(channel.BINDING_NAME, channel._handle_binding_event)
+ page.on.assert_called_once()
+ assert channel._page_console_captures[page].cdp_session is None
+
+ @pytest.mark.asyncio
+ async def test_exfiltrate_rearms_existing_page_without_duplicate_listeners(self) -> None:
+ channel, _ = _make_channel()
+ page = _make_page()
+ page.context.new_cdp_session = AsyncMock(side_effect=RuntimeError("boom"))
+
+ await channel.exfiltrate(page)
+ page.on.reset_mock()
+ page.add_init_script.reset_mock()
+ page.evaluate.reset_mock()
+
+ result = await channel.exfiltrate(page)
+
+ assert result is channel
+ page.on.assert_not_called()
+ page.add_init_script.assert_not_awaited()
+ assert page.evaluate.await_count == 2
diff --git a/tests/unit/test_missing_totp_source.py b/tests/unit/test_missing_totp_source.py
new file mode 100644
index 000000000..86ca7ef12
--- /dev/null
+++ b/tests/unit/test_missing_totp_source.py
@@ -0,0 +1,337 @@
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from skyvern.errors.errors import (
+ MissingTOTPSourceError,
+ TimeoutGetTOTPVerificationCodeError,
+)
+from skyvern.exceptions import SkyvernException
+from skyvern.forge.agent import _build_totp_timeout_reasoning
+from skyvern.forge.sdk.schemas.totp_codes import OTPType
+from skyvern.webeye.actions.action_types import ActionType
+from skyvern.webeye.actions.actions import TerminateAction, VerificationCodeAction
+from skyvern.webeye.actions.parse_actions import generate_cua_fallback_actions
+
+
+def _make_task(
+ *,
+ totp_verification_url: str | None = None,
+ totp_identifier: str | None = None,
+ organization_id: str | None = "o_test",
+ workflow_run_id: str | None = "wr_test",
+) -> MagicMock:
+ task = MagicMock()
+ task.task_id = "tsk_test"
+ task.organization_id = organization_id
+ task.workflow_run_id = workflow_run_id
+ task.workflow_permanent_id = "wpid_test"
+ task.totp_verification_url = totp_verification_url
+ task.totp_identifier = totp_identifier
+ task.navigation_payload = None
+ task.navigation_goal = "log in"
+ return task
+
+
+def _make_step() -> MagicMock:
+ step = MagicMock()
+ step.step_id = "stp_test"
+ step.order = 0
+ return step
+
+
+def _patch_cua_fallback_common(monkeypatch: pytest.MonkeyPatch, action_type: str) -> None:
+ """Patch the LLM handler and credential helpers common to most tests."""
+
+ async def _fake_llm(*args, **kwargs): # type: ignore[no-untyped-def]
+ return {"action": action_type}
+
+ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.app.LLM_API_HANDLER", AsyncMock(side_effect=_fake_llm))
+ monkeypatch.setattr(
+ "skyvern.webeye.actions.parse_actions.extract_totp_from_navigation_inputs",
+ lambda *_: None,
+ )
+ monkeypatch.setattr(
+ "skyvern.webeye.actions.parse_actions.try_generate_totp_from_credential",
+ lambda *_: None,
+ )
+
+
+# ---------- get_verification_code branch ----------
+
+
+async def test_get_verification_code_with_no_source_emits_missing_totp_source(monkeypatch: pytest.MonkeyPatch) -> None:
+ task = _make_task()
+ step = _make_step()
+ _patch_cua_fallback_common(monkeypatch, "get_verification_code")
+ monkeypatch.setattr(
+ "skyvern.webeye.actions.parse_actions._has_credential_totp_candidate",
+ lambda *_: False,
+ )
+
+ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)
+
+ assert len(actions) == 1
+ action = actions[0]
+ assert isinstance(action, TerminateAction)
+ assert action.action_type == ActionType.TERMINATE
+ assert action.errors
+ assert action.errors[0].error_code == MissingTOTPSourceError().error_code
+ assert "no TOTP source is configured" in action.reasoning
+
+
+async def test_get_verification_code_with_url_configured_emits_configured_but_empty_reasoning(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # URL set, poll returns None (silent delivery or org-token-missing) → OTP_ERROR
+ # so programmatic consumers can distinguish from wrong-type case.
+ task = _make_task(totp_verification_url="https://example.invalid/totp?secret=abc123")
+ step = _make_step()
+ _patch_cua_fallback_common(monkeypatch, "get_verification_code")
+ monkeypatch.setattr(
+ "skyvern.webeye.actions.parse_actions._has_credential_totp_candidate",
+ lambda *_: False,
+ )
+
+ async def _no_poll(*args, **kwargs): # type: ignore[no-untyped-def]
+ return None
+
+ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.poll_otp_value", _no_poll)
+
+ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)
+
+ assert len(actions) == 1
+ action = actions[0]
+ assert isinstance(action, TerminateAction)
+ assert all(err.error_code != MissingTOTPSourceError().error_code for err in action.errors)
+ assert action.errors
+ assert action.errors[0].error_code == "OTP_ERROR"
+ # Reasoning surfaces the polled URL, with the secret-bearing query string stripped.
+ assert "Configured TOTP source" in action.reasoning
+ assert "totp_verification_url=https://example.invalid/totp" in action.reasoning
+ assert "secret=abc123" not in action.reasoning
+
+
+async def test_get_verification_code_with_only_credential_configured_but_failing_emits_otp_error(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # When the ONLY configured source is a credential and generation fails (returns
+ # None), the customer must get OTP_ERROR with a credential-specific reasoning —
+ # not silently fall into the wrong-type case with no errors.
+ task = _make_task()
+ step = _make_step()
+ _patch_cua_fallback_common(monkeypatch, "get_verification_code")
+ monkeypatch.setattr(
+ "skyvern.webeye.actions.parse_actions._has_credential_totp_candidate",
+ lambda *_: True,
+ )
+
+ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)
+
+ assert len(actions) == 1
+ action = actions[0]
+ assert isinstance(action, TerminateAction)
+ assert action.errors
+ assert action.errors[0].error_code == "OTP_ERROR"
+ assert "credential parameter with TOTP" in action.reasoning
+
+
+async def test_get_verification_code_with_credential_totp_returns_verification(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Guards against the diagnostic branches swallowing valid runs.
+ task = _make_task()
+ step = _make_step()
+ _patch_cua_fallback_common(monkeypatch, "get_verification_code")
+
+ fake_otp_value = MagicMock()
+ fake_otp_value.value = "123456"
+ fake_otp_value.get_otp_type.return_value = OTPType.TOTP
+
+ monkeypatch.setattr(
+ "skyvern.webeye.actions.parse_actions.try_generate_totp_from_credential",
+ lambda *_: fake_otp_value,
+ )
+
+ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)
+
+ assert len(actions) == 1
+ action = actions[0]
+ assert isinstance(action, VerificationCodeAction)
+ assert action.verification_code == "123456"
+
+
+async def test_get_verification_code_with_multiple_unselected_credentials_emits_missing_source(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Multi-no-active: try_generate_totp_from_credential returns None per
+ # otp_service.py's selection logic, and the helper reports no usable source.
+ # MISSING_TOTP_SOURCE must fire — not OTP_ERROR — because nothing was actually
+ # configured for THIS run.
+ task = _make_task()
+ step = _make_step()
+ _patch_cua_fallback_common(monkeypatch, "get_verification_code")
+ monkeypatch.setattr(
+ "skyvern.webeye.actions.parse_actions._has_credential_totp_candidate",
+ lambda *_: False, # mirroring multi-no-active selection result
+ )
+
+ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)
+
+ assert len(actions) == 1
+ action = actions[0]
+ assert isinstance(action, TerminateAction)
+ assert action.errors
+ assert action.errors[0].error_code == MissingTOTPSourceError().error_code
+
+
+# ---------- get_magic_link branch ----------
+
+
+async def test_get_magic_link_with_no_source_emits_missing_totp_source(monkeypatch: pytest.MonkeyPatch) -> None:
+ task = _make_task()
+ step = _make_step()
+ _patch_cua_fallback_common(monkeypatch, "get_magic_link")
+
+ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)
+
+ assert len(actions) == 1
+ action = actions[0]
+ assert isinstance(action, TerminateAction)
+ assert action.errors
+ assert action.errors[0].error_code == MissingTOTPSourceError().error_code
+
+
+async def test_get_magic_link_with_credential_candidate_still_emits_missing_source(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Magic-link cannot be satisfied by credential TOTP (credentials emit only
+ # OTPType.TOTP). The classifier must not consult the credential probe here.
+ task = _make_task()
+ step = _make_step()
+ _patch_cua_fallback_common(monkeypatch, "get_magic_link")
+
+ # Even if _has_credential_totp_candidate would return True, the magic-link
+ # branch must NOT consult it. Patch to True to prove the branch ignores it.
+ monkeypatch.setattr(
+ "skyvern.webeye.actions.parse_actions._has_credential_totp_candidate",
+ lambda *_: True,
+ )
+
+ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)
+
+ assert len(actions) == 1
+ action = actions[0]
+ assert isinstance(action, TerminateAction)
+ assert action.errors
+ assert action.errors[0].error_code == MissingTOTPSourceError().error_code
+
+
+async def test_get_magic_link_with_url_but_poll_returns_none_emits_otp_error(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Symmetry with get_verification_code: configured-but-empty must attach
+ # OTP_ERROR so webhook consumers can branch failures the same way across
+ # both OTP-type paths.
+ task = _make_task(totp_verification_url="https://example.invalid/magic?secret=abc")
+ step = _make_step()
+ _patch_cua_fallback_common(monkeypatch, "get_magic_link")
+
+ async def _no_poll(*args, **kwargs): # type: ignore[no-untyped-def]
+ return None
+
+ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.poll_otp_value", _no_poll)
+
+ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)
+
+ assert len(actions) == 1
+ action = actions[0]
+ assert isinstance(action, TerminateAction)
+ assert action.errors
+ assert action.errors[0].error_code == "OTP_ERROR"
+ assert "No magic link found" in action.reasoning
+
+
+async def test_get_magic_link_with_failed_delivery_emits_otp_error(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # FailedToGetTOTPVerificationCode -> broken delivery endpoint, not silent
+ # timeout. Must attach OTP_ERROR with the underlying reason surfaced.
+ from skyvern.exceptions import FailedToGetTOTPVerificationCode
+
+ task = _make_task(totp_verification_url="https://example.invalid/magic")
+ step = _make_step()
+ _patch_cua_fallback_common(monkeypatch, "get_magic_link")
+
+ async def _fail_poll(*args, **kwargs): # type: ignore[no-untyped-def]
+ raise FailedToGetTOTPVerificationCode(reason="upstream 502")
+
+ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.poll_otp_value", _fail_poll)
+
+ actions = await generate_cua_fallback_actions(task, step, assistant_message=None, reasoning=None)
+
+ assert len(actions) == 1
+ action = actions[0]
+ assert isinstance(action, TerminateAction)
+ assert action.errors
+ assert action.errors[0].error_code == "OTP_ERROR"
+ assert "upstream 502" in action.reasoning
+
+
+# ---------- agent.py _build_totp_timeout_reasoning helper ----------
+
+
+def test_non_cua_timeout_reasoning_uses_url_when_present() -> None:
+ # URL precedence mirrors poll_otp_value's if/elif. Query string stripped to
+ # avoid leaking auth tokens into webhook payloads.
+ task = _make_task(
+ totp_verification_url="https://example.invalid/totp?token=secret",
+ totp_identifier="[email protected]",
+ )
+ reasoning = _build_totp_timeout_reasoning(task)
+ assert "totp_verification_url=https://example.invalid/totp" in reasoning
+ assert "token=secret" not in reasoning
+ # Identifier is suppressed — only the URL was actually polled.
+ assert "totp_identifier=" not in reasoning
+
+
+def test_non_cua_timeout_reasoning_uses_identifier_when_url_absent() -> None:
+ task = _make_task(totp_identifier="[email protected]")
+ reasoning = _build_totp_timeout_reasoning(task)
+ assert "[email protected]" in reasoning
+ assert "totp_verification_url=" not in reasoning
+
+
+def test_non_cua_timeout_reasoning_raises_when_called_without_source() -> None:
+ # SkyvernException (not assert) so the guard survives python -O.
+ task = _make_task(totp_verification_url=None, totp_identifier=None)
+ with pytest.raises(SkyvernException):
+ _build_totp_timeout_reasoning(task)
+
+
+# ---------- wire-format pins ----------
+
+
+def test_missing_totp_source_error_to_user_defined_error_shape() -> None:
+ # Pins the wire format. Customers may have alerts keyed on the exact string.
+ err = MissingTOTPSourceError().to_user_defined_error()
+ assert err.error_code == "MISSING_TOTP_SOURCE"
+ assert err.confidence_float == 1.0
+ assert "totp_verification_url" in err.reasoning or "totp_identifier" in err.reasoning
+
+
+def test_timeout_error_unchanged_for_backcompat() -> None:
+ # OTP_TIMEOUT wire string must remain stable — customers may key alerting on it.
+ err = TimeoutGetTOTPVerificationCodeError().to_user_defined_error()
+ assert err.error_code == "OTP_TIMEOUT"