Skip to content

Automate browser based workflows with AI

fix: remove ReDoS ambiguity in PdfFillBlock test regex (code scanning 329) (#7371)

Shuchang Zheng801f6ae05a9a
diff --git a/docs/plans/2026-07-13-totp-uri-placeholder-fix.md b/docs/plans/2026-07-13-totp-uri-placeholder-fix.md
new file mode 100644
index 000000000..d65cacdfc
--- /dev/null
+++ b/docs/plans/2026-07-13-totp-uri-placeholder-fix.md
@@ -0,0 +1,28 @@
+# TOTP issuer mismatch and placeholder fail-closed plan
+
+## Scope
+
+Fix standard `otpauth://` TOTP URIs whose label issuer differs from the `issuer` query parameter, while retaining strict validation of the secret and runtime configuration. Prevent unresolved TOTP placeholders from reaching browser input paths.
+
+## Implementation
+
+1. Add regression tests in `tests/unit/test_parse_totp_secret.py` for raw and percent-encoded issuer mismatches, non-default TOTP settings, and malformed configurations that must remain rejected.
+2. Update `skyvern/forge/sdk/services/credentials.py` so strict `pyotp.parse_uri()` validation remains the default and only the issuer-mismatch error gets a validation-only retry without issuer query metadata.
+3. Add handler and generated-script tests proving missing TOTP material raises a typed failure and browser typing is never called across DOM, selectable, CUA, and autocomplete paths.
+4. Update `skyvern/webeye/actions/handler.py` and `skyvern/core/script_generations/skyvern_page.py` so every input path resolves a TOTP marker before branching or typing.
+
+## Acceptance criteria
+
+- Mismatched label/query issuers generate the correct code while preserving algorithm, digit count, and period.
+- Matching-issuer URIs and raw Base32 secrets behave unchanged.
+- Missing or invalid secrets, unsupported algorithms or digit counts, malformed periods, unknown parameters, and HOTP URIs remain rejected.
+- No unresolved `placeholder_*_totp` value is passed to a browser typing method.
+- Failures do not expose the URI, secret, placeholder, or generated code.
+
+## Verification
+
+- Run targeted parser and handler tests during TDD.
+- Compile and lint every modified Python file.
+- Run the broader credential/TOTP regression suites and all unit tests.
+- Run pre-commit on the changed files and `git diff --check`.
+- Obtain independent parser/security and action-failure reviews, then complete a separate QA pass.
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.py
index 9af6435bc..b197e3bb4 100644
--- a/skyvern/core/script_generations/real_skyvern_page_ai.py
+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py
@@ -13,7 +13,7 @@ from playwright.async_api import Page
from skyvern.config import settings
from skyvern.constants import SKYVERN_PAGE_MAX_SCRAPING_RETRIES, SPECIAL_FIELD_VERIFICATION_CODE
from skyvern.core.script_generations.skyvern_page_ai import SYSTEM_PROMPT_UNSET, SkyvernPageAi
-from skyvern.exceptions import SkyvernActionFailed, WorkflowRunContextNotInitialized
+from skyvern.exceptions import NoTOTPSecretFound, SkyvernActionFailed, WorkflowRunContextNotInitialized
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.files import validate_download_url
@@ -21,6 +21,7 @@ from skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extract
from skyvern.forge.sdk.cache import extraction_cache
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.schemas.totp_codes import OTPType
+from skyvern.forge.sdk.services.credentials import is_unresolved_totp_value
from skyvern.schemas.workflows import BlockStatus
from skyvern.services import script_service
from skyvern.services.otp_service import poll_otp_value
@@ -310,7 +311,8 @@ class RealSkyvernPageAi(SkyvernPageAi):
"""Input text into an element using AI to determine the value."""
# v3 mid-run hook — see ai_click for the contract.
- if v3_parent_episode_id is None:
+ # Never expose an unresolved TOTP value to a reviewer that can write its prompt value directly to the page.
+ if v3_parent_episode_id is None and not is_unresolved_totp_value(value):
ep_id, class_a = await self._maybe_run_v3_midrun(
action_type="fill",
failed_selector=failed_selector,
@@ -435,7 +437,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
if actions and isinstance(actions[0], InputTextAction):
action = cast(InputTextAction, actions[0])
except Exception:
- LOG.exception(f"Failed to adapt value for input text action on selector={selector}, value={value}")
+ LOG.exception("Failed to adapt value for input text action", selector=selector)
if action and organization_id and task and step:
result = await handle_input_text_action(action, self.page, self.scraped_page, task, step)
@@ -452,6 +454,8 @@ class RealSkyvernPageAi(SkyvernPageAi):
v3_parent_episode_id=v3_parent_episode_id,
)
else:
+ if is_unresolved_totp_value(transformed_value):
+ raise NoTOTPSecretFound()
locator = self.page.locator(selector)
await handler_utils.input_sequentially(locator, transformed_value, timeout=timeout)
return value
diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.py
index 7aba3b198..392c1ece4 100644
--- a/skyvern/core/script_generations/skyvern_page.py
+++ b/skyvern/core/script_generations/skyvern_page.py
@@ -19,12 +19,13 @@ from playwright.async_api import TimeoutError as PlaywrightTimeoutError
from skyvern.config import settings
from skyvern.core.script_generations.fuzzy_matcher import match_option as _match_option
from skyvern.core.script_generations.skyvern_page_ai import SkyvernPageAi
-from skyvern.exceptions import ScriptTerminationException
+from skyvern.exceptions import NoTOTPSecretFound, ScriptTerminationException
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.files import download_file as download_file_from_url
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.event.factory import EventStrategyFactory
+from skyvern.forge.sdk.services.credentials import is_unresolved_totp_value
from skyvern.library.ai_locator import AILocator
from skyvern.webeye.actions import handler_utils
from skyvern.webeye.actions.action_types import ActionType
@@ -168,6 +169,28 @@ class SkyvernPage(Page):
) -> str:
return value
+ async def _resolve_totp_placeholder_or_raise(
+ self,
+ value: str,
+ totp_identifier: str | None = None,
+ totp_url: str | None = None,
+ ) -> str:
+ if not is_unresolved_totp_value(value):
+ return value
+ try:
+ resolved_value = await self.get_actual_value(
+ value,
+ totp_identifier=totp_identifier,
+ totp_url=totp_url,
+ )
+ except NoTOTPSecretFound:
+ raise
+ except Exception as exc:
+ raise NoTOTPSecretFound() from exc
+ if is_unresolved_totp_value(resolved_value):
+ raise NoTOTPSecretFound()
+ return resolved_value
+
async def get_totp_digit(
self,
context: Any,
@@ -574,6 +597,11 @@ class SkyvernPage(Page):
raise ValueError("mode='direct' requires a selector.")
if value is None:
raise ValueError("mode='direct' requires a value.")
+ value = await self._resolve_totp_placeholder_or_raise(
+ value,
+ totp_identifier=totp_identifier,
+ totp_url=totp_url,
+ )
timeout = kwargs.pop("timeout", settings.BROWSER_ACTION_TIMEOUT_MS)
locator = self._locator_scope.locator(selector).first
await locator.fill(value, timeout=timeout, **kwargs)
@@ -747,6 +775,14 @@ class SkyvernPage(Page):
if context and context.ai_mode_override:
ai = context.ai_mode_override
+ resolved_totp_value: str | None = None
+ if value is not None and is_unresolved_totp_value(value):
+ resolved_totp_value = await self._resolve_totp_placeholder_or_raise(
+ value,
+ totp_identifier=kwargs.get("totp_identifier"),
+ totp_url=kwargs.get("totp_url"),
+ )
+
timeout = kwargs.pop("timeout", settings.BROWSER_ACTION_TIMEOUT_MS)
data = kwargs.pop("data", None)
@@ -774,15 +810,20 @@ class SkyvernPage(Page):
)
raise ValueError("Selector is required but was not provided")
- actual_value = value or ""
- try:
- actual_value = await self.get_actual_value(
- actual_value,
- totp_identifier=kwargs.get("totp_identifier"),
- totp_url=kwargs.get("totp_url"),
- )
- except Exception:
- pass # use original value
+ actual_value = resolved_totp_value if resolved_totp_value is not None else value or ""
+ if resolved_totp_value is None:
+ try:
+ actual_value = await self.get_actual_value(
+ actual_value,
+ totp_identifier=kwargs.get("totp_identifier"),
+ totp_url=kwargs.get("totp_url"),
+ )
+ except NoTOTPSecretFound:
+ raise
+ except Exception:
+ pass # use original value
+ if is_unresolved_totp_value(actual_value):
+ raise NoTOTPSecretFound()
try:
result = await self._do_autocomplete(
@@ -802,7 +843,7 @@ class SkyvernPage(Page):
if prompt:
return await self._ai.ai_input_text(
selector=None,
- value=actual_value,
+ value=value if resolved_totp_value is not None else actual_value,
intention=prompt,
data=data,
timeout=timeout,
@@ -987,6 +1028,15 @@ class SkyvernPage(Page):
if context and context.ai_mode_override:
ai = context.ai_mode_override
+ original_value = value
+ resolved_totp_value: str | None = None
+ if is_unresolved_totp_value(value):
+ resolved_totp_value = await self._resolve_totp_placeholder_or_raise(
+ value,
+ totp_identifier=totp_identifier,
+ totp_url=totp_url,
+ )
+
# For single-digit TOTP values (from multi-field TOTP inputs), force fallback mode
# so that we use the exact digit value instead of having AI generate a new one
if value and len(value) == 1 and value.isdigit() and ai == "proactive":
@@ -995,15 +1045,19 @@ class SkyvernPage(Page):
# format the text with the actual value of the parameter if it's a secret when running a workflow
if ai == "fallback":
error_to_raise = None
- original_value = value
original_selector = selector # preserve for fallback episode recording
if selector:
try:
- value = await self.get_actual_value(
- value,
- totp_identifier=totp_identifier,
- totp_url=totp_url,
- )
+ if resolved_totp_value is not None:
+ value = resolved_totp_value
+ else:
+ value = await self.get_actual_value(
+ value,
+ totp_identifier=totp_identifier,
+ totp_url=totp_url,
+ )
+ if is_unresolved_totp_value(value):
+ raise NoTOTPSecretFound()
# Retry selector lookup to handle page transitions (redirects,
# slow renders) before burning an expensive AI fallback call.
if _skip_element_prep:
@@ -1011,12 +1065,20 @@ class SkyvernPage(Page):
else:
locator = await self._wait_for_selector_with_retry(selector, timeout=timeout)
await self._prepare_element(locator, timeout=timeout)
+ if resolved_totp_value is not None:
+ value = await self._resolve_totp_placeholder_or_raise(
+ original_value,
+ totp_identifier=totp_identifier,
+ totp_url=totp_url,
+ )
# Use locator.fill() (programmatic, single-shot) instead of typing
# character-by-character. Sequential typing triggers autocomplete
# dropdowns on search bars and typeaheads which destabilise the DOM
# and cause the locator to time out mid-input.
await locator.fill(value, timeout=timeout)
return original_value
+ except NoTOTPSecretFound:
+ raise
except Exception as e:
LOG.warning(
"CSS selector fill failed, falling back to AI",
@@ -1029,7 +1091,7 @@ class SkyvernPage(Page):
if intention:
return await self._ai.ai_input_text(
selector=selector,
- value=value,
+ value=original_value if resolved_totp_value is not None else value,
intention=intention,
data=data,
totp_identifier=totp_identifier,
@@ -1060,6 +1122,12 @@ class SkyvernPage(Page):
raise ValueError("Selector is required but was not provided")
locator = self._locator_scope.locator(selector).first
+ if resolved_totp_value is not None:
+ value = await self._resolve_totp_placeholder_or_raise(
+ original_value,
+ totp_identifier=totp_identifier,
+ totp_url=totp_url,
+ )
await locator.fill(value, timeout=timeout)
return value
@@ -1642,6 +1710,7 @@ class SkyvernPage(Page):
for idx, value in sorted(mapping.items()):
if idx >= len(form_fields) or value is None:
continue
+ totp_placeholder = value if isinstance(value, str) and is_unresolved_totp_value(value) else None
field = form_fields[idx]
selector = field.get("selector", "")
@@ -1650,7 +1719,9 @@ class SkyvernPage(Page):
label = field.get("label") or field.get("name") or "unknown"
try:
- if field_type in ("radio_group", "checkbox_group"):
+ if totp_placeholder is not None:
+ await self.fill(selector=selector, value=totp_placeholder, ai=None)
+ elif field_type in ("radio_group", "checkbox_group"):
LOG.info(
"fill_from_mapping: processing group field",
field_label=label[:50],
@@ -1797,6 +1868,8 @@ class SkyvernPage(Page):
else:
await self.fill(selector=selector, value=str(value), ai=None)
+ except NoTOTPSecretFound:
+ raise
except Exception:
LOG.warning(
"fill_from_mapping: field fill failed, trying AI fallback",
@@ -1810,7 +1883,10 @@ class SkyvernPage(Page):
continue
try:
self._track_ai_call()
- if field_type in ("radio_group", "checkbox_group"):
+ if totp_placeholder is not None:
+ prompt = f"Fill the '{label}' field with the provided verification code"
+ await self.fill(selector=selector, value=totp_placeholder, ai="fallback", prompt=prompt)
+ elif field_type in ("radio_group", "checkbox_group"):
prompt = f"Select '{value}' for the question '{label}'"
await self.click(selector=selector, ai="fallback", prompt=prompt)
elif field_type in ("radio", "checkbox"):
@@ -1822,6 +1898,8 @@ class SkyvernPage(Page):
else:
prompt = f"Fill the '{label}' field with: {value}"
await self.fill(selector=selector, ai="fallback", prompt=prompt)
+ except NoTOTPSecretFound:
+ raise
except Exception:
LOG.warning("fill_from_mapping: AI fallback also failed", field_label=label, exc_info=True)
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 59450b529..2031e1ff6 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -48,6 +48,13 @@ class SkyvernException(Exception):
self.message = message
super().__init__(message)
+ @property
+ def user_facing_type_name(self) -> str:
+ # Class name safe to render in a user-facing message. Subclasses whose real class
+ # name carries sensitive info (e.g. a remote-browser vendor identity) override this
+ # so the concrete name stays in logs/monitoring but never reaches end users.
+ return type(self).__name__
+
class SkyvernExtraNotInstalled(ImportError):
def __init__(self, feature: str, extra: str = "server"):
@@ -128,6 +135,16 @@ def _is_browser_connection_error(message: str) -> bool:
return any(pattern in message for pattern in _BROWSER_CONNECTION_PATTERNS)
+# A raw CDP connect failure (e.g. from playwright.chromium.connect_over_cdp) echoes the
+# ws/wss endpoint URL, which can carry the remote-browser vendor host, a session-bearing
+# path/query, or credentials embedded as user:pass@host. Redact it before it reaches a user.
+_CDP_WS_URL_RE = re.compile(r"wss?://\S+", re.IGNORECASE)
+
+
+def _redact_cdp_endpoint_urls(message: str) -> str:
+ return _CDP_WS_URL_RE.sub("[remote browser endpoint]", message)
+
+
def get_user_facing_exception_message(exception: Exception) -> str:
if isinstance(exception, SkyvernException):
return exception.message or str(exception)
@@ -473,16 +490,23 @@ class UnknownErrorWhileCreatingBrowserContext(SkyvernException):
SUPPORT_GUIDANCE = "Please try re-running. If this continues, contact [email protected]."
def __init__(self, browser_type: str, exception: Exception) -> None:
- exception_type = type(exception).__name__
+ # browser_type can be a concrete remote-browser vendor identity (settings.BROWSER_TYPE);
+ # keep it on the exception for structured logs but never surface it in the user message.
+ self.browser_type = browser_type
+ # A SkyvernException may redact its own class name (e.g. a vendor-named rate-limit
+ # error whose real name is kept for logs/monitoring but must not reach end users).
+ exception_type = (
+ exception.user_facing_type_name if isinstance(exception, SkyvernException) else type(exception).__name__
+ )
detail = self._get_detail(exception)
- super().__init__(f"Failed to create browser context for {browser_type} ({exception_type}). {detail}")
+ super().__init__(f"Failed to create browser context ({exception_type}). {detail}")
@staticmethod
def _get_detail(exception: Exception) -> str:
if isinstance(exception, CdpConnectionConfigurationError):
return exception.message or str(exception)
- raw_message = str(exception).strip()
+ raw_message = _redact_cdp_endpoint_urls(str(exception).strip())
raw_lower = raw_message.lower()
# Browser launch environment errors: worker cannot initialize the
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index d24cc0a36..acf5ab165 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -378,6 +378,24 @@ async def _cancel_pending_prefetch_task(task: asyncio.Task | None) -> None:
await task
+class _BackgroundArtifactTaskTracker:
+ # Instantiated fresh per agent_step call so the in-flight artifact task is isolated
+ # to that call. A shared (module-global or instance) tracker would let concurrent
+ # agent_step invocations await each other's background writes.
+ def __init__(self) -> None:
+ self.task: asyncio.Task | None = None
+
+ async def drain(self) -> None:
+ if self.task is None:
+ return
+ task = self.task
+ self.task = None
+ try:
+ await task
+ except Exception:
+ LOG.warning("Background artifact task failed, continuing", exc_info=True)
+
+
def _build_totp_timeout_reasoning(task: Task) -> str:
# Mirror poll_otp_value's URL-then-identifier precedence so we only report the
# source that was actually queried. URL goes through strip_query_params to keep
@@ -1480,18 +1498,7 @@ class ForgeAgent:
cua_response=None,
)
prefetched_summary_task: asyncio.Task[dict[str, Any]] | None = None
- current_artifact_task: asyncio.Task | None = None
-
- async def await_background_artifact_task() -> None:
- nonlocal current_artifact_task
- if current_artifact_task is None:
- return
- task = current_artifact_task
- current_artifact_task = None
- try:
- await task
- except Exception:
- LOG.warning("Background artifact task failed, continuing", exc_info=True)
+ artifact_tracker = _BackgroundArtifactTaskTracker()
try:
LOG.info(
@@ -1602,302 +1609,30 @@ class ForgeAgent:
)
return step, detailed_agent_step_output
- # Execute the actions
- LOG.info(
- "Executing actions",
- sampling=True,
- step_order=step.order,
- step_retry=step.retry_index,
+ actions, early_return = await self._execute_step_actions(
+ task=task,
+ step=step,
+ browser_state=browser_state,
+ engine=engine,
+ scraped_page=scraped_page,
+ complete_verification=complete_verification,
actions=actions,
+ detailed_agent_step_output=detailed_agent_step_output,
+ pdf_auto_download_src=pdf_auto_download_src,
+ pdf_auto_download_used_bytes=pdf_auto_download_used_bytes,
+ _step_span=_step_span,
+ artifact_tracker=artifact_tracker,
)
- action_results: list[ActionResult] = []
- detailed_agent_step_output.action_results = action_results
- # filter out wait action if there are other actions in the list
- # we do this because WAIT action is considered as a failure
- # which will block following actions if we don't remove it from the list
- # if the list only contains WAIT action, we will execute WAIT action(s)
- if len(actions) > 1:
- wait_actions_to_skip = [action for action in actions if action.action_type == ActionType.WAIT]
- wait_actions_len = len(wait_actions_to_skip)
- # if there are wait actions and there are other actions in the list, skip wait actions
- # if we are using cached action plan, we don't skip wait actions
- if wait_actions_len > 0 and wait_actions_len < len(actions):
- actions = [action for action in actions if action.action_type != ActionType.WAIT]
- LOG.info(
- "Skipping wait actions",
- wait_actions_to_skip=wait_actions_to_skip,
- actions=actions,
- )
-
- # initialize list of tuples and set actions as the first element of each tuple so that in the case
- # of an exception, we can still see all the actions
- detailed_agent_step_output.actions_and_results = [(action, []) for action in actions]
-
- # build a linked action chain by the action_idx
- action_linked_list: list[ActionLinkedNode] = []
- element_id_to_action_index: dict[str, int] = dict()
- for action_idx, action in enumerate(actions):
- node = ActionLinkedNode(action=action)
- action_linked_list.append(node)
-
- if not isinstance(action, WebAction):
- continue
-
- previous_action_idx = element_id_to_action_index.get(action.element_id)
- if previous_action_idx is not None:
- previous_node = action_linked_list[previous_action_idx]
- previous_node.next = node
-
- element_id_to_action_index[action.element_id] = action_idx
-
- element_id_to_last_action: dict[str, int] = dict()
- try:
- wait_config = await get_or_create_wait_config(task.task_id, task.workflow_run_id, task.organization_id)
- base_delay = get_wait_time(wait_config, "inter_action_delay", default=0.5)
- except Exception:
- base_delay = 0.5
- for action_idx, action_node in enumerate(action_linked_list):
- await await_background_artifact_task()
-
- context = skyvern_context.ensure_context()
- if context.refresh_working_page:
- LOG.warning(
- "Detected the signal to reload the page, going to reload and skip the rest of the actions",
- step_order=step.order,
- )
- await browser_state.reload_page()
- context.refresh_working_page = False
- action_result = ActionSuccess()
- action_result.step_order = step.order
- action_result.step_retry_number = step.retry_index
- action = ReloadPageAction(
- reasoning="Something wrong with the current page, reload to continue",
- status=ActionStatus.completed,
- organization_id=task.organization_id,
- workflow_run_id=task.workflow_run_id,
- task_id=task.task_id,
- step_id=step.step_id,
- step_order=step.order,
- action_order=action_idx,
- )
- detailed_agent_step_output.actions_and_results[action_idx] = (action, [action_result])
- action.action_id = (await app.DATABASE.workflow_params.create_action(action=action)).action_id
- current_artifact_task = asyncio.create_task(
- self.record_artifacts_after_action(task, step, browser_state, engine, action)
- )
- break
-
- action = action_node.action
- if isinstance(action, WebAction):
- previous_action_idx = element_id_to_last_action.get(action.element_id)
- if previous_action_idx is not None:
- LOG.warning(
- "Duplicate action element id.",
- step_order=step.order,
- action=action,
- )
-
- previous_action, previous_result = detailed_agent_step_output.actions_and_results[
- previous_action_idx
- ]
- if len(previous_result) > 0 and previous_result[-1].success:
- LOG.info(
- "Previous action succeeded, but we'll still continue.",
- step_order=step.order,
- previous_action=previous_action,
- previous_result=previous_result,
- )
- else:
- LOG.warning(
- "Previous action failed, so handle the next action.",
- step_order=step.order,
- previous_action=previous_action,
- previous_result=previous_result,
- )
-
- element_id_to_last_action[action.element_id] = action_idx
-
- if engine != RunEngine.openai_cua:
- self.async_operation_pool.run_operation(task.task_id, AgentPhase.action)
- current_page = await browser_state.must_get_working_page()
- if isinstance(action, CompleteAction) and not complete_verification:
- # Do not verify the complete action when complete_verification is False
- # set verified to True will skip the completion verification
- action.verified = True
-
- # Pass TOTP secret to handler for multi-field TOTP sequences
- # Handler will generate TOTP at execution time
- if (
- action.action_type == ActionType.INPUT_TEXT
- and self._is_multi_field_totp_sequence(actions)
- and (totp_secret := skyvern_context.ensure_context().totp_codes.get(f"{task.task_id}_secret"))
- ):
- # Pass TOTP secret to handler for execution-time generation
- action.totp_timing_info = {
- "is_totp_sequence": True,
- "action_index": action_idx,
- "totp_secret": totp_secret,
- "is_retry": step.retry_index > 0,
- }
-
- # Tell the handler to skip the auto-completion Tab hack when the
- # next batched action would be broken by a focus change — e.g. a
- # KEYPRESS Enter or another action on the same element.
- if action.action_type == ActionType.INPUT_TEXT:
- next_action = (
- action_linked_list[action_idx + 1].action if action_idx + 1 < len(action_linked_list) else None
- )
- if isinstance(next_action, KeypressAction) or (
- isinstance(next_action, WebAction) and next_action.element_id == action.element_id
- ):
- action.skip_auto_complete_tab = True
- action.stop_batch_after_dropdown_select = should_stop_batch_after_dropdown_select(next_action)
-
- results = await ActionHandler.handle_action(
- scraped_page=scraped_page,
- task=task,
- step=step,
- page=current_page,
- action=action,
- )
- await app.AGENT_FUNCTION.post_action_execution(action)
- detailed_agent_step_output.actions_and_results[action_idx] = (
- action,
- results,
- )
-
- # Determine wait time between actions
- wait_time = random.uniform(base_delay, base_delay * 2) if base_delay > 0 else 0.0
-
- # For multi-field TOTP sequences, use zero delay between all digits for fast execution
- if action.action_type == ActionType.INPUT_TEXT and self._is_multi_field_totp_sequence(actions):
- current_text = action.text if hasattr(action, "text") else None
-
- if current_text and len(current_text) == 1 and current_text.isdigit():
- # Zero delay between all TOTP digits for fast execution
- wait_time = 0.0
- LOG.debug(
- "TOTP: zero delay for digit",
- task_id=task.task_id,
- action_idx=action_idx,
- digit=current_text,
- )
-
- # Skip sleep and post-action artifacts for page-level SCROLL to preserve
- # scroll-driven JS state. Many pages enable buttons only while scrolled to
- # bottom (e.g. T&C "Agree" buttons) and re-disable them after any delay or
- # programmatic scroll. Sub-container scrolls (strategies 1 & 2) don't affect
- # page position, so they keep normal sleep and artifact recording.
- is_page_level_scroll = action.action_type == ActionType.SCROLL and any(
- r.success and isinstance(r.data, dict) and r.data.get("page_level_scroll") for r in results
- )
- if is_page_level_scroll:
- wait_time = 0.0
-
- _step_span.add_event(
- "action.post_wait",
- attributes={"wait_time_ms": int(wait_time * 1000), "action_idx": action_idx},
- )
- await asyncio.sleep(wait_time)
- if not is_page_level_scroll:
- current_artifact_task = asyncio.create_task(
- self.record_artifacts_after_action(task, step, browser_state, engine, action)
- )
- else:
- LOG.info(
- "Skipping post-action artifacts for page-level scroll",
- step_order=step.order,
- step_retry=step.retry_index,
- action_idx=action_idx,
- )
- for result in results:
- result.step_retry_number = step.retry_index
- result.step_order = step.order
- step.output = detailed_agent_step_output.to_agent_step_output()
- action_results.extend(results)
- # Check the last result for this action. If that succeeded, assume the entire action is successful
- if results and results[-1].success:
- if pdf_auto_download_src and isinstance(action, DownloadFileAction):
- actually_downloaded = pdf_auto_download_used_bytes or results[-1].download_triggered
- if actually_downloaded:
- mark_pdf_source_downloaded(pdf_auto_download_src)
- pdf_auto_download_src = None
- LOG.info(
- "Action succeeded",
- sampling=True,
- step_order=step.order,
- step_retry=step.retry_index,
- action_idx=action_idx,
- action=action,
- action_result=results,
- )
- if results[-1].skip_remaining_actions:
- LOG.warning(
- "Going to stop executing the remaining actions",
- step_order=step.order,
- step_retry=step.retry_index,
- action_idx=action_idx,
- action=action,
- action_result=results,
- )
- break
-
- elif results and isinstance(action, DecisiveAction):
- LOG.warning(
- "DecisiveAction failed, but not stopping execution and not retrying the step",
- step_order=step.order,
- step_retry=step.retry_index,
- action_idx=action_idx,
- action=action,
- action_result=results,
- )
- elif results and not results[-1].success and not results[-1].stop_execution_on_failure:
- LOG.warning(
- "Action failed, but not stopping execution",
- step_order=step.order,
- step_retry=step.retry_index,
- action_idx=action_idx,
- action=action,
- action_result=results,
- )
- else:
- if action_node.next is not None:
- LOG.warning(
- "Action failed, but have duplicated element id in the action list. Continue excuting.",
- step_order=step.order,
- step_retry=step.retry_index,
- action_idx=action_idx,
- action=action,
- next_action=action_node.next.action,
- action_result=results,
- )
- continue
-
- LOG.warning(
- "Action failed, marking step as failed",
- step_order=step.order,
- step_retry=step.retry_index,
- action_idx=action_idx,
- action=action,
- action_result=results,
- actions_and_results=detailed_agent_step_output.actions_and_results,
- )
- # if the action failed, don't execute the rest of the actions, mark the step as failed, and retry
- failed_step = await self.update_step(
- step=step,
- status=StepStatus.failed,
- output=detailed_agent_step_output.to_agent_step_output(),
- )
- return failed_step, detailed_agent_step_output.get_clean_detailed_output()
-
- await await_background_artifact_task()
+ if early_return is not None:
+ return early_return
+ assert detailed_agent_step_output.actions_and_results is not None
LOG.info(
"Actions executed successfully, marking step as completed",
sampling=True,
step_order=step.order,
step_retry=step.retry_index,
- action_results=action_results,
+ action_results=detailed_agent_step_output.action_results,
)
# Clean up TOTP cache after multi-field TOTP sequence completion
@@ -2052,7 +1787,314 @@ class ForgeAgent:
return failed_step, detailed_agent_step_output.get_clean_detailed_output()
finally:
await _cancel_pending_prefetch_task(prefetched_summary_task)
- await await_background_artifact_task()
+ await artifact_tracker.drain()
+
+ async def _execute_step_actions(
+ self,
+ *,
+ task: Task,
+ step: Step,
+ browser_state: BrowserState,
+ engine: RunEngine,
+ scraped_page: ScrapedPage,
+ complete_verification: bool,
+ actions: list[Action],
+ detailed_agent_step_output: DetailedAgentStepOutput,
+ pdf_auto_download_src: str | None,
+ pdf_auto_download_used_bytes: bool,
+ _step_span: otel_trace.Span,
+ artifact_tracker: _BackgroundArtifactTaskTracker,
+ ) -> tuple[list[Action], tuple[Step, DetailedAgentStepOutput] | None]:
+ # Execute the actions
+ LOG.info(
+ "Executing actions",
+ sampling=True,
+ step_order=step.order,
+ step_retry=step.retry_index,
+ actions=actions,
+ )
+ action_results: list[ActionResult] = []
+ detailed_agent_step_output.action_results = action_results
+ # filter out wait action if there are other actions in the list
+ # we do this because WAIT action is considered as a failure
+ # which will block following actions if we don't remove it from the list
+ # if the list only contains WAIT action, we will execute WAIT action(s)
+ if len(actions) > 1:
+ wait_actions_to_skip = [action for action in actions if action.action_type == ActionType.WAIT]
+ wait_actions_len = len(wait_actions_to_skip)
+ # if there are wait actions and there are other actions in the list, skip wait actions
+ # if we are using cached action plan, we don't skip wait actions
+ if wait_actions_len > 0 and wait_actions_len < len(actions):
+ actions = [action for action in actions if action.action_type != ActionType.WAIT]
+ LOG.info(
+ "Skipping wait actions",
+ wait_actions_to_skip=wait_actions_to_skip,
+ actions=actions,
+ )
+
+ # initialize list of tuples and set actions as the first element of each tuple so that in the case
+ # of an exception, we can still see all the actions
+ detailed_agent_step_output.actions_and_results = [(action, []) for action in actions]
+
+ # build a linked action chain by the action_idx
+ action_linked_list: list[ActionLinkedNode] = []
+ element_id_to_action_index: dict[str, int] = dict()
+ for action_idx, action in enumerate(actions):
+ node = ActionLinkedNode(action=action)
+ action_linked_list.append(node)
+
+ if not isinstance(action, WebAction):
+ continue
+
+ previous_action_idx = element_id_to_action_index.get(action.element_id)
+ if previous_action_idx is not None:
+ previous_node = action_linked_list[previous_action_idx]
+ previous_node.next = node
+
+ element_id_to_action_index[action.element_id] = action_idx
+
+ element_id_to_last_action: dict[str, int] = dict()
+ try:
+ wait_config = await get_or_create_wait_config(task.task_id, task.workflow_run_id, task.organization_id)
+ base_delay = get_wait_time(wait_config, "inter_action_delay", default=0.5)
+ except Exception:
+ base_delay = 0.5
+ for action_idx, action_node in enumerate(action_linked_list):
+ await artifact_tracker.drain()
+
+ context = skyvern_context.ensure_context()
+ if context.refresh_working_page:
+ LOG.warning(
+ "Detected the signal to reload the page, going to reload and skip the rest of the actions",
+ step_order=step.order,
+ )
+ await browser_state.reload_page()
+ context.refresh_working_page = False
+ action_result = ActionSuccess()
+ action_result.step_order = step.order
+ action_result.step_retry_number = step.retry_index
+ action = ReloadPageAction(
+ reasoning="Something wrong with the current page, reload to continue",
+ status=ActionStatus.completed,
+ organization_id=task.organization_id,
+ workflow_run_id=task.workflow_run_id,
+ task_id=task.task_id,
+ step_id=step.step_id,
+ step_order=step.order,
+ action_order=action_idx,
+ )
+ detailed_agent_step_output.actions_and_results[action_idx] = (action, [action_result])
+ action.action_id = (await app.DATABASE.workflow_params.create_action(action=action)).action_id
+ artifact_tracker.task = asyncio.create_task(
+ self.record_artifacts_after_action(task, step, browser_state, engine, action)
+ )
+ break
+
+ action = action_node.action
+ if isinstance(action, WebAction):
+ previous_action_idx = element_id_to_last_action.get(action.element_id)
+ if previous_action_idx is not None:
+ LOG.warning(
+ "Duplicate action element id.",
+ step_order=step.order,
+ action=action,
+ )
+
+ previous_action, previous_result = detailed_agent_step_output.actions_and_results[
+ previous_action_idx
+ ]
+ if len(previous_result) > 0 and previous_result[-1].success:
+ LOG.info(
+ "Previous action succeeded, but we'll still continue.",
+ step_order=step.order,
+ previous_action=previous_action,
+ previous_result=previous_result,
+ )
+ else:
+ LOG.warning(
+ "Previous action failed, so handle the next action.",
+ step_order=step.order,
+ previous_action=previous_action,
+ previous_result=previous_result,
+ )
+
+ element_id_to_last_action[action.element_id] = action_idx
+
+ if engine != RunEngine.openai_cua:
+ self.async_operation_pool.run_operation(task.task_id, AgentPhase.action)
+ current_page = await browser_state.must_get_working_page()
+ if isinstance(action, CompleteAction) and not complete_verification:
+ # Do not verify the complete action when complete_verification is False
+ # set verified to True will skip the completion verification
+ action.verified = True
+
+ # Pass TOTP secret to handler for multi-field TOTP sequences
+ # Handler will generate TOTP at execution time
+ if (
+ action.action_type == ActionType.INPUT_TEXT
+ and self._is_multi_field_totp_sequence(actions)
+ and (totp_secret := skyvern_context.ensure_context().totp_codes.get(f"{task.task_id}_secret"))
+ ):
+ # Pass TOTP secret to handler for execution-time generation
+ action.totp_timing_info = {
+ "is_totp_sequence": True,
+ "action_index": action_idx,
+ "totp_secret": totp_secret,
+ "is_retry": step.retry_index > 0,
+ }
+
+ # Tell the handler to skip the auto-completion Tab hack when the
+ # next batched action would be broken by a focus change — e.g. a
+ # KEYPRESS Enter or another action on the same element.
+ if action.action_type == ActionType.INPUT_TEXT:
+ next_action = (
+ action_linked_list[action_idx + 1].action if action_idx + 1 < len(action_linked_list) else None
+ )
+ if isinstance(next_action, KeypressAction) or (
+ isinstance(next_action, WebAction) and next_action.element_id == action.element_id
+ ):
+ action.skip_auto_complete_tab = True
+ action.stop_batch_after_dropdown_select = should_stop_batch_after_dropdown_select(next_action)
+
+ results = await ActionHandler.handle_action(
+ scraped_page=scraped_page,
+ task=task,
+ step=step,
+ page=current_page,
+ action=action,
+ )
+ await app.AGENT_FUNCTION.post_action_execution(action)
+ detailed_agent_step_output.actions_and_results[action_idx] = (
+ action,
+ results,
+ )
+
+ # Determine wait time between actions
+ wait_time = random.uniform(base_delay, base_delay * 2) if base_delay > 0 else 0.0
+
+ # For multi-field TOTP sequences, use zero delay between all digits for fast execution
+ if action.action_type == ActionType.INPUT_TEXT and self._is_multi_field_totp_sequence(actions):
+ current_text = action.text if hasattr(action, "text") else None
+
+ if current_text and len(current_text) == 1 and current_text.isdigit():
+ # Zero delay between all TOTP digits for fast execution
+ wait_time = 0.0
+ LOG.debug(
+ "TOTP: zero delay for digit",
+ task_id=task.task_id,
+ action_idx=action_idx,
+ digit=current_text,
+ )
+
+ # Skip sleep and post-action artifacts for page-level SCROLL to preserve
+ # scroll-driven JS state. Many pages enable buttons only while scrolled to
+ # bottom (e.g. T&C "Agree" buttons) and re-disable them after any delay or
+ # programmatic scroll. Sub-container scrolls (strategies 1 & 2) don't affect
+ # page position, so they keep normal sleep and artifact recording.
+ is_page_level_scroll = action.action_type == ActionType.SCROLL and any(
+ r.success and isinstance(r.data, dict) and r.data.get("page_level_scroll") for r in results
+ )
+ if is_page_level_scroll:
+ wait_time = 0.0
+
+ _step_span.add_event(
+ "action.post_wait",
+ attributes={"wait_time_ms": int(wait_time * 1000), "action_idx": action_idx},
+ )
+ await asyncio.sleep(wait_time)
+ if not is_page_level_scroll:
+ artifact_tracker.task = asyncio.create_task(
+ self.record_artifacts_after_action(task, step, browser_state, engine, action)
+ )
+ else:
+ LOG.info(
+ "Skipping post-action artifacts for page-level scroll",
+ step_order=step.order,
+ step_retry=step.retry_index,
+ action_idx=action_idx,
+ )
+ for result in results:
+ result.step_retry_number = step.retry_index
+ result.step_order = step.order
+ step.output = detailed_agent_step_output.to_agent_step_output()
+ action_results.extend(results)
+ # Check the last result for this action. If that succeeded, assume the entire action is successful
+ if results and results[-1].success:
+ if pdf_auto_download_src and isinstance(action, DownloadFileAction):
+ actually_downloaded = pdf_auto_download_used_bytes or results[-1].download_triggered
+ if actually_downloaded:
+ mark_pdf_source_downloaded(pdf_auto_download_src)
+ pdf_auto_download_src = None
+ LOG.info(
+ "Action succeeded",
+ sampling=True,
+ step_order=step.order,
+ step_retry=step.retry_index,
+ action_idx=action_idx,
+ action=action,
+ action_result=results,
+ )
+ if results[-1].skip_remaining_actions:
+ LOG.warning(
+ "Going to stop executing the remaining actions",
+ step_order=step.order,
+ step_retry=step.retry_index,
+ action_idx=action_idx,
+ action=action,
+ action_result=results,
+ )
+ break
+
+ elif results and isinstance(action, DecisiveAction):
+ LOG.warning(
+ "DecisiveAction failed, but not stopping execution and not retrying the step",
+ step_order=step.order,
+ step_retry=step.retry_index,
+ action_idx=action_idx,
+ action=action,
+ action_result=results,
+ )
+ elif results and not results[-1].success and not results[-1].stop_execution_on_failure:
+ LOG.warning(
+ "Action failed, but not stopping execution",
+ step_order=step.order,
+ step_retry=step.retry_index,
+ action_idx=action_idx,
+ action=action,
+ action_result=results,
+ )
+ else:
+ if action_node.next is not None:
+ LOG.warning(
+ "Action failed, but have duplicated element id in the action list. Continue excuting.",
+ step_order=step.order,
+ step_retry=step.retry_index,
+ action_idx=action_idx,
+ action=action,
+ next_action=action_node.next.action,
+ action_result=results,
+ )
+ continue
+
+ LOG.warning(
+ "Action failed, marking step as failed",
+ step_order=step.order,
+ step_retry=step.retry_index,
+ action_idx=action_idx,
+ action=action,
+ action_result=results,
+ actions_and_results=detailed_agent_step_output.actions_and_results,
+ )
+ # if the action failed, don't execute the rest of the actions, mark the step as failed, and retry
+ failed_step = await self.update_step(
+ step=step,
+ status=StepStatus.failed,
+ output=detailed_agent_step_output.to_agent_step_output(),
+ )
+ return actions, (failed_step, detailed_agent_step_output.get_clean_detailed_output())
+
+ await artifact_tracker.drain()
+ return actions, None
async def _generate_step_actions(
self,
diff --git a/skyvern/forge/sdk/services/credentials.py b/skyvern/forge/sdk/services/credentials.py
index e35a5c2be..46b265ca9 100644
--- a/skyvern/forge/sdk/services/credentials.py
+++ b/skyvern/forge/sdk/services/credentials.py
@@ -1,7 +1,7 @@
import re
from dataclasses import dataclass
from enum import StrEnum
-from urllib.parse import unquote, urlsplit
+from urllib.parse import SplitResult, parse_qsl, unquote, urlencode, urlsplit, urlunsplit
import pyotp
import structlog
@@ -9,6 +9,9 @@ import structlog
LOG = structlog.get_logger()
_SECRET_QUERY_PATTERN = re.compile(r"(?i)(?:^|[?&])secret=([^&#]+)")
+_ISSUER_MISMATCH_ERROR = "If issuer is specified in both label and parameters, it should be equal."
+_TOTP_PLACEHOLDER_PATTERN = re.compile(r"placeholder_[A-Za-z0-9]+_totp")
+_TOTP_MARKER_VALUES = frozenset({"OP_TOTP", "BW_TOTP", "AZ_TOTP"})
# 1Password's Python SDK forwards generic 5xx upstream failures as plain Exceptions
# whose stringified message embeds the HTTP status.
@@ -82,11 +85,32 @@ def _compact_secret_value(totp_secret: str) -> str:
return _strip_totp_input_whitespace(totp_secret).replace("-", "")
+def _safe_urlsplit(value: str) -> SplitResult | None:
+ try:
+ return urlsplit(value)
+ except ValueError:
+ return None
+
+
def _is_otpauth_uri(totp_secret: str) -> bool:
- parsed = urlsplit(totp_secret)
+ parsed = _safe_urlsplit(totp_secret)
+ if parsed is None:
+ return False
return parsed.scheme.lower() == "otpauth" and parsed.netloc.lower() == "totp"
+def is_unresolved_totp_placeholder(value: object) -> bool:
+ """Return whether a value contains an unexpanded workflow TOTP placeholder."""
+ return isinstance(value, str) and bool(_TOTP_PLACEHOLDER_PATTERN.search(value))
+
+
+def is_unresolved_totp_value(value: object) -> bool:
+ """Return whether a value still contains a TOTP placeholder or provider marker."""
+ return isinstance(value, str) and (
+ is_unresolved_totp_placeholder(value) or any(marker in value for marker in _TOTP_MARKER_VALUES)
+ )
+
+
def _validate_base32_secret(totp_secret: str) -> str:
normalized_secret = _compact_secret_value(totp_secret)
try:
@@ -97,14 +121,51 @@ def _validate_base32_secret(totp_secret: str) -> str:
def _parse_uri_totp(uri: str) -> pyotp.TOTP | None:
+ parsed_uri = _safe_urlsplit(uri)
+ if parsed_uri is None:
+ LOG.warning("TOTP URI is malformed")
+ return None
+ query_pairs = parse_qsl(parsed_uri.query, keep_blank_values=True)
+ issuer_count = sum(key == "issuer" for key, _value in query_pairs)
+ if issuer_count > 1:
+ LOG.warning("TOTP URI contains duplicate issuer parameters")
+ return None
+
try:
parsed_otp = pyotp.parse_uri(uri)
- except Exception:
- LOG.warning("Failed to parse TOTP config from URI", exc_info=True)
+ except ValueError as exc:
+ if str(exc) != _ISSUER_MISMATCH_ERROR:
+ LOG.warning("Failed to parse TOTP config from URI", exception_type=type(exc).__name__)
+ return None
+
+ # Issuer metadata does not affect code generation, but PyOTP rejects conflicts before validating the config.
+ if issuer_count != 1:
+ LOG.warning("TOTP URI is missing issuer metadata required for mismatch recovery")
+ return None
+ query_without_issuer = urlencode([(key, value) for key, value in query_pairs if key != "issuer"])
+ validation_uri = urlunsplit(parsed_uri._replace(query=query_without_issuer))
+ try:
+ parsed_otp = pyotp.parse_uri(validation_uri)
+ except Exception as fallback_exc:
+ LOG.warning(
+ "Failed to parse TOTP config from URI after ignoring mismatched issuer metadata",
+ exception_type=type(fallback_exc).__name__,
+ )
+ return None
+ except Exception as exc:
+ LOG.warning("Failed to parse TOTP config from URI", exception_type=type(exc).__name__)
return None
if not isinstance(parsed_otp, pyotp.TOTP):
LOG.warning("Parsed OTP URI is not a TOTP config")
return None
+ if parsed_otp.interval <= 0:
+ LOG.warning("TOTP URI contains a nonpositive period")
+ return None
+ try:
+ parsed_otp.byte_secret()
+ except Exception as exc:
+ LOG.warning("TOTP URI contains an invalid secret", exception_type=type(exc).__name__)
+ return None
return parsed_otp
@@ -124,7 +185,12 @@ def parse_totp_secret(totp_secret: str) -> str:
uri_candidates.append(unquoted_input)
for candidate in uri_candidates:
- if not _is_otpauth_uri(candidate):
+ parsed_candidate = _safe_urlsplit(candidate)
+ if parsed_candidate is None:
+ return ""
+ if not (parsed_candidate.scheme.lower() == "otpauth" and parsed_candidate.netloc.lower() == "totp"):
+ if parsed_candidate.scheme.lower() == "otpauth":
+ return ""
continue
parsed_otp = _parse_uri_totp(candidate)
if not parsed_otp:
diff --git a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
index f8280ec98..9b49fe51f 100644
--- a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
+++ b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
@@ -57,6 +57,26 @@ FLAT_PDF_OCR_MISSING_MESSAGE = (
)
FLAT_FILL_RENDER_RESOLUTION = 150
FLAT_FILL_FONT_SIZE = 11
+FLAT_FILL_MIN_FONT_SIZE = 7.0
+FLAT_FILL_FONT_SIZE_GAP = 3.0
+FLAT_FILL_TEXT_END_GAP_PT = 3.0
+# Helvetica AFM: Ascender 718, Descender -207 (per 1000 em). The overlay value's
+# ink extends this far above/below its baseline; the OCR anchor box is the printed
+# label, not the value, so collision bands are modeled from the value's own ink.
+FLAT_FILL_GLYPH_ASCENT_RATIO = 0.718
+FLAT_FILL_GLYPH_DESCENT_RATIO = 0.207
+FLAT_FILL_LINE_EPS_PX = 2.0
+# Adobe AFM glyph widths (per 1000 units) for Helvetica, chars 32..126; fallback 600.
+# fmt: off
+_HELVETICA_WIDTHS_1000 = (
+ 278, 278, 355, 556, 556, 889, 667, 191, 333, 333, 389, 584, 278, 333, 278, 278,
+ 556, 556, 556, 556, 556, 556, 556, 556, 556, 556, 278, 278, 584, 584, 584, 556,
+ 1015, 667, 667, 722, 722, 667, 611, 778, 722, 278, 500, 667, 556, 833, 722, 778,
+ 667, 778, 722, 667, 611, 722, 667, 944, 667, 667, 611, 278, 278, 278, 469, 556,
+ 333, 556, 556, 500, 556, 556, 278, 556, 556, 222, 222, 500, 222, 833, 556, 556,
+ 556, 556, 333, 500, 278, 556, 500, 722, 500, 500, 500, 334, 260, 334, 584,
+)
+# fmt: on
FLAT_FILL_OCR_MIN_CONFIDENCE = 30.0
FLAT_FILL_FONT_RESOURCE = "/SkyvernHelv"
# Tesseract merges visually aligned table cells into one line; split anchors on column-sized gaps.
@@ -68,6 +88,25 @@ FLAT_FILL_MAX_PAGES = 25
FLAT_FILL_OCR_LANGUAGES = os.getenv("FLAT_FILL_OCR_LANGUAGES", DEFAULT_FLAT_FILL_OCR_LANGUAGES).strip()
+def _flat_text_width_pt(value: str, font_size: float) -> float:
+ single_line = " ".join(value.split())
+ total = 0
+ for ch in single_line:
+ code = ord(ch)
+ total += _HELVETICA_WIDTHS_1000[code - 32] if 32 <= code <= 126 else 600
+ return total * font_size / 1000.0
+
+
+def _flat_value_band_px(position: str, anchor: FlatPdfAnchor, scale_y: float, font_size: float) -> tuple[float, float]:
+ if position == "below":
+ baseline_px = anchor.bottom + (FLAT_FILL_FONT_SIZE_GAP + font_size) / scale_y
+ else:
+ baseline_px = anchor.bottom + 1 / scale_y
+ ascent_px = FLAT_FILL_GLYPH_ASCENT_RATIO * font_size / scale_y
+ descent_px = FLAT_FILL_GLYPH_DESCENT_RATIO * font_size / scale_y
+ return baseline_px - ascent_px, baseline_px + descent_px
+
+
@dataclass(frozen=True)
class PdfFieldInventory:
name: str
@@ -99,6 +138,15 @@ class FlatPlacement:
position: str
+@dataclass(frozen=True)
+class ResolvedFlatText:
+ value: str
+ x: float
+ y: float
+ font_size: float
+ overflowed: bool = False
+
+
class PdfFillBlock(Block):
block_type: Literal[BlockType.PDF_FILL] = BlockType.PDF_FILL # type: ignore
@@ -697,7 +745,138 @@ class PdfFillBlock(Block):
single_line = " ".join(value.split())
return single_line.replace("\\", r"\\").replace("(", r"\(").replace(")", r"\)")
- async def _fill_flat_overlay(self, reader: PdfReader, placements: list[FlatPlacement], output_path: Path) -> bytes:
+ def _resolve_flat_page_layout(
+ self,
+ page_placements: list[FlatPlacement],
+ page_anchors: list[FlatPdfAnchor],
+ *,
+ page_width: float,
+ page_height: float,
+ origin_x: float,
+ origin_y: float,
+ ) -> list[ResolvedFlatText]:
+ resolved: list[ResolvedFlatText] = []
+ resolved_rects: list[tuple[float, float, float, float]] = []
+
+ for placement in sorted(page_placements, key=lambda item: (item.anchor.top, item.anchor.x0)):
+ anchor = placement.anchor
+ if not anchor.page_width_px or not anchor.page_height_px:
+ continue
+ scale_x = page_width / anchor.page_width_px
+ scale_y = page_height / anchor.page_height_px
+ chosen_position = "below" if placement.position == "below" else "right"
+ alternate_position = "right" if chosen_position == "below" else "below"
+ start_x_by_position = {
+ "right": anchor.x1 + 6 / scale_x,
+ "below": anchor.x0 + 8 / scale_x,
+ }
+
+ def available_span(position: str) -> float:
+ start_x_px = start_x_by_position[position]
+ band_top, band_bottom = _flat_value_band_px(position, anchor, scale_y, float(FLAT_FILL_FONT_SIZE))
+ obstacles: list[tuple[float, float]] = []
+ for other_anchor in page_anchors:
+ if other_anchor.anchor_id == anchor.anchor_id:
+ continue
+ if position == "below" and other_anchor.bottom <= anchor.bottom + FLAT_FILL_LINE_EPS_PX:
+ continue
+ if min(band_bottom, other_anchor.bottom) > max(band_top, other_anchor.top):
+ obstacles.append((float(other_anchor.x0), float(other_anchor.x1)))
+ for rect_x0, rect_x1, rect_top, rect_bottom in resolved_rects:
+ if min(band_bottom, rect_bottom) > max(band_top, rect_top):
+ obstacles.append((rect_x0, rect_x1))
+
+ if any(obstacle_x0 <= start_x_px < obstacle_x1 for obstacle_x0, obstacle_x1 in obstacles):
+ return 0.0
+ page_right_px = anchor.page_width_px - 6 / scale_x
+ next_obstacle_x = min(
+ (obstacle_x0 for obstacle_x0, _ in obstacles if obstacle_x0 > start_x_px),
+ default=page_right_px,
+ )
+ return min(page_right_px, next_obstacle_x) - start_x_px
+
+ spans = {position: available_span(position) for position in ("right", "below")}
+
+ def fits(position: str, font_size: float) -> bool:
+ width_px = _flat_text_width_pt(placement.value, font_size) / scale_x
+ return width_px <= spans[position] - FLAT_FILL_TEXT_END_GAP_PT / scale_x
+
+ # Reject below ink past the page bottom, keeping bottom-row values visible to the right.
+ def on_page(position: str, font_size: float) -> bool:
+ if position != "below":
+ return True
+ _, band_bottom_px = _flat_value_band_px("below", anchor, scale_y, font_size)
+ return band_bottom_px <= anchor.page_height_px
+
+ candidates: list[tuple[str, float]] = [
+ (chosen_position, float(FLAT_FILL_FONT_SIZE)),
+ (alternate_position, float(FLAT_FILL_FONT_SIZE)),
+ ]
+ font_size = FLAT_FILL_FONT_SIZE - 0.5
+ while font_size >= FLAT_FILL_MIN_FONT_SIZE:
+ candidates.append((chosen_position, font_size))
+ font_size -= 0.5
+ font_size = FLAT_FILL_FONT_SIZE - 0.5
+ while font_size >= FLAT_FILL_MIN_FONT_SIZE:
+ candidates.append((alternate_position, font_size))
+ font_size -= 0.5
+
+ selected = next((candidate for candidate in candidates if on_page(*candidate) and fits(*candidate)), None)
+ overflowed = selected is None
+ if selected is None:
+ fallback_positions = ["right"] + (["below"] if on_page("below", FLAT_FILL_MIN_FONT_SIZE) else [])
+ final_position = max(fallback_positions, key=lambda position: spans[position])
+ final_font_size = FLAT_FILL_MIN_FONT_SIZE
+ LOG.warning(
+ "Flat PDF text placement overflow",
+ value_length=len(placement.value),
+ right_span_px=spans["right"],
+ below_span_px=spans["below"],
+ )
+ else:
+ final_position, final_font_size = selected
+
+ if final_position != chosen_position or final_font_size != FLAT_FILL_FONT_SIZE:
+ LOG.info(
+ "Adjusted flat PDF text placement",
+ anchor_id=anchor.anchor_id,
+ from_position=chosen_position,
+ to_position=final_position,
+ from_font_size=FLAT_FILL_FONT_SIZE,
+ to_font_size=final_font_size,
+ )
+
+ if final_position == "below":
+ text_x = origin_x + anchor.x0 * scale_x + 8
+ text_y = origin_y + page_height - anchor.bottom * scale_y - final_font_size - FLAT_FILL_FONT_SIZE_GAP
+ else:
+ text_x = origin_x + anchor.x1 * scale_x + 6
+ text_y = origin_y + page_height - anchor.bottom * scale_y - 1
+ resolved.append(
+ ResolvedFlatText(
+ value=placement.value,
+ x=text_x,
+ y=text_y,
+ font_size=final_font_size,
+ overflowed=overflowed,
+ )
+ )
+ start_x_px = start_x_by_position[final_position]
+ band_top, band_bottom = _flat_value_band_px(final_position, anchor, scale_y, final_font_size)
+ width_px = _flat_text_width_pt(placement.value, final_font_size) / scale_x
+ resolved_rects.append((start_x_px, start_x_px + width_px, band_top, band_bottom))
+
+ return resolved
+
+ async def _fill_flat_overlay(
+ self,
+ reader: PdfReader,
+ placements: list[FlatPlacement],
+ anchors: list[FlatPdfAnchor],
+ output_path: Path,
+ *,
+ overflow_records: list[dict[str, Any]] | None = None,
+ ) -> bytes:
writer = PdfWriter()
writer.clone_document_from_reader(reader)
@@ -716,23 +895,35 @@ class PdfFillBlock(Block):
page_height = float(crop_box.height)
origin_x = float(crop_box.left)
origin_y = float(crop_box.bottom)
- text_ops: list[str] = []
- for placement in page_placements:
- anchor = placement.anchor
- if not anchor.page_width_px or not anchor.page_height_px:
- continue
- scale_x = page_width / anchor.page_width_px
- scale_y = page_height / anchor.page_height_px
- if placement.position == "below":
- text_x = origin_x + anchor.x0 * scale_x + 8
- text_y = origin_y + page_height - anchor.bottom * scale_y - FLAT_FILL_FONT_SIZE - 3
- else:
- text_x = origin_x + anchor.x1 * scale_x + 6
- text_y = origin_y + page_height - anchor.bottom * scale_y - 1
- text_ops.append(
- f"BT {FLAT_FILL_FONT_RESOURCE} {FLAT_FILL_FONT_SIZE} Tf 0 0 0 rg "
- f"{text_x:.1f} {text_y:.1f} Td ({self._escape_pdf_text(placement.value)}) Tj ET"
- )
+ page_anchors = [anchor for anchor in anchors if anchor.page_index == page_index]
+ resolvable_placements = [
+ placement
+ for placement in sorted(page_placements, key=lambda item: (item.anchor.top, item.anchor.x0))
+ if placement.anchor.page_width_px and placement.anchor.page_height_px
+ ]
+ resolved_texts = self._resolve_flat_page_layout(
+ resolvable_placements,
+ page_anchors,
+ page_width=page_width,
+ page_height=page_height,
+ origin_x=origin_x,
+ origin_y=origin_y,
+ )
+ if overflow_records is not None:
+ for placement, resolved in zip(resolvable_placements, resolved_texts, strict=True):
+ if resolved.overflowed:
+ overflow_records.append(
+ {
+ "anchor_id": placement.anchor.anchor_id,
+ "value": resolved.value,
+ "reason": "Placement overflowed the available space; value may overlap adjacent content",
+ }
+ )
+ text_ops = [
+ f"BT {FLAT_FILL_FONT_RESOURCE} {resolved.font_size:g} Tf 0 0 0 rg "
+ f"{resolved.x:.1f} {resolved.y:.1f} Td ({self._escape_pdf_text(resolved.value)}) Tj ET"
+ for resolved in resolved_texts
+ ]
if not text_ops:
continue
@@ -752,6 +943,7 @@ class PdfFillBlock(Block):
NameObject("/Type"): NameObject("/Font"),
NameObject("/Subtype"): NameObject("/Type1"),
NameObject("/BaseFont"): NameObject("/Helvetica"),
+ NameObject("/Encoding"): NameObject("/WinAnsiEncoding"),
}
)
overlay_page[NameObject("/Resources")] = DictionaryObject(
@@ -905,6 +1097,8 @@ class PdfFillBlock(Block):
)
fill_mode = "acroform" if inventory else "flat_overlay"
+ skipped_fields: list[dict[str, Any]] = []
+ overflowed_placements: list[dict[str, Any]] = []
try:
output_path = self._output_path(workflow_run_id, workflow_run_block_id)
if inventory:
@@ -969,7 +1163,13 @@ class PdfFillBlock(Block):
output_parameter_value={"fields": {}, "skipped_fields": skipped_fields},
)
fields = {placement.anchor.text: placement.value for placement in placements}
- pdf_bytes = await self._fill_flat_overlay(reader, placements, output_path)
+ pdf_bytes = await self._fill_flat_overlay(
+ reader,
+ placements,
+ anchors,
+ output_path,
+ overflow_records=overflowed_placements,
+ )
except Exception as e:
return await self._record_failure(
workflow_run_context,
@@ -1007,6 +1207,7 @@ class PdfFillBlock(Block):
"fill_mode": fill_mode,
"fields": fields,
"skipped_fields": skipped_fields,
+ "overflowed_placements": overflowed_placements,
"file_path": str(output_path),
"file_name": output_path.name,
"file_size": output_path.stat().st_size,
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index a9fc0396e..eb4e39a3b 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -62,6 +62,7 @@ from skyvern.exceptions import (
NoIncrementalElementFoundForAutoCompletion,
NoIncrementalElementFoundForCustomSelection,
NoSuitableAutoCompleteOption,
+ NoTOTPSecretFound,
OptionIndexOutOfBound,
PhoneNumberInputMismatch,
SkyvernException,
@@ -95,6 +96,8 @@ from skyvern.forge.sdk.services.credentials import (
AzureVaultConstants,
OnePasswordConstants,
generate_totp_code,
+ is_unresolved_totp_placeholder,
+ is_unresolved_totp_value,
parse_totp_config,
)
from skyvern.forge.sdk.settings_manager import SettingsManager
@@ -3009,9 +3012,54 @@ async def handle_input_text_action(
) -> list[ActionResult]:
if not action.element_id:
# This is a CUA type action
- await EventStrategyFactory.type_text(page, None, action.text)
+ text_result = get_actual_value_of_parameter_if_secret_with_task(task, action.text)
+ if text_result is None:
+ return [ActionFailure(FailedToFetchSecret())]
+ if is_unresolved_totp_placeholder(text_result):
+ return [ActionFailure(NoTOTPSecretFound())]
+ cua_text = text_result
+ if cua_text in (BitwardenConstants.TOTP, OnePasswordConstants.TOTP, AzureVaultConstants.TOTP):
+ try:
+ cua_totp_secret = get_totp_secret_with_task(task=task, parameter=action.text)
+ cua_text = generate_totp_value_from_secret(cua_totp_secret)
+ except NoTOTPSecretFound as exc:
+ return [ActionFailure(exc)]
+ elif is_unresolved_totp_value(cua_text):
+ return [ActionFailure(NoTOTPSecretFound())]
+ await EventStrategyFactory.type_text(page, None, cua_text)
return [ActionSuccess()]
+ totp_secret: str | None = None
+ is_multi_field_totp = bool(action.totp_timing_info and action.totp_timing_info.get("is_totp_sequence"))
+ if is_multi_field_totp:
+ text = ""
+ current_text_target = action.text
+ is_totp_value = False
+ is_secret_value = True
+ else:
+ text_result = get_actual_value_of_parameter_if_secret_with_task(task, action.text)
+ if text_result is None:
+ return [ActionFailure(FailedToFetchSecret())]
+ if is_unresolved_totp_placeholder(text_result):
+ return [ActionFailure(NoTOTPSecretFound())]
+ is_totp_value = text_result in (
+ BitwardenConstants.TOTP,
+ OnePasswordConstants.TOTP,
+ AzureVaultConstants.TOTP,
+ )
+ if not is_totp_value and is_unresolved_totp_value(text_result):
+ return [ActionFailure(NoTOTPSecretFound())]
+ if is_totp_value:
+ try:
+ totp_secret = get_totp_secret_with_task(task=task, parameter=action.text)
+ except NoTOTPSecretFound as exc:
+ return [ActionFailure(exc)]
+ text = ""
+ else:
+ text = text_result
+ current_text_target = text_result
+ is_secret_value = is_totp_value or text != action.text
+
dom = DomUtil(scraped_page, page)
skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)
skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())
@@ -3019,28 +3067,12 @@ async def handle_input_text_action(
timeout = settings.BROWSER_ACTION_TIMEOUT_MS
current_text = await get_input_value(skyvern_element.get_tag_name(), skyvern_element.get_locator())
- if current_text == action.text:
+ if not is_totp_value and current_text == current_text_target:
return [ActionSuccess()]
# before filling text, we need to validate if the element can be filled if it's not one of COMMON_INPUT_TAGS
tag_name = scraped_page.id_to_element_dict[action.element_id]["tagName"].lower()
- # Check if this is multi-field TOTP first - if so, skip secret resolution
- if action.totp_timing_info and action.totp_timing_info.get("is_totp_sequence"):
- # For multi-field TOTP, we'll set text directly in the TOTP logic below
- text: str = ""
- else:
- # For regular inputs, resolve secrets
- text_result = get_actual_value_of_parameter_if_secret_with_task(task, action.text)
- if text_result is None:
- return [ActionFailure(FailedToFetchSecret())]
- text = text_result
-
- is_totp_value = (
- text == BitwardenConstants.TOTP or text == OnePasswordConstants.TOTP or text == AzureVaultConstants.TOTP
- )
- is_secret_value = text != action.text
-
# dynamically validate the attr, since it could change into enabled after the previous actions
if await skyvern_element.is_disabled(dynamic=True):
LOG.warning(
@@ -3050,14 +3082,20 @@ async def handle_input_text_action(
)
return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]
- select_action = SelectOptionAction(
- reasoning=action.reasoning,
- element_id=skyvern_element.get_id(),
- option=SelectOption(label=text),
- intention=action.intention,
- input_or_select_context=action.input_or_select_context,
- )
if await skyvern_element.get_selectable():
+ select_text = text
+ if is_totp_value:
+ try:
+ select_text = generate_totp_value_from_secret(totp_secret)
+ except NoTOTPSecretFound as exc:
+ return [ActionFailure(exc)]
+ select_action = SelectOptionAction(
+ reasoning=action.reasoning,
+ element_id=skyvern_element.get_id(),
+ option=SelectOption(label=select_text),
+ intention=action.intention,
+ input_or_select_context=action.input_or_select_context,
+ )
LOG.info(
"Input element is selectable, doing select actions",
element_id=skyvern_element.get_id(),
@@ -3066,6 +3104,14 @@ async def handle_input_text_action(
action.set_has_mini_agent()
return await handle_select_option_action(select_action, page, scraped_page, task, step)
+ select_action = SelectOptionAction(
+ reasoning=action.reasoning,
+ element_id=skyvern_element.get_id(),
+ option=SelectOption(label=text),
+ intention=action.intention,
+ input_or_select_context=action.input_or_select_context,
+ )
+
incremental_element: list[dict] = []
auto_complete_hacky_flag: bool = False
@@ -3285,7 +3331,10 @@ async def handle_input_text_action(
class_name: str | None = await skyvern_element.get_attr("class")
if class_name and "blinking-cursor" in class_name.lower():
if is_totp_value:
- text = generate_totp_value_with_task(task=task, parameter=action.text)
+ try:
+ text = generate_totp_value_from_secret(totp_secret)
+ except NoTOTPSecretFound as exc:
+ return [ActionFailure(exc)]
await skyvern_element.press_fill(text=text)
return [ActionSuccess()]
@@ -3340,7 +3389,10 @@ async def handle_input_text_action(
if is_totp_value:
LOG.info("Skipping the auto completion logic since it's a TOTP input")
- text = generate_totp_value_with_task(task=task, parameter=action.text)
+ try:
+ text = generate_totp_value_from_secret(totp_secret)
+ except NoTOTPSecretFound as exc:
+ return [ActionFailure(exc)]
await skyvern_element.input(text)
return [ActionSuccess()]
@@ -4787,35 +4839,127 @@ def get_actual_value_of_parameter_if_secret_with_task(task: Task, parameter: str
return get_actual_value_of_parameter_if_secret(task.workflow_run_id, parameter)
-def generate_totp_value(workflow_run_id: str, parameter: str) -> str:
+def get_totp_secret(workflow_run_id: str, parameter: str) -> str:
workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(workflow_run_id)
+ if not workflow_run_context:
+ raise NoTOTPSecretFound()
totp_secret_key = workflow_run_context.totp_secret_value_key(parameter)
totp_secret = workflow_run_context.get_original_secret_value_or_none(totp_secret_key)
if not totp_secret:
- LOG.warning("No TOTP secret found, returning the parameter value as is", parameter=parameter)
- return parameter
- return generate_totp_code(totp_secret)
+ LOG.warning("No TOTP secret found")
+ raise NoTOTPSecretFound()
+ if parse_totp_config(totp_secret) is None:
+ LOG.warning("Failed to parse TOTP credential secret")
+ raise NoTOTPSecretFound()
+ return totp_secret
+
+
+def get_totp_secret_with_task(task: Task, parameter: str) -> str:
+ if task.workflow_run_id is None:
+ raise NoTOTPSecretFound()
+ return get_totp_secret(task.workflow_run_id, parameter)
+
+
+def generate_totp_value_from_secret(totp_secret: str | None) -> str:
+ if not totp_secret:
+ raise NoTOTPSecretFound()
+ try:
+ return generate_totp_code(totp_secret)
+ except Exception as exc:
+ LOG.warning("Failed to generate TOTP from credential secret", exception_type=type(exc).__name__)
+ raise NoTOTPSecretFound() from exc
+
+
+def generate_totp_value(workflow_run_id: str, parameter: str) -> str:
+ return generate_totp_value_from_secret(get_totp_secret(workflow_run_id, parameter))
def generate_totp_value_with_task(task: Task, parameter: str) -> str:
if task.workflow_run_id is None:
- return parameter
+ raise NoTOTPSecretFound()
return generate_totp_value(task.workflow_run_id, parameter)
async def _did_page_respond(
incremental_scraped: IncrementalScrapePage,
+ frame: Page | Frame,
+ original_url: str,
skyvern_frame: SkyvernFrame | None = None,
) -> bool:
try:
if skyvern_frame:
await skyvern_frame.safe_wait_for_animation_end(caller="page_respond")
- return (await incremental_scraped.get_incremental_elements_num()) > 0
+ return frame.url != original_url or (await incremental_scraped.get_incremental_elements_num()) > 0
except Exception:
LOG.debug("Failed to check incremental elements after click", exc_info=True)
return True
+async def _click_in_javascript_and_verify(
+ action: ClickAction | UploadFileAction,
+ page: Page,
+ skyvern_element: SkyvernElement,
+ incremental_scraped: IncrementalScrapePage,
+ skyvern_frame: SkyvernFrame | None,
+ *,
+ anchor: str,
+ no_response_msg: str,
+) -> ActionResult:
+ try:
+ frame = skyvern_frame.get_frame() if skyvern_frame else page
+ original_url = frame.url
+ await skyvern_element.click_in_javascript()
+ if await _did_page_respond(incremental_scraped, frame, original_url, skyvern_frame):
+ return ActionSuccess()
+ LOG.info(
+ "Chain click: JS click did not trigger a page response",
+ action=action,
+ element=str(skyvern_element),
+ )
+ return ActionFailure(FailToClick(action.element_id, anchor=anchor, msg=no_response_msg))
+ except Exception as e:
+ return ActionFailure(FailToClick(action.element_id, anchor=anchor, msg=str(e)))
+
+
+async def _is_likely_sibling_control(element: SkyvernElement, timeout: int) -> bool:
+ try:
+ metrics = await element.get_locator().evaluate(
+ """(element) => {
+ const rect = element.getBoundingClientRect();
+ const root = document.documentElement;
+ return {
+ width: rect.width,
+ height: rect.height,
+ viewport_width: window.innerWidth || root.clientWidth || 0,
+ viewport_height: window.innerHeight || root.clientHeight || 0,
+ inside_modal: Boolean(element.closest(
+ 'dialog,[role="dialog" i],[role="alertdialog" i],[aria-modal="true" i]'
+ )),
+ };
+ }""",
+ timeout=timeout,
+ )
+ except Exception:
+ LOG.debug("Failed to classify blocking sibling", exc_info=True, element_id=element.get_id())
+ return False
+
+ if not isinstance(metrics, dict) or metrics.get("inside_modal") is not False:
+ return False
+ width = metrics.get("width")
+ height = metrics.get("height")
+ viewport_width = metrics.get("viewport_width")
+ viewport_height = metrics.get("viewport_height")
+ dimensions = (width, height, viewport_width, viewport_height)
+ numeric_dimensions: list[float] = []
+ for value in dimensions:
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
+ return False
+ numeric_dimensions.append(float(value))
+ numeric_width, numeric_height, numeric_viewport_width, numeric_viewport_height = numeric_dimensions
+ # ponytail: non-modal overlays are size-gated; add style/z-index signals if smaller overlays surface.
+ return numeric_width < numeric_viewport_width / 2 and numeric_height < numeric_viewport_height / 2
+
+
def _get_click_count(action: ClickAction | UploadFileAction) -> int:
if isinstance(action, ClickAction):
return action.repeat
@@ -5032,6 +5176,46 @@ async def chain_click(
)
action_results.append(ActionSuccess())
return action_results
+
+ # A non-anchor click (e.g. a primary form submit) blocked by an
+ # untracked overlay (a consent / opt-out / FCRA modal over the
+ # control): a coordinate click would land on whatever pixel is on
+ # top — the overlay's own button — so dispatch on the intended
+ # element instead (element.click() fires on the target node,
+ # bypassing hit-testing and never touching the overlay). This
+ # needs the observer to confirm the click landed; without one we
+ # fail cleanly rather than guess, like the blocker path below.
+ if (
+ isinstance(action, ClickAction)
+ and not action.file_url
+ and not action.download
+ and action.x is None
+ and action.y is None
+ and click_count == 1
+ and incremental_scraped is not None
+ and skyvern_element.get_tag_name() != InteractiveElement.A
+ ):
+ LOG.info(
+ "Chain click: click blocked by an untracked overlay; dispatching on the intended element "
+ "instead of coordinate-clicking into the overlay",
+ action=action,
+ element=str(skyvern_element),
+ locator=locator,
+ )
+ action_results.append(
+ await _click_in_javascript_and_verify(
+ action,
+ page,
+ skyvern_element,
+ incremental_scraped,
+ skyvern_frame,
+ anchor="overlay_blocked",
+ no_response_msg=(
+ "click blocked by overlay; no page response after intended-element dispatch"
+ ),
+ )
+ )
+ return action_results
else:
# Element is visible and elementFromPoint returns the target itself,
# but Playwright's click still failed (e.g. element transiently
@@ -5075,9 +5259,11 @@ async def chain_click(
element=str(blocking_element),
locator=locator,
)
- if await blocking_element.is_parent_of(
- await skyvern_element.get_element_handler()
- ) or await blocking_element.is_sibling_of(await skyvern_element.get_element_handler()):
+ target_handler = await skyvern_element.get_element_handler()
+ if await blocking_element.is_parent_of(target_handler) or (
+ await blocking_element.is_sibling_of(target_handler)
+ and (incremental_scraped is None or await _is_likely_sibling_control(blocking_element, timeout))
+ ):
LOG.info(
"Chain click: element is blocked by other elements, going to click on the blocking element",
action=action,
@@ -5099,28 +5285,23 @@ async def chain_click(
# JS click dispatches directly on the DOM node, bypassing hit-testing.
LOG.info(
- "Chain click: blocker is not parent/sibling, trying JS click on original element",
+ "Chain click: blocker is not a safe retarget, trying JS click on original element",
action=action,
element=str(skyvern_element),
locator=locator,
)
- try:
- await skyvern_element.click_in_javascript()
- if await _did_page_respond(incremental_scraped, skyvern_frame):
- action_results.append(ActionSuccess())
- return action_results
- LOG.info(
- "Chain click: JS click did not trigger a page response",
- action=action,
- element=str(skyvern_element),
- )
- action_results.append(
- ActionFailure(FailToClick(action.element_id, anchor="self_js", msg="no page response after click"))
+ action_results.append(
+ await _click_in_javascript_and_verify(
+ action,
+ page,
+ skyvern_element,
+ incremental_scraped,
+ skyvern_frame,
+ anchor="self_js",
+ no_response_msg="no page response after click",
)
- return action_results
- except Exception as e:
- action_results.append(ActionFailure(FailToClick(action.element_id, anchor="self_js", msg=str(e))))
- return action_results
+ )
+ return action_results
finally:
click_succeeded = any(isinstance(r, ActionSuccess) for r in action_results)
diff --git a/tests/unit/test_agent_step_artifact_tracker.py b/tests/unit/test_agent_step_artifact_tracker.py
new file mode 100644
index 000000000..046a10388
--- /dev/null
+++ b/tests/unit/test_agent_step_artifact_tracker.py
@@ -0,0 +1,186 @@
+"""Isolation guarantees for the per-call background-artifact tracker introduced in SKY-11698.
+
+The action-execution loop was extracted from ForgeAgent.agent_step into
+_execute_step_actions. The in-flight background artifact task, previously held
+by a nested closure over a `nonlocal` local, is now a _BackgroundArtifactTaskTracker
+instantiated fresh inside agent_step on every call. ForgeAgent is a process-wide
+singleton (app.agent), so concurrent agent_step calls share `self`; a tracker stored
+on the instance or as a module global would let one call await — or clobber — another
+call's background write. These tests pin that the tracker stays per-call.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+
+import skyvern.forge.agent as agent_mod
+from skyvern.forge.agent import _BackgroundArtifactTaskTracker
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.models import StepStatus
+from tests.unit.test_agent_step_characterization import make_agent_step_rig
+
+
+class _RecordingTracker(_BackgroundArtifactTaskTracker):
+ """Real tracker semantics plus a record of which task results each instance drained."""
+
+ instances: list[_RecordingTracker] = []
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.drained_results: list[object] = []
+ _RecordingTracker.instances.append(self)
+
+ async def drain(self) -> None:
+ if self.task is None:
+ return
+ task = self.task
+ self.task = None
+ try:
+ self.drained_results.append(await task)
+ except Exception:
+ agent_mod.LOG.warning("Background artifact task failed, continuing", exc_info=True)
+
+
+def _tag_background_writes(agent: agent_mod.ForgeAgent) -> None:
+ """Make the background artifact task resolve to its owning task_id so drains are attributable."""
+
+ async def _record(task, step, browser_state, engine, action) -> str:
+ return task.task_id
+
+ agent.record_artifacts_after_action = _record # type: ignore[method-assign]
+
+
+# --- _BackgroundArtifactTaskTracker contract (parity with the old closure) ---
+
+
+async def test_drain_is_noop_when_no_task() -> None:
+ tracker = _BackgroundArtifactTaskTracker()
+ assert tracker.task is None
+ await tracker.drain() # must not raise
+ assert tracker.task is None
+
+
+async def test_drain_awaits_task_and_clears_it() -> None:
+ awaited = asyncio.Event()
+
+ async def _work() -> None:
+ awaited.set()
+
+ tracker = _BackgroundArtifactTaskTracker()
+ tracker.task = asyncio.create_task(_work())
+ await tracker.drain()
+
+ assert awaited.is_set()
+ assert tracker.task is None
+
+
+async def test_drain_swallows_task_exception() -> None:
+ async def _boom() -> None:
+ raise RuntimeError("artifact write failed")
+
+ tracker = _BackgroundArtifactTaskTracker()
+ tracker.task = asyncio.create_task(_boom())
+
+ await tracker.drain() # closure logged-and-continued; tracker must too
+
+ assert tracker.task is None
+
+
+# --- Per-call isolation through agent_step ---
+
+
+async def test_agent_step_uses_a_fresh_drained_tracker_per_call(monkeypatch: pytest.MonkeyPatch) -> None:
+ _RecordingTracker.instances.clear()
+ monkeypatch.setattr(agent_mod, "_BackgroundArtifactTaskTracker", _RecordingTracker)
+
+ rig_a = make_agent_step_rig(monkeypatch, task_overrides={"task_id": "task-seq-a"})
+ _tag_background_writes(rig_a.agent)
+ rig_b = make_agent_step_rig(monkeypatch, task_overrides={"task_id": "task-seq-b"})
+ _tag_background_writes(rig_b.agent)
+
+ step_a, _ = await rig_a.run()
+ step_b, _ = await rig_b.run()
+
+ assert step_a.status == StepStatus.completed
+ assert step_b.status == StepStatus.completed
+
+ assert len(_RecordingTracker.instances) == 2
+ first, second = _RecordingTracker.instances
+ assert first is not second
+ # each call drained (task cleared) and drained only its own background write
+ assert first.task is None and second.task is None
+ assert first.drained_results == ["task-seq-a"]
+ assert second.drained_results == ["task-seq-b"]
+
+
+async def test_tracker_is_never_shared_state(monkeypatch: pytest.MonkeyPatch) -> None:
+ rig = make_agent_step_rig(monkeypatch)
+ _tag_background_writes(rig.agent)
+
+ step, _ = await rig.run()
+ assert step.status == StepStatus.completed
+
+ # A stored-on-self tracker would let concurrent calls on the app.agent singleton bleed.
+ assert not any(isinstance(v, _BackgroundArtifactTaskTracker) for v in vars(rig.agent).values())
+ # A module-global tracker instance would bleed across every call.
+ assert not any(isinstance(v, _BackgroundArtifactTaskTracker) for v in vars(agent_mod).values())
+
+
+async def test_concurrent_agent_steps_do_not_bleed_tracker_state(monkeypatch: pytest.MonkeyPatch) -> None:
+ _RecordingTracker.instances.clear()
+ monkeypatch.setattr(agent_mod, "_BackgroundArtifactTaskTracker", _RecordingTracker)
+
+ rig_a = make_agent_step_rig(monkeypatch, task_overrides={"task_id": "task-cc-a"})
+ rig_b = make_agent_step_rig(monkeypatch, task_overrides={"task_id": "task-cc-b"})
+
+ # Hold both calls' background writes open until both are scheduled, forcing the
+ # window where both trackers are simultaneously live.
+ both_scheduled = asyncio.Event()
+ started = 0
+
+ async def _blocking_record(task, step, browser_state, engine, action) -> str:
+ nonlocal started
+ started += 1
+ if started >= 2:
+ both_scheduled.set()
+ await both_scheduled.wait()
+ return task.task_id
+
+ rig_a.agent.record_artifacts_after_action = _blocking_record # type: ignore[method-assign]
+ rig_b.agent.record_artifacts_after_action = _blocking_record # type: ignore[method-assign]
+
+ # Separate asyncio tasks so each call gets its own copy of the skyvern_context ContextVar.
+ async def _run(rig) -> StepStatus:
+ skyvern_context.set(rig.context)
+ try:
+ step, _ = await rig.agent.agent_step(
+ task=rig.task,
+ step=rig.step,
+ browser_state=rig.browser_state,
+ organization=rig.organization,
+ )
+ return step.status
+ finally:
+ skyvern_context.reset()
+
+ status_a, status_b = await asyncio.gather(
+ asyncio.create_task(_run(rig_a)),
+ asyncio.create_task(_run(rig_b)),
+ )
+
+ assert status_a == StepStatus.completed
+ assert status_b == StepStatus.completed
+
+ assert len(_RecordingTracker.instances) == 2
+ # No tracker drained another call's background write.
+ for tracker in _RecordingTracker.instances:
+ assert len(set(tracker.drained_results)) == 1
+ assert {r for t in _RecordingTracker.instances for r in t.drained_results} == {"task-cc-a", "task-cc-b"}
diff --git a/tests/unit/test_chain_click_blocked_anchor_navigation.py b/tests/unit/test_chain_click_blocked_anchor_navigation.py
index 62d8095cd..6e7bc384d 100644
--- a/tests/unit/test_chain_click_blocked_anchor_navigation.py
+++ b/tests/unit/test_chain_click_blocked_anchor_navigation.py
@@ -9,6 +9,7 @@ fallback for the ``blocking_element is None and blocked is True`` branch only.
from __future__ import annotations
+from typing import NamedTuple
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -18,6 +19,14 @@ from skyvern.webeye.utils import dom as dom_module
from skyvern.webeye.utils.dom import InteractiveElement, SkyvernElement
+class _ChainClickRun(NamedTuple):
+ results: list
+ coordinate_click: AsyncMock
+ try_navigate: AsyncMock
+ click_in_javascript: AsyncMock
+ blocking_click: AsyncMock | None
+
+
def _make_anchor_element(
href: str | None = "/target",
tag: str = InteractiveElement.A,
@@ -303,10 +312,26 @@ class TestChainClickBlockedAnchorNavigation:
action_y: int | None = None,
action_download: bool = False,
navigate_result: str | None = "https://portal.example.com/billpay",
- ) -> tuple[list, AsyncMock, AsyncMock]:
- """Invoke chain_click through the ``blocking_element is None`` path
- with tightly-scoped stubs. Returns (results, coordinate_click_mock,
- try_navigate_mock)."""
+ with_incremental: bool = False,
+ page_responded: bool | None = True,
+ input_type: str | None = None,
+ js_click_url: str | None = None,
+ blocking_is_parent: bool | None = None,
+ blocking_is_sibling: bool | None = None,
+ blocking_rect: dict[str, float] | None = None,
+ blocking_viewport: tuple[float, float] = (1920, 1080),
+ blocking_inside_modal: bool = False,
+ ) -> _ChainClickRun:
+ """Invoke chain_click through the ``find_blocking_element`` fallback with
+ tightly-scoped stubs.
+
+ ``with_incremental`` supplies an ``incremental_scraped`` observer (as the
+ real ClickAction path does) and monkeypatches ``_did_page_respond`` to
+ return ``page_responded`` so the intended-element JS-dispatch fallback can
+ be verified. When ``blocking_is_parent``/``blocking_is_sibling`` are set,
+ ``find_blocking_element`` returns a tracked blocking element with those
+ relationships (Path 2); otherwise it returns ``(None, blocked)`` (Path 1).
+ """
from skyvern.webeye.actions import handler as handler_module
from skyvern.webeye.actions.actions import ClickAction
@@ -352,6 +377,8 @@ class TestChainClickBlockedAnchorNavigation:
return href
if name == "onclick":
return None
+ if name == "type":
+ return input_type
return None
elem.get_attr = AsyncMock(side_effect=_get_attr) # type: ignore[method-assign]
@@ -359,10 +386,37 @@ class TestChainClickBlockedAnchorNavigation:
elem.find_bound_label_by_attr_id = AsyncMock(return_value=None) # type: ignore[method-assign]
elem.find_bound_label_by_direct_parent = AsyncMock(return_value=None) # type: ignore[method-assign]
elem.is_visible = AsyncMock(return_value=True) # type: ignore[method-assign]
- elem.find_blocking_element = AsyncMock(return_value=(None, blocked)) # type: ignore[method-assign]
+ elem.get_element_handler = AsyncMock(return_value=MagicMock()) # type: ignore[method-assign]
elem.coordinate_click = AsyncMock(return_value=None) # type: ignore[method-assign]
elem.click_in_javascript = AsyncMock(return_value=None) # type: ignore[method-assign]
+ blocking_click: AsyncMock | None = None
+ if blocking_is_parent is not None or blocking_is_sibling is not None:
+ blocking_element = MagicMock()
+ blocking_element.get_id = MagicMock(return_value="BLK1")
+ blocking_element.is_parent_of = AsyncMock(return_value=bool(blocking_is_parent))
+ blocking_element.is_sibling_of = AsyncMock(return_value=bool(blocking_is_sibling))
+ blocking_click = AsyncMock(return_value=None)
+ blocking_locator = MagicMock()
+ blocking_locator.click = blocking_click
+ blocking_locator.evaluate = AsyncMock(
+ return_value=(
+ {
+ "width": blocking_rect["width"],
+ "height": blocking_rect["height"],
+ "viewport_width": blocking_viewport[0],
+ "viewport_height": blocking_viewport[1],
+ "inside_modal": blocking_inside_modal,
+ }
+ if blocking_rect is not None
+ else None
+ )
+ )
+ blocking_element.get_locator = MagicMock(return_value=blocking_locator)
+ elem.find_blocking_element = AsyncMock(return_value=(blocking_element, blocked)) # type: ignore[method-assign]
+ else:
+ elem.find_blocking_element = AsyncMock(return_value=(None, blocked)) # type: ignore[method-assign]
+
try_navigate = AsyncMock(return_value=navigate_result)
elem.try_navigate_via_href = try_navigate # type: ignore[method-assign]
@@ -370,6 +424,12 @@ class TestChainClickBlockedAnchorNavigation:
page.url = "https://portal.example.com/dashboard"
page.goto = AsyncMock(return_value=None)
page.on = MagicMock()
+ if js_click_url is not None:
+
+ async def _navigate_on_js_click() -> None:
+ page.url = js_click_url
+
+ elem.click_in_javascript = AsyncMock(side_effect=_navigate_on_js_click) # type: ignore[method-assign]
task = MagicMock()
task.organization_id = "org_test"
@@ -382,14 +442,32 @@ class TestChainClickBlockedAnchorNavigation:
y=action_y,
)
+ incremental = None
+ skyvern_frame = None
+ if with_incremental:
+ incremental = MagicMock()
+ skyvern_frame = MagicMock()
+ skyvern_frame.get_frame.return_value = page
+ if page_responded is None:
+ incremental.get_incremental_elements_num = AsyncMock(return_value=0)
+ skyvern_frame.safe_wait_for_animation_end = AsyncMock(return_value=None)
+ else:
+ monkeypatch.setattr(
+ handler_module,
+ "_did_page_respond",
+ AsyncMock(return_value=page_responded),
+ )
+
results = await handler_module.chain_click(
task=task,
scraped_page=scraped_page,
page=page,
action=action,
skyvern_element=elem,
+ incremental_scraped=incremental,
+ skyvern_frame=skyvern_frame,
)
- return results, elem.coordinate_click, try_navigate
+ return _ChainClickRun(results, elem.coordinate_click, try_navigate, elem.click_in_javascript, blocking_click)
@pytest.mark.asyncio
async def test_blocked_anchor_navigates_via_href_and_skips_coordinate_click(
@@ -397,7 +475,7 @@ class TestChainClickBlockedAnchorNavigation:
) -> None:
from skyvern.webeye.actions.responses import ActionSuccess
- results, coord_click, navigate = await self._run_chain_click(
+ results, coord_click, navigate, *_ = await self._run_chain_click(
monkeypatch,
blocked=True,
tag=InteractiveElement.A,
@@ -410,7 +488,7 @@ class TestChainClickBlockedAnchorNavigation:
@pytest.mark.asyncio
async def test_blocked_non_anchor_still_uses_coordinate_click(self, monkeypatch: pytest.MonkeyPatch) -> None:
- results, coord_click, navigate = await self._run_chain_click(
+ results, coord_click, navigate, *_ = await self._run_chain_click(
monkeypatch,
blocked=True,
tag=InteractiveElement.BUTTON,
@@ -424,7 +502,7 @@ class TestChainClickBlockedAnchorNavigation:
async def test_unblocked_anchor_still_uses_coordinate_click(self, monkeypatch: pytest.MonkeyPatch) -> None:
# blocked=False = the transient-instability React re-render case; the
# coordinate fallback is safe there. We must not divert to href nav.
- results, coord_click, navigate = await self._run_chain_click(
+ results, coord_click, navigate, *_ = await self._run_chain_click(
monkeypatch,
blocked=False,
tag=InteractiveElement.A,
@@ -438,7 +516,7 @@ class TestChainClickBlockedAnchorNavigation:
async def test_blocked_anchor_with_upload_action_skips_href_navigation(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
- results, coord_click, navigate = await self._run_chain_click(
+ results, coord_click, navigate, *_ = await self._run_chain_click(
monkeypatch,
blocked=True,
tag=InteractiveElement.A,
@@ -456,7 +534,7 @@ class TestChainClickBlockedAnchorNavigation:
# JS-driven downloads (onclick builds a blob/POST) would fetch the
# wrong static resource if we diverted to ``frame.goto(href)``.
# Downloads must keep going through the normal click path.
- results, coord_click, navigate = await self._run_chain_click(
+ results, coord_click, navigate, *_ = await self._run_chain_click(
monkeypatch,
blocked=True,
tag=InteractiveElement.A,
@@ -471,7 +549,7 @@ class TestChainClickBlockedAnchorNavigation:
async def test_blocked_anchor_with_explicit_coordinates_skips_href_navigation(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
- results, coord_click, navigate = await self._run_chain_click(
+ results, coord_click, navigate, *_ = await self._run_chain_click(
monkeypatch,
blocked=True,
tag=InteractiveElement.A,
@@ -489,7 +567,7 @@ class TestChainClickBlockedAnchorNavigation:
) -> None:
# If the helper cannot resolve a safe URL (e.g. javascript:) it returns
# None; the existing coordinate fallback must still run.
- results, coord_click, navigate = await self._run_chain_click(
+ results, coord_click, navigate, *_ = await self._run_chain_click(
monkeypatch,
blocked=True,
tag=InteractiveElement.A,
@@ -501,6 +579,279 @@ class TestChainClickBlockedAnchorNavigation:
coord_click.assert_awaited_once()
+class TestChainClickOverlayBlockedSubmit:
+ """A primary-submit click intercepted by a consent / opt-out / FCRA overlay
+ must not be silently redirected onto the overlay. On the observer-backed
+ ClickAction path (``incremental_scraped`` present), chain_click dispatches on
+ the *intended* element (bypassing hit-testing) instead of coordinate-clicking
+ into whatever pixel is on top, and never retargets onto a mere sibling
+ overlay. Callers without an observer (custom checkbox/radio retarget) keep
+ their existing coordinate-click / sibling-retarget behavior.
+ """
+
+ _run_chain_click = staticmethod(TestChainClickBlockedAnchorNavigation._run_chain_click)
+
+ @staticmethod
+ def _has_success(results: list) -> bool:
+ from skyvern.webeye.actions.responses import ActionSuccess
+
+ return any(isinstance(r, ActionSuccess) for r in results)
+
+ @staticmethod
+ def _has_failure(results: list) -> bool:
+ from skyvern.webeye.actions.responses import ActionFailure
+
+ return any(isinstance(r, ActionFailure) for r in results)
+
+ @pytest.mark.asyncio
+ async def test_untracked_overlay_dispatches_on_intended_element_not_overlay(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ run = await self._run_chain_click(
+ monkeypatch,
+ blocked=True,
+ tag=InteractiveElement.BUTTON,
+ href=None,
+ with_incremental=True,
+ page_responded=True,
+ )
+
+ run.click_in_javascript.assert_awaited_once()
+ run.coordinate_click.assert_not_called()
+ assert self._has_success(run.results)
+
+ @pytest.mark.asyncio
+ async def test_untracked_overlay_no_page_response_fails_clean(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ run = await self._run_chain_click(
+ monkeypatch,
+ blocked=True,
+ tag=InteractiveElement.BUTTON,
+ href=None,
+ with_incremental=True,
+ page_responded=False,
+ )
+
+ run.click_in_javascript.assert_awaited_once()
+ run.coordinate_click.assert_not_called()
+ assert self._has_failure(run.results)
+ assert not self._has_success(run.results)
+
+ @pytest.mark.asyncio
+ async def test_untracked_overlay_input_submit_dispatches_on_intended_element(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ run = await self._run_chain_click(
+ monkeypatch,
+ blocked=True,
+ tag=InteractiveElement.INPUT,
+ href=None,
+ with_incremental=True,
+ page_responded=True,
+ )
+
+ run.click_in_javascript.assert_awaited_once()
+ run.coordinate_click.assert_not_called()
+ assert self._has_success(run.results)
+
+ @pytest.mark.asyncio
+ async def test_untracked_overlay_navigation_after_dispatch_counts_as_success(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ run = await self._run_chain_click(
+ monkeypatch,
+ blocked=True,
+ tag=InteractiveElement.BUTTON,
+ href=None,
+ with_incremental=True,
+ page_responded=None,
+ js_click_url="https://portal.example.com/submitted",
+ )
+
+ run.click_in_javascript.assert_awaited_once()
+ run.coordinate_click.assert_not_called()
+ assert self._has_success(run.results)
+
+ @pytest.mark.asyncio
+ async def test_explicit_coordinate_click_still_coordinate_clicks_under_overlay(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ # An action with explicit x/y is a deliberate coordinate click; keep it.
+ run = await self._run_chain_click(
+ monkeypatch,
+ blocked=True,
+ tag=InteractiveElement.BUTTON,
+ href=None,
+ with_incremental=True,
+ action_x=100,
+ action_y=200,
+ )
+
+ run.coordinate_click.assert_awaited_once()
+ run.click_in_javascript.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_parent_container_blocker_still_retargets(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ run = await self._run_chain_click(
+ monkeypatch,
+ blocked=True,
+ tag=InteractiveElement.BUTTON,
+ href=None,
+ with_incremental=True,
+ blocking_is_parent=True,
+ blocking_is_sibling=False,
+ )
+
+ assert run.blocking_click is not None
+ run.blocking_click.assert_awaited_once()
+ run.click_in_javascript.assert_not_called()
+ assert self._has_success(run.results)
+
+ @pytest.mark.asyncio
+ async def test_sibling_overlay_with_observer_not_retargeted(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ run = await self._run_chain_click(
+ monkeypatch,
+ blocked=True,
+ tag=InteractiveElement.BUTTON,
+ href=None,
+ with_incremental=True,
+ page_responded=True,
+ blocking_is_parent=False,
+ blocking_is_sibling=True,
+ )
+
+ assert run.blocking_click is not None
+ run.blocking_click.assert_not_called()
+ run.click_in_javascript.assert_awaited_once()
+ assert self._has_success(run.results)
+
+ @pytest.mark.parametrize("input_type", ["checkbox", "radio"])
+ @pytest.mark.asyncio
+ async def test_sibling_form_control_with_observer_still_retargets(
+ self, monkeypatch: pytest.MonkeyPatch, input_type: str
+ ) -> None:
+ run = await self._run_chain_click(
+ monkeypatch,
+ blocked=True,
+ tag=InteractiveElement.INPUT,
+ input_type=input_type,
+ href=None,
+ with_incremental=True,
+ blocking_is_parent=False,
+ blocking_is_sibling=True,
+ blocking_rect={"x": 0, "y": 0, "width": 24, "height": 24},
+ )
+
+ assert run.blocking_click is not None
+ run.blocking_click.assert_awaited_once()
+ run.click_in_javascript.assert_not_called()
+ assert self._has_success(run.results)
+
+ @pytest.mark.asyncio
+ async def test_small_custom_sibling_with_observer_still_retargets(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ run = await self._run_chain_click(
+ monkeypatch,
+ blocked=True,
+ tag="div",
+ href=None,
+ with_incremental=True,
+ blocking_is_parent=False,
+ blocking_is_sibling=True,
+ blocking_rect={"x": 0, "y": 0, "width": 24, "height": 24},
+ )
+
+ assert run.blocking_click is not None
+ run.blocking_click.assert_awaited_once()
+ run.click_in_javascript.assert_not_called()
+ assert self._has_success(run.results)
+
+ @pytest.mark.asyncio
+ async def test_full_cover_sibling_overlay_over_checkbox_is_not_retargeted(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ run = await self._run_chain_click(
+ monkeypatch,
+ blocked=True,
+ tag=InteractiveElement.INPUT,
+ input_type="checkbox",
+ href=None,
+ with_incremental=True,
+ page_responded=True,
+ blocking_is_parent=False,
+ blocking_is_sibling=True,
+ blocking_rect={"x": 0, "y": 0, "width": 1920, "height": 1080},
+ )
+
+ assert run.blocking_click is not None
+ run.blocking_click.assert_not_called()
+ run.click_in_javascript.assert_awaited_once()
+ assert self._has_success(run.results)
+
+ @pytest.mark.asyncio
+ async def test_small_semantic_modal_sibling_over_checkbox_is_not_retargeted(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ run = await self._run_chain_click(
+ monkeypatch,
+ blocked=True,
+ tag=InteractiveElement.INPUT,
+ input_type="checkbox",
+ href=None,
+ with_incremental=True,
+ page_responded=True,
+ blocking_is_parent=False,
+ blocking_is_sibling=True,
+ blocking_rect={"x": 0, "y": 0, "width": 600, "height": 400},
+ blocking_inside_modal=True,
+ )
+
+ assert run.blocking_click is not None
+ run.blocking_click.assert_not_called()
+ run.click_in_javascript.assert_awaited_once()
+ assert self._has_success(run.results)
+
+ @pytest.mark.asyncio
+ async def test_iframe_covering_sibling_overlay_over_checkbox_is_not_retargeted(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ run = await self._run_chain_click(
+ monkeypatch,
+ blocked=True,
+ tag=InteractiveElement.INPUT,
+ input_type="checkbox",
+ href=None,
+ with_incremental=True,
+ page_responded=True,
+ blocking_is_parent=False,
+ blocking_is_sibling=True,
+ blocking_rect={"x": 0, "y": 0, "width": 800, "height": 500},
+ blocking_viewport=(800, 500),
+ )
+
+ assert run.blocking_click is not None
+ run.blocking_click.assert_not_called()
+ run.click_in_javascript.assert_awaited_once()
+ assert self._has_success(run.results)
+
+ @pytest.mark.asyncio
+ async def test_sibling_blocker_without_observer_still_retargets(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ # No observer = the custom checkbox/radio retarget callers; a sibling
+ # decoration (e.g. a styled span over the real control) must still be
+ # clicked to activate the intended control.
+ run = await self._run_chain_click(
+ monkeypatch,
+ blocked=True,
+ tag=InteractiveElement.BUTTON,
+ href=None,
+ with_incremental=False,
+ blocking_is_parent=False,
+ blocking_is_sibling=True,
+ )
+
+ assert run.blocking_click is not None
+ run.blocking_click.assert_awaited_once()
+ assert self._has_success(run.results)
+
+
def test_dom_module_exports_try_navigate_via_href() -> None:
"""Sanity check that the helper is a method on ``SkyvernElement``."""
assert hasattr(dom_module.SkyvernElement, "try_navigate_via_href")
diff --git a/tests/unit/test_exception_messages.py b/tests/unit/test_exception_messages.py
index b07a870ed..8986bb847 100644
--- a/tests/unit/test_exception_messages.py
+++ b/tests/unit/test_exception_messages.py
@@ -38,6 +38,59 @@ def test_unknown_error_while_creating_browser_context_strips_call_log() -> None:
assert "[email protected]" in message
+def test_unknown_error_omits_browser_type_from_user_facing_message() -> None:
+ # The browser_type label can be a remote-browser vendor identity; keep it on the
+ # exception for structured logs but never surface it in the user-facing message.
+ sentinel = "zzz-secret-browser-label"
+ error = UnknownErrorWhileCreatingBrowserContext(sentinel, RuntimeError("setup failed"))
+ message = str(error)
+
+ assert sentinel not in message
+ assert error.browser_type == sentinel
+ assert "Failed to create browser context" in message
+
+
+def test_unknown_error_renders_redacted_inner_type_name() -> None:
+ # A SkyvernException may hide a sensitive class name behind user_facing_type_name; the
+ # wrapper must render that neutral token, never the real class name.
+ class _SecretVendorRateLimitError(SkyvernException):
+ @property
+ def user_facing_type_name(self) -> str:
+ return "RemoteBrowserRateLimitError"
+
+ error = UnknownErrorWhileCreatingBrowserContext("dynamic-browser", _SecretVendorRateLimitError("slow down"))
+ message = str(error)
+
+ assert "SecretVendor" not in message
+ assert "RemoteBrowserRateLimitError" in message
+
+
+def test_unknown_error_preserves_real_inner_type_name_by_default() -> None:
+ # Failure classification keys off the parenthesized inner class name in the message, so a
+ # plain SkyvernException (no redaction) must still surface its real class name.
+ class _PlainProxyError(SkyvernException):
+ pass
+
+ error = UnknownErrorWhileCreatingBrowserContext("dynamic-browser", _PlainProxyError("no proxy available"))
+ assert "_PlainProxyError" in str(error)
+
+
+def test_unknown_error_redacts_raw_cdp_endpoint_url() -> None:
+ # A raw connect_over_cdp failure echoes the ws/wss endpoint, which can carry the vendor
+ # host, a session-bearing query, or embedded credentials — none may reach the user.
+ inner_exception = Exception(
+ "browserType.connectOverCDP: WebSocket error: "
+ "wss://user:[email protected]/session/tok-9f3a?apiKey=SEKRET connect ECONNREFUSED"
+ )
+ message = str(UnknownErrorWhileCreatingBrowserContext("dynamic-browser", inner_exception))
+
+ assert "wss://" not in message
+ assert "tok-9f3a" not in message
+ assert "SEKRET" not in message
+ assert "remote.example.internal" not in message
+ assert "[remote browser endpoint]" in message
+
+
def test_captcha_not_solved_in_time_is_captcha_solve_error() -> None:
# The action handler catches CaptchaSolveError before its generic arm; this
# subclass relationship is what routes captcha-solve failures to the handled
diff --git a/tests/unit/test_multi_field_totp.py b/tests/unit/test_multi_field_totp.py
index b357d24ec..6bea24120 100644
--- a/tests/unit/test_multi_field_totp.py
+++ b/tests/unit/test_multi_field_totp.py
@@ -7,8 +7,16 @@ import pytest
from skyvern.core.script_generations.generate_script import _annotate_multi_field_totp_sequence
from skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPage
+from skyvern.exceptions import NoTOTPSecretFound
+from skyvern.forge.sdk.services.credentials import OnePasswordConstants
from skyvern.webeye.actions.action_types import ActionType
-from skyvern.webeye.actions.handler import _handle_multi_field_totp_sequence
+from skyvern.webeye.actions.actions import InputTextAction
+from skyvern.webeye.actions.handler import (
+ _handle_multi_field_totp_sequence,
+ generate_totp_value,
+ handle_input_text_action,
+)
+from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess
class _FakeTotp:
@@ -25,6 +33,297 @@ class _FakeTotp:
return "current-code"
+class _FakeWorkflowRunContext:
+ placeholder = "placeholder_AbCd_totp"
+
+ def __init__(self, totp_secret: str | None) -> None:
+ self.secrets = {
+ self.placeholder: OnePasswordConstants.TOTP,
+ self.totp_secret_value_key(self.placeholder): totp_secret,
+ }
+
+ def get_original_secret_value_or_none(self, key: str) -> str | None:
+ return self.secrets.get(key)
+
+ def totp_secret_value_key(self, totp_secret_id: str) -> str:
+ return f"{totp_secret_id}_value"
+
+ def find_credential_parameter_key_for_secret(self, _secret_id: str) -> None:
+ return None
+
+ def find_embedded_placeholder_tokens(self, text: str) -> list[str]:
+ return [text]
+
+
+def _patch_workflow_context(monkeypatch: pytest.MonkeyPatch, workflow_context: _FakeWorkflowRunContext) -> None:
+ from skyvern.webeye.actions import handler
+
+ monkeypatch.setattr(
+ handler,
+ "app",
+ SimpleNamespace(
+ WORKFLOW_CONTEXT_MANAGER=SimpleNamespace(
+ get_workflow_run_context=lambda _workflow_run_id: workflow_context,
+ )
+ ),
+ )
+
+
+def test_generate_totp_value_raises_when_secret_is_missing(monkeypatch: pytest.MonkeyPatch) -> None:
+ workflow_context = _FakeWorkflowRunContext(totp_secret="")
+ _patch_workflow_context(monkeypatch, workflow_context)
+
+ with pytest.raises(NoTOTPSecretFound, match="No TOTP secret found"):
+ generate_totp_value("wr_test", workflow_context.placeholder)
+
+
+def test_generate_totp_value_maps_invalid_secret_to_typed_failure(monkeypatch: pytest.MonkeyPatch) -> None:
+ workflow_context = _FakeWorkflowRunContext(totp_secret="NOT-VALID!")
+ _patch_workflow_context(monkeypatch, workflow_context)
+
+ with pytest.raises(NoTOTPSecretFound, match="No TOTP secret found"):
+ generate_totp_value("wr_test", workflow_context.placeholder)
+
+
+async def test_cua_input_does_not_type_unresolved_totp_placeholder(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.webeye.actions import handler
+
+ workflow_context = _FakeWorkflowRunContext(totp_secret="")
+ _patch_workflow_context(monkeypatch, workflow_context)
+ type_text = AsyncMock()
+ monkeypatch.setattr(handler.EventStrategyFactory, "type_text", type_text)
+
+ results = await handle_input_text_action(
+ InputTextAction(element_id="", text=workflow_context.placeholder),
+ MagicMock(),
+ MagicMock(),
+ SimpleNamespace(workflow_run_id="wr_test"),
+ MagicMock(),
+ )
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionFailure)
+ assert results[0].stop_execution_on_failure is True
+ assert results[0].exception_type == "NoTOTPSecretFound"
+ assert workflow_context.placeholder not in (results[0].exception_message or "")
+ type_text.assert_not_awaited()
+
+
+async def test_cua_input_types_generated_totp_instead_of_placeholder(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.webeye.actions import handler
+
+ workflow_context = _FakeWorkflowRunContext(totp_secret="JBSWY3DPEHPK3PXP")
+ _patch_workflow_context(monkeypatch, workflow_context)
+ type_text = AsyncMock()
+ monkeypatch.setattr(handler.EventStrategyFactory, "type_text", type_text)
+ monkeypatch.setattr(handler, "generate_totp_code", MagicMock(return_value="654321"))
+ page = MagicMock()
+
+ results = await handle_input_text_action(
+ InputTextAction(element_id="", text=workflow_context.placeholder),
+ page,
+ MagicMock(),
+ SimpleNamespace(workflow_run_id="wr_test"),
+ MagicMock(),
+ )
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionSuccess)
+ type_text.assert_awaited_once_with(page, None, "654321")
+
+
+async def test_dom_input_does_not_reach_browser_when_totp_secret_is_missing(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.webeye.actions import handler
+
+ workflow_context = _FakeWorkflowRunContext(totp_secret="")
+ _patch_workflow_context(monkeypatch, workflow_context)
+ dom_type = MagicMock()
+ monkeypatch.setattr(handler, "DomUtil", dom_type)
+
+ results = await handle_input_text_action(
+ InputTextAction(element_id="otp", text=workflow_context.placeholder),
+ MagicMock(),
+ MagicMock(),
+ SimpleNamespace(workflow_run_id="wr_test"),
+ MagicMock(),
+ )
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionFailure)
+ assert results[0].exception_type == "NoTOTPSecretFound"
+ dom_type.assert_not_called()
+
+
+async def test_dom_input_rejects_embedded_totp_provider_marker(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.webeye.actions import handler
+
+ monkeypatch.setattr(
+ handler,
+ "get_actual_value_of_parameter_if_secret_with_task",
+ MagicMock(return_value="code=OP_TOTP user=resolved-secret"),
+ )
+ dom_type = MagicMock()
+ monkeypatch.setattr(handler, "DomUtil", dom_type)
+
+ results = await handle_input_text_action(
+ InputTextAction(element_id="otp", text="composite credential value"),
+ MagicMock(),
+ MagicMock(),
+ SimpleNamespace(workflow_run_id="wr_test"),
+ MagicMock(),
+ )
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionFailure)
+ assert results[0].exception_type == "NoTOTPSecretFound"
+ dom_type.assert_not_called()
+
+
+async def test_dom_input_generates_totp_immediately_before_write(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.webeye.actions import handler
+
+ workflow_context = _FakeWorkflowRunContext(totp_secret="JBSWY3DPEHPK3PXP")
+ _patch_workflow_context(monkeypatch, workflow_context)
+ events: list[str] = []
+ monkeypatch.setattr(
+ handler,
+ "generate_totp_code",
+ MagicMock(side_effect=lambda _secret: events.append("generate") or "654321"),
+ )
+
+ locator = MagicMock()
+ locator.focus = AsyncMock()
+ element = MagicMock()
+ element.get_id.return_value = "otp"
+ element.get_frame.return_value = MagicMock()
+ element.get_tag_name.return_value = "input"
+ element.get_locator.return_value = locator
+ element.get_selectable = AsyncMock(return_value=False)
+ element.is_disabled = AsyncMock(return_value=False)
+ element.supports_text_input = AsyncMock(return_value=True)
+ element.has_hidden_attr = AsyncMock(return_value=False)
+ element.is_readonly = AsyncMock(return_value=False)
+ element.get_attr = AsyncMock(return_value=None)
+ element.is_spinbtn_input = AsyncMock(return_value=False)
+ element.is_editable = AsyncMock(return_value=True)
+ element.find_blocking_element = AsyncMock(
+ side_effect=lambda **_kwargs: events.append("blocking-check") or (None, False)
+ )
+ element.input = AsyncMock(side_effect=lambda _text: events.append("input"))
+
+ dom = MagicMock()
+ dom.get_skyvern_element_by_id = AsyncMock(return_value=element)
+ monkeypatch.setattr(handler, "DomUtil", MagicMock(return_value=dom))
+ frame = MagicMock()
+ frame.safe_wait_for_animation_end = AsyncMock(side_effect=lambda **_kwargs: events.append("animation-wait"))
+ monkeypatch.setattr(handler.SkyvernFrame, "create_instance", AsyncMock(return_value=frame))
+ monkeypatch.setattr(handler, "IncrementalScrapePage", MagicMock())
+ monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value=""))
+ monkeypatch.setattr(handler, "_get_input_or_select_context", AsyncMock(return_value=None))
+ scraped_page = SimpleNamespace(id_to_element_dict={"otp": {"tagName": "input"}})
+
+ results = await handle_input_text_action(
+ InputTextAction(element_id="otp", text=workflow_context.placeholder),
+ MagicMock(),
+ scraped_page,
+ SimpleNamespace(workflow_run_id="wr_test", task_id="task_test"),
+ SimpleNamespace(step_id="step_test"),
+ )
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionSuccess)
+ assert events == ["animation-wait", "blocking-check", "generate", "input"]
+ element.input.assert_awaited_once_with("654321")
+
+
+async def test_cua_input_keeps_ordinary_text_unchanged(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.webeye.actions import handler
+
+ type_text = AsyncMock()
+ monkeypatch.setattr(handler.EventStrategyFactory, "type_text", type_text)
+ page = MagicMock()
+
+ results = await handle_input_text_action(
+ InputTextAction(element_id="", text="ordinary text"),
+ page,
+ MagicMock(),
+ SimpleNamespace(workflow_run_id=None),
+ MagicMock(),
+ )
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionSuccess)
+ type_text.assert_awaited_once_with(page, None, "ordinary text")
+
+
+async def test_cua_input_rejects_raw_totp_placeholder_without_workflow_context(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skyvern.webeye.actions import handler
+
+ type_text = AsyncMock()
+ monkeypatch.setattr(handler.EventStrategyFactory, "type_text", type_text)
+
+ results = await handle_input_text_action(
+ InputTextAction(element_id="", text="placeholder_AbCd_totp"),
+ MagicMock(),
+ MagicMock(),
+ SimpleNamespace(workflow_run_id=None),
+ MagicMock(),
+ )
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionFailure)
+ assert results[0].exception_type == "NoTOTPSecretFound"
+ type_text.assert_not_awaited()
+
+
+async def test_selectable_input_receives_generated_totp_instead_of_marker(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.webeye.actions import handler
+
+ workflow_context = _FakeWorkflowRunContext(totp_secret="JBSWY3DPEHPK3PXP")
+ _patch_workflow_context(monkeypatch, workflow_context)
+ monkeypatch.setattr(handler, "generate_totp_code", MagicMock(return_value="654321"))
+
+ element = MagicMock()
+ element.get_id.return_value = "otp"
+ element.get_frame.return_value = MagicMock()
+ element.get_selectable = AsyncMock(return_value=True)
+ element.is_disabled = AsyncMock(return_value=False)
+ dom = MagicMock()
+ dom.get_skyvern_element_by_id = AsyncMock(return_value=element)
+ dom_type = MagicMock(return_value=dom)
+ monkeypatch.setattr(handler, "DomUtil", dom_type)
+ monkeypatch.setattr(handler.SkyvernFrame, "create_instance", AsyncMock(return_value=MagicMock()))
+ monkeypatch.setattr(handler, "IncrementalScrapePage", MagicMock())
+ monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value=""))
+ select_mock = AsyncMock(return_value=[ActionSuccess()])
+ monkeypatch.setattr(handler, "handle_select_option_action", select_mock)
+ scraped_page = SimpleNamespace(id_to_element_dict={"otp": {"tagName": "input"}})
+
+ results = await handle_input_text_action(
+ InputTextAction(element_id="otp", text=workflow_context.placeholder),
+ MagicMock(),
+ scraped_page,
+ SimpleNamespace(workflow_run_id="wr_test"),
+ MagicMock(),
+ )
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionSuccess)
+ select_action = select_mock.await_args.args[0]
+ assert select_action.option.label == "654321"
+ assert select_action.option.label != workflow_context.placeholder
+
+
class TestAnnotateMultiFieldTotpSequence:
"""Tests for _annotate_multi_field_totp_sequence function."""
diff --git a/tests/unit/test_parse_totp_secret.py b/tests/unit/test_parse_totp_secret.py
index e9d8bf5b9..31d329e40 100644
--- a/tests/unit/test_parse_totp_secret.py
+++ b/tests/unit/test_parse_totp_secret.py
@@ -1,14 +1,27 @@
+import hashlib
from urllib.parse import quote
import pyotp
+import pytest
-from skyvern.forge.sdk.services.credentials import generate_totp_code, normalize_totp_config, parse_totp_secret
+from skyvern.forge.sdk.services.credentials import (
+ generate_totp_code,
+ is_unresolved_totp_value,
+ normalize_totp_config,
+ parse_totp_config,
+ parse_totp_secret,
+)
def test_empty_string_returns_empty() -> None:
assert parse_totp_secret("") == ""
[email protected]("marker", ["OP_TOTP", "BW_TOTP", "AZ_TOTP"])
+def test_unresolved_totp_value_detects_embedded_provider_marker(marker: str) -> None:
+ assert is_unresolved_totp_value(f"code={marker} user=resolved-secret")
+
+
def test_valid_base32_secret() -> None:
secret = "JBSWY3DPEHPK3PXP"
assert parse_totp_secret(secret) == secret
@@ -47,6 +60,95 @@ def test_generate_totp_code_uses_otpauth_uri_params() -> None:
assert len(generate_totp_code(uri, for_time=0)) == 8
+def test_otpauth_uri_with_mismatched_issuer_preserves_generation_config() -> None:
+ uri = (
+ "otpauth://totp/LabelIssuer:[email protected]"
+ "?secret=JBSWY3DPEHPK3PXP&issuer=QueryIssuer&algorithm=SHA256&digits=8&period=60"
+ )
+ expected = pyotp.TOTP(
+ "JBSWY3DPEHPK3PXP",
+ digest=hashlib.sha256,
+ digits=8,
+ interval=60,
+ ).at(0)
+
+ assert parse_totp_secret(uri) == "JBSWY3DPEHPK3PXP"
+ assert normalize_totp_config(uri) == uri
+ assert generate_totp_code(uri, for_time=0) == expected
+
+
+def test_url_encoded_otpauth_uri_with_mismatched_issuer_is_accepted() -> None:
+ uri = (
+ "otpauth://totp/LabelIssuer:[email protected]"
+ "?secret=JBSWY3DPEHPK3PXP&issuer=QueryIssuer&algorithm=SHA512&digits=7&period=45"
+ )
+ expected = pyotp.TOTP(
+ "JBSWY3DPEHPK3PXP",
+ digest=hashlib.sha512,
+ digits=7,
+ interval=45,
+ ).at(0)
+
+ encoded_uri = quote(uri, safe="")
+
+ assert parse_totp_secret(encoded_uri) == "JBSWY3DPEHPK3PXP"
+ assert normalize_totp_config(encoded_uri) == uri
+ assert generate_totp_code(encoded_uri, for_time=0) == expected
+
+
+ "query",
+ [
+ "issuer=QueryIssuer",
+ "secret=NOT_VALID!&issuer=QueryIssuer",
+ "secret=JBSWY3DPEHPK3PXP&issuer=QueryIssuer&algorithm=NOPE",
+ "secret=JBSWY3DPEHPK3PXP&issuer=QueryIssuer&digits=9",
+ "secret=JBSWY3DPEHPK3PXP&issuer=QueryIssuer&period=invalid",
+ "secret=JBSWY3DPEHPK3PXP&issuer=QueryIssuer&period=0",
+ "secret=JBSWY3DPEHPK3PXP&issuer=QueryIssuer&period=-30",
+ "secret=JBSWY3DPEHPK3PXP&issuer=QueryIssuer&unknown=value",
+ "secret=JBSWY3DPEHPK3PXP&issuer=FirstIssuer&issuer=SecondIssuer",
+ "secret=JBSWY3DPEHPK3PXP&issuer=QueryIssuer&issuer=QueryIssuer",
+ "secret=JBSWY3DPEHPK3PXP&issuer=QueryIssuer&iss%75er=QueryIssuer",
+ ],
+)
+def test_issuer_mismatch_does_not_bypass_invalid_config(query: str) -> None:
+ uri = f"otpauth://totp/LabelIssuer:[email protected]?{query}"
+
+ assert parse_totp_secret(uri) == ""
+ assert normalize_totp_config(uri) == ""
+
+
[email protected]("period", ["0", "-30"])
+def test_matching_issuer_uri_rejects_nonpositive_period(period: str) -> None:
+ uri = f"otpauth://totp/Example:[email protected]?secret=JBSWY3DPEHPK3PXP&issuer=Example&period={period}"
+
+ assert parse_totp_secret(uri) == ""
+ assert normalize_totp_config(uri) == ""
+
+
[email protected]("second_key", ["issuer", "iss%75er"])
+def test_matching_issuer_uri_rejects_duplicate_issuer(second_key: str) -> None:
+ uri = f"otpauth://totp/Example:[email protected]?secret=JBSWY3DPEHPK3PXP&issuer=Example&{second_key}=Example"
+
+ assert parse_totp_secret(uri) == ""
+ assert normalize_totp_config(uri) == ""
+
+
+def test_hotp_uri_is_not_coerced_into_totp() -> None:
+ uri = "otpauth://hotp/Example:[email protected]?secret=JBSWY3DPEHPK3PXP&issuer=Example&counter=0"
+
+ assert parse_totp_secret(uri) == ""
+ assert normalize_totp_config(uri) == ""
+
+
[email protected]("value", ["otpauth://[", quote("otpauth://[", safe="")])
+def test_malformed_otpauth_authority_is_rejected_without_raising(value: str) -> None:
+ assert parse_totp_secret(value) == ""
+ assert parse_totp_config(value) is None
+ assert normalize_totp_config(value) == ""
+
+
def test_normalize_totp_config_preserves_url_encoded_otpauth_uri() -> None:
uri = (
"otpauth://totp/Example:[email protected]"
diff --git a/tests/unit/test_pdf_fill_block.py b/tests/unit/test_pdf_fill_block.py
index a638e067e..2d9bbd7b1 100644
--- a/tests/unit/test_pdf_fill_block.py
+++ b/tests/unit/test_pdf_fill_block.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+import io
+import re
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
@@ -52,6 +54,77 @@ def _make_block(**overrides: Any) -> PdfFillBlock:
return PdfFillBlock(**data)
+def _flat_text_ops(pdf_bytes: bytes) -> list[tuple[str, float, float, float, str]]:
+ reader = PdfReader(io.BytesIO(pdf_bytes))
+ contents = reader.pages[0].get_contents()
+ assert contents is not None
+ data = contents.get_data().decode("latin-1")
+ pattern = re.compile(
+ r"BT\s+/SkyvernHelv(?:-\d+)?\s+(?P<font>\d+(?:\.\d+)?)\s+Tf\s+0\s+0\s+0\s+rg\s+"
+ r"(?P<x>-?\d+(?:\.\d+)?)\s+(?P<y>-?\d+(?:\.\d+)?)\s+Td\s+"
+ r"\((?P<value>(?:\\.|[^\\)])*)\)\s+Tj\s+ET"
+ )
+ operations: list[tuple[str, float, float, float, str]] = []
+ for match in pattern.finditer(data):
+ value = re.sub(
+ r"\\([0-7]{3}|.)",
+ lambda escaped: chr(int(escaped.group(1), 8)) if escaped.group(1).isdigit() else escaped.group(1),
+ match.group("value"),
+ )
+ font = match.group("font")
+ x = match.group("x")
+ y = match.group("y")
+ operation = f"BT /SkyvernHelv {font} Tf 0 0 0 rg {float(x):.1f} {float(y):.1f} Td ({value}) Tj ET"
+ operations.append((operation, float(x), float(y), float(font), value))
+ return operations
+
+
+def _flat_collision_anchors() -> list[FlatPdfAnchor]:
+ dimensions = {"page_width_px": 1275, "page_height_px": 1651}
+ return [
+ FlatPdfAnchor(anchor_id=0, page_index=0, text="a. Name:", x0=88, x1=218, top=309, bottom=322, **dimensions),
+ FlatPdfAnchor(
+ anchor_id=1, page_index=0, text="b. Date of birth", x0=494, x1=672, top=309, bottom=322, **dimensions
+ ),
+ FlatPdfAnchor(
+ anchor_id=2, page_index=0, text="c. Member ID #:", x0=761, x1=952, top=309, bottom=325, **dimensions
+ ),
+ FlatPdfAnchor(
+ anchor_id=3,
+ page_index=0,
+ text="d. Street address:",
+ x0=88,
+ x1=285,
+ top=371,
+ bottom=384,
+ **dimensions,
+ ),
+ FlatPdfAnchor(anchor_id=4, page_index=0, text="e. City:", x0=106, x1=138, top=395, bottom=411, **dimensions),
+ FlatPdfAnchor(anchor_id=5, page_index=0, text="f. State:", x0=493, x1=594, top=394, bottom=408, **dimensions),
+ FlatPdfAnchor(
+ anchor_id=6, page_index=0, text="g. Zip code:", x0=744, x1=829, top=395, bottom=411, **dimensions
+ ),
+ ]
+
+
+def _tight_flat_anchors() -> list[FlatPdfAnchor]:
+ dimensions = {"page_width_px": 1275, "page_height_px": 1651}
+ return [
+ FlatPdfAnchor(anchor_id=0, page_index=0, text="Ref #:", x0=100, x1=160, top=100, bottom=113, **dimensions),
+ FlatPdfAnchor(anchor_id=1, page_index=0, text="Next:", x0=300, x1=360, top=100, bottom=113, **dimensions),
+ FlatPdfAnchor(anchor_id=2, page_index=0, text="Below:", x0=100, x1=200, top=120, bottom=133, **dimensions),
+ ]
+
+
+def _write_blank_flat_pdf(tmp_path: Path) -> Path:
+ source_path = tmp_path / "source.pdf"
+ writer = PdfWriter()
+ writer.add_blank_page(width=612, height=792)
+ with source_path.open("wb") as file:
+ writer.write(file)
+ return source_path
+
+
def _install_context(monkeypatch: pytest.MonkeyPatch, run_id: str = "run_pdf_fill") -> WorkflowRunContext:
context = WorkflowRunContext(
workflow_title="PDF Fill Test",
@@ -190,6 +263,7 @@ async def test_skyvern_engine_fills_pdf_with_llm_mapping(monkeypatch: pytest.Mon
assert output["file_size"] == output_path.stat().st_size
assert output["fields"] == {"name": "Jane", "subscribe": "/Yes"}
assert output["skipped_fields"] == []
+ assert output["overflowed_placements"] == []
output_reader = PdfReader(str(output_path))
fields = output_reader.get_fields()
@@ -424,6 +498,7 @@ async def test_flat_pdf_overlay_fills_with_ocr_anchors(monkeypatch: pytest.Monke
assert output["fill_mode"] == "flat_overlay"
assert output["fields"] == {"Enrollee name:": "Luis Ortiz"}
assert output["skipped_fields"] == []
+ assert output["overflowed_placements"] == []
import pdfplumber
@@ -456,6 +531,453 @@ async def test_flat_pdf_over_page_limit_fails(monkeypatch: pytest.MonkeyPatch, t
assert "supports up to 25 pages" in (result.failure_reason or "")
+async def test_flat_overlay_right_value_moves_below_when_line_is_full(tmp_path: Path) -> None:
+ source_path = _write_blank_flat_pdf(tmp_path)
+ anchors = _flat_collision_anchors()
+ dob_label = anchors[1]
+ member_label = anchors[2]
+ placements = [
+ FlatPlacement(anchor=dob_label, value="01/01/2000", position="right"),
+ FlatPlacement(anchor=member_label, value="10000001", position="right"),
+ ]
+ block = _make_block(file_url=str(source_path))
+ result_bytes = await block._fill_flat_overlay(
+ PdfReader(str(source_path)), placements, anchors, tmp_path / "filled.pdf"
+ )
+
+ PdfReader(io.BytesIO(result_bytes))
+ operations = _flat_text_ops(result_bytes)
+ assert len(operations) == 2
+ operations_by_value = {operation[4]: operation for operation in operations}
+ date_op = operations_by_value["01/01/2000"]
+ member_op = operations_by_value["10000001"]
+ scale_x = 612 / 1275
+ scale_y = 792 / 1651
+ expected_date_x = 494 * scale_x + 8
+ expected_date_y = 792 - 322 * scale_y - 11 - 3
+ assert date_op[1] == pytest.approx(expected_date_x, abs=0.05)
+ assert date_op[2] == pytest.approx(expected_date_y, abs=0.05)
+ assert date_op[3] == 11
+ assert member_op[1] == pytest.approx(952 * scale_x + 6, abs=0.05)
+ assert member_op[2] == pytest.approx(792 - 325 * scale_y - 1, abs=0.05)
+ assert member_op[3] == 11
+ assert date_op[1] + pdf_fill_block._flat_text_width_pt("01/01/2000", 11) <= member_label.x0 * scale_x
+
+
+async def test_flat_overlay_below_value_moves_right_when_next_row_occupied(tmp_path: Path) -> None:
+ source_path = _write_blank_flat_pdf(tmp_path)
+ anchors = _flat_collision_anchors()
+ addr_label = anchors[3]
+ city_label = anchors[4]
+ placements = [
+ FlatPlacement(anchor=addr_label, value="350 Test Street", position="below"),
+ FlatPlacement(anchor=city_label, value="Detroit", position="right"),
+ ]
+ block = _make_block(file_url=str(source_path))
+ result_bytes = await block._fill_flat_overlay(
+ PdfReader(str(source_path)), placements, anchors, tmp_path / "filled.pdf"
+ )
+
+ PdfReader(io.BytesIO(result_bytes))
+ operations = _flat_text_ops(result_bytes)
+ assert len(operations) == 2
+ operations_by_value = {operation[4]: operation for operation in operations}
+ address_op = operations_by_value["350 Test Street"]
+ city_op = operations_by_value["Detroit"]
+ scale_x = 612 / 1275
+ scale_y = 792 / 1651
+ expected_address_x = 285 * scale_x + 6
+ expected_address_y = 792 - 384 * scale_y - 1
+ assert address_op[1] == pytest.approx(expected_address_x, abs=0.05)
+ assert address_op[2] == pytest.approx(expected_address_y, abs=0.05)
+ assert address_op[3] == 11
+ assert city_op[1] == pytest.approx(138 * scale_x + 6, abs=0.05)
+ assert city_op[2] == pytest.approx(792 - 411 * scale_y - 1, abs=0.05)
+ assert city_op[3] == 11
+
+ address_rect = (
+ address_op[1],
+ address_op[2],
+ address_op[1] + pdf_fill_block._flat_text_width_pt("350 Test Street", 11),
+ address_op[2] + 11,
+ )
+ city_rect = (
+ city_op[1],
+ city_op[2],
+ city_op[1] + pdf_fill_block._flat_text_width_pt("Detroit", 11),
+ city_op[2] + 11,
+ )
+ assert not (
+ address_rect[0] < city_rect[2]
+ and address_rect[2] > city_rect[0]
+ and address_rect[1] < city_rect[3]
+ and address_rect[3] > city_rect[1]
+ )
+ city_label_rect = (
+ city_label.x0 * scale_x,
+ 792 - city_label.bottom * scale_y,
+ city_label.x1 * scale_x,
+ 792 - city_label.top * scale_y,
+ )
+ assert not (
+ address_rect[0] < city_label_rect[2]
+ and address_rect[2] > city_label_rect[0]
+ and address_rect[1] < city_label_rect[3]
+ and address_rect[3] > city_label_rect[1]
+ )
+
+
+async def test_flat_overlay_below_rejected_when_it_would_fall_off_page(tmp_path: Path) -> None:
+ source_path = _write_blank_flat_pdf(tmp_path)
+ dimensions = {"page_width_px": 1275, "page_height_px": 1651}
+ anchor = FlatPdfAnchor(
+ anchor_id=0, page_index=0, text="Bottom:", x0=88, x1=200, top=1600, bottom=1632, **dimensions
+ )
+ obstacle = FlatPdfAnchor(anchor_id=1, page_index=0, text="X:", x0=210, x1=280, top=1600, bottom=1632, **dimensions)
+ block = _make_block(file_url=str(source_path))
+ result_bytes = await block._fill_flat_overlay(
+ PdfReader(str(source_path)),
+ [FlatPlacement(anchor=anchor, value="W" * 100, position="right")],
+ [anchor, obstacle],
+ tmp_path / "filled.pdf",
+ )
+
+ operation = _flat_text_ops(result_bytes)[0]
+ scale_x = 612 / 1275
+ scale_y = 792 / 1651
+ right_x = anchor.x1 * scale_x + 6
+ below_x = anchor.x0 * scale_x + 8
+ below_y = (
+ 792 - anchor.bottom * scale_y - pdf_fill_block.FLAT_FILL_MIN_FONT_SIZE - pdf_fill_block.FLAT_FILL_FONT_SIZE_GAP
+ )
+ assert operation[1] == pytest.approx(right_x, abs=0.05)
+ assert operation[1] != pytest.approx(below_x, abs=0.05)
+ assert 0 <= operation[2] <= 792
+ assert below_y < 0
+
+
+async def test_flat_overlay_right_avoids_obstacle_touching_band_edge(tmp_path: Path) -> None:
+ source_path = _write_blank_flat_pdf(tmp_path)
+ dimensions = {"page_width_px": 1275, "page_height_px": 1651}
+ anchor = FlatPdfAnchor(anchor_id=0, page_index=0, text="Ref:", x0=88, x1=200, top=300, bottom=313, **dimensions)
+ obstacle = FlatPdfAnchor(anchor_id=1, page_index=0, text="X:", x0=250, x1=320, top=286, bottom=300, **dimensions)
+ value = "WWWWWWWWWW"
+ block = _make_block(file_url=str(source_path))
+ result_bytes = await block._fill_flat_overlay(
+ PdfReader(str(source_path)),
+ [FlatPlacement(anchor=anchor, value=value, position="right")],
+ [anchor, obstacle],
+ tmp_path / "filled.pdf",
+ )
+
+ operation = _flat_text_ops(result_bytes)[0]
+ scale_x = 612 / 1275
+ right_x = anchor.x1 * scale_x + 6
+ below_x = anchor.x0 * scale_x + 8
+ text_end_x = operation[1] + pdf_fill_block._flat_text_width_pt(value, operation[3])
+ assert operation[1] != pytest.approx(right_x, abs=0.05) or operation[3] < 11
+ assert operation[1] == pytest.approx(below_x, abs=0.05) or text_end_x <= obstacle.x0 * scale_x
+
+
+async def test_flat_overlay_below_detects_tall_obstacle_spanning_label_line(tmp_path: Path) -> None:
+ source_path = _write_blank_flat_pdf(tmp_path)
+ dimensions = {"page_width_px": 1275, "page_height_px": 1651}
+ anchor = FlatPdfAnchor(anchor_id=0, page_index=0, text="Notes:", x0=88, x1=180, top=300, bottom=313, **dimensions)
+ obstacle = FlatPdfAnchor(anchor_id=1, page_index=0, text="Box", x0=95, x1=180, top=305, bottom=340, **dimensions)
+ block = _make_block(file_url=str(source_path))
+ result_bytes = await block._fill_flat_overlay(
+ PdfReader(str(source_path)),
+ [FlatPlacement(anchor=anchor, value="OK", position="below")],
+ [anchor, obstacle],
+ tmp_path / "filled.pdf",
+ )
+
+ operation = _flat_text_ops(result_bytes)[0]
+ scale_x = 612 / 1275
+ assert operation[1] == pytest.approx(anchor.x1 * scale_x + 6, abs=0.05)
+ assert operation[3] == 11
+
+
+async def test_flat_overlay_zero_height_anchor_detects_obstacle(tmp_path: Path) -> None:
+ source_path = _write_blank_flat_pdf(tmp_path)
+ dimensions = {"page_width_px": 1275, "page_height_px": 1651}
+ anchor = FlatPdfAnchor(anchor_id=0, page_index=0, text="Ref:", x0=88, x1=200, top=300, bottom=300, **dimensions)
+ obstacle = FlatPdfAnchor(anchor_id=1, page_index=0, text="X:", x0=250, x1=320, top=286, bottom=305, **dimensions)
+ value = "WWWWWWWWWW"
+ block = _make_block(file_url=str(source_path))
+ result_bytes = await block._fill_flat_overlay(
+ PdfReader(str(source_path)),
+ [FlatPlacement(anchor=anchor, value=value, position="right")],
+ [anchor, obstacle],
+ tmp_path / "filled.pdf",
+ )
+
+ operation = _flat_text_ops(result_bytes)[0]
+ scale_x = 612 / 1275
+ right_x = anchor.x1 * scale_x + 6
+ below_x = anchor.x0 * scale_x + 8
+ text_end_x = operation[1] + pdf_fill_block._flat_text_width_pt(value, operation[3])
+ assert operation[1] != pytest.approx(right_x, abs=0.05) or operation[3] < 11
+ assert operation[1] == pytest.approx(below_x, abs=0.05) or text_end_x <= obstacle.x0 * scale_x
+
+
+async def test_flat_overlay_fitting_placement_is_byte_identical(tmp_path: Path) -> None:
+ source_path = _write_blank_flat_pdf(tmp_path)
+ anchors = _flat_collision_anchors()
+ city_label = anchors[4]
+ block = _make_block(file_url=str(source_path))
+ result_bytes = await block._fill_flat_overlay(
+ PdfReader(str(source_path)),
+ [FlatPlacement(anchor=city_label, value="Detroit", position="right")],
+ anchors,
+ tmp_path / "filled.pdf",
+ )
+
+ PdfReader(io.BytesIO(result_bytes))
+ operations = _flat_text_ops(result_bytes)
+ assert len(operations) == 1
+ scale_x = 612 / 1275
+ scale_y = 792 / 1651
+ expected_op = (
+ f"BT /SkyvernHelv 11 Tf 0 0 0 rg {138 * scale_x + 6:.1f} {792 - 411 * scale_y - 1:.1f} Td (Detroit) Tj ET"
+ )
+ assert operations[0][0] == expected_op
+
+
+async def test_flat_overlay_shrinks_font_when_neither_position_fits_at_full_size(tmp_path: Path) -> None:
+ source_path = _write_blank_flat_pdf(tmp_path)
+ anchors = _tight_flat_anchors()
+ block = _make_block(file_url=str(source_path))
+ value = "ABCDEFGHIJ"
+ result_bytes = await block._fill_flat_overlay(
+ PdfReader(str(source_path)),
+ [FlatPlacement(anchor=anchors[0], value=value, position="right")],
+ anchors,
+ tmp_path / "filled.pdf",
+ )
+
+ PdfReader(io.BytesIO(result_bytes))
+ operations = _flat_text_ops(result_bytes)
+ assert len(operations) == 1
+ operation = operations[0]
+ assert 7 <= operation[3] <= 10.5
+ scale_x = 612 / 1275
+ assert operation[1] + pdf_fill_block._flat_text_width_pt(value, operation[3]) <= 300 * scale_x
+
+
+async def test_flat_overlay_shrunk_below_does_not_overblock_later_placement(tmp_path: Path) -> None:
+ source_path = _write_blank_flat_pdf(tmp_path)
+ dimensions = {"page_width_px": 1275, "page_height_px": 1651}
+ first_anchor = FlatPdfAnchor(
+ anchor_id=0, page_index=0, text="Primary:", x0=100, x1=150, top=90, bottom=100, **dimensions
+ )
+ right_blocker = FlatPdfAnchor(
+ anchor_id=1, page_index=0, text="Side:", x0=160, x1=200, top=90, bottom=105, **dimensions
+ )
+ below_blocker = FlatPdfAnchor(
+ anchor_id=2, page_index=0, text="Limit:", x0=230, x1=260, top=127, bottom=135, **dimensions
+ )
+ later_anchor = FlatPdfAnchor(
+ anchor_id=3, page_index=0, text="Next:", x0=120, x1=140, top=140, bottom=140, **dimensions
+ )
+ first_value = "WWWWWWW"
+ block = _make_block(file_url=str(source_path))
+ result_bytes = await block._fill_flat_overlay(
+ PdfReader(str(source_path)),
+ [
+ FlatPlacement(anchor=first_anchor, value=first_value, position="below"),
+ FlatPlacement(anchor=later_anchor, value="OK", position="right"),
+ ],
+ [first_anchor, right_blocker, below_blocker, later_anchor],
+ tmp_path / "filled.pdf",
+ )
+
+ operations_by_value = {operation[4]: operation for operation in _flat_text_ops(result_bytes)}
+ assert operations_by_value[first_value][3] == 7.5
+ scale_x = 612 / 1275
+ scale_y = 792 / 1651
+ later_op = operations_by_value["OK"]
+ assert later_op[1] == pytest.approx(later_anchor.x1 * scale_x + 6, abs=0.05)
+ assert later_op[2] == pytest.approx(792 - later_anchor.bottom * scale_y - 1, abs=0.05)
+ assert later_op[3] == 11
+
+
+async def test_flat_overlay_overflow_falls_back_to_min_font(tmp_path: Path) -> None:
+ source_path = _write_blank_flat_pdf(tmp_path)
+ anchors = _tight_flat_anchors()
+ block = _make_block(file_url=str(source_path))
+ result_bytes = await block._fill_flat_overlay(
+ PdfReader(str(source_path)),
+ [FlatPlacement(anchor=anchors[0], value="W" * 100, position="right")],
+ anchors,
+ tmp_path / "filled.pdf",
+ )
+
+ PdfReader(io.BytesIO(result_bytes))
+ operations = _flat_text_ops(result_bytes)
+ assert len(operations) == 1
+ assert operations[0][3] == 7
+
+
+async def test_flat_overlay_overflow_recorded_in_output(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+ run_id = "run_flat_pdf_overflow"
+ _install_context(monkeypatch, run_id)
+ flat_pdf = _run_local_pdf_path(monkeypatch, tmp_path, run_id, "flat.pdf")
+ _write_flat_pdf(flat_pdf)
+ dimensions = {"page_width_px": 600, "page_height_px": 400}
+ anchor = FlatPdfAnchor(anchor_id=0, page_index=0, text="Ref:", x0=20, x1=120, top=30, bottom=44, **dimensions)
+ obstacle = FlatPdfAnchor(anchor_id=1, page_index=0, text="X:", x0=130, x1=190, top=30, bottom=44, **dimensions)
+ value = "W" * 100
+
+ async def _placement_llm_response(**_: Any) -> dict[str, Any]:
+ return {"placements": [{"anchor_id": 0, "value": value, "position": "right"}], "thought": "ok"}
+
+ monkeypatch.setattr(PdfFillBlock, "_tesseract_available", staticmethod(lambda: True))
+ monkeypatch.setattr(PdfFillBlock, "_extract_flat_anchors", AsyncMock(return_value=[anchor, obstacle]))
+ monkeypatch.setattr(PdfFillBlock, "_resolve_default_llm_handler", AsyncMock(return_value=_placement_llm_response))
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.workflow.models.pdf_fill_block.get_path_for_workflow_download_directory",
+ lambda workflow_run_id: tmp_path / "downloads" / workflow_run_id,
+ )
+
+ block = _make_block(file_url=str(flat_pdf), payload={"reference": value})
+ result = await block.execute(workflow_run_id=run_id, workflow_run_block_id="", organization_id=None)
+
+ assert result.success is True
+ overflowed_placements = result.output_parameter_value["overflowed_placements"]
+ assert overflowed_placements == [
+ {
+ "anchor_id": anchor.anchor_id,
+ "value": value,
+ "reason": "Placement overflowed the available space; value may overlap adjacent content",
+ }
+ ]
+ assert "overlap" in overflowed_placements[0]["reason"]
+
+
+async def test_flat_overlay_second_below_value_avoids_first(tmp_path: Path) -> None:
+ source_path = _write_blank_flat_pdf(tmp_path)
+ dimensions = {"page_width_px": 1275, "page_height_px": 1651}
+ notes_label = FlatPdfAnchor(
+ anchor_id=0, page_index=0, text="Notes:", x0=88, x1=218, top=309, bottom=322, **dimensions
+ )
+ code_label = FlatPdfAnchor(
+ anchor_id=1, page_index=0, text="Code:", x0=494, x1=672, top=309, bottom=322, **dimensions
+ )
+ anchors = [notes_label, code_label]
+ first_value = "A" * 65
+ block = _make_block(file_url=str(source_path))
+ result_bytes = await block._fill_flat_overlay(
+ PdfReader(str(source_path)),
+ [
+ FlatPlacement(anchor=notes_label, value=first_value, position="below"),
+ FlatPlacement(anchor=code_label, value="X7", position="below"),
+ ],
+ anchors,
+ tmp_path / "filled.pdf",
+ )
+
+ PdfReader(io.BytesIO(result_bytes))
+ operations = _flat_text_ops(result_bytes)
+ assert len(operations) == 2
+ operations_by_value = {operation[4]: operation for operation in operations}
+ first_op = operations_by_value[first_value]
+ second_op = operations_by_value["X7"]
+ first_rect = (
+ first_op[1],
+ first_op[2],
+ first_op[1] + pdf_fill_block._flat_text_width_pt(first_value, first_op[3]),
+ first_op[2] + first_op[3],
+ )
+ second_rect = (
+ second_op[1],
+ second_op[2],
+ second_op[1] + pdf_fill_block._flat_text_width_pt("X7", second_op[3]),
+ second_op[2] + second_op[3],
+ )
+ assert not (
+ first_rect[0] < second_rect[2]
+ and first_rect[2] > second_rect[0]
+ and first_rect[1] < second_rect[3]
+ and first_rect[3] > second_rect[1]
+ )
+ scale_x = 612 / 1275
+ scale_y = 792 / 1651
+ assert second_op[1] == pytest.approx(672 * scale_x + 6, abs=0.05)
+ assert second_op[2] == pytest.approx(792 - 322 * scale_y - 1, abs=0.05)
+
+
+async def test_flat_overlay_font_uses_winansi_encoding(tmp_path: Path) -> None:
+ source_path = _write_blank_flat_pdf(tmp_path)
+ anchor = FlatPdfAnchor(
+ anchor_id=0,
+ page_index=0,
+ text="Name:",
+ x0=88,
+ x1=180,
+ top=300,
+ bottom=313,
+ page_width_px=1275,
+ page_height_px=1651,
+ )
+ block = _make_block(file_url=str(source_path))
+ result_bytes = await block._fill_flat_overlay(
+ PdfReader(str(source_path)),
+ [FlatPlacement(anchor=anchor, value="Renée", position="right")],
+ [anchor],
+ tmp_path / "filled.pdf",
+ )
+
+ reader = PdfReader(io.BytesIO(result_bytes))
+ font = reader.pages[0]["/Resources"]["/Font"][pdf_fill_block.FLAT_FILL_FONT_RESOURCE].get_object()
+ assert font["/Encoding"] == "/WinAnsiEncoding"
+ assert font["/BaseFont"] == "/Helvetica"
+ operations = _flat_text_ops(result_bytes)
+ assert len(operations) == 1
+ assert operations[0][4] == "Renée"
+
+
+def test_resolve_flat_page_layout_emits_exact_coords_for_fitting_value() -> None:
+ anchor = FlatPdfAnchor(
+ anchor_id=0,
+ page_index=0,
+ text="Ref:",
+ x0=88,
+ x1=200,
+ top=300,
+ bottom=313,
+ page_width_px=1275,
+ page_height_px=1651,
+ )
+ block = _make_block()
+ resolved = block._resolve_flat_page_layout(
+ [FlatPlacement(anchor=anchor, value="OK", position="right")],
+ [anchor],
+ page_width=612,
+ page_height=792,
+ origin_x=0,
+ origin_y=0,
+ )
+
+ assert len(resolved) == 1
+ assert resolved[0].x == 200 * (612 / 1275) + 6
+ assert resolved[0].y == 792 - 313 * (792 / 1651) - 1
+ assert resolved[0].font_size == 11.0
+
+
@pytest.mark.asyncio
async def test_flat_overlay_respects_shifted_mediabox(tmp_path: Path) -> None:
flat_pdf = tmp_path / "shifted.pdf"
@@ -482,6 +1004,7 @@ async def test_flat_overlay_respects_shifted_mediabox(tmp_path: Path) -> None:
await block._fill_flat_overlay(
PdfReader(str(flat_pdf)),
[FlatPlacement(anchor=anchor, value="Jane", position="right")],
+ [anchor],
output_path,
)
@@ -529,6 +1052,7 @@ async def test_flat_overlay_respects_cropbox_distinct_from_mediabox(tmp_path: Pa
await block._fill_flat_overlay(
PdfReader(str(flat_pdf)),
[FlatPlacement(anchor=anchor, value="Jane", position="right")],
+ [anchor],
output_path,
)
diff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.py
index f87921151..481238eb1 100644
--- a/tests/unit/test_script_skyvern_page.py
+++ b/tests/unit/test_script_skyvern_page.py
@@ -17,9 +17,10 @@ import pytest
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
from skyvern.config import settings
+from skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAi
from skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPage
from skyvern.core.script_generations.skyvern_page import SkyvernPage
-from skyvern.exceptions import IllegitCompleteScriptTermination, ScriptTerminationException
+from skyvern.exceptions import IllegitCompleteScriptTermination, NoTOTPSecretFound, ScriptTerminationException
class _KeyboardStub:
@@ -859,6 +860,385 @@ async def test_fill_autocomplete_value_none_proactive_unchanged(mock_scraped_pag
script_page._ai.ai_input_text.assert_called_once()
+async def test_fill_autocomplete_does_not_fall_back_to_unresolved_totp_placeholder(mock_scraped_page, mock_ai):
+ mock_page = create_mock_page()
+
+ with patch(
+ "skyvern.core.script_generations.skyvern_page.Page.__init__",
+ return_value=None,
+ ):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=mock_page,
+ ai=mock_ai,
+ )
+
+ script_page.get_actual_value = AsyncMock(side_effect=NoTOTPSecretFound())
+ script_page._do_autocomplete = AsyncMock(return_value="placeholder_AbCd_totp")
+
+ with pytest.raises(NoTOTPSecretFound):
+ await script_page.fill_autocomplete(
+ selector="#otp",
+ value="placeholder_AbCd_totp",
+ ai="fallback",
+ )
+
+ script_page._do_autocomplete.assert_not_awaited()
+
+
+async def test_fill_autocomplete_rejects_raw_totp_placeholder_after_resolution(mock_scraped_page, mock_ai):
+ mock_page = create_mock_page()
+
+ with patch(
+ "skyvern.core.script_generations.skyvern_page.Page.__init__",
+ return_value=None,
+ ):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=mock_page,
+ ai=mock_ai,
+ )
+
+ script_page.get_actual_value = AsyncMock(return_value="placeholder_AbCd_totp")
+ script_page._do_autocomplete = AsyncMock(return_value="placeholder_AbCd_totp")
+
+ with pytest.raises(NoTOTPSecretFound):
+ await script_page.fill_autocomplete(
+ selector="#otp",
+ value="placeholder_AbCd_totp",
+ ai="fallback",
+ )
+
+ script_page._do_autocomplete.assert_not_awaited()
+
+
+async def test_fill_autocomplete_proactive_rejects_raw_totp_placeholder_before_ai(mock_scraped_page, mock_ai):
+ mock_page = create_mock_page()
+
+ with patch(
+ "skyvern.core.script_generations.skyvern_page.Page.__init__",
+ return_value=None,
+ ):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=mock_page,
+ ai=mock_ai,
+ )
+
+ script_page.get_actual_value = AsyncMock(return_value="placeholder_AbCd_totp")
+ script_page._ai.ai_input_text = AsyncMock(return_value="placeholder_AbCd_totp")
+
+ with pytest.raises(NoTOTPSecretFound):
+ await script_page.fill_autocomplete(
+ selector="#otp",
+ value="placeholder_AbCd_totp",
+ prompt="Enter the verification code",
+ ai="proactive",
+ )
+
+ script_page._ai.ai_input_text.assert_not_awaited()
+
+
+async def test_fill_autocomplete_proactive_validates_totp_without_exposing_code_to_ai(mock_scraped_page, mock_ai):
+ mock_page = create_mock_page()
+
+ with patch(
+ "skyvern.core.script_generations.skyvern_page.Page.__init__",
+ return_value=None,
+ ):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=mock_page,
+ ai=mock_ai,
+ )
+
+ script_page.get_actual_value = AsyncMock(return_value="654321")
+ script_page._ai.ai_input_text = AsyncMock(return_value="654321")
+
+ result = await script_page.fill_autocomplete(
+ selector="#otp",
+ value="placeholder_AbCd_totp",
+ prompt="Enter the verification code",
+ ai="proactive",
+ )
+
+ assert result == "654321"
+ assert script_page._ai.ai_input_text.await_args.kwargs["value"] == "placeholder_AbCd_totp"
+ assert "654321" not in script_page._ai.ai_input_text.await_args.kwargs.values()
+
+
+async def test_input_text_does_not_send_unresolved_totp_placeholder_to_ai_fallback(mock_scraped_page, mock_ai):
+ mock_page = create_mock_page()
+
+ with patch(
+ "skyvern.core.script_generations.skyvern_page.Page.__init__",
+ return_value=None,
+ ):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=mock_page,
+ ai=mock_ai,
+ )
+
+ script_page.get_actual_value = AsyncMock(side_effect=NoTOTPSecretFound())
+ script_page._ai.ai_input_text = AsyncMock(return_value="placeholder_AbCd_totp")
+
+ with pytest.raises(NoTOTPSecretFound):
+ await script_page._input_text(
+ selector="#otp",
+ value="placeholder_AbCd_totp",
+ ai="fallback",
+ intention="Enter the verification code",
+ )
+
+ script_page._ai.ai_input_text.assert_not_awaited()
+
+
+async def test_input_text_regenerates_totp_after_selector_preparation(mock_scraped_page, mock_ai):
+ mock_page = create_mock_page()
+
+ with patch(
+ "skyvern.core.script_generations.skyvern_page.Page.__init__",
+ return_value=None,
+ ):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=mock_page,
+ ai=mock_ai,
+ )
+
+ events: list[str] = []
+ generated_codes = iter(("123456", "654321"))
+ script_page.get_actual_value = AsyncMock(
+ side_effect=lambda *_args, **_kwargs: events.append("generate") or next(generated_codes)
+ )
+ locator = MagicMock()
+ locator.fill = AsyncMock(side_effect=lambda value, **_kwargs: events.append(f"fill:{value}"))
+ script_page._wait_for_selector_with_retry = AsyncMock(
+ side_effect=lambda *_args, **_kwargs: events.append("selector") or locator
+ )
+ script_page._prepare_element = AsyncMock(side_effect=lambda *_args, **_kwargs: events.append("prepare"))
+
+ result = await script_page._input_text(
+ selector="#otp",
+ value="placeholder_AbCd_totp",
+ ai="fallback",
+ )
+
+ assert result == "placeholder_AbCd_totp"
+ assert events == ["generate", "selector", "prepare", "generate", "fill:654321"]
+ locator.fill.assert_awaited_once_with("654321", timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
+
+
+async def test_fill_from_mapping_keeps_totp_placeholder_out_of_ai_prompt(mock_scraped_page, mock_ai):
+ mock_page = create_mock_page()
+
+ with patch(
+ "skyvern.core.script_generations.skyvern_page.Page.__init__",
+ return_value=None,
+ ):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=mock_page,
+ ai=mock_ai,
+ )
+
+ script_page._resolve_totp_placeholder_or_raise = AsyncMock(return_value="654321")
+ script_page.fill = AsyncMock(side_effect=(RuntimeError("selector fill failed"), "placeholder_AbCd_totp"))
+
+ await script_page.fill_from_mapping(
+ form_fields=[{"selector": "#otp", "type": "text", "tag": "input", "label": "Verification code"}],
+ mapping={0: "placeholder_AbCd_totp"},
+ )
+
+ script_page._resolve_totp_placeholder_or_raise.assert_not_awaited()
+ assert script_page.fill.await_count == 2
+ first_fill = script_page.fill.await_args_list[0].kwargs
+ fallback_fill = script_page.fill.await_args_list[1].kwargs
+ assert first_fill == {"selector": "#otp", "value": "placeholder_AbCd_totp", "ai": None}
+ assert fallback_fill["value"] == "placeholder_AbCd_totp"
+ assert fallback_fill["ai"] == "fallback"
+ assert "654321" not in str(fallback_fill)
+
+
+ "fill_side_effect, expected_fill_count",
+ [
+ ((NoTOTPSecretFound(),), 1),
+ ((RuntimeError("selector fill failed"), NoTOTPSecretFound()), 2),
+ ],
+)
+async def test_fill_from_mapping_propagates_totp_resolution_failures(
+ mock_scraped_page,
+ mock_ai,
+ fill_side_effect,
+ expected_fill_count,
+):
+ mock_page = create_mock_page()
+
+ with patch(
+ "skyvern.core.script_generations.skyvern_page.Page.__init__",
+ return_value=None,
+ ):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=mock_page,
+ ai=mock_ai,
+ )
+
+ script_page.fill = AsyncMock(side_effect=fill_side_effect)
+
+ with pytest.raises(NoTOTPSecretFound):
+ await script_page.fill_from_mapping(
+ form_fields=[{"selector": "#otp", "type": "text", "tag": "input", "label": "Verification code"}],
+ mapping={0: "placeholder_AbCd_totp"},
+ )
+
+ assert script_page.fill.await_count == expected_fill_count
+
+
+async def test_input_text_rejects_raw_totp_placeholder_after_resolution(mock_scraped_page, mock_ai):
+ mock_page = create_mock_page()
+
+ with patch(
+ "skyvern.core.script_generations.skyvern_page.Page.__init__",
+ return_value=None,
+ ):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=mock_page,
+ ai=mock_ai,
+ )
+
+ script_page.get_actual_value = AsyncMock(return_value="placeholder_AbCd_totp")
+ script_page._ai.ai_input_text = AsyncMock(return_value="placeholder_AbCd_totp")
+
+ with pytest.raises(NoTOTPSecretFound):
+ await script_page._input_text(
+ selector="#otp",
+ value="placeholder_AbCd_totp",
+ ai="fallback",
+ intention="Enter the verification code",
+ )
+
+ script_page._ai.ai_input_text.assert_not_awaited()
+
+
[email protected]("ai", [None, "proactive"])
+async def test_input_text_rejects_raw_totp_placeholder_in_nonfallback_modes(mock_scraped_page, mock_ai, ai):
+ mock_page = create_mock_page()
+
+ with patch(
+ "skyvern.core.script_generations.skyvern_page.Page.__init__",
+ return_value=None,
+ ):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=mock_page,
+ ai=mock_ai,
+ )
+
+ script_page.get_actual_value = AsyncMock(return_value="placeholder_AbCd_totp")
+ script_page._ai.ai_input_text = AsyncMock(return_value="placeholder_AbCd_totp")
+
+ with pytest.raises(NoTOTPSecretFound):
+ await script_page._input_text(
+ selector="#otp",
+ value="placeholder_AbCd_totp",
+ ai=ai,
+ intention="Enter the verification code",
+ )
+
+ script_page._ai.ai_input_text.assert_not_awaited()
+
+
+async def test_input_text_proactive_validates_totp_without_exposing_code_to_ai(mock_scraped_page, mock_ai):
+ mock_page = create_mock_page()
+
+ with patch(
+ "skyvern.core.script_generations.skyvern_page.Page.__init__",
+ return_value=None,
+ ):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=mock_page,
+ ai=mock_ai,
+ )
+
+ script_page.get_actual_value = AsyncMock(return_value="654321")
+ script_page._ai.ai_input_text = AsyncMock(return_value="654321")
+
+ result = await script_page._input_text(
+ selector="#otp",
+ value="placeholder_AbCd_totp",
+ ai="proactive",
+ intention="Enter the verification code",
+ )
+
+ assert result == "654321"
+ assert script_page._ai.ai_input_text.await_args.kwargs["value"] == "placeholder_AbCd_totp"
+ assert "654321" not in script_page._ai.ai_input_text.await_args.kwargs.values()
+
+
[email protected]("marker", ["OP_TOTP", "BW_TOTP", "AZ_TOTP"])
+async def test_input_text_rejects_raw_totp_markers_in_direct_mode(mock_scraped_page, mock_ai, marker):
+ mock_page = create_mock_page()
+
+ with patch(
+ "skyvern.core.script_generations.skyvern_page.Page.__init__",
+ return_value=None,
+ ):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=mock_page,
+ ai=mock_ai,
+ )
+
+ script_page.get_actual_value = AsyncMock(return_value=marker)
+
+ with pytest.raises(NoTOTPSecretFound):
+ await script_page._input_text(selector="#otp", value=marker, ai=None)
+
+
+async def test_real_page_ai_rejects_raw_totp_placeholder_before_fallback() -> None:
+ page_ai = RealSkyvernPageAi.__new__(RealSkyvernPageAi)
+ page_ai._maybe_run_v3_midrun = AsyncMock(return_value=(None, False))
+
+ with (
+ patch(
+ "skyvern.core.script_generations.real_skyvern_page_ai.skyvern_context.ensure_context",
+ return_value=SimpleNamespace(
+ organization_id=None,
+ task_id=None,
+ step_id=None,
+ workflow_run_id=None,
+ ),
+ ),
+ pytest.raises(NoTOTPSecretFound),
+ ):
+ await page_ai.ai_input_text(
+ selector="#otp",
+ value="placeholder_AbCd_totp",
+ intention="",
+ failed_selector="#stale-otp",
+ )
+
+ page_ai._maybe_run_v3_midrun.assert_not_awaited()
+
+
# =============================================================================
# Tests for wait() — timeout_ms support
# =============================================================================
@@ -1671,6 +2051,69 @@ def _skyvern_page_with_locator(mock_ai, locator: _RecordingLocator) -> SkyvernPa
return skyvern_page
+async def test_direct_fill_rejects_raw_totp_placeholder_before_browser_write(mock_scraped_page, mock_ai):
+ locator = _RecordingLocator()
+ script_page = _direct_fill_page(mock_scraped_page, mock_ai, locator)
+ script_page.get_actual_value = AsyncMock(return_value="placeholder_AbCd_totp")
+
+ with pytest.raises(NoTOTPSecretFound):
+ await script_page.fill("#otp", "placeholder_AbCd_totp", mode="direct")
+
+ assert locator.calls == []
+
+
+ "unresolved_value",
+ [
+ " placeholder_AbCd_totp",
+ "placeholder_AbCd_totp ",
+ "code: placeholder_AbCd_totp",
+ "prefixplaceholder_AbCd_totpsuffix",
+ ],
+)
+async def test_direct_fill_rejects_embedded_totp_placeholder_before_browser_write(
+ mock_scraped_page,
+ mock_ai,
+ unresolved_value,
+):
+ locator = _RecordingLocator()
+ script_page = _direct_fill_page(mock_scraped_page, mock_ai, locator)
+ script_page.get_actual_value = AsyncMock(return_value=unresolved_value)
+
+ with pytest.raises(NoTOTPSecretFound):
+ await script_page.fill("#otp", unresolved_value, mode="direct")
+
+ assert locator.calls == []
+
+
+async def test_direct_fill_resolves_totp_placeholder_before_browser_write(mock_scraped_page, mock_ai):
+ locator = _RecordingLocator()
+ script_page = _direct_fill_page(mock_scraped_page, mock_ai, locator)
+ script_page.get_actual_value = AsyncMock(return_value="654321")
+
+ result = await script_page.fill("#otp", "placeholder_AbCd_totp", mode="direct")
+
+ assert result == "654321"
+ assert ("fill", "654321") in locator.calls
+ assert ("fill", "placeholder_AbCd_totp") not in locator.calls
+
+
[email protected]("marker", ["OP_TOTP", "BW_TOTP", "AZ_TOTP"])
+async def test_direct_fill_rejects_raw_totp_markers_before_browser_write(mock_scraped_page, mock_ai, marker):
+ locator = _RecordingLocator()
+ script_page = _direct_fill_page(mock_scraped_page, mock_ai, locator)
+ script_page.get_actual_value = AsyncMock(return_value=marker)
+
+ with pytest.raises(NoTOTPSecretFound):
+ await script_page.fill("#otp", marker, mode="direct")
+
+ assert locator.calls == []
+
+
@pytest.mark.asyncio
async def test_direct_fill_dispatches_change_and_blur_after_value_set(mock_scraped_page, mock_ai):
"""mode='direct' must dispatch change/blur after the value-set so a JS gate