Skip to content

Automate browser based workflows with AI

Improve screenshot capture failure observability (#8547)

LawyZheng4822876e8083
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index a17f90583..571458ab4 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -3453,6 +3453,25 @@ def _is_prefix_loss_truncation(*, tag_name: str, intended: str, rendered: str |
return intended_cf.endswith(rendered_cf)
+def _is_same_length_reorder(*, tag_name: str, intended: str, rendered: str | None) -> bool:
+ # The per-character type(tail) path re-focuses the field on every character; a field that resets the caret on
+ # the input event lays the characters down out of order, so the rendered value is the same length as intended
+ # and a permutation of its characters, but not equal. After the permitted textarea CRLF/CR->LF
+ # normalization and case-folding: strict signature = not-equal, equal length, equal character multiset. This
+ # excludes a legitimate normalization (a case/format change alters the multiset), an autocomplete expansion (a
+ # different length), and any arbitrary mismatch (a different multiset) -- none of which is refilled.
+ if rendered is None:
+ return False
+ if tag_name == "textarea":
+ intended = _normalize_textarea_line_endings(intended)
+ rendered = _normalize_textarea_line_endings(rendered)
+ intended_cf = intended.casefold()
+ rendered_cf = rendered.casefold()
+ if len(rendered_cf) != len(intended_cf) or rendered_cf == intended_cf:
+ return False
+ return sorted(rendered_cf) == sorted(intended_cf)
+
+
async def _observe_input_value(
*,
skyvern_element: SkyvernElement,
@@ -3646,36 +3665,59 @@ async def _heal_truncated_freetext_input(
tag_name: str,
text: str,
is_secret_value: bool = False,
+ has_autocomplete_evidence: bool = False,
+ input_type: str | None = None,
+ maxlength: str | None = None,
engine_selection: BrowserEngineSelection | None = None,
) -> ActionFailure | None:
- # Preserves the deployed SKY-13631 coverage for the residual per-character seam: after the fill-first
- # default (SKY-13821) an ordinary native input fills atomically and cannot lose a prefix, but the paths
- # still typed character-by-character (tel formatting, a combobox/search-bar/in-context input) keep this
- # observational truncation guard. Only values longer than the split boundary can lose a prefix, so shorter
- # values and non-free-text tags are skipped without a read-back. Secret values are excluded outright --
- # their exact length must not reach the logs and an unmasked secret must not be rewritten from this generic
- # path; _fill_secret_with_readback owns that recovery. On the exact prefix-loss signature, re-enter the
- # value once with a single atomic fill; a matching or autocomplete-expanded value is left as typed so the
- # normal keystroke/autocomplete behavior is preserved. The pre-refill read-back is observational only:
- # bounded and best-effort, an unobtainable read-back detects no truncation and heals nothing. Once a refill
- # has run, the seam is integrity-gated:
- # confirmation requires a full case-folded match with the intended value (not merely the absence of the
- # loss signature). The sole accepted normalization is a textarea's browser-defined CRLF/lone-CR to LF
- # canonicalization. An unconfirmed or unobservable post-refill value fails closed with a structured
- # ActionFailure so a persistent partial value is never reported as success or followed by a batched Submit
- # (SKY-13631). No second write is ever attempted. Logs carry only lengths.
- if is_secret_value or tag_name not in _NATIVE_FILL_TAGS or len(text) <= TEXT_PRESS_MAX_LENGTH:
+ # Observational read-back guard for the residual per-character seam. An ordinary native input fills
+ # atomically and never reaches here; the paths still typed character-by-character (tel formatting, a
+ # combobox/search-bar/in-context input) do. One bounded read-back covers two corruption signatures a
+ # caret-resetting field produces:
+ # - prefix loss: the field wipes the atomic leading fill and keeps only the per-character tail, so the
+ # rendered value is a short proper suffix of the intended text. Only values longer than the split
+ # boundary can lose a prefix.
+ # - same-length reorder: the caret reset lays the tail down out of order, so the rendered value is a
+ # same-length permutation of the intended text. Repaired only on a round-trip-safe input (textarea or an
+ # exact-value type) with no structural autocomplete/combobox evidence -- an atomic refill on a
+ # keyboard-driven widget would suppress its option surfacing. tel/number/date/time and maxlength-truncating
+ # inputs are excluded; a genuine typeahead keeps its keystrokes.
+ # Secret values are excluded outright -- their exact length must not reach the logs and an unmasked secret
+ # must not be rewritten from this generic path; _fill_secret_with_readback owns that recovery. On either
+ # signature, re-enter the value once with a single atomic fill; a matching, autocomplete-expanded, or
+ # otherwise-mismatched value is left as typed. The pre-refill read-back is observational only: bounded and
+ # best-effort, an unobtainable read-back detects nothing and heals nothing.
+ can_prefix_loss = not is_secret_value and tag_name in _NATIVE_FILL_TAGS and len(text) > TEXT_PRESS_MAX_LENGTH
+ normalized_input_type = (input_type or "").strip().lower()
+ round_trip_safe = tag_name == "textarea" or normalized_input_type in _EXACT_VALUE_INPUT_TYPES
+ can_reorder = (
+ not is_secret_value
+ and tag_name in _NATIVE_FILL_TAGS
+ and len(text) > 1
+ and not has_autocomplete_evidence
+ and round_trip_safe
+ and normalized_input_type != "tel"
+ and not _maxlength_truncates_value(text, maxlength)
+ )
+ if not (can_prefix_loss or can_reorder):
return None
observed, rendered = await _observe_input_value(
skyvern_element=skyvern_element, tag_name=tag_name, engine_selection=engine_selection
)
- if not observed or not _is_prefix_loss_truncation(tag_name=tag_name, intended=text, rendered=rendered):
+ if not observed:
+ return None
+ lost_prefix = can_prefix_loss and _is_prefix_loss_truncation(tag_name=tag_name, intended=text, rendered=rendered)
+ reordered = can_reorder and _is_same_length_reorder(tag_name=tag_name, intended=text, rendered=rendered)
+ if not (lost_prefix or reordered):
return None
LOG.warning(
- "Free-text input lost its leading fill and kept only a trailing suffix; re-entering atomically",
+ "Free-text input read-back does not match the intended value (prefix loss or same-length reorder); "
+ "re-entering atomically",
element_id=skyvern_element.get_id(),
intended_length=len(text),
rendered_length=len(rendered or ""),
+ lost_prefix=lost_prefix,
+ reordered=reordered,
)
await skyvern_element.refresh_locator_if_stale()
await skyvern_element.input_fill(text=text)
@@ -8881,14 +8923,18 @@ async def _handle_input_text_action(
# an atomic write emits no key events, so the option tree stays empty and the action reports success
# with uncommitted display text. Computed once here so it both excludes secret typed-widgets from the
# first-write transport below and drives the ordinary-branch keeps_typing decision (SKY-13821).
- is_typed_widget = (
- (
- input_or_select_context is not None
- and bool(input_or_select_context.is_search_bar or input_or_select_context.is_location_input)
- )
- or await skyvern_element.is_auto_completion_input()
- or await _is_combobox_or_typeahead(skyvern_element)
+ # Structural autocomplete identity (role=combobox / aria-autocomplete / autocomplete-class markers) is a
+ # load-time property, so read it once here on the still-attached element rather than re-probing after the
+ # write: the final input event can remount/navigate/detach the control, and a post-write live attribute
+ # read would then raise and turn a successful input into a failure. Computed unconditionally (not behind
+ # the LLM search/location short-circuit) so the reorder heal below can reuse it.
+ structural_autocomplete = await skyvern_element.is_auto_completion_input() or await _is_combobox_or_typeahead(
+ skyvern_element
)
+ is_typed_widget = (
+ input_or_select_context is not None
+ and bool(input_or_select_context.is_search_bar or input_or_select_context.is_location_input)
+ ) or structural_autocomplete
# A positive maxlength shorter than the secret is an auto-advancing split field; an atomic write leaves
# a truncated prefix and, since it cannot round-trip, reports success unverified. Route it to the seam
# (below) so the per-character focus advance carries the rest to the sibling boxes (SKY-13821).
@@ -8990,13 +9036,20 @@ async def _handle_input_text_action(
return [ActionSuccess()]
else:
await skyvern_element.input_sequentially(text=text)
- # The residual per-character seam can still lose a leading prefix on a caret-resetting
- # field; preserve the deployed SKY-13631 truncation heal here (SKY-13821).
+ # The residual per-character seam can lose a leading prefix or lay the value down reordered on a
+ # caret-resetting field; one read-back here heals both. The reorder repair is gated on the
+ # structural autocomplete/combobox context read before typing (surfaced-option evidence is not
+ # available at this seam, so a marker-less option-only autocomplete is treated as
+ # no-autocomplete here; the strict same-length permutation signature keeps that rare case
+ # narrow).
truncation_failure = await _heal_truncated_freetext_input(
skyvern_element=skyvern_element,
tag_name=tag_name,
text=text,
is_secret_value=is_secret_value,
+ has_autocomplete_evidence=structural_autocomplete,
+ input_type=secret_input_type,
+ maxlength=element_maxlength,
engine_selection=engine_selection,
)
if isinstance(truncation_failure, ActionFailure):
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index cac0459eb..07ec17fce 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -835,6 +835,89 @@ class ScreenshotMode(StrEnum):
DETAILED = "detailed"
+class ScreenshotArm(StrEnum):
+ CONTROL = "control"
+ TREATMENT = "treatment"
+
+
+class ScreenshotPrimitive(StrEnum):
+ PLAYWRIGHT = "playwright"
+ CDP_RESCUE = "cdp_rescue"
+
+
+class ScreenshotStage(StrEnum):
+ ATTACH = "attach"
+ GEOMETRY = "geometry"
+ CAPTURE = "capture"
+ VALIDATION = "validation"
+ DETACH = "detach"
+ TERMINAL = "terminal"
+ UNKNOWN = "unknown"
+
+
+class ScreenshotOutcome(StrEnum):
+ SUCCESS = "success"
+ TIMEOUT = "timeout"
+ TARGET_CLOSED = "target_closed"
+ DECLINED = "declined"
+ ERROR = "error"
+
+
+class ScreenshotEligibility(StrEnum):
+ ELIGIBLE = "eligible"
+ INELIGIBLE_FULL_PAGE = "ineligible_full_page"
+ INELIGIBLE_ENGINE = "ineligible_engine"
+ INELIGIBLE_BROWSER = "ineligible_browser"
+
+
+def _screenshot_observation_fields(
+ *,
+ arm: ScreenshotArm,
+ primitive: ScreenshotPrimitive,
+ stage: ScreenshotStage,
+ outcome: ScreenshotOutcome,
+ elapsed_ms: float | None = None,
+ timeout_budget_ms: float | None = None,
+ eligibility: ScreenshotEligibility | None = None,
+) -> dict[str, Any]:
+ """Fixed, bounded, low-cardinality screenshot telemetry for Datadog aggregation.
+
+ Enum-typed keys keep cardinality bounded; numeric measures are clamped non-negative and integral.
+ No page-derived token crosses this boundary.
+ """
+ fields: dict[str, Any] = {
+ "screenshot.arm": arm.value,
+ "screenshot.primitive": primitive.value,
+ "screenshot.stage": stage.value,
+ "screenshot.outcome": outcome.value,
+ }
+ if elapsed_ms is not None:
+ fields["screenshot.elapsed_ms"] = max(0, int(elapsed_ms))
+ if timeout_budget_ms is not None:
+ fields["screenshot.timeout_budget_ms"] = max(0, int(timeout_budget_ms))
+ if eligibility is not None:
+ fields["screenshot.eligibility"] = eligibility.value
+ return fields
+
+
+def _cdp_rescue_eligibility(
+ *,
+ full_page: bool,
+ engine_selection: BrowserEngineSelection | None,
+ page: Page,
+) -> ScreenshotEligibility:
+ """Classify why the raw-CDP rescue path is (in)eligible. Mirrors the gate in ``_page_screenshot_helper``
+ without changing it; used only for telemetry."""
+ if full_page:
+ return ScreenshotEligibility.INELIGIBLE_FULL_PAGE
+ if engine_selection is not None and engine_selection.name == SKYCDP_ENGINE_NAME:
+ return ScreenshotEligibility.INELIGIBLE_ENGINE
+ browser = page.context.browser
+ if browser is None or browser.browser_type.name != "chromium":
+ return ScreenshotEligibility.INELIGIBLE_BROWSER
+ return ScreenshotEligibility.ELIGIBLE
+
+
async def _restore_invalid_viewport_before_screenshot(page: Page) -> None:
viewport = page.viewport_size
if not isinstance(viewport, dict):
@@ -886,12 +969,19 @@ async def _page_screenshot_helper(
raise
if page.is_closed():
raise
- if (
- not full_page
- and (engine_selection is None or engine_selection.name != SKYCDP_ENGINE_NAME)
- and (browser := page.context.browser) is not None
- and browser.browser_type.name == "chromium"
- ):
+ eligibility = _cdp_rescue_eligibility(full_page=full_page, engine_selection=engine_selection, page=page)
+ LOG.info(
+ "Screenshot playwright attempt timed out; evaluating raw CDP rescue eligibility",
+ **_screenshot_observation_fields(
+ arm=ScreenshotArm.CONTROL,
+ primitive=ScreenshotPrimitive.PLAYWRIGHT,
+ stage=ScreenshotStage.CAPTURE,
+ outcome=ScreenshotOutcome.TIMEOUT,
+ timeout_budget_ms=timeout,
+ eligibility=eligibility,
+ ),
+ )
+ if eligibility == ScreenshotEligibility.ELIGIBLE:
# A failed rescue ends this attempt; scaled viewports retain the animation retry.
rescued = await _cdp_rescue_screenshot(page=page, file_path=file_path)
if page.is_closed():
@@ -943,6 +1033,7 @@ async def _current_viewpoint_screenshot_helper(
except Exception:
viewport_info = "unknown"
+ capture_start = time.time()
try:
if mode == ScreenshotMode.DETAILED:
await _wait_for_screenshot_load_state(
@@ -972,12 +1063,21 @@ async def _current_viewpoint_screenshot_helper(
"Screenshot taking time",
screenshot_time=end_time - start_time,
file_path=file_path,
+ **_screenshot_observation_fields(
+ arm=ScreenshotArm.CONTROL,
+ primitive=ScreenshotPrimitive.PLAYWRIGHT,
+ stage=ScreenshotStage.TERMINAL,
+ outcome=ScreenshotOutcome.SUCCESS,
+ elapsed_ms=(end_time - capture_start) * 1000,
+ timeout_budget_ms=timeout,
+ ),
)
skyvern_context.record_browser_success()
return screenshot
except Exception as e:
if engine_selection is not None and not _is_engine_error(e, engine_selection):
raise
+ terminal_elapsed_ms = (time.time() - capture_start) * 1000
if page.is_closed() or _is_screenshot_target_closed(e, engine_selection):
LOG.info(
"Skipping screenshot because target closed during capture",
@@ -985,6 +1085,14 @@ async def _current_viewpoint_screenshot_helper(
viewport=viewport_info,
full_page=full_page,
mode=mode.value if hasattr(mode, "value") else str(mode),
+ **_screenshot_observation_fields(
+ arm=ScreenshotArm.CONTROL,
+ primitive=ScreenshotPrimitive.PLAYWRIGHT,
+ stage=ScreenshotStage.TERMINAL,
+ outcome=ScreenshotOutcome.TARGET_CLOSED,
+ elapsed_ms=terminal_elapsed_ms,
+ timeout_budget_ms=timeout,
+ ),
)
raise ScreenshotTargetClosed(error_message=str(e)) from e
if is_engine_timeout(e, engine_selection):
@@ -998,6 +1106,14 @@ async def _current_viewpoint_screenshot_helper(
mode=mode.value if hasattr(mode, "value") else str(mode),
screenshot_stage=_extract_playwright_screenshot_stage(e),
error=str(e),
+ **_screenshot_observation_fields(
+ arm=ScreenshotArm.CONTROL,
+ primitive=ScreenshotPrimitive.PLAYWRIGHT,
+ stage=ScreenshotStage.TERMINAL,
+ outcome=ScreenshotOutcome.TIMEOUT,
+ elapsed_ms=terminal_elapsed_ms,
+ timeout_budget_ms=timeout,
+ ),
)
raise FailedToTakeScreenshot(error_message=str(e)) from e
LOG.error(
@@ -1007,6 +1123,14 @@ async def _current_viewpoint_screenshot_helper(
full_page=full_page,
error=str(e),
exc_info=True,
+ **_screenshot_observation_fields(
+ arm=ScreenshotArm.CONTROL,
+ primitive=ScreenshotPrimitive.PLAYWRIGHT,
+ stage=ScreenshotStage.TERMINAL,
+ outcome=ScreenshotOutcome.ERROR,
+ elapsed_ms=terminal_elapsed_ms,
+ timeout_budget_ms=timeout,
+ ),
)
raise FailedToTakeScreenshot(error_message=str(e)) from e
@@ -1024,11 +1148,29 @@ class _RescueDeclined:
_RESCUE_DECLINED = _RescueDeclined()
-async def _cdp_rescue_screenshot(page: Page, file_path: str | None) -> bytes | _RescueDeclined | None:
+async def _cdp_rescue_screenshot(
+ page: Page,
+ file_path: str | None,
+ *,
+ arm: ScreenshotArm = ScreenshotArm.CONTROL,
+) -> bytes | _RescueDeclined | None:
"""Return PNG bytes, a scaled-viewport decline, or None after a failed rescue."""
session = None
+ stage = ScreenshotStage.ATTACH
+ start = time.time()
+
+ def _fields(reached: ScreenshotStage, outcome: ScreenshotOutcome) -> dict[str, Any]:
+ return _screenshot_observation_fields(
+ arm=arm,
+ primitive=ScreenshotPrimitive.CDP_RESCUE,
+ stage=reached,
+ outcome=outcome,
+ elapsed_ms=(time.time() - start) * 1000,
+ )
+
try:
session = await asyncio.wait_for(page.context.new_cdp_session(page), timeout=CDP_RESCUE_SESSION_TIMEOUT_SECONDS)
+ stage = ScreenshotStage.GEOMETRY
async with asyncio.timeout(CDP_RESCUE_CAPTURE_TIMEOUT_SECONDS):
# The renderer round-trip shares the capture budget and can still time out on a stuck renderer.
geometry = await session.send(
@@ -1041,24 +1183,66 @@ async def _cdp_rescue_screenshot(page: Page, file_path: str | None) -> bytes | _
)
if geometry["result"]["value"] != {"deviceScaleFactor": 1, "viewportScale": 1}:
# A scaled CDP clip can reset Chromium's emulated DPR, so leave scaled captures to Playwright.
+ LOG.info(
+ "Raw CDP rescue screenshot declined scaled viewport",
+ **_fields(ScreenshotStage.GEOMETRY, ScreenshotOutcome.DECLINED),
+ )
return _RESCUE_DECLINED
+ stage = ScreenshotStage.CAPTURE
result = await session.send("Page.captureScreenshot", {"format": "png", "captureBeyondViewport": False})
+ stage = ScreenshotStage.VALIDATION
data = base64.b64decode(result.get("data", ""), validate=False)
if data[: len(_PNG_SIGNATURE)] != _PNG_SIGNATURE:
- LOG.warning("Raw CDP rescue screenshot returned a non-PNG payload", payload_bytes=len(data))
+ LOG.warning(
+ "Raw CDP rescue screenshot returned a non-PNG payload",
+ payload_bytes=len(data),
+ **_fields(ScreenshotStage.VALIDATION, ScreenshotOutcome.ERROR),
+ )
return None
if file_path is not None:
Path(file_path).parent.mkdir(parents=True, exist_ok=True)
Path(file_path).write_bytes(data)
+ LOG.info(
+ "Raw CDP rescue screenshot captured",
+ **_fields(ScreenshotStage.VALIDATION, ScreenshotOutcome.SUCCESS),
+ )
return data
- except Exception:
- LOG.warning("Raw CDP rescue screenshot failed", exc_info=True)
+ except Exception as exc:
+ LOG.warning(
+ "Raw CDP rescue screenshot failed",
+ exc_info=True,
+ **_fields(
+ stage,
+ ScreenshotOutcome.TIMEOUT
+ if isinstance(exc, TimeoutError) or is_driver_timeout_error(exc)
+ else ScreenshotOutcome.ERROR,
+ ),
+ )
return None
finally:
if session is not None:
- detach_task = asyncio.create_task(
- asyncio.wait_for(session.detach(), timeout=CDP_RESCUE_DETACH_TIMEOUT_SECONDS)
- )
+
+ async def detach_with_observation() -> None:
+ detach_start = time.time()
+ try:
+ await asyncio.wait_for(session.detach(), timeout=CDP_RESCUE_DETACH_TIMEOUT_SECONDS)
+ except Exception as exc:
+ LOG.warning(
+ "Raw CDP rescue screenshot detach failed",
+ **_screenshot_observation_fields(
+ arm=arm,
+ primitive=ScreenshotPrimitive.CDP_RESCUE,
+ stage=ScreenshotStage.DETACH,
+ outcome=ScreenshotOutcome.TIMEOUT
+ if isinstance(exc, TimeoutError) or is_driver_timeout_error(exc)
+ else ScreenshotOutcome.ERROR,
+ elapsed_ms=(time.time() - detach_start) * 1000,
+ timeout_budget_ms=CDP_RESCUE_DETACH_TIMEOUT_SECONDS * 1000,
+ ),
+ )
+ raise
+
+ detach_task = asyncio.create_task(detach_with_observation())
with contextlib.suppress(Exception):
try:
await asyncio.shield(detach_task)
diff --git a/tests/unit/test_freetext_truncation_selfheal.py b/tests/unit/test_freetext_truncation_selfheal.py
index 3cf91d82d..18dca1240 100644
--- a/tests/unit/test_freetext_truncation_selfheal.py
+++ b/tests/unit/test_freetext_truncation_selfheal.py
@@ -31,6 +31,7 @@ from skyvern.webeye.actions.handler import (
_freetext_mismatch_failure,
_heal_truncated_freetext_input,
_is_prefix_loss_truncation,
+ _is_same_length_reorder,
_static_declared_constraint_evidence,
)
from skyvern.webeye.actions.responses import ActionFailure, ActionResult
@@ -131,11 +132,18 @@ def _assert_fails_closed(result: ActionResult | None, *, intended_length: int =
async def _run_heal(
- intended: str, rendered: str | None, tag_name: str = "input", is_secret_value: bool = False
+ intended: str,
+ rendered: str | None,
+ tag_name: str = "input",
+ is_secret_value: bool = False,
+ has_autocomplete_evidence: bool = False,
+ input_type: str | None = None,
+ maxlength: str | None = None,
) -> tuple[AsyncMock, ActionResult | None]:
element = make_input_element_mock(element_id="EL1")
+ read_back = AsyncMock(return_value=rendered)
with (
- patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(return_value=rendered)),
+ patch("skyvern.webeye.actions.handler.get_input_value", new=read_back),
patch(_STATIC_PATCH, new=AsyncMock(return_value=None)),
):
result = await _heal_truncated_freetext_input(
@@ -143,13 +151,22 @@ async def _run_heal(
tag_name=tag_name,
text=intended,
is_secret_value=is_secret_value,
+ has_autocomplete_evidence=has_autocomplete_evidence,
+ input_type=input_type,
+ maxlength=maxlength,
engine_selection=None,
)
+ element._read_back = read_back
return element, result
async def _run_heal_capture(
- intended: str, readbacks: list[str | None], tag_name: str = "input"
+ intended: str,
+ readbacks: list[str | None],
+ tag_name: str = "input",
+ has_autocomplete_evidence: bool = False,
+ input_type: str | None = None,
+ maxlength: str | None = None,
) -> tuple[AsyncMock, MagicMock, ActionResult | None]:
element = make_input_element_mock(element_id="EL1")
with (
@@ -158,7 +175,13 @@ async def _run_heal_capture(
patch("skyvern.webeye.actions.handler.LOG") as log,
):
result = await _heal_truncated_freetext_input(
- skyvern_element=element, tag_name=tag_name, text=intended, engine_selection=None
+ skyvern_element=element,
+ tag_name=tag_name,
+ text=intended,
+ has_autocomplete_evidence=has_autocomplete_evidence,
+ input_type=input_type,
+ maxlength=maxlength,
+ engine_selection=None,
)
return element, log, result
@@ -260,13 +283,15 @@ async def test_heal_is_noop_for_autocomplete_expansion() -> None:
@pytest.mark.asyncio
-async def test_heal_skips_short_values_without_reading_back() -> None:
- short_value = "a" * TEXT_PRESS_MAX_LENGTH # not longer than the split boundary: no prefix to lose
+async def test_heal_skips_short_value_ineligible_for_both_signatures_without_reading_back() -> None:
+ # A short value cannot lose a prefix (not longer than the split boundary), and a non-round-trip-safe type
+ # (tel) is excluded from reorder repair -- so neither signature applies and the read-back is skipped.
+ short_value = "a" * TEXT_PRESS_MAX_LENGTH
element = make_input_element_mock(element_id="EL1")
read_back = AsyncMock(return_value="a")
with patch("skyvern.webeye.actions.handler.get_input_value", new=read_back):
result = await _heal_truncated_freetext_input(
- skyvern_element=element, tag_name="input", text=short_value, engine_selection=None
+ skyvern_element=element, tag_name="input", text=short_value, input_type="tel", engine_selection=None
)
read_back.assert_not_awaited()
element.input_fill.assert_not_awaited()
@@ -1036,3 +1061,128 @@ async def test_heal_does_not_run_static_when_post_refill_unobservable() -> None:
element.press_fill.assert_not_called()
assert isinstance(result, ActionFailure)
assert "unlikely to succeed" in (result.exception_message or "")
+
+
+# --------------------------------------------------------------------------- #
+# Same-length reorder repair — the single heal call also repairs a strict same-length permutation on a
+# round-trip-safe input, gated on no autocomplete evidence. A markerless option-only autocomplete (one that
+# surfaces options without a structural aria-autocomplete/combobox marker) is not distinguishable at this
+# read-back seam, so the strict permutation signature is what keeps that rare case narrow.
+# --------------------------------------------------------------------------- #
+_R_INTENDED = "abcdef"
+_R_REORDER = "fabcde" # same length, same multiset, not equal -> strict reorder signature
+
+
+ "intended, rendered, expected",
+ [
+ (_LONG, _LONG[1:] + _LONG[:1], True), # rotation
+ (_LONG, _LONG[::-1], True), # reversal
+ (_R_INTENDED, _R_REORDER, True), # short rotation
+ ("AbcDef", "FEDCAB", True), # casefold before compare
+ (_R_INTENDED, _R_INTENDED, False), # exact echo
+ (_LONG, _LONG.upper(), False), # case-only
+ (_R_INTENDED, "abcdefg", False), # different length (expansion)
+ (_R_INTENDED, "abcde", False), # different length (truncation)
+ (_R_INTENDED, "abcdeg", False), # same length, different multiset (arbitrary mismatch)
+ (_R_INTENDED, "", False), # empty
+ (_R_INTENDED, None, False), # unobservable
+ ],
+)
+def test_is_same_length_reorder_predicate(intended: str, rendered: str | None, expected: bool) -> None:
+ assert _is_same_length_reorder(tag_name="input", intended=intended, rendered=rendered) is expected
+
+
+def test_is_same_length_reorder_textarea_normalizes_newlines() -> None:
+ # After the permitted CRLF/CR->LF normalization the value is fully present -> not a reorder.
+ assert _is_same_length_reorder(tag_name="textarea", intended="ab\r\ncd", rendered="ab\ncd") is False
+
+
+async def test_reorder_repair_heals_and_continues() -> None:
+ # No autocomplete evidence + a same-length reorder read-back -> one atomic refill, exact confirm, then the
+ # existing helper contract: return None so the caller continues normally.
+ element, log, result = await _run_heal_capture(_R_INTENDED, readbacks=[_R_REORDER, _R_INTENDED], input_type="text")
+ element.input_fill.assert_awaited_once_with(text=_R_INTENDED)
+ assert log.info.call_args.kwargs["refill_confirmed"] is True
+ assert result is None
+
+
+async def test_reorder_repair_fail_closed_on_persistent_reorder() -> None:
+ element, log, result = await _run_heal_capture(_R_INTENDED, readbacks=[_R_REORDER, _R_REORDER], input_type="text")
+ element.input_fill.assert_awaited_once_with(text=_R_INTENDED)
+ assert log.info.call_args.kwargs["refill_confirmed"] is False
+ _assert_fails_closed(result, intended_length=len(_R_INTENDED))
+
+
+async def test_reorder_repair_skips_when_autocomplete_evidence_present() -> None:
+ element, result = await _run_heal(
+ _R_INTENDED, rendered=_R_REORDER, has_autocomplete_evidence=True, input_type="text"
+ )
+ element._read_back.assert_not_awaited() # gated out before any read-back
+ element.input_fill.assert_not_awaited()
+ assert result is None
+
+
+async def test_reorder_repair_skips_non_reorder_mismatch() -> None:
+ # Arbitrary (different-multiset) mismatch must not refill.
+ element, result = await _run_heal(_R_INTENDED, rendered="abcdeg", input_type="text")
+ element.input_fill.assert_not_awaited()
+ assert result is None
+
+
[email protected]("input_type", ["number", "date", "datetime-local", "time", "tel"])
+async def test_reorder_repair_skips_ineligible_input_types(input_type: str) -> None:
+ element, result = await _run_heal(_R_INTENDED, rendered=_R_REORDER, input_type=input_type)
+ element._read_back.assert_not_awaited()
+ element.input_fill.assert_not_awaited()
+ assert result is None
+
+
+async def test_reorder_repair_skips_maxlength_truncating() -> None:
+ element, result = await _run_heal(_R_INTENDED, rendered=_R_REORDER, input_type="text", maxlength="3")
+ element._read_back.assert_not_awaited()
+ element.input_fill.assert_not_awaited()
+ assert result is None
+
+
+async def test_reorder_repair_skips_secret_value() -> None:
+ element, result = await _run_heal(_R_INTENDED, rendered=_R_REORDER, input_type="text", is_secret_value=True)
+ element._read_back.assert_not_awaited()
+ element.input_fill.assert_not_awaited()
+ assert result is None
+
+
[email protected]("input_type", ["", "text", "search", "url", "email", "password"])
+async def test_reorder_repair_eligible_exact_value_types(input_type: str) -> None:
+ element, log, result = await _run_heal_capture(
+ _R_INTENDED, readbacks=[_R_REORDER, _R_INTENDED], input_type=input_type
+ )
+ element.input_fill.assert_awaited_once_with(text=_R_INTENDED)
+ assert result is None
+
+
+async def test_reorder_repair_eligible_textarea() -> None:
+ element, log, result = await _run_heal_capture(
+ _R_INTENDED, readbacks=[_R_REORDER, _R_INTENDED], tag_name="textarea", input_type=""
+ )
+ element.input_fill.assert_awaited_once_with(text=_R_INTENDED)
+ assert result is None
+
+
+async def test_single_call_evaluates_both_prefix_loss_and_reorder() -> None:
+ # The one heal call after input_sequentially evaluates BOTH signatures: a long round-trip-safe value whose
+ # read-back is a same-length permutation (not a lost-prefix suffix) is still detected and repaired.
+ element, log, result = await _run_heal_capture(_LONG, readbacks=[_LONG[1:] + _LONG[:1], _LONG], input_type="text")
+ element.input_fill.assert_awaited_once_with(text=_LONG)
+ assert log.info.call_args.kwargs["refill_confirmed"] is True
+ assert result is None
diff --git a/tests/unit/test_input_text_search_bar_skip.py b/tests/unit/test_input_text_search_bar_skip.py
index 70dc9adcf..bb3f4e2ce 100644
--- a/tests/unit/test_input_text_search_bar_skip.py
+++ b/tests/unit/test_input_text_search_bar_skip.py
@@ -13,12 +13,13 @@ from datetime import UTC, datetime
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from playwright._impl._errors import Error as PlaywrightError
from skyvern.forge.sdk.models import StepStatus
from skyvern.webeye.actions.actions import InputOrSelectContext, InputTextAction, KeypressAction
from skyvern.webeye.actions.handler import handle_input_text_action
from skyvern.webeye.actions.handler_utils import keys_include_enter, should_stop_batch_after_dropdown_select
-from skyvern.webeye.actions.responses import ActionSuccess
+from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess
from tests.unit.conftest import make_input_element_mock
from tests.unit.helpers import make_organization, make_step, make_task
@@ -150,3 +151,146 @@ async def test_plain_search_input_does_not_stop_batch() -> None:
results = await _run_search_bar_input(stop_flag=True, incremental=[])
assert len(results) == 1 and isinstance(results[0], ActionSuccess)
assert not results[0].skip_remaining_actions
+
+
+# --------------------------------------------------------------------------- #
+# Caller wiring for the single heal call after input_sequentially: at that seam the caller computes the live
+# structural autocomplete gate (is_auto_completion_input / _is_combobox_or_typeahead) and passes it into the one
+# heal call. A marker-less field with no autocomplete evidence and a reordered read-back gets one atomic refill
+# (then continues); a structurally genuine autocomplete is left typed. (The reorder detection/gating itself is
+# unit-tested in test_freetext_truncation_selfheal.)
+# --------------------------------------------------------------------------- #
+async def _run_reorder_scenario(
+ *, is_combobox: bool, incremental: list[dict], post_fill_value: str
+) -> tuple[list, MagicMock]:
+ el = make_input_element_mock(element_id="AADC") # no type attr -> round-trip-safe ""
+ el.is_auto_completion_input = AsyncMock(return_value=False)
+ state = {"filled": False}
+
+ async def _fill(*_a: object, **_k: object) -> None:
+ state["filled"] = True
+
+ el.input_fill = AsyncMock(side_effect=_fill)
+
+ async def _read(*_a: object, **_k: object) -> str:
+ return post_fill_value if state["filled"] else "654321" # sequential seam left a reordered read-back
+
+ dom_instance = MagicMock()
+ dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=el)
+ inc = MagicMock()
+ inc.start_listen_dom_increment = AsyncMock()
+ inc.stop_listen_dom_increment = AsyncMock()
+ inc.get_incremental_element_tree = AsyncMock(return_value=incremental)
+ skyvern_frame = MagicMock()
+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()
+ scraped_page = MagicMock()
+ scraped_page.id_to_element_dict = {"AADC": {"tagName": "input"}}
+ context = InputOrSelectContext(field="Account", is_search_bar=True, is_location_input=False)
+
+ with (
+ patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_instance),
+ patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance", new=AsyncMock(return_value=skyvern_frame)),
+ patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc),
+ patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(side_effect=_read)),
+ patch(
+ "skyvern.webeye.actions.handler.get_actual_value_of_parameter_if_secret_with_task", return_value="123456"
+ ),
+ patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=context)),
+ patch("skyvern.webeye.actions.handler._is_combobox_or_typeahead", new=AsyncMock(return_value=is_combobox)),
+ patch("skyvern.webeye.actions.handler._wait_custom_select_render_settle", new=AsyncMock()),
+ patch("skyvern.webeye.actions.handler.sequentially_select_from_dropdown", new=AsyncMock()),
+ ):
+ results = await handle_input_text_action(
+ action=_input(), page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP
+ )
+ return results, el
+
+
+async def test_reorder_repaired_when_no_autocomplete_evidence() -> None:
+ # Marker-less, no options, reordered read-back -> caller computes has_autocomplete_evidence=False and the
+ # reorder-repair call refills the intended value, then continues (ActionSuccess).
+ results, el = await _run_reorder_scenario(is_combobox=False, incremental=[], post_fill_value="123456")
+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)
+ el.input_fill.assert_awaited_once_with(text="123456")
+
+
+async def test_reorder_not_repaired_when_structural_combobox() -> None:
+ # A structurally genuine combobox -> has_autocomplete_evidence=True -> no reorder refill (typed preserved).
+ results, el = await _run_reorder_scenario(is_combobox=True, incremental=[], post_fill_value="123456")
+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)
+ el.input_fill.assert_not_awaited()
+
+
+async def test_reorder_repair_fail_closed_when_unconfirmed() -> None:
+ # The refill does not restore the value -> batch-terminal ActionFailure from the helper.
+ results, el = await _run_reorder_scenario(is_combobox=False, incremental=[], post_fill_value="654321")
+ assert len(results) == 1 and isinstance(results[0], ActionFailure)
+ assert results[0].skip_remaining_actions is True
+
+
+async def test_post_type_autocomplete_probe_failure_does_not_fail_successful_input() -> None:
+ # The control is a marker-less field pre-typing (structural probe returns False), but the final input event
+ # detaches/remounts it, so a structural autocomplete probe run AFTER input_sequentially would raise a live
+ # element-lifecycle error. That must not convert a successfully-typed-and-healed value into a failure: the
+ # autocomplete gate is taken from the pre-typing (attached) evidence, so the reordered read-back is repaired
+ # and the action continues (ActionSuccess). The probe raises with a message not in the incremental handler's
+ # tolerated navigation substrings, so on the pre-fix head it is re-raised and fails the action.
+ el = make_input_element_mock(element_id="AADC") # no type attr -> round-trip-safe ""
+ state = {"typed": False, "filled": False}
+
+ async def _type(*_a: object, **_k: object) -> None:
+ state["typed"] = True
+
+ el.input_sequentially = AsyncMock(side_effect=_type)
+
+ async def _fill(*_a: object, **_k: object) -> None:
+ state["filled"] = True
+
+ el.input_fill = AsyncMock(side_effect=_fill)
+
+ async def _auto_completion(*_a: object, **_k: object) -> bool:
+ if state["typed"]:
+ raise PlaywrightError("element is not attached to the DOM") # detached by the final input event
+ return False # attached pre-typing: no structural autocomplete markers
+
+ el.is_auto_completion_input = AsyncMock(side_effect=_auto_completion)
+
+ async def _read(*_a: object, **_k: object) -> str:
+ return "123456" if state["filled"] else "654321" # reordered read-back until the heal refills
+
+ dom_instance = MagicMock()
+ dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=el)
+ inc = MagicMock()
+ inc.start_listen_dom_increment = AsyncMock()
+ inc.stop_listen_dom_increment = AsyncMock()
+ inc.get_incremental_element_tree = AsyncMock(return_value=[])
+ skyvern_frame = MagicMock()
+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()
+ scraped_page = MagicMock()
+ scraped_page.id_to_element_dict = {"AADC": {"tagName": "input"}}
+ context = InputOrSelectContext(field="Account", is_search_bar=True, is_location_input=False)
+
+ with (
+ patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_instance),
+ patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance", new=AsyncMock(return_value=skyvern_frame)),
+ patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc),
+ patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(side_effect=_read)),
+ patch(
+ "skyvern.webeye.actions.handler.get_actual_value_of_parameter_if_secret_with_task", return_value="123456"
+ ),
+ patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=context)),
+ # Structural combobox probe stays False throughout -- isolates the new post-type probe (which calls
+ # is_auto_completion_input) from the unrelated pre-existing settle-block combobox probe.
+ patch("skyvern.webeye.actions.handler._is_combobox_or_typeahead", new=AsyncMock(return_value=False)),
+ patch("skyvern.webeye.actions.handler._wait_custom_select_render_settle", new=AsyncMock()),
+ patch("skyvern.webeye.actions.handler.sequentially_select_from_dropdown", new=AsyncMock()),
+ ):
+ results = await handle_input_text_action(
+ action=_input(), page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP
+ )
+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)
+ el.input_fill.assert_awaited_once_with(text="123456") # pre-typing evidence -> reorder repaired, not failed
diff --git a/tests/unit/test_input_text_tel_card_routing.py b/tests/unit/test_input_text_tel_card_routing.py
index 67388f57b..b43896e0f 100644
--- a/tests/unit/test_input_text_tel_card_routing.py
+++ b/tests/unit/test_input_text_tel_card_routing.py
@@ -897,6 +897,22 @@ async def test_atomic_fill_path_skips_truncation_heal() -> None:
heal.assert_not_awaited()
+async def test_atomic_fill_path_does_no_reorder_readback() -> None:
+ # An ordinary atomic-fill input is populated in one fill and never invokes the heal, so it pays no reorder
+ # (or prefix-loss) read-back. _observe_input_value is only reached from inside the heal, which is not called
+ # on the atomic-fill common path.
+ el = _mock_input({"type": "text", "autocomplete": None, "name": "full-name"})
+ observe = AsyncMock(return_value=(False, None))
+
+ with patch("skyvern.webeye.actions.handler._observe_input_value", new=observe):
+ results, *_ = await _run_input_text(el, "Ada Lovelace")
+
+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)
+ el.input_fill.assert_awaited_once_with("Ada Lovelace")
+ observe.assert_not_awaited() # no reorder (or prefix-loss) read-back on the atomic-fill common path
+
+
@pytest.mark.asyncio
async def test_search_bar_input_keeps_sequential_typing() -> None:
# A search-bar surfaces its options only as the value is typed, so it keeps the per-character seam.
diff --git a/tests/unit/webeye/test_screenshot_observability.py b/tests/unit/webeye/test_screenshot_observability.py
new file mode 100644
index 000000000..c5f079689
--- /dev/null
+++ b/tests/unit/webeye/test_screenshot_observability.py
@@ -0,0 +1,350 @@
+"""Structured, low-cardinality screenshot telemetry: arm/primitive/stage/outcome must be truthful.
+
+These tests fix the observability contract; they must not constrain capture selection or retry, which
+``test_screenshot_cdp_fallback.py`` owns.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+from io import BytesIO
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from PIL import Image
+from playwright.async_api import Error as PlaywrightError
+from playwright.async_api import Page
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+
+from skyvern.forge.sdk.settings_manager import SettingsManager
+from skyvern.webeye.browser_engine import SKYCDP_ENGINE_NAME
+from skyvern.webeye.utils import page as page_module
+from skyvern.webeye.utils.page import (
+ ScreenshotArm,
+ ScreenshotEligibility,
+ ScreenshotOutcome,
+ ScreenshotPrimitive,
+ ScreenshotStage,
+ _current_viewpoint_screenshot_helper,
+ _screenshot_observation_fields,
+)
+
+
+def _page() -> MagicMock:
+ page = MagicMock(spec=Page)
+ page.is_closed.return_value = False
+ page.url = "https://example.test/screenshot"
+ page.viewport_size = {"width": 80, "height": 60}
+ page.context.browser.browser_type.name = "chromium"
+ page.wait_for_load_state = AsyncMock()
+ page.screenshot = AsyncMock(side_effect=PlaywrightTimeoutError("waiting for fonts to load"))
+ buffer = BytesIO()
+ with Image.new("RGB", (80, 60), (20, 70, 180)) as image:
+ image.save(buffer, format="PNG")
+ session = AsyncMock()
+
+ async def send(method: str, params: dict) -> dict:
+ if method == "Runtime.evaluate":
+ return {"result": {"value": {"deviceScaleFactor": 1, "viewportScale": 1}}}
+ return {"data": base64.b64encode(buffer.getvalue()).decode()}
+
+ session.send.side_effect = send
+ page.context.new_cdp_session = AsyncMock(return_value=session)
+ return page
+
+
[email protected](autouse=True)
+def _settings(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(SettingsManager.get_settings(), "BROWSER_CURSOR_VISUALIZATION", False)
+ for name in ("SESSION", "CAPTURE", "DETACH"):
+ monkeypatch.setattr(page_module, f"CDP_RESCUE_{name}_TIMEOUT_SECONDS", 0.05)
+
+
+def _log_events(log: MagicMock) -> list[dict[str, Any]]:
+ events: list[dict[str, Any]] = []
+ for level in ("debug", "info", "warning", "error"):
+ for call in getattr(log, level).call_args_list:
+ events.append(call.kwargs)
+ return events
+
+
+def _observations(log: MagicMock) -> list[dict[str, Any]]:
+ return [kwargs for kwargs in _log_events(log) if "screenshot.outcome" in kwargs]
+
+
+# --- pure builder ---------------------------------------------------------------------------------
+
+
+def test_observation_fields_are_bounded_and_truthful() -> None:
+ fields = _screenshot_observation_fields(
+ arm=ScreenshotArm.CONTROL,
+ primitive=ScreenshotPrimitive.CDP_RESCUE,
+ stage=ScreenshotStage.CAPTURE,
+ outcome=ScreenshotOutcome.SUCCESS,
+ elapsed_ms=-5,
+ timeout_budget_ms=1234.7,
+ eligibility=ScreenshotEligibility.ELIGIBLE,
+ )
+ assert fields == {
+ "screenshot.arm": "control",
+ "screenshot.primitive": "cdp_rescue",
+ "screenshot.stage": "capture",
+ "screenshot.outcome": "success",
+ "screenshot.elapsed_ms": 0,
+ "screenshot.timeout_budget_ms": 1234,
+ "screenshot.eligibility": "eligible",
+ }
+
+
+def test_observation_fields_omit_optional_measures_when_absent() -> None:
+ fields = _screenshot_observation_fields(
+ arm=ScreenshotArm.CONTROL,
+ primitive=ScreenshotPrimitive.PLAYWRIGHT,
+ stage=ScreenshotStage.TERMINAL,
+ outcome=ScreenshotOutcome.ERROR,
+ )
+ assert set(fields) == {
+ "screenshot.arm",
+ "screenshot.primitive",
+ "screenshot.stage",
+ "screenshot.outcome",
+ }
+
+
+# --- CDP rescue stage matrix ----------------------------------------------------------------------
+
+
+async def test_cdp_rescue_success_reports_validation_after_file_write(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ log = MagicMock()
+ monkeypatch.setattr(page_module, "LOG", log)
+ page = _page()
+ path = tmp_path / "nested" / "screenshot.png"
+ result = await _current_viewpoint_screenshot_helper(page, file_path=str(path))
+ assert path.read_bytes() == result
+ cdp = [o for o in _observations(log) if o.get("screenshot.primitive") == "cdp_rescue"]
+ assert [(o["screenshot.arm"], o["screenshot.stage"], o["screenshot.outcome"]) for o in cdp] == [
+ ("control", "validation", "success")
+ ]
+
+
[email protected]("stage", ["attach", "geometry", "capture", "validation"])
+ "failure,expected_outcome",
+ [
+ (TimeoutError("deadline exceeded"), "timeout"),
+ (PlaywrightTimeoutError("protocol timeout"), "timeout"),
+ (PlaywrightError("protocol failure"), "error"),
+ ],
+)
+async def test_cdp_rescue_failure_reports_stage_and_outcome(
+ stage: str, failure: Exception, expected_outcome: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ log = MagicMock()
+ monkeypatch.setattr(page_module, "LOG", log)
+ page = _page()
+ original_timeout = page.screenshot.side_effect
+ session = page.context.new_cdp_session.return_value
+ if stage == "attach":
+ page.context.new_cdp_session.side_effect = failure
+ elif stage in {"geometry", "capture"}:
+
+ async def fail(method: str, params: dict) -> dict:
+ if stage == "geometry" or method == "Page.captureScreenshot":
+ raise failure
+ return {"result": {"value": {"deviceScaleFactor": 1, "viewportScale": 1}}}
+
+ session.send.side_effect = fail
+ else:
+ monkeypatch.setattr(Path, "write_bytes", MagicMock(side_effect=failure))
+
+ with pytest.raises(page_module.FailedToTakeScreenshot) as raised:
+ await _current_viewpoint_screenshot_helper(page, file_path=str(tmp_path / "screenshot.png"))
+ assert raised.value.__cause__ is original_timeout
+ cdp = [o for o in _observations(log) if o.get("screenshot.primitive") == "cdp_rescue"]
+ assert [(o["screenshot.stage"], o["screenshot.outcome"]) for o in cdp] == [(stage, expected_outcome)]
+
+
+async def test_cdp_rescue_invalid_png_reports_validation_error(monkeypatch: pytest.MonkeyPatch) -> None:
+ log = MagicMock()
+ monkeypatch.setattr(page_module, "LOG", log)
+ page = _page()
+ session = page.context.new_cdp_session.return_value
+
+ async def bad_png(method: str, params: dict) -> dict:
+ if method == "Runtime.evaluate":
+ return {"result": {"value": {"deviceScaleFactor": 1, "viewportScale": 1}}}
+ return {"data": base64.b64encode(b"not-a-png").decode()}
+
+ session.send.side_effect = bad_png
+ with pytest.raises(page_module.FailedToTakeScreenshot):
+ await _current_viewpoint_screenshot_helper(page)
+ cdp = [o for o in _observations(log) if o.get("screenshot.primitive") == "cdp_rescue"]
+ assert [(o["screenshot.stage"], o["screenshot.outcome"]) for o in cdp] == [("validation", "error")]
+
+
[email protected]("cancel_stage", [None, "capture", "detach"])
[email protected]("failure", ["timeout", "error"])
+async def test_detach_failure_is_observable_without_changing_capture_or_cancellation(
+ cancel_stage: str | None, failure: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ log = MagicMock()
+ monkeypatch.setattr(page_module, "LOG", log)
+ page = _page()
+ session = page.context.new_cdp_session.return_value
+ capture_entered = asyncio.Event()
+ detach_entered = asyncio.Event()
+ release_detach = asyncio.Event()
+ detach_finished = asyncio.Event()
+
+ async def capture(*args: object, **kwargs: object) -> None:
+ capture_entered.set()
+ await asyncio.Event().wait()
+
+ async def detach() -> None:
+ detach_entered.set()
+ try:
+ if failure == "timeout":
+ await asyncio.Event().wait()
+ else:
+ await release_detach.wait()
+ raise PlaywrightError("detach failed")
+ finally:
+ detach_finished.set()
+
+ if cancel_stage == "capture":
+ session.send.side_effect = capture
+ session.detach.side_effect = detach
+ existing_tasks = asyncio.all_tasks()
+ task = asyncio.create_task(_current_viewpoint_screenshot_helper(page))
+ try:
+ if cancel_stage == "capture":
+ await asyncio.wait_for(capture_entered.wait(), timeout=0.5)
+ task.cancel()
+ await asyncio.wait_for(detach_entered.wait(), timeout=0.5)
+ if cancel_stage == "detach":
+ task.cancel()
+ release_detach.set()
+ if cancel_stage is not None:
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, timeout=0.5)
+ else:
+ assert (await asyncio.wait_for(task, timeout=0.5)).startswith(page_module._PNG_SIGNATURE)
+ assert detach_finished.is_set()
+ assert not (asyncio.all_tasks() - existing_tasks)
+ detach_observations = [o for o in _observations(log) if o.get("screenshot.stage") == "detach"]
+ assert len(detach_observations) == 1
+ observation = detach_observations[0]
+ assert observation["screenshot.arm"] == "control"
+ assert observation["screenshot.primitive"] == "cdp_rescue"
+ assert observation["screenshot.outcome"] == failure
+ assert observation["screenshot.timeout_budget_ms"] == 50
+ assert observation["screenshot.elapsed_ms"] >= 0
+ finally:
+ release_detach.set()
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+
+
+async def test_scaled_viewport_decline_emits_declined_outcome(monkeypatch: pytest.MonkeyPatch) -> None:
+ log = MagicMock()
+ monkeypatch.setattr(page_module, "LOG", log)
+ page = _page()
+ session = page.context.new_cdp_session.return_value
+ session.send.side_effect = None
+ session.send.return_value = {"result": {"value": {"deviceScaleFactor": 2, "viewportScale": 1}}}
+ page.screenshot.side_effect = [PlaywrightTimeoutError("first"), b"animation-retry-bytes"]
+ assert await _current_viewpoint_screenshot_helper(page) == b"animation-retry-bytes"
+ cdp = [o for o in _observations(log) if o.get("screenshot.primitive") == "cdp_rescue"]
+ assert any(o["screenshot.outcome"] == "declined" and o["screenshot.stage"] == "geometry" for o in cdp)
+
+
+# --- eligibility ----------------------------------------------------------------------------------
+
+
+ "kind,reason",
+ [
+ ("full_page", "ineligible_full_page"),
+ ("firefox", "ineligible_browser"),
+ ("unknown_browser", "ineligible_browser"),
+ ("skycdp", "ineligible_engine"),
+ ],
+)
+async def test_ineligible_paths_emit_decline_reason(kind: str, reason: str, monkeypatch: pytest.MonkeyPatch) -> None:
+ log = MagicMock()
+ monkeypatch.setattr(page_module, "LOG", log)
+ page = _page()
+ if kind == "firefox":
+ page.context.browser.browser_type.name = "firefox"
+ elif kind == "unknown_browser":
+ page.context.browser = None
+ selection = None
+ if kind == "skycdp":
+ selection = SimpleNamespace(
+ name=SKYCDP_ENGINE_NAME,
+ is_engine_error=lambda exc: isinstance(exc, PlaywrightError),
+ is_engine_timeout_error=lambda exc: isinstance(exc, PlaywrightTimeoutError),
+ )
+ page.screenshot.side_effect = [PlaywrightTimeoutError("first"), b"playwright-retry"]
+ assert (
+ await _current_viewpoint_screenshot_helper(page, full_page=kind == "full_page", engine_selection=selection)
+ == b"playwright-retry"
+ )
+ eligibilities = {o.get("screenshot.eligibility") for o in _observations(log)}
+ assert reason in eligibilities
+
+
+async def test_eligible_timeout_records_eligible(monkeypatch: pytest.MonkeyPatch) -> None:
+ log = MagicMock()
+ monkeypatch.setattr(page_module, "LOG", log)
+ page = _page()
+ await _current_viewpoint_screenshot_helper(page)
+ eligibilities = {o.get("screenshot.eligibility") for o in _observations(log)}
+ assert "eligible" in eligibilities
+
+
+# --- terminal outcome -----------------------------------------------------------------------------
+
+
+async def test_terminal_success_emits_playwright_success(monkeypatch: pytest.MonkeyPatch) -> None:
+ log = MagicMock()
+ monkeypatch.setattr(page_module, "LOG", log)
+ page = _page()
+ page.screenshot.side_effect = None
+ page.screenshot.return_value = b"ordinary-frame"
+ assert await _current_viewpoint_screenshot_helper(page, timeout=123) == b"ordinary-frame"
+ terminal = [o for o in _observations(log) if o.get("screenshot.stage") == "terminal"]
+ assert any(
+ o["screenshot.outcome"] == "success"
+ and o["screenshot.arm"] == "control"
+ and o["screenshot.timeout_budget_ms"] == 123
+ for o in terminal
+ )
+
+
+async def test_terminal_timeout_emits_timeout_outcome(monkeypatch: pytest.MonkeyPatch) -> None:
+ log = MagicMock()
+ monkeypatch.setattr(page_module, "LOG", log)
+ page = _page()
+ session = page.context.new_cdp_session.return_value
+ session.send.side_effect = PlaywrightError("CDP failure")
+ with pytest.raises(page_module.FailedToTakeScreenshot):
+ await _current_viewpoint_screenshot_helper(page)
+ terminal = [o for o in _observations(log) if o.get("screenshot.stage") == "terminal"]
+ assert any(o["screenshot.outcome"] == "timeout" for o in terminal)