Skip to content

Automate browser based workflows with AI

Raise the file-parser LLM call deadline from 30 s to 250 s (#8888)

Shuchang Zheng95fc8a7e221b
diff --git a/skyvern/constants.py b/skyvern/constants.py
index b01ddf07c..60260a810 100644
--- a/skyvern/constants.py
+++ b/skyvern/constants.py
@@ -161,6 +161,13 @@ PDF_OCR_PAGE_CONCURRENCY = 5
# alone cannot bound it; this keeps a pathological document to one failed block instead of
# letting it consume the run's whole activity budget.
FILE_PARSE_STEP_TIMEOUT_SECONDS = 300
+# Wall-clock ceiling for one file-parser LLM call (page OCR or structured extraction); the slowest legitimate call seen
+# is a ~245 s large-output extraction. The LLM client's timeout only bounds the gap between received bytes, which a
+# gateway's keep-alive bytes keep resetting on a stalled call.
+FILE_PARSER_LLM_CALL_TIMEOUT_SECONDS = 250
+# Wall-clock ceiling for OCR'ing every page of one scanned PDF: MAX_PDF_OCR_PAGES in rounds of PDF_OCR_PAGE_CONCURRENCY,
+# one retry per page, at the ~30 s a slow OCR call takes. It, not the per-call ceiling, bounds a scan whose pages stall.
+PDF_OCR_TOTAL_TIMEOUT_SECONDS = 600
MAX_IMAGE_MESSAGES = 10
SCROLL_AMOUNT_MULTIPLIER = 100
EXTRACT_ACTION_SCROLL_AMOUNT = 500 # pixels per scroll action from extract-action prompt
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 7460dd2f7..42a0b7c6f 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -28,7 +28,7 @@ import unicodedata
import uuid
import zipfile
from collections import defaultdict, deque
-from collections.abc import Iterator, Mapping, Sequence
+from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from dataclasses import dataclass, replace
from datetime import UTC, date, datetime, time
from email.message import EmailMessage
@@ -81,12 +81,14 @@ from skyvern.constants import (
AZURE_BLOB_STORAGE_MAX_UPLOAD_FILE_COUNT,
BROWSER_DOWNLOADING_SUFFIX,
FILE_PARSE_STEP_TIMEOUT_SECONDS,
+ FILE_PARSER_LLM_CALL_TIMEOUT_SECONDS,
GET_DOWNLOADED_FILES_TIMEOUT,
MAX_FILE_PARSE_INPUT_TOKENS,
MAX_PDF_OCR_PAGES,
MAX_UPLOAD_FILE_COUNT,
NAVIGATION_MAX_RETRY_TIME,
PDF_OCR_PAGE_CONCURRENCY,
+ PDF_OCR_TOTAL_TIMEOUT_SECONDS,
PROXY_TRANSPORT_NAV_ERRORS,
SAVE_DOWNLOADED_FILES_TIMEOUT,
)
@@ -13615,6 +13617,32 @@ async def _run_blocking_parse_step(
raise FileParseTimeout(file_url=file_url, step=step, timeout_seconds=FILE_PARSE_STEP_TIMEOUT_SECONDS)
+class _FileParserLLMTimeout(TimeoutError):
+ pass
+
+
+async def _file_parser_llm_deadline(step: str, file_url: str, llm_key: str | None) -> AsyncIterator[None]:
+ """Bound one file-parser LLM call by wall-clock time, so a stalled provider fails the block."""
+ try:
+ async with asyncio.timeout(FILE_PARSER_LLM_CALL_TIMEOUT_SECONDS) as deadline:
+ yield
+ except TimeoutError:
+ # A TimeoutError this scope did not raise came from inside the handler; relabelling it would bury its cause.
+ if not deadline.expired():
+ raise
+ LOG.warning(
+ "File parser LLM call exceeded its time budget",
+ step=step,
+ timeout_seconds=FILE_PARSER_LLM_CALL_TIMEOUT_SECONDS,
+ file_url=_log_safe_url(file_url),
+ llm_key=llm_key,
+ )
+ raise _FileParserLLMTimeout(
+ f"The model did not answer within {FILE_PARSER_LLM_CALL_TIMEOUT_SECONDS} seconds during {step}."
+ ) from None
+
+
# csv.field_size_limit is process-global. Raising it once at import (rather than
# set/restore around each parse) avoids a race between concurrent CSV parses on
# separate worker threads stepping on each other's limit.
@@ -14180,6 +14208,7 @@ class FileParserBlock(Block):
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
self.override_llm_key_for_organization(organization_id), default=default_handler
)
+ llm_key = _handler_llm_key(llm_api_handler)
semaphore = asyncio.Semaphore(PDF_OCR_PAGE_CONCURRENCY)
async def _ocr_page(page_image: bytes) -> str:
@@ -14189,15 +14218,16 @@ class FileParserBlock(Block):
try:
# OCR transcription intentionally skips system_prompt; it still applies
# to the downstream extract-information-from-file-text call.
- llm_response = await llm_api_handler(
- prompt=prompt_for_attempt,
- prompt_name="extract-text-from-image",
- screenshots=[page_image],
- # Schema validation must inspect the raw parsed root; dict coercion can hide bad OCR JSON.
- force_dict=False,
- workflow_run_block_id=workflow_run_block_id,
- organization_id=organization_id,
- )
+ async with _file_parser_llm_deadline("PDF page OCR", self.file_url, llm_key=llm_key):
+ llm_response = await llm_api_handler(
+ prompt=prompt_for_attempt,
+ prompt_name="extract-text-from-image",
+ screenshots=[page_image],
+ # Schema validation must inspect the raw parsed root; dict coercion can hide bad OCR JSON.
+ force_dict=False,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
except (InvalidLLMResponseFormat, InvalidLLMResponseType) as e:
failure_reason = _llm_response_format_failure_reason(e)
will_retry = attempt + 1 < self.ocr_validation_max_attempts
@@ -14232,11 +14262,31 @@ class FileParserBlock(Block):
prompt_for_attempt = self._build_ocr_validation_retry_prompt(llm_prompt, ocr_failure_reason)
raise RuntimeError("OCR retry loop exhausted without returning or raising.")
- page_results = await asyncio.gather(
- *(_ocr_page(page_image) for page_image in page_images),
- return_exceptions=True,
- )
+ try:
+ async with asyncio.timeout(PDF_OCR_TOTAL_TIMEOUT_SECONDS) as ocr_deadline:
+ page_results = await asyncio.gather(
+ *(_ocr_page(page_image) for page_image in page_images),
+ return_exceptions=True,
+ )
+ except TimeoutError:
+ if not ocr_deadline.expired():
+ raise
+ LOG.warning(
+ "File parser PDF OCR exceeded its time budget",
+ timeout_seconds=PDF_OCR_TOTAL_TIMEOUT_SECONDS,
+ pages=len(page_images),
+ file_url=_log_safe_url(self.file_url),
+ llm_key=llm_key,
+ )
+ raise _FileParserLLMTimeout(
+ f"The model did not transcribe the PDF's {len(page_images)} pages within "
+ f"{PDF_OCR_TOTAL_TIMEOUT_SECONDS} seconds."
+ ) from None
+ # A page the model never answered would silently drop its text from the extraction, so it fails the block.
+ stalled_pages = [r for r in page_results if isinstance(r, _FileParserLLMTimeout)]
+ if stalled_pages:
+ raise stalled_pages[0]
# A total OCR outage must fail the block, not record an empty success — match the
# prior single-call path, which propagated OCR errors. Partial failures are skipped below.
errors = [r for r in page_results if isinstance(r, BaseException)]
@@ -14305,20 +14355,22 @@ class FileParserBlock(Block):
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(
self.override_llm_key_for_organization(organization_id), default=default_handler
)
+ llm_key = _handler_llm_key(llm_api_handler)
# OCR transcription intentionally skips system_prompt — see
# _parse_pdf_file_with_vision_ocr for rationale.
prompt_for_attempt = llm_prompt
for attempt in range(self.ocr_validation_max_attempts):
try:
- llm_response = await llm_api_handler(
- prompt=prompt_for_attempt,
- prompt_name="extract-text-from-image",
- screenshots=[image_bytes],
- # Schema validation must inspect the raw parsed root; dict coercion can hide bad OCR JSON.
- force_dict=False,
- workflow_run_block_id=workflow_run_block_id,
- organization_id=organization_id,
- )
+ async with _file_parser_llm_deadline("image OCR", self.file_url, llm_key=llm_key):
+ llm_response = await llm_api_handler(
+ prompt=prompt_for_attempt,
+ prompt_name="extract-text-from-image",
+ screenshots=[image_bytes],
+ # Schema validation must inspect the raw parsed root; dict coercion can hide bad OCR JSON.
+ force_dict=False,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
except (InvalidLLMResponseFormat, InvalidLLMResponseType) as e:
failure_reason = _llm_response_format_failure_reason(e)
will_retry = attempt + 1 < self.ocr_validation_max_attempts
@@ -14673,15 +14725,16 @@ class FileParserBlock(Block):
for attempt in range(self.schema_validation_max_attempts):
telemetry.extraction_attempts = attempt + 1
try:
- llm_response = await llm_api_handler(
- prompt=prompt_for_attempt,
- prompt_name="extract-information-from-file-text",
- # Schema validation must inspect the raw parsed root; dict coercion can hide wrong-root responses.
- force_dict=False,
- system_prompt=self.workflow_system_prompt,
- workflow_run_block_id=workflow_run_block_id,
- organization_id=organization_id,
- )
+ async with _file_parser_llm_deadline("AI extraction", self.file_url, llm_key=telemetry.llm_key):
+ llm_response = await llm_api_handler(
+ prompt=prompt_for_attempt,
+ prompt_name="extract-information-from-file-text",
+ # Schema validation must inspect the raw parsed root; dict coercion can hide wrong-root responses.
+ force_dict=False,
+ system_prompt=self.workflow_system_prompt,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
except (InvalidLLMResponseFormat, InvalidLLMResponseType) as e:
failure_reason = _llm_response_format_failure_reason(e)
will_retry = attempt + 1 < self.schema_validation_max_attempts
@@ -15173,15 +15226,27 @@ class PDFParserBlock(Block):
)
llm_response: dict[str, Any] | list | str | None = None
+ llm_api_handler = get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)
+ llm_key = _handler_llm_key(llm_api_handler)
prompt_for_attempt = llm_prompt
for attempt in range(self.schema_validation_max_attempts):
try:
- llm_response = await get_org_aware_primary_llm_api_handler(default=app.LLM_API_HANDLER)(
- prompt=prompt_for_attempt,
- prompt_name="extract-information-from-file-text",
- # Schema validation must inspect the raw parsed root; dict coercion can hide wrong-root responses.
- force_dict=False,
- system_prompt=self.workflow_system_prompt,
+ async with _file_parser_llm_deadline("AI extraction", self.file_url, llm_key=llm_key):
+ llm_response = await llm_api_handler(
+ prompt=prompt_for_attempt,
+ prompt_name="extract-information-from-file-text",
+ # Schema validation must inspect the raw parsed root; dict coercion can hide wrong-root responses.
+ force_dict=False,
+ system_prompt=self.workflow_system_prompt,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ except _FileParserLLMTimeout as e:
+ return await self.build_block_result(
+ success=False,
+ failure_reason=str(e),
+ output_parameter_value=None,
+ status=BlockStatus.failed,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
)
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 2c43ac3b5..4eff2f8a0 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -8230,6 +8230,9 @@ def _disambiguate_digest_bodies(
OBSERVE_GROUP_TEXT_TOTAL_CAP = 4000
+# Points at a read, never at a position: the text beside an unbound group is often a hint, an error or another
+# question.
+_NO_QUESTION_MARKER = "(no question found; read the question in get_html before answering)"
# Display width of each masked-then-capped field of the observe digest. Every render site reads its
# width here, so the retain margin below is always sized for the widest window.
OBSERVE_DISPLAY_WIDTHS = {
@@ -8826,11 +8829,107 @@ async (__tv3TextArgs) => {
}
return _normText(out);
};
+ const _TITLE_SEL = 'h1,h2,h3,h4,h5,h6,[role~=heading],legend';
+ const _NOTE_SEL = '[role~=alert],[role~=status],[aria-live]:not([aria-live=off])';
+ const _END = {};
+ const _NOTE = {};
+ // Every visible character, no depth cap: a capped read of a deep question looks empty and the walk moves on.
+ const _allText = (el) => {
+ if (_unseen(el)) return '';
+ let out = '';
+ for (let c = _firstChildOf.call(el); c; c = _nextOf.call(c)) {
+ const kind = _nodeTypeOf.call(c);
+ if (kind === 3) out += ' ' + _contentOf.call(c);
+ else if (kind === 1) out += ' ' + _allText(c);
+ }
+ return _normText(out);
+ };
+ const _noteIds = (g, el) => {
+ const ids = new Set();
+ const add = (n) => { for (const a of ['aria-describedby', 'aria-errormessage']) for (const id of (n.getAttribute(a) || '').split(/\s+/)) if (id) ids.add(id); };
+ add(el);
+ if (g.box) { add(g.box); for (const c of _qsa.call(g.box, _CHOICE_SEL)) add(c); }
+ return ids;
+ };
+
+ // Built once per call: under each composed ancestor, every control observe enumerated (those it counts as
+ // omitted included).
+ let _listedUnder = null;
+ const _listedIn = (n) => {
+ if (_listedUnder) return _listedUnder.get(n) || [];
+ const m = new Map();
+ for (const e of els) {
+ for (let p = _composedParent(e.el); p; p = _composedParent(p)) {
+ const l = m.get(p);
+ if (l) l.push(e.el); else m.set(p, [e.el]);
+ }
+ }
+ // Published only once whole, so a throw mid-build leaves it null for the next call.
+ return (_listedUnder = m).get(n) || [];
+ };
+ const _holdsAnotherControl = (card, el, g, kin) =>
+ _listedIn(card).some((c) => !_matches.call(c, kin) || !(_inGroup(c, el, g, kin) || _proxied(c, kin)));
+ // One block of a card: null when absent or empty, _NOTE for a live region (or a block that is only one), _END
+ // for a heading, else its text.
+ const _cardBlock = (c) => {
+ const kind = _nodeTypeOf.call(c);
+ if (kind === 3) { const t = _normText(_contentOf.call(c)); return t.length < 2 ? null : t; }
+ if (kind !== 1) return null;
+ const t = _allText(c);
+ if (t.length < 2) return null;
+ const note = _matches.call(c, _NOTE_SEL) ? c : _qs.call(c, _NOTE_SEL);
+ if (note && (note === c || _allText(note) === t)) return _NOTE;
+ return _matches.call(c, _TITLE_SEL) || _qs.call(c, _TITLE_SEL) ? _END : t;
+ };
+ const _fieldOrChoice = (c) => { const k = _controlsOf(c); return k.nonChoice || k.choices.length > 0; };
+ // A label placed before the option it names is that option's caption, not a question.
+ const _beforeOwnOption = (c, el, g, kin) => {
+ if (_nodeTypeOf.call(c) !== 1 || !_matches.call(c, 'label')) return false;
+ try { const x = c.control; return !!x && _matches.call(x, kin) && _inGroup(x, el, g, kin); } catch (e) { return false; }
+ };
+ // Exactly one candidate block across the whole card, read in order from the options box up to the largest
+ // ancestor holding nothing but this group. At the box, content after the first option is its captions, and a
+ // level with content but no candidate is never climbed past.
+ const _cardQuestion = (el, g) => {
+ const kin = g.radio ? _RADIO_SEL : _CHECK_SEL;
+ if (_noteIds(g, el).size) return '';
+ let found = '';
+ let below = false;
+ for (let card = g.box, inner = null; card; inner = card, card = _parentOf.call(card)) {
+ if (_openShadowOf(card)) break;
+ if (_holdsAnotherControl(card, el, g, kin)) {
+ // The level that ends the card: text beside its own link or button, back to the previous answered field, may be
+ // a question row the control split off the card.
+ for (let c = inner && _prevOf.call(inner); c && !(_nodeTypeOf.call(c) === 1 && _fieldOrChoice(c)); c = _prevOf.call(c)) {
+ if (_nodeTypeOf.call(c) === 1 && _listedIn(c).length && _cardBlock(c) !== null) return '';
+ }
+ break;
+ }
+ let after = false;
+ let here = false;
+ for (let c = _firstChildOf.call(card); c; c = _nextOf.call(c)) {
+ if (!after && (inner ? c === inner : _nodeTypeOf.call(c) === 1 && _hasChoice(c))) { after = true; continue; }
+ if (after && !inner) break;
+ if (!inner && _beforeOwnOption(c, el, g, kin)) continue;
+ const b = _cardBlock(c);
+ if (b === null) continue;
+ if (b === _END || (b !== _NOTE && (after || found || below))) return '';
+ here = true;
+ if (b !== _NOTE) found = b;
+ }
+ below = below || here;
+ }
+ return found && !_captionsOnly(g.box, found) ? found : '';
+ };
const _choiceQuestion = (el, g) => {
const declared = _declaredGroupName(el, g);
if (declared || !g.box || g.multi) return declared;
const t = _boxQuestion(g.box, g);
- if (!t || _matches.call(g.box, 'tr,[role=row]')) return t;
+ if (_matches.call(g.box, 'tr,[role=row]')) return t;
+ if (!t) {
+ if (g.entry) g.entry.nearby = true;
+ return _cardQuestion(el, g);
+ }
return _namesOnlyThisGroup(g.box, el, g) && !_captionsOnly(g.box, t) ? t : '';
};
const _isAutocomplete = """
@@ -10159,9 +10258,10 @@ async (__tv3TextArgs) => {
if (el.type === 'radio' || role === 'radio') {
// A fresh object per element on purpose: a control with no group entry is its own group.
const entry = (g && g.entry) || {};
- // Found means the page gave a question, printed or not: the page budget or the same-text dedupe
- // may hold it back. A description is a hint or an error, never the group's question.
- if (ownGroup) entry.found = true;
+ // Found means the page gave a question, even one the budget or the same-text dedupe held back; a question read
+ // from the card counts only once shown, so a withheld one keeps the marker.
+ const withheld = !rec.group && groupTotal + gt.length > _GROUP_TEXT_TOTAL_CAP;
+ if (ownGroup && !(withheld && entry.nearby)) entry.found = true;
else if (!entry.found) {
if (!entry.missing) entry.missing = ++radioGroupsMissing;
rec.group_missing = entry.missing;
@@ -12118,6 +12218,8 @@ async def _observable_child_frames(page: Any) -> tuple[list[Any], int, int]:
# "the main frame" and must not decay into it: the main frame is a real document a selector will happily
# resolve in.
_UNKNOWN_REALM = object()
+# Set by the click echo for one call; the click path records the kind of choice control it found under the target.
+_CLICK_CHOICE_KIND: ContextVar[list[str] | None] = ContextVar("taskv3_click_choice_kind", default=None)
class _RealmChangedDuringRead(Exception):
@@ -13282,7 +13384,7 @@ def build_browser_tools(
extra += f" group={_field(e['group'], OBSERVE_DISPLAY_WIDTHS['group'])}"
elif e.get("group_missing") and missing_key not in missing_groups_shown:
missing_groups_shown.add(missing_key)
- extra += " group=(no question found)"
+ extra += f" group={_NO_QUESTION_MARKER}"
if e.get("a11yRemoved"):
# What the page declared, not a prediction: aria-hidden with a negative tabindex does
# not stop a click landing or a field filling (measured), and a control the page
@@ -14997,6 +15099,8 @@ def build_browser_tools(
await asyncio.sleep(0.1)
# Enabling can re-render or move the control, so the label point is measured again after the wait.
pre_click_arg = await _probe_arg(page, selector)
+ if isinstance(skin_probe, dict) and "radio" in skin_probe and (kinds := _CLICK_CHOICE_KIND.get()) is not None:
+ kinds.append("radio" if skin_probe["radio"] else "checkbox")
if isinstance(skin_probe, dict) and skin_probe.get("drawnLabel") and (label_over_control or _acted_realm):
# A drawn input under its own label is already force-clicked below, and a frame's label point is
# frame-relative, so neither takes the label route.
@@ -21618,6 +21722,7 @@ def build_browser_tools(
except Exception:
pass
entry["handle"] = matches[0]
+ entry["reresolved"] = True
return selector, None
async def _realm_for_typed_selector(page: Any, selector: str) -> tuple[Any, ToolResult | None]:
@@ -21923,6 +22028,36 @@ def build_browser_tools(
_pick_realms[key] = realm if prior == realm or gone else _UNKNOWN_REALM
return entry["question"], entry.get("option") or ""
+ def _with_pick_echo(handler: ToolHandler) -> ToolHandler:
+ # Names the question observe bound the clicked option to, so a wrong pairing is visible in the result.
+ @functools.wraps(handler)
+ async def wrapped(args: dict[str, Any]) -> ToolResult:
+ ref = REF_SELECTOR_RE.match(str(args.get("selector") or "")) if args.get("mark") is None else None
+ entry = (_observe_manifest.get(int(ref.group(1))) if ref else None) or {}
+ kinds: list[str] = []
+ token = _CLICK_CHOICE_KIND.set(kinds)
+ try:
+ result = await handler(args)
+ finally:
+ _CLICK_CHOICE_KIND.reset(token)
+ kind = entry.get("choice") or (kinds[0] if kinds else "")
+ question = None if entry.get("reresolved") else entry.get("question")
+ # A lone checkbox is mostly a consent that names itself; an unknown question there says nothing.
+ if result.status != "ok" or result.entry_unconfirmed or not kind or (kind == "checkbox" and not question):
+ return result
+ mark = args.get("mark")
+ option = entry.get("option") or _mask_refs(
+ str(_look_manifest.get(mark, {}).get("label") or "") if isinstance(mark, int) else ""
+ )
+ width = OBSERVE_DISPLAY_WIDTHS["group"]
+ what = repr(str(option)[:width]) if option else "this option"
+ line = f"{'picked' if kind == 'radio' else 'clicked'} {what} for " + (
+ f"question {question[:width]!r}" if question else "an unknown question"
+ )
+ return dataclasses.replace(result, content=f"{result.content}\n{line}")
+
+ return wrapped
+
async def _choice_checked(selector: str) -> bool | None:
realm = _pick_realms.get(selector, (None, ""))
if realm is _UNKNOWN_REALM:
@@ -22190,6 +22325,7 @@ def build_browser_tools(
# Again outermost, for the refusals the mark and ref resolvers raise themselves.
_tool_spec.handler = _with_refusal_result(_tool_spec.handler)
if _tool_spec.name == "click":
+ _tool_spec.handler = _with_pick_echo(_tool_spec.handler)
_tool_spec.toggle_probe = _click_targets_toggle
_tool_spec.address_probe = _click_address
_tool_spec.choice_probe = _click_choice
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index c9b892676..92063c06d 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -4102,6 +4102,8 @@ async def _select_deterministic_normal_option(
action: actions.SelectOptionAction,
skyvern_element: SkyvernElement,
locator: Locator,
+ task: Task,
+ target_value: str,
matched_label: str | None,
matched_value: str | None,
matched_index: int | None,
@@ -4113,6 +4115,11 @@ async def _select_deterministic_normal_option(
value = matched_value if matched_value is not None else matched_label
if value is not None and not _select_value_is_ambiguous(skyvern_element.get_options(), value):
+ cancellation_failure = await _native_select_cancellation_failure(
+ task=task, target_value=target_value, matched_label=matched_label
+ )
+ if cancellation_failure is not None:
+ return [cancellation_failure]
try:
await locator.select_option(
value=value,
@@ -4130,6 +4137,11 @@ async def _select_deterministic_normal_option(
)
if not is_success and matched_label is not None and matched_label != value:
+ cancellation_failure = await _native_select_cancellation_failure(
+ task=task, target_value=target_value, matched_label=matched_label
+ )
+ if cancellation_failure is not None:
+ return [cancellation_failure]
try:
await locator.select_option(
label=matched_label,
@@ -4155,6 +4167,11 @@ async def _select_deterministic_normal_option(
locator=locator,
)
else:
+ cancellation_failure = await _native_select_cancellation_failure(
+ task=task, target_value=target_value, matched_label=matched_label
+ )
+ if cancellation_failure is not None:
+ return [cancellation_failure]
try:
await locator.select_option(
index=matched_index,
@@ -10513,9 +10530,18 @@ async def handle_select_option_action(
# normal_select can raise before returning (e.g. an LLM/provider error). Don't lose
# the styled-dropdown fallback below — record the failure and keep going.
LOG.warning("normal_select raised; falling back to blocking-element detection", exc_info=True)
+ cancellation_failure = await _native_select_cancellation_failure(
+ task=task,
+ target_value=_select_option_target_value(action.option) or "",
+ matched_label=None,
+ )
+ if cancellation_failure is not None:
+ return [cancellation_failure]
normal_select_result = [ActionFailure(exception=e)]
if _normal_select_successful(normal_select_result):
return normal_select_result
+ if normal_select_result and normal_select_result[-1].skip_remaining_actions:
+ return normal_select_result
blocking_element: SkyvernElement | None = None
exist = False
@@ -13973,6 +13999,22 @@ def _is_terminal_custom_select_failure(action_result: ActionResult | None) -> bo
)
+async def _native_select_cancellation_failure(
+ *,
+ task: Task,
+ target_value: str,
+ matched_label: str | None,
+) -> ActionFailure | None:
+ if await read_run_cancellation(task) is None:
+ return None
+ action_failure, _ = _terminal_custom_select_failure(
+ target_value=target_value,
+ matched_label=matched_label,
+ reason=RUN_STOPPED_BEFORE_SELECTING_REASON,
+ )
+ return action_failure
+
+
async def _select_deterministic_custom_option(
*,
target_value: str | None,
@@ -15643,14 +15685,26 @@ async def normal_select(
organization_id=task.organization_id,
)
if not resolution.fallback_to_llm and resolution.matched_index is not None:
+ if await read_run_cancellation(task) is not None:
+ action_failure, _ = _terminal_custom_select_failure(
+ target_value=target_value,
+ matched_label=resolution.matched_label,
+ reason=RUN_STOPPED_BEFORE_SELECTING_REASON,
+ )
+ return [action_failure]
+
deterministic_result = await _select_deterministic_normal_option(
action=action,
skyvern_element=skyvern_element,
locator=locator,
+ task=task,
+ target_value=target_value,
matched_label=resolution.matched_label,
matched_value=resolution.matched_value,
matched_index=resolution.matched_index,
)
+ if deterministic_result and _is_terminal_custom_select_failure(deterministic_result[-1]):
+ return deterministic_result
if _normal_select_successful(deterministic_result) and not await _normal_select_readback_contradicts(
locator=locator,
matched_index=resolution.matched_index,
@@ -15686,6 +15740,14 @@ async def normal_select(
json_response = await get_org_aware_secondary_llm_api_handler(default=app.NORMAL_SELECT_AGENT_LLM_API_HANDLER)(
prompt=prompt, step=step, prompt_name="normal-select"
)
+ if await read_run_cancellation(task) is not None:
+ action_failure, _ = _terminal_custom_select_failure(
+ target_value=target_value or "",
+ matched_label=None,
+ reason=RUN_STOPPED_BEFORE_SELECTING_REASON,
+ )
+ return [action_failure]
+
index: int | None = json_response.get("index")
value: str | None = json_response.get("value")
_log_select_shadow_match(
@@ -15703,6 +15765,11 @@ async def normal_select(
await _best_effort_focus_click_before_select(locator=locator, action=action)
if not is_success and value is not None:
+ cancellation_failure = await _native_select_cancellation_failure(
+ task=task, target_value=target_value or "", matched_label=None
+ )
+ if cancellation_failure is not None:
+ return [cancellation_failure]
try:
# click by value (if it matches)
await locator.select_option(
@@ -15721,6 +15788,11 @@ async def normal_select(
)
if not is_success and value is not None:
+ cancellation_failure = await _native_select_cancellation_failure(
+ task=task, target_value=target_value or "", matched_label=None
+ )
+ if cancellation_failure is not None:
+ return [cancellation_failure]
try:
# click by label (if it matches)
await locator.select_option(
@@ -15747,6 +15819,11 @@ async def normal_select(
locator=locator,
)
else:
+ cancellation_failure = await _native_select_cancellation_failure(
+ task=task, target_value=target_value or "", matched_label=None
+ )
+ if cancellation_failure is not None:
+ return [cancellation_failure]
try:
# This means the supplied index was for the select element, not a reference to the css dict
await locator.select_option(
diff --git a/tests/unit/test_normal_select_deterministic.py b/tests/unit/test_normal_select_deterministic.py
index 48b881f3f..078315fcd 100644
--- a/tests/unit/test_normal_select_deterministic.py
+++ b/tests/unit/test_normal_select_deterministic.py
@@ -8,6 +8,8 @@ import pytest
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
from skyvern.forge.agent_functions import AgentFunction
+from skyvern.forge.sdk.schemas.tasks import TaskStatus
+from skyvern.services.run_cancellation import RunCancellation
from skyvern.webeye.actions import handler
from skyvern.webeye.actions.actions import InputOrSelectContext, SelectOption, SelectOptionAction
from tests.unit.helpers import make_organization, make_task
@@ -101,12 +103,16 @@ async def _run_normal_select(
evaluate_side_effect: BaseException | None = None,
collapse_gate: AsyncMock | None = None,
agent_function: object | None = None,
+ normal_select_llm: AsyncMock | None = None,
+ cancellation_reader: AsyncMock | None = None,
) -> tuple[list, MagicMock, AsyncMock]:
provider = SimpleNamespace(
is_feature_enabled_cached=AsyncMock(return_value=feature_enabled),
resolve_feature_enabled_unrecorded=AsyncMock(return_value=feature_enabled),
)
- normal_select_llm = AsyncMock(return_value=llm_response or {"value": "fallback", "index": None})
+ normal_select_llm = normal_select_llm or AsyncMock(
+ return_value=llm_response or {"value": "fallback", "index": None}
+ )
if collapse_gate is None:
monkeypatch.setattr(handler.app, "EXPERIMENTATION_PROVIDER", provider)
@@ -114,6 +120,7 @@ async def _run_normal_select(
monkeypatch.setattr(handler, "_is_collapse_select_fanout_enabled", collapse_gate)
monkeypatch.setattr(handler.app, "AGENT_FUNCTION", agent_function or AgentFunction())
monkeypatch.setattr(handler.app, "NORMAL_SELECT_AGENT_LLM_API_HANDLER", normal_select_llm)
+ monkeypatch.setattr(handler, "read_run_cancellation", cancellation_reader or AsyncMock(return_value=None))
monkeypatch.setattr(handler.prompt_engine, "load_prompt", MagicMock(return_value="prompt"))
monkeypatch.setattr(handler.skyvern_context, "ensure_context", MagicMock(return_value=SimpleNamespace(tz_info=UTC)))
@@ -134,6 +141,169 @@ async def _run_normal_select(
return result, locator, normal_select_llm
[email protected](("cancel_during_llm", "selects"), [(True, False), (False, True)])
+async def test_normal_select_does_not_select_after_cancel_during_llm(
+ monkeypatch: pytest.MonkeyPatch,
+ cancel_during_llm: bool,
+ selects: bool,
+) -> None:
+ llm_finished = False
+
+ async def llm_side_effect(**_kwargs: object) -> dict[str, str | int | None]:
+ nonlocal llm_finished
+ llm_finished = True
+ return {"value": "US", "index": None}
+
+ cancellation = RunCancellation(TaskStatus.canceled, from_workflow_run=False)
+ cancellation_reader = AsyncMock(
+ side_effect=lambda *_args, **_kwargs: cancellation if llm_finished and cancel_during_llm else None
+ )
+ result, locator, normal_select_llm = await _run_normal_select(
+ monkeypatch,
+ feature_enabled=False,
+ action=_select_action("United States"),
+ options=[{"optionIndex": 0, "text": "United States", "value": "US"}],
+ selected_value="US",
+ normal_select_llm=AsyncMock(side_effect=llm_side_effect),
+ cancellation_reader=cancellation_reader,
+ )
+
+ assert locator.select_option.await_count == (1 if selects else 0)
+ normal_select_llm.assert_awaited_once()
+ assert handler._is_terminal_custom_select_failure(result[-1]) is (not selects)
+ if not selects:
+ cancellation_reader.assert_awaited_once()
+
+
+async def test_normal_select_does_not_deterministically_select_after_cancel_during_context_parse(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ context_parse_finished = False
+
+ async def parse_context(**_kwargs: object) -> InputOrSelectContext:
+ nonlocal context_parse_finished
+ context_parse_finished = True
+ return InputOrSelectContext(field="Department", is_required=True)
+
+ context_parser = AsyncMock(side_effect=parse_context)
+ monkeypatch.setattr(handler, "_get_input_or_select_context", context_parser)
+ cancellation = RunCancellation(TaskStatus.canceled, from_workflow_run=False)
+ cancellation_reader = AsyncMock(
+ side_effect=lambda *_args, **_kwargs: cancellation if context_parse_finished else None
+ )
+ agent_function = SimpleNamespace(
+ resolve_field_option=AsyncMock(
+ return_value=SimpleNamespace(
+ fallback_to_llm=False,
+ matched_index=0,
+ matched_label="United States",
+ matched_value="US",
+ )
+ )
+ )
+ result, locator, normal_select_llm = await _run_normal_select(
+ monkeypatch,
+ feature_enabled=True,
+ action=_select_action("United States"),
+ options=[{"optionIndex": 0, "text": "United States", "value": "US"}],
+ selected_value="US",
+ collapse_gate=AsyncMock(return_value=True),
+ agent_function=agent_function,
+ cancellation_reader=cancellation_reader,
+ )
+
+ assert handler._is_terminal_custom_select_failure(result[-1])
+ locator.select_option.assert_not_awaited()
+ normal_select_llm.assert_not_awaited()
+ context_parser.assert_awaited_once()
+ cancellation_reader.assert_awaited_once()
+
+
+async def test_normal_select_does_not_deterministically_select_after_cancel_during_focus(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ focus_finished = False
+
+ async def focus_before_select(**_kwargs: object) -> None:
+ nonlocal focus_finished
+ focus_finished = True
+
+ monkeypatch.setattr(handler, "_best_effort_focus_click_before_select", focus_before_select)
+ cancellation = RunCancellation(TaskStatus.canceled, from_workflow_run=False)
+ cancellation_reader = AsyncMock(side_effect=lambda *_args, **_kwargs: cancellation if focus_finished else None)
+ agent_function = SimpleNamespace(
+ resolve_field_option=AsyncMock(
+ return_value=SimpleNamespace(
+ fallback_to_llm=False,
+ matched_index=0,
+ matched_label="United States",
+ matched_value="US",
+ )
+ )
+ )
+
+ result, locator, normal_select_llm = await _run_normal_select(
+ monkeypatch,
+ feature_enabled=True,
+ action=_select_action("United States"),
+ options=[{"optionIndex": 0, "text": "United States", "value": "US"}],
+ selected_value="US",
+ collapse_gate=AsyncMock(return_value=True),
+ agent_function=agent_function,
+ cancellation_reader=cancellation_reader,
+ )
+
+ assert handler._is_terminal_custom_select_failure(result[-1])
+ locator.select_option.assert_not_awaited()
+ normal_select_llm.assert_not_awaited()
+ assert cancellation_reader.await_count == 2
+
+
[email protected]("feature_enabled", [False, True])
+async def test_normal_select_does_not_try_fallback_after_cancel_between_attempts(
+ monkeypatch: pytest.MonkeyPatch,
+ feature_enabled: bool,
+) -> None:
+ cancellation = RunCancellation(TaskStatus.canceled, from_workflow_run=False)
+ cancellation_reader = AsyncMock(side_effect=[None, None, cancellation])
+
+ async def focus_before_select(*, locator: MagicMock, **_kwargs: object) -> None:
+ locator.select_option.side_effect = Exception("timed out")
+
+ monkeypatch.setattr(handler, "_best_effort_focus_click_before_select", focus_before_select)
+ agent_function = SimpleNamespace(
+ resolve_field_option=AsyncMock(
+ return_value=SimpleNamespace(
+ fallback_to_llm=False,
+ matched_index=0,
+ matched_label="United States",
+ matched_value="US",
+ )
+ )
+ )
+ result, locator, _ = await _run_normal_select(
+ monkeypatch,
+ feature_enabled=feature_enabled,
+ action=_select_action("United States"),
+ options=[{"optionIndex": 0, "text": "United States", "value": "US"}],
+ selected_value="US",
+ llm_response={"value": "US", "index": 0},
+ collapse_gate=AsyncMock(return_value=feature_enabled),
+ agent_function=agent_function,
+ cancellation_reader=cancellation_reader,
+ )
+
+ assert handler._is_terminal_custom_select_failure(result[-1])
+ assert locator.select_option.await_args_list == [
+ call(value="US", timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS)
+ ]
+ assert cancellation_reader.await_count == 3
+
+
@pytest.mark.parametrize(
("target", "options", "selected_value"),
[
diff --git a/tests/unit/test_select_option_blocking_overlay.py b/tests/unit/test_select_option_blocking_overlay.py
index e3af06d3b..a6859c034 100644
--- a/tests/unit/test_select_option_blocking_overlay.py
+++ b/tests/unit/test_select_option_blocking_overlay.py
@@ -47,6 +47,7 @@ class TestSelectOptionFirst:
blocker_is_surrogate: bool = True,
select_visible: bool = True,
preselected: str | None = None,
+ normal_result: object | None = None,
) -> tuple:
from skyvern.webeye.actions import handler as handler_module
from skyvern.webeye.actions.actions import SelectOption, SelectOptionAction
@@ -62,6 +63,8 @@ class TestSelectOptionFirst:
if normal_raises:
normal_select = AsyncMock(side_effect=Exception("LLM provider error"))
+ elif normal_result is not None:
+ normal_select = AsyncMock(return_value=[normal_result])
else:
normal_result = [ActionSuccess()] if normal_success else [ActionFailure(Exception("no option"))]
normal_select = AsyncMock(return_value=normal_result)
@@ -134,6 +137,30 @@ class TestSelectOptionFirst:
handle_checkbox.assert_awaited_once()
assert results == ["CHECKBOX_ROUTED"]
+ @pytest.mark.asyncio
+ async def test_canceled_normal_select_does_not_fall_back_to_styled_widget(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ from skyvern.webeye.actions import handler as handler_module
+
+ terminal_failure, _ = handler_module._terminal_custom_select_failure(
+ target_value="California",
+ matched_label=None,
+ reason=handler_module.RUN_STOPPED_BEFORE_SELECTING_REASON,
+ )
+ blocker = MagicMock()
+ results, _, select_element, handle_checkbox = await self._run(
+ monkeypatch,
+ normal_success=False,
+ normal_result=terminal_failure,
+ blocking_return=(blocker, True),
+ blocker_is_checkbox=True,
+ )
+
+ assert results == [terminal_failure]
+ select_element.find_blocking_element.assert_not_awaited()
+ handle_checkbox.assert_not_awaited()
+
@pytest.mark.asyncio
async def test_select_option_fails_with_modal_blocker_no_hijack(self, monkeypatch: pytest.MonkeyPatch) -> None:
# select_option failed AND the overlapping element is an unrelated modal (not a dropdown
@@ -167,6 +194,35 @@ class TestSelectOptionFirst:
handle_checkbox.assert_awaited_once()
assert results == ["CHECKBOX_ROUTED"]
+ @pytest.mark.asyncio
+ async def test_canceled_normal_select_exception_does_not_fall_back_to_styled_widget(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ from skyvern.forge.sdk.schemas.tasks import TaskStatus
+ from skyvern.services.run_cancellation import RunCancellation
+ from skyvern.webeye.actions import handler as handler_module
+
+ cancellation = RunCancellation(TaskStatus.canceled, from_workflow_run=False)
+ monkeypatch.setattr(handler_module, "read_run_cancellation", AsyncMock(return_value=cancellation))
+ terminal_failure, _ = handler_module._terminal_custom_select_failure(
+ target_value="California",
+ matched_label=None,
+ reason=handler_module.RUN_STOPPED_BEFORE_SELECTING_REASON,
+ )
+ blocker = MagicMock()
+ results, normal_select, select_element, handle_checkbox = await self._run(
+ monkeypatch,
+ normal_success=False,
+ normal_raises=True,
+ blocking_return=(blocker, True),
+ blocker_is_checkbox=True,
+ )
+
+ assert results == [terminal_failure]
+ normal_select.assert_awaited_once()
+ select_element.find_blocking_element.assert_not_awaited()
+ handle_checkbox.assert_not_awaited()
+
@pytest.mark.asyncio
async def test_hidden_backing_select_click_navigates_widget_even_without_role(
self, monkeypatch: pytest.MonkeyPatch
@@ -278,11 +334,14 @@ class TestSelectCommitsUnderOverlay:
locator.click.assert_awaited_once()
@pytest.mark.asyncio
- async def test_deterministic_select_commits_when_preclick_intercepted(self) -> None:
+ async def test_deterministic_select_commits_when_preclick_intercepted(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
from skyvern.webeye.actions import handler as handler_module
from skyvern.webeye.actions.actions import SelectOption, SelectOptionAction
from skyvern.webeye.actions.responses import ActionSuccess
+ monkeypatch.setattr(handler_module, "read_run_cancellation", AsyncMock(return_value=None))
locator = MagicMock()
locator.click = AsyncMock(side_effect=Exception("click intercepted by overlay"))
locator.select_option = AsyncMock(return_value=None) # DOM select succeeds regardless
@@ -296,6 +355,8 @@ class TestSelectCommitsUnderOverlay:
action=action,
skyvern_element=skyvern_element,
locator=locator,
+ task=MagicMock(),
+ target_value="California",
matched_label="California",
matched_value="CA",
matched_index=0,
diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.py
index 3301fd608..ed8b44fd6 100644
--- a/tests/unit/test_select_shadow_match.py
+++ b/tests/unit/test_select_shadow_match.py
@@ -78,6 +78,7 @@ async def _run_normal_select_with_shadow_log(
)
monkeypatch.setattr(handler.prompt_engine, "load_prompt", lambda *_args, **_kwargs: "prompt")
monkeypatch.setattr(handler.app, "NORMAL_SELECT_AGENT_LLM_API_HANDLER", AsyncMock(return_value=llm_response))
+ monkeypatch.setattr(handler, "read_run_cancellation", AsyncMock(return_value=None))
monkeypatch.setattr(handler.settings, "SKYVERN_SELECT_SHADOW_MATCH", True)
context = SkyvernContext(tz_info=ZoneInfo("UTC"))
@@ -86,9 +87,7 @@ async def _run_normal_select_with_shadow_log(
results = await handler.normal_select(
action=action,
skyvern_element=_FakeSelectElement(options),
- task=SimpleNamespace(
- navigation_goal="goal", navigation_payload={}, organization_id=None, workflow_permanent_id=None
- ),
+ task=_shadow_task(),
step=SimpleNamespace(step_id="step-1"),
builder=SimpleNamespace(),
)
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 53dc0ebb2..406638a76 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -79,6 +79,7 @@ from skyvern.forge.taskv3.loop import (
from skyvern.forge.taskv3.run_arms import LOGIN_PACE_FLAG
from skyvern.forge.taskv3.tools import (
_MENU_OPTION_TEXTS_JS,
+ _NO_QUESTION_MARKER,
_OPAQUE_ID_RUN_RE,
_SEMANTIC_COMMIT_STATE_JS,
_TYPE_TARGET_PROBE_JS,
@@ -566,6 +567,8 @@ async def test_observe_renders_checkbox_checked_state() -> None:
assert _ref_line(r.content, "I agree") and "checked=" in r.content
+_MARKER = f"group={_NO_QUESTION_MARKER}"
+
_DISABLED_SHAPES_HTML = """
<!doctype html><html><body>
<input type="button" id="plain-on" value="Alpha plain">
@@ -4115,7 +4118,7 @@ def _assert_marked_not_guessed(data: dict[str, Any], content: str, markers: int,
assert not e.get("group"), e
for text in never:
assert text not in json.dumps({k: v for k, v in e.items() if k != "placement"}), (text, e)
- assert content.count("group=(no question found)") == markers, content
+ assert content.count(_MARKER) == markers, content
@_skip_no_browser
@@ -4341,8 +4344,11 @@ async def test_observe_renders_the_no_question_marker_once_per_radio_group() ->
r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
radio_lines = [line for line in r.content.splitlines() if "/radio" in line]
assert len(radio_lines) == 4, r.content
- assert r.content.count("group=(no question found)") == 2, r.content
- assert "(no question found)" in radio_lines[0] and "(no question found)" in radio_lines[2], r.content
+ assert r.content.count(_MARKER) == 2, r.content
+ assert _NO_QUESTION_MARKER in radio_lines[0] and _NO_QUESTION_MARKER in radio_lines[2], r.content
+ # The marker points at a read, never at a position: the text beside an unbound group is often the wrong text.
+ assert "get_html" in _NO_QUESTION_MARKER, _NO_QUESTION_MARKER
+ assert not re.search(r"\b(look|above|below|beside|right|near)\b", _NO_QUESTION_MARKER), _NO_QUESTION_MARKER
@_skip_no_browser
@@ -4483,8 +4489,13 @@ async def test_observe_radios_in_each_options_own_shadow_root_are_one_group(para
r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
radios = _radio_records(data)
assert [e.get("label") for e in radios] == ["Light", "Medium", "Dark"], radios
+ if paragraph:
+ # The paragraph is the one text block of the card holding only these options.
+ assert [e.get("question") for e in radios] == [paragraph] * 3, radios
+ assert _NO_QUESTION_MARKER not in r.content, r.content
+ return
assert len({e.get("group_missing") for e in radios}) == 1 and radios[0].get("group_missing"), radios
- _assert_marked_not_guessed(data, r.content, 1, [paragraph] if paragraph else [])
+ _assert_marked_not_guessed(data, r.content, 1, [])
@_skip_no_browser
@@ -5102,7 +5113,9 @@ def _aria_radio(caption: str) -> str:
],
)
async def test_observe_a_section_name_over_several_groups_is_never_their_question(body: str, section: str) -> None:
- async with _content_page(f"<!doctype html><html><body><form>{body}</form></body></html>") as page:
+ # Without their own captions, the section name still never stands in for either group's question.
+ bare = re.sub(r"<p>[^<]*</p>", "", body)
+ async with _content_page(f"<!doctype html><html><body><form>{bare}</form></body></html>") as page:
data = await _observe_data(page)
r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
_assert_marked_not_guessed(data, r.content, 2, [section])
@@ -5138,9 +5151,10 @@ async def test_observe_a_section_holding_several_questions_gives_none_of_them_to
items = "".join(
f'<li><label><input type="{kind}" name="{k}" value="{k}-{v}">{v}</label></li>' for v in ("Yes", "No")
)
- return f'<div class="row"><p>{questions[k]}</p><ul>{items}</ul></div>'
+ return f'<div class="row"><ul>{items}</ul></div>'
- html = f"<!doctype html><html><body><form><fieldset>{row('a')}{row('b')}</fieldset></form></body></html>"
+ captions = "".join(f"<p>{q}</p>" for q in questions.values())
+ html = f"<!doctype html><html><body><form><fieldset>{captions}{row('a')}{row('b')}</fieldset></form></body></html>"
async with _content_page(html) as page:
data = await _observe_data(page)
r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
@@ -5157,16 +5171,16 @@ async def test_observe_a_section_holding_several_questions_gives_none_of_them_to
@pytest.mark.asyncio
async def test_observe_a_frames_radio_group_keeps_its_marker_beside_the_pages_own() -> None:
# Each realm numbers its groups from 1, so the page's first group and the frame's share a number.
- frame = "<form><div><p>Do you smoke?</p><div>{}</div></div></form>".format(_yes_no_radios("f").replace('"', "'"))
+ frame = "<form><div><div>{}</div></div></form>".format(_yes_no_radios("f").replace('"', "'"))
html = (
- f"<!doctype html><html><body><form><div><p>Do you drive?</p><div>{_yes_no_radios('m')}</div></div></form>"
+ f"<!doctype html><html><body><form><div><div>{_yes_no_radios('m')}</div></div></form>"
f'<iframe srcdoc="{frame}" width="400" height="200"></iframe></body></html>'
)
async with _live_page(html) as page:
r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
radio_lines = [line for line in r.content.splitlines() if "/radio" in line]
assert len(radio_lines) == 4, r.content
- assert [("(no question found)" in line) for line in radio_lines] == [True, False, True, False], r.content
+ assert [(_NO_QUESTION_MARKER in line) for line in radio_lines] == [True, False, True, False], r.content
def _two_realm_page(page_script: str, frame_script: str) -> str:
@@ -5275,7 +5289,7 @@ async def test_observe_a_radio_groups_description_never_puts_its_marker_mid_grou
r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
radio_lines = [line for line in r.content.splitlines() if "/radio" in line]
assert len(radio_lines) == 2, r.content
- assert "(no question found)" not in radio_lines[1], r.content
+ assert _NO_QUESTION_MARKER not in radio_lines[1], r.content
@_skip_no_browser
@@ -5297,7 +5311,7 @@ async def test_observe_a_short_hint_never_silences_an_undeclared_radio_groups_ma
data = await _observe_data(page)
r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
assert not any(e.get("group") for e in _radio_records(data)), _radio_records(data)
- assert r.content.count("group=(no question found)") == 1, r.content
+ assert r.content.count(_MARKER) == 1, r.content
_TWO_CHECKBOXES = (
@@ -5441,7 +5455,7 @@ async def test_observe_a_choice_groups_own_text_outranks_an_outer_declared_name(
for e in choices.values():
for text in wrong:
assert text not in (e.get("group") or ""), choices
- assert "(no question found)" not in r.content, r.content
+ assert _NO_QUESTION_MARKER not in r.content, r.content
@_skip_no_browser
@@ -5497,7 +5511,9 @@ async def test_observe_a_choice_groups_own_text_outranks_an_outer_declared_name(
async def test_observe_a_declared_name_that_is_not_this_whole_groups_gets_the_marker(
body: str, never: list[str]
) -> None:
- async with _content_page(_choice_form(body)) as page:
+ # Without the group's own caption, the outer name still never stands in for its question.
+ bare = re.sub(r"<p>[^<]*</p>", "", body)
+ async with _content_page(_choice_form(bare)) as page:
data = await _observe_data(page)
r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
_assert_marked_not_guessed(data, r.content, 1, never)
@@ -5540,7 +5556,7 @@ async def test_observe_a_per_option_name_defers_to_the_named_group_above_it(body
return
radios = {str(e.get("value")): e for e in _radio_records(data)}
assert question in (radios["a-yes"].get("group") or ""), radios
- assert "(no question found)" not in r.content, r.content
+ assert _NO_QUESTION_MARKER not in r.content, r.content
def _aria_yes_no(k: str) -> str:
@@ -5556,9 +5572,9 @@ def _aria_yes_no(k: str) -> str:
("body", "question", "markers"),
[
pytest.param(
- '<div class="q"><p id="q-label">Are you over 18?</p><div role="radiogroup" aria-labelledby="q-label">'
- f'{_aria_yes_no("qa")}</div></div><div class="q"><p id="q-label">Have you ever been convicted of a felony?</p>'
- f'<div role="radiogroup" aria-labelledby="q-label">{_aria_yes_no("qb")}</div></div>',
+ '<p id="q-label">Are you over 18?</p><p id="q-label">Have you ever been convicted of a felony?</p>'
+ f'<div class="q"><div role="radiogroup" aria-labelledby="q-label">{_aria_yes_no("qa")}</div></div>'
+ f'<div class="q"><div role="radiogroup" aria-labelledby="q-label">{_aria_yes_no("qb")}</div></div>',
"Are you over 18?",
2,
id="templated-question-reusing-its-id",
@@ -5581,6 +5597,377 @@ async def test_observe_a_declared_name_that_may_be_another_questions_gets_the_ma
_assert_marked_not_guessed(data, r.content, markers, [question])
+@_skip_no_browser
+async def test_click_on_a_choice_option_names_the_question_observe_bound_it_to() -> None:
+ # Two adjacent caption cards, then a group with no question.
+ cards = "".join(
+ f"<div class='q'><div>{q}</div><div>{_yes_no_radios(k)}</div></div>"
+ for k, q in (("s", "Will you now or in the future require sponsorship?"), ("w", "Can you work here?"))
+ )
+ html = f"<!doctype html><html><body><form>{cards}<div>{_yes_no_radios('u')}</div></form></body></html>"
+ async with _live_page(html) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "observe").handler({})
+ refs = [int(m.group(1)) for m in re.finditer(r"^ref=(\d+) .*/radio", r.content, re.M)]
+ assert len(refs) == 6, r.content
+ picked = [await _tool(tools, "click").handler({"selector": f"ref={refs[i]}"}) for i in (3, 5)]
+ by_selector = await _tool(tools, "click").handler({"selector": "input[value=s-no]"})
+ assert picked[0].status == "ok" and picked[0].content.endswith("\npicked 'No' for question 'Can you work here?'")
+ assert picked[1].content.endswith("\npicked 'No' for an unknown question"), picked[1].content
+ # A pick by a raw selector has no observe record to name the question from.
+ assert by_selector.content.endswith("\npicked this option for an unknown question"), by_selector.content
+
+
+@_skip_no_browser
+async def test_click_echo_stays_silent_when_the_pick_could_not_be_confirmed() -> None:
+ # The card leaves the page on click, so its state cannot be read back.
+ html = (
+ "<!doctype html><html><body><form><div class='q' id=card><div>Can you work here?</div>"
+ f"<div>{_yes_no_radios('w')}</div></div></form>"
+ "<script>document.getElementById('card').addEventListener('click', (e) => e.currentTarget.remove());</script>"
+ "</body></html>"
+ )
+ async with _live_page(html) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "observe").handler({})
+ ref = int(re.findall(r"^ref=(\d+) .*/radio", r.content, re.M)[1])
+ picked = await _tool(tools, "click").handler({"selector": f"ref={ref}"})
+ assert picked.entry_unconfirmed, picked
+ assert "picked" not in picked.content, picked.content
+
+
+_AUTH_Q = "Are you legally authorized to work in this country?"
+_FUTURE_SPONSOR_Q = "Will you now or in the future require visa sponsorship?"
+_EMAIL = "<label>Email <input type=text name=e></label>"
+
+
+def _list_radios(name: str) -> str:
+ return (
+ f"<ul><li><label><input type=radio name={name} value={name}-yes><span>Yes</span></label></li>"
+ f"<li><label><input type=radio name={name} value={name}-no><span>No</span></label></li></ul>"
+ )
+
+
+def _choice_records(data: dict[str, Any]) -> list[dict[str, Any]]:
+ kinds = ("radio", "checkbox")
+ return [e for e in data["elements"] if e.get("type") in kinds or e.get("role") in kinds]
+
+
+def _choice_lines(content: str) -> list[str]:
+ return [line for line in content.splitlines() if "/radio" in line or "/checkbox" in line]
+
+
+def _opts(name: str, extra: str = "") -> str:
+ return (
+ f"<div class='opts'><label><input type=radio name={name} value={name}-yes{extra}>Yes</label>"
+ f"<label><input type=radio name={name} value={name}-no{extra}>No</label></div>"
+ )
+
+
+def _required_card(name: str, question: str) -> str:
+ return (
+ f"<li class=item><div class=caption>{question}<span class=req>✱</span></div>"
+ f"<div class=answers>{_list_radios(name)}</div></li>"
+ )
+
+
+# The card rung: a group with no declared question takes the one text block of the largest ancestor that holds
+# nothing observe lists but this group's options.
+@_skip_no_browser
+ ("body", "questions", "never"),
+ [
+ pytest.param(
+ f"<ul class=questions>{_required_card('s', _FUTURE_SPONSOR_Q)}{_required_card('a', _AUTH_Q)}</ul>" + _EMAIL,
+ [f"{_FUTURE_SPONSOR_Q} ✱"] * 2 + [f"{_AUTH_Q} ✱"] * 2,
+ [],
+ id="list-of-required-question-cards",
+ ),
+ pytest.param(
+ "<div class=page><h4>WORK AUTHORIZATION</h4><p>Answer every question below.</p>"
+ f"<ul class=questions>{_required_card('a', _AUTH_Q)}</ul><h4>SPONSORSHIP</h4>"
+ f"<ul class=questions>{_required_card('s', _FUTURE_SPONSOR_Q)}</ul></div>" + _EMAIL,
+ [f"{_AUTH_Q} ✱"] * 2 + [f"{_FUTURE_SPONSOR_Q} ✱"] * 2,
+ ["WORK AUTHORIZATION", "Answer every question", "SPONSORSHIP"],
+ id="question-card-under-a-section-heading-and-notice",
+ ),
+ pytest.param(
+ f"<div class=card>{_AUTH_Q}<div class=opts>{_yes_no_radios('t')}</div></div>" + _EMAIL,
+ [_AUTH_Q] * 2,
+ [],
+ id="question-as-a-bare-text-node-in-the-card",
+ ),
+ pytest.param(
+ f"<div class=card><p>{_AUTH_Q}</p> * <span>✱</span><div class=opts>{_yes_no_radios('m')}</div></div>"
+ + _EMAIL,
+ [_AUTH_Q] * 2,
+ [],
+ id="question-beside-one-character-required-markers",
+ ),
+ pytest.param(
+ f"<div class=card><p>{_AUTH_Q}</p><div role='alert status'>This field is required</div>"
+ f"<div class=opts>{_yes_no_radios('e')}</div></div>" + _EMAIL,
+ [_AUTH_Q] * 2,
+ ["This field is required"],
+ id="question-then-an-error-with-two-roles",
+ ),
+ pytest.param(
+ f"<div class=card><p>{_AUTH_Q}</p><div class=error><span role=alert>Required</span></div>"
+ f"<div class=opts>{_yes_no_radios('n')}</div></div>" + _EMAIL,
+ [_AUTH_Q] * 2,
+ ["Required"],
+ id="question-then-an-error-wrapping-its-alert",
+ ),
+ pytest.param(
+ f"<div class=card><p aria-live=off>{_AUTH_Q}</p><div class=opts>{_yes_no_radios('f')}</div></div>" + _EMAIL,
+ [_AUTH_Q] * 2,
+ [],
+ id="question-in-a-region-whose-announcements-are-off",
+ ),
+ pytest.param(
+ f"<div class=card><p>{_AUTH_Q} <span role=status>(required)</span></p>"
+ f"<div class=opts>{_yes_no_radios('i')}</div></div>" + _EMAIL,
+ [f"{_AUTH_Q} (required)"] * 2,
+ [],
+ id="question-holding-an-inline-status",
+ ),
+ pytest.param(
+ f"<div class=q><p>{_AUTH_Q}</p><div role=alert>This field is required</div><div class=opts>"
+ f"{_yes_no_radios('a')}</div></div>" + _EMAIL,
+ [_AUTH_Q] * 2,
+ ["This field is required"],
+ id="question-then-alert-error",
+ ),
+ pytest.param(
+ "<div class=q><div>Do you require sponsorship?</div>"
+ "<label><input type=checkbox name=sy value=s-yes>Yes</label>"
+ "<label><input type=checkbox name=sn value=s-no>No</label></div>" + _EMAIL,
+ ["Do you require sponsorship?"] * 2,
+ [],
+ id="checkbox-list-under-its-question",
+ ),
+ ],
+)
+async def test_observe_binds_each_choice_group_to_exactly_its_own_question(
+ body: str, questions: list[str], never: list[str]
+) -> None:
+ async with _content_page(f"<!doctype html><html><body><form>{body}</form></body></html>") as page:
+ data = await _observe_data(page)
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
+ choices = _choice_records(data)
+ assert [e.get("question") for e in choices] == questions, choices
+ for e, own in zip(choices, questions, strict=True):
+ record = json.dumps({k: v for k, v in e.items() if k != "placement"})
+ for text in [*never, "Email", *(q for q in set(questions) if q != own)]:
+ assert text not in record, (text, e)
+ lines = _choice_lines(r.content)
+ for q in set(questions):
+ assert any(q in line for line in lines), f"{q!r} is on no choice line:\n{r.content}"
+ assert _MARKER not in r.content, r.content
+
+
+# Known residual (SKY-18213), pinned as today's output because the browser gate fails on an xfail: the legacy
+# rung reads a whole fieldset, so its hint and captions fold into the legend's question. Invert when it closes.
+@_skip_no_browser
+async def test_a_fieldset_read_whole_by_the_legacy_rung_still_folds_its_hint_and_captions_in() -> None:
+ body = (
+ "<fieldset><legend>Have you ever been excluded from a federal health care program?</legend><p>Select one</p>"
+ f"{_yes_no_radios('m1')}</fieldset>" + _EMAIL
+ )
+ async with _content_page(f"<!doctype html><html><body><form>{body}</form></body></html>") as page:
+ data = await _observe_data(page)
+ folded = "Have you ever been excluded from a federal health care program? Select one YesNo"
+ assert [e.get("question") for e in _choice_records(data)] == [folded] * 2
+
+
+# A section panel drawing its `label` attribute above its slotted content.
+_SECTION_COMPONENT_JS = """<script>
+customElements.define('x-section', class extends HTMLElement {
+ constructor() { super(); this.attachShadow({mode: 'open'}).innerHTML = '<div class=t></div><slot></slot>'; }
+ connectedCallback() {
+ this.style.display = 'block';
+ this.shadowRoot.querySelector('.t').textContent = this.getAttribute('label');
+ }
+});
+</script>"""
+
+
+# The marker, never a guess, when the card does not single out one question.
+@_skip_no_browser
+ ("body", "never"),
+ [
+ pytest.param(
+ f"<div class=q><p>Do you accept the <a href='#'>policy</a>?</p><div class=opts>{_yes_no_radios('p')}"
+ "</div></div>" + _EMAIL,
+ ["Do you accept the", "policy"],
+ id="link-inside-the-question",
+ ),
+ pytest.param(
+ "<div class='card'><h4>VISA STATUS</h4><ul><li class='item'>"
+ "<div class='caption'>Will you require sponsorship?<span class='req'>*</span></div>"
+ f"<div class='answers'>{_list_radios('k')}</div></li></ul></div>",
+ ["VISA STATUS"],
+ id="single-question-card-under-a-heading",
+ ),
+ pytest.param(
+ f"<div class='q'><div class='step'>Step 3 of 5</div><p>{_AUTH_Q}</p><div>{_yes_no_radios('d')}</div>"
+ "</div>" + _EMAIL,
+ ["Step 3 of 5", _AUTH_Q],
+ id="step-counter-then-question",
+ ),
+ pytest.param(
+ f"{_SECTION_COMPONENT_JS}<x-section label='Work eligibility'><p>{_FUTURE_SPONSOR_Q}</p>"
+ f"{_yes_no_radios('s')}</x-section>" + _EMAIL,
+ ["Work eligibility", _FUTURE_SPONSOR_Q],
+ id="section-component-label-over-a-slotted-question",
+ ),
+ pytest.param(
+ f"<div class=blk><div class=row><span>{_AUTH_Q}</span> <a href='#help'>Help</a></div><div class=wrap>"
+ f"<small>Select one option</small><div class=opts>{_yes_no_radios('w')}</div></div></div>" + _EMAIL,
+ ["Select one option", _AUTH_Q],
+ id="hint-in-the-card-beside-a-question-row-holding-a-help-link",
+ ),
+ pytest.param(
+ f"<p>{_AUTH_Q}</p><label>Notes <input name=notes></label><div class=card><label for=cy>Yes</label>"
+ "<input type=radio id=cy name=c value=y><label for=cn>No</label><input type=radio id=cn name=c value=n></div>"
+ + _EMAIL,
+ [_AUTH_Q],
+ id="caption-placed-before-its-own-option",
+ ),
+ pytest.param(
+ f"<div class=wrap><div class=title><h4>CONTACT PREFERENCES</h4></div><div class=opts>{_yes_no_radios('h')}"
+ "</div></div>" + _EMAIL,
+ ["CONTACT PREFERENCES"],
+ id="section-title-nested-in-a-wrapper-block",
+ ),
+ pytest.param(
+ "<div class=wrap><div role='heading text' aria-level=4>CONTACT PREFERENCES</div>"
+ f"<div class=opts>{_yes_no_radios('k')}</div></div>" + _EMAIL,
+ ["CONTACT PREFERENCES"],
+ id="section-title-with-two-roles",
+ ),
+ pytest.param(
+ "<div class=card><span>Yes No</span><div><input type=radio name=t aria-label=Yes>"
+ "<input type=radio name=t aria-label=No></div></div>" + _EMAIL,
+ [],
+ id="card-text-that-is-only-its-options-captions",
+ ),
+ ],
+)
+async def test_observe_marks_a_choice_group_whose_card_does_not_single_out_its_question(
+ body: str, never: list[str]
+) -> None:
+ async with _content_page(f"<!doctype html><html><body><form>{body}</form></body></html>") as page:
+ data = await _observe_data(page)
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
+ _assert_marked_not_guessed(data, r.content, 1, never)
+
+
+# Each group's `group=` field in order: its question, or None for the marker.
+@_skip_no_browser
+ ("body", "groups"),
+ [
+ pytest.param(
+ f"<div class=wrap><p>Tell us about your eligibility.</p><div class=card2><div id=qa>{_AUTH_Q}</div>"
+ f"{_opts('au', ' aria-describedby=qa')}</div></div>" + _EMAIL,
+ [_AUTH_Q],
+ id="described-question-under-a-wrapper-intro",
+ ),
+ pytest.param(
+ f"<div id=qa>{_AUTH_Q}</div><label>Notes <input></label>"
+ f"<div class=card><div>This field is required</div>{_opts('au', ' aria-describedby=qa')}</div>" + _EMAIL,
+ [_AUTH_Q],
+ id="question-described-from-outside-a-card-holding-an-error",
+ ),
+ pytest.param(
+ "<div class=wrap><p>Tell us about your eligibility.</p><div class=card2><div role=status>Required</div>"
+ f"{_opts('au')}</div></div>" + _EMAIL,
+ [None],
+ id="intro-above-a-card-holding-only-a-status-note",
+ ),
+ pytest.param(
+ f"<div class=card><div id=qa>{_AUTH_Q}</div><div>This field is required</div>"
+ f"{_opts('au', ' aria-describedby=qa')}</div>" + _EMAIL,
+ [_AUTH_Q],
+ id="question-described-inside-a-card-holding-an-error",
+ ),
+ pytest.param(
+ f"<div class=card>{_opts('au')}<div>This field is required</div></div>" + _EMAIL,
+ [None],
+ id="error-after-the-options-of-a-bare-group",
+ ),
+ ],
+)
+async def test_observe_prints_each_choice_group_its_own_question_or_the_marker(
+ body: str, groups: list[str | None]
+) -> None:
+ html = f"<!doctype html><html><body><form>{body}</form></body></html>"
+ async with _content_page(html) as page:
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
+ printed = [line for line in _choice_lines(r.content) if " group=" in line]
+ assert len(printed) == len(groups), r.content
+ for line, question in zip(printed, groups, strict=True):
+ assert (_MARKER if question is None else f" group={question!r}") in line, (question, r.content)
+
+
+@_skip_no_browser
+async def test_observe_marks_a_choice_group_whose_question_the_group_budget_withholds() -> None:
+ filler = "".join(
+ f"<div class=q><div>Question {i}: please describe in your own words whether you have previously held a "
+ f"position that involved {'x' * 60} responsibilities of kind {i}?</div><div>{_yes_no_radios(f'f{i}')}</div></div>"
+ for i in range(30)
+ )
+ legal = "".join(
+ f"<div class=q><div>{q}</div><div>{_yes_no_radios(k)}</div></div>"
+ for k, q in (("au", _AUTH_Q), ("sp", _FUTURE_SPONSOR_Q))
+ )
+ async with _content_page(_choice_form(filler + legal)) as page:
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
+ firsts = [line for line in r.content.splitlines() if "/radio" in line][0::2]
+ assert len(firsts) == 32, r.content
+ # A group whose question the budget withheld still says so, rather than printing nothing.
+ assert all(" group=" in line for line in firsts), r.content
+ assert 0 < sum(_MARKER in line for line in firsts) < 32, r.content
+
+
+@_skip_no_browser
+async def test_click_echo_names_no_question_for_an_option_the_page_replaced_since_observe() -> None:
+ def screen(q: str) -> str:
+ return (
+ f"<div class='q'><p>{q}</p><div><label><input type=radio id='opt-yes' name='x' value='y'>Yes</label>"
+ "<label><input type=radio id='opt-no' name='x' value='n'>No</label></div></div>"
+ )
+
+ html = (
+ f"<!doctype html><html><body><form><div id='screen'>{screen(_FUTURE_SPONSOR_Q)}</div>{_EMAIL}</form>"
+ f"<script>function next(){{document.getElementById('screen').innerHTML = {json.dumps(screen(_AUTH_Q))};}}"
+ "</script></body></html>"
+ )
+ async with _live_page(html) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "observe").handler({})
+ refs = [int(m.group(1)) for m in re.finditer(r"^ref=(\d+) .*/radio", r.content, re.M)]
+ assert len(refs) == 2 and _FUTURE_SPONSOR_Q in r.content, r.content
+ await page.evaluate("next()")
+ click = _tool(tools, "click")
+ c = await click.handler({"selector": f"ref={refs[1]}"})
+ assert await page.evaluate("document.querySelector('#opt-no').checked"), c.content
+ again = await click.handler({"selector": f"ref={refs[1]}"})
+ for res in (c, again):
+ assert res.status == "ok" and "\npicked 'No' for an unknown question" in res.content, res.content
+ assert _FUTURE_SPONSOR_Q not in res.content, res.content
+
+
@_skip_no_browser
@pytest.mark.asyncio
async def test_observe_a_legend_never_names_a_radio_group_it_holds_only_part_of() -> None:
@@ -5592,7 +5979,7 @@ async def test_observe_a_legend_never_names_a_radio_group_it_holds_only_part_of(
data = await _observe_data(page)
r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
assert "Contact preference" not in json.dumps(_radio_records(data)), _radio_records(data)
- assert "group=(no question found)" in r.content, r.content
+ assert _MARKER in r.content, r.content
@_skip_no_browser
@@ -5654,7 +6041,7 @@ async def test_observe_a_later_members_question_drops_an_earlier_members_marker(
r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
radios = {str(e.get("value")): e for e in _radio_records(data)}
assert "Extra fee applies" in (radios["express"].get("group") or ""), radios
- assert "(no question found)" not in r.content, r.content
+ assert _NO_QUESTION_MARKER not in r.content, r.content
@_skip_no_browser
@@ -5728,7 +6115,7 @@ async def test_observe_a_legend_names_proxy_radios_whose_option_wrappers_hold_on
if sel.startswith("#notify"):
assert "Subscription Plan" not in text, groups
assert text not in ("Basic", "Pro", "Email", "SMS"), groups
- assert "(no question found)" not in r.content, r.content
+ assert _NO_QUESTION_MARKER not in r.content, r.content
@_skip_no_browser
@@ -5779,7 +6166,7 @@ async def test_observe_an_options_only_field_container_gets_the_marker_not_its_c
assert e.get("group") in (None, own), e
if e.get("group"):
shown.add(own)
- assert r.content.count("group=(no question found)") == len(questions) - len(shown), r.content
+ assert r.content.count(_MARKER) == len(questions) - len(shown), r.content
checkboxes = [e for e in data["elements"] if e.get("type") == "checkbox"]
assert len(checkboxes) == 40, checkboxes
assert all(e.get("group") in (None, "Language skills") for e in checkboxes), checkboxes
@@ -5822,7 +6209,7 @@ async def test_observe_a_declared_question_held_back_from_printing_is_not_marked
)
async with _content_page(f"<!doctype html><html><body><form>{body}</form></body></html>") as page:
r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
- assert "(no question found)" not in r.content, r.content
+ assert _NO_QUESTION_MARKER not in r.content, r.content
@_skip_no_browser
diff --git a/tests/unit/workflow/test_file_parser_block.py b/tests/unit/workflow/test_file_parser_block.py
index c305f37ff..e69f2317c 100644
--- a/tests/unit/workflow/test_file_parser_block.py
+++ b/tests/unit/workflow/test_file_parser_block.py
@@ -778,6 +778,25 @@ class TestPDFParserSchemaValidation:
assert "expected type object, got array" in prompts[1]
record_output_parameter_value.assert_awaited_once()
+ async def test_stalled_model_fails_the_block_at_the_deadline(self) -> None:
+ block = _make_pdf_parser_block("https://example.com/resume.pdf")
+
+ async def stalled_handler(**kwargs: Any) -> Any:
+ await asyncio.Event().wait()
+
+ with pytest.MonkeyPatch.context() as mp:
+ self._patch_execute_dependencies(mp, block, AsyncMock(side_effect=stalled_handler))
+ mp.setattr("skyvern.forge.sdk.workflow.models.block.FILE_PARSER_LLM_CALL_TIMEOUT_SECONDS", 0.05)
+
+ # The outer bound turns a missing deadline into a failure rather than a hung test run.
+ result = await asyncio.wait_for(
+ block.execute("workflow-run", "workflow-run-block", organization_id="org-1"), timeout=10
+ )
+
+ assert result.success is False
+ assert result.status == BlockStatus.failed
+ assert "did not answer within" in (result.failure_reason or "")
+
@pytest.mark.asyncio
class TestOcrPdfPages:
@@ -903,6 +922,31 @@ class TestOcrPdfPages:
with pytest.raises(RuntimeError, match="vision model outage"):
await block._ocr_pdf_pages([b"p1", b"p2", b"p3"])
+ @pytest.mark.parametrize(
+ ("call_timeout", "pass_timeout", "expected_reason"),
+ [(0.05, 10, "did not answer within"), (10, 0.05, "did not transcribe")],
+ ids=["page_deadline", "whole_pass_deadline"],
+ )
+ async def test_stalled_page_fails_the_pass_instead_of_being_skipped(
+ self, call_timeout: float, pass_timeout: float, expected_reason: str
+ ) -> None:
+ """Skipping a page the model never answered would hand extraction a document with that page missing."""
+ block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)
+
+ async def fake_handler(**kwargs: Any) -> dict[str, str]:
+ (image,) = kwargs["screenshots"]
+ if image == b"stalled":
+ await asyncio.Event().wait()
+ return {"extracted_text": f"text for {image.decode()}"}
+
+ with pytest.MonkeyPatch.context() as mp:
+ self._patch_handler(mp, AsyncMock(side_effect=fake_handler))
+ mp.setattr("skyvern.forge.sdk.workflow.models.block.FILE_PARSER_LLM_CALL_TIMEOUT_SECONDS", call_timeout)
+ mp.setattr("skyvern.forge.sdk.workflow.models.block.PDF_OCR_TOTAL_TIMEOUT_SECONDS", pass_timeout)
+
+ with pytest.raises(TimeoutError, match=expected_reason):
+ await asyncio.wait_for(block._ocr_pdf_pages([b"good-1", b"stalled", b"good-3"]), timeout=5)
+
async def test_empty_pages_contribute_nothing(self) -> None:
"""Pages that OCR to empty text add neither a marker nor content."""
block = _make_file_parser_block("https://example.com/scan.pdf", FileType.PDF)
@@ -1254,6 +1298,62 @@ class TestParseStepTimeout:
assert captured["error_codes"] == ["FILE_PARSER_ERROR"]
assert "timed out" in captured["failure_reason"].lower()
+ @pytest.mark.parametrize(
+ ("handler_error", "expected_reason"),
+ [
+ (None, "did not answer within"),
+ (TimeoutError("gateway gave up on the request"), "gateway gave up on the request"),
+ ],
+ ids=["stalled_model_hits_the_deadline", "handler_timeout_keeps_its_own_cause"],
+ )
+ async def test_stalled_extraction_model_fails_the_block_at_the_deadline(
+ self, handler_error: Exception | None, expected_reason: str
+ ) -> None:
+ """A provider that never answers fails the block at the deadline instead of holding the run's slot."""
+ block = _make_file_parser_block("https://example.com/resume.pdf", FileType.PDF)
+ block.json_schema = {"type": "object", "properties": {"name": {"type": "string"}}}
+ mock_ctx = _mock_workflow_run_context()
+
+ async def stalled_handler(**kwargs: Any) -> Any:
+ if handler_error is not None:
+ raise handler_error
+ await asyncio.Event().wait()
+
+ with pytest.MonkeyPatch.context() as mp:
+ mp.setattr(FileParserBlock, "get_workflow_run_context", lambda *a, **kw: mock_ctx)
+ mp.setattr(FileParserBlock, "format_potential_template_parameters", lambda *a, **kw: None)
+ mp.setattr(
+ "skyvern.forge.sdk.workflow.models.block.resolve_local_or_download_file",
+ AsyncMock(return_value="/tmp/resume.pdf"),
+ )
+ mp.setattr(FileParserBlock, "validate_file_type", lambda *a, **kw: None)
+ mp.setattr(
+ "skyvern.forge.sdk.workflow.models.block.extract_pdf_file", lambda *a, **kw: "Jane Doe, engineer"
+ )
+ mp.setattr(
+ "skyvern.forge.sdk.workflow.models.block.LLMAPIHandlerFactory.get_override_llm_api_handler",
+ lambda *a, **kw: stalled_handler,
+ )
+ mp.setattr("skyvern.forge.sdk.workflow.models.block.FILE_PARSER_LLM_CALL_TIMEOUT_SECONDS", 0.05)
+ mp.setattr(FileParserBlock, "record_output_parameter_value", AsyncMock())
+
+ captured: dict[str, Any] = {}
+
+ async def fake_build_block_result(*args: Any, **kwargs: Any) -> Any:
+ captured.update(kwargs)
+ return MagicMock(success=kwargs.get("success"))
+
+ mp.setattr(FileParserBlock, "build_block_result", fake_build_block_result)
+
+ # The outer bound turns a missing deadline into a failure rather than a hung test run.
+ result = await asyncio.wait_for(
+ block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="org_test"),
+ timeout=10,
+ )
+
+ assert result.success is False
+ assert expected_reason in captured["failure_reason"]
+
async def test_validation_timeout_fails_the_block(self) -> None:
block = _make_file_parser_block("https://example.com/huge.pdf", FileType.PDF)
mock_ctx = _mock_workflow_run_context()