Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(RUS-5): preserve pinned selection in handler cleanup paths (#7879)
LawyZheng4f433105f5d0
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 8c439828f..bd4b9e64d 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -142,7 +142,7 @@ from skyvern.webeye.actions.actions import (WebAction,)from skyvern.webeye.actions.responses import ActionAbort, ActionFailure, ActionResult, ActionSuccess-from skyvern.webeye.browser_engine import BrowserEngineSelection, resolve_engine_selection_for_task+from skyvern.webeye.browser_engine import UNSET_SELECTION, BrowserEngineSelection, resolve_engine_selection_for_taskfrom skyvern.webeye.browser_factory import initialize_download_dir, resolve_artifact_pathfrom skyvern.webeye.browser_state import BrowserStatefrom skyvern.webeye.cdp_download_interceptor import (@@ -556,12 +556,16 @@ async def _reset_autocomplete_for_llm_fallback(text: str,task: Task,step: Step,+ engine_selection: BrowserEngineSelection | None = UNSET_SELECTION,) -> tuple[IncrementalScrapePage, list[dict], list[dict], str, list[str]]:+ if engine_selection is UNSET_SELECTION:+ engine_selection = resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)await current_incremental_scraped.stop_listen_dom_increment()await skyvern_element.input_clear()incremental_scraped = IncrementalScrapePage(- skyvern_frame=skyvern_frame, engine_selection=resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)+ skyvern_frame=skyvern_frame,+ engine_selection=engine_selection,)await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())await skyvern_element.press_fill(text)@@ -571,6 +575,7 @@ async def _reset_autocomplete_for_llm_fallback(task=task,step=step,check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],+ engine_selection=engine_selection,),)@@ -1333,12 +1338,15 @@ async def filter_out_elements(def clean_and_remove_element_tree_factory(- task: Task, step: Step, check_filter_funcs: list[CheckFilterOutElementIDFunc]+ task: Task,+ step: Step,+ check_filter_funcs: list[CheckFilterOutElementIDFunc],+ engine_selection: BrowserEngineSelection | None = UNSET_SELECTION,) -> CleanupElementTreeFunc:async def helper_func(frame: Page | Frame, url: str, element_tree: list[dict]) -> list[dict]:- element_tree = await app.AGENT_FUNCTION.cleanup_element_tree_factory(task=task, step=step)(- frame, url, element_tree- )+ element_tree = await app.AGENT_FUNCTION.cleanup_element_tree_factory(+ task=task, step=step, engine_selection=engine_selection+ )(frame, url, element_tree)for check_filter in check_filter_funcs:element_tree = await filter_out_elements(frame=frame, element_tree=element_tree, check_filter=check_filter)@@ -3562,10 +3570,13 @@ async def handle_click_action(else:incremental_scraped: IncrementalScrapePage | None = Nonetry:- skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())+ engine_selection = resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)+ skyvern_frame = await SkyvernFrame.create_instance(+ skyvern_element.get_frame(), engine_selection=engine_selection+ )incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame,- engine_selection=resolve_engine_selection_for_task(task, app.BROWSER_MANAGER),+ engine_selection=engine_selection,)await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())@@ -3692,7 +3703,10 @@ async def handle_sequential_click_for_dropdown(incremental_elements = await incremental_scraped.get_incremental_element_tree(clean_and_remove_element_tree_factory(- task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)]+ task=task,+ step=step,+ check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],+ engine_selection=skyvern_frame.engine_selection,),)@@ -3755,7 +3769,9 @@ async def handle_sequential_click_for_dropdown(),skyvern_element=anchor_element,element_tree_builder=scraped_page,+ task=task,step=step,+ engine_selection=skyvern_frame.engine_selection,)if dropdown_select_context.is_date_related:@@ -3784,6 +3800,7 @@ async def handle_sequential_click_for_dropdown(scraped_page=scraped_page,step=step,task=task,+ engine_selection=skyvern_frame.engine_selection,entry_action_type="click",scraped_page_after_open=scraped_page_after_open,new_interactable_element_ids=new_interactable_element_ids,@@ -4147,7 +4164,7 @@ async def handle_input_text_action(can_input_text = Trueengine_selection = resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)- skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())+ skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame(), engine_selection=engine_selection)incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame, engine_selection=engine_selection)timeout = settings.BROWSER_ACTION_TIMEOUT_MS@@ -4219,7 +4236,9 @@ async def handle_input_text_action(action=action,element_tree_builder=scraped_page,skyvern_element=skyvern_element,+ task=task,step=step,+ engine_selection=engine_selection,)if not can_input_text:target_is_hidden = target_hidden if target_hidden is not None else await skyvern_element.has_hidden_attr()@@ -4303,7 +4322,10 @@ async def handle_input_text_action(await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=wait_sec, caller="input_text.autocomplete")incremental_element = await incremental_scraped.get_incremental_element_tree(clean_and_remove_element_tree_factory(- task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)]+ task=task,+ step=step,+ check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],+ engine_selection=engine_selection,),)if len(incremental_element) == 0:@@ -4705,6 +4727,7 @@ async def handle_input_text_action(task=task,step=step,check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],+ engine_selection=engine_selection,),)if len(incremental_element) > 0:@@ -4909,7 +4932,7 @@ async def _wait_for_upload_processing(page: Page, engine_selection: BrowserEngin# Settle delay: let the page react to the file-input change and mount# upload UI (spinner, progress bar, XHR) before polling for readiness.await asyncio.sleep(0.5)- skyvern_frame = await SkyvernFrame.create_instance(page)+ skyvern_frame = await SkyvernFrame.create_instance(page, engine_selection=engine_selection)await skyvern_frame.wait_for_page_ready(loading_indicator_timeout_ms=3000,network_idle_timeout_ms=3000,@@ -5002,14 +5025,13 @@ async def handle_upload_file_action(if is_file_input:LOG.info("Taking UploadFileAction. Found file input tag", action=action)if file_path:+ engine_selection = resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)await locator.set_input_files(file_path,timeout=settings.BROWSER_ACTION_TIMEOUT_MS,)- await _wait_for_upload_processing(- page, engine_selection=resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)- )+ await _wait_for_upload_processing(page, engine_selection=engine_selection)return [ActionSuccess()]else:@@ -5116,6 +5138,7 @@ async def handle_select_option_action() -> list[ActionResult]:dom = DomUtil(scraped_page, page)skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)+ engine_selection = resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)tag_name = skyvern_element.get_tag_name()element_dict = scraped_page.id_to_element_dict[action.element_id]@@ -5216,7 +5239,12 @@ async def handle_select_option_action(if select_is_visible:try:normal_select_result = await normal_select(- action=action, skyvern_element=skyvern_element, builder=dom.scraped_page, task=task, step=step+ action=action,+ skyvern_element=skyvern_element,+ builder=dom.scraped_page,+ task=task,+ step=step,+ engine_selection=engine_selection,)except Exception as e:# normal_select can raise before returning (e.g. an LLM/provider error). Don't lose@@ -5314,10 +5342,8 @@ async def handle_select_option_action()timeout = settings.BROWSER_ACTION_TIMEOUT_MS- skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())- incremental_scraped = IncrementalScrapePage(- skyvern_frame=skyvern_frame, engine_selection=resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)- )+ skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame(), engine_selection=engine_selection)+ incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame, engine_selection=engine_selection)is_open = Falsesuggested_value: str | None = Noneresults: list[ActionResult] = []@@ -5331,7 +5357,7 @@ async def handle_select_option_action(page=page,dom=dom,timeout=timeout,- engine_selection=resolve_engine_selection_for_task(task, app.BROWSER_MANAGER),+ engine_selection=engine_selection,)# The click opens the widget: mark it open now (not only on the incremental path below) so the# finally cleanup dismisses it on every exit — including an emerging-path optional miss that@@ -5342,7 +5368,10 @@ async def handle_select_option_action(incremental_element = await incremental_scraped.get_incremental_element_tree(clean_and_remove_element_tree_factory(- task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)]+ task=task,+ step=step,+ check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],+ engine_selection=engine_selection,),)@@ -5357,12 +5386,20 @@ async def handle_select_option_action(await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=0.5, caller="select_option.arrowdown")incremental_element = await incremental_scraped.get_incremental_element_tree(clean_and_remove_element_tree_factory(- task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)]+ task=task,+ step=step,+ check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],+ engine_selection=engine_selection,),)input_or_select_context = await _get_input_or_select_context(- action=action, element_tree_builder=scraped_page, step=step, skyvern_element=skyvern_element+ action=action,+ element_tree_builder=scraped_page,+ task=task,+ step=step,+ skyvern_element=skyvern_element,+ engine_selection=engine_selection,)if len(incremental_element) == 0:@@ -5383,6 +5420,7 @@ async def handle_select_option_action(task=task,step=step,entry_action_type=entry_action_type,+ engine_selection=engine_selection,))return results@@ -6693,7 +6731,7 @@ async def choose_auto_completion_dropdown(engine_selection = resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)current_frame = skyvern_element.get_frame()- skyvern_frame = await SkyvernFrame.create_instance(current_frame)+ skyvern_frame = await SkyvernFrame.create_instance(current_frame, engine_selection=engine_selection)incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame, engine_selection=engine_selection)await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())@@ -6703,7 +6741,10 @@ async def choose_auto_completion_dropdown(await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1, caller="autocomplete.fill")incremental_element = await incremental_scraped.get_incremental_element_tree(clean_and_remove_element_tree_factory(- task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)]+ task=task,+ step=step,+ check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],+ engine_selection=engine_selection,),)@@ -6730,9 +6771,9 @@ async def choose_auto_completion_dropdown(confirmed_preserved_list.append(current_element)if len(confirmed_preserved_list) > 0:- confirmed_preserved_list = await app.AGENT_FUNCTION.cleanup_element_tree_factory(task=task, step=step)(- skyvern_frame.get_frame(), skyvern_frame.get_frame().url, copy.deepcopy(confirmed_preserved_list)- )+ confirmed_preserved_list = await app.AGENT_FUNCTION.cleanup_element_tree_factory(+ task=task, step=step, engine_selection=engine_selection+ )(skyvern_frame.get_frame(), skyvern_frame.get_frame().url, copy.deepcopy(confirmed_preserved_list))confirmed_preserved_list = trim_element_tree(copy.deepcopy(confirmed_preserved_list))incremental_element.extend(confirmed_preserved_list)@@ -6784,6 +6825,7 @@ async def choose_auto_completion_dropdown(text=text,task=task,step=step,+ engine_selection=engine_selection,)result.incremental_elements = copy.deepcopy(fallback_incremental_elements)cleaned_incremental_element = shadow_candidate_elements@@ -6836,6 +6878,7 @@ async def choose_auto_completion_dropdown(text=text,task=task,step=step,+ engine_selection=engine_selection,)result.incremental_elements = copy.deepcopy(fallback_incremental_elements)cleaned_incremental_element = shadow_candidate_elements@@ -6865,6 +6908,7 @@ async def choose_auto_completion_dropdown(text=text,task=task,step=step,+ engine_selection=engine_selection,)result.incremental_elements = copy.deepcopy(fallback_incremental_elements)cleaned_incremental_element = shadow_candidate_elements@@ -7240,9 +7284,11 @@ async def discover_and_select_from_full_dropdown(return Nonecurrent_frame = skyvern_element.get_frame()- skyvern_frame = await SkyvernFrame.create_instance(current_frame)+ engine_selection = resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)+ skyvern_frame = await SkyvernFrame.create_instance(current_frame, engine_selection=engine_selection)incremental_scraped = IncrementalScrapePage(- skyvern_frame=skyvern_frame, engine_selection=resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)+ skyvern_frame=skyvern_frame,+ engine_selection=engine_selection,)await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())@@ -7265,6 +7311,7 @@ async def discover_and_select_from_full_dropdown(task=task,step=step,check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],+ engine_selection=engine_selection,)incremental_element = await incremental_scraped.get_incremental_element_tree(cleanup_func)@@ -7277,7 +7324,7 @@ async def discover_and_select_from_full_dropdown(try:await skyvern_element.press_key("ArrowDown")except Exception as exc:- if not _is_selected_engine_timeout(exc, resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)):+ if not _is_selected_engine_timeout(exc, engine_selection):raiseLOG.info("Timeout pressing ArrowDown in discover fallback, continuing",@@ -7508,6 +7555,7 @@ async def sequentially_select_from_dropdown(task=task,step=step,check_filter_funcs=check_filter_funcs,+ engine_selection=skyvern_frame.engine_selection,))if len(secondary_increment_element) == 0:@@ -8136,7 +8184,11 @@ async def _select_deterministic_custom_option(selection_group_id: str | None = None,select_depth: int = 0,on_click_attempted: Callable[[], None] | None = None,+ engine_selection: BrowserEngineSelection | None = UNSET_SELECTION,) -> tuple[ActionResult, str | None] | None:+ if engine_selection is UNSET_SELECTION:+ engine_selection = resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)+started_at = time.monotonic()selection_group_id = selection_group_id or str(uuid.uuid4())option_count: int | None = None@@ -8290,9 +8342,7 @@ async def _select_deterministic_custom_option(click_attempted = Trueif on_click_attempted is not None:on_click_attempted()- await selected_element.click(- page=page, engine_selection=resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)- )+ await selected_element.click(page=page, engine_selection=engine_selection)verified = await _verify_custom_select_option_with_settle(matched_element=selected_element,readback_scope_element=readback_scope_element,@@ -8330,7 +8380,7 @@ async def _select_deterministic_custom_option(reset_verified = await _reset_custom_select_combobox_input(readback_scope_element,page,- engine_selection=resolve_engine_selection_for_task(task, app.BROWSER_MANAGER),+ engine_selection=engine_selection,)if reset_verified:LOG.info(@@ -8446,12 +8496,15 @@ async def select_from_emerging_elements(entry_action_type: str = "select_option",scraped_page_after_open: ScrapedPage | None = None,new_interactable_element_ids: list[str] | None = None,+ engine_selection: BrowserEngineSelection | None = UNSET_SELECTION,) -> ActionResult:"""This is the function to select an element from the new showing elements.Currently mainly used for the dropdown menu selection."""+ if engine_selection is UNSET_SELECTION:+ engine_selection = resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)selection_group_id = str(uuid.uuid4())# TODO: support to handle the case when options are loaded by scrollscraped_page_after_open = scraped_page_after_open or await scraped_page.generate_scraped_page_without_screenshots()@@ -8532,6 +8585,7 @@ async def select_from_emerging_elements(selection_group_id=selection_group_id,select_depth=0,on_click_attempted=_mark_widget_mutated,+ engine_selection=engine_selection,)if deterministic_result is not None:action_result, _matched_label = deterministic_result@@ -8604,7 +8658,7 @@ async def select_from_emerging_elements(current_text = await get_input_value(input_element.get_tag_name(),input_element.get_locator(),- engine_selection=resolve_engine_selection_for_task(task, app.BROWSER_MANAGER),+ engine_selection=engine_selection,)if current_text == actual_value:return ActionSuccess()@@ -8626,9 +8680,7 @@ async def select_from_emerging_elements(return ActionFailure(exception=InteractWithDropdownContainer(element_id=element_id))await selected_element.scroll_into_view()- await selected_element.click(- page=page, engine_selection=resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)- )+ await selected_element.click(page=page, engine_selection=engine_selection)return ActionSuccess()@@ -8692,7 +8744,12 @@ async def select_from_dropdown()trimmed_element_tree = await incremental_scraped.get_incremental_element_tree(- clean_and_remove_element_tree_factory(task=task, step=step, check_filter_funcs=check_filter_funcs),+ clean_and_remove_element_tree_factory(+ task=task,+ step=step,+ check_filter_funcs=check_filter_funcs,+ engine_selection=skyvern_frame.engine_selection,+ ),)incremental_scraped.set_element_tree_trimmed(trimmed_element_tree)html = incremental_scraped.build_element_tree(html_need_skyvern_attrs=True)@@ -8717,6 +8774,7 @@ async def select_from_dropdown(selection_group_id=selection_group_id or str(uuid.uuid4()),select_depth=len(select_history),on_click_attempted=_mark_widget_mutated,+ engine_selection=skyvern_frame.engine_selection,)if deterministic_result is not None:action_result, matched_label = deterministic_result@@ -8802,7 +8860,7 @@ async def select_from_dropdown(current_text = await get_input_value(input_element.get_tag_name(),input_element.get_locator(),- engine_selection=resolve_engine_selection_for_task(task, app.BROWSER_MANAGER),+ engine_selection=skyvern_frame.engine_selection,)if current_text == actual_value:single_select_result.action_result = ActionSuccess()@@ -8838,7 +8896,12 @@ async def select_from_dropdown(input_or_select_context=context,)results = await normal_select(- action=action, skyvern_element=selected_element, task=task, step=step, builder=incremental_scraped+ action=action,+ skyvern_element=selected_element,+ task=task,+ step=step,+ builder=incremental_scraped,+ engine_selection=skyvern_frame.engine_selection,)assert len(results) > 0single_select_result.action_result = results[0]@@ -8854,7 +8917,7 @@ async def select_from_dropdown(await selected_element.click(page=page,timeout=timeout,- engine_selection=resolve_engine_selection_for_task(task, app.BROWSER_MANAGER),+ engine_selection=skyvern_frame.engine_selection,)single_select_result.action_result = ActionSuccess()return single_select_result@@ -8907,7 +8970,10 @@ async def select_from_dropdown_by_value(timeout = settings.BROWSER_ACTION_TIMEOUT_MSawait incremental_scraped.get_incremental_element_tree(clean_and_remove_element_tree_factory(- task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)]+ task=task,+ step=step,+ check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],+ engine_selection=skyvern_frame.engine_selection,),)@@ -8944,7 +9010,10 @@ async def select_from_dropdown_by_value(async def continue_callback(incre_scraped: IncrementalScrapePage) -> bool:await incre_scraped.get_incremental_element_tree(clean_and_remove_element_tree_factory(- task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)]+ task=task,+ step=step,+ check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],+ engine_selection=skyvern_frame.engine_selection,),)@@ -9070,7 +9139,9 @@ async def locate_dropdown_menu(x, y = await skyvern_frame.get_scroll_x_y()try:screenshot = await take_element_screenshot(- head_element.get_locator(), timeout=settings.BROWSER_SCREENSHOT_TIMEOUT_MS+ head_element.get_locator(),+ timeout=settings.BROWSER_SCREENSHOT_TIMEOUT_MS,+ engine_selection=skyvern_frame.engine_selection,)except FailedToTakeScreenshot:LOG.debug(@@ -9210,6 +9281,7 @@ async def normal_select(task: Task,step: Step,builder: ElementTreeBuilder,+ engine_selection: BrowserEngineSelection | None = UNSET_SELECTION,) -> List[ActionResult]:collapse_select_fanout_enabled = await _is_collapse_select_fanout_enabled(task)if not collapse_select_fanout_enabled:@@ -9229,8 +9301,10 @@ async def normal_select(input_or_select_context = await _get_input_or_select_context(action=action,element_tree_builder=builder,+ task=task,step=step,skyvern_element=skyvern_element,+ engine_selection=engine_selection,)LOG.debug("Parsed input/select context",@@ -9967,13 +10041,17 @@ async def _get_input_or_select_context(element_tree_builder: ElementTreeBuilder,step: Step,ancestor_depth: int = 5,+ task: Task | None = None,+ engine_selection: BrowserEngineSelection | None = UNSET_SELECTION,) -> InputOrSelectContext:# Early return optimization: if action already has input_or_select_context, use itif not isinstance(action, AbstractActionForContextParse) and action.input_or_select_context is not None:return action.input_or_select_context# Ancestor depth optimization: use ancestor element for deep DOM structures- skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())+ if engine_selection is UNSET_SELECTION:+ engine_selection = resolve_engine_selection_for_task(task, app.BROWSER_MANAGER)+ skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame(), engine_selection=engine_selection)try:depth = await skyvern_frame.get_element_dom_depth(await skyvern_element.get_element_handler())except Exception:@@ -9991,7 +10069,9 @@ async def _get_input_or_select_context(starter=element_handle,frame=skyvern_element.get_frame_id(),)- clean_up_func = app.AGENT_FUNCTION.cleanup_element_tree_factory(step=step)+ clean_up_func = app.AGENT_FUNCTION.cleanup_element_tree_factory(+ step=step, engine_selection=engine_selection+ )element_tree = await clean_up_func(skyvern_element.get_frame(), "", copy.deepcopy(element_tree))element_tree_trimmed = trim_element_tree(copy.deepcopy(element_tree))element_tree_builder = ScrapedPage(diff --git a/tests/unit/test_auto_completion_discover_fallback.py b/tests/unit/test_auto_completion_discover_fallback.pyindex bc7a8bc1e..6d9b09b06 100644--- a/tests/unit/test_auto_completion_discover_fallback.py+++ b/tests/unit/test_auto_completion_discover_fallback.py@@ -120,6 +120,8 @@ async def test_discover_fallback_succeeds_when_options_appear() -> None:frame = _mock_frame(locator_count=1)skyvern_el = _mock_skyvern_element(frame)inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)+ selection = object()+ cleanup_factory = MagicMock(return_value=AsyncMock())llm_response = {"auto_completion_attempt": True,@@ -141,6 +143,14 @@ async def test_discover_fallback_succeeds_when_options_appear() -> None:patch("skyvern.webeye.actions.handler.app") as mock_app,patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,+ patch(+ "skyvern.webeye.actions.handler.resolve_engine_selection_for_task",+ return_value=selection,+ ) as resolve,+ patch(+ "skyvern.webeye.actions.handler.clean_and_remove_element_tree_factory",+ cleanup_factory,+ ),):mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)mock_prompt.load_prompt.return_value = "mocked prompt"@@ -163,6 +173,9 @@ async def test_discover_fallback_succeeds_when_options_appear() -> None:# Types the discovered value, then finds and clicks matched option via Playwright locatorskyvern_el.press_fill.assert_awaited_with("I do not wish to disclose")inc_scrape.stop_listen_dom_increment.assert_awaited()+ resolve.assert_called_once_with(_TASK, mock_app.BROWSER_MANAGER)+ assert cleanup_factory.call_count == 1+ assert cleanup_factory.call_args.kwargs["engine_selection"] is selection@pytest.mark.asynciodiff --git a/tests/unit/test_element_engine_selection_threading.py b/tests/unit/test_element_engine_selection_threading.pyindex d7280b23e..1ee19591b 100644--- a/tests/unit/test_element_engine_selection_threading.py+++ b/tests/unit/test_element_engine_selection_threading.py@@ -8,6 +8,8 @@ import pytestfrom skyvern.forge import agent_functionsfrom skyvern.webeye.actions import handler+from skyvern.webeye.actions.actions import InputTextAction, SelectOption, SelectOptionAction+from skyvern.webeye.actions.responses import ActionFailurefrom skyvern.webeye.browser_engine import BrowserEngineMetadata, BrowserEngineSelection@@ -146,7 +148,7 @@ async def test_autocomplete_threads_one_fresh_selection_snapshot(monkeypatch: pyselection = _selection()task = _task()frame = MagicMock()- locator = MagicMock(count=AsyncMock(return_value=1))+ locator = MagicMock(count=AsyncMock(return_value=1), element_handle=AsyncMock(return_value=MagicMock()))frame.locator.return_value = locatorsource_element = MagicMock(get_frame=MagicMock(return_value=frame),@@ -155,11 +157,19 @@ async def test_autocomplete_threads_one_fresh_selection_snapshot(monkeypatch: pyinput_clear=AsyncMock(),is_visible=AsyncMock(return_value=False),)- skyvern_frame = MagicMock(safe_wait_for_animation_end=AsyncMock())+ skyvern_frame = MagicMock(+ safe_wait_for_animation_end=AsyncMock(), parse_element_from_html=AsyncMock(return_value={})+ )+ cleanup = AsyncMock(side_effect=lambda _frame, _url, tree: tree)+ cleanup_factory = MagicMock(return_value=cleanup)++ async def scrape(cleaner: object) -> list[dict]:+ return await cleaner(frame, "", [{"id": "option-1", "text": "match"}]) # type: ignore[operator]+incremental = MagicMock(start_listen_dom_increment=AsyncMock(),stop_listen_dom_increment=AsyncMock(),- get_incremental_element_tree=AsyncMock(return_value=[{"id": "option-1", "text": "match"}]),+ get_incremental_element_tree=AsyncMock(side_effect=scrape),build_html_tree=MagicMock(return_value="<div>match</div>"),id_to_element_dict={"option-1": {"id": "option-1", "tagName": "div"}},)@@ -171,6 +181,7 @@ async def test_autocomplete_threads_one_fresh_selection_snapshot(monkeypatch: pyincremental_constructor = MagicMock(return_value=incremental)monkeypatch.setattr(handler, "IncrementalScrapePage", incremental_constructor)monkeypatch.setattr(handler, "SkyvernElement", constructor)+ monkeypatch.setattr(handler.app.AGENT_FUNCTION, "cleanup_element_tree_factory", cleanup_factory)monkeypatch.setattr(handler.app,"AUTO_COMPLETION_LLM_API_HANDLER",@@ -183,7 +194,6 @@ async def test_autocomplete_threads_one_fresh_selection_snapshot(monkeypatch: py"ensure_context",MagicMock(return_value=SimpleNamespace(tz_info=UTC)),)-await handler.choose_auto_completion_dropdown(context=MagicMock(is_search_bar=False, field="field", intention=None),page=MagicMock(),@@ -193,14 +203,153 @@ async def test_autocomplete_threads_one_fresh_selection_snapshot(monkeypatch: pyskyvern_element=source_element,step=MagicMock(),task=task,+ preserved_elements=[{"id": "preserved"}],+ )+ await handler._reset_autocomplete_for_llm_fallback(+ current_incremental_scraped=incremental,+ skyvern_frame=skyvern_frame,+ skyvern_element=source_element,+ page=MagicMock(),+ scraped_page=MagicMock(),+ dom=MagicMock(),+ text="match",+ task=task,+ step=MagicMock(),+ engine_selection=selection,)resolver.assert_called_once_with(task, handler.app.BROWSER_MANAGER)+ assert cleanup.await_count == 3+ assert all(call.kwargs["engine_selection"] is selection for call in cleanup_factory.call_args_list)assert incremental_constructor.call_args.kwargs["engine_selection"] is selection- assert constructor.call_args.kwargs["engine_selection"] is selection+ assert any(call.kwargs["engine_selection"] is selection for call in constructor.call_args_list)selected_element.click.assert_awaited_once()assert selected_element.click.await_args.kwargs["engine_selection"] is selection+async def test_input_handler_passes_its_selection_snapshot_to_context_parser(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ selection = _selection()+ task = _task()+ element = MagicMock(+ supports_text_input=AsyncMock(return_value=False),+ has_hidden_attr=AsyncMock(return_value=True),+ get_selectable=AsyncMock(return_value=False),+ )+ element.get_frame.return_value = MagicMock()+ element.get_tag_name.return_value = "input"+ element.get_id.return_value = "input-1"+ dom = MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=element))+ scraped_page = MagicMock(id_to_element_dict={"input-1": {"tagName": "input"}})+ resolver = MagicMock(return_value=selection)+ context_parser = AsyncMock(return_value=MagicMock(is_date_related=False))+ monkeypatch.setattr(handler, "DomUtil", MagicMock(return_value=dom))+ monkeypatch.setattr(handler, "resolve_engine_selection_for_task", resolver)+ monkeypatch.setattr(handler.SkyvernFrame, "create_instance", AsyncMock(return_value=MagicMock()))+ monkeypatch.setattr(handler, "IncrementalScrapePage", MagicMock())+ monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value=""))+ monkeypatch.setattr(handler, "get_actual_value_of_parameter_if_secret_with_task", MagicMock(return_value="hello"))+ monkeypatch.setattr(handler.SkyvernElement, "wait_until_enabled", AsyncMock(return_value=True))+ monkeypatch.setattr(handler, "_get_input_or_select_context", context_parser)+ await handler.handle_input_text_action(+ InputTextAction(element_id="input-1", text="hello"),+ MagicMock(),+ scraped_page,+ task,+ MagicMock(),+ )+ resolver.assert_called_once_with(task, handler.app.BROWSER_MANAGER)+ context_parser.assert_awaited_once()+ assert context_parser.await_args.kwargs["engine_selection"] is selection+++async def test_custom_select_handler_passes_its_selection_snapshot_to_context_parser(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ selection = _selection()+ task = _task()+ element = _element_double()+ element.get_id.return_value, element.get_tag_name.return_value = "select-1", "div"+ element.get_frame.return_value = MagicMock()+ element.is_selectable = AsyncMock(return_value=True)+ for method in ("is_checkbox", "is_radio", "is_btn_input"):+ setattr(element, method, AsyncMock(return_value=False))+ element.is_visible = AsyncMock(return_value=False)+ element.blur = AsyncMock()+ dom = MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=element))+ scraped_page = MagicMock(id_to_element_dict={"select-1": {"tagName": "div"}})+ resolver = MagicMock(return_value=selection)+ context_parser = AsyncMock(+ return_value=MagicMock(is_date_related=False, intention=None, field=None, is_required=False)+ )+ incremental = MagicMock(+ start_listen_dom_increment=AsyncMock(),+ stop_listen_dom_increment=AsyncMock(),+ get_incremental_element_tree=AsyncMock(return_value=[]),+ )+ monkeypatch.setattr(handler, "DomUtil", MagicMock(return_value=dom))+ monkeypatch.setattr(handler, "resolve_engine_selection_for_task", resolver)+ monkeypatch.setattr(+ handler.SkyvernFrame,+ "create_instance",+ AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),+ )+ monkeypatch.setattr(handler, "IncrementalScrapePage", MagicMock(return_value=incremental))+ monkeypatch.setattr(handler.SkyvernElement, "wait_until_enabled", AsyncMock(return_value=True))+ monkeypatch.setattr(handler, "_get_input_or_select_context", context_parser)+ monkeypatch.setattr(+ handler,+ "select_from_emerging_elements",+ AsyncMock(return_value=ActionFailure(Exception("stop after context parsing"))),+ )+ await handler.handle_select_option_action(+ SelectOptionAction(element_id="select-1", option=SelectOption(label="Choice")),+ MagicMock(),+ scraped_page,+ task,+ MagicMock(),+ )+ resolver.assert_called_once_with(task, handler.app.BROWSER_MANAGER)+ context_parser.assert_awaited_once()+ assert context_parser.await_args.kwargs["engine_selection"] is selection+++async def test_dropdown_screenshot_uses_frame_selection_and_skips_selected_timeout(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ selection = _selection()+ locator = MagicMock(+ page=MagicMock(is_closed=MagicMock(return_value=False)),+ screenshot=AsyncMock(side_effect=_SelectedTimeout("timed out")),+ )+ candidate = MagicMock(+ get_locator=MagicMock(return_value=locator),+ is_next_to_element=AsyncMock(return_value=True),+ find_children_element_id_by_callback=AsyncMock(return_value=None),+ get_attr=AsyncMock(return_value=None),+ )+ frame = MagicMock(+ engine_selection=selection,+ get_element_visible=AsyncMock(return_value=True),+ get_scroll_x_y=AsyncMock(return_value=(4, 8)),+ safe_scroll_to_x_y=AsyncMock(),+ )+ incremental = MagicMock(skyvern_frame=frame, element_tree=[{"id": "candidate"}])+ anchor = MagicMock(is_visible=AsyncMock(return_value=True))+ monkeypatch.setattr(handler.SkyvernElement, "create_from_incremental", AsyncMock(return_value=candidate))+ llm = AsyncMock()+ monkeypatch.setattr(handler.app, "SECONDARY_LLM_API_HANDLER", llm)++ assert await handler.locate_dropdown_menu(anchor, incremental, MagicMock(), _task()) is None++ assert locator.screenshot.await_count == 2+ frame.safe_scroll_to_x_y.assert_awaited_once_with(4, 8)+ llm.assert_not_awaited()++def test_selected_engine_rejects_foreign_error_family() -> None:selection = _selection()assert selection.is_engine_error(_SelectedError())diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.pyindex c29a3f14b..a9a379769 100644--- a/tests/unit/test_select_shadow_match.py+++ b/tests/unit/test_select_shadow_match.py@@ -807,7 +807,7 @@ async def test_click_ingress_routes_no_target_and_emits_nothing(monkeypatch: pytanchor_element=MagicMock(get_id=Mock(return_value="field-id")),dom=MagicMock(),page=SimpleNamespace(url="https://test"),- skyvern_frame=SimpleNamespace(safe_wait_for_animation_end=AsyncMock()),+ skyvern_frame=SimpleNamespace(safe_wait_for_animation_end=AsyncMock(), engine_selection=object()),scraped_page=scraped_page,incremental_scraped=incremental_scraped,task=_OUTCOME_TASK,diff --git a/tests/unit/test_upload_file_wait.py b/tests/unit/test_upload_file_wait.pyindex 9fc46e746..41f012440 100644--- a/tests/unit/test_upload_file_wait.py+++ b/tests/unit/test_upload_file_wait.py@@ -6,7 +6,7 @@ import astimport asyncioimport inspectimport textwrap-from unittest.mock import AsyncMock, patch+from unittest.mock import AsyncMock, MagicMock, Mock, patchimport pytestfrom playwright._impl._errors import Error as PlaywrightError@@ -14,6 +14,7 @@ from playwright.async_api import TimeoutError as PlaywrightTimeoutErrorfrom skyvern.exceptions import SkyvernPageAnalysisTimeoutfrom skyvern.webeye.actions import handler as handler_module+from skyvern.webeye.actions.actions import UploadFileActionfrom skyvern.webeye.actions.handler import _wait_for_upload_processingfrom skyvern.webeye.browser_engine import BrowserEngineMetadata, BrowserEngineSelection@@ -56,16 +57,19 @@ async def _run_with_page_ready_error(error: BaseException, engine_selection: Bro@pytest.mark.asyncioasync def test_calls_wait_for_page_ready_with_settle_delay() -> None:+ page = AsyncMock()+ engine_selection = _engine_selection()mock_frame = AsyncMock()with (patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance", new_callable=AsyncMock) as mock_create,patch("skyvern.webeye.actions.handler.asyncio.sleep", new_callable=AsyncMock) as mock_sleep,):mock_create.return_value = mock_frame- await _wait_for_upload_processing(AsyncMock())+ await _wait_for_upload_processing(page, engine_selection=engine_selection)# Settle delay before readiness pollingmock_sleep.assert_awaited_once_with(0.5)+ mock_create.assert_awaited_once_with(page, engine_selection=engine_selection)mock_frame.wait_for_page_ready.assert_awaited_once_with(loading_indicator_timeout_ms=3000,network_idle_timeout_ms=3000,@@ -74,6 +78,76 @@ async def test_calls_wait_for_page_ready_with_settle_delay() -> None:)[email protected]("engine_selection", [_engine_selection(), None])+async def test_input_or_select_context_reuses_task_engine_selection(+ engine_selection: BrowserEngineSelection | None,+) -> None:+ task = Mock()+ step = Mock()+ frame = Mock()+ element = Mock()+ element.get_frame.return_value = frame+ element.get_element_handler = AsyncMock(return_value=Mock())+ element.get_frame_id.return_value = "frame-id"+ element.get_locator.return_value.locator.return_value.element_handle = AsyncMock(return_value=Mock())+ skyvern_frame = AsyncMock()+ skyvern_frame.get_element_dom_depth.return_value = 6+ skyvern_frame.build_tree_from_element.return_value = ([], [])+ cleanup_factory = Mock(return_value=AsyncMock(return_value=[]))+ app_instance = object.__getattribute__(handler_module.app, "_inst")++ with (+ patch.object(handler_module, "resolve_engine_selection_for_task", return_value=engine_selection) as resolve,+ patch.object(handler_module.SkyvernFrame, "create_instance", AsyncMock(return_value=skyvern_frame)) as create,+ patch.object(handler_module.app.AGENT_FUNCTION, "cleanup_element_tree_factory", cleanup_factory),+ patch.object(app_instance, "PARSE_SELECT_LLM_API_HANDLER", AsyncMock(return_value={}), create=True),+ ):+ await handler_module._get_input_or_select_context(+ action=handler_module.AbstractActionForContextParse(reasoning=None, element_id="element", intention=None),+ skyvern_element=element,+ element_tree_builder=Mock(),+ task=task,+ step=step,+ engine_selection=engine_selection,+ )+ resolve.assert_not_called()+ create.assert_awaited_once_with(frame, engine_selection=engine_selection)+ cleanup_factory.assert_called_once_with(step=step, engine_selection=engine_selection)++[email protected]("selection", [_engine_selection(), None])+async def test_upload_resolves_before_dispatch_and_reuses_exact_selection(selection: object | None) -> None:+ events: list[str] = []+ locator = MagicMock(set_input_files=AsyncMock(side_effect=lambda *args, **kwargs: events.append("dispatch")))+ element = MagicMock(locator=locator, is_file_input=AsyncMock(return_value=True))+ dom = MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=element))+ task = MagicMock(navigation_goal="https://example.test/file", navigation_payload={}, organization_id="org")+ resolver = Mock(side_effect=lambda *args: events.append("resolve") or selection)+ wait = AsyncMock(side_effect=lambda *args, **kwargs: events.append("wait"))+ with (+ patch.object(handler_module, "DomUtil", return_value=dom),+ patch.object(+ handler_module, "get_actual_value_of_parameter_if_secret_with_task", side_effect=lambda _, url: url+ ),+ patch.object(handler_module.handler_utils, "download_file", AsyncMock(return_value="/tmp/file")),+ patch.object(handler_module.SkyvernElement, "wait_until_enabled", AsyncMock(return_value=True)),+ patch.object(handler_module, "resolve_engine_selection_for_task", resolver),+ patch.object(handler_module, "_wait_for_upload_processing", wait),+ ):+ await handler_module.handle_upload_file_action(+ UploadFileAction(element_id="file", file_url="https://example.test/file"),+ MagicMock(),+ MagicMock(),+ task,+ MagicMock(),+ )+ assert events == ["resolve", "dispatch", "wait"]+ resolver.assert_called_once_with(task, handler_module.app.BROWSER_MANAGER)+ assert wait.await_args.kwargs["engine_selection"] is selection++@pytest.mark.asyncioasync def test_swallows_playwright_timeout() -> None:mock_frame = AsyncMock()