Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9334): defer file chooser listener across steps for two-step uploads (#5937)
LawyZheng94458a9c22a2
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex 23dc8cab0..338081e28 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -3,7 +3,7 @@ from __future__ import annotationsfrom contextlib import contextmanagerfrom contextvars import ContextVar, Tokenfrom dataclasses import dataclass, field-from typing import TYPE_CHECKING, Any, Iterator, TypedDict+from typing import TYPE_CHECKING, Any, Callable, Iterator, TypedDictfrom zoneinfo import ZoneInfoimport structlog@@ -11,7 +11,7 @@ import structlogfrom skyvern.config import settingsif TYPE_CHECKING:- from playwright.async_api import Frame, Page+ from playwright.async_api import FileChooser, Frame, Pagefrom skyvern.forge.sdk.db.enums import WorkflowRunTriggerType@@ -31,6 +31,22 @@ class DialogEntry(TypedDict):count: int+@dataclass+class PendingFileChooserListener:+ page: Page+ file_paths: list[str] | str+ handler: Callable[[FileChooser], Any] | None = None+ triggered: bool = False++ def cleanup(self) -> None:+ if self.handler is not None:+ try:+ self.page.remove_listener("filechooser", self.handler)+ except Exception:+ LOG.debug("Failed to remove filechooser listener during cleanup", exc_info=True)+ self.handler = None++@dataclassclass SkyvernContext:request_id: str | None = None@@ -145,6 +161,17 @@ class SkyvernContext:# alongside the existing input_tokens / llm_cost on the same row.last_prompt_breakdown: dict[str, Any] | None = None+ # Deferred file chooser listener — survives across steps so a popup-intercepted upload+ # can be completed when a subsequent click triggers the actual file chooser.+ pending_file_chooser: PendingFileChooserListener | None = None++ def cleanup_pending_file_chooser(self) -> None:+ if self.pending_file_chooser is not None:+ if not self.pending_file_chooser.triggered:+ LOG.warning("Cleaning up unconsumed pending file chooser listener")+ self.pending_file_chooser.cleanup()+ self.pending_file_chooser = None+def __repr__(self) -> str:return f"SkyvernContext(request_id={self.request_id}, organization_id={self.organization_id}, task_id={self.task_id}, step_id={self.step_id}, workflow_id={self.workflow_id}, workflow_run_id={self.workflow_run_id}, task_v2_id={self.task_v2_id}, max_steps_override={self.max_steps_override}, run_id={self.run_id}, copilot_session_id={self.copilot_session_id})"@@ -292,13 +319,16 @@ def replace(context: SkyvernContext) -> None:Returns:None"""- _flush_feature_flags_if_needed(current())+ _cleanup_outgoing_context(current())_context.set(context)-def _flush_feature_flags_if_needed(context: SkyvernContext | None) -> None:- if context is not None and context.feature_flag_entries:+def _cleanup_outgoing_context(context: SkyvernContext | None) -> None:+ if context is None:+ return+ if context.feature_flag_entries:context.flush_feature_flags()+ context.cleanup_pending_file_chooser()def _restore(token: Token[SkyvernContext | None]) -> None:@@ -311,7 +341,7 @@ def _restore(token: Token[SkyvernContext | None]) -> None:Returns:None"""- _flush_feature_flags_if_needed(current())+ _cleanup_outgoing_context(current())_context.reset(token)@@ -340,5 +370,5 @@ def reset() -> None:Returns:None"""- _flush_feature_flags_if_needed(current())+ _cleanup_outgoing_context(current())_context.set(None)diff --git a/skyvern/services/action_service.py b/skyvern/services/action_service.pyindex 121a2479c..65b3bbe87 100644--- a/skyvern/services/action_service.py+++ b/skyvern/services/action_service.py@@ -53,6 +53,7 @@ async def get_action_history("exception_type","exception_message","download_triggered",+ "upload_file_triggered",},),}diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex df1886f76..8f96a3503 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -57,7 +57,6 @@ from skyvern.exceptions import (NoIncrementalElementFoundForCustomSelection,NoSuitableAutoCompleteOption,OptionIndexOutOfBound,- WrongElementToUploadFile,)from skyvern.experimentation.wait_utils import get_or_create_wait_config, get_wait_timefrom skyvern.forge import app@@ -73,6 +72,7 @@ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extraction_resultfrom skyvern.forge.sdk.cache import extraction_cache, extraction_shadowfrom skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import PendingFileChooserListenerfrom skyvern.forge.sdk.core.skyvern_context import current as skyvern_currentfrom skyvern.forge.sdk.core.skyvern_context import ensure_contextfrom skyvern.forge.sdk.event.factory import EventStrategyFactory@@ -2659,28 +2659,47 @@ async def chain_click(file = await handler_utils.download_file(file_url, action.model_dump(), task.organization_id)is_filechooser_trigger = False+ is_upload_action = bool(action.file_url)+ context = skyvern_context.current()+ has_pending = (+ context is not None and context.pending_file_chooser is not None and context.pending_file_chooser.page is page+ )++ if is_upload_action and has_pending and context is not None:+ LOG.info("New UPLOAD_FILE action arrived, cleaning up stale pending file chooser listener")+ context.cleanup_pending_file_chooser()+ has_pending = Falseasync def fc_func(fc: FileChooser) -> None:nonlocal is_filechooser_triggeris_filechooser_trigger = Trueawait fc.set_files(files=file)- page.on("filechooser", fc_func)- LOG.info("Registered file chooser listener", action=action, path=file)+ if not has_pending:+ page.on("filechooser", fc_func)+ LOG.info("Registered file chooser listener", action=action, path=file)+ else:+ LOG.info(+ "Skipping defensive file chooser listener — pending deferred listener exists",+ action=action,+ )"""Clicks on an element identified by the css and its parent if failed.:param css: css of the element to click"""+ # Tracks the return value so the finally block can inspect click success.+ action_results: list[ActionResult] = []try:if not await skyvern_element.navigate_to_a_href(page=page):await EventStrategyFactory.move_to_element(page, locator)await locator.click(timeout=timeout)LOG.info("Chain click: main element click succeeded", action=action, locator=locator)- return [ActionSuccess()]+ action_results = [ActionSuccess()]+ return action_resultsexcept Exception as e:- action_results: list[ActionResult] = [ActionFailure(FailToClick(action.element_id, msg=str(e)))]+ action_results = [ActionFailure(FailToClick(action.element_id, msg=str(e)))]if skyvern_element.get_tag_name() == "label":try:@@ -2831,21 +2850,61 @@ async def chain_click(return action_resultsfinally:- LOG.info("Remove file chooser listener", action=action)-# FIXME: use 'page.wait_for_event("filechooser", timeout)' to wait for the file to be uploaded instead of hardcoding sleeping time- # Sleep for 15 seconds after uploading a file to let the page process it- # Removing this breaks file uploads using the filechooser- if file:- await asyncio.sleep(15)- page.remove_listener("filechooser", fc_func)-- if action.file_url and not is_filechooser_trigger:+ click_succeeded = any(isinstance(r, ActionSuccess) for r in action_results)++ if is_filechooser_trigger:+ # File chooser opened during this click — upload completed normally+ LOG.info("File chooser triggered during this click", action=action)+ if file:+ await asyncio.sleep(15)+ if not has_pending:+ page.remove_listener("filechooser", fc_func)+ if context is not None and context.pending_file_chooser is not None:+ context.cleanup_pending_file_chooser()++ elif is_upload_action and file and click_succeeded and context is not None:+ # UPLOAD_FILE click succeeded but file chooser didn't open (e.g. popup intercepted).+ # Defer the listener so a subsequent click can trigger it.+ if not has_pending:+ page.remove_listener("filechooser", fc_func)LOG.warning(- "Action has file_url, but filechoose even hasn't been triggered. Upload file attempt seems to fail",+ "UPLOAD_FILE click succeeded but file chooser was not triggered — deferring listener",action=action,)- return [ActionFailure(WrongElementToUploadFile(action.element_id))]+ # Clean up any existing pending listener (may be on a different page)+ if context.pending_file_chooser is not None:+ context.cleanup_pending_file_chooser()++ pending = PendingFileChooserListener(page=page, file_paths=file)++ async def deferred_fc_handler(fc: FileChooser) -> None:+ pending.triggered = True+ await fc.set_files(files=pending.file_paths)+ # Auto-remove after firing to prevent double-consumption+ pending.cleanup()++ pending.handler = deferred_fc_handler+ page.on("filechooser", deferred_fc_handler)+ context.pending_file_chooser = pending++ elif (+ context is not None and context.pending_file_chooser is not None and context.pending_file_chooser.triggered+ ):+ # A previous UPLOAD_FILE's deferred listener was consumed by this click+ LOG.info("Pending file chooser from previous UPLOAD_FILE was consumed by this click", action=action)+ await asyncio.sleep(15)+ context.cleanup_pending_file_chooser()++ else:+ # No file chooser involved — just clean up the defensive listener+ if not has_pending:+ page.remove_listener("filechooser", fc_func)++ if is_upload_action:+ for r in action_results:+ if isinstance(r, ActionSuccess):+ r.upload_file_triggered = is_filechooser_trigger@traced(name="skyvern.agent.dropdown.auto_completion")diff --git a/skyvern/webeye/actions/responses.py b/skyvern/webeye/actions/responses.pyindex 07f473ebb..aeea6dbdc 100644--- a/skyvern/webeye/actions/responses.py+++ b/skyvern/webeye/actions/responses.py@@ -14,6 +14,7 @@ class ActionResult(BaseModel):step_retry_number: int | None = Nonestep_order: int | None = Nonedownload_triggered: bool | None = None+ upload_file_triggered: bool | None = Nonedownloaded_files: list[str] | None = None # Actual file names that were downloaded# None is used for old data so that we can differentiate between old and new data which only has booleaninteracted_with_sibling: bool | None = None@@ -34,6 +35,8 @@ class ActionResult(BaseModel):results.append(f"step_retry_number={self.step_retry_number}")if self.download_triggered is not None:results.append(f"download_triggered={self.download_triggered}")+ if self.upload_file_triggered is not None:+ results.append(f"upload_file_triggered={self.upload_file_triggered}")if self.downloaded_files is not None:results.append(f"downloaded_files={self.downloaded_files}")if self.interacted_with_sibling is not None:diff --git a/tests/unit/test_pending_file_chooser.py b/tests/unit/test_pending_file_chooser.pynew file mode 100644index 000000000..f64cfbc49--- /dev/null+++ b/tests/unit/test_pending_file_chooser.py@@ -0,0 +1,130 @@+"""Tests for PendingFileChooserListener and SkyvernContext.cleanup_pending_file_chooser."""++from __future__ import annotations++from unittest.mock import MagicMock, patch++from skyvern.forge.sdk.core.skyvern_context import (+ PendingFileChooserListener,+ SkyvernContext,+ _cleanup_outgoing_context,+)+++def _make_mock_page() -> MagicMock:+ page = MagicMock()+ page.is_closed.return_value = False+ return page+++def _make_pending(page: MagicMock | None = None, triggered: bool = False) -> PendingFileChooserListener:+ page = page or _make_mock_page()+ handler = MagicMock()+ return PendingFileChooserListener(page=page, file_paths=["/tmp/test.pdf"], handler=handler, triggered=triggered)+++class TestPendingFileChooserListenerCleanup:+ def test_cleanup_removes_listener(self) -> None:+ page = _make_mock_page()+ pending = _make_pending(page=page)+ handler = pending.handler++ pending.cleanup()++ page.remove_listener.assert_called_once_with("filechooser", handler)+ assert pending.handler is None++ def test_cleanup_idempotent(self) -> None:+ pending = _make_pending()+ pending.cleanup()+ pending.cleanup()++ assert pending.handler is None++ def test_cleanup_handles_closed_page(self) -> None:+ page = _make_mock_page()+ page.remove_listener.side_effect = Exception("Target page closed")+ pending = _make_pending(page=page)++ pending.cleanup()++ assert pending.handler is None+++class TestSkyvernContextCleanupPendingFileChooser:+ def test_cleanup_when_no_pending(self) -> None:+ ctx = SkyvernContext()+ ctx.cleanup_pending_file_chooser()+ assert ctx.pending_file_chooser is None++ def test_cleanup_unconsumed_logs_warning(self) -> None:+ ctx = SkyvernContext()+ ctx.pending_file_chooser = _make_pending(triggered=False)++ with patch("skyvern.forge.sdk.core.skyvern_context.LOG") as mock_log:+ ctx.cleanup_pending_file_chooser()+ mock_log.warning.assert_called_once()++ assert ctx.pending_file_chooser is None++ def test_cleanup_consumed_no_warning(self) -> None:+ ctx = SkyvernContext()+ ctx.pending_file_chooser = _make_pending(triggered=True)++ with patch("skyvern.forge.sdk.core.skyvern_context.LOG") as mock_log:+ ctx.cleanup_pending_file_chooser()+ mock_log.warning.assert_not_called()++ assert ctx.pending_file_chooser is None++ def test_cleanup_removes_listener_from_page(self) -> None:+ page = _make_mock_page()+ ctx = SkyvernContext()+ pending = _make_pending(page=page, triggered=True)+ handler = pending.handler+ ctx.pending_file_chooser = pending++ ctx.cleanup_pending_file_chooser()++ page.remove_listener.assert_called_once_with("filechooser", handler)++ def test_new_pending_replaces_old(self) -> None:+ page_a = _make_mock_page()+ page_b = _make_mock_page()+ ctx = SkyvernContext()++ old_pending = _make_pending(page=page_a)+ old_handler = old_pending.handler+ ctx.pending_file_chooser = old_pending++ ctx.cleanup_pending_file_chooser()+ page_a.remove_listener.assert_called_once_with("filechooser", old_handler)++ new_pending = _make_pending(page=page_b)+ ctx.pending_file_chooser = new_pending+ assert ctx.pending_file_chooser.page is page_b+++class TestCleanupOutgoingContext:+ def test_none_context_is_safe(self) -> None:+ _cleanup_outgoing_context(None)++ def test_cleans_pending_file_chooser(self) -> None:+ ctx = SkyvernContext()+ pending = _make_pending(triggered=False)+ ctx.pending_file_chooser = pending++ _cleanup_outgoing_context(ctx)++ assert ctx.pending_file_chooser is None++ def test_flushes_feature_flags_and_pending(self) -> None:+ ctx = SkyvernContext()+ ctx.feature_flag_entries = {"FLAG_A": True}+ ctx.pending_file_chooser = _make_pending(triggered=False)++ with patch.object(ctx, "flush_feature_flags") as mock_flush:+ _cleanup_outgoing_context(ctx)+ mock_flush.assert_called_once()++ assert ctx.pending_file_chooser is None