Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-12641): give the LLM clear no-file-download feedback (#7690)
LawyZhenga49b8a808c30
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex ead1cf480..ff3723c6d 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -179,6 +179,11 @@ LOG = structlog.get_logger()UPLOAD_PENDING_FOLLOWUP_MESSAGE = "Upload is not complete yet. Continue the upload flow."+DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE = (+ "No file download was observed or credited after this action. "+ "If the goal still requires this file, keep trying to download it rather than reporting the goal complete."+)+FIX_TEL_INPUT_DIGIT_DROP_FLAG = "FIX_TEL_INPUT_DIGIT_DROP"COLLAPSE_SELECT_FANOUT_FLAG = "COLLAPSE_SELECT_FANOUT"COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG = "COLLAPSE_CUSTOM_SELECT_FANOUT"@@ -2862,6 +2867,9 @@ class ActionHandler:)else:results[-1].download_triggered = False+ if isinstance(results[-1], ActionSuccess):+ results[-1].needs_followup = True+ results[-1].followup_message = DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGEaction.download_triggered = Falsereturn resultsresults[-1].download_triggered = Truediff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.pyindex 3b32b60ee..23ccc9d64 100644--- a/tests/unit/test_download_file_action_handler.py+++ b/tests/unit/test_download_file_action_handler.py@@ -15,6 +15,7 @@ from skyvern.errors.errors import UserDefinedErrorfrom skyvern.forge.sdk.models import StepStatusfrom skyvern.webeye.actions.actions import ClickAction, DownloadFileActionfrom skyvern.webeye.actions.handler import (+ DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE,ActionHandler,ScopedXhrDownloadCapture,_cleanup_captured_download_popup,@@ -56,6 +57,20 @@ def _download(*, path: Path | None = None, failure: str | None = None, save_as:return download+def test_download_not_triggered_message_claims_only_observation() -> None:+ # download_triggered=false proves only that Skyvern did not observe/credit a+ # download after the action — NOT categorically that no download started or no+ # file was saved (late/missed artifacts are possible). Pin the exact intended+ # observation-only string so no categorical save/start wording can creep back in.+ assert DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE == (+ "No file download was observed or credited after this action. "+ "If the goal still requires this file, keep trying to download it rather than reporting the goal complete."+ )+ lowered = DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE.lower()+ assert "not saved" not in lowered+ assert "download started" not in lowered++@pytest.mark.asyncioasync def test_persist_captured_download_cancellation_cleans_owned_target(tmp_path: Path) -> None:async def save_as(target: Path) -> None:@@ -1601,7 +1616,7 @@ async def test_handle_action_navigates_back_from_blank_page_after_download(),patch("skyvern.webeye.actions.handler.app", mock_app),):- await ActionHandler.handle_action(+ results = await ActionHandler.handle_action(scraped_page=scraped_page,task=task,step=step,@@ -1611,6 +1626,10 @@ async def test_handle_action_navigates_back_from_blank_page_after_download(# The blank-page recovery should have navigated back to the original URLbrowser_state.navigate_to_url.assert_called_once_with(page=page, url=original_url)+ # A successful download must not attach the no-download followup feedback.+ assert results[-1].download_triggered is True+ assert results[-1].needs_followup is None+ assert results[-1].followup_message is Nonespan_attrs = _download_wait_span_attrs(span_exporter)assert span_attrs["download_signal_observed"] is Trueassert span_attrs["download_signal_source"] == "download_file_detected"@@ -1765,6 +1784,8 @@ async def test_handle_action_download_no_signal_fails_fast(span_exporter: InMemoassert elapsed < 1.0assert results[-1].download_triggered is Falseassert action.download_triggered is False+ assert results[-1].needs_followup is True+ assert results[-1].followup_message == DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGEassert wait_for_downloads.await_count == 0page.off.assert_called_once()span_attrs = _download_wait_span_attrs(span_exporter)@@ -1775,6 +1796,58 @@ async def test_handle_action_download_no_signal_fails_fast(span_exporter: InMemoassert "download_signal_poll_iterations" not in span_attrs+async def test_handle_action_download_no_signal_preserves_action_failure() -> None:+ now = datetime.now(UTC)+ organization = make_organization(now)+ task, step, page, browser_state, scraped_page, action = _make_download_click_context(+ now=now,+ organization=organization,+ page_url="https://example.com/no-download",+ )+ action.errors = []+ failure = ActionFailure(RuntimeError("click failed"))++ async def mock_inner_handle_action(*args: object, **kwargs: object) -> list[ActionFailure]:+ return [failure]++ with tempfile.TemporaryDirectory() as temp_dir:+ mock_app = MagicMock()+ mock_app.BROWSER_MANAGER.get_for_task.return_value = browser_state+ mock_app.DATABASE.workflow_params.create_action = AsyncMock(return_value=action)+ mock_app.STORAGE = MagicMock()++ with (+ patch.object(ActionHandler, "_handle_action", side_effect=mock_inner_handle_action),+ patch("skyvern.webeye.actions.handler.BROWSER_DOWNLOAD_NO_SIGNAL_GRACE_TIME", 0.01),+ patch("skyvern.webeye.actions.handler.get_download_dir", return_value=temp_dir),+ patch("skyvern.webeye.actions.handler.list_files_in_directory", return_value=[]),+ patch("skyvern.webeye.actions.handler.skyvern_context.current", return_value=None),+ patch(+ "skyvern.webeye.actions.handler.check_downloading_files_and_wait_for_download_to_complete",+ new=AsyncMock(),+ ),+ patch("skyvern.webeye.actions.handler.app", mock_app),+ ):+ results = await ActionHandler.handle_action(+ scraped_page=scraped_page,+ task=task,+ step=step,+ page=page,+ action=action,+ )++ assert results == [failure]+ assert results[-1] is failure+ assert isinstance(results[-1], ActionFailure)+ assert results[-1].success is False+ assert results[-1].download_triggered is False+ assert action.download_triggered is False+ assert action.errors == []+ assert results[-1].needs_followup is None+ assert results[-1].followup_message is None++@pytest.mark.asyncioasync def test_handle_action_download_fails_on_transient_user_defined_error_text(span_exporter: InMemorySpanExporter,@@ -1860,6 +1933,9 @@ async def test_handle_action_download_fails_on_transient_user_defined_error_textassert results[-1].download_triggered is Falseassert "download failure says the generated archive could not be saved" in (results[-1].exception_message or "")assert action.download_triggered is False+ # Page-confirmed terminal user errors are definitive: no "keep trying" followup.+ assert results[-1].needs_followup is None+ assert results[-1].followup_message is Noneassert action.errors is not Noneassert [error.error_code for error in action.errors] == ["previous_error", "data_not_downloadable"]assert action.terminal_user_errors is Truediff --git a/tests/unit/test_upload_pending_action_history.py b/tests/unit/test_upload_pending_action_history.pyindex 29cb93790..acace5548 100644--- a/tests/unit/test_upload_pending_action_history.py+++ b/tests/unit/test_upload_pending_action_history.py@@ -1,5 +1,12 @@-from skyvern.webeye.actions.handler import UPLOAD_PENDING_FOLLOWUP_MESSAGE-from skyvern.webeye.actions.responses import ActionResult, ActionSuccess+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from skyvern.services.action_service import get_action_history+from skyvern.webeye.actions.actions import ClickAction+from skyvern.webeye.actions.handler import DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE, UPLOAD_PENDING_FOLLOWUP_MESSAGE+from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccessclass TestUploadFollowupActionHistory:@@ -64,3 +71,60 @@ class TestUploadFollowupActionHistory:d = r.model_dump(exclude_none=True)assert "needs_followup" not in dassert "followup_message" not in d+++async def _download_action_history(result: ActionResult) -> dict:+ """Run the real get_action_history over a single download-intent action/result."""+ action = ClickAction(element_id="download-link", download=True)+ step = MagicMock()+ step.output = SimpleNamespace(actions_and_results=[(action, [result])])+ task = MagicMock(task_id="tsk_1", organization_id="o_1")+ with patch("skyvern.services.action_service.app") as app_mock:+ app_mock.DATABASE.tasks.get_task_steps = AsyncMock(return_value=[])+ history = await get_action_history(task, current_step=step)+ assert len(history) == 1+ return history[0]+++class TestDownloadFollowupActionHistory:+ """The no-download feedback must survive action-history serialization for the next prompt."""++ @pytest.mark.asyncio+ async def test_no_download_success_result_carries_feedback_through_history(self) -> None:+ r = ActionSuccess()+ r.download_triggered = False+ r.needs_followup = True+ r.followup_message = DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE++ entry = await _download_action_history(r)++ assert entry["action"]["download"] is True+ assert entry["result"]["success"] is True+ assert entry["result"]["download_triggered"] is False+ assert entry["result"]["needs_followup"] is True+ assert entry["result"]["followup_message"] == DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE++ @pytest.mark.asyncio+ async def test_terminal_failure_result_omits_followup_from_history(self) -> None:+ # Negative control: a page-confirmed terminal user error yields an+ # ActionFailure with download_triggered=false and NO followup fields, so the+ # serialized history must not carry a contradictory "keep trying" signal.+ r = ActionFailure(Exception("data not downloadable"), download_triggered=False)++ entry = await _download_action_history(r)++ assert entry["result"]["success"] is False+ assert entry["result"]["download_triggered"] is False+ assert "needs_followup" not in entry["result"]+ assert "followup_message" not in entry["result"]++ @pytest.mark.asyncio+ async def test_successful_download_omits_followup_from_history(self) -> None:+ r = ActionSuccess()+ r.download_triggered = True++ entry = await _download_action_history(r)++ assert entry["result"]["download_triggered"] is True+ assert "needs_followup" not in entry["result"]+ assert "followup_message" not in entry["result"]