Skip to content

Automate browser based workflows with AI

fix(SKY-12641): give the LLM clear no-file-download feedback (#7690)

LawyZhenga49b8a808c30
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index ead1cf480..ff3723c6d 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -179,6 +179,11 @@ LOG = structlog.get_logger()
UPLOAD_PENDING_FOLLOWUP_MESSAGE = "Upload is not complete yet. Continue the upload flow."
+DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE = (
+ "No file download was observed or credited after this action. "
+ "If the goal still requires this file, keep trying to download it rather than reporting the goal complete."
+)
+
FIX_TEL_INPUT_DIGIT_DROP_FLAG = "FIX_TEL_INPUT_DIGIT_DROP"
COLLAPSE_SELECT_FANOUT_FLAG = "COLLAPSE_SELECT_FANOUT"
COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG = "COLLAPSE_CUSTOM_SELECT_FANOUT"
@@ -2862,6 +2867,9 @@ class ActionHandler:
)
else:
results[-1].download_triggered = False
+ if isinstance(results[-1], ActionSuccess):
+ results[-1].needs_followup = True
+ results[-1].followup_message = DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE
action.download_triggered = False
return results
results[-1].download_triggered = True
diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.py
index 3b32b60ee..23ccc9d64 100644
--- a/tests/unit/test_download_file_action_handler.py
+++ b/tests/unit/test_download_file_action_handler.py
@@ -15,6 +15,7 @@ from skyvern.errors.errors import UserDefinedError
from skyvern.forge.sdk.models import StepStatus
from skyvern.webeye.actions.actions import ClickAction, DownloadFileAction
from skyvern.webeye.actions.handler import (
+ DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE,
ActionHandler,
ScopedXhrDownloadCapture,
_cleanup_captured_download_popup,
@@ -56,6 +57,20 @@ def _download(*, path: Path | None = None, failure: str | None = None, save_as:
return download
+def test_download_not_triggered_message_claims_only_observation() -> None:
+ # download_triggered=false proves only that Skyvern did not observe/credit a
+ # download after the action — NOT categorically that no download started or no
+ # file was saved (late/missed artifacts are possible). Pin the exact intended
+ # observation-only string so no categorical save/start wording can creep back in.
+ assert DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE == (
+ "No file download was observed or credited after this action. "
+ "If the goal still requires this file, keep trying to download it rather than reporting the goal complete."
+ )
+ lowered = DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE.lower()
+ assert "not saved" not in lowered
+ assert "download started" not in lowered
+
+
@pytest.mark.asyncio
async def test_persist_captured_download_cancellation_cleans_owned_target(tmp_path: Path) -> None:
async def save_as(target: Path) -> None:
@@ -1601,7 +1616,7 @@ async def test_handle_action_navigates_back_from_blank_page_after_download(
),
patch("skyvern.webeye.actions.handler.app", mock_app),
):
- await ActionHandler.handle_action(
+ results = await ActionHandler.handle_action(
scraped_page=scraped_page,
task=task,
step=step,
@@ -1611,6 +1626,10 @@ async def test_handle_action_navigates_back_from_blank_page_after_download(
# The blank-page recovery should have navigated back to the original URL
browser_state.navigate_to_url.assert_called_once_with(page=page, url=original_url)
+ # A successful download must not attach the no-download followup feedback.
+ assert results[-1].download_triggered is True
+ assert results[-1].needs_followup is None
+ assert results[-1].followup_message is None
span_attrs = _download_wait_span_attrs(span_exporter)
assert span_attrs["download_signal_observed"] is True
assert span_attrs["download_signal_source"] == "download_file_detected"
@@ -1765,6 +1784,8 @@ async def test_handle_action_download_no_signal_fails_fast(span_exporter: InMemo
assert elapsed < 1.0
assert results[-1].download_triggered is False
assert action.download_triggered is False
+ assert results[-1].needs_followup is True
+ assert results[-1].followup_message == DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE
assert wait_for_downloads.await_count == 0
page.off.assert_called_once()
span_attrs = _download_wait_span_attrs(span_exporter)
@@ -1775,6 +1796,58 @@ async def test_handle_action_download_no_signal_fails_fast(span_exporter: InMemo
assert "download_signal_poll_iterations" not in span_attrs
+async def test_handle_action_download_no_signal_preserves_action_failure() -> None:
+ now = datetime.now(UTC)
+ organization = make_organization(now)
+ task, step, page, browser_state, scraped_page, action = _make_download_click_context(
+ now=now,
+ organization=organization,
+ page_url="https://example.com/no-download",
+ )
+ action.errors = []
+ failure = ActionFailure(RuntimeError("click failed"))
+
+ async def mock_inner_handle_action(*args: object, **kwargs: object) -> list[ActionFailure]:
+ return [failure]
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ mock_app = MagicMock()
+ mock_app.BROWSER_MANAGER.get_for_task.return_value = browser_state
+ mock_app.DATABASE.workflow_params.create_action = AsyncMock(return_value=action)
+ mock_app.STORAGE = MagicMock()
+
+ with (
+ patch.object(ActionHandler, "_handle_action", side_effect=mock_inner_handle_action),
+ patch("skyvern.webeye.actions.handler.BROWSER_DOWNLOAD_NO_SIGNAL_GRACE_TIME", 0.01),
+ patch("skyvern.webeye.actions.handler.get_download_dir", return_value=temp_dir),
+ patch("skyvern.webeye.actions.handler.list_files_in_directory", return_value=[]),
+ patch("skyvern.webeye.actions.handler.skyvern_context.current", return_value=None),
+ patch(
+ "skyvern.webeye.actions.handler.check_downloading_files_and_wait_for_download_to_complete",
+ new=AsyncMock(),
+ ),
+ patch("skyvern.webeye.actions.handler.app", mock_app),
+ ):
+ results = await ActionHandler.handle_action(
+ scraped_page=scraped_page,
+ task=task,
+ step=step,
+ page=page,
+ action=action,
+ )
+
+ assert results == [failure]
+ assert results[-1] is failure
+ assert isinstance(results[-1], ActionFailure)
+ assert results[-1].success is False
+ assert results[-1].download_triggered is False
+ assert action.download_triggered is False
+ assert action.errors == []
+ assert results[-1].needs_followup is None
+ assert results[-1].followup_message is None
+
+
@pytest.mark.asyncio
async def test_handle_action_download_fails_on_transient_user_defined_error_text(
span_exporter: InMemorySpanExporter,
@@ -1860,6 +1933,9 @@ async def test_handle_action_download_fails_on_transient_user_defined_error_text
assert results[-1].download_triggered is False
assert "download failure says the generated archive could not be saved" in (results[-1].exception_message or "")
assert action.download_triggered is False
+ # Page-confirmed terminal user errors are definitive: no "keep trying" followup.
+ assert results[-1].needs_followup is None
+ assert results[-1].followup_message is None
assert action.errors is not None
assert [error.error_code for error in action.errors] == ["previous_error", "data_not_downloadable"]
assert action.terminal_user_errors is True
diff --git a/tests/unit/test_upload_pending_action_history.py b/tests/unit/test_upload_pending_action_history.py
index 29cb93790..acace5548 100644
--- a/tests/unit/test_upload_pending_action_history.py
+++ b/tests/unit/test_upload_pending_action_history.py
@@ -1,5 +1,12 @@
-from skyvern.webeye.actions.handler import UPLOAD_PENDING_FOLLOWUP_MESSAGE
-from skyvern.webeye.actions.responses import ActionResult, ActionSuccess
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from skyvern.services.action_service import get_action_history
+from skyvern.webeye.actions.actions import ClickAction
+from skyvern.webeye.actions.handler import DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE, UPLOAD_PENDING_FOLLOWUP_MESSAGE
+from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess
class TestUploadFollowupActionHistory:
@@ -64,3 +71,60 @@ class TestUploadFollowupActionHistory:
d = r.model_dump(exclude_none=True)
assert "needs_followup" not in d
assert "followup_message" not in d
+
+
+async def _download_action_history(result: ActionResult) -> dict:
+ """Run the real get_action_history over a single download-intent action/result."""
+ action = ClickAction(element_id="download-link", download=True)
+ step = MagicMock()
+ step.output = SimpleNamespace(actions_and_results=[(action, [result])])
+ task = MagicMock(task_id="tsk_1", organization_id="o_1")
+ with patch("skyvern.services.action_service.app") as app_mock:
+ app_mock.DATABASE.tasks.get_task_steps = AsyncMock(return_value=[])
+ history = await get_action_history(task, current_step=step)
+ assert len(history) == 1
+ return history[0]
+
+
+class TestDownloadFollowupActionHistory:
+ """The no-download feedback must survive action-history serialization for the next prompt."""
+
+ @pytest.mark.asyncio
+ async def test_no_download_success_result_carries_feedback_through_history(self) -> None:
+ r = ActionSuccess()
+ r.download_triggered = False
+ r.needs_followup = True
+ r.followup_message = DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE
+
+ entry = await _download_action_history(r)
+
+ assert entry["action"]["download"] is True
+ assert entry["result"]["success"] is True
+ assert entry["result"]["download_triggered"] is False
+ assert entry["result"]["needs_followup"] is True
+ assert entry["result"]["followup_message"] == DOWNLOAD_NOT_TRIGGERED_FOLLOWUP_MESSAGE
+
+ @pytest.mark.asyncio
+ async def test_terminal_failure_result_omits_followup_from_history(self) -> None:
+ # Negative control: a page-confirmed terminal user error yields an
+ # ActionFailure with download_triggered=false and NO followup fields, so the
+ # serialized history must not carry a contradictory "keep trying" signal.
+ r = ActionFailure(Exception("data not downloadable"), download_triggered=False)
+
+ entry = await _download_action_history(r)
+
+ assert entry["result"]["success"] is False
+ assert entry["result"]["download_triggered"] is False
+ assert "needs_followup" not in entry["result"]
+ assert "followup_message" not in entry["result"]
+
+ @pytest.mark.asyncio
+ async def test_successful_download_omits_followup_from_history(self) -> None:
+ r = ActionSuccess()
+ r.download_triggered = True
+
+ entry = await _download_action_history(r)
+
+ assert entry["result"]["download_triggered"] is True
+ assert "needs_followup" not in entry["result"]
+ assert "followup_message" not in entry["result"]