Skip to content

Automate browser based workflows with AI

fix: fail cached download block when no file is downloaded locally (#SKY-8433) (#5127)

pedrohsdbcb34d86cbadc
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index aa2937b67..e08874812 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -1849,6 +1849,22 @@ async def download(
await _run_cached_function(cached_fn)
+ # Check local filesystem for newly downloaded files.
+ # This is the primary verification — it doesn't depend on S3/remote
+ # storage and catches the case where download_selector() returned None
+ # causing the click to silently succeed without downloading anything.
+ local_files_after_download = (
+ list_files_in_directory(local_download_dir) if local_download_dir.exists() else []
+ )
+ # Filter out incomplete .crdownload files from both lists for an accurate comparison
+ local_new_files = set(local_files_after_download) - set(local_files_before)
+ local_new_complete_files = [f for f in local_new_files if not f.endswith(".crdownload")]
+ if not local_new_complete_files:
+ raise Exception(
+ "Cached download block did not produce a new file on the local filesystem. "
+ f"Local files before: {len(local_files_before)}, after: {len(local_files_after_download)}"
+ )
+
# Rename newly downloaded files using download_suffix if provided.
# Rename runs BEFORE S3 upload so that remote storage receives the
# correctly-named file and subsequent blocks get the right URLs.
diff --git a/tests/unit/test_cached_download_s3_upload.py b/tests/unit/test_cached_download_s3_upload.py
index 633d53856..e6a21a62b 100644
--- a/tests/unit/test_cached_download_s3_upload.py
+++ b/tests/unit/test_cached_download_s3_upload.py
@@ -110,9 +110,14 @@ def _cleanup(refs):
@pytest.mark.asyncio
-async def test_cached_download_calls_save_downloaded_files(setup):
+async def test_cached_download_calls_save_downloaded_files(setup, tmp_path):
"""save_downloaded_files must be called so get_downloaded_files can find the file."""
- refs = setup(get_side_effect=[[], ["file1.pdf"]])
+ download_dir = tmp_path / "downloads"
+ local_file = str(download_dir / "file1.pdf")
+ refs = setup(
+ get_side_effect=[[], ["file1.pdf"]],
+ list_files_side_effect=[[], [local_file]], # before=empty, after=has file (local verification)
+ )
try:
from skyvern.services.script_service import download
@@ -146,7 +151,7 @@ async def test_rename_happens_before_s3_upload(setup, tmp_path):
refs = setup(
get_side_effect=[[], ["invoice.pdf"]],
save_side_effect=track_save,
- list_files_side_effect=[[], [fake_file]],
+ list_files_side_effect=[[], [fake_file], [fake_file]], # before, local verify, rename
rename_mock=rename_mock,
)
try:
@@ -169,7 +174,7 @@ async def test_download_suffix_rename_uses_file_path_directly(setup, tmp_path):
rename_mock = MagicMock(return_value=str(download_dir / "invoice.pdf"))
refs = setup(
get_side_effect=[[], ["invoice.pdf"]],
- list_files_side_effect=[[], [abs_path]],
+ list_files_side_effect=[[], [abs_path], [abs_path]], # before, local verify, rename
rename_mock=rename_mock,
)
try:
@@ -184,9 +189,12 @@ async def test_download_suffix_rename_uses_file_path_directly(setup, tmp_path):
@pytest.mark.asyncio
async def test_verification_raises_when_no_new_file(setup):
- """When get_downloaded_files shows no increase, the cached path should raise
+ """When no new file appears on local filesystem, the cached path should raise
and fall back to AI."""
- refs = setup(get_side_effect=[[], [], [], []])
+ refs = setup(
+ get_side_effect=[[], [], [], []],
+ list_files_side_effect=[[], []], # before=empty, after=still empty (local verification fails)
+ )
try:
from skyvern.services.script_service import download
@@ -194,16 +202,21 @@ async def test_verification_raises_when_no_new_file(setup):
refs["fallback"].assert_called_once()
error_arg = refs["fallback"].call_args.kwargs.get("error")
- assert "Files before: 0, after: 0" in str(error_arg)
+ assert "did not produce a new file on the local filesystem" in str(error_arg)
finally:
_cleanup(refs)
@pytest.mark.asyncio
-async def test_verification_retries_before_failing(setup):
+async def test_verification_retries_before_failing(setup, tmp_path):
"""get_downloaded_files retries up to 3 times before declaring failure."""
+ download_dir = tmp_path / "downloads"
+ local_file = str(download_dir / "file.pdf")
# Before: 0 files. After: [], [], ["file.pdf"] (succeeds on 3rd attempt)
- refs = setup(get_side_effect=[[], [], [], ["file.pdf"]])
+ refs = setup(
+ get_side_effect=[[], [], [], ["file.pdf"]],
+ list_files_side_effect=[[], [local_file]], # local verification passes
+ )
try:
from skyvern.services.script_service import download
@@ -216,19 +229,23 @@ async def test_verification_retries_before_failing(setup):
@pytest.mark.asyncio
-async def test_save_timeout_skips_verification(setup):
- """TimeoutError on save_downloaded_files should skip verification entirely.
- No point retrying get_downloaded_files when we know S3 is degraded."""
+async def test_save_timeout_skips_verification(setup, tmp_path):
+ """TimeoutError on save_downloaded_files should skip S3 verification entirely.
+ No point retrying get_downloaded_files when we know S3 is degraded.
+ Local file verification still passes since a file was downloaded."""
+ download_dir = tmp_path / "downloads"
+ local_file = str(download_dir / "file.pdf")
refs = setup(
save_side_effect=asyncio.TimeoutError(),
get_side_effect=[[]], # only the before-check runs; after-check is skipped
+ list_files_side_effect=[[], [local_file]], # local verification passes
)
try:
from skyvern.services.script_service import download
await download(prompt="Download invoice", label="test_block")
- # Verification skipped → block completes, no AI fallback
+ # S3 verification skipped, but local verification passed → block completes
refs["fallback"].assert_not_called()
refs["update_block"].assert_called_once()
# get_downloaded_files called only once (before-check), not 3 more times for after
@@ -238,19 +255,23 @@ async def test_save_timeout_skips_verification(setup):
@pytest.mark.asyncio
-async def test_save_generic_exception_skips_verification(setup):
- """Non-timeout S3 failure (e.g., permission error) should also skip verification.
- Matches agent.py which catches both TimeoutError and generic Exception."""
+async def test_save_generic_exception_skips_verification(setup, tmp_path):
+ """Non-timeout S3 failure (e.g., permission error) should also skip S3 verification.
+ Matches agent.py which catches both TimeoutError and generic Exception.
+ Local file verification still passes since a file was downloaded."""
+ download_dir = tmp_path / "downloads"
+ local_file = str(download_dir / "file.pdf")
refs = setup(
save_side_effect=RuntimeError("S3 permission denied"),
get_side_effect=[[]], # only the before-check runs; after-check is skipped
+ list_files_side_effect=[[], [local_file]], # local verification passes
)
try:
from skyvern.services.script_service import download
await download(prompt="Download invoice", label="test_block")
- # Verification skipped → block completes, no AI fallback
+ # S3 verification skipped, but local verification passed → block completes
refs["fallback"].assert_not_called()
refs["update_block"].assert_called_once()
# get_downloaded_files called only once (before-check), not 3 more times for after
@@ -260,24 +281,33 @@ async def test_save_generic_exception_skips_verification(setup):
@pytest.mark.asyncio
-async def test_get_before_timeout_skips_verification(setup):
- """If the before-check times out, verification should be skipped entirely
- to avoid spurious AI fallbacks under degraded storage."""
- refs = setup(get_side_effect=asyncio.TimeoutError())
+async def test_get_before_timeout_skips_verification(setup, tmp_path):
+ """If the before-check times out, S3 verification should be skipped entirely
+ to avoid spurious AI fallbacks under degraded storage.
+ Local file verification still passes since a file was downloaded."""
+ download_dir = tmp_path / "downloads"
+ local_file = str(download_dir / "file.pdf")
+ refs = setup(
+ get_side_effect=asyncio.TimeoutError(),
+ list_files_side_effect=[[], [local_file]], # local verification passes
+ )
try:
from skyvern.services.script_service import download
await download(prompt="Download invoice", label="test_block")
- # Should NOT fall back — verification was skipped
+ # Should NOT fall back — local verification passed, S3 verification skipped
refs["fallback"].assert_not_called()
finally:
_cleanup(refs)
@pytest.mark.asyncio
-async def test_get_after_timeout_skips_verification(setup):
- """If the after-check times out, verification should be skipped."""
+async def test_get_after_timeout_skips_verification(setup, tmp_path):
+ """If the after-check times out, S3 verification should be skipped.
+ Local file verification still passes since a file was downloaded."""
+ download_dir = tmp_path / "downloads"
+ local_file = str(download_dir / "file.pdf")
call_count = {"n": 0}
async def get_side_effect(**kwargs):
@@ -286,7 +316,10 @@ async def test_get_after_timeout_skips_verification(setup):
return [] # before-check succeeds
raise asyncio.TimeoutError()
- refs = setup(get_side_effect=get_side_effect)
+ refs = setup(
+ get_side_effect=get_side_effect,
+ list_files_side_effect=[[], [local_file]], # local verification passes
+ )
try:
from skyvern.services.script_service import download
@@ -298,10 +331,16 @@ async def test_get_after_timeout_skips_verification(setup):
@pytest.mark.asyncio
-async def test_no_rename_without_download_suffix(setup):
+async def test_no_rename_without_download_suffix(setup, tmp_path):
"""When download_suffix is not provided, rename should not be called."""
+ download_dir = tmp_path / "downloads"
+ local_file = str(download_dir / "file.pdf")
rename_mock = MagicMock()
- refs = setup(get_side_effect=[[], ["file.pdf"]], rename_mock=rename_mock)
+ refs = setup(
+ get_side_effect=[[], ["file.pdf"]],
+ rename_mock=rename_mock,
+ list_files_side_effect=[[], [local_file]], # local verification passes
+ )
try:
from skyvern.services.script_service import download
@@ -322,7 +361,7 @@ async def test_rename_skips_crdownload_files(setup, tmp_path):
rename_mock = MagicMock(return_value=str(download_dir / "renamed.pdf"))
refs = setup(
get_side_effect=[[], ["invoice.pdf"]],
- list_files_side_effect=[[], [incomplete, complete]],
+ list_files_side_effect=[[], [incomplete, complete], [incomplete, complete]], # before, local verify, rename
rename_mock=rename_mock,
)
try:
@@ -347,7 +386,7 @@ async def test_rename_handles_name_collision(setup, tmp_path):
rename_mock = MagicMock(return_value=str(download_dir / "invoice_1.pdf"))
refs = setup(
get_side_effect=[[], ["invoice_1.pdf"]],
- list_files_side_effect=[[], [new_file]],
+ list_files_side_effect=[[], [new_file], [new_file]], # before, local verify, rename
rename_mock=rename_mock,
)
try:
@@ -361,9 +400,14 @@ async def test_rename_handles_name_collision(setup, tmp_path):
@pytest.mark.asyncio
-async def test_block_marked_completed_on_success(setup):
+async def test_block_marked_completed_on_success(setup, tmp_path):
"""On successful download + verification, block should be marked completed."""
- refs = setup(get_side_effect=[[], ["file.pdf"]])
+ download_dir = tmp_path / "downloads"
+ local_file = str(download_dir / "file.pdf")
+ refs = setup(
+ get_side_effect=[[], ["file.pdf"]],
+ list_files_side_effect=[[], [local_file]], # local verification passes
+ )
try:
from skyvern.services.script_service import download
@@ -373,3 +417,72 @@ async def test_block_marked_completed_on_success(setup):
refs["fallback"].assert_not_called()
finally:
_cleanup(refs)
+
+
+async def test_local_verification_fails_when_no_file_downloaded(setup):
+ """SKY-8433: When download_selector() returns None and click silently succeeds
+ without downloading anything, the local filesystem check should detect
+ no new files and trigger AI fallback instead of silently succeeding."""
+ refs = setup(
+ get_side_effect=[[]], # S3 before-check only; local check fails before S3 after-check
+ list_files_side_effect=[[], []], # before=empty, after=still empty
+ )
+ try:
+ from skyvern.services.script_service import download
+
+ await download(prompt="Download invoice", label="test_block")
+
+ refs["fallback"].assert_called_once()
+ error_arg = refs["fallback"].call_args.kwargs.get("error")
+ assert "did not produce a new file on the local filesystem" in str(error_arg)
+ # S3 save should NOT be called since local check fails first
+ refs["storage"].save_downloaded_files.assert_not_called()
+ finally:
+ _cleanup(refs)
+
+
+async def test_local_verification_ignores_crdownload_files(setup, tmp_path):
+ """Incomplete .crdownload files should not count as successful downloads
+ for local filesystem verification."""
+ download_dir = tmp_path / "downloads"
+ incomplete_file = str(download_dir / "invoice.pdf.crdownload")
+ refs = setup(
+ get_side_effect=[[]],
+ list_files_side_effect=[[], [incomplete_file]], # only incomplete file appeared
+ )
+ try:
+ from skyvern.services.script_service import download
+
+ await download(prompt="Download invoice", label="test_block")
+
+ refs["fallback"].assert_called_once()
+ error_arg = refs["fallback"].call_args.kwargs.get("error")
+ assert "did not produce a new file on the local filesystem" in str(error_arg)
+ finally:
+ _cleanup(refs)
+
+
+async def test_local_verification_passes_with_complete_file(setup, tmp_path):
+ """When a complete file appears on the local filesystem after the cached
+ function runs, the local verification should pass and allow the block
+ to proceed to S3 upload and completion."""
+ download_dir = tmp_path / "downloads"
+ local_file = str(download_dir / "invoice.pdf")
+ refs = setup(
+ get_side_effect=[[], [local_file]], # S3 verification also passes
+ list_files_side_effect=[[], [local_file]], # local verification passes
+ )
+ try:
+ from skyvern.services.script_service import download
+
+ await download(prompt="Download invoice", label="test_block")
+
+ refs["fallback"].assert_not_called()
+ refs["update_block"].assert_called_once()
+ # S3 save should be called since local check passed
+ refs["storage"].save_downloaded_files.assert_called_once()
+ finally:
+ _cleanup(refs)