Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix: fail cached download block when no file is downloaded locally (#SKY-8433) (#5127)
pedrohsdbcb34d86cbadc
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex aa2937b67..e08874812 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -1849,6 +1849,22 @@ async def download(await _run_cached_function(cached_fn)+ # Check local filesystem for newly downloaded files.+ # This is the primary verification — it doesn't depend on S3/remote+ # storage and catches the case where download_selector() returned None+ # causing the click to silently succeed without downloading anything.+ local_files_after_download = (+ list_files_in_directory(local_download_dir) if local_download_dir.exists() else []+ )+ # Filter out incomplete .crdownload files from both lists for an accurate comparison+ local_new_files = set(local_files_after_download) - set(local_files_before)+ local_new_complete_files = [f for f in local_new_files if not f.endswith(".crdownload")]+ if not local_new_complete_files:+ raise Exception(+ "Cached download block did not produce a new file on the local filesystem. "+ f"Local files before: {len(local_files_before)}, after: {len(local_files_after_download)}"+ )+# Rename newly downloaded files using download_suffix if provided.# Rename runs BEFORE S3 upload so that remote storage receives the# correctly-named file and subsequent blocks get the right URLs.diff --git a/tests/unit/test_cached_download_s3_upload.py b/tests/unit/test_cached_download_s3_upload.pyindex 633d53856..e6a21a62b 100644--- a/tests/unit/test_cached_download_s3_upload.py+++ b/tests/unit/test_cached_download_s3_upload.py@@ -110,9 +110,14 @@ def _cleanup(refs):@pytest.mark.asyncio-async def test_cached_download_calls_save_downloaded_files(setup):+async def test_cached_download_calls_save_downloaded_files(setup, tmp_path):"""save_downloaded_files must be called so get_downloaded_files can find the file."""- refs = setup(get_side_effect=[[], ["file1.pdf"]])+ download_dir = tmp_path / "downloads"+ local_file = str(download_dir / "file1.pdf")+ refs = setup(+ get_side_effect=[[], ["file1.pdf"]],+ list_files_side_effect=[[], [local_file]], # before=empty, after=has file (local verification)+ )try:from skyvern.services.script_service import download@@ -146,7 +151,7 @@ async def test_rename_happens_before_s3_upload(setup, tmp_path):refs = setup(get_side_effect=[[], ["invoice.pdf"]],save_side_effect=track_save,- list_files_side_effect=[[], [fake_file]],+ list_files_side_effect=[[], [fake_file], [fake_file]], # before, local verify, renamerename_mock=rename_mock,)try:@@ -169,7 +174,7 @@ async def test_download_suffix_rename_uses_file_path_directly(setup, tmp_path):rename_mock = MagicMock(return_value=str(download_dir / "invoice.pdf"))refs = setup(get_side_effect=[[], ["invoice.pdf"]],- list_files_side_effect=[[], [abs_path]],+ list_files_side_effect=[[], [abs_path], [abs_path]], # before, local verify, renamerename_mock=rename_mock,)try:@@ -184,9 +189,12 @@ async def test_download_suffix_rename_uses_file_path_directly(setup, tmp_path):@pytest.mark.asyncioasync def test_verification_raises_when_no_new_file(setup):- """When get_downloaded_files shows no increase, the cached path should raise+ """When no new file appears on local filesystem, the cached path should raiseand fall back to AI."""- refs = setup(get_side_effect=[[], [], [], []])+ refs = setup(+ get_side_effect=[[], [], [], []],+ list_files_side_effect=[[], []], # before=empty, after=still empty (local verification fails)+ )try:from skyvern.services.script_service import download@@ -194,16 +202,21 @@ async def test_verification_raises_when_no_new_file(setup):refs["fallback"].assert_called_once()error_arg = refs["fallback"].call_args.kwargs.get("error")- assert "Files before: 0, after: 0" in str(error_arg)+ assert "did not produce a new file on the local filesystem" in str(error_arg)finally:_cleanup(refs)@pytest.mark.asyncio-async def test_verification_retries_before_failing(setup):+async def test_verification_retries_before_failing(setup, tmp_path):"""get_downloaded_files retries up to 3 times before declaring failure."""+ download_dir = tmp_path / "downloads"+ local_file = str(download_dir / "file.pdf")# Before: 0 files. After: [], [], ["file.pdf"] (succeeds on 3rd attempt)- refs = setup(get_side_effect=[[], [], [], ["file.pdf"]])+ refs = setup(+ get_side_effect=[[], [], [], ["file.pdf"]],+ list_files_side_effect=[[], [local_file]], # local verification passes+ )try:from skyvern.services.script_service import download@@ -216,19 +229,23 @@ async def test_verification_retries_before_failing(setup):@pytest.mark.asyncio-async def test_save_timeout_skips_verification(setup):- """TimeoutError on save_downloaded_files should skip verification entirely.- No point retrying get_downloaded_files when we know S3 is degraded."""+async def test_save_timeout_skips_verification(setup, tmp_path):+ """TimeoutError on save_downloaded_files should skip S3 verification entirely.+ No point retrying get_downloaded_files when we know S3 is degraded.+ Local file verification still passes since a file was downloaded."""+ download_dir = tmp_path / "downloads"+ local_file = str(download_dir / "file.pdf")refs = setup(save_side_effect=asyncio.TimeoutError(),get_side_effect=[[]], # only the before-check runs; after-check is skipped+ list_files_side_effect=[[], [local_file]], # local verification passes)try:from skyvern.services.script_service import downloadawait download(prompt="Download invoice", label="test_block")- # Verification skipped → block completes, no AI fallback+ # S3 verification skipped, but local verification passed → block completesrefs["fallback"].assert_not_called()refs["update_block"].assert_called_once()# get_downloaded_files called only once (before-check), not 3 more times for after@@ -238,19 +255,23 @@ async def test_save_timeout_skips_verification(setup):@pytest.mark.asyncio-async def test_save_generic_exception_skips_verification(setup):- """Non-timeout S3 failure (e.g., permission error) should also skip verification.- Matches agent.py which catches both TimeoutError and generic Exception."""+async def test_save_generic_exception_skips_verification(setup, tmp_path):+ """Non-timeout S3 failure (e.g., permission error) should also skip S3 verification.+ Matches agent.py which catches both TimeoutError and generic Exception.+ Local file verification still passes since a file was downloaded."""+ download_dir = tmp_path / "downloads"+ local_file = str(download_dir / "file.pdf")refs = setup(save_side_effect=RuntimeError("S3 permission denied"),get_side_effect=[[]], # only the before-check runs; after-check is skipped+ list_files_side_effect=[[], [local_file]], # local verification passes)try:from skyvern.services.script_service import downloadawait download(prompt="Download invoice", label="test_block")- # Verification skipped → block completes, no AI fallback+ # S3 verification skipped, but local verification passed → block completesrefs["fallback"].assert_not_called()refs["update_block"].assert_called_once()# get_downloaded_files called only once (before-check), not 3 more times for after@@ -260,24 +281,33 @@ async def test_save_generic_exception_skips_verification(setup):@pytest.mark.asyncio-async def test_get_before_timeout_skips_verification(setup):- """If the before-check times out, verification should be skipped entirely- to avoid spurious AI fallbacks under degraded storage."""- refs = setup(get_side_effect=asyncio.TimeoutError())+async def test_get_before_timeout_skips_verification(setup, tmp_path):+ """If the before-check times out, S3 verification should be skipped entirely+ to avoid spurious AI fallbacks under degraded storage.+ Local file verification still passes since a file was downloaded."""+ download_dir = tmp_path / "downloads"+ local_file = str(download_dir / "file.pdf")+ refs = setup(+ get_side_effect=asyncio.TimeoutError(),+ list_files_side_effect=[[], [local_file]], # local verification passes+ )try:from skyvern.services.script_service import downloadawait download(prompt="Download invoice", label="test_block")- # Should NOT fall back — verification was skipped+ # Should NOT fall back — local verification passed, S3 verification skippedrefs["fallback"].assert_not_called()finally:_cleanup(refs)@pytest.mark.asyncio-async def test_get_after_timeout_skips_verification(setup):- """If the after-check times out, verification should be skipped."""+async def test_get_after_timeout_skips_verification(setup, tmp_path):+ """If the after-check times out, S3 verification should be skipped.+ Local file verification still passes since a file was downloaded."""+ download_dir = tmp_path / "downloads"+ local_file = str(download_dir / "file.pdf")call_count = {"n": 0}async def get_side_effect(**kwargs):@@ -286,7 +316,10 @@ async def test_get_after_timeout_skips_verification(setup):return [] # before-check succeedsraise asyncio.TimeoutError()- refs = setup(get_side_effect=get_side_effect)+ refs = setup(+ get_side_effect=get_side_effect,+ list_files_side_effect=[[], [local_file]], # local verification passes+ )try:from skyvern.services.script_service import download@@ -298,10 +331,16 @@ async def test_get_after_timeout_skips_verification(setup):@pytest.mark.asyncio-async def test_no_rename_without_download_suffix(setup):+async def test_no_rename_without_download_suffix(setup, tmp_path):"""When download_suffix is not provided, rename should not be called."""+ download_dir = tmp_path / "downloads"+ local_file = str(download_dir / "file.pdf")rename_mock = MagicMock()- refs = setup(get_side_effect=[[], ["file.pdf"]], rename_mock=rename_mock)+ refs = setup(+ get_side_effect=[[], ["file.pdf"]],+ rename_mock=rename_mock,+ list_files_side_effect=[[], [local_file]], # local verification passes+ )try:from skyvern.services.script_service import download@@ -322,7 +361,7 @@ async def test_rename_skips_crdownload_files(setup, tmp_path):rename_mock = MagicMock(return_value=str(download_dir / "renamed.pdf"))refs = setup(get_side_effect=[[], ["invoice.pdf"]],- list_files_side_effect=[[], [incomplete, complete]],+ list_files_side_effect=[[], [incomplete, complete], [incomplete, complete]], # before, local verify, renamerename_mock=rename_mock,)try:@@ -347,7 +386,7 @@ async def test_rename_handles_name_collision(setup, tmp_path):rename_mock = MagicMock(return_value=str(download_dir / "invoice_1.pdf"))refs = setup(get_side_effect=[[], ["invoice_1.pdf"]],- list_files_side_effect=[[], [new_file]],+ list_files_side_effect=[[], [new_file], [new_file]], # before, local verify, renamerename_mock=rename_mock,)try:@@ -361,9 +400,14 @@ async def test_rename_handles_name_collision(setup, tmp_path):@pytest.mark.asyncio-async def test_block_marked_completed_on_success(setup):+async def test_block_marked_completed_on_success(setup, tmp_path):"""On successful download + verification, block should be marked completed."""- refs = setup(get_side_effect=[[], ["file.pdf"]])+ download_dir = tmp_path / "downloads"+ local_file = str(download_dir / "file.pdf")+ refs = setup(+ get_side_effect=[[], ["file.pdf"]],+ list_files_side_effect=[[], [local_file]], # local verification passes+ )try:from skyvern.services.script_service import download@@ -373,3 +417,72 @@ async def test_block_marked_completed_on_success(setup):refs["fallback"].assert_not_called()finally:_cleanup(refs)+++async def test_local_verification_fails_when_no_file_downloaded(setup):+ """SKY-8433: When download_selector() returns None and click silently succeeds+ without downloading anything, the local filesystem check should detect+ no new files and trigger AI fallback instead of silently succeeding."""+ refs = setup(+ get_side_effect=[[]], # S3 before-check only; local check fails before S3 after-check+ list_files_side_effect=[[], []], # before=empty, after=still empty+ )+ try:+ from skyvern.services.script_service import download++ await download(prompt="Download invoice", label="test_block")++ refs["fallback"].assert_called_once()+ error_arg = refs["fallback"].call_args.kwargs.get("error")+ assert "did not produce a new file on the local filesystem" in str(error_arg)+ # S3 save should NOT be called since local check fails first+ refs["storage"].save_downloaded_files.assert_not_called()+ finally:+ _cleanup(refs)+++async def test_local_verification_ignores_crdownload_files(setup, tmp_path):+ """Incomplete .crdownload files should not count as successful downloads+ for local filesystem verification."""+ download_dir = tmp_path / "downloads"+ incomplete_file = str(download_dir / "invoice.pdf.crdownload")+ refs = setup(+ get_side_effect=[[]],+ list_files_side_effect=[[], [incomplete_file]], # only incomplete file appeared+ )+ try:+ from skyvern.services.script_service import download++ await download(prompt="Download invoice", label="test_block")++ refs["fallback"].assert_called_once()+ error_arg = refs["fallback"].call_args.kwargs.get("error")+ assert "did not produce a new file on the local filesystem" in str(error_arg)+ finally:+ _cleanup(refs)+++async def test_local_verification_passes_with_complete_file(setup, tmp_path):+ """When a complete file appears on the local filesystem after the cached+ function runs, the local verification should pass and allow the block+ to proceed to S3 upload and completion."""+ download_dir = tmp_path / "downloads"+ local_file = str(download_dir / "invoice.pdf")+ refs = setup(+ get_side_effect=[[], [local_file]], # S3 verification also passes+ list_files_side_effect=[[], [local_file]], # local verification passes+ )+ try:+ from skyvern.services.script_service import download++ await download(prompt="Download invoice", label="test_block")++ refs["fallback"].assert_not_called()+ refs["update_block"].assert_called_once()+ # S3 save should be called since local check passed+ refs["storage"].save_downloaded_files.assert_called_once()+ finally:+ _cleanup(refs)