Skip to content

Automate browser based workflows with AI

feat: dynamic download selectors for file_download blocks in for-loops (#5077)

pedrohsdbf894be1344a5
diff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.py
index 0a7e4b4d4..3ea075288 100644
--- a/skyvern/core/script_generations/generate_script.py
+++ b/skyvern/core/script_generations/generate_script.py
@@ -1147,6 +1147,7 @@ def _build_block_fn(
actions: list[dict[str, Any]],
value_to_param: dict[str, str] | None = None,
use_semantic_selectors: bool = False,
+ is_in_for_loop: bool = False,
) -> FunctionDef:
# Check if this block is form-filling (4+ input_text/select actions)
if use_semantic_selectors and _is_form_filling_block(actions):
@@ -1162,28 +1163,37 @@ def _build_block_fn(
if block.get("url"):
body_stmts.append(cst.parse_statement(f"await page.goto({repr(block['url'])})"))
- for act in actions:
- if act["action_type"] in [
- ActionType.COMPLETE,
- ActionType.TERMINATE,
- ActionType.NULL_ACTION,
- ]:
- continue
-
- # For extraction blocks, assign extract action results to output variable
- assign_to_output = act["action_type"] == "extract"
+ # For file_download blocks inside for-loops, generate a dynamic click that uses
+ # per-iteration context instead of hardcoded xpath/prompt from iteration 0.
+ block_type = block.get("block_type")
+ if is_in_for_loop and block_type == "file_download":
body_stmts.append(
- _action_to_stmt(
- act,
- block,
- assign_to_output=assign_to_output,
- value_to_param=value_to_param,
- use_semantic_selectors=use_semantic_selectors,
+ cst.parse_statement(
+ 'await page.click(selector=context.download_selector(), prompt=context.prompt, ai="fallback")'
)
)
+ else:
+ for act in actions:
+ if act["action_type"] in [
+ ActionType.COMPLETE,
+ ActionType.TERMINATE,
+ ActionType.NULL_ACTION,
+ ]:
+ continue
+
+ # For extraction blocks, assign extract action results to output variable
+ assign_to_output = act["action_type"] == "extract"
+ body_stmts.append(
+ _action_to_stmt(
+ act,
+ block,
+ assign_to_output=assign_to_output,
+ value_to_param=value_to_param,
+ use_semantic_selectors=use_semantic_selectors,
+ )
+ )
# add complete action
- block_type = block.get("block_type")
if block_type in SCRIPT_TASK_BLOCKS_WITH_COMPLETE_ACTION:
complete_action = {"action_type": "complete"}
body_stmts.append(_action_to_stmt(complete_action, block))
@@ -2818,6 +2828,7 @@ async def generate_workflow_script_python_code(
inner_actions,
value_to_param=value_to_param,
use_semantic_selectors=use_semantic_selectors,
+ is_in_for_loop=True,
)
inner_block_code = cst.Module(body=[inner_fn_def]).code
diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.py
index 774c91f89..b74ef2355 100644
--- a/skyvern/core/script_generations/skyvern_page.py
+++ b/skyvern/core/script_generations/skyvern_page.py
@@ -2917,3 +2917,43 @@ class RunContext:
self.actions_and_results: list[tuple[Action, list[ActionResult]]] = []
# Pre-extracted values from applicant context, keyed by canonical category name
self.extracted_params: dict[str, str | None] = extracted_params or {}
+
+ @property
+ def prompt(self) -> str | None:
+ """Return the per-iteration prompt from SkyvernContext (set by script_service per loop iteration)."""
+ ctx = skyvern_context.current()
+ return ctx.prompt if ctx else None
+
+ @property
+ def loop_value(self) -> Any | None:
+ """Return the current loop iteration value from SkyvernContext.loop_metadata."""
+ ctx = skyvern_context.current()
+ if ctx and ctx.loop_metadata:
+ return ctx.loop_metadata.get("current_value")
+ return None
+
+ def download_selector(self) -> str | None:
+ """Build a CSS selector targeting a download link from the current loop value.
+
+ Scans the loop_value dict for URL-like strings, extracts the filename,
+ and returns a selector like ``a[href*="filename.pdf"]``. Returns None
+ if no URL is found.
+ """
+ value = self.loop_value
+ if not value or not isinstance(value, dict):
+ return None
+ for v in value.values():
+ if not isinstance(v, str):
+ continue
+ # Match URL-like strings (http/https or paths with file extensions)
+ if re.match(r"https?://", v) or re.match(r"/.*\.\w+", v):
+ # Extract filename from URL path
+ filename = v.rstrip("/").rsplit("/", 1)[-1]
+ # Strip query params
+ filename = filename.split("?")[0]
+ if filename and "." in filename:
+ # Escape CSS special characters to avoid malformed selectors
+ filename = re.sub(r'["\[\]\\]', "", filename)
+ if filename:
+ return f'a[href*="{filename}"]'
+ return None
diff --git a/skyvern/forge/prompts/skyvern/script-reviewer.j2 b/skyvern/forge/prompts/skyvern/script-reviewer.j2
index 0b6f7f54b..f55974bd6 100644
--- a/skyvern/forge/prompts/skyvern/script-reviewer.j2
+++ b/skyvern/forge/prompts/skyvern/script-reviewer.j2
@@ -27,7 +27,9 @@ You are a script reviewer for a browser automation system. Your job is to update
await page.fill(selector='...', ai='proactive', prompt='Select how you heard about this job')
```
10. **URL PATTERNS**: When different variants come from different URL hosts or paths, add `url_patterns` to the `page.classify()` call for FREE Tier-0 disambiguation. Use regex on the URL path: `url_patterns={"variant_a": "example\\.com/form-a"}`. URL patterns are checked BEFORE text patterns and cost nothing.
-11. **FOR-LOOP BLOCKS**: When code runs inside a for-loop (e.g. by iterating over extracted items), every `page.click()`, `page.goto()`, or `page.download_file()` call MUST use the loop variable (`current_value`) to target the specific item. Never use a static selector or generic prompt that would resolve to the same element on every iteration. Prefer `page.goto(current_value["url"])` for direct URLs, or include the item's title/identifier in the selector: `selector=f'a:has-text("{current_value["title"]}")'`.
+11. **FOR-LOOP BLOCKS**: When code runs inside a for-loop (e.g. by iterating over extracted items), every `page.click()`, `page.goto()`, or `page.download_file()` call MUST use per-iteration data to target the specific item. Never use a static selector or generic prompt that would resolve to the same element on every iteration.
+ - **File download blocks**: Use `context.download_selector()` which builds a CSS selector (e.g. `a[href*="filename.pdf"]`) from URL-like fields in the current loop value. Combine with `context.prompt` for AI fallback: `await page.click(selector=context.download_selector(), prompt=context.prompt, ai="fallback")`. If `download_selector()` returns `None` (no URL in loop value), fall back to a text-based selector using `context.loop_value`.
+ - **Other blocks**: Use `context.loop_value` (dict) to build selectors. Prefer `page.goto(context.loop_value["url"])` for direct URLs, or include the item's title/identifier in the selector: `selector=f'a:has-text("{context.loop_value["title"]}")'`.
## Block Navigation Goal
```
@@ -384,6 +386,12 @@ async def block_fn(page, context):
### Properties
- `page.url` — Current page URL (str property, free, no LLM call)
+### Context Properties (for-loop blocks only)
+These properties are available on the `context` object when code runs inside a for-loop:
+- `context.loop_value` — Current loop iteration value (dict). Contains the extracted data for this iteration (e.g. `{"title": "...", "download_url": "...", "unique_id": "..."}`).
+- `context.prompt` — Per-iteration navigation goal/prompt (str or None). Set automatically from the block's navigation_goal with the current loop value substituted.
+- `context.download_selector()` — Builds a CSS selector targeting a download link from `context.loop_value`. Scans the dict values for URL-like strings, extracts the filename, and returns e.g. `'a[href*="filename.pdf"]'`. Returns `None` if no URL is found — in that case, build a selector manually from `context.loop_value`.
+
### DO NOT USE (these do NOT exist and will crash at runtime)
- ~~`page.completed`~~ — Does NOT exist. Use `await page.complete()` to finish.
- ~~`page.current_url`~~ — Does NOT exist. Use `page.url` instead.
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index 71b845732..e308a6a4f 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -1820,8 +1820,52 @@ async def download(
try:
await _prepare_cached_block_inputs(cache_key, prompt)
+
+ # Count downloaded files before running cached function so we can
+ # verify that the download actually produced a new file.
+ org_id = context.organization_id or ""
+ run_id = context.workflow_run_id or ""
+ files_before: list = []
+ files_before_ok = False
+ try:
+ async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):
+ files_before = await app.STORAGE.get_downloaded_files(
+ organization_id=org_id,
+ run_id=run_id,
+ )
+ files_before_ok = True
+ except asyncio.TimeoutError:
+ LOG.warning("Timeout getting downloaded files before cached download")
+
await _run_cached_function(cached_fn)
+ # Verify a new file was actually downloaded.
+ # Retry briefly — file may not be visible in storage immediately after the click.
+ files_after: list = []
+ files_after_ok = False
+ for _attempt in range(3):
+ try:
+ async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):
+ files_after = await app.STORAGE.get_downloaded_files(
+ organization_id=org_id,
+ run_id=run_id,
+ )
+ files_after_ok = True
+ except asyncio.TimeoutError:
+ LOG.warning("Timeout getting downloaded files after cached download")
+ if len(files_after) > len(files_before):
+ break
+ if _attempt < 2:
+ await asyncio.sleep(2)
+
+ # Only raise if both calls succeeded — if either timed out, skip
+ # the check to avoid spurious AI fallbacks under degraded storage.
+ if files_before_ok and files_after_ok and len(files_after) <= len(files_before):
+ raise Exception(
+ "Cached download function did not produce a new file. "
+ f"Files before: {len(files_before)}, after: {len(files_after)}"
+ )
+
# Update block status to completed if workflow block was created
if workflow_run_block_id:
await _update_workflow_block(