Skip to content

Automate browser based workflows with AI

Class the file-attached-no-request upload result apart from a failed upload, and accept an upload control that is not the input (SKY-16449) (#8582)

pedrohsdbf0e263bc42f9
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index fd6fc4c1f..c2cce32ae 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -62,6 +62,12 @@ ToolErrorClass = Literal[
"covered",
"inert",
"unreachable",
+ # `file_upload`: the target is not a file input, holds no single one, and opened no file picker;
+ # or it is one whose click would submit its form, so it was not clicked.
+ "no_file_input",
+ "submits_form",
+ # `file_upload`: the input could not be read back and no upload request was seen.
+ "attach_unconfirmed",
# The field resolved and the page cooperated, but the requested VALUE named no single option.
# Each of these names only what was READ: an absence claim holds solely over a list read in full,
# which is why a declared-but-truncated list gets `rows_unread` rather than `no_matching_row`.
@@ -95,6 +101,11 @@ ToolOkClass = Literal[
"solved",
"absent",
"attempts_exhausted",
+ # `file_upload`. `attached_no_activity` is the file confirmed on the input with no upload request
+ # seen -- a form that sends the file with the submit lands here, and so does an unwired handler.
+ "upload_seen",
+ "consumed_shown",
+ "attached_no_activity",
]
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 0e018ab56..77f6abb38 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -7119,16 +7119,58 @@ def _newly_rendered_lines(before: str, after: str) -> list[str]:
return [line.strip() for line in after.splitlines() if line.strip() and line.strip() not in seen]
-async def _input_holds_file(el: Any) -> bool:
+async def _input_holds_file(el: Any) -> bool | None:
"""Playwright-layer readback that set_input_files populated the control — proves the file attached to
- the input element, not that the site registered it. Fail-open: an unreadable control must never turn a
- real upload into a false negative."""
+ the input element, not that the site registered it. None when the control is unreadable, which must
+ never turn a real upload into a false negative, nor count as confirmation."""
try:
count = await el.evaluate("e => (e && e.files) ? e.files.length : 0")
return bool(count) and int(count) > 0
except Exception:
- LOG.info("taskv3 file-input populate readback failed, assuming populated", exc_info=True)
- return True
+ LOG.info("taskv3 file-input populate readback failed", exc_info=True)
+ return None
+
+
+# Where a file_upload target's file goes, answered as v1's upload handler does: the target itself when
+# it is a file input, its control when it is a <label> for one, else the single file input inside it,
+# open shadow roots included. More than one inside is ambiguous and yields nothing. The control, not
+# the label, because the populate readback reads `.files` off whatever this returns.
+_FILE_INPUT_FOR_JS = (
+ r"""(e) => {
+ const _shadowRoots = """
+ + _SHADOW_ROOTS_JS
+ + r""";
+ const isFile = (n) => !!n && n.tagName === 'INPUT' && String(n.type || '').toLowerCase() === 'file';
+ if (isFile(e)) return e;
+ if (e.tagName === 'LABEL' && isFile(e.control)) return e.control;
+ const starts = [e];
+ if (e.shadowRoot && e.shadowRoot.nodeType === 11) starts.push(e.shadowRoot);
+ const found = new Set();
+ for (const root of starts.flatMap(_shadowRoots)) {
+ try { for (const n of root.querySelectorAll('input')) if (isFile(n)) found.add(n); } catch (err) {}
+ }
+ return found.size === 1 ? found.values().next().value : null;
+}"""
+)
+
+# `type` is `submit` on a typeless <button> too, so this covers the implicit default button of a form.
+_SUBMITS_FORM_JS = "e => !!e.form && (e.tagName === 'BUTTON' || e.tagName === 'INPUT') && ['submit', 'image'].includes(String(e.type).toLowerCase())"
+
+_FILE_CHOOSER_TIMEOUT_MS = 3000
+
+
+async def _file_input_for(el: Any) -> Any | None:
+ """The element to set files on, or None when no file input is associated with `el`. An unreadable
+ target keeps `el`, so the driver's own error still reports it."""
+ try:
+ handle = await el.evaluate_handle(_FILE_INPUT_FOR_JS)
+ except Exception:
+ LOG.info("taskv3 file_upload file-input resolution failed, using the target as given", exc_info=True)
+ return el
+ target = handle.as_element()
+ if target is None:
+ await handle.dispose()
+ return target
# Counts fields holding in-progress state a reload would discard, piercing shadow roots. Unlike the
@@ -11793,6 +11835,16 @@ def build_browser_tools(
{**staged, "page_state_changed": True},
error_class="stale_selector",
)
+ file_input = await _file_input_for(el)
+ if file_input is None and await el.evaluate(_SUBMITS_FORM_JS) is True:
+ # v1 would click it and hope for a file picker; a click here can send the form instead.
+ return ToolResult(
+ "error",
+ f"{selector} is not a file input and clicking it would submit the form, so nothing was "
+ f"clicked — target the file input or the control that opens the file picker",
+ staged,
+ error_class="submits_form",
+ )
# Verify the upload took EFFECT, not just that set_input_files did not raise. Watch upload-like
# network dispatches across the set_input_files + settle window (the window we already dwell in,
# so this adds no latency); a genuine upload dispatches at least one, a silent no-op none.
@@ -11812,8 +11864,28 @@ def build_browser_tools(
text_before = await _whole_page_text(_current_page())
probe.start()
try:
- await el.set_input_files([local_path])
- populated = await _input_holds_file(el)
+ if file_input is None:
+ # A styled dropzone or button with no file input inside it: click it and fill the
+ # picker it opens, as v1 does. The picker belongs to the page, whichever frame opened it.
+ try:
+ async with _current_page().expect_file_chooser(timeout=_FILE_CHOOSER_TIMEOUT_MS) as chooser_info:
+ await el.click(timeout=_FILE_CHOOSER_TIMEOUT_MS)
+ chooser = await chooser_info.value
+ except Exception as exc:
+ if not is_driver_timeout_error(exc):
+ raise
+ return ToolResult(
+ "error",
+ f"{selector} is not a file input, holds no single file input, and clicking it opened "
+ f"no file picker — target the file input or the control that opens the file picker",
+ {**staged, "page_state_changed": True},
+ error_class="no_file_input",
+ )
+ await chooser.set_files([local_path])
+ file_input = chooser.element
+ else:
+ await file_input.set_input_files([local_path])
+ populated = await _input_holds_file(file_input)
# Settle + a small randomized delay so the upload and a following submit are not dispatched
# in the same instant, matching v1's upload cadence (the engine that clears this step reliably).
# Page-level too: it reuses v1's network-idle/DOM-stability wait, which describes the whole
@@ -11822,7 +11894,11 @@ def build_browser_tools(
await _upload_submit_delay()
finally:
probe.stop()
- if not populated:
+ if populated is not False and not probe.saw_upload():
+ # With no request seen, the readback alone decides the ok, so it must describe the input as it
+ # is now: a change handler can clear or reject the file during the settle.
+ populated = await _input_holds_file(file_input)
+ if populated is False:
# A consume-and-clear dropzone reads the file on change, uploads it and resets the input, so
# an empty control after a genuine upload is normal there. Confirming it needs every signal
# a silent no-op cannot fake at once: the file's own name newly rendered on the page AND an
@@ -11846,6 +11922,7 @@ def build_browser_tools(
return ToolResult.ok(
f"uploaded 1 file to {selector} (the site consumed the file and now shows it: {said!r})",
staged,
+ ok_class="consumed_shown",
)
LOG.info(
"taskv3 file_upload input cleared after attach; page names the file without confirming it",
@@ -11858,18 +11935,27 @@ def build_browser_tools(
staged,
)
return ToolResult("error", f"file did not attach to {selector} — re-observe the field", staged)
- if not probe.saw_upload():
- # The file is on the input but the site never reacted: report a recoverable error (not a
- # confident OK) so the loop re-verifies before submitting. A submit-time-upload form lands
- # here too and costs one re-plan turn, never a lost file.
+ if probe.saw_upload():
+ return ToolResult.ok(f"uploaded 1 file to {selector}", staged, ok_class="upload_seen")
+ if populated is None:
return ToolResult(
"error",
- f"attached the file to {selector} but observed no upload activity — re-observe the field "
- f"to confirm the file is shown before submitting; if the form uploads on submit this may "
- f"be expected",
+ f"set the file on {selector} but could neither read the input back nor see an upload "
+ f"request — re-observe the field to confirm the file is shown before submitting",
staged,
+ error_class="attach_unconfirmed",
)
- return ToolResult.ok(f"uploaded 1 file to {selector}", staged)
+ # The tool's postcondition is the file on the input, and the readback confirmed it; whether the
+ # page sends it now or with the submit is the form's behaviour. An unwired change handler also
+ # lands here: the recoverable error this used to be drew no retry from the model either.
+ return ToolResult.ok(
+ f"uploaded 1 file to {selector}: the input holds it, and the site sent no upload request yet — "
+ f"normal for a form that sends the file when it is submitted. A hidden file input does not show "
+ f"in observe, so re-checking it will not show the file; look instead for an upload error or a "
+ f"still-required file field on the page",
+ staged,
+ ok_class="attached_no_activity",
+ )
async def select_combobox(args: dict[str, Any]) -> ToolResult:
# Explicit typeahead fill (type() also drives this automatically). Routes through the shared
@@ -12451,9 +12537,9 @@ def build_browser_tools(
if not kind:
return
# A FILL is recorded whether or not the VERDICT was ok, because mutating the control and
- # returning an error are independent outcomes. `file_upload` reports an error with the file
- # ALREADY on the input when it sees no upload activity -- the ordinary submit-time-upload form --
- # and a typeahead commit can fail with the text typed. Gated on the verdict, that work is
+ # returning an error are independent outcomes. `file_upload` can report an error after the file
+ # was set (an unreadable input with no upload request seen), and a typeahead commit can fail with
+ # the text typed. Gated on the verdict, that work is
# invisible to the reload guard and a same-url `navigate()` discards it, the file included.
#
# Recording an attempt that changed nothing is the benign direction: it costs one recoverable
@@ -12759,7 +12845,8 @@ def build_browser_tools(
_spec("navigate", "Navigate the browser to a URL.", _obj({"url": {"type": "string"}}, ["url"]), navigate),
_spec(
"file_upload",
- "Upload a file (local path or URL) into a file input by its observe ref (e.g. ref=12) or a CSS selector.",
+ "Upload a file (local path or URL) into a file input, or into the upload button/dropzone that "
+ "holds or opens one, by its observe ref (e.g. ref=12) or a CSS selector.",
_obj({"selector": {"type": "string"}, "file": {"type": "string"}}, ["selector", "file"]),
file_upload,
),
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 5b9efb5d8..f5bd2db11 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -14850,31 +14850,45 @@ def _patch_upload_download(monkeypatch: pytest.MonkeyPatch) -> None:
@pytest.mark.asyncio
-async def test_file_upload_no_upload_activity_returns_actionable_error_not_false_ok(
- monkeypatch: pytest.MonkeyPatch,
[email protected]("input_state", ["holds_file", "unreadable", "cleared_during_settle"])
+async def test_file_upload_with_no_upload_activity_is_ok_only_when_the_input_confirms_the_file(
+ monkeypatch: pytest.MonkeyPatch, input_state: str
) -> None:
- # The core fix: set_input_files can populate the control at the Playwright layer yet the site
- # register nothing (post-navigation the change handler is not wired) — zero upload requests
- # dispatched. file_upload must return a recoverable non-OK there, not a confident OK that makes the
- # agent submit with no file. A submit-time-upload form lands here too as an accepted false-negative.
- # RED against pre-fix code, which returned "uploaded 1 file" regardless of activity.
+ # A form that sends the file with the submit dispatches nothing at attach time. The input holding
+ # the file when the tool returns is its postcondition, so that is an ok; an unreadable input, or one
+ # a change handler emptied while the page settled, confirms nothing and stays an error.
import skyvern.forge.taskv3.tools as tools_module
page = _FakePage()
- page.element.emit_upload_on_set = False # file lands in the input, but the site never reacts
+ page.element.emit_upload_on_set = False
+ if input_state == "unreadable":
+
+ async def _unreadable(_js: str, _arg: Any = None) -> Any:
+ raise RuntimeError("Execution context was destroyed")
+
+ monkeypatch.setattr(page.element, "evaluate", _unreadable)
_patch_upload_dwell(monkeypatch, tools_module)
+ if input_state == "cleared_during_settle":
+
+ async def _settle_clears(_page: Any) -> None:
+ page.element._files = []
+
+ monkeypatch.setattr(tools_module, "_settle_after_upload", _settle_clears)
_patch_upload_download(monkeypatch)
tools = build_browser_tools(_fixed_page_provider(page))
r = await _tool(tools, "file_upload").handler({"selector": "#cv", "file": "resume.pdf"})
- assert r.status == "error", r.content
- assert "no upload activity" in r.content
- assert "uploaded 1 file" not in r.content
- # The file was still attached at the Playwright layer, so the staged-download key must persist so
- # the download-signal wrapper still suppresses the staged file.
+ if input_state == "holds_file":
+ assert (r.status, r.ok_class) == ("ok", "attached_no_activity"), r.content
+ assert "sent no upload request" in r.content
+ elif input_state == "unreadable":
+ assert (r.status, r.error_class) == ("error", "attach_unconfirmed"), r.content
+ else:
+ assert r.status == "error", r.content
+ assert "did not attach" in r.content
+ # The staged-download key must persist so the download-signal wrapper still suppresses the file.
assert (r.data or {}).get("staged_download") == "cv.pdf"
- # The request listener must be removed after the call — no leaked/accumulating listeners.
assert page._request_listeners == []
@@ -15218,11 +15232,95 @@ async def test_file_upload_ignores_non_upload_network_noise(monkeypatch: pytest.
tools = build_browser_tools(_fixed_page_provider(page))
r = await _tool(tools, "file_upload").handler({"selector": "#cv", "file": "resume.pdf"})
- assert r.status == "error", r.content
- assert "no upload activity" in r.content
+ assert (r.status, r.ok_class) == ("ok", "attached_no_activity"), r.content
assert page._request_listeners == []
+_UPLOAD_TARGETS_HTML = """<!doctype html><html><body>
+<form id="apply" onsubmit="event.preventDefault(); window.__submitted = true;">
+ <div id="dropzone" style="border:1px dashed;padding:20px">Drop your resume
+ <input id="inner" type="file" style="display:none">
+ </div>
+ <input id="picker-input" type="file" style="display:none">
+ <input id="labelled-input" type="file" style="display:none">
+ <label id="choose" for="labelled-input">Choose file</label>
+ <button id="attach" type="button" onclick="document.getElementById('picker-input').click()">Attach</button>
+ <div id="inert" style="padding:10px">Nothing to click</div>
+ <button id="send">Send application</button>
+</form>
+<input id="formless-input" type="file" style="display:none">
+<button id="formless-attach" onclick="document.getElementById('formless-input').click()">Select resume</button>
+</body></html>"""
+
+
+@_skip_no_browser
+ ("selector", "receiving_input"),
+ [
+ ("#dropzone", "#inner"),
+ ("#attach", "#picker-input"),
+ ("#choose", "#labelled-input"),
+ # A typeless button outside any form submits nothing, so it is clicked.
+ ("#formless-attach", "#formless-input"),
+ ],
+)
+async def test_file_upload_on_an_upload_control_that_is_not_the_input_lands_the_file(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, selector: str, receiving_input: str
+) -> None:
+ # The model targets the visible control, since observe does not list a hidden file input. The
+ # driver refuses a non-input ("Node is not an HTMLInputElement"); v1 sets the file on the input
+ # inside the control, or clicks it and fills the picker it opens.
+ import skyvern.forge.sdk.api.files as files_module
+ import skyvern.forge.taskv3.tools as tools_module
+
+ cv = tmp_path / "cv.pdf"
+ cv.write_bytes(b"%PDF-1.4 synthetic")
+
+ async def _fake_download(source: str, output_dir: str | None = None, organization_id: str | None = None) -> str:
+ return str(cv)
+
+ monkeypatch.setattr(files_module, "download_file", _fake_download)
+ _patch_upload_dwell(monkeypatch, tools_module)
+ async with _content_page(_UPLOAD_TARGETS_HTML) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "file_upload").handler({"selector": selector, "file": "resume.pdf"})
+
+ assert (r.status, r.ok_class) == ("ok", "attached_no_activity"), r.content
+ assert await page.eval_on_selector(receiving_input, "e => e.files.length") == 1
+ assert not await page.evaluate("() => !!window.__submitted")
+
+
+@_skip_no_browser
+ ("selector", "error_class"),
+ [("#inert", "no_file_input"), ("#send", "submits_form")],
+)
+async def test_file_upload_on_a_control_with_no_file_input_reports_it_without_submitting(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, selector: str, error_class: str
+) -> None:
+ import skyvern.forge.sdk.api.files as files_module
+ import skyvern.forge.taskv3.tools as tools_module
+
+ cv = tmp_path / "cv.pdf"
+ cv.write_bytes(b"%PDF-1.4 synthetic")
+
+ async def _fake_download(source: str, output_dir: str | None = None, organization_id: str | None = None) -> str:
+ return str(cv)
+
+ monkeypatch.setattr(files_module, "download_file", _fake_download)
+ monkeypatch.setattr(tools_module, "_FILE_CHOOSER_TIMEOUT_MS", 300)
+ _patch_upload_dwell(monkeypatch, tools_module)
+ async with _content_page(_UPLOAD_TARGETS_HTML) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "file_upload").handler({"selector": selector, "file": "resume.pdf"})
+
+ assert not await page.evaluate("() => !!window.__submitted"), r.content
+ assert r.status == "error", r.content
+ assert r.error_class == error_class, r.content
+
+
@pytest.mark.asyncio
async def test_non_upload_actions_do_not_settle_or_delay(monkeypatch: pytest.MonkeyPatch) -> None:
# The dwell is upload-specific: click/type must NOT incur the settle or the delay, so the tool