Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Class the file-attached-no-request upload result apart from a failed upload, and accept an upload control that is not the input (SKY-16449) (#8582)
pedrohsdbf0e263bc42f9
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex fd6fc4c1f..c2cce32ae 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -62,6 +62,12 @@ ToolErrorClass = Literal["covered","inert","unreachable",+ # `file_upload`: the target is not a file input, holds no single one, and opened no file picker;+ # or it is one whose click would submit its form, so it was not clicked.+ "no_file_input",+ "submits_form",+ # `file_upload`: the input could not be read back and no upload request was seen.+ "attach_unconfirmed",# The field resolved and the page cooperated, but the requested VALUE named no single option.# Each of these names only what was READ: an absence claim holds solely over a list read in full,# which is why a declared-but-truncated list gets `rows_unread` rather than `no_matching_row`.@@ -95,6 +101,11 @@ ToolOkClass = Literal["solved","absent","attempts_exhausted",+ # `file_upload`. `attached_no_activity` is the file confirmed on the input with no upload request+ # seen -- a form that sends the file with the submit lands here, and so does an unwired handler.+ "upload_seen",+ "consumed_shown",+ "attached_no_activity",]diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex 0e018ab56..77f6abb38 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -7119,16 +7119,58 @@ def _newly_rendered_lines(before: str, after: str) -> list[str]:return [line.strip() for line in after.splitlines() if line.strip() and line.strip() not in seen]-async def _input_holds_file(el: Any) -> bool:+async def _input_holds_file(el: Any) -> bool | None:"""Playwright-layer readback that set_input_files populated the control — proves the file attached to- the input element, not that the site registered it. Fail-open: an unreadable control must never turn a- real upload into a false negative."""+ the input element, not that the site registered it. None when the control is unreadable, which must+ never turn a real upload into a false negative, nor count as confirmation."""try:count = await el.evaluate("e => (e && e.files) ? e.files.length : 0")return bool(count) and int(count) > 0except Exception:- LOG.info("taskv3 file-input populate readback failed, assuming populated", exc_info=True)- return True+ LOG.info("taskv3 file-input populate readback failed", exc_info=True)+ return None+++# Where a file_upload target's file goes, answered as v1's upload handler does: the target itself when+# it is a file input, its control when it is a <label> for one, else the single file input inside it,+# open shadow roots included. More than one inside is ambiguous and yields nothing. The control, not+# the label, because the populate readback reads `.files` off whatever this returns.+_FILE_INPUT_FOR_JS = (+ r"""(e) => {+ const _shadowRoots = """+ + _SHADOW_ROOTS_JS+ + r""";+ const isFile = (n) => !!n && n.tagName === 'INPUT' && String(n.type || '').toLowerCase() === 'file';+ if (isFile(e)) return e;+ if (e.tagName === 'LABEL' && isFile(e.control)) return e.control;+ const starts = [e];+ if (e.shadowRoot && e.shadowRoot.nodeType === 11) starts.push(e.shadowRoot);+ const found = new Set();+ for (const root of starts.flatMap(_shadowRoots)) {+ try { for (const n of root.querySelectorAll('input')) if (isFile(n)) found.add(n); } catch (err) {}+ }+ return found.size === 1 ? found.values().next().value : null;+}"""+)++# `type` is `submit` on a typeless <button> too, so this covers the implicit default button of a form.+_SUBMITS_FORM_JS = "e => !!e.form && (e.tagName === 'BUTTON' || e.tagName === 'INPUT') && ['submit', 'image'].includes(String(e.type).toLowerCase())"++_FILE_CHOOSER_TIMEOUT_MS = 3000+++async def _file_input_for(el: Any) -> Any | None:+ """The element to set files on, or None when no file input is associated with `el`. An unreadable+ target keeps `el`, so the driver's own error still reports it."""+ try:+ handle = await el.evaluate_handle(_FILE_INPUT_FOR_JS)+ except Exception:+ LOG.info("taskv3 file_upload file-input resolution failed, using the target as given", exc_info=True)+ return el+ target = handle.as_element()+ if target is None:+ await handle.dispose()+ return target# Counts fields holding in-progress state a reload would discard, piercing shadow roots. Unlike the@@ -11793,6 +11835,16 @@ def build_browser_tools({**staged, "page_state_changed": True},error_class="stale_selector",)+ file_input = await _file_input_for(el)+ if file_input is None and await el.evaluate(_SUBMITS_FORM_JS) is True:+ # v1 would click it and hope for a file picker; a click here can send the form instead.+ return ToolResult(+ "error",+ f"{selector} is not a file input and clicking it would submit the form, so nothing was "+ f"clicked — target the file input or the control that opens the file picker",+ staged,+ error_class="submits_form",+ )# Verify the upload took EFFECT, not just that set_input_files did not raise. Watch upload-like# network dispatches across the set_input_files + settle window (the window we already dwell in,# so this adds no latency); a genuine upload dispatches at least one, a silent no-op none.@@ -11812,8 +11864,28 @@ def build_browser_tools(text_before = await _whole_page_text(_current_page())probe.start()try:- await el.set_input_files([local_path])- populated = await _input_holds_file(el)+ if file_input is None:+ # A styled dropzone or button with no file input inside it: click it and fill the+ # picker it opens, as v1 does. The picker belongs to the page, whichever frame opened it.+ try:+ async with _current_page().expect_file_chooser(timeout=_FILE_CHOOSER_TIMEOUT_MS) as chooser_info:+ await el.click(timeout=_FILE_CHOOSER_TIMEOUT_MS)+ chooser = await chooser_info.value+ except Exception as exc:+ if not is_driver_timeout_error(exc):+ raise+ return ToolResult(+ "error",+ f"{selector} is not a file input, holds no single file input, and clicking it opened "+ f"no file picker — target the file input or the control that opens the file picker",+ {**staged, "page_state_changed": True},+ error_class="no_file_input",+ )+ await chooser.set_files([local_path])+ file_input = chooser.element+ else:+ await file_input.set_input_files([local_path])+ populated = await _input_holds_file(file_input)# Settle + a small randomized delay so the upload and a following submit are not dispatched# in the same instant, matching v1's upload cadence (the engine that clears this step reliably).# Page-level too: it reuses v1's network-idle/DOM-stability wait, which describes the whole@@ -11822,7 +11894,11 @@ def build_browser_tools(await _upload_submit_delay()finally:probe.stop()- if not populated:+ if populated is not False and not probe.saw_upload():+ # With no request seen, the readback alone decides the ok, so it must describe the input as it+ # is now: a change handler can clear or reject the file during the settle.+ populated = await _input_holds_file(file_input)+ if populated is False:# A consume-and-clear dropzone reads the file on change, uploads it and resets the input, so# an empty control after a genuine upload is normal there. Confirming it needs every signal# a silent no-op cannot fake at once: the file's own name newly rendered on the page AND an@@ -11846,6 +11922,7 @@ def build_browser_tools(return ToolResult.ok(f"uploaded 1 file to {selector} (the site consumed the file and now shows it: {said!r})",staged,+ ok_class="consumed_shown",)LOG.info("taskv3 file_upload input cleared after attach; page names the file without confirming it",@@ -11858,18 +11935,27 @@ def build_browser_tools(staged,)return ToolResult("error", f"file did not attach to {selector} — re-observe the field", staged)- if not probe.saw_upload():- # The file is on the input but the site never reacted: report a recoverable error (not a- # confident OK) so the loop re-verifies before submitting. A submit-time-upload form lands- # here too and costs one re-plan turn, never a lost file.+ if probe.saw_upload():+ return ToolResult.ok(f"uploaded 1 file to {selector}", staged, ok_class="upload_seen")+ if populated is None:return ToolResult("error",- f"attached the file to {selector} but observed no upload activity — re-observe the field "- f"to confirm the file is shown before submitting; if the form uploads on submit this may "- f"be expected",+ f"set the file on {selector} but could neither read the input back nor see an upload "+ f"request — re-observe the field to confirm the file is shown before submitting",staged,+ error_class="attach_unconfirmed",)- return ToolResult.ok(f"uploaded 1 file to {selector}", staged)+ # The tool's postcondition is the file on the input, and the readback confirmed it; whether the+ # page sends it now or with the submit is the form's behaviour. An unwired change handler also+ # lands here: the recoverable error this used to be drew no retry from the model either.+ return ToolResult.ok(+ f"uploaded 1 file to {selector}: the input holds it, and the site sent no upload request yet — "+ f"normal for a form that sends the file when it is submitted. A hidden file input does not show "+ f"in observe, so re-checking it will not show the file; look instead for an upload error or a "+ f"still-required file field on the page",+ staged,+ ok_class="attached_no_activity",+ )async def select_combobox(args: dict[str, Any]) -> ToolResult:# Explicit typeahead fill (type() also drives this automatically). Routes through the shared@@ -12451,9 +12537,9 @@ def build_browser_tools(if not kind:return# A FILL is recorded whether or not the VERDICT was ok, because mutating the control and- # returning an error are independent outcomes. `file_upload` reports an error with the file- # ALREADY on the input when it sees no upload activity -- the ordinary submit-time-upload form --- # and a typeahead commit can fail with the text typed. Gated on the verdict, that work is+ # returning an error are independent outcomes. `file_upload` can report an error after the file+ # was set (an unreadable input with no upload request seen), and a typeahead commit can fail with+ # the text typed. Gated on the verdict, that work is# invisible to the reload guard and a same-url `navigate()` discards it, the file included.## Recording an attempt that changed nothing is the benign direction: it costs one recoverable@@ -12759,7 +12845,8 @@ def build_browser_tools(_spec("navigate", "Navigate the browser to a URL.", _obj({"url": {"type": "string"}}, ["url"]), navigate),_spec("file_upload",- "Upload a file (local path or URL) into a file input by its observe ref (e.g. ref=12) or a CSS selector.",+ "Upload a file (local path or URL) into a file input, or into the upload button/dropzone that "+ "holds or opens one, by its observe ref (e.g. ref=12) or a CSS selector.",_obj({"selector": {"type": "string"}, "file": {"type": "string"}}, ["selector", "file"]),file_upload,),diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.pyindex 5b9efb5d8..f5bd2db11 100644--- a/tests/unit/test_taskv3_tools.py+++ b/tests/unit/test_taskv3_tools.py@@ -14850,31 +14850,45 @@ def _patch_upload_download(monkeypatch: pytest.MonkeyPatch) -> None:@pytest.mark.asyncio-async def test_file_upload_no_upload_activity_returns_actionable_error_not_false_ok(- monkeypatch: pytest.MonkeyPatch,[email protected]("input_state", ["holds_file", "unreadable", "cleared_during_settle"])+async def test_file_upload_with_no_upload_activity_is_ok_only_when_the_input_confirms_the_file(+ monkeypatch: pytest.MonkeyPatch, input_state: str) -> None:- # The core fix: set_input_files can populate the control at the Playwright layer yet the site- # register nothing (post-navigation the change handler is not wired) — zero upload requests- # dispatched. file_upload must return a recoverable non-OK there, not a confident OK that makes the- # agent submit with no file. A submit-time-upload form lands here too as an accepted false-negative.- # RED against pre-fix code, which returned "uploaded 1 file" regardless of activity.+ # A form that sends the file with the submit dispatches nothing at attach time. The input holding+ # the file when the tool returns is its postcondition, so that is an ok; an unreadable input, or one+ # a change handler emptied while the page settled, confirms nothing and stays an error.import skyvern.forge.taskv3.tools as tools_modulepage = _FakePage()- page.element.emit_upload_on_set = False # file lands in the input, but the site never reacts+ page.element.emit_upload_on_set = False+ if input_state == "unreadable":++ async def _unreadable(_js: str, _arg: Any = None) -> Any:+ raise RuntimeError("Execution context was destroyed")++ monkeypatch.setattr(page.element, "evaluate", _unreadable)_patch_upload_dwell(monkeypatch, tools_module)+ if input_state == "cleared_during_settle":++ async def _settle_clears(_page: Any) -> None:+ page.element._files = []++ monkeypatch.setattr(tools_module, "_settle_after_upload", _settle_clears)_patch_upload_download(monkeypatch)tools = build_browser_tools(_fixed_page_provider(page))r = await _tool(tools, "file_upload").handler({"selector": "#cv", "file": "resume.pdf"})- assert r.status == "error", r.content- assert "no upload activity" in r.content- assert "uploaded 1 file" not in r.content- # The file was still attached at the Playwright layer, so the staged-download key must persist so- # the download-signal wrapper still suppresses the staged file.+ if input_state == "holds_file":+ assert (r.status, r.ok_class) == ("ok", "attached_no_activity"), r.content+ assert "sent no upload request" in r.content+ elif input_state == "unreadable":+ assert (r.status, r.error_class) == ("error", "attach_unconfirmed"), r.content+ else:+ assert r.status == "error", r.content+ assert "did not attach" in r.content+ # The staged-download key must persist so the download-signal wrapper still suppresses the file.assert (r.data or {}).get("staged_download") == "cv.pdf"- # The request listener must be removed after the call — no leaked/accumulating listeners.assert page._request_listeners == []@@ -15218,11 +15232,95 @@ async def test_file_upload_ignores_non_upload_network_noise(monkeypatch: pytest.tools = build_browser_tools(_fixed_page_provider(page))r = await _tool(tools, "file_upload").handler({"selector": "#cv", "file": "resume.pdf"})- assert r.status == "error", r.content- assert "no upload activity" in r.content+ assert (r.status, r.ok_class) == ("ok", "attached_no_activity"), r.contentassert page._request_listeners == []+_UPLOAD_TARGETS_HTML = """<!doctype html><html><body>+<form id="apply" onsubmit="event.preventDefault(); window.__submitted = true;">+ <div id="dropzone" style="border:1px dashed;padding:20px">Drop your resume+ <input id="inner" type="file" style="display:none">+ </div>+ <input id="picker-input" type="file" style="display:none">+ <input id="labelled-input" type="file" style="display:none">+ <label id="choose" for="labelled-input">Choose file</label>+ <button id="attach" type="button" onclick="document.getElementById('picker-input').click()">Attach</button>+ <div id="inert" style="padding:10px">Nothing to click</div>+ <button id="send">Send application</button>+</form>+<input id="formless-input" type="file" style="display:none">+<button id="formless-attach" onclick="document.getElementById('formless-input').click()">Select resume</button>+</body></html>"""+++@_skip_no_browser+ ("selector", "receiving_input"),+ [+ ("#dropzone", "#inner"),+ ("#attach", "#picker-input"),+ ("#choose", "#labelled-input"),+ # A typeless button outside any form submits nothing, so it is clicked.+ ("#formless-attach", "#formless-input"),+ ],+)+async def test_file_upload_on_an_upload_control_that_is_not_the_input_lands_the_file(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, selector: str, receiving_input: str+) -> None:+ # The model targets the visible control, since observe does not list a hidden file input. The+ # driver refuses a non-input ("Node is not an HTMLInputElement"); v1 sets the file on the input+ # inside the control, or clicks it and fills the picker it opens.+ import skyvern.forge.sdk.api.files as files_module+ import skyvern.forge.taskv3.tools as tools_module++ cv = tmp_path / "cv.pdf"+ cv.write_bytes(b"%PDF-1.4 synthetic")++ async def _fake_download(source: str, output_dir: str | None = None, organization_id: str | None = None) -> str:+ return str(cv)++ monkeypatch.setattr(files_module, "download_file", _fake_download)+ _patch_upload_dwell(monkeypatch, tools_module)+ async with _content_page(_UPLOAD_TARGETS_HTML) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "file_upload").handler({"selector": selector, "file": "resume.pdf"})++ assert (r.status, r.ok_class) == ("ok", "attached_no_activity"), r.content+ assert await page.eval_on_selector(receiving_input, "e => e.files.length") == 1+ assert not await page.evaluate("() => !!window.__submitted")+++@_skip_no_browser+ ("selector", "error_class"),+ [("#inert", "no_file_input"), ("#send", "submits_form")],+)+async def test_file_upload_on_a_control_with_no_file_input_reports_it_without_submitting(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, selector: str, error_class: str+) -> None:+ import skyvern.forge.sdk.api.files as files_module+ import skyvern.forge.taskv3.tools as tools_module++ cv = tmp_path / "cv.pdf"+ cv.write_bytes(b"%PDF-1.4 synthetic")++ async def _fake_download(source: str, output_dir: str | None = None, organization_id: str | None = None) -> str:+ return str(cv)++ monkeypatch.setattr(files_module, "download_file", _fake_download)+ monkeypatch.setattr(tools_module, "_FILE_CHOOSER_TIMEOUT_MS", 300)+ _patch_upload_dwell(monkeypatch, tools_module)+ async with _content_page(_UPLOAD_TARGETS_HTML) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "file_upload").handler({"selector": selector, "file": "resume.pdf"})++ assert not await page.evaluate("() => !!window.__submitted"), r.content+ assert r.status == "error", r.content+ assert r.error_class == error_class, r.content++@pytest.mark.asyncioasync def test_non_upload_actions_do_not_settle_or_delay(monkeypatch: pytest.MonkeyPatch) -> None:# The dwell is upload-specific: click/type must NOT incur the settle or the delay, so the tool