Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fix the Task V3 answer audit's quote check, dropdown and placeholder plumbing, and bound its holds per task (#9043)
pedrohsdbc5bd690ffa1f
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonindex e0cfb4bb0..315c7ecc5 100644--- a/docs/api-reference/openapi.json+++ b/docs/api-reference/openapi.json@@ -7814,6 +7814,18 @@],"title": "File Url"},+ "download_url": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Download Url",+ "description": "Origin URL recorded by a download action."+ },"download": {"anyOf": [{diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.pyindex 68ebf8bf8..2e600ccc1 100644--- a/skyvern/forge/sdk/workflow/models/workflow.py+++ b/skyvern/forge/sdk/workflow/models/workflow.py@@ -599,6 +599,9 @@ class WorkflowRunResponseBase(BaseModel):cached_credits_used: int = 0task_v2: TaskV2 | None = Noneworkflow_title: str | None = None+ copilot_session_id: str | None = Field(+ default=None, description="ID of the Copilot chat associated with this workflow run."+ )browser_session_id: str | None = Nonebrowser_profile_id: str | None = Nonebrowser_seed_source: BrowserSeedSource | None = Nonediff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 5484aa537..65af48c3d 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -755,6 +755,7 @@ ACTION_TEXT_FIELDS = ("reasoning","intention","file_url",+ "download_url","file_name","description","data_extraction_goal",@@ -15619,6 +15620,7 @@ class WorkflowService:credits_used=workflow_run.credits_used,cached_credits_used=workflow_run.cached_credits_used,workflow_title=workflow.title,+ copilot_session_id=workflow_run.copilot_session_id,browser_session_id=workflow_run.browser_session_id,browser_profile_id=workflow_run.browser_profile_id,browser_seed_source=workflow_run.browser_seed_source,diff --git a/skyvern/forge/taskv3/answer_basis_audit.py b/skyvern/forge/taskv3/answer_basis_audit.pyindex 761cd62fd..f76381ec7 100644--- a/skyvern/forge/taskv3/answer_basis_audit.py+++ b/skyvern/forge/taskv3/answer_basis_audit.py@@ -2,7 +2,8 @@The audit decides which recorded answers have no basis in the run's data; this module only records what wasanswered and refuses every moving action while a flagged answer is unchanged, so the model switches it to a decline-option, clears it or stops. With no audit supplied (the default), nothing is wrapped.+option, clears it or stops. The tenth refused call ends the task. With no audit supplied (the default), nothing is+wrapped."""from __future__ import annotations@@ -16,11 +17,15 @@ from typing import Any, Protocolimport structlogfrom skyvern.forge.taskv3.loop import (+ ANSWER_BASIS_HOLD_BOUND_GUARD,CODE_TOOL_NAME,+ GUARD_END_DATA_KEY,OPENED_LIST_ERROR_CLASSES,PICKED_OPTION_DATA_KEY,TARGET_KIND_DATA_KEY,TARGET_LABEL_DATA_KEY,+ TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY,+ GuardEnd,ToolRefusal,ToolResult,ToolSpec,@@ -36,12 +41,16 @@ _MENU_NOTE_KEY = "menu_note"_FREE_TEXT_KINDS = frozenset({"textbox", "textarea", "combobox"})# Longer typed text is a cover letter or a description, never a legal-standing answer._MAX_AUDITED_WORDS = 25+# The refused call that ends the task (holds count per call, per engine run of one task).+ANSWER_BASIS_MAX_HOLDS = 10@dataclass(frozen=True)class RecordedAnswer:question: strvalue: str+ # The question is the field's placeholder, the only text it shows; the audit decides whether it asks anything.+ label_is_placeholder: bool = Falseclass AnswerBasisAudit(Protocol):@@ -80,6 +89,11 @@ def apply_answer_basis_audit(tools: list[ToolSpec], audit: AnswerBasisAudit | Noflagged: dict[Field, str] = {}refused: set[tuple[Field, str]] = set()open_menu: dict[str, Field | None] = {"field": None}+ placeholders: set[Field] = set()+ holds = 0++ def answer(field: Field, value: str) -> RecordedAnswer:+ return RecordedAnswer(field[0], value, field in placeholders)def record(field: Field, value: str) -> None:previous = pending.pop(field, None)@@ -88,7 +102,7 @@ def apply_answer_basis_audit(tools: list[ToolSpec], audit: AnswerBasisAudit | No# An emptied field holds no answer, so its flag goes with it.flagged.pop(field, None)if field[0] and value:- pending[field] = (value, asyncio.create_task(audit.unsupported([RecordedAnswer(field[0], value)])))+ pending[field] = (value, asyncio.create_task(audit.unsupported([answer(field, value)])))async def found(job: Awaitable[list[RecordedAnswer]]) -> bool:try:@@ -112,7 +126,7 @@ def apply_answer_basis_audit(tools: list[ToolSpec], audit: AnswerBasisAudit | Noif entering is not None and flagged.get(entering[0]) != entering[1]:if not appends:shown.pop(entering[0], None)- if entering in refused or await found(audit.unsupported([RecordedAnswer(entering[0][0], entering[1])])):+ if entering in refused or await found(audit.unsupported([answer(*entering)])):refused.add(entering)shown[entering[0]] = entering[1]untyped = entering[0]@@ -124,19 +138,28 @@ def apply_answer_basis_audit(tools: list[ToolSpec], audit: AnswerBasisAudit | Noselector, menu_row = await spec.address_probe(args) if spec.address_probe else (_selector(args), False)if menu_row or any(selector == field[1] for field in flagged):return True- LOG.info("taskv3 answer basis hold", tool=spec.name, flagged=len(shown))+ nonlocal holds+ holds += 1+ LOG.info("taskv3 answer basis hold", tool=spec.name, flagged=len(shown), holds=holds)named = "; ".join(f"'{q}' with '{v}' (field {s}; not typed, nothing was entered or pressed)"if (q, s) == untypedelse f"'{q}' answered '{v}' (field {s})"for (q, s), v in shown.items())+ questions = "; ".join(f"'{q}'" for q, _ in shown)+ end = GuardEnd(+ ANSWER_BASIS_HOLD_BOUND_GUARD,+ f"Stopped without submitting: the answer audit found nothing in the data that supports the answer to "+ f"{questions}.",+ )raise ToolRefusal(f"Not done: nothing in the data supports these answers: {named}. For each, choose the field's decline ""option, such as prefer not to say, if it offers one. Otherwise, if the question is required, stop, naming ""it; if it is optional, clear it and continue. Until each answered one is changed, only a click on that field ""or on a row of its open list goes through.",error_class="answer_without_basis",+ data={GUARD_END_DATA_KEY: end} if holds >= ANSWER_BASIS_MAX_HOLDS else None,)for spec in tools:@@ -154,9 +177,11 @@ def apply_answer_basis_audit(tools: list[ToolSpec], audit: AnswerBasisAudit | No# Its own Enter can submit the form, so the text is judged first, appended (clear=false) or not.enters = args.get("press_enter") and 0 < len(text.split()) <= _MAX_AUDITED_WORDSif _spec.answer_probe is not None and enters:- selector, label, kind = await _spec.answer_probe(args)+ selector, label, kind, from_placeholder = await _spec.answer_probe(args)if label.strip() and kind in _FREE_TEXT_KINDS:entering = ((label.strip(), selector), text)+ if from_placeholder:+ placeholders.add(entering[0])elif not selector:LOG.info("taskv3 answer basis probe missed", tool=_tool)repair = _may_move_on(_tool, args) and await hold(_spec, args, entering, not args.get("clear", True))@@ -180,13 +205,17 @@ def apply_answer_basis_audit(tools: list[ToolSpec], audit: AnswerBasisAudit | Noreturn result# The ref resolver rewrites args["selector"] in place, so this is the selector the call acted on.field = (_label(result), _selector(args))+ if data.get(TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY):+ placeholders.add(field)+ else:+ placeholders.discard(field)if entering is not None and field == entering[0]:record(field, "")return resultif _tool == "select_combobox":record(field, str(args.get("value") or ""))elif _tool == "select_option":- record(field, str(args.get("label") or args.get("value") or ""))+ record(field, str(data.get(PICKED_OPTION_DATA_KEY) or args.get("label") or args.get("value") or ""))elif _tool == "type" and str(data.get(TARGET_KIND_DATA_KEY) or "") in _FREE_TEXT_KINDS:text = str(args.get("text") or "")if len(text.split()) <= _MAX_AUDITED_WORDS:diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex 46944fdc4..59bb0e855 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -338,11 +338,22 @@ class ToolRefusal(Exception):# tokens. The label composition itself -- the floor vocabulary, the shape filter, the secret matcher --# lives in target_label.py; this module only carries the two raw values from probe to `RoundAction`.TARGET_LABEL_DATA_KEY = "target_label"-# The option text a click on a menu row committed (page text: never on the call record).+# The option text a click on a menu row, or a native select, committed (page text: never on the call record).PICKED_OPTION_DATA_KEY = "picked_option"# A list the control opened but could not settle a row in: the list stays open for a pick.OPENED_LIST_ERROR_CLASSES = frozenset({"list_no_match", "list_unread", "list_ambiguous", "identical_rows"})TARGET_KIND_DATA_KEY = "target_kind"+# Set when the target label is the field's placeholder, a hint rather than its question.+TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY = "target_label_is_placeholder"+# A refusal whose data carries a GuardEnd under this key ends the run with that guard's verdict.+GUARD_END_DATA_KEY = "guard_end"+++class GuardEnd(NamedTuple):+ guard: str+ reason: str++# The durable selector of the element a `drag` dropped onto, for the pending-submit watch.DROP_TARGET_DATA_KEY = "drop_target"# The tool-result `data` key a recorded action's outcome rides on: the machine facts about what the@@ -780,8 +791,9 @@ class ToolSpec:# The selector a click's arguments resolve to as the call's record keeps it after dispatch (a ref's durable# selector, a mark's act tag), and whether it names a row of an open list; set on click alone.address_probe: Callable[[dict[str, Any]], Awaitable[tuple[str, bool]]] | None = None- # The selector, name and kind of the field a type call is about to enter, read before it acts; set on type alone.- answer_probe: Callable[[dict[str, Any]], Awaitable[tuple[str, str, str]]] | None = None+ # The selector, name and kind of the field a type call is about to enter, and whether the name is its placeholder,+ # read before it acts; set on type alone.+ answer_probe: Callable[[dict[str, Any]], Awaitable[tuple[str, str, str, bool]]] | None = None@propertydef touches_page(self) -> bool:@@ -1085,6 +1097,8 @@ NAV_DEAD_END_GUARD = "navigation_dead_end"# A page-level handler kept asking for the page to be reloaded past the per-run cap: the page cannot# be stabilized, and acting on it would mean acting on a page declared stale.PAGE_REFRESH_EXHAUSTED_GUARD = "page_refresh_exhausted"+# The answer-basis audit refused moving on ANSWER_BASIS_MAX_HOLDS times in one run.+ANSWER_BASIS_HOLD_BOUND_GUARD = "answer_basis_hold_bound"# The browser or its page is gone: the page accessor behind every tool already tried to reopen a lost# tab and could not, so every later call raises the same error and the model spends turns on a browser# that cannot come back (SKY-18903). The only guard whose verdict is `failed` rather than `terminated`:@@ -5906,6 +5920,13 @@ async def run_agent_tool_loop(# A successful navigate moved the run off any dead page seen earlier this batch.st.pending_nav_dead_end = None+ guard_end = result_data.get(GUARD_END_DATA_KEY) if result.refused else None+ if isinstance(guard_end, GuardEnd):+ LOG.info("taskv3 loop guard ended the run", tool=tool_name, turn=st.turns, guard=guard_end.guard)+ st.outcome = _guard_verdict(guard_end.guard, guard_end.reason)+ _append_skipped_tool_results(st.messages, tool_calls[idx + 1 :], guard_end.reason)+ break+if spec is not None and spec.terminal and result.status == "ok":data = result.data or {}st.outcome = LoopOutcome(diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex 5a7afbb46..bd25b2412 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -93,6 +93,7 @@ from skyvern.forge.taskv3.loop import (REF_SELECTOR_RE,TARGET_KIND_DATA_KEY,TARGET_LABEL_DATA_KEY,+ TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY,CoveredBranch,CoveredLayerKind,DownloadFailureFacts,@@ -5457,8 +5458,11 @@ _TARGET_NAME_JS = (// from one.const textEntry = editable || tag === 'TEXTAREA' || _matches(el, '[role=textbox],[role=searchbox]')|| (tag === 'INPUT' && ['button', 'submit', 'reset', 'image', 'checkbox', 'radio', 'file', 'range'].indexOf(type) === -1);- const name = nrm(_attr(el, 'aria-label')) || labelled() || labelledBy() || contents || caption- || nrm(_attr(el, 'alt')) || nrm(_attr(el, 'placeholder')) || (textEntry ? '' : nrm(_attr(el, 'title')));+ const named = nrm(_attr(el, 'aria-label')) || labelled() || labelledBy() || contents || caption+ || nrm(_attr(el, 'alt'));+ // A placeholder is a hint, often a sample answer, not the question: the name says when it came from one.+ const hint = named ? '' : nrm(_attr(el, 'placeholder'));+ const name = named || hint || (textEntry ? '' : nrm(_attr(el, 'title')));// Refused here rather than truncated, and this is the only place that can be exact: `length` is// UTF-16 code units while Python counts code points, so a name whose bound falls on an astral// character (an emoji is one code point, two units) would arrive UNDER the Python cap and be@@ -5498,7 +5502,7 @@ _TARGET_NAME_JS = (kind = 'textbox';}}- return { name: finalName, kind: kind };+ return { name: finalName, kind: kind, placeholder: Boolean(hint) && finalName !== null };}""")@@ -15318,6 +15322,7 @@ def build_browser_tools(selector = args.get("selector")name: str | None = Nonekind: str | None = None+ from_placeholder = False# The page comes from the preflight handoff, read and not popped -- the handler's own# _resolve_page consumes it. Never page_provider(): a second must_get_working_page can# recover or relaunch a page, and a label is not worth a side effect. Every labelled tool@@ -15355,6 +15360,7 @@ def build_browser_tools(name = raw_name if isinstance(raw_name, str) and raw_name.strip() else Noneraw_kind = raw.get("kind")kind = raw_kind if isinstance(raw_kind, str) and raw_kind else None+ from_placeholder = raw.get("placeholder") is Trueresult = await handler(args)# Only a call that acted carries a name/kind. The probe resolves the selector on its own,# so on a refusal the two can disagree about which element the selector meant -- a cloned@@ -15367,6 +15373,8 @@ def build_browser_tools(data = {**(result.data or {})}if name is not None:data[TARGET_LABEL_DATA_KEY] = name+ if from_placeholder:+ data[TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY] = Trueif kind is not None:data[TARGET_KIND_DATA_KEY] = kind# `replace`, not a positional rebuild: this carries every field the result already has,@@ -21668,12 +21676,17 @@ def build_browser_tools(f"selected on {selector} — {reason}, so the selection could not be verified; re-observe ""before relying on it")+ # The visible text of what is now selected: a call by value names only an internal code.+ shown = readback.get("selectedLabels" if is_multi else "selectedLabel") if isinstance(readback, dict) else None+ shown = ", ".join(x for x in shown if isinstance(x, str)) if isinstance(shown, list) else shown+ data = {PICKED_OPTION_DATA_KEY: shown} if isinstance(shown, str) and shown.strip() else Noneif is_multi:return ToolResult.ok(- f"selected on {selector} — it now holds {len(held)} option(s): {_selection_report(held)}"+ f"selected on {selector} — it now holds {len(held)} option(s): {_selection_report(held)}", data=data)return ToolResult.ok(- f"selected on {selector} (hidden native select, set directly)" if force else f"selected on {selector}"+ f"selected on {selector} (hidden native select, set directly)" if force else f"selected on {selector}",+ data=data,)async def press_key(args: dict[str, Any]) -> ToolResult:@@ -22965,13 +22978,14 @@ def build_browser_tools(_with_ref_resolution("answer_probe", _with_selector_guard(_answer_target)))- async def _answer_field(args: dict[str, Any]) -> tuple[str, str, str]:+ async def _answer_field(args: dict[str, Any]) -> tuple[str, str, str, bool]:resolved = dict(args)try:data = (await _resolve_answer_target(resolved)).data or {}except Exception:- return "", "", ""- return str(resolved.get("selector") or ""), str(data.get("name") or ""), str(data.get("kind") or "")+ return "", "", "", False+ name, kind = str(data.get("name") or ""), str(data.get("kind") or "")+ return str(resolved.get("selector") or ""), name, kind, data.get("placeholder") is Truetools = [_spec(diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.pyindex 4f7ea048c..1c28b9201 100644--- a/skyvern/webeye/actions/actions.py+++ b/skyvern/webeye/actions/actions.py@@ -175,6 +175,7 @@ class Action(BaseModel):# WebAction fieldsfile_name: str | None = Nonefile_url: str | None = None+ download_url: str | None = Field(default=None, description="Origin URL recorded by a download action.")download: bool | None = Nonedownload_triggered: bool | None = None # Whether a download was triggered by this actiondownloaded_files: list[str] | None = None # List of file names downloaded by this action@@ -402,7 +403,6 @@ class DownloadFileAction(Action):download: bool = Truefile_name: strbyte: Annotated[bytes | None, Field(exclude=True)] = None # bytes data- download_url: str | None = None # URL to download file fromdef __repr__(self) -> str:return f"DownloadFileAction(file_name={self.file_name}, download_url={self.download_url}, has_byte={self.byte is not None})"diff --git a/tests/unit/test_output_parameter_size_guards.py b/tests/unit/test_output_parameter_size_guards.pyindex 593ac9213..50f8b8390 100644--- a/tests/unit/test_output_parameter_size_guards.py+++ b/tests/unit/test_output_parameter_size_guards.py@@ -1282,6 +1282,8 @@ async def test_get_workflow_run_timeline_caps_action_text(monkeypatch: pytest.Motask_id="tsk_1",response="r" * (cap * 3),text="t" * (cap * 3),+ file_url="https://files.invalid/" + "f" * (cap * 3),+ download_url="https://files.invalid/" + "d" * (cap * 3),)monkeypatch.setattr(workflow_service_module.app,@@ -1298,7 +1300,7 @@ async def test_get_workflow_run_timeline_caps_action_text(monkeypatch: pytest.Mo)capped_action = timeline[0].block.actions[0]- for field in ("response", "text"):+ for field in ("response", "text", "file_url", "download_url"):value = getattr(capped_action, field)assert isinstance(value, str), fieldassert "[truncated" in value, fielddiff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.pyindex e34cf23ea..8721cbfad 100644--- a/tests/unit/test_script_skyvern_page.py+++ b/tests/unit/test_script_skyvern_page.py@@ -1889,6 +1889,55 @@ class TestActionSubclassPersistence:kwargs=kwargs,)+ @pytest.mark.asyncio+ async def test_native_download_records_the_url_fetch_even_with_ignored_byte_kwargs(+ self, mock_scraped_page, mock_ai+ ):+ script_page = self._build_script_page(mock_scraped_page, mock_ai)+ script_page._wait_for_page_ready_before_action = AsyncMock()+ script_page._create_screenshot_after_execution = AsyncMock()+ script_page._create_html_action_after_execution = AsyncMock()+ context = self._build_context()+ captured = []++ async def save_action(action):+ captured.append(action)+ return action++ with (+ patch("skyvern.core.script_generations.script_skyvern_page.skyvern_context.current", return_value=context),+ patch(+ "skyvern.core.script_generations.skyvern_page.download_file_from_url",+ new=AsyncMock(return_value="/synthetic/report.pdf"),+ ) as fetch,+ patch(+ "skyvern.core.script_generations.script_skyvern_page.app.DATABASE.workflow_params.create_action",+ new=AsyncMock(side_effect=save_action),+ ),+ patch(+ "skyvern.core.script_generations.script_skyvern_page.app.DATABASE.workflow_params.update_action_reasoning",+ new=AsyncMock(),+ ),+ patch(+ "skyvern.core.script_generations.script_skyvern_page.script_run_context_manager.get_run_context",+ return_value=None,+ ),+ patch.object(settings, "CACHED_ACTION_DELAY_SECONDS", 0),+ ):+ path = await script_page.download_file(+ file_name="report.pdf", download_url="https://example.com/report.pdf", byte=b"ignored bytes"+ )++ assert path == "/synthetic/report.pdf"+ fetch.assert_awaited_once_with(+ "https://example.com/report.pdf", filename="report.pdf", organization_id="o_test"+ )+ action = captured[0]+ assert action.status == "completed" and action.created_by == "script"+ assert action.download is True+ assert action.file_name == "report.pdf" and action.download_url == "https://example.com/report.pdf"+ assert action.download_triggered is None and action.downloaded_files is None+@pytest.mark.asyncioasync def test_move_persists_x_and_y(self, mock_scraped_page, mock_ai):from skyvern.webeye.actions.action_types import ActionTypediff --git a/tests/unit/test_taskv3_answer_basis_audit.py b/tests/unit/test_taskv3_answer_basis_audit.pyindex 061415156..2a760feab 100644--- a/tests/unit/test_taskv3_answer_basis_audit.py+++ b/tests/unit/test_taskv3_answer_basis_audit.py@@ -9,6 +9,7 @@ from skyvern.forge.taskv3.loop import (PICKED_OPTION_DATA_KEY,TARGET_KIND_DATA_KEY,TARGET_LABEL_DATA_KEY,+ TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY,ToolRefusal,ToolResult,ToolSpec,@@ -39,11 +40,12 @@ async def _address(args: dict[str, Any]) -> tuple[str, bool]:return _REFS.get(selector, (selector, False))-async def _answer(args: dict[str, Any]) -> tuple[str, str, str]:+async def _answer(args: dict[str, Any]) -> tuple[str, str, str, bool]:# What the field reads as before the call acts; `_probed` stands in for a probe that resolves elsewhere or fails.if args.get("_probed") == "":- return "", "", ""- return str(args.get("_probed", args.get("selector"))), str(args.get("_label", "")), args.get("_kind", "textbox")+ return "", "", "", False+ selector, label = str(args.get("_probed", args.get("selector"))), str(args.get("_label", ""))+ return selector, label, args.get("_kind", "textbox"), bool(args.get("_placeholder"))def _tools(dispatched: list[tuple[str, dict[str, Any]]]) -> list[ToolSpec]:@@ -63,7 +65,12 @@ def _tools(dispatched: list[tuple[str, dict[str, Any]]]) -> list[ToolSpec]:if name == "select_combobox" and args.get("_no_match"):content = f"opened {sel} but no option matched {args['value']!r}"return ToolResult("error", content, {TARGET_LABEL_DATA_KEY: label}, error_class="list_no_match")- return ToolResult("ok", f"{name} done", {TARGET_LABEL_DATA_KEY: label, TARGET_KIND_DATA_KEY: "textbox"})+ data = {TARGET_LABEL_DATA_KEY: label, TARGET_KIND_DATA_KEY: "textbox"}+ if "_shown" in args:+ data[PICKED_OPTION_DATA_KEY] = args["_shown"]+ if args.get("_placeholder"):+ data[TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY] = True+ return ToolResult("ok", f"{name} done", data)probe = _answer if name == "type" else Nonereturn ToolSpec(@@ -309,3 +316,23 @@ async def test_a_typed_enter_the_probe_cannot_place_is_still_recorded_after_the_await _by_name(tools, "type").handler({**TYPED, "_probed": "#elsewhere", "text": "Yes"})await _by_name(tools, "click").handler({"selector": "#submit"})assert [name for name, _ in dispatched] == ["type", "click"] and len(audit.seen) == 2+++async def test_the_audit_records_what_the_page_shows_and_where_its_question_came_from() -> None:+ audit = _Audit()+ tools = _tools([])+ apply_answer_basis_audit(tools, audit)+ select, type_ = _by_name(tools, "select_option").handler, _by_name(tools, "type").handler++ await select({"selector": "#auth", "value": "10", "_label": DEBARMENT, "_shown": "No"})+ # A field named only by its placeholder is still audited; the audit is told the name is a placeholder.+ await type_({"selector": "#country", "text": "Canada", "_label": "Country", "_placeholder": True})+ await type_({"selector": "#visa", "text": "No", "_label": "Visa", "_placeholder": True, "press_enter": True})+ await _by_name(tools, "click").handler({"selector": "#submit"})++ assert audit.seen == [+ [RecordedAnswer(DEBARMENT, "No")],+ [RecordedAnswer("Country", "Canada", label_is_placeholder=True)],+ [RecordedAnswer("Visa", "No", label_is_placeholder=True)],+ ]diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex da2835ed6..26fe747c9 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -41,6 +41,7 @@ from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.taskv3 import loop as loop_modulefrom skyvern.forge.taskv3 import tools as taskv3_tools_module+from skyvern.forge.taskv3.answer_basis_audit import ANSWER_BASIS_MAX_HOLDS, RecordedAnswer, apply_answer_basis_auditfrom skyvern.forge.taskv3.auth_tools import _MAX_GIVEUP_DEFERRALS, VerificationFailure, VerificationStatefrom skyvern.forge.taskv3.engine import MAX_TOKENS_CEILING, MAX_TOKENS_PER_ACTION_STEP, taskv3_runaway_backstopsfrom skyvern.forge.taskv3.goal_check import GoalVerdict, ToolTrail, run_goal_check@@ -55,6 +56,7 @@ from skyvern.forge.taskv3.loop import (ACTION_LOOP_NUDGE_AFTER,ACTION_LOOP_TERMINATE_AFTER,ACTION_OUTCOME_DATA_KEY,+ ANSWER_BASIS_HOLD_BOUND_GUARD,BROWSER_LOST_EVENT,BROWSER_LOST_GUARD,BROWSER_LOST_REASON,@@ -89,6 +91,8 @@ from skyvern.forge.taskv3.loop import (SUBMISSION_HELD_WAIT_MAX_SECONDS,SUBMISSION_RENDER_MARGIN_SECONDS,TARGET_ACTION_ROUND_EVENT,+ TARGET_KIND_DATA_KEY,+ TARGET_LABEL_DATA_KEY,TOKEN_BUDGET_EXTENDED_EVENT,TOKEN_BUDGET_EXTENSION_REFUSED_EVENT,UNCHARGED_REFUSAL_GRACE,@@ -12455,6 +12459,41 @@ async def test_a_lost_browser_ends_the_run_where_any_other_raise_comes_back_as_aassert (lost["guard"], lost["tool"]) == (BROWSER_LOST_GUARD, "click")+async def test_the_answer_audit_ends_a_run_it_keeps_holding_with_a_verdict_naming_the_question() -> None:+ # The bound's hold ends the run terminated; the holds before it stay ordinary refusals the model reads.+ clicks: list[tuple[str, dict[str, Any]]] = []++ class _FlagsEverything:+ async def unsupported(self, answers: list[RecordedAnswer]) -> list[RecordedAnswer]:+ return answers++ async def type_answer(args: dict[str, Any]) -> ToolResult:+ return ToolResult.ok(+ "typed", data={TARGET_LABEL_DATA_KEY: "Work authorization", TARGET_KIND_DATA_KEY: "textbox"}+ )++ tools = [+ ToolSpec(name="type", description="t", parameters={}, handler=type_answer, billable=True),+ _recording_tool("click", clicks, billable=True),+ make_finish_tool(),+ ]+ apply_answer_basis_audit(tools, _FlagsEverything())+ # A different control each turn, so the repeated-action guard never fires first.+ submits = [[("click", {"selector": f"#next{i}"})] for i in range(ANSWER_BASIS_MAX_HOLDS + 1)]+ script = [+ [("type", {"selector": "#auth", "text": "Yes"})],+ *submits,+ [("finish", {"status": "completed", "reason": "x"})],+ ]+ outcome, caller = await _run(script, tools, max_turns=30)++ assert (outcome.status, outcome.guard) == ("terminated", ANSWER_BASIS_HOLD_BOUND_GUARD)+ assert "'Work authorization'" in outcome.reason and "Yes" not in outcome.reason+ assert caller.calls == 1 + ANSWER_BASIS_MAX_HOLDS+ assert clicks == []++def test_every_driver_timeout_type_installed_here_classifies_as_a_driver_timeout() -> None:# Written against the driver-type TABLE rather than a hard-coded package, because the hazard is# precisely that CI and production do not install the same driver: the browser image repointsdiff --git a/tests/unit/test_workflow_run_detail_template_cross_org.py b/tests/unit/test_workflow_run_detail_template_cross_org.pyindex f9d152b2e..d890270d6 100644--- a/tests/unit/test_workflow_run_detail_template_cross_org.py+++ b/tests/unit/test_workflow_run_detail_template_cross_org.py@@ -1,15 +1,21 @@-from datetime import datetime, timezone+from datetime import UTC, datetimefrom types import SimpleNamespacefrom unittest.mock import AsyncMock+import httpximport pytest+from fastapi import FastAPIfrom skyvern.forge import app+from skyvern.forge.sdk.routes import agent_protocol+from skyvern.forge.sdk.services import org_auth_servicefrom skyvern.forge.sdk.workflow.models.workflow import (Workflow,WorkflowDefinition,+ WorkflowRun,WorkflowRunResponseBase,WorkflowRunStatus,+ WorkflowRunWithWorkflowResponse,WorkflowStatus,)from skyvern.forge.sdk.workflow.service import WorkflowService@@ -20,7 +26,7 @@ CALLER_ORG_ID = "o_caller"def _make_template_workflow() -> Workflow:- now = datetime.now(timezone.utc)+ now = datetime.now(UTC)return Workflow(workflow_id="w_tmpl",organization_id=TEMPLATE_ORG_ID,@@ -36,37 +42,22 @@ def _make_template_workflow() -> Workflow:)-def _make_workflow_run(workflow_run_id: str = "wr_tmpl", organization_id: str = CALLER_ORG_ID) -> SimpleNamespace:- now = datetime.now(timezone.utc)- return SimpleNamespace(+def _make_workflow_run(+ workflow_run_id: str = "wr_tmpl",+ organization_id: str = CALLER_ORG_ID,+ copilot_session_id: str | None = None,+) -> WorkflowRun:+ now = datetime.now(UTC)+ return WorkflowRun(workflow_run_id=workflow_run_id,workflow_id="w_tmpl",+ workflow_permanent_id="wpid_tmpl",organization_id=organization_id,+ copilot_session_id=copilot_session_id,status=WorkflowRunStatus.running,- failure_reason=None,- failure_category=None,- retried_from_workflow_run_id=None,- proxy_location=None,- webhook_callback_url=None,- webhook_failure_reason=None,- totp_verification_url=None,- totp_identifier=None,- extra_http_headers=None,- cdp_connect_headers=None,- queued_at=None,started_at=now,- finished_at=None,created_at=now,modified_at=now,- credits_used=0,- cached_credits_used=0,- browser_session_id=None,- browser_profile_id=None,- browser_seed_source=None,- max_screenshot_scrolls=None,- browser_address=None,- run_with=None,- script_run=None,)@@ -86,7 +77,13 @@ def template_run_db(monkeypatch: pytest.MonkeyPatch) -> tuple[AsyncMock, AsyncMoget_workflow_by_permanent_id=get_by_wpid,),observer=SimpleNamespace(get_task_v2_by_workflow_run_id=AsyncMock(return_value=None)),- tasks=SimpleNamespace(get_tasks_by_workflow_run_id=AsyncMock(return_value=[])),+ tasks=SimpleNamespace(+ get_tasks_by_workflow_run_id=AsyncMock(return_value=[]),+ get_step_counts_by_task_ids=AsyncMock(return_value=(0, 0)),+ ),+ browser_sessions=SimpleNamespace(+ get_persistent_browser_session_by_runnable_id=AsyncMock(return_value=None)+ ),workflow_runs=SimpleNamespace(get_workflow_run_parameters=AsyncMock(return_value=[]),get_workflow_run_block_errors=AsyncMock(return_value=[]),@@ -99,11 +96,14 @@ def template_run_db(monkeypatch: pytest.MonkeyPatch) -> tuple[AsyncMock, AsyncMo@pytest.mark.asyncio[email protected]("copilot_session_id", [None, "wcc_123456"])async def test_template_run_detail_resolves_via_run_join(- monkeypatch: pytest.MonkeyPatch, template_run_db: tuple[AsyncMock, AsyncMock]+ monkeypatch: pytest.MonkeyPatch,+ template_run_db: tuple[AsyncMock, AsyncMock],+ copilot_session_id: str | None,) -> None:get_by_run, get_by_wpid = template_run_db- workflow_run = _make_workflow_run()+ workflow_run = _make_workflow_run(copilot_session_id=copilot_session_id)# A real service instance: the stub app's WORKFLOW_SERVICE is a lazy auto-mock, so we must# exercise the actual build_workflow_run_status_response rather than app.WORKFLOW_SERVICE.@@ -125,6 +125,7 @@ async def test_template_run_detail_resolves_via_run_join(assert isinstance(response, WorkflowRunResponseBase)assert response.workflow_run_id == "wr_tmpl"assert response.workflow_title == "Job Application Workflow"+ assert response.copilot_session_id == copilot_session_id# Resolution must go through the run join (scoped by the run's org), not the caller-org# permanent-id lookup that returns None for cross-org template workflows.@@ -133,6 +134,26 @@ async def test_template_run_detail_resolves_via_run_join(assert get_by_run.await_args.kwargs.get("organization_id") == CALLER_ORG_IDget_by_wpid.assert_not_awaited()+ monkeypatch.setattr(service, "get_workflow_by_workflow_run_id", AsyncMock(return_value=get_by_run.return_value))+ monkeypatch.setattr(app, "WORKFLOW_SERVICE", service)+ monkeypatch.setattr(app.AGENT_FUNCTION, "calculate_workflow_run_total_cost", AsyncMock(return_value=0.0))+ canonical = next(+ route for route in agent_protocol.base_router.routes if route.path == "/workflows/runs/{workflow_run_id}"+ )+ assert canonical.endpoint is agent_protocol.get_workflow_and_run_from_workflow_run_id+ assert canonical.response_model is WorkflowRunWithWorkflowResponse+ api = FastAPI()+ api.dependency_overrides[org_auth_service.get_current_org] = lambda: SimpleNamespace(organization_id=CALLER_ORG_ID)+ api.include_router(agent_protocol.base_router, prefix="/v1")+ async with httpx.AsyncClient(+ transport=httpx.ASGITransport(app=api), base_url="http://test", trust_env=False+ ) as client:+ serialized = await client.get("/v1/workflows/runs/wr_tmpl")++ assert serialized.status_code == 200+ assert serialized.json()["copilot_session_id"] == copilot_session_id+ assert serialized.json()["workflow"]["organization_id"] == TEMPLATE_ORG_ID+@pytest.mark.asyncio@pytest.mark.usefixtures("template_run_db")diff --git a/tests/unit/workflow/test_workflow_run_timeline_tree.py b/tests/unit/workflow/test_workflow_run_timeline_tree.pyindex d176a1efb..65f8ce5b8 100644--- a/tests/unit/workflow/test_workflow_run_timeline_tree.py+++ b/tests/unit/workflow/test_workflow_run_timeline_tree.py@@ -4,19 +4,24 @@ from __future__ import annotationsimport jsonfrom datetime import UTC, datetime, timedelta, timezone+from types import SimpleNamespacefrom typing import Anyfrom unittest.mock import AsyncMock+import httpximport pytest+from fastapi import FastAPIfrom pydantic import TypeAdapterfrom skyvern.forge import appfrom skyvern.forge.sdk.db.repositories.tasks import TasksRepositoryfrom skyvern.forge.sdk.routes import agent_protocolfrom skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock, WorkflowRunTimeline, WorkflowRunTimelineType+from skyvern.forge.sdk.services import org_auth_servicefrom skyvern.forge.sdk.workflow.service import WorkflowService+from skyvern.schemas.run_enums import RunTypefrom skyvern.schemas.workflows import BlockType-from skyvern.webeye.actions.actions import Action, ExtractAction+from skyvern.webeye.actions.actions import Action, DownloadFileAction, ExtractActionfrom tests.unit.helpers import (ONE_TIME_CODE,assert_one_time_code_withheld,@@ -160,6 +165,46 @@ async def test_timeline_attaches_actions_to_code_blocks(mock_db: AsyncMock) -> Nassert timeline[0].block.actions == [action]+async def test_native_download_url_survives_registered_timeline_response(+ mock_db: AsyncMock, monkeypatch: pytest.MonkeyPatch+) -> None:+ stamp = datetime(2026, 5, 4, 12, 0, tzinfo=UTC)+ mock_db.return_value = [_block("wrb_download", created_at=stamp, block_type=BlockType.CODE, task_id="tsk_download")]+ app.DATABASE.tasks.get_tasks_actions.return_value = [+ DownloadFileAction(+ action_id="act_download",+ organization_id="o_test",+ workflow_run_id="wr_test",+ task_id="tsk_download",+ step_id="stp_download",+ status="completed",+ file_name="report.pdf",+ download_url="https://example.com/files/report.pdf?token=synthetic-private-query",+ byte=b"PRIVATE_DOWNLOAD_BYTES",+ created_by="script",+ )+ ]+ monkeypatch.setattr(+ app.DATABASE.tasks, "get_run", AsyncMock(return_value=SimpleNamespace(task_run_type=RunType.workflow_run))+ )+ monkeypatch.setattr(app.DATABASE.observer, "get_task_v2_by_workflow_run_id", AsyncMock(return_value=None))+ monkeypatch.setattr(app, "WORKFLOW_SERVICE", WorkflowService())+ monkeypatch.setattr(agent_protocol.analytics, "capture", lambda *_args, **_kwargs: None)+ api = FastAPI()+ api.include_router(agent_protocol.base_router, prefix="/v1")+ api.dependency_overrides[org_auth_service.get_current_org] = lambda: SimpleNamespace(organization_id="o_test")++ async with httpx.AsyncClient(transport=httpx.ASGITransport(app=api), base_url="http://test") as client:+ response = await client.get("/v1/runs/wr_test/timeline")++ assert response.status_code == 200+ action = response.json()[0]["block"]["actions"][0]+ assert action["download_url"] == "https://example.com/files/report.pdf?token=synthetic-private-query"+ assert action["file_name"] == "report.pdf" and action["created_by"] == "script"+ assert "byte" not in action and "PRIVATE_DOWNLOAD_BYTES" not in response.text++@pytest.mark.asyncio@pytest.mark.parametrize("cap_output_values", [True, False])async def test_timeline_serves_the_typed_value_and_withholds_the_one_time_code(