Skip to content

Automate browser based workflows with AI

Fix the Task V3 answer audit's quote check, dropdown and placeholder plumbing, and bound its holds per task (#9043)

pedrohsdbc5bd690ffa1f
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index e0cfb4bb0..315c7ecc5 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -7814,6 +7814,18 @@
],
"title": "File Url"
},
+ "download_url": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Download Url",
+ "description": "Origin URL recorded by a download action."
+ },
"download": {
"anyOf": [
{
diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.py
index 68ebf8bf8..2e600ccc1 100644
--- a/skyvern/forge/sdk/workflow/models/workflow.py
+++ b/skyvern/forge/sdk/workflow/models/workflow.py
@@ -599,6 +599,9 @@ class WorkflowRunResponseBase(BaseModel):
cached_credits_used: int = 0
task_v2: TaskV2 | None = None
workflow_title: str | None = None
+ copilot_session_id: str | None = Field(
+ default=None, description="ID of the Copilot chat associated with this workflow run."
+ )
browser_session_id: str | None = None
browser_profile_id: str | None = None
browser_seed_source: BrowserSeedSource | None = None
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 5484aa537..65af48c3d 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -755,6 +755,7 @@ ACTION_TEXT_FIELDS = (
"reasoning",
"intention",
"file_url",
+ "download_url",
"file_name",
"description",
"data_extraction_goal",
@@ -15619,6 +15620,7 @@ class WorkflowService:
credits_used=workflow_run.credits_used,
cached_credits_used=workflow_run.cached_credits_used,
workflow_title=workflow.title,
+ copilot_session_id=workflow_run.copilot_session_id,
browser_session_id=workflow_run.browser_session_id,
browser_profile_id=workflow_run.browser_profile_id,
browser_seed_source=workflow_run.browser_seed_source,
diff --git a/skyvern/forge/taskv3/answer_basis_audit.py b/skyvern/forge/taskv3/answer_basis_audit.py
index 761cd62fd..f76381ec7 100644
--- a/skyvern/forge/taskv3/answer_basis_audit.py
+++ b/skyvern/forge/taskv3/answer_basis_audit.py
@@ -2,7 +2,8 @@
The audit decides which recorded answers have no basis in the run's data; this module only records what was
answered and refuses every moving action while a flagged answer is unchanged, so the model switches it to a decline
-option, clears it or stops. With no audit supplied (the default), nothing is wrapped.
+option, clears it or stops. The tenth refused call ends the task. With no audit supplied (the default), nothing is
+wrapped.
"""
from __future__ import annotations
@@ -16,11 +17,15 @@ from typing import Any, Protocol
import structlog
from skyvern.forge.taskv3.loop import (
+ ANSWER_BASIS_HOLD_BOUND_GUARD,
CODE_TOOL_NAME,
+ GUARD_END_DATA_KEY,
OPENED_LIST_ERROR_CLASSES,
PICKED_OPTION_DATA_KEY,
TARGET_KIND_DATA_KEY,
TARGET_LABEL_DATA_KEY,
+ TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY,
+ GuardEnd,
ToolRefusal,
ToolResult,
ToolSpec,
@@ -36,12 +41,16 @@ _MENU_NOTE_KEY = "menu_note"
_FREE_TEXT_KINDS = frozenset({"textbox", "textarea", "combobox"})
# Longer typed text is a cover letter or a description, never a legal-standing answer.
_MAX_AUDITED_WORDS = 25
+# The refused call that ends the task (holds count per call, per engine run of one task).
+ANSWER_BASIS_MAX_HOLDS = 10
@dataclass(frozen=True)
class RecordedAnswer:
question: str
value: str
+ # The question is the field's placeholder, the only text it shows; the audit decides whether it asks anything.
+ label_is_placeholder: bool = False
class AnswerBasisAudit(Protocol):
@@ -80,6 +89,11 @@ def apply_answer_basis_audit(tools: list[ToolSpec], audit: AnswerBasisAudit | No
flagged: dict[Field, str] = {}
refused: set[tuple[Field, str]] = set()
open_menu: dict[str, Field | None] = {"field": None}
+ placeholders: set[Field] = set()
+ holds = 0
+
+ def answer(field: Field, value: str) -> RecordedAnswer:
+ return RecordedAnswer(field[0], value, field in placeholders)
def record(field: Field, value: str) -> None:
previous = pending.pop(field, None)
@@ -88,7 +102,7 @@ def apply_answer_basis_audit(tools: list[ToolSpec], audit: AnswerBasisAudit | No
# An emptied field holds no answer, so its flag goes with it.
flagged.pop(field, None)
if field[0] and value:
- pending[field] = (value, asyncio.create_task(audit.unsupported([RecordedAnswer(field[0], value)])))
+ pending[field] = (value, asyncio.create_task(audit.unsupported([answer(field, value)])))
async def found(job: Awaitable[list[RecordedAnswer]]) -> bool:
try:
@@ -112,7 +126,7 @@ def apply_answer_basis_audit(tools: list[ToolSpec], audit: AnswerBasisAudit | No
if entering is not None and flagged.get(entering[0]) != entering[1]:
if not appends:
shown.pop(entering[0], None)
- if entering in refused or await found(audit.unsupported([RecordedAnswer(entering[0][0], entering[1])])):
+ if entering in refused or await found(audit.unsupported([answer(*entering)])):
refused.add(entering)
shown[entering[0]] = entering[1]
untyped = entering[0]
@@ -124,19 +138,28 @@ def apply_answer_basis_audit(tools: list[ToolSpec], audit: AnswerBasisAudit | No
selector, menu_row = await spec.address_probe(args) if spec.address_probe else (_selector(args), False)
if menu_row or any(selector == field[1] for field in flagged):
return True
- LOG.info("taskv3 answer basis hold", tool=spec.name, flagged=len(shown))
+ nonlocal holds
+ holds += 1
+ LOG.info("taskv3 answer basis hold", tool=spec.name, flagged=len(shown), holds=holds)
named = "; ".join(
f"'{q}' with '{v}' (field {s}; not typed, nothing was entered or pressed)"
if (q, s) == untyped
else f"'{q}' answered '{v}' (field {s})"
for (q, s), v in shown.items()
)
+ questions = "; ".join(f"'{q}'" for q, _ in shown)
+ end = GuardEnd(
+ ANSWER_BASIS_HOLD_BOUND_GUARD,
+ f"Stopped without submitting: the answer audit found nothing in the data that supports the answer to "
+ f"{questions}.",
+ )
raise ToolRefusal(
f"Not done: nothing in the data supports these answers: {named}. For each, choose the field's decline "
"option, such as prefer not to say, if it offers one. Otherwise, if the question is required, stop, naming "
"it; if it is optional, clear it and continue. Until each answered one is changed, only a click on that field "
"or on a row of its open list goes through.",
error_class="answer_without_basis",
+ data={GUARD_END_DATA_KEY: end} if holds >= ANSWER_BASIS_MAX_HOLDS else None,
)
for spec in tools:
@@ -154,9 +177,11 @@ def apply_answer_basis_audit(tools: list[ToolSpec], audit: AnswerBasisAudit | No
# Its own Enter can submit the form, so the text is judged first, appended (clear=false) or not.
enters = args.get("press_enter") and 0 < len(text.split()) <= _MAX_AUDITED_WORDS
if _spec.answer_probe is not None and enters:
- selector, label, kind = await _spec.answer_probe(args)
+ selector, label, kind, from_placeholder = await _spec.answer_probe(args)
if label.strip() and kind in _FREE_TEXT_KINDS:
entering = ((label.strip(), selector), text)
+ if from_placeholder:
+ placeholders.add(entering[0])
elif not selector:
LOG.info("taskv3 answer basis probe missed", tool=_tool)
repair = _may_move_on(_tool, args) and await hold(_spec, args, entering, not args.get("clear", True))
@@ -180,13 +205,17 @@ def apply_answer_basis_audit(tools: list[ToolSpec], audit: AnswerBasisAudit | No
return result
# The ref resolver rewrites args["selector"] in place, so this is the selector the call acted on.
field = (_label(result), _selector(args))
+ if data.get(TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY):
+ placeholders.add(field)
+ else:
+ placeholders.discard(field)
if entering is not None and field == entering[0]:
record(field, "")
return result
if _tool == "select_combobox":
record(field, str(args.get("value") or ""))
elif _tool == "select_option":
- record(field, str(args.get("label") or args.get("value") or ""))
+ record(field, str(data.get(PICKED_OPTION_DATA_KEY) or args.get("label") or args.get("value") or ""))
elif _tool == "type" and str(data.get(TARGET_KIND_DATA_KEY) or "") in _FREE_TEXT_KINDS:
text = str(args.get("text") or "")
if len(text.split()) <= _MAX_AUDITED_WORDS:
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index 46944fdc4..59bb0e855 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -338,11 +338,22 @@ class ToolRefusal(Exception):
# tokens. The label composition itself -- the floor vocabulary, the shape filter, the secret matcher --
# lives in target_label.py; this module only carries the two raw values from probe to `RoundAction`.
TARGET_LABEL_DATA_KEY = "target_label"
-# The option text a click on a menu row committed (page text: never on the call record).
+# The option text a click on a menu row, or a native select, committed (page text: never on the call record).
PICKED_OPTION_DATA_KEY = "picked_option"
# A list the control opened but could not settle a row in: the list stays open for a pick.
OPENED_LIST_ERROR_CLASSES = frozenset({"list_no_match", "list_unread", "list_ambiguous", "identical_rows"})
TARGET_KIND_DATA_KEY = "target_kind"
+# Set when the target label is the field's placeholder, a hint rather than its question.
+TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY = "target_label_is_placeholder"
+# A refusal whose data carries a GuardEnd under this key ends the run with that guard's verdict.
+GUARD_END_DATA_KEY = "guard_end"
+
+
+class GuardEnd(NamedTuple):
+ guard: str
+ reason: str
+
+
# The durable selector of the element a `drag` dropped onto, for the pending-submit watch.
DROP_TARGET_DATA_KEY = "drop_target"
# The tool-result `data` key a recorded action's outcome rides on: the machine facts about what the
@@ -780,8 +791,9 @@ class ToolSpec:
# The selector a click's arguments resolve to as the call's record keeps it after dispatch (a ref's durable
# selector, a mark's act tag), and whether it names a row of an open list; set on click alone.
address_probe: Callable[[dict[str, Any]], Awaitable[tuple[str, bool]]] | None = None
- # The selector, name and kind of the field a type call is about to enter, read before it acts; set on type alone.
- answer_probe: Callable[[dict[str, Any]], Awaitable[tuple[str, str, str]]] | None = None
+ # The selector, name and kind of the field a type call is about to enter, and whether the name is its placeholder,
+ # read before it acts; set on type alone.
+ answer_probe: Callable[[dict[str, Any]], Awaitable[tuple[str, str, str, bool]]] | None = None
@property
def touches_page(self) -> bool:
@@ -1085,6 +1097,8 @@ NAV_DEAD_END_GUARD = "navigation_dead_end"
# A page-level handler kept asking for the page to be reloaded past the per-run cap: the page cannot
# be stabilized, and acting on it would mean acting on a page declared stale.
PAGE_REFRESH_EXHAUSTED_GUARD = "page_refresh_exhausted"
+# The answer-basis audit refused moving on ANSWER_BASIS_MAX_HOLDS times in one run.
+ANSWER_BASIS_HOLD_BOUND_GUARD = "answer_basis_hold_bound"
# The browser or its page is gone: the page accessor behind every tool already tried to reopen a lost
# tab and could not, so every later call raises the same error and the model spends turns on a browser
# that cannot come back (SKY-18903). The only guard whose verdict is `failed` rather than `terminated`:
@@ -5906,6 +5920,13 @@ async def run_agent_tool_loop(
# A successful navigate moved the run off any dead page seen earlier this batch.
st.pending_nav_dead_end = None
+ guard_end = result_data.get(GUARD_END_DATA_KEY) if result.refused else None
+ if isinstance(guard_end, GuardEnd):
+ LOG.info("taskv3 loop guard ended the run", tool=tool_name, turn=st.turns, guard=guard_end.guard)
+ st.outcome = _guard_verdict(guard_end.guard, guard_end.reason)
+ _append_skipped_tool_results(st.messages, tool_calls[idx + 1 :], guard_end.reason)
+ break
+
if spec is not None and spec.terminal and result.status == "ok":
data = result.data or {}
st.outcome = LoopOutcome(
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 5a7afbb46..bd25b2412 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -93,6 +93,7 @@ from skyvern.forge.taskv3.loop import (
REF_SELECTOR_RE,
TARGET_KIND_DATA_KEY,
TARGET_LABEL_DATA_KEY,
+ TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY,
CoveredBranch,
CoveredLayerKind,
DownloadFailureFacts,
@@ -5457,8 +5458,11 @@ _TARGET_NAME_JS = (
// from one.
const textEntry = editable || tag === 'TEXTAREA' || _matches(el, '[role=textbox],[role=searchbox]')
|| (tag === 'INPUT' && ['button', 'submit', 'reset', 'image', 'checkbox', 'radio', 'file', 'range'].indexOf(type) === -1);
- const name = nrm(_attr(el, 'aria-label')) || labelled() || labelledBy() || contents || caption
- || nrm(_attr(el, 'alt')) || nrm(_attr(el, 'placeholder')) || (textEntry ? '' : nrm(_attr(el, 'title')));
+ const named = nrm(_attr(el, 'aria-label')) || labelled() || labelledBy() || contents || caption
+ || nrm(_attr(el, 'alt'));
+ // A placeholder is a hint, often a sample answer, not the question: the name says when it came from one.
+ const hint = named ? '' : nrm(_attr(el, 'placeholder'));
+ const name = named || hint || (textEntry ? '' : nrm(_attr(el, 'title')));
// Refused here rather than truncated, and this is the only place that can be exact: `length` is
// UTF-16 code units while Python counts code points, so a name whose bound falls on an astral
// character (an emoji is one code point, two units) would arrive UNDER the Python cap and be
@@ -5498,7 +5502,7 @@ _TARGET_NAME_JS = (
kind = 'textbox';
}
}
- return { name: finalName, kind: kind };
+ return { name: finalName, kind: kind, placeholder: Boolean(hint) && finalName !== null };
}"""
)
@@ -15318,6 +15322,7 @@ def build_browser_tools(
selector = args.get("selector")
name: str | None = None
kind: str | None = None
+ from_placeholder = False
# The page comes from the preflight handoff, read and not popped -- the handler's own
# _resolve_page consumes it. Never page_provider(): a second must_get_working_page can
# recover or relaunch a page, and a label is not worth a side effect. Every labelled tool
@@ -15355,6 +15360,7 @@ def build_browser_tools(
name = raw_name if isinstance(raw_name, str) and raw_name.strip() else None
raw_kind = raw.get("kind")
kind = raw_kind if isinstance(raw_kind, str) and raw_kind else None
+ from_placeholder = raw.get("placeholder") is True
result = await handler(args)
# Only a call that acted carries a name/kind. The probe resolves the selector on its own,
# so on a refusal the two can disagree about which element the selector meant -- a cloned
@@ -15367,6 +15373,8 @@ def build_browser_tools(
data = {**(result.data or {})}
if name is not None:
data[TARGET_LABEL_DATA_KEY] = name
+ if from_placeholder:
+ data[TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY] = True
if kind is not None:
data[TARGET_KIND_DATA_KEY] = kind
# `replace`, not a positional rebuild: this carries every field the result already has,
@@ -21668,12 +21676,17 @@ def build_browser_tools(
f"selected on {selector} — {reason}, so the selection could not be verified; re-observe "
"before relying on it"
)
+ # The visible text of what is now selected: a call by value names only an internal code.
+ shown = readback.get("selectedLabels" if is_multi else "selectedLabel") if isinstance(readback, dict) else None
+ shown = ", ".join(x for x in shown if isinstance(x, str)) if isinstance(shown, list) else shown
+ data = {PICKED_OPTION_DATA_KEY: shown} if isinstance(shown, str) and shown.strip() else None
if is_multi:
return ToolResult.ok(
- f"selected on {selector} — it now holds {len(held)} option(s): {_selection_report(held)}"
+ f"selected on {selector} — it now holds {len(held)} option(s): {_selection_report(held)}", data=data
)
return ToolResult.ok(
- f"selected on {selector} (hidden native select, set directly)" if force else f"selected on {selector}"
+ f"selected on {selector} (hidden native select, set directly)" if force else f"selected on {selector}",
+ data=data,
)
async def press_key(args: dict[str, Any]) -> ToolResult:
@@ -22965,13 +22978,14 @@ def build_browser_tools(
_with_ref_resolution("answer_probe", _with_selector_guard(_answer_target))
)
- async def _answer_field(args: dict[str, Any]) -> tuple[str, str, str]:
+ async def _answer_field(args: dict[str, Any]) -> tuple[str, str, str, bool]:
resolved = dict(args)
try:
data = (await _resolve_answer_target(resolved)).data or {}
except Exception:
- return "", "", ""
- return str(resolved.get("selector") or ""), str(data.get("name") or ""), str(data.get("kind") or "")
+ return "", "", "", False
+ name, kind = str(data.get("name") or ""), str(data.get("kind") or "")
+ return str(resolved.get("selector") or ""), name, kind, data.get("placeholder") is True
tools = [
_spec(
diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.py
index 4f7ea048c..1c28b9201 100644
--- a/skyvern/webeye/actions/actions.py
+++ b/skyvern/webeye/actions/actions.py
@@ -175,6 +175,7 @@ class Action(BaseModel):
# WebAction fields
file_name: str | None = None
file_url: str | None = None
+ download_url: str | None = Field(default=None, description="Origin URL recorded by a download action.")
download: bool | None = None
download_triggered: bool | None = None # Whether a download was triggered by this action
downloaded_files: list[str] | None = None # List of file names downloaded by this action
@@ -402,7 +403,6 @@ class DownloadFileAction(Action):
download: bool = True
file_name: str
byte: Annotated[bytes | None, Field(exclude=True)] = None # bytes data
- download_url: str | None = None # URL to download file from
def __repr__(self) -> str:
return f"DownloadFileAction(file_name={self.file_name}, download_url={self.download_url}, has_byte={self.byte is not None})"
diff --git a/tests/unit/test_output_parameter_size_guards.py b/tests/unit/test_output_parameter_size_guards.py
index 593ac9213..50f8b8390 100644
--- a/tests/unit/test_output_parameter_size_guards.py
+++ b/tests/unit/test_output_parameter_size_guards.py
@@ -1282,6 +1282,8 @@ async def test_get_workflow_run_timeline_caps_action_text(monkeypatch: pytest.Mo
task_id="tsk_1",
response="r" * (cap * 3),
text="t" * (cap * 3),
+ file_url="https://files.invalid/" + "f" * (cap * 3),
+ download_url="https://files.invalid/" + "d" * (cap * 3),
)
monkeypatch.setattr(
workflow_service_module.app,
@@ -1298,7 +1300,7 @@ async def test_get_workflow_run_timeline_caps_action_text(monkeypatch: pytest.Mo
)
capped_action = timeline[0].block.actions[0]
- for field in ("response", "text"):
+ for field in ("response", "text", "file_url", "download_url"):
value = getattr(capped_action, field)
assert isinstance(value, str), field
assert "[truncated" in value, field
diff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.py
index e34cf23ea..8721cbfad 100644
--- a/tests/unit/test_script_skyvern_page.py
+++ b/tests/unit/test_script_skyvern_page.py
@@ -1889,6 +1889,55 @@ class TestActionSubclassPersistence:
kwargs=kwargs,
)
+ @pytest.mark.asyncio
+ async def test_native_download_records_the_url_fetch_even_with_ignored_byte_kwargs(
+ self, mock_scraped_page, mock_ai
+ ):
+ script_page = self._build_script_page(mock_scraped_page, mock_ai)
+ script_page._wait_for_page_ready_before_action = AsyncMock()
+ script_page._create_screenshot_after_execution = AsyncMock()
+ script_page._create_html_action_after_execution = AsyncMock()
+ context = self._build_context()
+ captured = []
+
+ async def save_action(action):
+ captured.append(action)
+ return action
+
+ with (
+ patch("skyvern.core.script_generations.script_skyvern_page.skyvern_context.current", return_value=context),
+ patch(
+ "skyvern.core.script_generations.skyvern_page.download_file_from_url",
+ new=AsyncMock(return_value="/synthetic/report.pdf"),
+ ) as fetch,
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.app.DATABASE.workflow_params.create_action",
+ new=AsyncMock(side_effect=save_action),
+ ),
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.app.DATABASE.workflow_params.update_action_reasoning",
+ new=AsyncMock(),
+ ),
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.script_run_context_manager.get_run_context",
+ return_value=None,
+ ),
+ patch.object(settings, "CACHED_ACTION_DELAY_SECONDS", 0),
+ ):
+ path = await script_page.download_file(
+ file_name="report.pdf", download_url="https://example.com/report.pdf", byte=b"ignored bytes"
+ )
+
+ assert path == "/synthetic/report.pdf"
+ fetch.assert_awaited_once_with(
+ "https://example.com/report.pdf", filename="report.pdf", organization_id="o_test"
+ )
+ action = captured[0]
+ assert action.status == "completed" and action.created_by == "script"
+ assert action.download is True
+ assert action.file_name == "report.pdf" and action.download_url == "https://example.com/report.pdf"
+ assert action.download_triggered is None and action.downloaded_files is None
+
@pytest.mark.asyncio
async def test_move_persists_x_and_y(self, mock_scraped_page, mock_ai):
from skyvern.webeye.actions.action_types import ActionType
diff --git a/tests/unit/test_taskv3_answer_basis_audit.py b/tests/unit/test_taskv3_answer_basis_audit.py
index 061415156..2a760feab 100644
--- a/tests/unit/test_taskv3_answer_basis_audit.py
+++ b/tests/unit/test_taskv3_answer_basis_audit.py
@@ -9,6 +9,7 @@ from skyvern.forge.taskv3.loop import (
PICKED_OPTION_DATA_KEY,
TARGET_KIND_DATA_KEY,
TARGET_LABEL_DATA_KEY,
+ TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY,
ToolRefusal,
ToolResult,
ToolSpec,
@@ -39,11 +40,12 @@ async def _address(args: dict[str, Any]) -> tuple[str, bool]:
return _REFS.get(selector, (selector, False))
-async def _answer(args: dict[str, Any]) -> tuple[str, str, str]:
+async def _answer(args: dict[str, Any]) -> tuple[str, str, str, bool]:
# What the field reads as before the call acts; `_probed` stands in for a probe that resolves elsewhere or fails.
if args.get("_probed") == "":
- return "", "", ""
- return str(args.get("_probed", args.get("selector"))), str(args.get("_label", "")), args.get("_kind", "textbox")
+ return "", "", "", False
+ selector, label = str(args.get("_probed", args.get("selector"))), str(args.get("_label", ""))
+ return selector, label, args.get("_kind", "textbox"), bool(args.get("_placeholder"))
def _tools(dispatched: list[tuple[str, dict[str, Any]]]) -> list[ToolSpec]:
@@ -63,7 +65,12 @@ def _tools(dispatched: list[tuple[str, dict[str, Any]]]) -> list[ToolSpec]:
if name == "select_combobox" and args.get("_no_match"):
content = f"opened {sel} but no option matched {args['value']!r}"
return ToolResult("error", content, {TARGET_LABEL_DATA_KEY: label}, error_class="list_no_match")
- return ToolResult("ok", f"{name} done", {TARGET_LABEL_DATA_KEY: label, TARGET_KIND_DATA_KEY: "textbox"})
+ data = {TARGET_LABEL_DATA_KEY: label, TARGET_KIND_DATA_KEY: "textbox"}
+ if "_shown" in args:
+ data[PICKED_OPTION_DATA_KEY] = args["_shown"]
+ if args.get("_placeholder"):
+ data[TARGET_LABEL_IS_PLACEHOLDER_DATA_KEY] = True
+ return ToolResult("ok", f"{name} done", data)
probe = _answer if name == "type" else None
return ToolSpec(
@@ -309,3 +316,23 @@ async def test_a_typed_enter_the_probe_cannot_place_is_still_recorded_after_the_
await _by_name(tools, "type").handler({**TYPED, "_probed": "#elsewhere", "text": "Yes"})
await _by_name(tools, "click").handler({"selector": "#submit"})
assert [name for name, _ in dispatched] == ["type", "click"] and len(audit.seen) == 2
+
+
+async def test_the_audit_records_what_the_page_shows_and_where_its_question_came_from() -> None:
+ audit = _Audit()
+ tools = _tools([])
+ apply_answer_basis_audit(tools, audit)
+ select, type_ = _by_name(tools, "select_option").handler, _by_name(tools, "type").handler
+
+ await select({"selector": "#auth", "value": "10", "_label": DEBARMENT, "_shown": "No"})
+ # A field named only by its placeholder is still audited; the audit is told the name is a placeholder.
+ await type_({"selector": "#country", "text": "Canada", "_label": "Country", "_placeholder": True})
+ await type_({"selector": "#visa", "text": "No", "_label": "Visa", "_placeholder": True, "press_enter": True})
+ await _by_name(tools, "click").handler({"selector": "#submit"})
+
+ assert audit.seen == [
+ [RecordedAnswer(DEBARMENT, "No")],
+ [RecordedAnswer("Country", "Canada", label_is_placeholder=True)],
+ [RecordedAnswer("Visa", "No", label_is_placeholder=True)],
+ ]
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index da2835ed6..26fe747c9 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -41,6 +41,7 @@ from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.taskv3 import loop as loop_module
from skyvern.forge.taskv3 import tools as taskv3_tools_module
+from skyvern.forge.taskv3.answer_basis_audit import ANSWER_BASIS_MAX_HOLDS, RecordedAnswer, apply_answer_basis_audit
from skyvern.forge.taskv3.auth_tools import _MAX_GIVEUP_DEFERRALS, VerificationFailure, VerificationState
from skyvern.forge.taskv3.engine import MAX_TOKENS_CEILING, MAX_TOKENS_PER_ACTION_STEP, taskv3_runaway_backstops
from skyvern.forge.taskv3.goal_check import GoalVerdict, ToolTrail, run_goal_check
@@ -55,6 +56,7 @@ from skyvern.forge.taskv3.loop import (
ACTION_LOOP_NUDGE_AFTER,
ACTION_LOOP_TERMINATE_AFTER,
ACTION_OUTCOME_DATA_KEY,
+ ANSWER_BASIS_HOLD_BOUND_GUARD,
BROWSER_LOST_EVENT,
BROWSER_LOST_GUARD,
BROWSER_LOST_REASON,
@@ -89,6 +91,8 @@ from skyvern.forge.taskv3.loop import (
SUBMISSION_HELD_WAIT_MAX_SECONDS,
SUBMISSION_RENDER_MARGIN_SECONDS,
TARGET_ACTION_ROUND_EVENT,
+ TARGET_KIND_DATA_KEY,
+ TARGET_LABEL_DATA_KEY,
TOKEN_BUDGET_EXTENDED_EVENT,
TOKEN_BUDGET_EXTENSION_REFUSED_EVENT,
UNCHARGED_REFUSAL_GRACE,
@@ -12455,6 +12459,41 @@ async def test_a_lost_browser_ends_the_run_where_any_other_raise_comes_back_as_a
assert (lost["guard"], lost["tool"]) == (BROWSER_LOST_GUARD, "click")
+async def test_the_answer_audit_ends_a_run_it_keeps_holding_with_a_verdict_naming_the_question() -> None:
+ # The bound's hold ends the run terminated; the holds before it stay ordinary refusals the model reads.
+ clicks: list[tuple[str, dict[str, Any]]] = []
+
+ class _FlagsEverything:
+ async def unsupported(self, answers: list[RecordedAnswer]) -> list[RecordedAnswer]:
+ return answers
+
+ async def type_answer(args: dict[str, Any]) -> ToolResult:
+ return ToolResult.ok(
+ "typed", data={TARGET_LABEL_DATA_KEY: "Work authorization", TARGET_KIND_DATA_KEY: "textbox"}
+ )
+
+ tools = [
+ ToolSpec(name="type", description="t", parameters={}, handler=type_answer, billable=True),
+ _recording_tool("click", clicks, billable=True),
+ make_finish_tool(),
+ ]
+ apply_answer_basis_audit(tools, _FlagsEverything())
+ # A different control each turn, so the repeated-action guard never fires first.
+ submits = [[("click", {"selector": f"#next{i}"})] for i in range(ANSWER_BASIS_MAX_HOLDS + 1)]
+ script = [
+ [("type", {"selector": "#auth", "text": "Yes"})],
+ *submits,
+ [("finish", {"status": "completed", "reason": "x"})],
+ ]
+ outcome, caller = await _run(script, tools, max_turns=30)
+
+ assert (outcome.status, outcome.guard) == ("terminated", ANSWER_BASIS_HOLD_BOUND_GUARD)
+ assert "'Work authorization'" in outcome.reason and "Yes" not in outcome.reason
+ assert caller.calls == 1 + ANSWER_BASIS_MAX_HOLDS
+ assert clicks == []
+
+
def test_every_driver_timeout_type_installed_here_classifies_as_a_driver_timeout() -> None:
# Written against the driver-type TABLE rather than a hard-coded package, because the hazard is
# precisely that CI and production do not install the same driver: the browser image repoints
diff --git a/tests/unit/test_workflow_run_detail_template_cross_org.py b/tests/unit/test_workflow_run_detail_template_cross_org.py
index f9d152b2e..d890270d6 100644
--- a/tests/unit/test_workflow_run_detail_template_cross_org.py
+++ b/tests/unit/test_workflow_run_detail_template_cross_org.py
@@ -1,15 +1,21 @@
-from datetime import datetime, timezone
+from datetime import UTC, datetime
from types import SimpleNamespace
from unittest.mock import AsyncMock
+import httpx
import pytest
+from fastapi import FastAPI
from skyvern.forge import app
+from skyvern.forge.sdk.routes import agent_protocol
+from skyvern.forge.sdk.services import org_auth_service
from skyvern.forge.sdk.workflow.models.workflow import (
Workflow,
WorkflowDefinition,
+ WorkflowRun,
WorkflowRunResponseBase,
WorkflowRunStatus,
+ WorkflowRunWithWorkflowResponse,
WorkflowStatus,
)
from skyvern.forge.sdk.workflow.service import WorkflowService
@@ -20,7 +26,7 @@ CALLER_ORG_ID = "o_caller"
def _make_template_workflow() -> Workflow:
- now = datetime.now(timezone.utc)
+ now = datetime.now(UTC)
return Workflow(
workflow_id="w_tmpl",
organization_id=TEMPLATE_ORG_ID,
@@ -36,37 +42,22 @@ def _make_template_workflow() -> Workflow:
)
-def _make_workflow_run(workflow_run_id: str = "wr_tmpl", organization_id: str = CALLER_ORG_ID) -> SimpleNamespace:
- now = datetime.now(timezone.utc)
- return SimpleNamespace(
+def _make_workflow_run(
+ workflow_run_id: str = "wr_tmpl",
+ organization_id: str = CALLER_ORG_ID,
+ copilot_session_id: str | None = None,
+) -> WorkflowRun:
+ now = datetime.now(UTC)
+ return WorkflowRun(
workflow_run_id=workflow_run_id,
workflow_id="w_tmpl",
+ workflow_permanent_id="wpid_tmpl",
organization_id=organization_id,
+ copilot_session_id=copilot_session_id,
status=WorkflowRunStatus.running,
- failure_reason=None,
- failure_category=None,
- retried_from_workflow_run_id=None,
- proxy_location=None,
- webhook_callback_url=None,
- webhook_failure_reason=None,
- totp_verification_url=None,
- totp_identifier=None,
- extra_http_headers=None,
- cdp_connect_headers=None,
- queued_at=None,
started_at=now,
- finished_at=None,
created_at=now,
modified_at=now,
- credits_used=0,
- cached_credits_used=0,
- browser_session_id=None,
- browser_profile_id=None,
- browser_seed_source=None,
- max_screenshot_scrolls=None,
- browser_address=None,
- run_with=None,
- script_run=None,
)
@@ -86,7 +77,13 @@ def template_run_db(monkeypatch: pytest.MonkeyPatch) -> tuple[AsyncMock, AsyncMo
get_workflow_by_permanent_id=get_by_wpid,
),
observer=SimpleNamespace(get_task_v2_by_workflow_run_id=AsyncMock(return_value=None)),
- tasks=SimpleNamespace(get_tasks_by_workflow_run_id=AsyncMock(return_value=[])),
+ tasks=SimpleNamespace(
+ get_tasks_by_workflow_run_id=AsyncMock(return_value=[]),
+ get_step_counts_by_task_ids=AsyncMock(return_value=(0, 0)),
+ ),
+ browser_sessions=SimpleNamespace(
+ get_persistent_browser_session_by_runnable_id=AsyncMock(return_value=None)
+ ),
workflow_runs=SimpleNamespace(
get_workflow_run_parameters=AsyncMock(return_value=[]),
get_workflow_run_block_errors=AsyncMock(return_value=[]),
@@ -99,11 +96,14 @@ def template_run_db(monkeypatch: pytest.MonkeyPatch) -> tuple[AsyncMock, AsyncMo
@pytest.mark.asyncio
[email protected]("copilot_session_id", [None, "wcc_123456"])
async def test_template_run_detail_resolves_via_run_join(
- monkeypatch: pytest.MonkeyPatch, template_run_db: tuple[AsyncMock, AsyncMock]
+ monkeypatch: pytest.MonkeyPatch,
+ template_run_db: tuple[AsyncMock, AsyncMock],
+ copilot_session_id: str | None,
) -> None:
get_by_run, get_by_wpid = template_run_db
- workflow_run = _make_workflow_run()
+ workflow_run = _make_workflow_run(copilot_session_id=copilot_session_id)
# A real service instance: the stub app's WORKFLOW_SERVICE is a lazy auto-mock, so we must
# exercise the actual build_workflow_run_status_response rather than app.WORKFLOW_SERVICE.
@@ -125,6 +125,7 @@ async def test_template_run_detail_resolves_via_run_join(
assert isinstance(response, WorkflowRunResponseBase)
assert response.workflow_run_id == "wr_tmpl"
assert response.workflow_title == "Job Application Workflow"
+ assert response.copilot_session_id == copilot_session_id
# Resolution must go through the run join (scoped by the run's org), not the caller-org
# permanent-id lookup that returns None for cross-org template workflows.
@@ -133,6 +134,26 @@ async def test_template_run_detail_resolves_via_run_join(
assert get_by_run.await_args.kwargs.get("organization_id") == CALLER_ORG_ID
get_by_wpid.assert_not_awaited()
+ monkeypatch.setattr(service, "get_workflow_by_workflow_run_id", AsyncMock(return_value=get_by_run.return_value))
+ monkeypatch.setattr(app, "WORKFLOW_SERVICE", service)
+ monkeypatch.setattr(app.AGENT_FUNCTION, "calculate_workflow_run_total_cost", AsyncMock(return_value=0.0))
+ canonical = next(
+ route for route in agent_protocol.base_router.routes if route.path == "/workflows/runs/{workflow_run_id}"
+ )
+ assert canonical.endpoint is agent_protocol.get_workflow_and_run_from_workflow_run_id
+ assert canonical.response_model is WorkflowRunWithWorkflowResponse
+ api = FastAPI()
+ api.dependency_overrides[org_auth_service.get_current_org] = lambda: SimpleNamespace(organization_id=CALLER_ORG_ID)
+ api.include_router(agent_protocol.base_router, prefix="/v1")
+ async with httpx.AsyncClient(
+ transport=httpx.ASGITransport(app=api), base_url="http://test", trust_env=False
+ ) as client:
+ serialized = await client.get("/v1/workflows/runs/wr_tmpl")
+
+ assert serialized.status_code == 200
+ assert serialized.json()["copilot_session_id"] == copilot_session_id
+ assert serialized.json()["workflow"]["organization_id"] == TEMPLATE_ORG_ID
+
@pytest.mark.asyncio
@pytest.mark.usefixtures("template_run_db")
diff --git a/tests/unit/workflow/test_workflow_run_timeline_tree.py b/tests/unit/workflow/test_workflow_run_timeline_tree.py
index d176a1efb..65f8ce5b8 100644
--- a/tests/unit/workflow/test_workflow_run_timeline_tree.py
+++ b/tests/unit/workflow/test_workflow_run_timeline_tree.py
@@ -4,19 +4,24 @@ from __future__ import annotations
import json
from datetime import UTC, datetime, timedelta, timezone
+from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock
+import httpx
import pytest
+from fastapi import FastAPI
from pydantic import TypeAdapter
from skyvern.forge import app
from skyvern.forge.sdk.db.repositories.tasks import TasksRepository
from skyvern.forge.sdk.routes import agent_protocol
from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock, WorkflowRunTimeline, WorkflowRunTimelineType
+from skyvern.forge.sdk.services import org_auth_service
from skyvern.forge.sdk.workflow.service import WorkflowService
+from skyvern.schemas.run_enums import RunType
from skyvern.schemas.workflows import BlockType
-from skyvern.webeye.actions.actions import Action, ExtractAction
+from skyvern.webeye.actions.actions import Action, DownloadFileAction, ExtractAction
from tests.unit.helpers import (
ONE_TIME_CODE,
assert_one_time_code_withheld,
@@ -160,6 +165,46 @@ async def test_timeline_attaches_actions_to_code_blocks(mock_db: AsyncMock) -> N
assert timeline[0].block.actions == [action]
+async def test_native_download_url_survives_registered_timeline_response(
+ mock_db: AsyncMock, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ stamp = datetime(2026, 5, 4, 12, 0, tzinfo=UTC)
+ mock_db.return_value = [_block("wrb_download", created_at=stamp, block_type=BlockType.CODE, task_id="tsk_download")]
+ app.DATABASE.tasks.get_tasks_actions.return_value = [
+ DownloadFileAction(
+ action_id="act_download",
+ organization_id="o_test",
+ workflow_run_id="wr_test",
+ task_id="tsk_download",
+ step_id="stp_download",
+ status="completed",
+ file_name="report.pdf",
+ download_url="https://example.com/files/report.pdf?token=synthetic-private-query",
+ byte=b"PRIVATE_DOWNLOAD_BYTES",
+ created_by="script",
+ )
+ ]
+ monkeypatch.setattr(
+ app.DATABASE.tasks, "get_run", AsyncMock(return_value=SimpleNamespace(task_run_type=RunType.workflow_run))
+ )
+ monkeypatch.setattr(app.DATABASE.observer, "get_task_v2_by_workflow_run_id", AsyncMock(return_value=None))
+ monkeypatch.setattr(app, "WORKFLOW_SERVICE", WorkflowService())
+ monkeypatch.setattr(agent_protocol.analytics, "capture", lambda *_args, **_kwargs: None)
+ api = FastAPI()
+ api.include_router(agent_protocol.base_router, prefix="/v1")
+ api.dependency_overrides[org_auth_service.get_current_org] = lambda: SimpleNamespace(organization_id="o_test")
+
+ async with httpx.AsyncClient(transport=httpx.ASGITransport(app=api), base_url="http://test") as client:
+ response = await client.get("/v1/runs/wr_test/timeline")
+
+ assert response.status_code == 200
+ action = response.json()[0]["block"]["actions"][0]
+ assert action["download_url"] == "https://example.com/files/report.pdf?token=synthetic-private-query"
+ assert action["file_name"] == "report.pdf" and action["created_by"] == "script"
+ assert "byte" not in action and "PRIVATE_DOWNLOAD_BYTES" not in response.text
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("cap_output_values", [True, False])
async def test_timeline_serves_the_typed_value_and_withholds_the_one_time_code(