Skip to content

Automate browser based workflows with AI

Show that Copilot questions can be answered in your own words (SKY-17389) (#8713)

Shuchang Zhengd6e13af4e4f3
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index 203fc5a45..8ee196d03 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -6571,6 +6571,7 @@ export function WorkflowCopilotChat({
inputValue,
setInputValue,
);
+ const trayPart = trayQuestion?.parts[questionStepper.index];
// handleSend is memoised and outlives a render; the stepper is rebuilt on every keystroke.
const questionStepperRef = useRef(questionStepper);
questionStepperRef.current = questionStepper;
@@ -10426,9 +10427,11 @@ export function WorkflowCopilotChat({
isLoading,
isWaitingForLiveBrowser,
latestTurnIsAsk,
- askPartHasChoices:
- (trayQuestion?.parts[questionStepper.index]?.choices.length ??
- 0) > 0,
+ askPartChoices: !trayPart?.choices.length
+ ? "none"
+ : questionStepper.choices[trayPart.part_id] === undefined
+ ? "unpicked"
+ : "picked",
})}
value={inputValue}
onChange={(e) => setInputValue(e.target.value)}
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/QuestionTray.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/QuestionTray.tsx
index 0e8c39e8c..b527a0b53 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/cards/QuestionTray.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/QuestionTray.tsx
@@ -186,11 +186,18 @@ export function QuestionTray({
);
})}
</div>
- ) : (
+ ) : null}
+ {/* Typing does not clear a picked choice, so the invitation to answer instead of the
+ choices only shows while none is picked. */}
+ {part.choices.length === 0 ? (
<p className="text-xs text-muted-foreground">
Type your answer in the message box below.
</p>
- )}
+ ) : stepper.choices[part.part_id] === undefined ? (
+ <p className="text-xs text-muted-foreground">
+ None of these fit? Type your own answer in the message box below.
+ </p>
+ ) : null}
</div>
) : null}
<div
diff --git a/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.test.ts b/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.test.ts
index 0c489cf5a..a2ce50c60 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.test.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.test.ts
@@ -18,9 +18,16 @@ describe("composerPlaceholder", () => {
composerPlaceholder({
...base,
latestTurnIsAsk: true,
- askPartHasChoices: true,
+ askPartChoices: "unpicked",
}),
- ).toBe("Add details (optional)…");
+ ).toBe("Or type your own…");
+ expect(
+ composerPlaceholder({
+ ...base,
+ latestTurnIsAsk: true,
+ askPartChoices: "picked",
+ }),
+ ).toBe("Add details…");
});
it("returns to the standing invitation once the ask is answered", () => {
diff --git a/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.ts b/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.ts
index 302ea810d..2873a99b3 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.ts
@@ -5,15 +5,16 @@ export function composerPlaceholder({
isLoading,
isWaitingForLiveBrowser,
latestTurnIsAsk,
- askPartHasChoices = false,
+ askPartChoices = "none",
}: {
// Whether a send adds to the queued message or replaces it (a programmatic one).
queuedPrompt: "add" | "replace" | null;
isLoading: boolean;
isWaitingForLiveBrowser: boolean;
latestTurnIsAsk: boolean;
- // Whether the question on screen offers choices; without them the composer is the answer.
- askPartHasChoices?: boolean;
+ // The question on screen's choices: without any the composer is the answer, and typing never
+ // clears a picked one, so text sent after a pick goes alongside it rather than replacing it.
+ askPartChoices?: "none" | "unpicked" | "picked";
}): string {
if (queuedPrompt === "add") return "Add to the queued message…";
if (queuedPrompt === "replace") return "Type to replace the queued message…";
@@ -22,6 +23,10 @@ export function composerPlaceholder({
// While a question is pending the composer is the text field for the question on screen, so
// it says so rather than inviting an unrelated new request.
if (latestTurnIsAsk)
- return askPartHasChoices ? "Add details (optional)…" : "Type your answer…";
+ return askPartChoices === "picked"
+ ? "Add details…"
+ : askPartChoices === "unpicked"
+ ? "Or type your own…"
+ : "Type your answer…";
return "Ask Copilot to build or change your workflow…";
}
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 03c3bba38..31861c2e2 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -2259,7 +2259,13 @@ class ForgeAgent:
)
from skyvern.forge.taskv3.loop import DEFAULT_MAX_SETTLE_DEFERRALS, CompletionBlocker, CompletionProbe
from skyvern.forge.taskv3.opaque_refs import mask_opaque_urls
- from skyvern.forge.taskv3.tools import _observable_child_frames, _recorded_work_frames, pending_marker
+ from skyvern.forge.taskv3.tools import (
+ _document_unidentifiable,
+ _observable_child_frames,
+ _realm_document_id,
+ _recorded_work_frames,
+ pending_marker,
+ )
from skyvern.forge.taskv3.workflow_position import PreviousBlockHandoff, select_previous_block
from skyvern.utils.token_counter import approx_count_tokens
@@ -2965,6 +2971,26 @@ class ForgeAgent:
nonce = await peek.evaluate(_DOCUMENT_NONCE_JS)
return f"{peek.url}|{nonce}"
+ # _page_probe plus every child document the run acted in, where a form can submit with the main
+ # document unchanged. Each realm pairs the browser-owned loaderId (unforgeable) with the nonce (which still
+ # catches a reload if the CDP session degrades to the url); a realm that cannot be identified raises, and
+ # the finish gate reads that as an unreadable identity.
+ async def _document_identity() -> str | None:
+ peek = await _fingerprint_page()
+ if peek is None:
+ return None
+
+ async def _realm_part(realm: Any) -> str:
+ realm_id = await _realm_document_id(realm)
+ if _document_unidentifiable(realm_id):
+ raise RuntimeError("taskv3 document identity: a realm could not be identified")
+ return f"{realm_id}|{await realm.evaluate(_DOCUMENT_NONCE_JS)}"
+
+ parts = [await _realm_part(peek)]
+ for frame in _recorded_work_frames(peek):
+ parts.append("detached" if frame.is_detached() else await _realm_part(frame))
+ return "\n".join(parts)
+
async def _reload_page() -> None:
# Observed by the action policy like the legacy internal refresh; the loop records it in
# the action round. A bare task pins one page, so it is passed explicitly.
@@ -3135,6 +3161,7 @@ class ForgeAgent:
page_free=page_free_validation,
page_fingerprint=_page_fingerprint,
page_probe=_page_probe,
+ document_identity=_document_identity,
reload_page=_reload_page,
restore_page_url=_restore_page_url,
download_attempts=_download_attempts,
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index 6b1654978..efb2353f8 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -300,6 +300,7 @@ async def run_task_v3_agent_loop(
label_secret_values: Callable[[], Collection[str]] | None = None,
login_identifier_tokens: Callable[[], Collection[str]] | None = None,
page_probe: Callable[[], Awaitable[str | None]] | None = None,
+ document_identity: Callable[[], Awaitable[str | None]] | None = None,
reload_page: Callable[[], Awaitable[None]] | None = None,
restore_page_url: Callable[[Any, str], Awaitable[None]] | None = None,
download_attempts: Callable[[], int | None] | None = None,
@@ -333,8 +334,9 @@ async def run_task_v3_agent_loop(
for a bounded re-verification turn; without one, pre-finish re-verification is prompt guidance
only. `max_settle_deferrals=0` disables that completed-side re-verification while leaving the
failure-evidence gate, which shares the sampler, intact. `page_probe` is a separate sampler (URL
- plus fingerprint) the loop uses to detect whether a failed batched call moved the page; a
- page-free run has no page to probe."""
+ plus document nonce) the loop uses to detect whether a failed batched call moved the page.
+ `document_identity` also covers the child frames the run acted in; the finish tool uses it to veto a re-ask
+ conversion whose document changed. A page-free run has no page to probe."""
loop_started_at = time.monotonic()
# Presigned file URLs in the payload carry an HMAC token the model would otherwise have to
# retype verbatim into a tool call; masking them here and resolving inside the tool handlers
@@ -555,6 +557,7 @@ async def run_task_v3_agent_loop(
goal_check=_goal_check if goal_check_on else None,
goal_check_enforce=goal_check_enforce,
unlisted_reask=_unlisted_reask if reask_on else None,
+ document_identity=None if page_free else document_identity,
)
tools = browser_tools + (extra_tools or []) + [finish_tool]
# The COMPLETE dispatch list, not just the browser tools: auth / captcha / code tools and finish
diff --git a/skyvern/forge/taskv3/goal_check.py b/skyvern/forge/taskv3/goal_check.py
index 30bd526e8..8eec252d4 100644
--- a/skyvern/forge/taskv3/goal_check.py
+++ b/skyvern/forge/taskv3/goal_check.py
@@ -438,6 +438,9 @@ class UnlistedReask:
reason: str = ""
veto: str | None = None
converted: bool = False
+ # The conversion's settle window: None when it did not run; rounds are fingerprint pairs.
+ settled: bool | None = None
+ settle_rounds: int = 0
# The goal check's verdict on the converted completion, when a goal judge is wired.
goal_check_verdict: str | None = None
llm_key: str | None = None
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index a3cad6307..f74abb4a2 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -2313,6 +2313,7 @@ def make_finish_tool(
goal_check: Callable[[], Awaitable[GoalVerdict]] | None = None,
goal_check_enforce: bool = False,
unlisted_reask: UnlistedReaskCheck | None = None,
+ document_identity: Callable[[], Awaitable[str | None]] | None = None,
) -> ToolSpec:
"""`page_fingerprint` samples an opaque fingerprint of the page's rendered content (None when no
page is available). A finish(completed) is deferred (bounded by `max_settle_deferrals`, then
@@ -2347,7 +2348,9 @@ def make_finish_tool(
`unlisted_reask` asks once per run whether a failed or terminated verdict that is about to stand only
missed a screen the site skipped. A grounded yes becomes completed unless a completed-side gate vetoes it;
- it never gives a turn back, and a completed verdict cannot reach it."""
+ it never gives a turn back, and a completed verdict cannot reach it. Its settle gate samples as many times as
+ the completed path would defer. When `document_identity` is wired, it is read before the judge and again after
+ the settle window, and a conversion whose identity changed or could not be read is refused, settled or not."""
deferrals = 0
failure_deferrals = 0
goal_check_held = False
@@ -2408,7 +2411,7 @@ def make_finish_tool(
return False # defer: the loop's cancellation check ends the run before another turn
return first == await _bounded_fingerprint()
- async def _conversion_veto(result: UnlistedReask) -> str | None:
+ async def _conversion_veto(result: UnlistedReask, identity_before: str | None) -> str | None:
"""The completed-side gates, as vetoes only: the model never claimed completion, so none may hold."""
try:
if should_cancel is not None and await should_cancel():
@@ -2433,10 +2436,34 @@ def make_finish_tool(
except Exception:
return "verification_blocker"
if page_fingerprint is not None:
- try:
- if not await _settled():
- return "unsettled"
- except Exception:
+ if document_identity is not None and identity_before is None:
+ return "identity_unreadable"
+ settled = False
+ for _ in range(max_settle_deferrals + 1):
+ result.settle_rounds += 1
+ try:
+ settled = await _settled()
+ except Exception:
+ pass # as on the completed path: an unreadable sample is an unsettled one
+ result.settled = settled
+ if settled:
+ break
+ try:
+ if should_cancel is not None and await should_cancel():
+ return "canceled"
+ except Exception:
+ return "canceled"
+ if deadline_at is not None and deadline_at - time.monotonic() <= 0:
+ return "deadline"
+ if deadline_at is not None and deadline_at - time.monotonic() <= 0:
+ return "deadline"
+ if document_identity is not None:
+ identity_after = await _sample_probe(document_identity, deadline_at=deadline_at)
+ if identity_after is None:
+ return "identity_unreadable"
+ if identity_before != identity_after:
+ return "navigating"
+ elif not settled:
return "unsettled"
if goal_check is not None:
try:
@@ -2453,13 +2480,19 @@ def make_finish_tool(
async def _reask(original: NonCompletedStatus, args: dict[str, Any]) -> ToolResult | None:
assert unlisted_reask is not None
reason = args.get("reason") or ""
+ # Before the judge: a navigation that commits while it runs must not become the baseline.
+ identity_before = (
+ await _sample_probe(document_identity, deadline_at=deadline_at)
+ if document_identity is not None and page_fingerprint is not None
+ else None
+ )
try:
result: UnlistedReask | None = await unlisted_reask(original, reason)
except Exception:
LOG.warning("taskv3 unlisted reask failed; keeping the verdict", exc_info=True)
result = None
if result is not None and result.converts:
- result.veto = await _conversion_veto(result)
+ result.veto = await _conversion_veto(result, identity_before)
result.converted = result.veto is None
LOG.info(
"taskv3 finish unlisted reask",
@@ -2469,6 +2502,8 @@ def make_finish_tool(
converts=result.converts if result is not None else False,
converted=result.converted if result is not None else False,
veto=result.veto if result is not None else None,
+ settled=result.settled if result is not None else None,
+ settle_rounds=result.settle_rounds if result is not None else 0,
goal_check_verdict=result.goal_check_verdict if result is not None else None,
llm_key=result.llm_key if result is not None else None,
skipped_reason=result.skipped_reason if result is not None else "reask_error",
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 250dec4b5..72b010407 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -9261,8 +9261,10 @@ _TEXT_DELTA_MAX_FAILED_READS = 2
_TEXT_DELTA_TEXT_MAX_CHARS = 2_000_000
# One evaluate, so the document identity and the text it is paired with come from the same document.
# The nonce is the one the loop's page probe keys on; [seed, limit] arrive as an argument, never as source.
-# Unlike `_realm_document_id` the nonce is page-forgeable, which is acceptable because it gates nothing
-# (a forged one only mis-scopes which lines are reported) and it saves a CDP call per action.
+# Unlike `_realm_document_id` the nonce is page-forgeable. That is acceptable: it saves a CDP call per action.
+# The page probe compares it for change, so a page that pre-seeds or forges the nonce can hide a reload from
+# that probe. The re-ask conversion's identity pairs it with `_realm_document_id`'s loaderId instead, which the
+# page cannot forge.
_TEXT_DELTA_READ_JS = (
r"""(args) => {
let nonce = window.__skyvern_doc_nonce;
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index 79af591d2..3cd0bca30 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -64,6 +64,7 @@ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from skyvern.forge.sdk.workflow.page_derived_templates import CLOSE as PAGE_DERIVED_CLOSE
from skyvern.forge.sdk.workflow.page_derived_templates import OPEN as PAGE_DERIVED_OPEN
from skyvern.forge.taskv3 import engine as taskv3_engine
+from skyvern.forge.taskv3 import tools as taskv3_tools
from skyvern.forge.taskv3.auth_tools import VerificationFailure, VerificationState
from skyvern.forge.taskv3.engine import DEFAULT_MAX_SETTLE_DEFERRALS, MIN_ACTION_STEPS, run_task_v3_agent_loop
from skyvern.forge.taskv3.goal_composition import PAGE_DATA_NOTE
@@ -84,7 +85,7 @@ from skyvern.forge.taskv3.run_arms import (
TYPE_COORDINATE_CLICK_FLAG,
run_arm_enabled,
)
-from skyvern.forge.taskv3.tools import PageProvider
+from skyvern.forge.taskv3.tools import PageProvider, _record_frame_work
from skyvern.schemas.runs import RunEngine
from skyvern.schemas.workflows import BlockStatus, BlockType
from skyvern.utils.secret_redaction import REDACTED_SECRET_PLACEHOLDER
@@ -5146,6 +5147,154 @@ async def test_execute_task_v3_page_fingerprint_samples_child_frames(monkeypatch
assert second != first
+async def test_execute_task_v3_document_identity_changes_with_an_acted_in_frame_only(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A form can submit inside a frame with the main document unchanged; the re-ask conversion's identity
+ # must see that frame, and only the frames the run acted in (an unrelated ad frame churns freely).
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=[])
+ main_frame = object()
+ acted = MagicMock()
+ acted.url = "https://example.com/form"
+ acted.is_detached = MagicMock(return_value=False)
+ acted.evaluate = AsyncMock(return_value="frame-nonce-a")
+ unrelated = MagicMock()
+ unrelated.url = "https://ads.example/frame"
+ unrelated.is_detached = MagicMock(return_value=False)
+ unrelated.evaluate = AsyncMock(return_value="ad-nonce-a")
+ pinned = MagicMock()
+ pinned.is_closed = MagicMock(return_value=False)
+ pinned.url = "https://example.com/"
+ pinned.main_frame = main_frame
+ pinned.frames = [main_frame, acted, unrelated]
+ pinned.evaluate = AsyncMock(return_value="main-nonce")
+
+ # A MagicMock realm has no real Playwright shape for `_realm_document_id`'s own CDP/frame-tree
+ # plumbing to key on, so the loaderId per realm is faked directly here, keyed on which mock object
+ # the identity closure passes in. `loader_ids` maps a realm to its current fake loaderId.
+ loader_ids: dict[int, str] = {}
+
+ async def fake_realm_document_id(target: Any) -> str:
+ return f"{target.url}|{loader_ids.get(id(target), 'loader-a')}"
+
+ monkeypatch.setattr(taskv3_tools, "_realm_document_id", fake_realm_document_id)
+
+ _step, _task, loop_mock, _post = await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ must_get_working_page_side_effect=[pinned],
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ assert loop_mock.await_args is not None
+ identity = loop_mock.await_args.kwargs["document_identity"]
+ main_only = await identity()
+ await _record_frame_work(pinned, acted, "#email", "filled", "doc-a")
+ baseline = await identity()
+ assert baseline != main_only
+
+ # Observable on the page but never acted in: it churns freely without moving the identity.
+ unrelated.evaluate = AsyncMock(return_value="ad-nonce-b")
+ unrelated.url = "https://ads.example/other-frame"
+ assert await identity() == baseline
+
+ acted.evaluate = AsyncMock(return_value="frame-nonce-b")
+ assert await identity() != baseline
+ acted.evaluate = AsyncMock(return_value="frame-nonce-a")
+ acted.url = "https://example.com/done"
+ assert await identity() != baseline
+ acted.url = "https://example.com/form"
+ assert await identity() == baseline
+
+ # A same-URL reload that PREDEFINES `window.__skyvern_doc_nonce` (or patches Math.random) forges
+ # the nonce steady, but the browser-owned loaderId behind `_realm_document_id` still moves --
+ # the combined identity must still change (SKY-17372).
+ loader_ids[id(acted)] = "loader-b"
+ assert await identity() != baseline
+ loader_ids[id(acted)] = "loader-a"
+ assert await identity() == baseline
+
+ acted.is_detached = MagicMock(return_value=True)
+ assert await identity() != baseline
+ acted.is_detached = MagicMock(return_value=False)
+ acted.evaluate = AsyncMock(side_effect=RuntimeError("execution context was destroyed"))
+ with pytest.raises(RuntimeError):
+ await identity()
+
+
+def _make_pinned_page_with_acted_frame() -> tuple[Any, Any]:
+ main_frame = object()
+ acted = MagicMock()
+ acted.url = "https://example.com/form"
+ acted.is_detached = MagicMock(return_value=False)
+ acted.evaluate = AsyncMock(return_value="frame-nonce-a")
+ pinned = MagicMock()
+ pinned.is_closed = MagicMock(return_value=False)
+ pinned.url = "https://example.com/"
+ pinned.main_frame = main_frame
+ pinned.frames = [main_frame, acted]
+ pinned.evaluate = AsyncMock(return_value="main-nonce")
+ return pinned, acted
+
+
+async def test_execute_task_v3_document_identity_raises_when_a_realm_is_unidentifiable(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # `_realm_document_id` marks a realm it could not name uniquely (e.g. sibling frames sharing a
+ # url) with `_UNIDENTIFIABLE_DOCUMENT` rather than raising itself. `_document_identity` must turn
+ # that marker into a raise of its own: a constant marker at both the before and after read would
+ # otherwise compare equal, and the finish tool's re-ask conversion veto would fail open on a
+ # document it never actually managed to identify (SKY-17372).
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=[])
+
+ # Case 1: an acted-in frame's realm cannot be named uniquely; the main page alone is fine, so the
+ # raise has to come from the loop over the acted-in frames, not just the main-realm read.
+ pinned, acted = _make_pinned_page_with_acted_frame()
+
+ async def frame_unidentifiable(target: Any) -> str:
+ if target is acted:
+ return f"{target.url}|{taskv3_tools._UNIDENTIFIABLE_DOCUMENT}"
+ return f"{target.url}|loader-a"
+
+ monkeypatch.setattr(taskv3_tools, "_realm_document_id", frame_unidentifiable)
+ _step, _task, loop_mock, _post = await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ must_get_working_page_side_effect=[pinned],
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ assert loop_mock.await_args is not None
+ identity = loop_mock.await_args.kwargs["document_identity"]
+ # Before the frame is ever acted in, the identity never looks at it and reads clean.
+ assert await identity() is not None
+ await _record_frame_work(pinned, acted, "#email", "filled", "doc-a")
+ with pytest.raises(RuntimeError):
+ await identity()
+
+ # Case 2: the MAIN page's own realm cannot be named uniquely -- read unconditionally, with no
+ # frame ever acted in.
+ pinned2, _acted2 = _make_pinned_page_with_acted_frame()
+
+ async def main_page_unidentifiable(target: Any) -> str:
+ return f"{target.url}|{taskv3_tools._UNIDENTIFIABLE_DOCUMENT}"
+
+ monkeypatch.setattr(taskv3_tools, "_realm_document_id", main_page_unidentifiable)
+ _step, _task, loop_mock2, _post = await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ must_get_working_page_side_effect=[pinned2],
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ assert loop_mock2.await_args is not None
+ identity2 = loop_mock2.await_args.kwargs["document_identity"]
+ with pytest.raises(RuntimeError):
+ await identity2()
+
+
@pytest.mark.asyncio
async def test_execute_task_v3_page_fingerprint_peeks_without_recovery(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.py
index 9db67d025..5740f91af 100644
--- a/tests/unit/test_browser_effect_sink_enumeration.py
+++ b/tests/unit/test_browser_effect_sink_enumeration.py
@@ -15,7 +15,7 @@ _CANDIDATE_METHODS = frozenset(
)
_DISCOVERED_BROWSER_API_CALLS = {
- "skyvern/forge/agent.py": Counter({"close": 1, "evaluate": 3, "new_page": 1}),
+ "skyvern/forge/agent.py": Counter({"close": 1, "evaluate": 4, "new_page": 1}),
"skyvern/forge/agent_functions.py": Counter({"close": 1, "scroll_into_view_if_needed": 1}),
"skyvern/webeye/actions/handler.py": Counter(
{
@@ -81,7 +81,10 @@ _EVALUATE_CALLERS = {
# _page_fingerprint samples TWICE: the page's own document, and each readable child frame. Both
# are the same read-only probe; the second exists because the settle check is a live gate and a
# main-frame-only sample reads a page whose child frame is still rendering as settled (SKY-14657).
- "skyvern/forge/agent.py": Counter({"_page_fingerprint": 2, "_page_probe": 1}),
+ # _document_identity reads the same nonce in the main document and each acted-in child frame, via
+ # the shared _realm_part helper (one evaluate call site, invoked once per realm) that also reads
+ # each realm's browser-owned _realm_document_id (SKY-17372).
+ "skyvern/forge/agent.py": Counter({"_page_fingerprint": 2, "_page_probe": 1, "_realm_part": 1}),
"skyvern/webeye/actions/multi_field_totp.py": Counter(
{
"_multi_field_totp_frame_gone": 1,
@@ -236,21 +239,21 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:
}
assert observed == _DISCOVERED_BROWSER_API_CALLS
- assert sum(sum(methods.values()) for methods in observed.values()) == 183
+ assert sum(sum(methods.values()) for methods in observed.values()) == 184
handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)
classified_non_browser = Counter(
{signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES}
)
assert classified_non_browser == _NON_BROWSER_CANDIDATES
assert sum(_NON_BROWSER_CANDIDATES.values()) == 7
- assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 176
+ assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 177
def test_every_raw_evaluate_call_is_classified() -> None:
observed = {path: callers for path in _owned_source_paths() if (callers := _callers_for_method(path, "evaluate"))}
assert observed == _EVALUATE_CALLERS
- assert sum(sum(callers.values()) for callers in observed.values()) == 36
+ assert sum(sum(callers.values()) for callers in observed.values()) == 37
def test_every_cdp_dispatch_is_classified_by_exact_command() -> None:
diff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.py
index 191840f1a..e2a03731f 100644
--- a/tests/unit/test_taskv3_engine.py
+++ b/tests/unit/test_taskv3_engine.py
@@ -632,35 +632,45 @@ async def test_engine_wires_failure_evidence_gate() -> None:
@pytest.mark.asyncio
-async def test_engine_forwards_the_page_probe_and_withholds_it_from_page_free_runs(
+async def test_engine_forwards_the_page_probe_and_document_identity_and_withholds_them_from_page_free_runs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- # Drop this kwarg anywhere on the way in and the loop classifies every unflagged error as
- # non-poisoning -- the fail-open state -- with every loop-level test still green.
+ # Drop either kwarg on the way in and a guard fails open with every loop-level test still green: the
+ # loop classifies every unflagged error as non-poisoning, and the finish tool converts a re-ask
+ # without comparing the document before and after its judge.
from skyvern.forge.taskv3 import engine as engine_mod
from skyvern.forge.taskv3.loop import LoopOutcome
- captured: list[object] = []
+ captured: list[tuple[object, object]] = []
+ finish_identities: list[object] = []
+ real_make = engine_mod.make_finish_tool
+
+ def _capture_finish(*args: Any, **kwargs: Any) -> Any:
+ finish_identities.append(kwargs.get("document_identity"))
+ return real_make(*args, **kwargs)
async def _capture(**kwargs: object) -> LoopOutcome:
- captured.append(kwargs.get("page_probe"))
+ captured.append((kwargs.get("page_probe"), finish_identities[-1]))
return LoopOutcome(status="completed", reason="ok")
async def probe() -> str | None:
return "doc-1"
+ async def identity() -> str | None:
+ return "doc-1|frame-1"
+
+ monkeypatch.setattr(engine_mod, "make_finish_tool", _capture_finish)
monkeypatch.setattr(engine_mod, "run_agent_tool_loop", _capture)
- await run_task_v3_agent_loop(
- page_provider=_fixed_page_provider(_FakePage()), llm_caller=_ScriptedCaller([]), goal="x", page_probe=probe
- )
- await run_task_v3_agent_loop(
- page_provider=_fixed_page_provider(_FakePage()),
- llm_caller=_ScriptedCaller([]),
- goal="x",
- page_probe=probe,
- page_free=True,
- )
- assert captured == [probe, None]
+ for page_free in (False, True):
+ await run_task_v3_agent_loop(
+ page_provider=_fixed_page_provider(_FakePage()),
+ llm_caller=_ScriptedCaller([]),
+ goal="x",
+ page_probe=probe,
+ document_identity=identity,
+ page_free=page_free,
+ )
+ assert captured == [(probe, identity), (None, None)]
@pytest.mark.asyncio
diff --git a/tests/unit/test_taskv3_goal_check.py b/tests/unit/test_taskv3_goal_check.py
index 28714a411..86bd4c087 100644
--- a/tests/unit/test_taskv3_goal_check.py
+++ b/tests/unit/test_taskv3_goal_check.py
@@ -1,7 +1,9 @@
from __future__ import annotations
import asyncio
+import itertools
import re
+import time
from pathlib import Path
from typing import Any
@@ -11,6 +13,7 @@ from structlog.testing import capture_logs
from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormat
from skyvern.forge.sdk.workflow.context_manager import RANDOM_SECRET_ID_PREFIX
+from skyvern.forge.taskv3 import loop as taskv3_loop
from skyvern.forge.taskv3.goal_check import (
GoalJudge,
GoalVerdict,
@@ -34,6 +37,7 @@ from skyvern.forge.taskv3.loop import (
run_agent_tool_loop,
)
from skyvern.utils.secret_redaction import REDACTED_SECRET_PLACEHOLDER, redact_secrets_from_text
+from tests.unit.scoped_asyncio import ScopedAsyncio
from tests.unit.test_taskv3_loop import _ScriptedCaller
PAGE_TEXT = "Order summary\nShipping address: 12 Example Road\nStatus: Draft"
@@ -850,6 +854,207 @@ async def test_a_grounded_reask_completes_once_and_a_completed_side_veto_keeps_t
assert len(spy.calls) == 1
+class _SettleWindowPage:
+ """Fingerprint pairs settle on the samples `settles` marks; the document identity moves once `moves_after`
+ settle samples have been taken (None: never)."""
+
+ def __init__(self, settles: tuple[bool, ...], moves_after: int | None) -> None:
+ self.settles = settles
+ self.moves_after = moves_after
+ self.fingerprint_calls = 0
+
+ async def fingerprint(self) -> str | None:
+ sample = self.fingerprint_calls // 2
+ self.fingerprint_calls += 1
+ if sample < len(self.settles) and self.settles[sample]:
+ return f"settled-{sample}"
+ return f"fp-{self.fingerprint_calls}"
+
+ async def identity(self) -> str | None:
+ moved = self.moves_after is not None and self.fingerprint_calls // 2 >= self.moves_after
+ return "https://example.com/next|nonce-b" if moved else "https://example.com/form|nonce-a"
+
+
+async def _conversion_veto_for(**overrides: Any) -> tuple[str | None, bool]:
+ kwargs = _gate_kwargs(pending=False, blocked=False, unsettled=False, goal=None)
+ kwargs.update(overrides)
+ finish = make_finish_tool(**kwargs, unlisted_reask=_ReaskSpy())
+ with capture_logs() as logs:
+ result = await finish.handler({"status": "terminated", "reason": "no PIN screen"})
+ (line,) = (log for log in logs if log["event"] == "taskv3 finish unlisted reask")
+ completed = result.data is not None and result.data["status"] == "completed"
+ assert completed == line["converted"]
+ return line["veto"], line["converted"]
+
+
+async def test_a_conversion_is_vetoed_by_the_settle_gate_only_when_the_document_changes_identity() -> None:
+ """With a readable identity, a conversion is refused by the settle gate only when the document changed
+ across the window, settled or not. An unchanged document is never refused."""
+ mismatches = []
+ for max_settle_deferrals in range(4):
+ window = max_settle_deferrals + 1
+ for settles in itertools.product([False, True], repeat=window):
+ for moves_after in [None, *range(1, window + 1)]:
+ page = _SettleWindowPage(settles, moves_after)
+ veto, converted = await _conversion_veto_for(
+ page_fingerprint=page.fingerprint,
+ document_identity=page.identity,
+ max_settle_deferrals=max_settle_deferrals,
+ )
+ samples_taken = settles.index(True) + 1 if any(settles) else window
+ moved = moves_after is not None and samples_taken >= moves_after
+ expected = "navigating" if moved else None
+ if veto != expected or converted != (expected is None):
+ mismatches.append((max_settle_deferrals, settles, moves_after, veto, converted))
+ assert mismatches == []
+
+
[email protected]("case", ["no_identity", "cancel", "cancel_raises"])
+async def test_a_conversion_on_a_page_that_never_settles_fails_closed_without_an_identity_or_on_cancel(
+ case: str,
+) -> None:
+ page = _SettleWindowPage(settles=(), moves_after=None)
+
+ async def should_cancel() -> bool:
+ # The veto's own first gate runs before any sample; only a cancel inside the window is under test.
+ if not case.startswith("cancel") or page.fingerprint_calls < 2:
+ return False
+ if case == "cancel_raises":
+ raise RuntimeError("cancel store unreachable")
+ return True
+
+ veto, converted = await _conversion_veto_for(
+ page_fingerprint=page.fingerprint,
+ document_identity=None if case == "no_identity" else page.identity,
+ should_cancel=should_cancel,
+ )
+
+ assert (veto, converted) == ("canceled" if case.startswith("cancel") else "unsettled", False)
+
+
[email protected]("settles", [(), (True,)])
[email protected]("read", ["before", "after"])
[email protected]("failure", ["none", "raises"])
+async def test_an_unreadable_document_identity_vetoes_a_conversion_even_on_a_settled_page(
+ settles: tuple[bool, ...], read: str, failure: str
+) -> None:
+ page = _SettleWindowPage(settles=settles, moves_after=None)
+ reads = 0
+
+ async def identity() -> str | None:
+ nonlocal reads
+ reads += 1
+ if (reads == 1) != (read == "before"):
+ return await page.identity()
+ if failure == "raises":
+ raise RuntimeError("execution context destroyed")
+ return None
+
+ veto, converted = await _conversion_veto_for(page_fingerprint=page.fingerprint, document_identity=identity)
+
+ assert (veto, converted) == ("identity_unreadable", False)
+
+
+async def test_a_navigation_during_the_judge_vetoes_a_conversion_even_once_the_new_page_settles() -> None:
+ page = _SettleWindowPage(settles=(True,), moves_after=None)
+
+ class _NavigatingJudge(_ReaskSpy):
+ async def __call__(self, status: NonCompletedStatus, reason: str) -> UnlistedReask:
+ page.moves_after = 0
+ return await super().__call__(status, reason)
+
+ finish = make_finish_tool(
+ **_gate_kwargs(pending=False, blocked=False, unsettled=False, goal=None)
+ | {"page_fingerprint": page.fingerprint, "document_identity": page.identity},
+ unlisted_reask=_NavigatingJudge(),
+ )
+ with capture_logs() as logs:
+ result = await finish.handler({"status": "terminated", "reason": "no PIN screen"})
+
+ assert result.data is not None and result.data["status"] == "terminated"
+ (line,) = (log for log in logs if log["event"] == "taskv3 finish unlisted reask")
+ assert (line["veto"], line["settled"], line["settle_rounds"]) == ("navigating", True, 1)
+
+
+class _ScopedClock:
+ def __init__(self) -> None:
+ self.now = 0.0
+
+ def monotonic(self) -> float:
+ return self.now
+
+ def __getattr__(self, name: str) -> object:
+ return getattr(time, name)
+
+
+async def test_a_deadline_inside_the_conversion_settle_window_vetoes_without_sleeping_past_it(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ clock = _ScopedClock()
+
+ async def sleep(seconds: float) -> None:
+ clock.now += seconds
+
+ monkeypatch.setattr(taskv3_loop, "time", clock)
+ monkeypatch.setattr(taskv3_loop, "asyncio", ScopedAsyncio(sleep=sleep))
+ page = _SettleWindowPage(settles=(), moves_after=None)
+
+ veto, converted = await _conversion_veto_for(
+ page_fingerprint=page.fingerprint,
+ document_identity=page.identity,
+ max_settle_deferrals=3,
+ settle_wait_seconds=4.0,
+ deadline_at=10.0,
+ )
+
+ assert (veto, converted) == ("deadline", False)
+ assert clock.now == 10.0
+
+
+async def test_a_deadline_that_elapses_during_the_judge_still_vetoes_as_deadline(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """_settled reads a missing sample as settled (its own docstring), so a deadline that elapses
+ between the pre-judge identity read and the settle loop's first round must still surface as
+ "deadline" -- not "identity_unreadable" from the closing read finding nothing left to sample."""
+ clock = _ScopedClock()
+
+ async def sleep(seconds: float) -> None:
+ clock.now += seconds
+
+ monkeypatch.setattr(taskv3_loop, "time", clock)
+ monkeypatch.setattr(taskv3_loop, "asyncio", ScopedAsyncio(sleep=sleep))
+ page = _SettleWindowPage(settles=(), moves_after=None)
+
+ class _SlowJudge(_ReaskSpy):
+ async def __call__(self, status: NonCompletedStatus, reason: str) -> UnlistedReask:
+ clock.now = 10.0 # the deadline elapses while the judge call is in flight
+ return await super().__call__(status, reason)
+
+ finish = make_finish_tool(
+ **_gate_kwargs(pending=False, blocked=False, unsettled=False, goal=None)
+ | {
+ "page_fingerprint": page.fingerprint,
+ "document_identity": page.identity,
+ "pending_marker": None,
+ "deadline_at": 10.0,
+ },
+ unlisted_reask=_SlowJudge(),
+ )
+ with capture_logs() as logs:
+ result = await finish.handler({"status": "terminated", "reason": "no PIN screen"})
+
+ assert result.data is not None and result.data["status"] == "terminated"
+ (line,) = (log for log in logs if log["event"] == "taskv3 finish unlisted reask")
+ assert line["veto"] == "deadline"
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("converts", [True, False])
async def test_a_run_canceled_during_the_reask_is_left_to_the_loops_cancellation(converts: bool) -> None: