Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Show that Copilot questions can be answered in your own words (SKY-17389) (#8713)
Shuchang Zhengd6e13af4e4f3
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex 203fc5a45..8ee196d03 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -6571,6 +6571,7 @@ export function WorkflowCopilotChat({inputValue,setInputValue,);+ const trayPart = trayQuestion?.parts[questionStepper.index];// handleSend is memoised and outlives a render; the stepper is rebuilt on every keystroke.const questionStepperRef = useRef(questionStepper);questionStepperRef.current = questionStepper;@@ -10426,9 +10427,11 @@ export function WorkflowCopilotChat({isLoading,isWaitingForLiveBrowser,latestTurnIsAsk,- askPartHasChoices:- (trayQuestion?.parts[questionStepper.index]?.choices.length ??- 0) > 0,+ askPartChoices: !trayPart?.choices.length+ ? "none"+ : questionStepper.choices[trayPart.part_id] === undefined+ ? "unpicked"+ : "picked",})}value={inputValue}onChange={(e) => setInputValue(e.target.value)}diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/QuestionTray.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/QuestionTray.tsxindex 0e8c39e8c..b527a0b53 100644--- a/skyvern-frontend/src/routes/workflows/copilot/cards/QuestionTray.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/QuestionTray.tsx@@ -186,11 +186,18 @@ export function QuestionTray({);})}</div>- ) : (+ ) : null}+ {/* Typing does not clear a picked choice, so the invitation to answer instead of the+ choices only shows while none is picked. */}+ {part.choices.length === 0 ? (<p className="text-xs text-muted-foreground">Type your answer in the message box below.</p>- )}+ ) : stepper.choices[part.part_id] === undefined ? (+ <p className="text-xs text-muted-foreground">+ None of these fit? Type your own answer in the message box below.+ </p>+ ) : null}</div>) : null}<divdiff --git a/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.test.ts b/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.test.tsindex 0c489cf5a..a2ce50c60 100644--- a/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.test.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.test.ts@@ -18,9 +18,16 @@ describe("composerPlaceholder", () => {composerPlaceholder({...base,latestTurnIsAsk: true,- askPartHasChoices: true,+ askPartChoices: "unpicked",}),- ).toBe("Add details (optional)…");+ ).toBe("Or type your own…");+ expect(+ composerPlaceholder({+ ...base,+ latestTurnIsAsk: true,+ askPartChoices: "picked",+ }),+ ).toBe("Add details…");});it("returns to the standing invitation once the ask is answered", () => {diff --git a/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.ts b/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.tsindex 302ea810d..2873a99b3 100644--- a/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/composerPlaceholder.ts@@ -5,15 +5,16 @@ export function composerPlaceholder({isLoading,isWaitingForLiveBrowser,latestTurnIsAsk,- askPartHasChoices = false,+ askPartChoices = "none",}: {// Whether a send adds to the queued message or replaces it (a programmatic one).queuedPrompt: "add" | "replace" | null;isLoading: boolean;isWaitingForLiveBrowser: boolean;latestTurnIsAsk: boolean;- // Whether the question on screen offers choices; without them the composer is the answer.- askPartHasChoices?: boolean;+ // The question on screen's choices: without any the composer is the answer, and typing never+ // clears a picked one, so text sent after a pick goes alongside it rather than replacing it.+ askPartChoices?: "none" | "unpicked" | "picked";}): string {if (queuedPrompt === "add") return "Add to the queued message…";if (queuedPrompt === "replace") return "Type to replace the queued message…";@@ -22,6 +23,10 @@ export function composerPlaceholder({// While a question is pending the composer is the text field for the question on screen, so// it says so rather than inviting an unrelated new request.if (latestTurnIsAsk)- return askPartHasChoices ? "Add details (optional)…" : "Type your answer…";+ return askPartChoices === "picked"+ ? "Add details…"+ : askPartChoices === "unpicked"+ ? "Or type your own…"+ : "Type your answer…";return "Ask Copilot to build or change your workflow…";}diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 03c3bba38..31861c2e2 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -2259,7 +2259,13 @@ class ForgeAgent:)from skyvern.forge.taskv3.loop import DEFAULT_MAX_SETTLE_DEFERRALS, CompletionBlocker, CompletionProbefrom skyvern.forge.taskv3.opaque_refs import mask_opaque_urls- from skyvern.forge.taskv3.tools import _observable_child_frames, _recorded_work_frames, pending_marker+ from skyvern.forge.taskv3.tools import (+ _document_unidentifiable,+ _observable_child_frames,+ _realm_document_id,+ _recorded_work_frames,+ pending_marker,+ )from skyvern.forge.taskv3.workflow_position import PreviousBlockHandoff, select_previous_blockfrom skyvern.utils.token_counter import approx_count_tokens@@ -2965,6 +2971,26 @@ class ForgeAgent:nonce = await peek.evaluate(_DOCUMENT_NONCE_JS)return f"{peek.url}|{nonce}"+ # _page_probe plus every child document the run acted in, where a form can submit with the main+ # document unchanged. Each realm pairs the browser-owned loaderId (unforgeable) with the nonce (which still+ # catches a reload if the CDP session degrades to the url); a realm that cannot be identified raises, and+ # the finish gate reads that as an unreadable identity.+ async def _document_identity() -> str | None:+ peek = await _fingerprint_page()+ if peek is None:+ return None++ async def _realm_part(realm: Any) -> str:+ realm_id = await _realm_document_id(realm)+ if _document_unidentifiable(realm_id):+ raise RuntimeError("taskv3 document identity: a realm could not be identified")+ return f"{realm_id}|{await realm.evaluate(_DOCUMENT_NONCE_JS)}"++ parts = [await _realm_part(peek)]+ for frame in _recorded_work_frames(peek):+ parts.append("detached" if frame.is_detached() else await _realm_part(frame))+ return "\n".join(parts)+async def _reload_page() -> None:# Observed by the action policy like the legacy internal refresh; the loop records it in# the action round. A bare task pins one page, so it is passed explicitly.@@ -3135,6 +3161,7 @@ class ForgeAgent:page_free=page_free_validation,page_fingerprint=_page_fingerprint,page_probe=_page_probe,+ document_identity=_document_identity,reload_page=_reload_page,restore_page_url=_restore_page_url,download_attempts=_download_attempts,diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.pyindex 6b1654978..efb2353f8 100644--- a/skyvern/forge/taskv3/engine.py+++ b/skyvern/forge/taskv3/engine.py@@ -300,6 +300,7 @@ async def run_task_v3_agent_loop(label_secret_values: Callable[[], Collection[str]] | None = None,login_identifier_tokens: Callable[[], Collection[str]] | None = None,page_probe: Callable[[], Awaitable[str | None]] | None = None,+ document_identity: Callable[[], Awaitable[str | None]] | None = None,reload_page: Callable[[], Awaitable[None]] | None = None,restore_page_url: Callable[[Any, str], Awaitable[None]] | None = None,download_attempts: Callable[[], int | None] | None = None,@@ -333,8 +334,9 @@ async def run_task_v3_agent_loop(for a bounded re-verification turn; without one, pre-finish re-verification is prompt guidanceonly. `max_settle_deferrals=0` disables that completed-side re-verification while leaving thefailure-evidence gate, which shares the sampler, intact. `page_probe` is a separate sampler (URL- plus fingerprint) the loop uses to detect whether a failed batched call moved the page; a- page-free run has no page to probe."""+ plus document nonce) the loop uses to detect whether a failed batched call moved the page.+ `document_identity` also covers the child frames the run acted in; the finish tool uses it to veto a re-ask+ conversion whose document changed. A page-free run has no page to probe."""loop_started_at = time.monotonic()# Presigned file URLs in the payload carry an HMAC token the model would otherwise have to# retype verbatim into a tool call; masking them here and resolving inside the tool handlers@@ -555,6 +557,7 @@ async def run_task_v3_agent_loop(goal_check=_goal_check if goal_check_on else None,goal_check_enforce=goal_check_enforce,unlisted_reask=_unlisted_reask if reask_on else None,+ document_identity=None if page_free else document_identity,)tools = browser_tools + (extra_tools or []) + [finish_tool]# The COMPLETE dispatch list, not just the browser tools: auth / captcha / code tools and finishdiff --git a/skyvern/forge/taskv3/goal_check.py b/skyvern/forge/taskv3/goal_check.pyindex 30bd526e8..8eec252d4 100644--- a/skyvern/forge/taskv3/goal_check.py+++ b/skyvern/forge/taskv3/goal_check.py@@ -438,6 +438,9 @@ class UnlistedReask:reason: str = ""veto: str | None = Noneconverted: bool = False+ # The conversion's settle window: None when it did not run; rounds are fingerprint pairs.+ settled: bool | None = None+ settle_rounds: int = 0# The goal check's verdict on the converted completion, when a goal judge is wired.goal_check_verdict: str | None = Nonellm_key: str | None = Nonediff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex a3cad6307..f74abb4a2 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -2313,6 +2313,7 @@ def make_finish_tool(goal_check: Callable[[], Awaitable[GoalVerdict]] | None = None,goal_check_enforce: bool = False,unlisted_reask: UnlistedReaskCheck | None = None,+ document_identity: Callable[[], Awaitable[str | None]] | None = None,) -> ToolSpec:"""`page_fingerprint` samples an opaque fingerprint of the page's rendered content (None when nopage is available). A finish(completed) is deferred (bounded by `max_settle_deferrals`, then@@ -2347,7 +2348,9 @@ def make_finish_tool(`unlisted_reask` asks once per run whether a failed or terminated verdict that is about to stand onlymissed a screen the site skipped. A grounded yes becomes completed unless a completed-side gate vetoes it;- it never gives a turn back, and a completed verdict cannot reach it."""+ it never gives a turn back, and a completed verdict cannot reach it. Its settle gate samples as many times as+ the completed path would defer. When `document_identity` is wired, it is read before the judge and again after+ the settle window, and a conversion whose identity changed or could not be read is refused, settled or not."""deferrals = 0failure_deferrals = 0goal_check_held = False@@ -2408,7 +2411,7 @@ def make_finish_tool(return False # defer: the loop's cancellation check ends the run before another turnreturn first == await _bounded_fingerprint()- async def _conversion_veto(result: UnlistedReask) -> str | None:+ async def _conversion_veto(result: UnlistedReask, identity_before: str | None) -> str | None:"""The completed-side gates, as vetoes only: the model never claimed completion, so none may hold."""try:if should_cancel is not None and await should_cancel():@@ -2433,10 +2436,34 @@ def make_finish_tool(except Exception:return "verification_blocker"if page_fingerprint is not None:- try:- if not await _settled():- return "unsettled"- except Exception:+ if document_identity is not None and identity_before is None:+ return "identity_unreadable"+ settled = False+ for _ in range(max_settle_deferrals + 1):+ result.settle_rounds += 1+ try:+ settled = await _settled()+ except Exception:+ pass # as on the completed path: an unreadable sample is an unsettled one+ result.settled = settled+ if settled:+ break+ try:+ if should_cancel is not None and await should_cancel():+ return "canceled"+ except Exception:+ return "canceled"+ if deadline_at is not None and deadline_at - time.monotonic() <= 0:+ return "deadline"+ if deadline_at is not None and deadline_at - time.monotonic() <= 0:+ return "deadline"+ if document_identity is not None:+ identity_after = await _sample_probe(document_identity, deadline_at=deadline_at)+ if identity_after is None:+ return "identity_unreadable"+ if identity_before != identity_after:+ return "navigating"+ elif not settled:return "unsettled"if goal_check is not None:try:@@ -2453,13 +2480,19 @@ def make_finish_tool(async def _reask(original: NonCompletedStatus, args: dict[str, Any]) -> ToolResult | None:assert unlisted_reask is not Nonereason = args.get("reason") or ""+ # Before the judge: a navigation that commits while it runs must not become the baseline.+ identity_before = (+ await _sample_probe(document_identity, deadline_at=deadline_at)+ if document_identity is not None and page_fingerprint is not None+ else None+ )try:result: UnlistedReask | None = await unlisted_reask(original, reason)except Exception:LOG.warning("taskv3 unlisted reask failed; keeping the verdict", exc_info=True)result = Noneif result is not None and result.converts:- result.veto = await _conversion_veto(result)+ result.veto = await _conversion_veto(result, identity_before)result.converted = result.veto is NoneLOG.info("taskv3 finish unlisted reask",@@ -2469,6 +2502,8 @@ def make_finish_tool(converts=result.converts if result is not None else False,converted=result.converted if result is not None else False,veto=result.veto if result is not None else None,+ settled=result.settled if result is not None else None,+ settle_rounds=result.settle_rounds if result is not None else 0,goal_check_verdict=result.goal_check_verdict if result is not None else None,llm_key=result.llm_key if result is not None else None,skipped_reason=result.skipped_reason if result is not None else "reask_error",diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex 250dec4b5..72b010407 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -9261,8 +9261,10 @@ _TEXT_DELTA_MAX_FAILED_READS = 2_TEXT_DELTA_TEXT_MAX_CHARS = 2_000_000# One evaluate, so the document identity and the text it is paired with come from the same document.# The nonce is the one the loop's page probe keys on; [seed, limit] arrive as an argument, never as source.-# Unlike `_realm_document_id` the nonce is page-forgeable, which is acceptable because it gates nothing-# (a forged one only mis-scopes which lines are reported) and it saves a CDP call per action.+# Unlike `_realm_document_id` the nonce is page-forgeable. That is acceptable: it saves a CDP call per action.+# The page probe compares it for change, so a page that pre-seeds or forges the nonce can hide a reload from+# that probe. The re-ask conversion's identity pairs it with `_realm_document_id`'s loaderId instead, which the+# page cannot forge._TEXT_DELTA_READ_JS = (r"""(args) => {let nonce = window.__skyvern_doc_nonce;diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex 79af591d2..3cd0bca30 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -64,6 +64,7 @@ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatusfrom skyvern.forge.sdk.workflow.page_derived_templates import CLOSE as PAGE_DERIVED_CLOSEfrom skyvern.forge.sdk.workflow.page_derived_templates import OPEN as PAGE_DERIVED_OPENfrom skyvern.forge.taskv3 import engine as taskv3_engine+from skyvern.forge.taskv3 import tools as taskv3_toolsfrom skyvern.forge.taskv3.auth_tools import VerificationFailure, VerificationStatefrom skyvern.forge.taskv3.engine import DEFAULT_MAX_SETTLE_DEFERRALS, MIN_ACTION_STEPS, run_task_v3_agent_loopfrom skyvern.forge.taskv3.goal_composition import PAGE_DATA_NOTE@@ -84,7 +85,7 @@ from skyvern.forge.taskv3.run_arms import (TYPE_COORDINATE_CLICK_FLAG,run_arm_enabled,)-from skyvern.forge.taskv3.tools import PageProvider+from skyvern.forge.taskv3.tools import PageProvider, _record_frame_workfrom skyvern.schemas.runs import RunEnginefrom skyvern.schemas.workflows import BlockStatus, BlockTypefrom skyvern.utils.secret_redaction import REDACTED_SECRET_PLACEHOLDER@@ -5146,6 +5147,154 @@ async def test_execute_task_v3_page_fingerprint_samples_child_frames(monkeypatchassert second != first+async def test_execute_task_v3_document_identity_changes_with_an_acted_in_frame_only(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # A form can submit inside a frame with the main document unchanged; the re-ask conversion's identity+ # must see that frame, and only the frames the run acted in (an unrelated ad frame churns freely).+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=[])+ main_frame = object()+ acted = MagicMock()+ acted.url = "https://example.com/form"+ acted.is_detached = MagicMock(return_value=False)+ acted.evaluate = AsyncMock(return_value="frame-nonce-a")+ unrelated = MagicMock()+ unrelated.url = "https://ads.example/frame"+ unrelated.is_detached = MagicMock(return_value=False)+ unrelated.evaluate = AsyncMock(return_value="ad-nonce-a")+ pinned = MagicMock()+ pinned.is_closed = MagicMock(return_value=False)+ pinned.url = "https://example.com/"+ pinned.main_frame = main_frame+ pinned.frames = [main_frame, acted, unrelated]+ pinned.evaluate = AsyncMock(return_value="main-nonce")++ # A MagicMock realm has no real Playwright shape for `_realm_document_id`'s own CDP/frame-tree+ # plumbing to key on, so the loaderId per realm is faked directly here, keyed on which mock object+ # the identity closure passes in. `loader_ids` maps a realm to its current fake loaderId.+ loader_ids: dict[int, str] = {}++ async def fake_realm_document_id(target: Any) -> str:+ return f"{target.url}|{loader_ids.get(id(target), 'loader-a')}"++ monkeypatch.setattr(taskv3_tools, "_realm_document_id", fake_realm_document_id)++ _step, _task, loop_mock, _post = await _run_execute_task_v3(+ monkeypatch,+ outcome,+ must_get_working_page_side_effect=[pinned],+ data_extraction_goal=None,+ extracted_information_schema=None,+ )+ assert loop_mock.await_args is not None+ identity = loop_mock.await_args.kwargs["document_identity"]+ main_only = await identity()+ await _record_frame_work(pinned, acted, "#email", "filled", "doc-a")+ baseline = await identity()+ assert baseline != main_only++ # Observable on the page but never acted in: it churns freely without moving the identity.+ unrelated.evaluate = AsyncMock(return_value="ad-nonce-b")+ unrelated.url = "https://ads.example/other-frame"+ assert await identity() == baseline++ acted.evaluate = AsyncMock(return_value="frame-nonce-b")+ assert await identity() != baseline+ acted.evaluate = AsyncMock(return_value="frame-nonce-a")+ acted.url = "https://example.com/done"+ assert await identity() != baseline+ acted.url = "https://example.com/form"+ assert await identity() == baseline++ # A same-URL reload that PREDEFINES `window.__skyvern_doc_nonce` (or patches Math.random) forges+ # the nonce steady, but the browser-owned loaderId behind `_realm_document_id` still moves --+ # the combined identity must still change (SKY-17372).+ loader_ids[id(acted)] = "loader-b"+ assert await identity() != baseline+ loader_ids[id(acted)] = "loader-a"+ assert await identity() == baseline++ acted.is_detached = MagicMock(return_value=True)+ assert await identity() != baseline+ acted.is_detached = MagicMock(return_value=False)+ acted.evaluate = AsyncMock(side_effect=RuntimeError("execution context was destroyed"))+ with pytest.raises(RuntimeError):+ await identity()+++def _make_pinned_page_with_acted_frame() -> tuple[Any, Any]:+ main_frame = object()+ acted = MagicMock()+ acted.url = "https://example.com/form"+ acted.is_detached = MagicMock(return_value=False)+ acted.evaluate = AsyncMock(return_value="frame-nonce-a")+ pinned = MagicMock()+ pinned.is_closed = MagicMock(return_value=False)+ pinned.url = "https://example.com/"+ pinned.main_frame = main_frame+ pinned.frames = [main_frame, acted]+ pinned.evaluate = AsyncMock(return_value="main-nonce")+ return pinned, acted+++async def test_execute_task_v3_document_identity_raises_when_a_realm_is_unidentifiable(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # `_realm_document_id` marks a realm it could not name uniquely (e.g. sibling frames sharing a+ # url) with `_UNIDENTIFIABLE_DOCUMENT` rather than raising itself. `_document_identity` must turn+ # that marker into a raise of its own: a constant marker at both the before and after read would+ # otherwise compare equal, and the finish tool's re-ask conversion veto would fail open on a+ # document it never actually managed to identify (SKY-17372).+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=[])++ # Case 1: an acted-in frame's realm cannot be named uniquely; the main page alone is fine, so the+ # raise has to come from the loop over the acted-in frames, not just the main-realm read.+ pinned, acted = _make_pinned_page_with_acted_frame()++ async def frame_unidentifiable(target: Any) -> str:+ if target is acted:+ return f"{target.url}|{taskv3_tools._UNIDENTIFIABLE_DOCUMENT}"+ return f"{target.url}|loader-a"++ monkeypatch.setattr(taskv3_tools, "_realm_document_id", frame_unidentifiable)+ _step, _task, loop_mock, _post = await _run_execute_task_v3(+ monkeypatch,+ outcome,+ must_get_working_page_side_effect=[pinned],+ data_extraction_goal=None,+ extracted_information_schema=None,+ )+ assert loop_mock.await_args is not None+ identity = loop_mock.await_args.kwargs["document_identity"]+ # Before the frame is ever acted in, the identity never looks at it and reads clean.+ assert await identity() is not None+ await _record_frame_work(pinned, acted, "#email", "filled", "doc-a")+ with pytest.raises(RuntimeError):+ await identity()++ # Case 2: the MAIN page's own realm cannot be named uniquely -- read unconditionally, with no+ # frame ever acted in.+ pinned2, _acted2 = _make_pinned_page_with_acted_frame()++ async def main_page_unidentifiable(target: Any) -> str:+ return f"{target.url}|{taskv3_tools._UNIDENTIFIABLE_DOCUMENT}"++ monkeypatch.setattr(taskv3_tools, "_realm_document_id", main_page_unidentifiable)+ _step, _task, loop_mock2, _post = await _run_execute_task_v3(+ monkeypatch,+ outcome,+ must_get_working_page_side_effect=[pinned2],+ data_extraction_goal=None,+ extracted_information_schema=None,+ )+ assert loop_mock2.await_args is not None+ identity2 = loop_mock2.await_args.kwargs["document_identity"]+ with pytest.raises(RuntimeError):+ await identity2()++@pytest.mark.asyncioasync def test_execute_task_v3_page_fingerprint_peeks_without_recovery(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.pyindex 9db67d025..5740f91af 100644--- a/tests/unit/test_browser_effect_sink_enumeration.py+++ b/tests/unit/test_browser_effect_sink_enumeration.py@@ -15,7 +15,7 @@ _CANDIDATE_METHODS = frozenset()_DISCOVERED_BROWSER_API_CALLS = {- "skyvern/forge/agent.py": Counter({"close": 1, "evaluate": 3, "new_page": 1}),+ "skyvern/forge/agent.py": Counter({"close": 1, "evaluate": 4, "new_page": 1}),"skyvern/forge/agent_functions.py": Counter({"close": 1, "scroll_into_view_if_needed": 1}),"skyvern/webeye/actions/handler.py": Counter({@@ -81,7 +81,10 @@ _EVALUATE_CALLERS = {# _page_fingerprint samples TWICE: the page's own document, and each readable child frame. Both# are the same read-only probe; the second exists because the settle check is a live gate and a# main-frame-only sample reads a page whose child frame is still rendering as settled (SKY-14657).- "skyvern/forge/agent.py": Counter({"_page_fingerprint": 2, "_page_probe": 1}),+ # _document_identity reads the same nonce in the main document and each acted-in child frame, via+ # the shared _realm_part helper (one evaluate call site, invoked once per realm) that also reads+ # each realm's browser-owned _realm_document_id (SKY-17372).+ "skyvern/forge/agent.py": Counter({"_page_fingerprint": 2, "_page_probe": 1, "_realm_part": 1}),"skyvern/webeye/actions/multi_field_totp.py": Counter({"_multi_field_totp_frame_gone": 1,@@ -236,21 +239,21 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:}assert observed == _DISCOVERED_BROWSER_API_CALLS- assert sum(sum(methods.values()) for methods in observed.values()) == 183+ assert sum(sum(methods.values()) for methods in observed.values()) == 184handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)classified_non_browser = Counter({signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES})assert classified_non_browser == _NON_BROWSER_CANDIDATESassert sum(_NON_BROWSER_CANDIDATES.values()) == 7- assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 176+ assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 177def test_every_raw_evaluate_call_is_classified() -> None:observed = {path: callers for path in _owned_source_paths() if (callers := _callers_for_method(path, "evaluate"))}assert observed == _EVALUATE_CALLERS- assert sum(sum(callers.values()) for callers in observed.values()) == 36+ assert sum(sum(callers.values()) for callers in observed.values()) == 37def test_every_cdp_dispatch_is_classified_by_exact_command() -> None:diff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.pyindex 191840f1a..e2a03731f 100644--- a/tests/unit/test_taskv3_engine.py+++ b/tests/unit/test_taskv3_engine.py@@ -632,35 +632,45 @@ async def test_engine_wires_failure_evidence_gate() -> None:@pytest.mark.asyncio-async def test_engine_forwards_the_page_probe_and_withholds_it_from_page_free_runs(+async def test_engine_forwards_the_page_probe_and_document_identity_and_withholds_them_from_page_free_runs(monkeypatch: pytest.MonkeyPatch,) -> None:- # Drop this kwarg anywhere on the way in and the loop classifies every unflagged error as- # non-poisoning -- the fail-open state -- with every loop-level test still green.+ # Drop either kwarg on the way in and a guard fails open with every loop-level test still green: the+ # loop classifies every unflagged error as non-poisoning, and the finish tool converts a re-ask+ # without comparing the document before and after its judge.from skyvern.forge.taskv3 import engine as engine_modfrom skyvern.forge.taskv3.loop import LoopOutcome- captured: list[object] = []+ captured: list[tuple[object, object]] = []+ finish_identities: list[object] = []+ real_make = engine_mod.make_finish_tool++ def _capture_finish(*args: Any, **kwargs: Any) -> Any:+ finish_identities.append(kwargs.get("document_identity"))+ return real_make(*args, **kwargs)async def _capture(**kwargs: object) -> LoopOutcome:- captured.append(kwargs.get("page_probe"))+ captured.append((kwargs.get("page_probe"), finish_identities[-1]))return LoopOutcome(status="completed", reason="ok")async def probe() -> str | None:return "doc-1"+ async def identity() -> str | None:+ return "doc-1|frame-1"++ monkeypatch.setattr(engine_mod, "make_finish_tool", _capture_finish)monkeypatch.setattr(engine_mod, "run_agent_tool_loop", _capture)- await run_task_v3_agent_loop(- page_provider=_fixed_page_provider(_FakePage()), llm_caller=_ScriptedCaller([]), goal="x", page_probe=probe- )- await run_task_v3_agent_loop(- page_provider=_fixed_page_provider(_FakePage()),- llm_caller=_ScriptedCaller([]),- goal="x",- page_probe=probe,- page_free=True,- )- assert captured == [probe, None]+ for page_free in (False, True):+ await run_task_v3_agent_loop(+ page_provider=_fixed_page_provider(_FakePage()),+ llm_caller=_ScriptedCaller([]),+ goal="x",+ page_probe=probe,+ document_identity=identity,+ page_free=page_free,+ )+ assert captured == [(probe, identity), (None, None)]@pytest.mark.asynciodiff --git a/tests/unit/test_taskv3_goal_check.py b/tests/unit/test_taskv3_goal_check.pyindex 28714a411..86bd4c087 100644--- a/tests/unit/test_taskv3_goal_check.py+++ b/tests/unit/test_taskv3_goal_check.py@@ -1,7 +1,9 @@from __future__ import annotationsimport asyncio+import itertoolsimport re+import timefrom pathlib import Pathfrom typing import Any@@ -11,6 +13,7 @@ from structlog.testing import capture_logsfrom skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormatfrom skyvern.forge.sdk.workflow.context_manager import RANDOM_SECRET_ID_PREFIX+from skyvern.forge.taskv3 import loop as taskv3_loopfrom skyvern.forge.taskv3.goal_check import (GoalJudge,GoalVerdict,@@ -34,6 +37,7 @@ from skyvern.forge.taskv3.loop import (run_agent_tool_loop,)from skyvern.utils.secret_redaction import REDACTED_SECRET_PLACEHOLDER, redact_secrets_from_text+from tests.unit.scoped_asyncio import ScopedAsynciofrom tests.unit.test_taskv3_loop import _ScriptedCallerPAGE_TEXT = "Order summary\nShipping address: 12 Example Road\nStatus: Draft"@@ -850,6 +854,207 @@ async def test_a_grounded_reask_completes_once_and_a_completed_side_veto_keeps_tassert len(spy.calls) == 1+class _SettleWindowPage:+ """Fingerprint pairs settle on the samples `settles` marks; the document identity moves once `moves_after`+ settle samples have been taken (None: never)."""++ def __init__(self, settles: tuple[bool, ...], moves_after: int | None) -> None:+ self.settles = settles+ self.moves_after = moves_after+ self.fingerprint_calls = 0++ async def fingerprint(self) -> str | None:+ sample = self.fingerprint_calls // 2+ self.fingerprint_calls += 1+ if sample < len(self.settles) and self.settles[sample]:+ return f"settled-{sample}"+ return f"fp-{self.fingerprint_calls}"++ async def identity(self) -> str | None:+ moved = self.moves_after is not None and self.fingerprint_calls // 2 >= self.moves_after+ return "https://example.com/next|nonce-b" if moved else "https://example.com/form|nonce-a"+++async def _conversion_veto_for(**overrides: Any) -> tuple[str | None, bool]:+ kwargs = _gate_kwargs(pending=False, blocked=False, unsettled=False, goal=None)+ kwargs.update(overrides)+ finish = make_finish_tool(**kwargs, unlisted_reask=_ReaskSpy())+ with capture_logs() as logs:+ result = await finish.handler({"status": "terminated", "reason": "no PIN screen"})+ (line,) = (log for log in logs if log["event"] == "taskv3 finish unlisted reask")+ completed = result.data is not None and result.data["status"] == "completed"+ assert completed == line["converted"]+ return line["veto"], line["converted"]+++async def test_a_conversion_is_vetoed_by_the_settle_gate_only_when_the_document_changes_identity() -> None:+ """With a readable identity, a conversion is refused by the settle gate only when the document changed+ across the window, settled or not. An unchanged document is never refused."""+ mismatches = []+ for max_settle_deferrals in range(4):+ window = max_settle_deferrals + 1+ for settles in itertools.product([False, True], repeat=window):+ for moves_after in [None, *range(1, window + 1)]:+ page = _SettleWindowPage(settles, moves_after)+ veto, converted = await _conversion_veto_for(+ page_fingerprint=page.fingerprint,+ document_identity=page.identity,+ max_settle_deferrals=max_settle_deferrals,+ )+ samples_taken = settles.index(True) + 1 if any(settles) else window+ moved = moves_after is not None and samples_taken >= moves_after+ expected = "navigating" if moved else None+ if veto != expected or converted != (expected is None):+ mismatches.append((max_settle_deferrals, settles, moves_after, veto, converted))+ assert mismatches == []++[email protected]("case", ["no_identity", "cancel", "cancel_raises"])+async def test_a_conversion_on_a_page_that_never_settles_fails_closed_without_an_identity_or_on_cancel(+ case: str,+) -> None:+ page = _SettleWindowPage(settles=(), moves_after=None)++ async def should_cancel() -> bool:+ # The veto's own first gate runs before any sample; only a cancel inside the window is under test.+ if not case.startswith("cancel") or page.fingerprint_calls < 2:+ return False+ if case == "cancel_raises":+ raise RuntimeError("cancel store unreachable")+ return True++ veto, converted = await _conversion_veto_for(+ page_fingerprint=page.fingerprint,+ document_identity=None if case == "no_identity" else page.identity,+ should_cancel=should_cancel,+ )++ assert (veto, converted) == ("canceled" if case.startswith("cancel") else "unsettled", False)++[email protected]("settles", [(), (True,)])[email protected]("read", ["before", "after"])[email protected]("failure", ["none", "raises"])+async def test_an_unreadable_document_identity_vetoes_a_conversion_even_on_a_settled_page(+ settles: tuple[bool, ...], read: str, failure: str+) -> None:+ page = _SettleWindowPage(settles=settles, moves_after=None)+ reads = 0++ async def identity() -> str | None:+ nonlocal reads+ reads += 1+ if (reads == 1) != (read == "before"):+ return await page.identity()+ if failure == "raises":+ raise RuntimeError("execution context destroyed")+ return None++ veto, converted = await _conversion_veto_for(page_fingerprint=page.fingerprint, document_identity=identity)++ assert (veto, converted) == ("identity_unreadable", False)+++async def test_a_navigation_during_the_judge_vetoes_a_conversion_even_once_the_new_page_settles() -> None:+ page = _SettleWindowPage(settles=(True,), moves_after=None)++ class _NavigatingJudge(_ReaskSpy):+ async def __call__(self, status: NonCompletedStatus, reason: str) -> UnlistedReask:+ page.moves_after = 0+ return await super().__call__(status, reason)++ finish = make_finish_tool(+ **_gate_kwargs(pending=False, blocked=False, unsettled=False, goal=None)+ | {"page_fingerprint": page.fingerprint, "document_identity": page.identity},+ unlisted_reask=_NavigatingJudge(),+ )+ with capture_logs() as logs:+ result = await finish.handler({"status": "terminated", "reason": "no PIN screen"})++ assert result.data is not None and result.data["status"] == "terminated"+ (line,) = (log for log in logs if log["event"] == "taskv3 finish unlisted reask")+ assert (line["veto"], line["settled"], line["settle_rounds"]) == ("navigating", True, 1)+++class _ScopedClock:+ def __init__(self) -> None:+ self.now = 0.0++ def monotonic(self) -> float:+ return self.now++ def __getattr__(self, name: str) -> object:+ return getattr(time, name)+++async def test_a_deadline_inside_the_conversion_settle_window_vetoes_without_sleeping_past_it(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ clock = _ScopedClock()++ async def sleep(seconds: float) -> None:+ clock.now += seconds++ monkeypatch.setattr(taskv3_loop, "time", clock)+ monkeypatch.setattr(taskv3_loop, "asyncio", ScopedAsyncio(sleep=sleep))+ page = _SettleWindowPage(settles=(), moves_after=None)++ veto, converted = await _conversion_veto_for(+ page_fingerprint=page.fingerprint,+ document_identity=page.identity,+ max_settle_deferrals=3,+ settle_wait_seconds=4.0,+ deadline_at=10.0,+ )++ assert (veto, converted) == ("deadline", False)+ assert clock.now == 10.0+++async def test_a_deadline_that_elapses_during_the_judge_still_vetoes_as_deadline(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """_settled reads a missing sample as settled (its own docstring), so a deadline that elapses+ between the pre-judge identity read and the settle loop's first round must still surface as+ "deadline" -- not "identity_unreadable" from the closing read finding nothing left to sample."""+ clock = _ScopedClock()++ async def sleep(seconds: float) -> None:+ clock.now += seconds++ monkeypatch.setattr(taskv3_loop, "time", clock)+ monkeypatch.setattr(taskv3_loop, "asyncio", ScopedAsyncio(sleep=sleep))+ page = _SettleWindowPage(settles=(), moves_after=None)++ class _SlowJudge(_ReaskSpy):+ async def __call__(self, status: NonCompletedStatus, reason: str) -> UnlistedReask:+ clock.now = 10.0 # the deadline elapses while the judge call is in flight+ return await super().__call__(status, reason)++ finish = make_finish_tool(+ **_gate_kwargs(pending=False, blocked=False, unsettled=False, goal=None)+ | {+ "page_fingerprint": page.fingerprint,+ "document_identity": page.identity,+ "pending_marker": None,+ "deadline_at": 10.0,+ },+ unlisted_reask=_SlowJudge(),+ )+ with capture_logs() as logs:+ result = await finish.handler({"status": "terminated", "reason": "no PIN screen"})++ assert result.data is not None and result.data["status"] == "terminated"+ (line,) = (log for log in logs if log["event"] == "taskv3 finish unlisted reask")+ assert line["veto"] == "deadline"++@pytest.mark.asyncio@pytest.mark.parametrize("converts", [True, False])async def test_a_run_canceled_during_the_reask_is_left_to_the_loops_cancellation(converts: bool) -> None: