Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Name the click hit class for what it measures, and time the probe where production calls it (#8521)
pedrohsdb89c9148ec915
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex 5fa236690..afeb38f3c 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -132,6 +132,10 @@ TARGET_KIND_DATA_KEY = "target_kind"# definitions with nothing on the record to cut on._RESOLVE_SECONDS: ContextVar[float | None] = ContextVar("taskv3_resolve_seconds", default=None)_FRAME_PERCEPTION: ContextVar[bool | None] = ContextVar("taskv3_frame_perception", default=None)+# Which case behind the click reach-probe's boolean applied, for the one click call this context covers.+# A context variable rather than a result field because `click` returns from many places, several of+# them after the probe has already answered -- the same reason `_RESOLVE_SECONDS` lives here.+_HIT_CLASS: ContextVar[dict[str, Any] | None] = ContextVar("taskv3_hit_class", default=None)def record_resolve_seconds(elapsed: float) -> None:@@ -139,6 +143,37 @@ def record_resolve_seconds(elapsed: float) -> None:_RESOLVE_SECONDS.set((_RESOLVE_SECONDS.get() or 0.0) + elapsed)+def record_hit_class(+ hit_class: str,+ *,+ needed: bool,+ probe_seconds: float | None = None,+ isolated: bool | None = None,+ raised: bool = False,+) -> None:+ """Record what this click's reach probe returned, the decision it produced, and what it cost.++ `needed` is on the record because the class alone cannot recover it: a shadow-rooted target+ answers `unknown` with needed=TRUE, so `unknown` would otherwise pool the rows that bought the+ second probe with the rows that answered nothing. Every `hit_class` value names what the hit test+ RETURNED, never why — nothing in that probe tells a real occluder from a hit that fell through,+ since a modal scrim drawn as `body::before` hit-tests as body and still blocks the click.+ `isolated` says which realm answered -- realms, not costs, since its `False` side spans both the+ cheapest fallback and the most expensive retry storm. `raised` marks a probe that threw, whose+ duration is a blow-up rather than a cost. The RECORDED copies are inert: the click path branches on+ its own `needed`, never on anything read back from here.+ """+ _HIT_CLASS.set(+ {+ "hit_class": hit_class,+ "needed": needed,+ "probe_seconds": probe_seconds,+ "isolated": isolated,+ "raised": raised,+ }+ )++def record_frame_perception(enabled: bool) -> None:"""Stamp which of the two definitions above produced this call's reading. Telemetry only."""_FRAME_PERCEPTION.set(enabled)@@ -1357,6 +1392,11 @@ _TOOL_CALL_RECORD_FIELDS = frozenset("action_key_hash","snapshot_digest","probe_first_time",+ "hit_class",+ "hit_needed",+ "hit_probe_seconds",+ "hit_probe_isolated",+ "hit_probe_raised",})@@ -2857,6 +2897,7 @@ async def run_agent_tool_loop(# Cleared per call, so a value can never carry over from the previous one in the batch._RESOLVE_SECONDS.set(None)_FRAME_PERCEPTION.set(None)+ _HIT_CLASS.set(None)tool_started_at = time.monotonic()if spec is None:result = ToolResult.error(f"unknown_tool: {tool_name}")@@ -2899,6 +2940,29 @@ async def run_agent_tool_loop(frame_perception = _FRAME_PERCEPTION.get()if frame_perception is not None:cost_fields["frame_perception"] = frame_perception+ # Every click row carries this, defaulting to `unknown`, because a groupBy DROPS rows+ # missing a facet -- a partial field would read as a clean result rather than a gap.+ # Gated on `spec is not None` for the same reason the `tool` field is: an unregistered+ # tool logs as `unknown_tool`, and such a row must not also carry a click-only facet.+ if spec is not None and tool_name == "click":+ hit = _HIT_CLASS.get() or {}+ cost_fields["hit_class"] = hit.get("hit_class") or "unknown"+ # Known for every click whether or not the probe ran, so it stays total alongside the+ # class. The two qualifiers below are NOT total by design: they describe a reading+ # that happened, and a fabricated `false` would be indistinguishable from a measured+ # one -- so group them WITHIN a hit_class bucket, never alone.++ # What the probe CALL cost at the production call site. Not the marginal round trip:+ # the first reading per realm also pays isolated-world construction, which is why the+ # median within one `hit_probe_isolated` bucket is the readable figure.+ # Present together, iff the probe was attempted: `hit_class` is the only total one.+ probe_seconds = hit.get("probe_seconds")+ if probe_seconds is not None:+ cost_fields["hit_probe_seconds"] = probe_seconds+ cost_fields["hit_needed"] = bool(hit.get("needed"))+ cost_fields["hit_probe_raised"] = bool(hit.get("raised"))+ if hit.get("isolated") is not None:+ cost_fields["hit_probe_isolated"] = hit["isolated"]# The action-loop guard's key and the perception ledger's digest, computed here (pure) so# their hashes ride the record below; the ledger itself is updated further down, unchanged.action_key = (tool_name, json.dumps(args, sort_keys=True, default=str))diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex 6cc10658a..9594cc98f 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -24,6 +24,7 @@ import timeimport unicodedataimport weakreffrom collections import Counter, defaultdict, deque+from contextvars import ContextVarfrom enum import Enumfrom typing import TYPE_CHECKING, Any, Awaitable, Callable, NamedTuple@@ -50,6 +51,7 @@ from skyvern.forge.taskv3.loop import (ToolResult,ToolSpec,record_frame_perception,+ record_hit_class,record_resolve_seconds,set_driver_timeout_predicate,)@@ -3028,25 +3030,49 @@ _REACH_PROBE_NEEDED_JS = ("""+ _NATIVE_LABEL_JS+ r"""+ // Answered as ONE STRING, "<0|1>:<class>", not an object. The decision has to survive the page-realm+ // fallback, and Playwright's serializer runs in the PAGE's world using bare Object.keys and+ // Object.prototype.toString: a page that clobbers either turns an object return into {}, into null,+ // or into truthy junk -- measured, all three -- which moves `needed` in both directions on the leg+ // this probe exists to be trusted on. A primitive passes through untouched.+ //+ // The class reports WHICH case produced the decision, because one boolean collapses seven distinct+ // conditions -- not found, shadow-rooted, zero-area, self, non-target, null hit, and a throw inside+ // the rect block -- into four classes. Telemetry only: no branch here reads it.+ //+ // `non_target` is named for what it measures -- the centre point hit something that is not the+ // target -- and NOT for occlusion. A row clipped out of its own scroll container lands here with+ // nothing painted over it and clicks fine; a pointer-events:none or visibility:hidden target lands+ // here too and does NOT click, it burns the full actionability wait. Nothing in this probe separates+ // any of them from a real occluder: a modal scrim drawn as `body::before` hit-tests AS body and+ // blocks the click. Answering that needs the scroll and composed descent the mature probe pays for.+ let needed = false;+ let hitClass = "unknown";+ const out = () => (needed ? "1:" : "0:") + hitClass;const el = _q.find(arg.sel) || (arg.el && arg.el.isConnected ? arg.el : null);- if (!el) return false;- try { if (Node.prototype.getRootNode.call(el) !== document) return true; } catch (e) { /* fall through */ }+ if (!el) return out();+ // A shadow-rooted target returns before any hit test runs, so it is unknown rather than no_hit.+ try { if (Node.prototype.getRootNode.call(el) !== document) { needed = true; return out(); } } catch (e) { /* fall through */ }// The cheap hit-test runs first and exits on an ordinary unoccluded hit; the DOM-wide label scan- // only runs for the null/foreign-hit cases where it can actually change the answer.+ // only runs for the null and non-target hit cases where it can actually change the answer.try {const rect = el.getBoundingClientRect();if (rect.width > 0 && rect.height > 0) {const cx = rect.left + rect.width / 2;const cy = rect.top + rect.height / 2;const hit = document.elementFromPoint(cx, cy);- if (hit === el || (hit && el.contains(hit))) return false;+ // Every value names what the hit test RETURNED, never why. A null hit is `no_hit`: it usually+ // means the point is outside the layout viewport, but that is an inference, and the boundary+ // between the layout and visual viewports does not agree with it.+ hitClass = (hit === el || (hit && el.contains(hit))) ? "self" : (hit ? "non_target" : "no_hit");+ if (hitClass === "self") return out();// A sibling <label for=id> drawn OVER its control trips the driver's containment check the same// way a slotted label does; a null hit (off-screen target) also earns the probe when labels exist.// Skipped when own-label granting is off (no isolated world): the scan only ever earns a bypass.- if (arg.allowOwnLabel !== false && (nativeLabelsOf(el).length || (_isLabel(el) && nativeControlOf(el)))) return true;+ if (arg.allowOwnLabel !== false && (nativeLabelsOf(el).length || (_isLabel(el) && nativeControlOf(el)))) needed = true;}} catch (e) { /* best-effort */ }- return false;+ return out();}""")@@ -7401,6 +7427,45 @@ def _merge_realm(into: dict[str, Any], other: dict[str, Any]) -> int:return len(taken)+_HIT_CLASSES = frozenset({"self", "non_target", "no_hit", "unknown"})+++def _reach_probe_needed(probe_result: Any) -> bool:+ """The reach probe's decision, from the `"<0|1>:<class>"` answer.++ Anything that is not that shape falls back to the value's own truthiness, which is what the bare+ boolean did -- so a probe that could not answer decides exactly as it decided before.+ """+ if isinstance(probe_result, str) and ":" in probe_result:+ return probe_result.split(":", 1)[0] == "1"+ return bool(probe_result)+++def _reach_hit_class(probe_result: Any) -> str:+ """Which hit-test case produced the decision. Total by construction: anything unrecognised is+ `unknown`, never absent, because a row without the facet vanishes from a groupBy instead of+ bucketing."""+ if isinstance(probe_result, str) and ":" in probe_result:+ value = probe_result.split(":", 1)[1]+ if value in _HIT_CLASSES:+ return value+ return "unknown"+++# Which realm answered the last `_probe_evaluate`. It separates REALMS, not costs, and it must not be+# read as a cost split: measured medians here are 0.56ms with an isolated world, 0.66ms on a cached+# miss (no CDP session for the page -- the miss is cached, so later probes fall back doing no CDP work+# at all), and 233ms when world creation keeps raising and the three-attempt loop runs. `False` spans+# the cheapest path AND the most expensive one, so it narrows a duration without partitioning it.+# Recorded because a duration that cannot say which realm answered cannot be compared across pages at+# all -- the same reason `frame_perception` rides every row carrying a css reading.+_PROBE_ISOLATED: ContextVar[bool | None] = ContextVar("taskv3_probe_isolated", default=None)+++def _probe_was_isolated() -> bool | None:+ return _PROBE_ISOLATED.get()++async def _probe_evaluate(target: Any, js: str, selector: str, arg: dict[str, Any]) -> Any:"""Isolated-world probe, falling back to the realm's OWN world with own-label granting disabled --a realm the page can patch must not be able to hand a label a hit-test bypass.@@ -7409,10 +7474,14 @@ async def _probe_evaluate(target: Any, js: str, selector: str, arg: dict[str, Anthan dropping to the page's: losing isolation is the trade this fallback has always made, butanswering about the wrong document is not a trade, it is a wrong answer wearing a verdict's clothes."""+ _PROBE_ISOLATED.set(None)isolated = await _evaluate_isolated(target, js, selector)if isolated is not None:+ _PROBE_ISOLATED.set(True)return isolated- return await target.evaluate(js, {**arg, "allowOwnLabel": False})+ answer = await target.evaluate(js, {**arg, "allowOwnLabel": False})+ _PROBE_ISOLATED.set(False)+ return answerdef build_browser_tools(@@ -8785,7 +8854,28 @@ def build_browser_tools(# <label for=id>) pays the extra round trip; any other light-DOM click keeps its single one.reach_pre = Nonetry:- if await _probe_evaluate(page, _REACH_PROBE_NEEDED_JS, selector, pre_click_arg):+ # Timed at the production call site rather than on a bench: a local page.evaluate bounds+ # only the browser-side half. The FIRST reading per realm also pays isolated-world+ # construction (~10x locally), so this is the probe CALL's cost, not the marginal round+ # trip -- read the median, and read it within one `hit_probe_isolated` bucket.+ probe_started_at = time.monotonic()+ reach_verdict: Any = None+ raised = True+ try:+ reach_verdict = await _probe_evaluate(page, _REACH_PROBE_NEEDED_JS, selector, pre_click_arg)+ raised = False+ finally:+ # Recorded even when the probe threw: the raise path correlates with re-render, so+ # dropping it would bias the sample toward the calm cases. Tagged, because a duration+ # that pools answers with blow-ups is two events under one name.+ record_hit_class(+ _reach_hit_class(reach_verdict),+ needed=_reach_probe_needed(reach_verdict),+ probe_seconds=time.monotonic() - probe_started_at,+ isolated=_probe_was_isolated(),+ raised=raised,+ )+ if _reach_probe_needed(reach_verdict):reach_pre = await _probe_evaluate(page, _TYPE_TARGET_PROBE_JS, selector, pre_click_arg)except Exception:reach_pre = Nonediff --git a/tests/unit/test_copilot_fill_credential_field.py b/tests/unit/test_copilot_fill_credential_field.pyindex 2cf4176d8..eaec21f79 100644--- a/tests/unit/test_copilot_fill_credential_field.py+++ b/tests/unit/test_copilot_fill_credential_field.py@@ -505,335 +505,343 @@ def _wire_impl(monkeypatch.setattr(credential_fill_module, "_register_scout_interaction_observation", fake_register)- @pytest.mark.asyncio- async def test_target_identity_is_captured_before_fill_and_effect_afterward(- self, monkeypatch: pytest.MonkeyPatch- ) -> None:- events: list[str] = []- class OrderedPage(_FakePage):- async def fill(self, *args: Any, **kwargs: Any) -> None:- events.append("fill")- await super().fill(*args, **kwargs)+async def test_target_identity_is_captured_before_fill_and_effect_afterward(monkeypatch: pytest.MonkeyPatch) -> None:+ events: list[str] = []- async def read_value(self, selector: str) -> str:- events.append("post_effect")- return await super().read_value(selector)+ class OrderedPage(_FakePage):+ async def fill(self, *args: Any, **kwargs: Any) -> None:+ events.append("fill")+ await super().fill(*args, **kwargs)- page = OrderedPage()- _wire_impl(monkeypatch, page)+ async def read_value(self, selector: str) -> str:+ events.append("post_effect")+ return await super().read_value(selector)- async def pre_fact(*_args: Any, **_kwargs: Any) -> int:- events.append("pre_fact")- return 1+ page = OrderedPage()+ _wire_impl(monkeypatch, page)- async def selector_candidates(ctx: Any, _selector: str) -> None:- events.append("selector_candidates")- ctx.pending_scout_selector_candidates = [{"selector": 'input[name="password"]', "source": "name"}]+ async def pre_fact(*_args: Any, **_kwargs: Any) -> int:+ events.append("pre_fact")+ return 1- monkeypatch.setattr(credential_fill_module, "_selector_live_match_count", pre_fact)- monkeypatch.setattr(credential_fill_module, "_capture_scout_selector_candidates", selector_candidates)+ async def selector_candidates(ctx: Any, _selector: str) -> None:+ events.append("selector_candidates")+ ctx.pending_scout_selector_candidates = [{"selector": 'input[name="password"]', "source": "name"}]- ctx = _ctx()- result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")+ monkeypatch.setattr(credential_fill_module, "_selector_live_match_count", pre_fact)+ monkeypatch.setattr(credential_fill_module, "_capture_scout_selector_candidates", selector_candidates)- assert result["ok"] is True- assert events.index("selector_candidates") < events.index("fill")- assert events.index("pre_fact") < events.index("fill") < events.index("post_effect")- assert ctx.scout_trajectory[-1]["selector_match_count"] == 1- assert ctx.scout_trajectory[-1]["observed_effects"]["value_landed"] is True- assert ctx.scout_trajectory[-1]["selector_candidates"] == [- {"selector": "#passwordInput", "source": "requested", "match_count": None},- {"selector": 'input[name="password"]', "source": "name", "match_count": None},- ]+ ctx = _ctx()+ result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")- def test_tool_layer_readback_cannot_verify_a_short_secret(self) -> None:- """Why the outcome is classified at the fill site, pinned against the real scrubber: a- registered secret returns from the tool layer as the placeholder, which no code recognises."""- otp = "mk-one"- ctx = SimpleNamespace(secret_scrub_values=[], browser_session_id=None)- register_secret_scrub_value(ctx, otp)- readback = scrub_secrets_from_structure(ctx, {"ok": True, "data": {"value": otp}})["data"]["value"]+ assert result["ok"] is True+ assert events.index("selector_candidates") < events.index("fill")+ assert events.index("pre_fact") < events.index("fill") < events.index("post_effect")+ assert ctx.scout_trajectory[-1]["selector_match_count"] == 1+ assert ctx.scout_trajectory[-1]["observed_effects"]["value_landed"] is True+ assert ctx.scout_trajectory[-1]["selector_candidates"] == [+ {"selector": "#passwordInput", "source": "requested", "match_count": None},+ {"selector": 'input[name="password"]', "source": "name", "match_count": None},+ ]- assert (- mcp_hooks_module._scout_readback_outcome(readback, otp) is mcp_hooks_module.ScoutReadbackOutcome.DIFFERENT- )- assert mcp_hooks_module._scout_readback_outcome(otp, otp) is mcp_hooks_module.ScoutReadbackOutcome.EXACT_MATCH- @pytest.mark.asyncio- async def test_multi_match_selector_still_reaches_an_outcome(self, monkeypatch: pytest.MonkeyPatch) -> None:- """Reading the un-narrowed locator raises Playwright strict mode, which would leave the- field unread; the readback narrows the way the fill did."""- page = _FakePage(readback="8675309" + "123456")- page.selector_match_count = 2- _wire_impl(monkeypatch, page, secret_value="123456")- ctx = _ctx()+def test_tool_layer_readback_cannot_verify_a_short_secret() -> None:+ """Why the outcome is classified at the fill site, pinned against the real scrubber: a+ registered secret returns from the tool layer as the placeholder, which no code recognises."""+ otp = "mk-one"+ ctx = SimpleNamespace(secret_scrub_values=[], browser_session_id=None)+ register_secret_scrub_value(ctx, otp)+ readback = scrub_secrets_from_structure(ctx, {"ok": True, "data": {"value": otp}})["data"]["value"]- result = await tools_module._fill_credential_field_impl(ctx, "input.otp", "cred_123", "totp")+ assert mcp_hooks_module._scout_readback_outcome(readback, otp) is mcp_hooks_module.ScoutReadbackOutcome.DIFFERENT+ assert mcp_hooks_module._scout_readback_outcome(otp, otp) is mcp_hooks_module.ScoutReadbackOutcome.EXACT_MATCH- assert result["ok"] is True- assert result["data"]["readback_outcome"] == "different"- @pytest.mark.asyncio- @pytest.mark.parametrize(- ("secret_value", "readback"),- [- ("marker-value-one", "marker-val"),- ("alphabetagamma", "alpha beta gamma"),- ("marker-value-longer", "marker-value-lo"),- ],- )- async def test_a_field_not_holding_what_was_typed_is_reported_without_failing_or_claiming_a_wrong_field(- self, monkeypatch: pytest.MonkeyPatch, secret_value: str, readback: str- ) -> None:- page = _FakePage(readback=readback)- _wire_impl(monkeypatch, page, secret_value=secret_value)- ctx = _ctx()+async def test_multi_match_selector_still_reaches_an_outcome(monkeypatch: pytest.MonkeyPatch) -> None:+ """Reading the un-narrowed locator raises Playwright strict mode, which would leave the+ field unread; the readback narrows the way the fill did."""+ page = _FakePage(readback="8675309" + "123456")+ page.selector_match_count = 2+ _wire_impl(monkeypatch, page, secret_value="123456")+ ctx = _ctx()++ result = await tools_module._fill_credential_field_impl(ctx, "input.otp", "cred_123", "totp")++ assert result["ok"] is True+ assert result["data"]["readback_outcome"] == "different"+++ ("secret_value", "readback"),+ [+ ("marker-value-one", "marker-val"),+ ("alphabetagamma", "alpha beta gamma"),+ ("marker-value-longer", "marker-value-lo"),+ ],+)+async def test_a_field_not_holding_what_was_typed_is_reported_without_failing_or_claiming_a_wrong_field(+ monkeypatch: pytest.MonkeyPatch, secret_value: str, readback: str+) -> None:+ page = _FakePage(readback=readback)+ _wire_impl(monkeypatch, page, secret_value=secret_value)+ ctx = _ctx()++ result = await tools_module._fill_credential_field_impl(ctx, "#totp", "cred_123", "totp")++ assert result["ok"] is True+ assert result["data"]["readback_outcome"] == "different"+ assert result["data"]["landing_inferred_from_navigation"] is False+ assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})+ assert "wrong" not in json.dumps(result)+ assert "different input" not in json.dumps(result)+++async def test_unreadable_field_records_the_fill_and_says_so(monkeypatch: pytest.MonkeyPatch) -> None:+ """An unreadable readback must not fail a fill that may well have landed, but must be visible."""++ class UnreadablePage(_FakePage):+ async def read_value(self, selector: str) -> str:+ raise RuntimeError("element is not an <input>")++ page = UnreadablePage()+ _wire_impl(monkeypatch, page, secret_value="123456")+ ctx = _ctx()+ with capture_logs() as logs:result = await tools_module._fill_credential_field_impl(ctx, "#totp", "cred_123", "totp")- assert result["ok"] is True- assert result["data"]["readback_outcome"] == "different"- assert result["data"]["landing_inferred_from_navigation"] is False- assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})- assert "wrong" not in json.dumps(result)- assert "different input" not in json.dumps(result)+ assert result["ok"] is True+ assert result["data"]["readback_outcome"] == "unavailable"+ assert result["data"]["landing_inferred_from_navigation"] is False+ assert ctx.scout_trajectory[-1]["credential_field"] == "totp"+ assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})+ assert any(+ entry.get("event") == "copilot fill_credential_field readback outcome" and entry.get("outcome") == "unavailable"+ for entry in logs+ )- @pytest.mark.asyncio- async def test_unreadable_field_records_the_fill_and_says_so(self, monkeypatch: pytest.MonkeyPatch) -> None:- """An unreadable readback must not fail a fill that may well have landed, but must be visible."""- class UnreadablePage(_FakePage):- async def read_value(self, selector: str) -> str:- raise RuntimeError("element is not an <input>")+async def test_happy_path_fills_and_records_value_free(monkeypatch: pytest.MonkeyPatch) -> None:+ page = _FakePage()+ _wire_impl(monkeypatch, page)+ ctx = _ctx()- page = UnreadablePage()- _wire_impl(monkeypatch, page, secret_value="123456")- ctx = _ctx()+ result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")- with capture_logs() as logs:- result = await tools_module._fill_credential_field_impl(ctx, "#totp", "cred_123", "totp")+ assert result["ok"] is True+ assert page.fill_calls == [("#passwordInput", _FAKE_PASSWORD)]+ assert page.fill_kwargs[0]["mode"] == "direct"+ assert result["data"]["typed_length"] == len(_FAKE_PASSWORD)+ assert result["data"]["credential_id"] == "cred_123"+ assert result["data"]["field"] == "password"+ assert result["data"]["observation_step"] == 3+ assert result["data"]["credential_name"] == "authtest simple"+ assert result["data"]["readback_outcome"] == "exact_match"+ assert result["data"]["landing_inferred_from_navigation"] is False+ assert "credential_parameter" not in result["data"]+ assert _FAKE_PASSWORD not in json.dumps(result)- assert result["ok"] is True- assert result["data"]["readback_outcome"] == "unavailable"- assert result["data"]["landing_inferred_from_navigation"] is False- assert ctx.scout_trajectory[-1]["credential_field"] == "totp"- assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})- assert any(- entry.get("event") == "copilot fill_credential_field readback outcome"- and entry.get("outcome") == "unavailable"- for entry in logs- )+ assert len(ctx.scouted_interactions) == 1+ recorded = ctx.scouted_interactions[0]+ assert recorded["tool_name"] == "fill_credential_field"+ assert recorded["credential_id"] == "cred_123"+ assert recorded["credential_field"] == "password"+ assert recorded["credential_name"] == "authtest simple"+ assert recorded["typed_length"] == len(_FAKE_PASSWORD)+ assert recorded["observed_effects"]["value_landed"] is True+ assert _FAKE_PASSWORD not in json.dumps(recorded)+ assert _FAKE_PASSWORD not in json.dumps(ctx.scout_trajectory)- @pytest.mark.asyncio- async def test_happy_path_fills_and_records_value_free(self, monkeypatch: pytest.MonkeyPatch) -> None:- page = _FakePage()- _wire_impl(monkeypatch, page)- ctx = _ctx()- result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")+async def test_email_otp_scout_failure_returns_only_factual_credential_identity(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ page = _FakePage()+ _wire_impl(monkeypatch, page)- assert result["ok"] is True- assert page.fill_calls == [("#passwordInput", _FAKE_PASSWORD)]- assert page.fill_kwargs[0]["mode"] == "direct"- assert result["data"]["typed_length"] == len(_FAKE_PASSWORD)- assert result["data"]["credential_id"] == "cred_123"- assert result["data"]["field"] == "password"- assert result["data"]["observation_step"] == 3- assert result["data"]["credential_name"] == "authtest simple"- assert result["data"]["readback_outcome"] == "exact_match"- assert result["data"]["landing_inferred_from_navigation"] is False- assert "credential_parameter" not in result["data"]- assert _FAKE_PASSWORD not in json.dumps(result)-- assert len(ctx.scouted_interactions) == 1- recorded = ctx.scouted_interactions[0]- assert recorded["tool_name"] == "fill_credential_field"- assert recorded["credential_id"] == "cred_123"- assert recorded["credential_field"] == "password"- assert recorded["credential_name"] == "authtest simple"- assert recorded["typed_length"] == len(_FAKE_PASSWORD)- assert recorded["observed_effects"]["value_landed"] is True- assert _FAKE_PASSWORD not in json.dumps(recorded)- assert _FAKE_PASSWORD not in json.dumps(ctx.scout_trajectory)-- @pytest.mark.asyncio- async def test_email_otp_scout_failure_returns_only_factual_credential_identity(- self, monkeypatch: pytest.MonkeyPatch- ) -> None:- page = _FakePage()- _wire_impl(monkeypatch, page)+ async def runtime_only_otp(_ctx: Any, _credential_id: str, _field: str) -> tuple[None, str, str]:+ return None, "authtest simple", "Email OTP requires workflow-run polling."- async def runtime_only_otp(_ctx: Any, _credential_id: str, _field: str) -> tuple[None, str, str]:- return None, "authtest simple", "Email OTP requires workflow-run polling."+ monkeypatch.setattr(credential_fill_module, "_resolve_credential_fill_value", runtime_only_otp)- monkeypatch.setattr(credential_fill_module, "_resolve_credential_fill_value", runtime_only_otp)+ result = await tools_module._fill_credential_field_impl(_ctx(), "#otp", "cred_123", "totp")- result = await tools_module._fill_credential_field_impl(_ctx(), "#otp", "cred_123", "totp")+ assert result["ok"] is False+ assert result["data"] == {+ "credential_id": "cred_123",+ "credential_name": "authtest simple",+ "credential_field": "totp",+ }+ assert page.fill_calls == []- assert result["ok"] is False- assert result["data"] == {- "credential_id": "cred_123",- "credential_name": "authtest simple",- "credential_field": "totp",- }- assert page.fill_calls == []- @pytest.mark.asyncio- async def test_fill_error_text_is_scrubbed(self, monkeypatch: pytest.MonkeyPatch) -> None:- page = _FakePage(fill_error=RuntimeError(f"could not type {_FAKE_PASSWORD} into element"))- _wire_impl(monkeypatch, page)+async def test_fill_error_text_is_scrubbed(monkeypatch: pytest.MonkeyPatch) -> None:+ page = _FakePage(fill_error=RuntimeError(f"could not type {_FAKE_PASSWORD} into element"))+ _wire_impl(monkeypatch, page)- result = await tools_module._fill_credential_field_impl(_ctx(), "#passwordInput", "cred_123", "password")+ result = await tools_module._fill_credential_field_impl(_ctx(), "#passwordInput", "cred_123", "password")- assert result["ok"] is False- assert _FAKE_PASSWORD not in result["error"]- assert "[REDACTED_SECRET]" in result["error"]+ assert result["ok"] is False+ assert _FAKE_PASSWORD not in result["error"]+ assert "[REDACTED_SECRET]" in result["error"]- @pytest.mark.asyncio- async def test_rejects_unknown_field(self, monkeypatch: pytest.MonkeyPatch) -> None:- page = _FakePage()- _wire_impl(monkeypatch, page)- result = await tools_module._fill_credential_field_impl(_ctx(), "#cvv", "cred_123", "cvv")+async def test_rejects_unknown_field(monkeypatch: pytest.MonkeyPatch) -> None:+ page = _FakePage()+ _wire_impl(monkeypatch, page)- assert result["ok"] is False- assert "username, password, totp" in result["error"]- assert page.fill_calls == []+ result = await tools_module._fill_credential_field_impl(_ctx(), "#cvv", "cred_123", "cvv")- @pytest.mark.asyncio- async def test_rejects_empty_selector(self, monkeypatch: pytest.MonkeyPatch) -> None:- page = _FakePage()- _wire_impl(monkeypatch, page)+ assert result["ok"] is False+ assert "username, password, totp" in result["error"]+ assert page.fill_calls == []- result = await tools_module._fill_credential_field_impl(_ctx(), " ", "cred_123", "password")- assert result["ok"] is False- assert page.fill_calls == []+async def test_rejects_empty_selector(monkeypatch: pytest.MonkeyPatch) -> None:+ page = _FakePage()+ _wire_impl(monkeypatch, page)- @pytest.mark.asyncio- async def test_unresolved_credential_never_reaches_vault_or_page(self, monkeypatch: pytest.MonkeyPatch) -> None:- page = _FakePage()- _wire_impl(monkeypatch, page)- resolver = AsyncMock()- monkeypatch.setattr(credential_fill_module, "_resolve_credential_fill_value", resolver)+ result = await tools_module._fill_credential_field_impl(_ctx(), " ", "cred_123", "password")- result = await tools_module._fill_credential_field_impl(_ctx(), "#passwordInput", "cred_999", "password")+ assert result["ok"] is False+ assert page.fill_calls == []- assert result["ok"] is False- assert "cred_999" in result["error"]- resolver.assert_not_awaited()- assert page.fill_calls == []- @pytest.mark.asyncio- async def test_standard_mode_is_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None:- page = _FakePage()- _wire_impl(monkeypatch, page)- ctx = _ctx(block_authoring_policy=BlockAuthoringPolicy.STANDARD)+async def test_unresolved_credential_never_reaches_vault_or_page(monkeypatch: pytest.MonkeyPatch) -> None:+ page = _FakePage()+ _wire_impl(monkeypatch, page)+ resolver = AsyncMock()+ monkeypatch.setattr(credential_fill_module, "_resolve_credential_fill_value", resolver)++ result = await tools_module._fill_credential_field_impl(_ctx(), "#passwordInput", "cred_999", "password")++ assert result["ok"] is False+ assert "cred_999" in result["error"]+ resolver.assert_not_awaited()+ assert page.fill_calls == []+++async def test_standard_mode_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None:+ page = _FakePage()+ _wire_impl(monkeypatch, page)+ ctx = _ctx(block_authoring_policy=BlockAuthoringPolicy.STANDARD)+ result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")++ assert result["ok"] is False+ assert page.fill_calls == []+++async def test_readback_failure_surfaces_and_skips_recording(monkeypatch: pytest.MonkeyPatch) -> None:+ page = _FakePage(readback="")+ _wire_impl(monkeypatch, page)+ ctx = _ctx()++ with capture_logs() as logs:result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")- assert result["ok"] is False- assert page.fill_calls == []+ assert result["ok"] is False+ assert "still empty" in result["error"]+ assert result["data"]["readback_outcome"] == "empty"+ assert result["data"]["selector"] == "#passwordInput"+ assert result["data"]["typed_length"] == len(_FAKE_PASSWORD)+ assert result["data"]["landing_inferred_from_navigation"] is False+ assert ctx.scouted_interactions == []+ assert any(+ entry.get("event") == "copilot fill_credential_field readback outcome" and entry.get("outcome") == "empty"+ for entry in logs+ )- @pytest.mark.asyncio- async def test_readback_failure_surfaces_and_skips_recording(self, monkeypatch: pytest.MonkeyPatch) -> None:- page = _FakePage(readback="")- _wire_impl(monkeypatch, page)- ctx = _ctx()- with capture_logs() as logs:- result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")+async def test_a_field_cleared_by_its_own_submit_is_not_reported_as_a_lost_fill(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ page = _FakePage(readback="")+ _wire_impl(monkeypatch, page, secret_value="123456")- assert result["ok"] is False- assert "still empty" in result["error"]- assert result["data"]["readback_outcome"] == "empty"- assert result["data"]["selector"] == "#passwordInput"- assert result["data"]["typed_length"] == len(_FAKE_PASSWORD)- assert result["data"]["landing_inferred_from_navigation"] is False- assert ctx.scouted_interactions == []- assert any(- entry.get("event") == "copilot fill_credential_field readback outcome" and entry.get("outcome") == "empty"- for entry in logs- )+ async def navigated(_ctx: Any) -> str:+ return _FIXTURE_LOGIN_URL + "verified/"- @pytest.mark.asyncio- async def test_a_field_cleared_by_its_own_submit_is_not_reported_as_a_lost_fill(- self, monkeypatch: pytest.MonkeyPatch- ) -> None:- page = _FakePage(readback="")- _wire_impl(monkeypatch, page, secret_value="123456")+ monkeypatch.setattr(credential_fill_module, "_live_working_page_url", navigated)+ ctx = _ctx()- async def navigated(_ctx: Any) -> str:- return _FIXTURE_LOGIN_URL + "verified/"+ result = await tools_module._fill_credential_field_impl(ctx, "#totpCode", "cred_123", "totp")- monkeypatch.setattr(credential_fill_module, "_live_working_page_url", navigated)- ctx = _ctx()+ # The form committed on the last digit and cleared its own field. Calling that a lost fill+ # sends the model back to re-type a code into a page the sign-in has already left.+ assert result["ok"] is True+ assert [entry["tool_name"] for entry in ctx.scouted_interactions] == ["fill_credential_field"]+ # The readback said the field was empty. The model is told the landing was inferred, not+ # that anyone saw the value sitting there.+ assert ctx.scouted_interactions[0]["observed_effects"]["landing_inferred_from_navigation"] is True+ assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})+ assert result["data"]["readback_outcome"] == "empty"+ assert result["data"]["landing_inferred_from_navigation"] is True- result = await tools_module._fill_credential_field_impl(ctx, "#totpCode", "cred_123", "totp")- # The form committed on the last digit and cleared its own field. Calling that a lost fill- # sends the model back to re-type a code into a page the sign-in has already left.- assert result["ok"] is True- assert [entry["tool_name"] for entry in ctx.scouted_interactions] == ["fill_credential_field"]- # The readback said the field was empty. The model is told the landing was inferred, not- # that anyone saw the value sitting there.- assert ctx.scouted_interactions[0]["observed_effects"]["landing_inferred_from_navigation"] is True- assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})- assert result["data"]["readback_outcome"] == "empty"- assert result["data"]["landing_inferred_from_navigation"] is True+async def test_a_navigation_cannot_stand_in_for_a_readback_nobody_took(monkeypatch: pytest.MonkeyPatch) -> None:+ class UnreadablePage(_FakePage):+ async def read_value(self, selector: str) -> str:+ raise RuntimeError("execution context was destroyed")- @pytest.mark.asyncio- async def test_a_navigation_cannot_stand_in_for_a_readback_nobody_took(- self, monkeypatch: pytest.MonkeyPatch- ) -> None:- class UnreadablePage(_FakePage):- async def read_value(self, selector: str) -> str:- raise RuntimeError("execution context was destroyed")+ page = UnreadablePage()+ _wire_impl(monkeypatch, page, secret_value="mk-one")- page = UnreadablePage()- _wire_impl(monkeypatch, page, secret_value="mk-one")+ async def navigated(_ctx: Any) -> str:+ return _FIXTURE_LOGIN_URL + "verified/"- async def navigated(_ctx: Any) -> str:- return _FIXTURE_LOGIN_URL + "verified/"+ monkeypatch.setattr(credential_fill_module, "_live_working_page_url", navigated)+ ctx = _ctx()- monkeypatch.setattr(credential_fill_module, "_live_working_page_url", navigated)- ctx = _ctx()+ result = await tools_module._fill_credential_field_impl(ctx, "#totpCode", "cred_123", "totp")- result = await tools_module._fill_credential_field_impl(ctx, "#totpCode", "cred_123", "totp")+ assert result["ok"] is True+ assert result["data"]["readback_outcome"] == "unavailable"+ assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})+ assert "landing_inferred_from_navigation" not in ctx.scouted_interactions[0].get("observed_effects", {})- assert result["ok"] is True- assert result["data"]["readback_outcome"] == "unavailable"- assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})- assert "landing_inferred_from_navigation" not in ctx.scouted_interactions[0].get("observed_effects", {})- @pytest.mark.asyncio- async def test_a_rejected_code_re_rendering_the_same_page_is_still_a_lost_fill(- self, monkeypatch: pytest.MonkeyPatch- ) -> None:- page = _FakePage(readback="")- _wire_impl(monkeypatch, page, secret_value="123456")+async def test_a_rejected_code_re_rendering_the_same_page_is_still_a_lost_fill(monkeypatch: pytest.MonkeyPatch) -> None:+ page = _FakePage(readback="")+ _wire_impl(monkeypatch, page, secret_value="123456")- async def same_page_with_error(_ctx: Any) -> str:- return _FIXTURE_LOGIN_URL + "?error=invalid"+ async def same_page_with_error(_ctx: Any) -> str:+ return _FIXTURE_LOGIN_URL + "?error=invalid"- monkeypatch.setattr(credential_fill_module, "_live_working_page_url", same_page_with_error)+ monkeypatch.setattr(credential_fill_module, "_live_working_page_url", same_page_with_error)- result = await tools_module._fill_credential_field_impl(_ctx(), "#totpCode", "cred_123", "totp")+ result = await tools_module._fill_credential_field_impl(_ctx(), "#totpCode", "cred_123", "totp")- # A rejected code re-renders the same page with an error param. That is not the form having- # carried the code away, so the empty field still means the fill did not land.- assert result["ok"] is False+ # A rejected code re-renders the same page with an error param. That is not the form having+ # carried the code away, so the empty field still means the fill did not land.+ assert result["ok"] is False- @pytest.mark.asyncio- async def test_a_verified_fill_is_still_reported_as_observed(self, monkeypatch: pytest.MonkeyPatch) -> None:- page = _FakePage(readback=_FAKE_PASSWORD)- _wire_impl(monkeypatch, page)- ctx = _ctx()- await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")+async def test_a_verified_fill_is_still_reported_as_observed(monkeypatch: pytest.MonkeyPatch) -> None:+ page = _FakePage(readback=_FAKE_PASSWORD)+ _wire_impl(monkeypatch, page)+ ctx = _ctx()++ await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")- assert "landing_inferred_from_navigation" not in ctx.scouted_interactions[0]["observed_effects"]+ assert "landing_inferred_from_navigation" not in ctx.scouted_interactions[0]["observed_effects"]class TestCredentialFillInCallSubmit:diff --git a/tests/unit/test_nested_test_def_collection_guard.py b/tests/unit/test_nested_test_def_collection_guard.pynew file mode 100644index 000000000..122c3c656--- /dev/null+++ b/tests/unit/test_nested_test_def_collection_guard.py@@ -0,0 +1,34 @@+"""A `test_*` def nested inside a helper function is never collected, so a deleted class header can+silently retire a whole block of tests (SKY-16226). A test nested inside another test is a local fake."""++import ast+from pathlib import Path++TESTS_ROOT = Path(__file__).resolve().parents[1]+++def _test_defs_nested_in_helpers(tree: ast.Module) -> list[tuple[int, str, str]]:+ offenders: list[tuple[int, str, str]] = []++ def walk(node: ast.AST, enclosing_helper: str | None) -> None:+ for child in ast.iter_child_nodes(node):+ helper = enclosing_helper+ if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):+ if child.name.startswith("test_") and enclosing_helper is not None:+ offenders.append((child.lineno, enclosing_helper, child.name))+ if not child.name.startswith("test_"):+ helper = helper or child.name+ walk(child, helper)++ walk(tree, None)+ return offenders+++def test_no_test_def_is_nested_inside_a_helper_function() -> None:+ offenders: list[str] = []+ for pattern in ("test_*.py", "*_test.py"):+ for path in sorted(TESTS_ROOT.rglob(pattern)):+ tree = ast.parse(path.read_text(), filename=str(path))+ for lineno, helper, name in _test_defs_nested_in_helpers(tree):+ offenders.append(f"{path.relative_to(TESTS_ROOT.parent)}:{lineno} {name} is nested inside {helper}")+ assert offenders == [], "\n".join(offenders)diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex d2e0afd86..8ccf51058 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -75,6 +75,7 @@ from skyvern.forge.taskv3.loop import (_RevisitMemory,make_finish_tool,record_frame_perception,+ record_hit_class,record_resolve_seconds,run_agent_tool_loop,)@@ -9588,3 +9589,70 @@ async def test_a_round_carries_billability_from_the_spec_not_from_a_name_list()acted = [entry for round_ in rounds for entry in round_ if entry.tool == "execute_python"]assert acted, "the billable tool should have produced a round entry"assert all(entry.billable for entry in acted)+++async def test_every_click_row_carries_a_hit_class_and_no_other_tool_does() -> None:+ """The facet has to be TOTAL over click rows: Datadog's groupBy drops rows missing a facet, so a+ click that returned before the probe ever ran would silently leave the denominator rather than+ showing up as the `unknown` it is. Gated on the tool so every other row keeps today's fields."""++ async def probed(args: dict[str, Any]) -> ToolResult:+ record_hit_class("non_target", needed=True, probe_seconds=0.012, isolated=True, raised=False)+ return ToolResult.ok("clicked")++ async def never_probed(args: dict[str, Any]) -> ToolResult:+ # A click that refused before reaching the hit test — the shape that must not vanish.+ return ToolResult.error("no element for selector '#gone'", error_class="stale_selector")++ async def other_tool(args: dict[str, Any]) -> ToolResult:+ return ToolResult.ok("hovered")++ tools = [+ ToolSpec(name="click", description="c", parameters={}, handler=probed, billable=True),+ ToolSpec(name="hover", description="h", parameters={}, handler=other_tool, billable=True),+ make_finish_tool(),+ ]+ script = [+ [("click", {"selector": "#a"})],+ [("hover", {"selector": "#b"})],+ [("finish", {"status": "completed", "reason": "ok"})],+ ]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools)+ assert outcome.status == "completed"+ by_tool = {e["tool"]: e for e in logs if e["event"] == "taskv3 tool call finished"}+ assert by_tool["click"]["hit_class"] == "non_target"+ # Which realm answered, whether the probe threw, and the decision it produced. The class alone+ # cannot recover the decision: a shadow-rooted target answers `unknown` with needed=True.+ assert by_tool["click"]["hit_probe_isolated"] is True+ assert by_tool["click"]["hit_probe_raised"] is False+ assert by_tool["click"]["hit_needed"] is True+ # What the probe cost where production calls it. The 15s actionability wait a pre-click probe+ # would short-circuit is only worth paying for if this is small, and nothing measured it before.+ assert by_tool["click"]["hit_probe_seconds"] == 0.012+ # A tool with no hit test carries no such field at all — absent, not "unknown".+ assert "hit_class" not in by_tool["hover"]++ tools[0] = ToolSpec(name="click", description="c", parameters={}, handler=never_probed, billable=True)+ with capture_logs() as logs:+ await _run(script, tools)+ unprobed = {e["tool"]: e for e in logs if e["event"] == "taskv3 tool call finished"}+ assert unprobed["click"]["hit_class"] == "unknown"+ # A probe that never ran carries no qualifiers -- absent, not a fabricated value.+ assert "hit_probe_isolated" not in unprobed["click"]+ assert "hit_probe_seconds" not in unprobed["click"]+ assert "hit_needed" not in unprobed["click"]+ assert "hit_probe_raised" not in unprobed["click"]++ # A run where `click` is not registered (page-free runs, or the REPLACE code surface) logs the+ # call as `unknown_tool`. Such a row must not carry a click-only facet: a groupBy on hit_class+ # without a tool filter would otherwise count it into `unknown`.+ with capture_logs() as logs:+ await _run(+ [[("click", {"selector": "#a"})], [("finish", {"status": "completed", "reason": "ok"})]],+ [make_finish_tool()],+ )+ unregistered = [e for e in logs if e["event"] == "taskv3 tool call finished"]+ assert any(e["tool"] == "unknown_tool" for e in unregistered)+ assert all("hit_class" not in e for e in unregistered)diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.pyindex 0a26a7e1a..2ef922244 100644--- a/tests/unit/test_taskv3_tools.py+++ b/tests/unit/test_taskv3_tools.py@@ -26,6 +26,7 @@ import pytestfrom playwright.async_api import Error as _PlaywrightErrorfrom structlog.testing import capture_logs+import skyvern.forge.taskv3.loop as taskv3_loopimport skyvern.forge.taskv3.tools as taskv3_toolsfrom skyvern.config import settingsfrom skyvern.forge.taskv3.code_surface import (@@ -23290,3 +23291,141 @@ def test_the_two_shared_row_refusals_carry_their_own_error_class() -> None:assert identical.error_class == "identical_rows", tags_live# Both wordings are one branch: the facet must not fork on which remedy the page allowed.assert "the field is NOT filled" in identical.content+++def test_the_widened_reach_probe_decides_exactly_what_the_boolean_decided() -> None:+ """Equivalence, not presence: the widened return must not move a single decision.++ A dict is always truthy in Python, so a caller left reading the raw return would run the expensive+ scrolling probe on EVERY click. That regression is invisible to any test that only asserts the new+ field is emitted, and it is on the path of every v3 click rather than only the covered ones.+ """+ needed = taskv3_tools._reach_probe_needed+ # The wire format is a PRIMITIVE, "<0|1>:<class>". An object return would be corruptible by the+ # page: Playwright's serializer runs in the page's world on bare Object.keys and+ # Object.prototype.toString, and a page that clobbers either turns an object into {}, into None,+ # or into truthy junk -- measured, all three -- which moves this decision in both directions.+ assert needed("1:non_target") is True+ assert needed("1:unknown") is True+ assert needed("0:self") is False+ assert needed("0:no_hit") is False+ # Anything that is not the protocol decides on its own truthiness, exactly as the bare boolean did.+ assert needed(True) is True+ assert needed(False) is False+ assert needed(None) is False+ assert needed({}) is False+ assert needed("garbage") is True+++def test_the_hit_class_is_total_over_every_shape_the_probe_can_answer_in() -> None:+ """A row with no class would vanish from a groupBy, so an absent reading must be a VALUE."""+ hit_class = taskv3_tools._reach_hit_class+ assert hit_class("0:self") == "self"+ # Both retired names are gone, not aliased: `foreign` claimed occlusion the probe cannot+ # establish, and `offscreen` claimed a cause for a null hit that its own boundary contradicts.+ assert hit_class("0:foreign") == "unknown"+ assert hit_class("0:offscreen") == "unknown"+ assert hit_class("1:non_target") == "non_target"+ assert hit_class("0:no_hit") == "no_hit"+ assert hit_class("1:unknown") == "unknown"+ # Anything the probe could not answer reads as unknown rather than as a missing field.+ assert hit_class(None) == "unknown"+ assert hit_class(True) == "unknown"+ assert hit_class({}) == "unknown"+ assert hit_class("0:invented_value") == "unknown"+++async def test_a_reach_probe_that_says_no_still_runs_only_one_probe() -> None:+ """The hot path: a clean hit must not buy the scrolling probe just because the answer got wider."""++ class _CountingPage(_FakePage):+ def __init__(self, answer: Any) -> None:+ super().__init__()+ self.answer = answer+ self.expensive_probes = 0++ async def evaluate(self, js: str, arg: Any = None) -> Any:+ # Matched on a string unique to the reach probe: the expensive probe also runs+ # elementFromPoint, so a looser match would swallow it and hide the regression.+ # Anchored on this probe's own wire format, which no other probe emits: the expensive+ # probe also runs elementFromPoint and the label scan, so a looser match swallows it.+ if '(needed ? "1:" : "0:")' in js:+ return self.answer+ # `slotted:` is unique to the expensive target probe; the skinned-checkbox probe also+ # calls scrollIntoView and runs regardless, so counting on that would never read zero.+ if "slotted:" in js:+ self.expensive_probes += 1+ return {"exists": True, "disabled": False}+ return await super().evaluate(js, arg)++ clean = _CountingPage("0:self")+ await _tool(build_browser_tools(_fixed_page_provider(clean)), "click").handler({"selector": "#submit"})+ assert clean.expensive_probes == 0+ assert (taskv3_loop._HIT_CLASS.get() or {}).get("hit_class") == "self"++ covered = _CountingPage("1:non_target")+ await _tool(build_browser_tools(_fixed_page_provider(covered)), "click").handler({"selector": "#submit"})+ assert covered.expensive_probes == 1+ recorded = taskv3_loop._HIT_CLASS.get() or {}+ assert recorded.get("hit_class") == "non_target"+ # The decision rides the record too: `unknown` alone cannot recover it, since a shadow-rooted+ # target answers `unknown` with needed=True.+ assert recorded.get("needed") is True+ assert recorded.get("raised") is False+ # Which realm answered -- the field is unwitnessed at this call site without an assertion here.+ assert recorded.get("isolated") in (True, False)+ # Nothing here claims occlusion: a real modal scrim drawn as `body::before` hit-tests as body and+ # still blocks the click, so no cheap signal in this probe separates an occluder from a fall-through.+ assert recorded.get("probe_seconds") is not None+ # The loop clears this per tool call; a unit test must not leave the process-global set either.+ taskv3_loop._HIT_CLASS.set(None)+++async def test_a_probe_that_raises_is_recorded_rather_than_dropped() -> None:+ """The raise path is not random: it correlates with re-render, which is the cohort of interest.++ Dropping those rows would bias the cost sample toward the calm cases — the defect that killed an+ earlier hover-provenance field on this same path. Recorded, and TAGGED: a duration that pools+ answers with blow-ups is two events under one name, so the cost figure filters on `raised`.+ """++ class _RaisingProbePage(_FakePage):+ async def evaluate(self, js: str, arg: Any = None) -> Any:+ if '(needed ? "1:" : "0:")' in js:+ raise RuntimeError("Execution context was destroyed")+ return await super().evaluate(js, arg)++ taskv3_loop._HIT_CLASS.set(None)+ await _tool(build_browser_tools(_fixed_page_provider(_RaisingProbePage())), "click").handler(+ {"selector": "#submit"}+ )++ recorded = taskv3_loop._HIT_CLASS.get() or {}+ assert recorded.get("raised") is True+ assert recorded.get("hit_class") == "unknown"+ # The decision a raising probe produces is the one the bare boolean produced: don't buy the+ # expensive probe on a question that was never answered.+ assert recorded.get("needed") is False+ # And the duration is still there, so the raise stratum has an n rather than vanishing.+ assert isinstance(recorded.get("probe_seconds"), float)+ taskv3_loop._HIT_CLASS.set(None)+++async def test_the_isolated_leg_is_recorded_as_isolated(monkeypatch: pytest.MonkeyPatch) -> None:+ """The True side of the realm flag had no witness: deleting the assignment left the suite green.++ A fake page has no CDP session, so every other test in this file exercises the fallback leg only —+ which is exactly how an unwitnessed branch survives a suite that looks thorough.+ """++ async def _isolated_answer(_target: Any, _js: str, _selector: str) -> Any:+ return "0:self"++ monkeypatch.setattr(taskv3_tools, "_evaluate_isolated", _isolated_answer)+ taskv3_loop._HIT_CLASS.set(None)+ await _tool(build_browser_tools(_fixed_page_provider(_FakePage())), "click").handler({"selector": "#submit"})+ assert (taskv3_loop._HIT_CLASS.get() or {}).get("isolated") is True+ taskv3_loop._HIT_CLASS.set(None)