Skip to content

Automate browser based workflows with AI

Name the click hit class for what it measures, and time the probe where production calls it (#8521)

pedrohsdb89c9148ec915
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index 5fa236690..afeb38f3c 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -132,6 +132,10 @@ TARGET_KIND_DATA_KEY = "target_kind"
# definitions with nothing on the record to cut on.
_RESOLVE_SECONDS: ContextVar[float | None] = ContextVar("taskv3_resolve_seconds", default=None)
_FRAME_PERCEPTION: ContextVar[bool | None] = ContextVar("taskv3_frame_perception", default=None)
+# Which case behind the click reach-probe's boolean applied, for the one click call this context covers.
+# A context variable rather than a result field because `click` returns from many places, several of
+# them after the probe has already answered -- the same reason `_RESOLVE_SECONDS` lives here.
+_HIT_CLASS: ContextVar[dict[str, Any] | None] = ContextVar("taskv3_hit_class", default=None)
def record_resolve_seconds(elapsed: float) -> None:
@@ -139,6 +143,37 @@ def record_resolve_seconds(elapsed: float) -> None:
_RESOLVE_SECONDS.set((_RESOLVE_SECONDS.get() or 0.0) + elapsed)
+def record_hit_class(
+ hit_class: str,
+ *,
+ needed: bool,
+ probe_seconds: float | None = None,
+ isolated: bool | None = None,
+ raised: bool = False,
+) -> None:
+ """Record what this click's reach probe returned, the decision it produced, and what it cost.
+
+ `needed` is on the record because the class alone cannot recover it: a shadow-rooted target
+ answers `unknown` with needed=TRUE, so `unknown` would otherwise pool the rows that bought the
+ second probe with the rows that answered nothing. Every `hit_class` value names what the hit test
+ RETURNED, never why — nothing in that probe tells a real occluder from a hit that fell through,
+ since a modal scrim drawn as `body::before` hit-tests as body and still blocks the click.
+ `isolated` says which realm answered -- realms, not costs, since its `False` side spans both the
+ cheapest fallback and the most expensive retry storm. `raised` marks a probe that threw, whose
+ duration is a blow-up rather than a cost. The RECORDED copies are inert: the click path branches on
+ its own `needed`, never on anything read back from here.
+ """
+ _HIT_CLASS.set(
+ {
+ "hit_class": hit_class,
+ "needed": needed,
+ "probe_seconds": probe_seconds,
+ "isolated": isolated,
+ "raised": raised,
+ }
+ )
+
+
def record_frame_perception(enabled: bool) -> None:
"""Stamp which of the two definitions above produced this call's reading. Telemetry only."""
_FRAME_PERCEPTION.set(enabled)
@@ -1357,6 +1392,11 @@ _TOOL_CALL_RECORD_FIELDS = frozenset(
"action_key_hash",
"snapshot_digest",
"probe_first_time",
+ "hit_class",
+ "hit_needed",
+ "hit_probe_seconds",
+ "hit_probe_isolated",
+ "hit_probe_raised",
}
)
@@ -2857,6 +2897,7 @@ async def run_agent_tool_loop(
# Cleared per call, so a value can never carry over from the previous one in the batch.
_RESOLVE_SECONDS.set(None)
_FRAME_PERCEPTION.set(None)
+ _HIT_CLASS.set(None)
tool_started_at = time.monotonic()
if spec is None:
result = ToolResult.error(f"unknown_tool: {tool_name}")
@@ -2899,6 +2940,29 @@ async def run_agent_tool_loop(
frame_perception = _FRAME_PERCEPTION.get()
if frame_perception is not None:
cost_fields["frame_perception"] = frame_perception
+ # Every click row carries this, defaulting to `unknown`, because a groupBy DROPS rows
+ # missing a facet -- a partial field would read as a clean result rather than a gap.
+ # Gated on `spec is not None` for the same reason the `tool` field is: an unregistered
+ # tool logs as `unknown_tool`, and such a row must not also carry a click-only facet.
+ if spec is not None and tool_name == "click":
+ hit = _HIT_CLASS.get() or {}
+ cost_fields["hit_class"] = hit.get("hit_class") or "unknown"
+ # Known for every click whether or not the probe ran, so it stays total alongside the
+ # class. The two qualifiers below are NOT total by design: they describe a reading
+ # that happened, and a fabricated `false` would be indistinguishable from a measured
+ # one -- so group them WITHIN a hit_class bucket, never alone.
+
+ # What the probe CALL cost at the production call site. Not the marginal round trip:
+ # the first reading per realm also pays isolated-world construction, which is why the
+ # median within one `hit_probe_isolated` bucket is the readable figure.
+ # Present together, iff the probe was attempted: `hit_class` is the only total one.
+ probe_seconds = hit.get("probe_seconds")
+ if probe_seconds is not None:
+ cost_fields["hit_probe_seconds"] = probe_seconds
+ cost_fields["hit_needed"] = bool(hit.get("needed"))
+ cost_fields["hit_probe_raised"] = bool(hit.get("raised"))
+ if hit.get("isolated") is not None:
+ cost_fields["hit_probe_isolated"] = hit["isolated"]
# The action-loop guard's key and the perception ledger's digest, computed here (pure) so
# their hashes ride the record below; the ledger itself is updated further down, unchanged.
action_key = (tool_name, json.dumps(args, sort_keys=True, default=str))
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 6cc10658a..9594cc98f 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -24,6 +24,7 @@ import time
import unicodedata
import weakref
from collections import Counter, defaultdict, deque
+from contextvars import ContextVar
from enum import Enum
from typing import TYPE_CHECKING, Any, Awaitable, Callable, NamedTuple
@@ -50,6 +51,7 @@ from skyvern.forge.taskv3.loop import (
ToolResult,
ToolSpec,
record_frame_perception,
+ record_hit_class,
record_resolve_seconds,
set_driver_timeout_predicate,
)
@@ -3028,25 +3030,49 @@ _REACH_PROBE_NEEDED_JS = (
"""
+ _NATIVE_LABEL_JS
+ r"""
+ // Answered as ONE STRING, "<0|1>:<class>", not an object. The decision has to survive the page-realm
+ // fallback, and Playwright's serializer runs in the PAGE's world using bare Object.keys and
+ // Object.prototype.toString: a page that clobbers either turns an object return into {}, into null,
+ // or into truthy junk -- measured, all three -- which moves `needed` in both directions on the leg
+ // this probe exists to be trusted on. A primitive passes through untouched.
+ //
+ // The class reports WHICH case produced the decision, because one boolean collapses seven distinct
+ // conditions -- not found, shadow-rooted, zero-area, self, non-target, null hit, and a throw inside
+ // the rect block -- into four classes. Telemetry only: no branch here reads it.
+ //
+ // `non_target` is named for what it measures -- the centre point hit something that is not the
+ // target -- and NOT for occlusion. A row clipped out of its own scroll container lands here with
+ // nothing painted over it and clicks fine; a pointer-events:none or visibility:hidden target lands
+ // here too and does NOT click, it burns the full actionability wait. Nothing in this probe separates
+ // any of them from a real occluder: a modal scrim drawn as `body::before` hit-tests AS body and
+ // blocks the click. Answering that needs the scroll and composed descent the mature probe pays for.
+ let needed = false;
+ let hitClass = "unknown";
+ const out = () => (needed ? "1:" : "0:") + hitClass;
const el = _q.find(arg.sel) || (arg.el && arg.el.isConnected ? arg.el : null);
- if (!el) return false;
- try { if (Node.prototype.getRootNode.call(el) !== document) return true; } catch (e) { /* fall through */ }
+ if (!el) return out();
+ // A shadow-rooted target returns before any hit test runs, so it is unknown rather than no_hit.
+ try { if (Node.prototype.getRootNode.call(el) !== document) { needed = true; return out(); } } catch (e) { /* fall through */ }
// The cheap hit-test runs first and exits on an ordinary unoccluded hit; the DOM-wide label scan
- // only runs for the null/foreign-hit cases where it can actually change the answer.
+ // only runs for the null and non-target hit cases where it can actually change the answer.
try {
const rect = el.getBoundingClientRect();
if (rect.width > 0 && rect.height > 0) {
const cx = rect.left + rect.width / 2;
const cy = rect.top + rect.height / 2;
const hit = document.elementFromPoint(cx, cy);
- if (hit === el || (hit && el.contains(hit))) return false;
+ // Every value names what the hit test RETURNED, never why. A null hit is `no_hit`: it usually
+ // means the point is outside the layout viewport, but that is an inference, and the boundary
+ // between the layout and visual viewports does not agree with it.
+ hitClass = (hit === el || (hit && el.contains(hit))) ? "self" : (hit ? "non_target" : "no_hit");
+ if (hitClass === "self") return out();
// A sibling <label for=id> drawn OVER its control trips the driver's containment check the same
// way a slotted label does; a null hit (off-screen target) also earns the probe when labels exist.
// Skipped when own-label granting is off (no isolated world): the scan only ever earns a bypass.
- if (arg.allowOwnLabel !== false && (nativeLabelsOf(el).length || (_isLabel(el) && nativeControlOf(el)))) return true;
+ if (arg.allowOwnLabel !== false && (nativeLabelsOf(el).length || (_isLabel(el) && nativeControlOf(el)))) needed = true;
}
} catch (e) { /* best-effort */ }
- return false;
+ return out();
}"""
)
@@ -7401,6 +7427,45 @@ def _merge_realm(into: dict[str, Any], other: dict[str, Any]) -> int:
return len(taken)
+_HIT_CLASSES = frozenset({"self", "non_target", "no_hit", "unknown"})
+
+
+def _reach_probe_needed(probe_result: Any) -> bool:
+ """The reach probe's decision, from the `"<0|1>:<class>"` answer.
+
+ Anything that is not that shape falls back to the value's own truthiness, which is what the bare
+ boolean did -- so a probe that could not answer decides exactly as it decided before.
+ """
+ if isinstance(probe_result, str) and ":" in probe_result:
+ return probe_result.split(":", 1)[0] == "1"
+ return bool(probe_result)
+
+
+def _reach_hit_class(probe_result: Any) -> str:
+ """Which hit-test case produced the decision. Total by construction: anything unrecognised is
+ `unknown`, never absent, because a row without the facet vanishes from a groupBy instead of
+ bucketing."""
+ if isinstance(probe_result, str) and ":" in probe_result:
+ value = probe_result.split(":", 1)[1]
+ if value in _HIT_CLASSES:
+ return value
+ return "unknown"
+
+
+# Which realm answered the last `_probe_evaluate`. It separates REALMS, not costs, and it must not be
+# read as a cost split: measured medians here are 0.56ms with an isolated world, 0.66ms on a cached
+# miss (no CDP session for the page -- the miss is cached, so later probes fall back doing no CDP work
+# at all), and 233ms when world creation keeps raising and the three-attempt loop runs. `False` spans
+# the cheapest path AND the most expensive one, so it narrows a duration without partitioning it.
+# Recorded because a duration that cannot say which realm answered cannot be compared across pages at
+# all -- the same reason `frame_perception` rides every row carrying a css reading.
+_PROBE_ISOLATED: ContextVar[bool | None] = ContextVar("taskv3_probe_isolated", default=None)
+
+
+def _probe_was_isolated() -> bool | None:
+ return _PROBE_ISOLATED.get()
+
+
async def _probe_evaluate(target: Any, js: str, selector: str, arg: dict[str, Any]) -> Any:
"""Isolated-world probe, falling back to the realm's OWN world with own-label granting disabled --
a realm the page can patch must not be able to hand a label a hit-test bypass.
@@ -7409,10 +7474,14 @@ async def _probe_evaluate(target: Any, js: str, selector: str, arg: dict[str, An
than dropping to the page's: losing isolation is the trade this fallback has always made, but
answering about the wrong document is not a trade, it is a wrong answer wearing a verdict's clothes.
"""
+ _PROBE_ISOLATED.set(None)
isolated = await _evaluate_isolated(target, js, selector)
if isolated is not None:
+ _PROBE_ISOLATED.set(True)
return isolated
- return await target.evaluate(js, {**arg, "allowOwnLabel": False})
+ answer = await target.evaluate(js, {**arg, "allowOwnLabel": False})
+ _PROBE_ISOLATED.set(False)
+ return answer
def build_browser_tools(
@@ -8785,7 +8854,28 @@ def build_browser_tools(
# <label for=id>) pays the extra round trip; any other light-DOM click keeps its single one.
reach_pre = None
try:
- if await _probe_evaluate(page, _REACH_PROBE_NEEDED_JS, selector, pre_click_arg):
+ # Timed at the production call site rather than on a bench: a local page.evaluate bounds
+ # only the browser-side half. The FIRST reading per realm also pays isolated-world
+ # construction (~10x locally), so this is the probe CALL's cost, not the marginal round
+ # trip -- read the median, and read it within one `hit_probe_isolated` bucket.
+ probe_started_at = time.monotonic()
+ reach_verdict: Any = None
+ raised = True
+ try:
+ reach_verdict = await _probe_evaluate(page, _REACH_PROBE_NEEDED_JS, selector, pre_click_arg)
+ raised = False
+ finally:
+ # Recorded even when the probe threw: the raise path correlates with re-render, so
+ # dropping it would bias the sample toward the calm cases. Tagged, because a duration
+ # that pools answers with blow-ups is two events under one name.
+ record_hit_class(
+ _reach_hit_class(reach_verdict),
+ needed=_reach_probe_needed(reach_verdict),
+ probe_seconds=time.monotonic() - probe_started_at,
+ isolated=_probe_was_isolated(),
+ raised=raised,
+ )
+ if _reach_probe_needed(reach_verdict):
reach_pre = await _probe_evaluate(page, _TYPE_TARGET_PROBE_JS, selector, pre_click_arg)
except Exception:
reach_pre = None
diff --git a/tests/unit/test_copilot_fill_credential_field.py b/tests/unit/test_copilot_fill_credential_field.py
index 2cf4176d8..eaec21f79 100644
--- a/tests/unit/test_copilot_fill_credential_field.py
+++ b/tests/unit/test_copilot_fill_credential_field.py
@@ -505,335 +505,343 @@ def _wire_impl(
monkeypatch.setattr(credential_fill_module, "_register_scout_interaction_observation", fake_register)
- @pytest.mark.asyncio
- async def test_target_identity_is_captured_before_fill_and_effect_afterward(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- events: list[str] = []
- class OrderedPage(_FakePage):
- async def fill(self, *args: Any, **kwargs: Any) -> None:
- events.append("fill")
- await super().fill(*args, **kwargs)
+async def test_target_identity_is_captured_before_fill_and_effect_afterward(monkeypatch: pytest.MonkeyPatch) -> None:
+ events: list[str] = []
- async def read_value(self, selector: str) -> str:
- events.append("post_effect")
- return await super().read_value(selector)
+ class OrderedPage(_FakePage):
+ async def fill(self, *args: Any, **kwargs: Any) -> None:
+ events.append("fill")
+ await super().fill(*args, **kwargs)
- page = OrderedPage()
- _wire_impl(monkeypatch, page)
+ async def read_value(self, selector: str) -> str:
+ events.append("post_effect")
+ return await super().read_value(selector)
- async def pre_fact(*_args: Any, **_kwargs: Any) -> int:
- events.append("pre_fact")
- return 1
+ page = OrderedPage()
+ _wire_impl(monkeypatch, page)
- async def selector_candidates(ctx: Any, _selector: str) -> None:
- events.append("selector_candidates")
- ctx.pending_scout_selector_candidates = [{"selector": 'input[name="password"]', "source": "name"}]
+ async def pre_fact(*_args: Any, **_kwargs: Any) -> int:
+ events.append("pre_fact")
+ return 1
- monkeypatch.setattr(credential_fill_module, "_selector_live_match_count", pre_fact)
- monkeypatch.setattr(credential_fill_module, "_capture_scout_selector_candidates", selector_candidates)
+ async def selector_candidates(ctx: Any, _selector: str) -> None:
+ events.append("selector_candidates")
+ ctx.pending_scout_selector_candidates = [{"selector": 'input[name="password"]', "source": "name"}]
- ctx = _ctx()
- result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")
+ monkeypatch.setattr(credential_fill_module, "_selector_live_match_count", pre_fact)
+ monkeypatch.setattr(credential_fill_module, "_capture_scout_selector_candidates", selector_candidates)
- assert result["ok"] is True
- assert events.index("selector_candidates") < events.index("fill")
- assert events.index("pre_fact") < events.index("fill") < events.index("post_effect")
- assert ctx.scout_trajectory[-1]["selector_match_count"] == 1
- assert ctx.scout_trajectory[-1]["observed_effects"]["value_landed"] is True
- assert ctx.scout_trajectory[-1]["selector_candidates"] == [
- {"selector": "#passwordInput", "source": "requested", "match_count": None},
- {"selector": 'input[name="password"]', "source": "name", "match_count": None},
- ]
+ ctx = _ctx()
+ result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")
- def test_tool_layer_readback_cannot_verify_a_short_secret(self) -> None:
- """Why the outcome is classified at the fill site, pinned against the real scrubber: a
- registered secret returns from the tool layer as the placeholder, which no code recognises."""
- otp = "mk-one"
- ctx = SimpleNamespace(secret_scrub_values=[], browser_session_id=None)
- register_secret_scrub_value(ctx, otp)
- readback = scrub_secrets_from_structure(ctx, {"ok": True, "data": {"value": otp}})["data"]["value"]
+ assert result["ok"] is True
+ assert events.index("selector_candidates") < events.index("fill")
+ assert events.index("pre_fact") < events.index("fill") < events.index("post_effect")
+ assert ctx.scout_trajectory[-1]["selector_match_count"] == 1
+ assert ctx.scout_trajectory[-1]["observed_effects"]["value_landed"] is True
+ assert ctx.scout_trajectory[-1]["selector_candidates"] == [
+ {"selector": "#passwordInput", "source": "requested", "match_count": None},
+ {"selector": 'input[name="password"]', "source": "name", "match_count": None},
+ ]
- assert (
- mcp_hooks_module._scout_readback_outcome(readback, otp) is mcp_hooks_module.ScoutReadbackOutcome.DIFFERENT
- )
- assert mcp_hooks_module._scout_readback_outcome(otp, otp) is mcp_hooks_module.ScoutReadbackOutcome.EXACT_MATCH
- @pytest.mark.asyncio
- async def test_multi_match_selector_still_reaches_an_outcome(self, monkeypatch: pytest.MonkeyPatch) -> None:
- """Reading the un-narrowed locator raises Playwright strict mode, which would leave the
- field unread; the readback narrows the way the fill did."""
- page = _FakePage(readback="8675309" + "123456")
- page.selector_match_count = 2
- _wire_impl(monkeypatch, page, secret_value="123456")
- ctx = _ctx()
+def test_tool_layer_readback_cannot_verify_a_short_secret() -> None:
+ """Why the outcome is classified at the fill site, pinned against the real scrubber: a
+ registered secret returns from the tool layer as the placeholder, which no code recognises."""
+ otp = "mk-one"
+ ctx = SimpleNamespace(secret_scrub_values=[], browser_session_id=None)
+ register_secret_scrub_value(ctx, otp)
+ readback = scrub_secrets_from_structure(ctx, {"ok": True, "data": {"value": otp}})["data"]["value"]
- result = await tools_module._fill_credential_field_impl(ctx, "input.otp", "cred_123", "totp")
+ assert mcp_hooks_module._scout_readback_outcome(readback, otp) is mcp_hooks_module.ScoutReadbackOutcome.DIFFERENT
+ assert mcp_hooks_module._scout_readback_outcome(otp, otp) is mcp_hooks_module.ScoutReadbackOutcome.EXACT_MATCH
- assert result["ok"] is True
- assert result["data"]["readback_outcome"] == "different"
- @pytest.mark.asyncio
- @pytest.mark.parametrize(
- ("secret_value", "readback"),
- [
- ("marker-value-one", "marker-val"),
- ("alphabetagamma", "alpha beta gamma"),
- ("marker-value-longer", "marker-value-lo"),
- ],
- )
- async def test_a_field_not_holding_what_was_typed_is_reported_without_failing_or_claiming_a_wrong_field(
- self, monkeypatch: pytest.MonkeyPatch, secret_value: str, readback: str
- ) -> None:
- page = _FakePage(readback=readback)
- _wire_impl(monkeypatch, page, secret_value=secret_value)
- ctx = _ctx()
+async def test_multi_match_selector_still_reaches_an_outcome(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Reading the un-narrowed locator raises Playwright strict mode, which would leave the
+ field unread; the readback narrows the way the fill did."""
+ page = _FakePage(readback="8675309" + "123456")
+ page.selector_match_count = 2
+ _wire_impl(monkeypatch, page, secret_value="123456")
+ ctx = _ctx()
+
+ result = await tools_module._fill_credential_field_impl(ctx, "input.otp", "cred_123", "totp")
+
+ assert result["ok"] is True
+ assert result["data"]["readback_outcome"] == "different"
+
+
+ ("secret_value", "readback"),
+ [
+ ("marker-value-one", "marker-val"),
+ ("alphabetagamma", "alpha beta gamma"),
+ ("marker-value-longer", "marker-value-lo"),
+ ],
+)
+async def test_a_field_not_holding_what_was_typed_is_reported_without_failing_or_claiming_a_wrong_field(
+ monkeypatch: pytest.MonkeyPatch, secret_value: str, readback: str
+) -> None:
+ page = _FakePage(readback=readback)
+ _wire_impl(monkeypatch, page, secret_value=secret_value)
+ ctx = _ctx()
+
+ result = await tools_module._fill_credential_field_impl(ctx, "#totp", "cred_123", "totp")
+
+ assert result["ok"] is True
+ assert result["data"]["readback_outcome"] == "different"
+ assert result["data"]["landing_inferred_from_navigation"] is False
+ assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})
+ assert "wrong" not in json.dumps(result)
+ assert "different input" not in json.dumps(result)
+
+
+async def test_unreadable_field_records_the_fill_and_says_so(monkeypatch: pytest.MonkeyPatch) -> None:
+ """An unreadable readback must not fail a fill that may well have landed, but must be visible."""
+
+ class UnreadablePage(_FakePage):
+ async def read_value(self, selector: str) -> str:
+ raise RuntimeError("element is not an <input>")
+
+ page = UnreadablePage()
+ _wire_impl(monkeypatch, page, secret_value="123456")
+ ctx = _ctx()
+ with capture_logs() as logs:
result = await tools_module._fill_credential_field_impl(ctx, "#totp", "cred_123", "totp")
- assert result["ok"] is True
- assert result["data"]["readback_outcome"] == "different"
- assert result["data"]["landing_inferred_from_navigation"] is False
- assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})
- assert "wrong" not in json.dumps(result)
- assert "different input" not in json.dumps(result)
+ assert result["ok"] is True
+ assert result["data"]["readback_outcome"] == "unavailable"
+ assert result["data"]["landing_inferred_from_navigation"] is False
+ assert ctx.scout_trajectory[-1]["credential_field"] == "totp"
+ assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})
+ assert any(
+ entry.get("event") == "copilot fill_credential_field readback outcome" and entry.get("outcome") == "unavailable"
+ for entry in logs
+ )
- @pytest.mark.asyncio
- async def test_unreadable_field_records_the_fill_and_says_so(self, monkeypatch: pytest.MonkeyPatch) -> None:
- """An unreadable readback must not fail a fill that may well have landed, but must be visible."""
- class UnreadablePage(_FakePage):
- async def read_value(self, selector: str) -> str:
- raise RuntimeError("element is not an <input>")
+async def test_happy_path_fills_and_records_value_free(monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _FakePage()
+ _wire_impl(monkeypatch, page)
+ ctx = _ctx()
- page = UnreadablePage()
- _wire_impl(monkeypatch, page, secret_value="123456")
- ctx = _ctx()
+ result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")
- with capture_logs() as logs:
- result = await tools_module._fill_credential_field_impl(ctx, "#totp", "cred_123", "totp")
+ assert result["ok"] is True
+ assert page.fill_calls == [("#passwordInput", _FAKE_PASSWORD)]
+ assert page.fill_kwargs[0]["mode"] == "direct"
+ assert result["data"]["typed_length"] == len(_FAKE_PASSWORD)
+ assert result["data"]["credential_id"] == "cred_123"
+ assert result["data"]["field"] == "password"
+ assert result["data"]["observation_step"] == 3
+ assert result["data"]["credential_name"] == "authtest simple"
+ assert result["data"]["readback_outcome"] == "exact_match"
+ assert result["data"]["landing_inferred_from_navigation"] is False
+ assert "credential_parameter" not in result["data"]
+ assert _FAKE_PASSWORD not in json.dumps(result)
- assert result["ok"] is True
- assert result["data"]["readback_outcome"] == "unavailable"
- assert result["data"]["landing_inferred_from_navigation"] is False
- assert ctx.scout_trajectory[-1]["credential_field"] == "totp"
- assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})
- assert any(
- entry.get("event") == "copilot fill_credential_field readback outcome"
- and entry.get("outcome") == "unavailable"
- for entry in logs
- )
+ assert len(ctx.scouted_interactions) == 1
+ recorded = ctx.scouted_interactions[0]
+ assert recorded["tool_name"] == "fill_credential_field"
+ assert recorded["credential_id"] == "cred_123"
+ assert recorded["credential_field"] == "password"
+ assert recorded["credential_name"] == "authtest simple"
+ assert recorded["typed_length"] == len(_FAKE_PASSWORD)
+ assert recorded["observed_effects"]["value_landed"] is True
+ assert _FAKE_PASSWORD not in json.dumps(recorded)
+ assert _FAKE_PASSWORD not in json.dumps(ctx.scout_trajectory)
- @pytest.mark.asyncio
- async def test_happy_path_fills_and_records_value_free(self, monkeypatch: pytest.MonkeyPatch) -> None:
- page = _FakePage()
- _wire_impl(monkeypatch, page)
- ctx = _ctx()
- result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")
+async def test_email_otp_scout_failure_returns_only_factual_credential_identity(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ page = _FakePage()
+ _wire_impl(monkeypatch, page)
- assert result["ok"] is True
- assert page.fill_calls == [("#passwordInput", _FAKE_PASSWORD)]
- assert page.fill_kwargs[0]["mode"] == "direct"
- assert result["data"]["typed_length"] == len(_FAKE_PASSWORD)
- assert result["data"]["credential_id"] == "cred_123"
- assert result["data"]["field"] == "password"
- assert result["data"]["observation_step"] == 3
- assert result["data"]["credential_name"] == "authtest simple"
- assert result["data"]["readback_outcome"] == "exact_match"
- assert result["data"]["landing_inferred_from_navigation"] is False
- assert "credential_parameter" not in result["data"]
- assert _FAKE_PASSWORD not in json.dumps(result)
-
- assert len(ctx.scouted_interactions) == 1
- recorded = ctx.scouted_interactions[0]
- assert recorded["tool_name"] == "fill_credential_field"
- assert recorded["credential_id"] == "cred_123"
- assert recorded["credential_field"] == "password"
- assert recorded["credential_name"] == "authtest simple"
- assert recorded["typed_length"] == len(_FAKE_PASSWORD)
- assert recorded["observed_effects"]["value_landed"] is True
- assert _FAKE_PASSWORD not in json.dumps(recorded)
- assert _FAKE_PASSWORD not in json.dumps(ctx.scout_trajectory)
-
- @pytest.mark.asyncio
- async def test_email_otp_scout_failure_returns_only_factual_credential_identity(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- page = _FakePage()
- _wire_impl(monkeypatch, page)
+ async def runtime_only_otp(_ctx: Any, _credential_id: str, _field: str) -> tuple[None, str, str]:
+ return None, "authtest simple", "Email OTP requires workflow-run polling."
- async def runtime_only_otp(_ctx: Any, _credential_id: str, _field: str) -> tuple[None, str, str]:
- return None, "authtest simple", "Email OTP requires workflow-run polling."
+ monkeypatch.setattr(credential_fill_module, "_resolve_credential_fill_value", runtime_only_otp)
- monkeypatch.setattr(credential_fill_module, "_resolve_credential_fill_value", runtime_only_otp)
+ result = await tools_module._fill_credential_field_impl(_ctx(), "#otp", "cred_123", "totp")
- result = await tools_module._fill_credential_field_impl(_ctx(), "#otp", "cred_123", "totp")
+ assert result["ok"] is False
+ assert result["data"] == {
+ "credential_id": "cred_123",
+ "credential_name": "authtest simple",
+ "credential_field": "totp",
+ }
+ assert page.fill_calls == []
- assert result["ok"] is False
- assert result["data"] == {
- "credential_id": "cred_123",
- "credential_name": "authtest simple",
- "credential_field": "totp",
- }
- assert page.fill_calls == []
- @pytest.mark.asyncio
- async def test_fill_error_text_is_scrubbed(self, monkeypatch: pytest.MonkeyPatch) -> None:
- page = _FakePage(fill_error=RuntimeError(f"could not type {_FAKE_PASSWORD} into element"))
- _wire_impl(monkeypatch, page)
+async def test_fill_error_text_is_scrubbed(monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _FakePage(fill_error=RuntimeError(f"could not type {_FAKE_PASSWORD} into element"))
+ _wire_impl(monkeypatch, page)
- result = await tools_module._fill_credential_field_impl(_ctx(), "#passwordInput", "cred_123", "password")
+ result = await tools_module._fill_credential_field_impl(_ctx(), "#passwordInput", "cred_123", "password")
- assert result["ok"] is False
- assert _FAKE_PASSWORD not in result["error"]
- assert "[REDACTED_SECRET]" in result["error"]
+ assert result["ok"] is False
+ assert _FAKE_PASSWORD not in result["error"]
+ assert "[REDACTED_SECRET]" in result["error"]
- @pytest.mark.asyncio
- async def test_rejects_unknown_field(self, monkeypatch: pytest.MonkeyPatch) -> None:
- page = _FakePage()
- _wire_impl(monkeypatch, page)
- result = await tools_module._fill_credential_field_impl(_ctx(), "#cvv", "cred_123", "cvv")
+async def test_rejects_unknown_field(monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _FakePage()
+ _wire_impl(monkeypatch, page)
- assert result["ok"] is False
- assert "username, password, totp" in result["error"]
- assert page.fill_calls == []
+ result = await tools_module._fill_credential_field_impl(_ctx(), "#cvv", "cred_123", "cvv")
- @pytest.mark.asyncio
- async def test_rejects_empty_selector(self, monkeypatch: pytest.MonkeyPatch) -> None:
- page = _FakePage()
- _wire_impl(monkeypatch, page)
+ assert result["ok"] is False
+ assert "username, password, totp" in result["error"]
+ assert page.fill_calls == []
- result = await tools_module._fill_credential_field_impl(_ctx(), " ", "cred_123", "password")
- assert result["ok"] is False
- assert page.fill_calls == []
+async def test_rejects_empty_selector(monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _FakePage()
+ _wire_impl(monkeypatch, page)
- @pytest.mark.asyncio
- async def test_unresolved_credential_never_reaches_vault_or_page(self, monkeypatch: pytest.MonkeyPatch) -> None:
- page = _FakePage()
- _wire_impl(monkeypatch, page)
- resolver = AsyncMock()
- monkeypatch.setattr(credential_fill_module, "_resolve_credential_fill_value", resolver)
+ result = await tools_module._fill_credential_field_impl(_ctx(), " ", "cred_123", "password")
- result = await tools_module._fill_credential_field_impl(_ctx(), "#passwordInput", "cred_999", "password")
+ assert result["ok"] is False
+ assert page.fill_calls == []
- assert result["ok"] is False
- assert "cred_999" in result["error"]
- resolver.assert_not_awaited()
- assert page.fill_calls == []
- @pytest.mark.asyncio
- async def test_standard_mode_is_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None:
- page = _FakePage()
- _wire_impl(monkeypatch, page)
- ctx = _ctx(block_authoring_policy=BlockAuthoringPolicy.STANDARD)
+async def test_unresolved_credential_never_reaches_vault_or_page(monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _FakePage()
+ _wire_impl(monkeypatch, page)
+ resolver = AsyncMock()
+ monkeypatch.setattr(credential_fill_module, "_resolve_credential_fill_value", resolver)
+
+ result = await tools_module._fill_credential_field_impl(_ctx(), "#passwordInput", "cred_999", "password")
+
+ assert result["ok"] is False
+ assert "cred_999" in result["error"]
+ resolver.assert_not_awaited()
+ assert page.fill_calls == []
+
+
+async def test_standard_mode_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _FakePage()
+ _wire_impl(monkeypatch, page)
+ ctx = _ctx(block_authoring_policy=BlockAuthoringPolicy.STANDARD)
+ result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")
+
+ assert result["ok"] is False
+ assert page.fill_calls == []
+
+
+async def test_readback_failure_surfaces_and_skips_recording(monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _FakePage(readback="")
+ _wire_impl(monkeypatch, page)
+ ctx = _ctx()
+
+ with capture_logs() as logs:
result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")
- assert result["ok"] is False
- assert page.fill_calls == []
+ assert result["ok"] is False
+ assert "still empty" in result["error"]
+ assert result["data"]["readback_outcome"] == "empty"
+ assert result["data"]["selector"] == "#passwordInput"
+ assert result["data"]["typed_length"] == len(_FAKE_PASSWORD)
+ assert result["data"]["landing_inferred_from_navigation"] is False
+ assert ctx.scouted_interactions == []
+ assert any(
+ entry.get("event") == "copilot fill_credential_field readback outcome" and entry.get("outcome") == "empty"
+ for entry in logs
+ )
- @pytest.mark.asyncio
- async def test_readback_failure_surfaces_and_skips_recording(self, monkeypatch: pytest.MonkeyPatch) -> None:
- page = _FakePage(readback="")
- _wire_impl(monkeypatch, page)
- ctx = _ctx()
- with capture_logs() as logs:
- result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")
+async def test_a_field_cleared_by_its_own_submit_is_not_reported_as_a_lost_fill(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ page = _FakePage(readback="")
+ _wire_impl(monkeypatch, page, secret_value="123456")
- assert result["ok"] is False
- assert "still empty" in result["error"]
- assert result["data"]["readback_outcome"] == "empty"
- assert result["data"]["selector"] == "#passwordInput"
- assert result["data"]["typed_length"] == len(_FAKE_PASSWORD)
- assert result["data"]["landing_inferred_from_navigation"] is False
- assert ctx.scouted_interactions == []
- assert any(
- entry.get("event") == "copilot fill_credential_field readback outcome" and entry.get("outcome") == "empty"
- for entry in logs
- )
+ async def navigated(_ctx: Any) -> str:
+ return _FIXTURE_LOGIN_URL + "verified/"
- @pytest.mark.asyncio
- async def test_a_field_cleared_by_its_own_submit_is_not_reported_as_a_lost_fill(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- page = _FakePage(readback="")
- _wire_impl(monkeypatch, page, secret_value="123456")
+ monkeypatch.setattr(credential_fill_module, "_live_working_page_url", navigated)
+ ctx = _ctx()
- async def navigated(_ctx: Any) -> str:
- return _FIXTURE_LOGIN_URL + "verified/"
+ result = await tools_module._fill_credential_field_impl(ctx, "#totpCode", "cred_123", "totp")
- monkeypatch.setattr(credential_fill_module, "_live_working_page_url", navigated)
- ctx = _ctx()
+ # The form committed on the last digit and cleared its own field. Calling that a lost fill
+ # sends the model back to re-type a code into a page the sign-in has already left.
+ assert result["ok"] is True
+ assert [entry["tool_name"] for entry in ctx.scouted_interactions] == ["fill_credential_field"]
+ # The readback said the field was empty. The model is told the landing was inferred, not
+ # that anyone saw the value sitting there.
+ assert ctx.scouted_interactions[0]["observed_effects"]["landing_inferred_from_navigation"] is True
+ assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})
+ assert result["data"]["readback_outcome"] == "empty"
+ assert result["data"]["landing_inferred_from_navigation"] is True
- result = await tools_module._fill_credential_field_impl(ctx, "#totpCode", "cred_123", "totp")
- # The form committed on the last digit and cleared its own field. Calling that a lost fill
- # sends the model back to re-type a code into a page the sign-in has already left.
- assert result["ok"] is True
- assert [entry["tool_name"] for entry in ctx.scouted_interactions] == ["fill_credential_field"]
- # The readback said the field was empty. The model is told the landing was inferred, not
- # that anyone saw the value sitting there.
- assert ctx.scouted_interactions[0]["observed_effects"]["landing_inferred_from_navigation"] is True
- assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})
- assert result["data"]["readback_outcome"] == "empty"
- assert result["data"]["landing_inferred_from_navigation"] is True
+async def test_a_navigation_cannot_stand_in_for_a_readback_nobody_took(monkeypatch: pytest.MonkeyPatch) -> None:
+ class UnreadablePage(_FakePage):
+ async def read_value(self, selector: str) -> str:
+ raise RuntimeError("execution context was destroyed")
- @pytest.mark.asyncio
- async def test_a_navigation_cannot_stand_in_for_a_readback_nobody_took(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- class UnreadablePage(_FakePage):
- async def read_value(self, selector: str) -> str:
- raise RuntimeError("execution context was destroyed")
+ page = UnreadablePage()
+ _wire_impl(monkeypatch, page, secret_value="mk-one")
- page = UnreadablePage()
- _wire_impl(monkeypatch, page, secret_value="mk-one")
+ async def navigated(_ctx: Any) -> str:
+ return _FIXTURE_LOGIN_URL + "verified/"
- async def navigated(_ctx: Any) -> str:
- return _FIXTURE_LOGIN_URL + "verified/"
+ monkeypatch.setattr(credential_fill_module, "_live_working_page_url", navigated)
+ ctx = _ctx()
- monkeypatch.setattr(credential_fill_module, "_live_working_page_url", navigated)
- ctx = _ctx()
+ result = await tools_module._fill_credential_field_impl(ctx, "#totpCode", "cred_123", "totp")
- result = await tools_module._fill_credential_field_impl(ctx, "#totpCode", "cred_123", "totp")
+ assert result["ok"] is True
+ assert result["data"]["readback_outcome"] == "unavailable"
+ assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})
+ assert "landing_inferred_from_navigation" not in ctx.scouted_interactions[0].get("observed_effects", {})
- assert result["ok"] is True
- assert result["data"]["readback_outcome"] == "unavailable"
- assert "value_landed" not in ctx.scouted_interactions[0].get("observed_effects", {})
- assert "landing_inferred_from_navigation" not in ctx.scouted_interactions[0].get("observed_effects", {})
- @pytest.mark.asyncio
- async def test_a_rejected_code_re_rendering_the_same_page_is_still_a_lost_fill(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- page = _FakePage(readback="")
- _wire_impl(monkeypatch, page, secret_value="123456")
+async def test_a_rejected_code_re_rendering_the_same_page_is_still_a_lost_fill(monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _FakePage(readback="")
+ _wire_impl(monkeypatch, page, secret_value="123456")
- async def same_page_with_error(_ctx: Any) -> str:
- return _FIXTURE_LOGIN_URL + "?error=invalid"
+ async def same_page_with_error(_ctx: Any) -> str:
+ return _FIXTURE_LOGIN_URL + "?error=invalid"
- monkeypatch.setattr(credential_fill_module, "_live_working_page_url", same_page_with_error)
+ monkeypatch.setattr(credential_fill_module, "_live_working_page_url", same_page_with_error)
- result = await tools_module._fill_credential_field_impl(_ctx(), "#totpCode", "cred_123", "totp")
+ result = await tools_module._fill_credential_field_impl(_ctx(), "#totpCode", "cred_123", "totp")
- # A rejected code re-renders the same page with an error param. That is not the form having
- # carried the code away, so the empty field still means the fill did not land.
- assert result["ok"] is False
+ # A rejected code re-renders the same page with an error param. That is not the form having
+ # carried the code away, so the empty field still means the fill did not land.
+ assert result["ok"] is False
- @pytest.mark.asyncio
- async def test_a_verified_fill_is_still_reported_as_observed(self, monkeypatch: pytest.MonkeyPatch) -> None:
- page = _FakePage(readback=_FAKE_PASSWORD)
- _wire_impl(monkeypatch, page)
- ctx = _ctx()
- await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")
+async def test_a_verified_fill_is_still_reported_as_observed(monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _FakePage(readback=_FAKE_PASSWORD)
+ _wire_impl(monkeypatch, page)
+ ctx = _ctx()
+
+ await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")
- assert "landing_inferred_from_navigation" not in ctx.scouted_interactions[0]["observed_effects"]
+ assert "landing_inferred_from_navigation" not in ctx.scouted_interactions[0]["observed_effects"]
class TestCredentialFillInCallSubmit:
diff --git a/tests/unit/test_nested_test_def_collection_guard.py b/tests/unit/test_nested_test_def_collection_guard.py
new file mode 100644
index 000000000..122c3c656
--- /dev/null
+++ b/tests/unit/test_nested_test_def_collection_guard.py
@@ -0,0 +1,34 @@
+"""A `test_*` def nested inside a helper function is never collected, so a deleted class header can
+silently retire a whole block of tests (SKY-16226). A test nested inside another test is a local fake."""
+
+import ast
+from pathlib import Path
+
+TESTS_ROOT = Path(__file__).resolve().parents[1]
+
+
+def _test_defs_nested_in_helpers(tree: ast.Module) -> list[tuple[int, str, str]]:
+ offenders: list[tuple[int, str, str]] = []
+
+ def walk(node: ast.AST, enclosing_helper: str | None) -> None:
+ for child in ast.iter_child_nodes(node):
+ helper = enclosing_helper
+ if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ if child.name.startswith("test_") and enclosing_helper is not None:
+ offenders.append((child.lineno, enclosing_helper, child.name))
+ if not child.name.startswith("test_"):
+ helper = helper or child.name
+ walk(child, helper)
+
+ walk(tree, None)
+ return offenders
+
+
+def test_no_test_def_is_nested_inside_a_helper_function() -> None:
+ offenders: list[str] = []
+ for pattern in ("test_*.py", "*_test.py"):
+ for path in sorted(TESTS_ROOT.rglob(pattern)):
+ tree = ast.parse(path.read_text(), filename=str(path))
+ for lineno, helper, name in _test_defs_nested_in_helpers(tree):
+ offenders.append(f"{path.relative_to(TESTS_ROOT.parent)}:{lineno} {name} is nested inside {helper}")
+ assert offenders == [], "\n".join(offenders)
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index d2e0afd86..8ccf51058 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -75,6 +75,7 @@ from skyvern.forge.taskv3.loop import (
_RevisitMemory,
make_finish_tool,
record_frame_perception,
+ record_hit_class,
record_resolve_seconds,
run_agent_tool_loop,
)
@@ -9588,3 +9589,70 @@ async def test_a_round_carries_billability_from_the_spec_not_from_a_name_list()
acted = [entry for round_ in rounds for entry in round_ if entry.tool == "execute_python"]
assert acted, "the billable tool should have produced a round entry"
assert all(entry.billable for entry in acted)
+
+
+async def test_every_click_row_carries_a_hit_class_and_no_other_tool_does() -> None:
+ """The facet has to be TOTAL over click rows: Datadog's groupBy drops rows missing a facet, so a
+ click that returned before the probe ever ran would silently leave the denominator rather than
+ showing up as the `unknown` it is. Gated on the tool so every other row keeps today's fields."""
+
+ async def probed(args: dict[str, Any]) -> ToolResult:
+ record_hit_class("non_target", needed=True, probe_seconds=0.012, isolated=True, raised=False)
+ return ToolResult.ok("clicked")
+
+ async def never_probed(args: dict[str, Any]) -> ToolResult:
+ # A click that refused before reaching the hit test — the shape that must not vanish.
+ return ToolResult.error("no element for selector '#gone'", error_class="stale_selector")
+
+ async def other_tool(args: dict[str, Any]) -> ToolResult:
+ return ToolResult.ok("hovered")
+
+ tools = [
+ ToolSpec(name="click", description="c", parameters={}, handler=probed, billable=True),
+ ToolSpec(name="hover", description="h", parameters={}, handler=other_tool, billable=True),
+ make_finish_tool(),
+ ]
+ script = [
+ [("click", {"selector": "#a"})],
+ [("hover", {"selector": "#b"})],
+ [("finish", {"status": "completed", "reason": "ok"})],
+ ]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools)
+ assert outcome.status == "completed"
+ by_tool = {e["tool"]: e for e in logs if e["event"] == "taskv3 tool call finished"}
+ assert by_tool["click"]["hit_class"] == "non_target"
+ # Which realm answered, whether the probe threw, and the decision it produced. The class alone
+ # cannot recover the decision: a shadow-rooted target answers `unknown` with needed=True.
+ assert by_tool["click"]["hit_probe_isolated"] is True
+ assert by_tool["click"]["hit_probe_raised"] is False
+ assert by_tool["click"]["hit_needed"] is True
+ # What the probe cost where production calls it. The 15s actionability wait a pre-click probe
+ # would short-circuit is only worth paying for if this is small, and nothing measured it before.
+ assert by_tool["click"]["hit_probe_seconds"] == 0.012
+ # A tool with no hit test carries no such field at all — absent, not "unknown".
+ assert "hit_class" not in by_tool["hover"]
+
+ tools[0] = ToolSpec(name="click", description="c", parameters={}, handler=never_probed, billable=True)
+ with capture_logs() as logs:
+ await _run(script, tools)
+ unprobed = {e["tool"]: e for e in logs if e["event"] == "taskv3 tool call finished"}
+ assert unprobed["click"]["hit_class"] == "unknown"
+ # A probe that never ran carries no qualifiers -- absent, not a fabricated value.
+ assert "hit_probe_isolated" not in unprobed["click"]
+ assert "hit_probe_seconds" not in unprobed["click"]
+ assert "hit_needed" not in unprobed["click"]
+ assert "hit_probe_raised" not in unprobed["click"]
+
+ # A run where `click` is not registered (page-free runs, or the REPLACE code surface) logs the
+ # call as `unknown_tool`. Such a row must not carry a click-only facet: a groupBy on hit_class
+ # without a tool filter would otherwise count it into `unknown`.
+ with capture_logs() as logs:
+ await _run(
+ [[("click", {"selector": "#a"})], [("finish", {"status": "completed", "reason": "ok"})]],
+ [make_finish_tool()],
+ )
+ unregistered = [e for e in logs if e["event"] == "taskv3 tool call finished"]
+ assert any(e["tool"] == "unknown_tool" for e in unregistered)
+ assert all("hit_class" not in e for e in unregistered)
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 0a26a7e1a..2ef922244 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -26,6 +26,7 @@ import pytest
from playwright.async_api import Error as _PlaywrightError
from structlog.testing import capture_logs
+import skyvern.forge.taskv3.loop as taskv3_loop
import skyvern.forge.taskv3.tools as taskv3_tools
from skyvern.config import settings
from skyvern.forge.taskv3.code_surface import (
@@ -23290,3 +23291,141 @@ def test_the_two_shared_row_refusals_carry_their_own_error_class() -> None:
assert identical.error_class == "identical_rows", tags_live
# Both wordings are one branch: the facet must not fork on which remedy the page allowed.
assert "the field is NOT filled" in identical.content
+
+
+def test_the_widened_reach_probe_decides_exactly_what_the_boolean_decided() -> None:
+ """Equivalence, not presence: the widened return must not move a single decision.
+
+ A dict is always truthy in Python, so a caller left reading the raw return would run the expensive
+ scrolling probe on EVERY click. That regression is invisible to any test that only asserts the new
+ field is emitted, and it is on the path of every v3 click rather than only the covered ones.
+ """
+ needed = taskv3_tools._reach_probe_needed
+ # The wire format is a PRIMITIVE, "<0|1>:<class>". An object return would be corruptible by the
+ # page: Playwright's serializer runs in the page's world on bare Object.keys and
+ # Object.prototype.toString, and a page that clobbers either turns an object into {}, into None,
+ # or into truthy junk -- measured, all three -- which moves this decision in both directions.
+ assert needed("1:non_target") is True
+ assert needed("1:unknown") is True
+ assert needed("0:self") is False
+ assert needed("0:no_hit") is False
+ # Anything that is not the protocol decides on its own truthiness, exactly as the bare boolean did.
+ assert needed(True) is True
+ assert needed(False) is False
+ assert needed(None) is False
+ assert needed({}) is False
+ assert needed("garbage") is True
+
+
+def test_the_hit_class_is_total_over_every_shape_the_probe_can_answer_in() -> None:
+ """A row with no class would vanish from a groupBy, so an absent reading must be a VALUE."""
+ hit_class = taskv3_tools._reach_hit_class
+ assert hit_class("0:self") == "self"
+ # Both retired names are gone, not aliased: `foreign` claimed occlusion the probe cannot
+ # establish, and `offscreen` claimed a cause for a null hit that its own boundary contradicts.
+ assert hit_class("0:foreign") == "unknown"
+ assert hit_class("0:offscreen") == "unknown"
+ assert hit_class("1:non_target") == "non_target"
+ assert hit_class("0:no_hit") == "no_hit"
+ assert hit_class("1:unknown") == "unknown"
+ # Anything the probe could not answer reads as unknown rather than as a missing field.
+ assert hit_class(None) == "unknown"
+ assert hit_class(True) == "unknown"
+ assert hit_class({}) == "unknown"
+ assert hit_class("0:invented_value") == "unknown"
+
+
+async def test_a_reach_probe_that_says_no_still_runs_only_one_probe() -> None:
+ """The hot path: a clean hit must not buy the scrolling probe just because the answer got wider."""
+
+ class _CountingPage(_FakePage):
+ def __init__(self, answer: Any) -> None:
+ super().__init__()
+ self.answer = answer
+ self.expensive_probes = 0
+
+ async def evaluate(self, js: str, arg: Any = None) -> Any:
+ # Matched on a string unique to the reach probe: the expensive probe also runs
+ # elementFromPoint, so a looser match would swallow it and hide the regression.
+ # Anchored on this probe's own wire format, which no other probe emits: the expensive
+ # probe also runs elementFromPoint and the label scan, so a looser match swallows it.
+ if '(needed ? "1:" : "0:")' in js:
+ return self.answer
+ # `slotted:` is unique to the expensive target probe; the skinned-checkbox probe also
+ # calls scrollIntoView and runs regardless, so counting on that would never read zero.
+ if "slotted:" in js:
+ self.expensive_probes += 1
+ return {"exists": True, "disabled": False}
+ return await super().evaluate(js, arg)
+
+ clean = _CountingPage("0:self")
+ await _tool(build_browser_tools(_fixed_page_provider(clean)), "click").handler({"selector": "#submit"})
+ assert clean.expensive_probes == 0
+ assert (taskv3_loop._HIT_CLASS.get() or {}).get("hit_class") == "self"
+
+ covered = _CountingPage("1:non_target")
+ await _tool(build_browser_tools(_fixed_page_provider(covered)), "click").handler({"selector": "#submit"})
+ assert covered.expensive_probes == 1
+ recorded = taskv3_loop._HIT_CLASS.get() or {}
+ assert recorded.get("hit_class") == "non_target"
+ # The decision rides the record too: `unknown` alone cannot recover it, since a shadow-rooted
+ # target answers `unknown` with needed=True.
+ assert recorded.get("needed") is True
+ assert recorded.get("raised") is False
+ # Which realm answered -- the field is unwitnessed at this call site without an assertion here.
+ assert recorded.get("isolated") in (True, False)
+ # Nothing here claims occlusion: a real modal scrim drawn as `body::before` hit-tests as body and
+ # still blocks the click, so no cheap signal in this probe separates an occluder from a fall-through.
+ assert recorded.get("probe_seconds") is not None
+ # The loop clears this per tool call; a unit test must not leave the process-global set either.
+ taskv3_loop._HIT_CLASS.set(None)
+
+
+async def test_a_probe_that_raises_is_recorded_rather_than_dropped() -> None:
+ """The raise path is not random: it correlates with re-render, which is the cohort of interest.
+
+ Dropping those rows would bias the cost sample toward the calm cases — the defect that killed an
+ earlier hover-provenance field on this same path. Recorded, and TAGGED: a duration that pools
+ answers with blow-ups is two events under one name, so the cost figure filters on `raised`.
+ """
+
+ class _RaisingProbePage(_FakePage):
+ async def evaluate(self, js: str, arg: Any = None) -> Any:
+ if '(needed ? "1:" : "0:")' in js:
+ raise RuntimeError("Execution context was destroyed")
+ return await super().evaluate(js, arg)
+
+ taskv3_loop._HIT_CLASS.set(None)
+ await _tool(build_browser_tools(_fixed_page_provider(_RaisingProbePage())), "click").handler(
+ {"selector": "#submit"}
+ )
+
+ recorded = taskv3_loop._HIT_CLASS.get() or {}
+ assert recorded.get("raised") is True
+ assert recorded.get("hit_class") == "unknown"
+ # The decision a raising probe produces is the one the bare boolean produced: don't buy the
+ # expensive probe on a question that was never answered.
+ assert recorded.get("needed") is False
+ # And the duration is still there, so the raise stratum has an n rather than vanishing.
+ assert isinstance(recorded.get("probe_seconds"), float)
+ taskv3_loop._HIT_CLASS.set(None)
+
+
+async def test_the_isolated_leg_is_recorded_as_isolated(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The True side of the realm flag had no witness: deleting the assignment left the suite green.
+
+ A fake page has no CDP session, so every other test in this file exercises the fallback leg only —
+ which is exactly how an unwitnessed branch survives a suite that looks thorough.
+ """
+
+ async def _isolated_answer(_target: Any, _js: str, _selector: str) -> Any:
+ return "0:self"
+
+ monkeypatch.setattr(taskv3_tools, "_evaluate_isolated", _isolated_answer)
+ taskv3_loop._HIT_CLASS.set(None)
+ await _tool(build_browser_tools(_fixed_page_provider(_FakePage())), "click").handler({"selector": "#submit"})
+ assert (taskv3_loop._HIT_CLASS.get() or {}).get("isolated") is True
+ taskv3_loop._HIT_CLASS.set(None)