Skip to content

Automate browser based workflows with AI

Name the branch behind each solve_captcha ok on the record (SKY-16237) (#8531)

pedrohsdbe3ecbcbe1b4b
diff --git a/skyvern/forge/taskv3/captcha_tools.py b/skyvern/forge/taskv3/captcha_tools.py
index df1ac3037..bd58ab924 100644
--- a/skyvern/forge/taskv3/captcha_tools.py
+++ b/skyvern/forge/taskv3/captcha_tools.py
@@ -6,6 +6,10 @@ submit. This tool gives the loop an explicit ``solve_captcha`` action that drive
ladder (which detects the challenge via DOM/iframe markers and operates it), then returns an honest
tri-state so the model stops blind-hammering. Solving routes through the ``AGENT_FUNCTION`` seam, so
this module stays OSS-clean.
+
+Solved and "no challenge detected" are both ``ok`` -- absent is not an error and must not force a
+retry -- so every ``ok`` branch here names an ``ok_class``. That is what keeps the tri-state legible
+on the tool-call record; ``tool_status`` alone cannot tell a working detector from a blind one.
"""
from __future__ import annotations
@@ -59,7 +63,8 @@ def build_captcha_tools(
if failed_attempts >= _MAX_SOLVE_ATTEMPTS:
return ToolResult.ok(
"captcha solve has already failed the maximum number of times for this task; do not call "
- "solve_captcha again — report the captcha as blocking or try another approach."
+ "solve_captcha again — report the captcha as blocking or try another approach.",
+ ok_class="attempts_exhausted",
)
try:
page = await page_provider()
@@ -97,11 +102,15 @@ def build_captcha_tools(
if solved:
# A real solve is progress; clear the failure streak so a later genuine captcha isn't disabled.
failed_attempts = 0
- return ToolResult.ok("captcha solved; re-observe the page and continue (e.g. retry submit).")
+ return ToolResult.ok(
+ "captcha solved; re-observe the page and continue (e.g. retry submit).",
+ ok_class="solved",
+ )
# Absent: no challenge present. Cheap structural no-op that does not count toward the failure cap.
return ToolResult.ok(
"no captcha challenge was detected in the page or its visible frames; nothing was solved. "
- "Re-observe before calling solve_captcha again."
+ "Re-observe before calling solve_captcha again.",
+ ok_class="absent",
)
tool = ToolSpec(
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index afeb38f3c..50d323c37 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -78,6 +78,20 @@ ToolErrorClass = Literal[
"other",
]
+# The `ok` counterpart to `ToolErrorClass`, for a tool whose success status spans outcomes that are
+# not the same event. `tool_status` is the only total outcome field on the record, so a tool that
+# returns `ok` for "did the thing", "there was nothing to do", and "declined to try" is indexed
+# identically on all three and a blind detector reads as a working one. Naming the branch is
+# telemetry, never a behaviour change: like `error_class` this is never serialized into the tool
+# message, so the model cannot see it (the loop sends `content` only).
+ToolOkClass = Literal[
+ # `solve_captcha`. Its three `ok` branches are genuinely different events and the `ok` for
+ # "absent" is correct -- no challenge present is not an error and must not force a retry.
+ "solved",
+ "absent",
+ "attempts_exhausted",
+]
+
@dataclass
class ToolResult:
@@ -91,10 +105,22 @@ class ToolResult:
# Telemetry only, and deliberately NOT in `data`: callers and tests pin `data` by equality, so a
# measurement riding in it would change an observable contract. Never shown to the model.
error_class: ToolErrorClass | None = None
+ # Same contract as `error_class`, on the other side of the status. Telemetry only, never shown
+ # to the model. Each is read only under its own status, so a result carrying the class for the
+ # OTHER one drops it silently; the classmethods below cannot express that pairing (neither
+ # accepts the other's kwarg) and the raw constructor is the only route that could.
+ ok_class: ToolOkClass | None = None
@classmethod
- def ok(cls, content: str, data: dict[str, Any] | None = None, screenshots: list[bytes] | None = None) -> ToolResult:
- return cls("ok", content, data, screenshots)
+ def ok(
+ cls,
+ content: str,
+ data: dict[str, Any] | None = None,
+ screenshots: list[bytes] | None = None,
+ *,
+ ok_class: ToolOkClass | None = None,
+ ) -> ToolResult:
+ return cls("ok", content, data, screenshots, ok_class=ok_class)
@classmethod
def error(
@@ -1383,6 +1409,7 @@ _TOOL_CALL_RECORD_FIELDS = frozenset(
"selector_present",
"selector_kind",
"tool_error_class",
+ "tool_ok_class",
"resolve_seconds",
"frame_perception",
"billable",
@@ -2929,6 +2956,16 @@ async def run_agent_tool_loop(
# vocabulary with Python exception names under one facet, and taskv3 emits on every
# erroring tool call so it would dominate the values.
cost_fields["tool_error_class"] = result.error_class or "other"
+ elif result.ok_class is not None:
+ # Prefixed for the same flat-index reason as `tool_error_class` above. NOT defaulted
+ # the way that field is: it is emitted only by tools whose `ok` spans distinct
+ # outcomes, so a fleet-wide default would put a field on every successful call to
+ # say nothing. WHAT A DENOMINATOR MEANS HERE, because a groupBy drops rows missing a
+ # facet: grouping a tool by this facet alone shows its `ok` calls ONLY and silently
+ # excludes its errors. `tool_status` is the total partition -- cut on it first, then
+ # read this within `ok` and `tool_error_class` within `error`. A tool that emits
+ # this at all must emit it on EVERY `ok` branch, or the buckets don't sum to `ok`.
+ cost_fields["tool_ok_class"] = result.ok_class
# Read off the context variable, not the result: on the raise path the loop built the
# result itself and the handler's own resolution time would otherwise be lost.
resolve_seconds = _RESOLVE_SECONDS.get()
diff --git a/tests/unit/test_taskv3_captcha_tools.py b/tests/unit/test_taskv3_captcha_tools.py
index 8b31412bd..93dfd1ec6 100644
--- a/tests/unit/test_taskv3_captcha_tools.py
+++ b/tests/unit/test_taskv3_captcha_tools.py
@@ -83,6 +83,96 @@ async def test_solve_captcha_absent_is_ok_and_steers_away(monkeypatch: pytest.Mo
assert ladder.await_count == 2
+async def test_every_ok_branch_names_a_distinct_outcome_class(monkeypatch: pytest.MonkeyPatch) -> None:
+ # The tool returns ok on three different events -- it solved one, there was none to solve, and it
+ # declined after the cap -- and `tool_status` cannot tell them apart, so a blind detector and a
+ # working one are the same row to every downstream reader. Drives all three through the real
+ # handler rather than pinning strings: the branches were previously separable only by
+ # `len(content)`, which is exactly the accident this replaces (it has already changed once).
+ ladder = AsyncMock(return_value=True)
+ monkeypatch.setattr(captcha_tools, "solve_challenge_ladder", ladder)
+ tools, _ = captcha_tools.build_captcha_tools(_task(), _provider(object()), organization_id="o_1")
+ handler = tools[0].handler
+
+ solved = await handler({})
+ ladder.return_value = False
+ absent = await handler({})
+ # Exhaust the cap on real failures, then the next call is the short-circuit branch.
+ ladder.side_effect = CaptchaChallengeUnsolvedError("x")
+ for _ in range(captcha_tools._MAX_SOLVE_ATTEMPTS):
+ await handler({})
+ declined = await handler({})
+
+ assert (solved.status, solved.ok_class) == ("ok", "solved")
+ assert (absent.status, absent.ok_class) == ("ok", "absent")
+ assert (declined.status, declined.ok_class) == ("ok", "attempts_exhausted")
+ # Distinctness is the property that matters: two branches sharing a value would reintroduce the
+ # conflation while every per-branch assertion above still passed.
+ assert len({solved.ok_class, absent.ok_class, declined.ok_class}) == 3
+
+
+def test_every_ok_construction_in_the_module_names_an_ok_class() -> None:
+ """Totality, which the three per-branch assertions above cannot give.
+
+ `tool_ok_class` is emitted only when the site names one, so a fourth `ok` branch added here
+ without an `ok_class` is silently absent from the facet: its rows still carry `tool_status=ok`,
+ the buckets stop summing to the tool's own ok total, and every test above stays green because
+ each pins a branch that exists. Read out of the module's own AST rather than from a list of
+ known call sites -- an enumerated list is the same defect one level up and would not see the
+ fourth branch either.
+ """
+ import ast
+ import inspect
+ import pathlib
+
+ module_path = pathlib.Path(inspect.getfile(captcha_tools))
+ source = module_path.read_text()
+ tree = ast.parse(source)
+
+ ok_calls = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "ok"
+ and isinstance(node.func.value, ast.Name)
+ and node.func.value.id == "ToolResult"
+ ]
+ # Anti-vacuity, both directions. A walk that matched nothing would satisfy the loop below in
+ # silence, and a walk that matched fewer sites than the file spells out is reading past some of
+ # them -- either way the check would vouch for branches it never saw.
+ assert len(ok_calls) == source.count("ToolResult.ok("), "the AST walk missed an ok construction"
+ assert len(ok_calls) >= 3, "the scan found no ok constructions to check; it is not looking"
+
+ unnamed = []
+ for node in ok_calls:
+ named = [k for k in node.keywords if k.arg == "ok_class"]
+ # `**mapping` carries keys this scan cannot read, so it cannot vouch for the site either.
+ readable = named and not any(isinstance(k.value, ast.Constant) and k.value.value is None for k in named)
+ if not readable:
+ unnamed.append(node.lineno)
+ assert not unnamed, f"ToolResult.ok() with no ok_class at {module_path.name} lines {unnamed}"
+
+ # The other route to an ok: the raw constructor takes the class positionally and can pair an
+ # `ok` status with an `error_class`, so it would evade the check above entirely. This module
+ # has no reason to use it, and the cheapest way to keep the scan total is to keep it that way.
+ raw = [
+ node.lineno
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "ToolResult"
+ ]
+ assert not raw, f"raw ToolResult(...) construction bypasses the ok_class check at lines {raw}"
+
+ classes = {
+ k.value.value
+ for node in ok_calls
+ for k in node.keywords
+ if k.arg == "ok_class" and isinstance(k.value, ast.Constant)
+ }
+ assert {"solved", "absent", "attempts_exhausted"} <= classes, "the scan is not reading the real sites"
+
+
@pytest.mark.asyncio
async def test_solve_captcha_unsolved_is_error(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index 8ccf51058..d596391ef 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -9187,6 +9187,51 @@ async def test_a_failed_call_records_its_error_class_and_a_successful_one_record
assert "frame_perception" not in by_tool["click"]
+async def test_an_ok_call_records_the_branch_behind_its_success_and_an_unclassified_one_records_none() -> None:
+ # `tool_status` is the only total outcome field, so a tool whose `ok` spans distinct events
+ # (solve_captcha: solved / nothing-there / declined) indexes all of them identically and a blind
+ # detector reads as a working one. This has already produced a published conclusion that was
+ # arithmetically right and substantively backwards. The branch must reach the record as its own
+ # groupable value -- recovering it from `result_chars` is an accident of string length that has
+ # already changed once across builds.
+ async def solved(args: dict[str, Any]) -> ToolResult:
+ return ToolResult.ok("done", ok_class="solved")
+
+ async def nothing_there(args: dict[str, Any]) -> ToolResult:
+ return ToolResult.ok("nothing to do", ok_class="absent")
+
+ async def unclassified(args: dict[str, Any]) -> ToolResult:
+ return ToolResult.ok("ok")
+
+ tools = [
+ ToolSpec(name="click", description="c", parameters={}, handler=solved, billable=True),
+ ToolSpec(name="hover", description="h", parameters={}, handler=nothing_there, billable=True),
+ ToolSpec(name="scroll", description="s", parameters={}, handler=unclassified, billable=True),
+ make_finish_tool(),
+ ]
+ script = [
+ [("click", {"selector": "#a"})],
+ [("hover", {"selector": "#b"})],
+ [("scroll", {"selector": "#c"})],
+ [("finish", {"status": "completed", "reason": "ok"})],
+ ]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools)
+ assert outcome.status == "completed"
+ by_tool = {e["tool"]: e for e in logs if e["event"] == "taskv3 tool call finished"}
+ # Distinct values, or the read this exists to enable cannot separate the branches.
+ assert by_tool["click"]["tool_ok_class"] == "solved"
+ assert by_tool["hover"]["tool_ok_class"] == "absent"
+ # Absent, not null and not a fleet-wide default: only a tool whose `ok` spans distinct outcomes
+ # emits this, so every other successful call keeps exactly the fields it has today.
+ assert "tool_ok_class" not in by_tool["scroll"]
+ assert "ok_class" not in by_tool["click"] # the record key is the prefixed one, never the raw field
+ # The two sides of the status stay on their own side: an ok call names no error class, and this
+ # field must never appear on an erroring row (`tool_status` is the total partition between them).
+ assert "tool_error_class" not in by_tool["click"]
+
+
@pytest.mark.asyncio
async def test_a_handler_that_raises_still_reports_the_time_it_spent_resolving_the_address() -> None:
# The cohort this telemetry exists to price is the one where the handler RAISES -- a driver
@@ -9274,9 +9319,22 @@ def test_every_driver_timeout_type_installed_here_classifies_as_a_driver_timeout
assert _raised_error_class(RuntimeError("something else")) == "handler_raised"
-def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:
+ ("field", "alias", "anti_vacuity"),
+ [
+ ("error_class", "ToolErrorClass", {"stale_selector", "disabled", "ambiguous_frame"}),
+ ("ok_class", "ToolOkClass", {"solved", "absent", "attempts_exhausted"}),
+ ],
+)
+def test_every_outcome_class_written_anywhere_is_in_the_closed_set(
+ field: str, alias: str, anti_vacuity: set[str]
+) -> None:
"""The `Literal` alias is the declaration; this is what actually checks it.
+ Runs over BOTH closed vocabularies on `ToolResult`. They are the same mechanism on either side
+ of the status and share every hazard below, so one census covers both -- a second copy would be
+ the one that goes stale when the scan is next widened.
+
`mypy.ini` sets `follow_imports = skip`, so `ToolResult` resolves to `Any` in every module that
imports it -- the annotation is enforced inside loop.py and nowhere else, and all but a handful
of the write sites are in tools.py. A typo or a near-synonym added there would split the facet
@@ -9285,7 +9343,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:
cannot all reach.
The population is DERIVED, not listed: every `ToolResult` construction under `skyvern/` and
- `cloud/`, plus every `error_class=` kwarg anywhere in the taskv3 package. A named module list is
+ `cloud/`, plus every `<field>=` kwarg anywhere in the taskv3 package. A named module list is
the same shape of hazard one level up -- it holds until someone writes the value somewhere the
list does not name, which is exactly what a tool registered through `extra_tools` does.
"""
@@ -9295,11 +9353,12 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:
from typing import get_args # noqa: PLC0415
import skyvern # noqa: PLC0415
- from skyvern.forge.taskv3.loop import ToolErrorClass, ToolResult # noqa: PLC0415
+ from skyvern.forge.taskv3 import loop as loop_for_census # noqa: PLC0415
+ from skyvern.forge.taskv3.loop import ToolResult # noqa: PLC0415
- # Derived, not the literal 4: the positional read is a coupling to FIELD ORDER, and a reorder
+ # Derived, not a literal index: the positional read is a coupling to FIELD ORDER, and a reorder
# would otherwise move the census onto a different argument without anything going red.
- error_class_position = [f.name for f in dataclasses.fields(ToolResult)].index("error_class")
+ field_position = [f.name for f in dataclasses.fields(ToolResult)].index(field)
def _is_tool_result_constructor(node: ast.Call, names: set[str], qualified: bool) -> bool:
# Every spelling of the CONSTRUCTOR -- bare, module-qualified, and import-aliased. It matters
@@ -9332,7 +9391,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:
repo_root = pathlib.Path(skyvern.__file__).resolve().parent.parent
taskv3 = repo_root / "skyvern" / "forge" / "taskv3"
- declared = set(get_args(ToolErrorClass))
+ declared = set(get_args(getattr(loop_for_census, alias)))
written: set[str] = set()
computed: set[str] = set()
for root in (repo_root / "skyvern", repo_root / "cloud"):
@@ -9342,7 +9401,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:
source = path.read_text(encoding="utf-8", errors="replace")
# `ToolResult` as well as the field name: a `**overrides` that carries the class need not
# spell it, so prefiltering on the field alone drops the file before it is ever parsed.
- if "error_class" not in source and "ToolResult" not in source:
+ if field not in source and "ToolResult" not in source:
continue
in_taskv3 = taskv3 in path.parents
tree = ast.parse(source)
@@ -9383,7 +9442,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
assigned = node.value if not isinstance(node, ast.AugAssign) else None
for target in targets:
- if not (isinstance(target, ast.Attribute) and target.attr == "error_class"):
+ if not (isinstance(target, ast.Attribute) and target.attr == field):
continue
if isinstance(assigned, ast.Constant) and isinstance(assigned.value, str):
written.add(assigned.value)
@@ -9408,7 +9467,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:
and len(node.args) == 3
):
attribute, assigned = node.args[1], node.args[2]
- named = isinstance(attribute, ast.Constant) and attribute.value == "error_class"
+ named = isinstance(attribute, ast.Constant) and attribute.value == field
if named and isinstance(assigned, ast.Constant) and isinstance(assigned.value, str):
written.add(assigned.value)
elif named:
@@ -9427,15 +9486,15 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:
):
continue
here = path.relative_to(repo_root).as_posix()
- # The field is the FIFTH positional, and the dataclass constructor accepts it there.
- # `.error()`/`.ok()` cannot: one takes it keyword-only, the other not at all.
+ # The dataclass constructor accepts either field positionally, at the index derived
+ # above. The classmethods cannot: each takes its own class keyword-only.
if _is_tool_result_constructor(node, constructor_names, imports_loop_module):
if any(isinstance(argument, ast.Starred) for argument in node.args):
# `ToolResult(*args)` -- the arity is a runtime property, so whether anything
# lands in the fifth slot is not decidable here. Unreadable, not absent.
computed.add(here)
- elif len(node.args) > error_class_position:
- positional = node.args[error_class_position]
+ elif len(node.args) > field_position:
+ positional = node.args[field_position]
if isinstance(positional, ast.Constant) and isinstance(positional.value, str):
written.add(positional.value)
else:
@@ -9451,7 +9510,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:
):
computed.add(here)
continue
- if keyword.arg != "error_class":
+ if keyword.arg != field:
continue
if isinstance(value, ast.Constant) and isinstance(value.value, str):
written.add(value.value)
@@ -9461,12 +9520,16 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:
# make the census read clean over exactly the site it cannot vouch for.
computed.add(here)
- # Anti-vacuity: a walk that matched nothing would satisfy the subset check silently. These three
- # live in tools.py, the module the subset check exists to cover.
- assert {"stale_selector", "disabled", "ambiguous_frame"} <= written
- assert written <= declared, f"error classes written but not declared: {sorted(written - declared)}"
- # Produced by the loop's fallback for an erroring call whose site named nothing, not by a kwarg.
- assert "other" in declared
+ # Anti-vacuity: a walk that matched nothing would satisfy the subset check silently. Each seed
+ # lives in the module that field is actually written from -- tools.py for `error_class`,
+ # captcha_tools.py for `ok_class` -- which is the module the subset check exists to cover.
+ assert anti_vacuity <= written, f"{field} census found none of its seed values; it is not looking"
+ assert written <= declared, f"{field} values written but not declared: {sorted(written - declared)}"
+ if field == "error_class":
+ # Produced by the loop's fallback for an erroring call whose site named nothing, not by a
+ # kwarg, so it is never `written` and has to be asserted against `declared` directly. There
+ # is deliberately no `ok_class` equivalent: an ok call that names no class emits no facet.
+ assert "other" in declared
# The other half of the guarantee, and the reason it is a set of FILES rather than a count: a
# non-literal value is unreadable here, so the only thing that can vouch for it is the `Literal`
# annotation -- which `follow_imports = skip` binds inside loop.py and nowhere else. Both sites
@@ -9478,7 +9541,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:
# sites it is anchored on, which claims a violation while naming nothing.
assert accessor_module in computed, "the scan found no unreadable write at all; it is not looking"
assert not computed - {accessor_module}, (
- f"error_class written from an unreadable expression outside loop.py, where the Literal "
+ f"{field} written from an unreadable expression outside loop.py, where the Literal "
f"annotation does not bind: {sorted(computed - {accessor_module})}"
)