Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Name the branch behind each solve_captcha ok on the record (SKY-16237) (#8531)
pedrohsdbe3ecbcbe1b4b
diff --git a/skyvern/forge/taskv3/captcha_tools.py b/skyvern/forge/taskv3/captcha_tools.pyindex df1ac3037..bd58ab924 100644--- a/skyvern/forge/taskv3/captcha_tools.py+++ b/skyvern/forge/taskv3/captcha_tools.py@@ -6,6 +6,10 @@ submit. This tool gives the loop an explicit ``solve_captcha`` action that driveladder (which detects the challenge via DOM/iframe markers and operates it), then returns an honesttri-state so the model stops blind-hammering. Solving routes through the ``AGENT_FUNCTION`` seam, sothis module stays OSS-clean.++Solved and "no challenge detected" are both ``ok`` -- absent is not an error and must not force a+retry -- so every ``ok`` branch here names an ``ok_class``. That is what keeps the tri-state legible+on the tool-call record; ``tool_status`` alone cannot tell a working detector from a blind one."""from __future__ import annotations@@ -59,7 +63,8 @@ def build_captcha_tools(if failed_attempts >= _MAX_SOLVE_ATTEMPTS:return ToolResult.ok("captcha solve has already failed the maximum number of times for this task; do not call "- "solve_captcha again — report the captcha as blocking or try another approach."+ "solve_captcha again — report the captcha as blocking or try another approach.",+ ok_class="attempts_exhausted",)try:page = await page_provider()@@ -97,11 +102,15 @@ def build_captcha_tools(if solved:# A real solve is progress; clear the failure streak so a later genuine captcha isn't disabled.failed_attempts = 0- return ToolResult.ok("captcha solved; re-observe the page and continue (e.g. retry submit).")+ return ToolResult.ok(+ "captcha solved; re-observe the page and continue (e.g. retry submit).",+ ok_class="solved",+ )# Absent: no challenge present. Cheap structural no-op that does not count toward the failure cap.return ToolResult.ok("no captcha challenge was detected in the page or its visible frames; nothing was solved. "- "Re-observe before calling solve_captcha again."+ "Re-observe before calling solve_captcha again.",+ ok_class="absent",)tool = ToolSpec(diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex afeb38f3c..50d323c37 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -78,6 +78,20 @@ ToolErrorClass = Literal["other",]+# The `ok` counterpart to `ToolErrorClass`, for a tool whose success status spans outcomes that are+# not the same event. `tool_status` is the only total outcome field on the record, so a tool that+# returns `ok` for "did the thing", "there was nothing to do", and "declined to try" is indexed+# identically on all three and a blind detector reads as a working one. Naming the branch is+# telemetry, never a behaviour change: like `error_class` this is never serialized into the tool+# message, so the model cannot see it (the loop sends `content` only).+ToolOkClass = Literal[+ # `solve_captcha`. Its three `ok` branches are genuinely different events and the `ok` for+ # "absent" is correct -- no challenge present is not an error and must not force a retry.+ "solved",+ "absent",+ "attempts_exhausted",+]+@dataclassclass ToolResult:@@ -91,10 +105,22 @@ class ToolResult:# Telemetry only, and deliberately NOT in `data`: callers and tests pin `data` by equality, so a# measurement riding in it would change an observable contract. Never shown to the model.error_class: ToolErrorClass | None = None+ # Same contract as `error_class`, on the other side of the status. Telemetry only, never shown+ # to the model. Each is read only under its own status, so a result carrying the class for the+ # OTHER one drops it silently; the classmethods below cannot express that pairing (neither+ # accepts the other's kwarg) and the raw constructor is the only route that could.+ ok_class: ToolOkClass | None = None@classmethod- def ok(cls, content: str, data: dict[str, Any] | None = None, screenshots: list[bytes] | None = None) -> ToolResult:- return cls("ok", content, data, screenshots)+ def ok(+ cls,+ content: str,+ data: dict[str, Any] | None = None,+ screenshots: list[bytes] | None = None,+ *,+ ok_class: ToolOkClass | None = None,+ ) -> ToolResult:+ return cls("ok", content, data, screenshots, ok_class=ok_class)@classmethoddef error(@@ -1383,6 +1409,7 @@ _TOOL_CALL_RECORD_FIELDS = frozenset("selector_present","selector_kind","tool_error_class",+ "tool_ok_class","resolve_seconds","frame_perception","billable",@@ -2929,6 +2956,16 @@ async def run_agent_tool_loop(# vocabulary with Python exception names under one facet, and taskv3 emits on every# erroring tool call so it would dominate the values.cost_fields["tool_error_class"] = result.error_class or "other"+ elif result.ok_class is not None:+ # Prefixed for the same flat-index reason as `tool_error_class` above. NOT defaulted+ # the way that field is: it is emitted only by tools whose `ok` spans distinct+ # outcomes, so a fleet-wide default would put a field on every successful call to+ # say nothing. WHAT A DENOMINATOR MEANS HERE, because a groupBy drops rows missing a+ # facet: grouping a tool by this facet alone shows its `ok` calls ONLY and silently+ # excludes its errors. `tool_status` is the total partition -- cut on it first, then+ # read this within `ok` and `tool_error_class` within `error`. A tool that emits+ # this at all must emit it on EVERY `ok` branch, or the buckets don't sum to `ok`.+ cost_fields["tool_ok_class"] = result.ok_class# Read off the context variable, not the result: on the raise path the loop built the# result itself and the handler's own resolution time would otherwise be lost.resolve_seconds = _RESOLVE_SECONDS.get()diff --git a/tests/unit/test_taskv3_captcha_tools.py b/tests/unit/test_taskv3_captcha_tools.pyindex 8b31412bd..93dfd1ec6 100644--- a/tests/unit/test_taskv3_captcha_tools.py+++ b/tests/unit/test_taskv3_captcha_tools.py@@ -83,6 +83,96 @@ async def test_solve_captcha_absent_is_ok_and_steers_away(monkeypatch: pytest.Moassert ladder.await_count == 2+async def test_every_ok_branch_names_a_distinct_outcome_class(monkeypatch: pytest.MonkeyPatch) -> None:+ # The tool returns ok on three different events -- it solved one, there was none to solve, and it+ # declined after the cap -- and `tool_status` cannot tell them apart, so a blind detector and a+ # working one are the same row to every downstream reader. Drives all three through the real+ # handler rather than pinning strings: the branches were previously separable only by+ # `len(content)`, which is exactly the accident this replaces (it has already changed once).+ ladder = AsyncMock(return_value=True)+ monkeypatch.setattr(captcha_tools, "solve_challenge_ladder", ladder)+ tools, _ = captcha_tools.build_captcha_tools(_task(), _provider(object()), organization_id="o_1")+ handler = tools[0].handler++ solved = await handler({})+ ladder.return_value = False+ absent = await handler({})+ # Exhaust the cap on real failures, then the next call is the short-circuit branch.+ ladder.side_effect = CaptchaChallengeUnsolvedError("x")+ for _ in range(captcha_tools._MAX_SOLVE_ATTEMPTS):+ await handler({})+ declined = await handler({})++ assert (solved.status, solved.ok_class) == ("ok", "solved")+ assert (absent.status, absent.ok_class) == ("ok", "absent")+ assert (declined.status, declined.ok_class) == ("ok", "attempts_exhausted")+ # Distinctness is the property that matters: two branches sharing a value would reintroduce the+ # conflation while every per-branch assertion above still passed.+ assert len({solved.ok_class, absent.ok_class, declined.ok_class}) == 3+++def test_every_ok_construction_in_the_module_names_an_ok_class() -> None:+ """Totality, which the three per-branch assertions above cannot give.++ `tool_ok_class` is emitted only when the site names one, so a fourth `ok` branch added here+ without an `ok_class` is silently absent from the facet: its rows still carry `tool_status=ok`,+ the buckets stop summing to the tool's own ok total, and every test above stays green because+ each pins a branch that exists. Read out of the module's own AST rather than from a list of+ known call sites -- an enumerated list is the same defect one level up and would not see the+ fourth branch either.+ """+ import ast+ import inspect+ import pathlib++ module_path = pathlib.Path(inspect.getfile(captcha_tools))+ source = module_path.read_text()+ tree = ast.parse(source)++ ok_calls = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr == "ok"+ and isinstance(node.func.value, ast.Name)+ and node.func.value.id == "ToolResult"+ ]+ # Anti-vacuity, both directions. A walk that matched nothing would satisfy the loop below in+ # silence, and a walk that matched fewer sites than the file spells out is reading past some of+ # them -- either way the check would vouch for branches it never saw.+ assert len(ok_calls) == source.count("ToolResult.ok("), "the AST walk missed an ok construction"+ assert len(ok_calls) >= 3, "the scan found no ok constructions to check; it is not looking"++ unnamed = []+ for node in ok_calls:+ named = [k for k in node.keywords if k.arg == "ok_class"]+ # `**mapping` carries keys this scan cannot read, so it cannot vouch for the site either.+ readable = named and not any(isinstance(k.value, ast.Constant) and k.value.value is None for k in named)+ if not readable:+ unnamed.append(node.lineno)+ assert not unnamed, f"ToolResult.ok() with no ok_class at {module_path.name} lines {unnamed}"++ # The other route to an ok: the raw constructor takes the class positionally and can pair an+ # `ok` status with an `error_class`, so it would evade the check above entirely. This module+ # has no reason to use it, and the cheapest way to keep the scan total is to keep it that way.+ raw = [+ node.lineno+ for node in ast.walk(tree)+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "ToolResult"+ ]+ assert not raw, f"raw ToolResult(...) construction bypasses the ok_class check at lines {raw}"++ classes = {+ k.value.value+ for node in ok_calls+ for k in node.keywords+ if k.arg == "ok_class" and isinstance(k.value, ast.Constant)+ }+ assert {"solved", "absent", "attempts_exhausted"} <= classes, "the scan is not reading the real sites"++@pytest.mark.asyncioasync def test_solve_captcha_unsolved_is_error(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr(diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex 8ccf51058..d596391ef 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -9187,6 +9187,51 @@ async def test_a_failed_call_records_its_error_class_and_a_successful_one_recordassert "frame_perception" not in by_tool["click"]+async def test_an_ok_call_records_the_branch_behind_its_success_and_an_unclassified_one_records_none() -> None:+ # `tool_status` is the only total outcome field, so a tool whose `ok` spans distinct events+ # (solve_captcha: solved / nothing-there / declined) indexes all of them identically and a blind+ # detector reads as a working one. This has already produced a published conclusion that was+ # arithmetically right and substantively backwards. The branch must reach the record as its own+ # groupable value -- recovering it from `result_chars` is an accident of string length that has+ # already changed once across builds.+ async def solved(args: dict[str, Any]) -> ToolResult:+ return ToolResult.ok("done", ok_class="solved")++ async def nothing_there(args: dict[str, Any]) -> ToolResult:+ return ToolResult.ok("nothing to do", ok_class="absent")++ async def unclassified(args: dict[str, Any]) -> ToolResult:+ return ToolResult.ok("ok")++ tools = [+ ToolSpec(name="click", description="c", parameters={}, handler=solved, billable=True),+ ToolSpec(name="hover", description="h", parameters={}, handler=nothing_there, billable=True),+ ToolSpec(name="scroll", description="s", parameters={}, handler=unclassified, billable=True),+ make_finish_tool(),+ ]+ script = [+ [("click", {"selector": "#a"})],+ [("hover", {"selector": "#b"})],+ [("scroll", {"selector": "#c"})],+ [("finish", {"status": "completed", "reason": "ok"})],+ ]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools)+ assert outcome.status == "completed"+ by_tool = {e["tool"]: e for e in logs if e["event"] == "taskv3 tool call finished"}+ # Distinct values, or the read this exists to enable cannot separate the branches.+ assert by_tool["click"]["tool_ok_class"] == "solved"+ assert by_tool["hover"]["tool_ok_class"] == "absent"+ # Absent, not null and not a fleet-wide default: only a tool whose `ok` spans distinct outcomes+ # emits this, so every other successful call keeps exactly the fields it has today.+ assert "tool_ok_class" not in by_tool["scroll"]+ assert "ok_class" not in by_tool["click"] # the record key is the prefixed one, never the raw field+ # The two sides of the status stay on their own side: an ok call names no error class, and this+ # field must never appear on an erroring row (`tool_status` is the total partition between them).+ assert "tool_error_class" not in by_tool["click"]++@pytest.mark.asyncioasync def test_a_handler_that_raises_still_reports_the_time_it_spent_resolving_the_address() -> None:# The cohort this telemetry exists to price is the one where the handler RAISES -- a driver@@ -9274,9 +9319,22 @@ def test_every_driver_timeout_type_installed_here_classifies_as_a_driver_timeoutassert _raised_error_class(RuntimeError("something else")) == "handler_raised"-def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:+ ("field", "alias", "anti_vacuity"),+ [+ ("error_class", "ToolErrorClass", {"stale_selector", "disabled", "ambiguous_frame"}),+ ("ok_class", "ToolOkClass", {"solved", "absent", "attempts_exhausted"}),+ ],+)+def test_every_outcome_class_written_anywhere_is_in_the_closed_set(+ field: str, alias: str, anti_vacuity: set[str]+) -> None:"""The `Literal` alias is the declaration; this is what actually checks it.+ Runs over BOTH closed vocabularies on `ToolResult`. They are the same mechanism on either side+ of the status and share every hazard below, so one census covers both -- a second copy would be+ the one that goes stale when the scan is next widened.+`mypy.ini` sets `follow_imports = skip`, so `ToolResult` resolves to `Any` in every module thatimports it -- the annotation is enforced inside loop.py and nowhere else, and all but a handfulof the write sites are in tools.py. A typo or a near-synonym added there would split the facet@@ -9285,7 +9343,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:cannot all reach.The population is DERIVED, not listed: every `ToolResult` construction under `skyvern/` and- `cloud/`, plus every `error_class=` kwarg anywhere in the taskv3 package. A named module list is+ `cloud/`, plus every `<field>=` kwarg anywhere in the taskv3 package. A named module list isthe same shape of hazard one level up -- it holds until someone writes the value somewhere thelist does not name, which is exactly what a tool registered through `extra_tools` does."""@@ -9295,11 +9353,12 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:from typing import get_args # noqa: PLC0415import skyvern # noqa: PLC0415- from skyvern.forge.taskv3.loop import ToolErrorClass, ToolResult # noqa: PLC0415+ from skyvern.forge.taskv3 import loop as loop_for_census # noqa: PLC0415+ from skyvern.forge.taskv3.loop import ToolResult # noqa: PLC0415- # Derived, not the literal 4: the positional read is a coupling to FIELD ORDER, and a reorder+ # Derived, not a literal index: the positional read is a coupling to FIELD ORDER, and a reorder# would otherwise move the census onto a different argument without anything going red.- error_class_position = [f.name for f in dataclasses.fields(ToolResult)].index("error_class")+ field_position = [f.name for f in dataclasses.fields(ToolResult)].index(field)def _is_tool_result_constructor(node: ast.Call, names: set[str], qualified: bool) -> bool:# Every spelling of the CONSTRUCTOR -- bare, module-qualified, and import-aliased. It matters@@ -9332,7 +9391,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:repo_root = pathlib.Path(skyvern.__file__).resolve().parent.parenttaskv3 = repo_root / "skyvern" / "forge" / "taskv3"- declared = set(get_args(ToolErrorClass))+ declared = set(get_args(getattr(loop_for_census, alias)))written: set[str] = set()computed: set[str] = set()for root in (repo_root / "skyvern", repo_root / "cloud"):@@ -9342,7 +9401,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:source = path.read_text(encoding="utf-8", errors="replace")# `ToolResult` as well as the field name: a `**overrides` that carries the class need not# spell it, so prefiltering on the field alone drops the file before it is ever parsed.- if "error_class" not in source and "ToolResult" not in source:+ if field not in source and "ToolResult" not in source:continuein_taskv3 = taskv3 in path.parentstree = ast.parse(source)@@ -9383,7 +9442,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:targets = node.targets if isinstance(node, ast.Assign) else [node.target]assigned = node.value if not isinstance(node, ast.AugAssign) else Nonefor target in targets:- if not (isinstance(target, ast.Attribute) and target.attr == "error_class"):+ if not (isinstance(target, ast.Attribute) and target.attr == field):continueif isinstance(assigned, ast.Constant) and isinstance(assigned.value, str):written.add(assigned.value)@@ -9408,7 +9467,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:and len(node.args) == 3):attribute, assigned = node.args[1], node.args[2]- named = isinstance(attribute, ast.Constant) and attribute.value == "error_class"+ named = isinstance(attribute, ast.Constant) and attribute.value == fieldif named and isinstance(assigned, ast.Constant) and isinstance(assigned.value, str):written.add(assigned.value)elif named:@@ -9427,15 +9486,15 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:):continuehere = path.relative_to(repo_root).as_posix()- # The field is the FIFTH positional, and the dataclass constructor accepts it there.- # `.error()`/`.ok()` cannot: one takes it keyword-only, the other not at all.+ # The dataclass constructor accepts either field positionally, at the index derived+ # above. The classmethods cannot: each takes its own class keyword-only.if _is_tool_result_constructor(node, constructor_names, imports_loop_module):if any(isinstance(argument, ast.Starred) for argument in node.args):# `ToolResult(*args)` -- the arity is a runtime property, so whether anything# lands in the fifth slot is not decidable here. Unreadable, not absent.computed.add(here)- elif len(node.args) > error_class_position:- positional = node.args[error_class_position]+ elif len(node.args) > field_position:+ positional = node.args[field_position]if isinstance(positional, ast.Constant) and isinstance(positional.value, str):written.add(positional.value)else:@@ -9451,7 +9510,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:):computed.add(here)continue- if keyword.arg != "error_class":+ if keyword.arg != field:continueif isinstance(value, ast.Constant) and isinstance(value.value, str):written.add(value.value)@@ -9461,12 +9520,16 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:# make the census read clean over exactly the site it cannot vouch for.computed.add(here)- # Anti-vacuity: a walk that matched nothing would satisfy the subset check silently. These three- # live in tools.py, the module the subset check exists to cover.- assert {"stale_selector", "disabled", "ambiguous_frame"} <= written- assert written <= declared, f"error classes written but not declared: {sorted(written - declared)}"- # Produced by the loop's fallback for an erroring call whose site named nothing, not by a kwarg.- assert "other" in declared+ # Anti-vacuity: a walk that matched nothing would satisfy the subset check silently. Each seed+ # lives in the module that field is actually written from -- tools.py for `error_class`,+ # captcha_tools.py for `ok_class` -- which is the module the subset check exists to cover.+ assert anti_vacuity <= written, f"{field} census found none of its seed values; it is not looking"+ assert written <= declared, f"{field} values written but not declared: {sorted(written - declared)}"+ if field == "error_class":+ # Produced by the loop's fallback for an erroring call whose site named nothing, not by a+ # kwarg, so it is never `written` and has to be asserted against `declared` directly. There+ # is deliberately no `ok_class` equivalent: an ok call that names no class emits no facet.+ assert "other" in declared# The other half of the guarantee, and the reason it is a set of FILES rather than a count: a# non-literal value is unreadable here, so the only thing that can vouch for it is the `Literal`# annotation -- which `follow_imports = skip` binds inside loop.py and nowhere else. Both sites@@ -9478,7 +9541,7 @@ def test_every_error_class_written_anywhere_is_in_the_closed_set() -> None:# sites it is anchored on, which claims a violation while naming nothing.assert accessor_module in computed, "the scan found no unreadable write at all; it is not looking"assert not computed - {accessor_module}, (- f"error_class written from an unreadable expression outside loop.py, where the Literal "+ f"{field} written from an unreadable expression outside loop.py, where the Literal "f"annotation does not bind: {sorted(computed - {accessor_module})}")