Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Say which criterion wins when both describe the page (SKY-16193) (#8522)
pedrohsdbbb59c6ea4ff1
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex e36ce121a..238c9d796 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -1872,6 +1872,9 @@ class ForgeAgent:None if context and context.complete_criterion_is_untrusted else task.complete_criterion),terminate_criterion=task.terminate_criterion,+ # Validation only: that is the task type whose criteria a decision-maker weighs against+ # each other in both engines, and the only one this was measured on (SKY-16193).+ criteria_precedence=task.task_type == TaskType.validation,error_code_mapping=task.error_code_mapping if offer_error_codes else None,framing=framing,block_context_section=block_context_section,diff --git a/skyvern/forge/taskv3/goal_composition.py b/skyvern/forge/taskv3/goal_composition.pyindex 95d69a2b8..3f554de11 100644--- a/skyvern/forge/taskv3/goal_composition.py+++ b/skyvern/forge/taskv3/goal_composition.py@@ -39,6 +39,11 @@ class GoalDirectives:complete_criterion: str | None = Noneterminate_criterion: str | None = Noneerror_code_mapping: dict[str, str] | None = None+ # Whether to state which criterion wins when both hold. Scoped by the caller to the task type the+ # measurement covers: on v1's general path the terminate criterion reaches no decision-maker at+ # all, so "which wins" is not the difference there and a rule written for one population would be+ # shipping ahead of its evidence.+ criteria_precedence: bool = Falseframing: str = ""block_context_section: str = ""@@ -70,6 +75,32 @@ def compose_goal(navigation_goal: str, directives: GoalDirectives) -> str:goal = (f"{goal}\n\nIf this becomes true, stop and finish with status=terminated: {directives.terminate_criterion}").strip()+ if directives.complete_criterion and directives.terminate_criterion and directives.criteria_precedence:+ # Two criteria can describe one page at once -- "no error is present" and "...or the pop up is+ # not available" both hold on a page with neither. v1's validation prompt offers ONE forced choice+ # and names completion first in its own rule; v3 read the termination clause as a standing+ # interrupt and terminated, on criteria that were authored against v1 (SKY-16193). Neither+ # engine evaluates the two in sequence -- the difference is how the choice is framed.+ #+ # Both criteria are NAMED rather than pointed at. A customer's terminate text routinely ends in+ # a coordinated pair ("...or a dual-eligible flag", "...or shows a login form") sitting thousands+ # of characters closer to this sentence than the two criteria are, so a demonstrative binds to+ # the wrong pair -- and on one real workflow it would land directly after an instruction saying+ # not to let anything override that guard.+ #+ # Worded without "the page": a page-free validation block is told it has no browser tools, so a+ # rule conditioned on what the page shows is dead text on exactly the task type this targets.+ #+ # Scoped to the OVERLAP and nothing else. An earlier draft added "terminate only when the+ # termination criterion holds", which reads as a ban on every other use of the status -- and+ # `terminated` is also how this engine reports being blocked or the goal being impossible+ # (engine.py's system prompt, loop.py's stuck options). v1 is broader still: its validation+ # prompt terminates "when the terminate criterion is met, OR when a complete criterion is+ # provided but not met". A precedence rule must not quietly narrow the status it mentions.+ goal = (+ f"{goal}\n\nIf the completion criterion and the termination criterion both hold at once, "+ "the completion criterion wins: finish with status=completed."+ ).strip()if directives.error_code_mapping:# v1 shows the model these codes in-loop (see the error_code_mapping_str prompt sites), so# a v1 terminal verdict names its own code. v3 did not, and the codes were instead matcheddiff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex bf81e74d0..fb92eb9db 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -444,6 +444,31 @@ async def test_execute_task_v3_surfaces_criteria_in_goal(monkeypatch: pytest.Mongoal = loop_mock.await_args.kwargs["goal"]assert "the confirmation page is shown" in goalassert "the posting is closed" in goal+ # The precedence rule is gated to validation tasks, and this seam is the only place that can catch+ # the gate regressing: compose_goal's own tests exercise its default, not the policy the caller+ # sets. `make_task` defaults to TaskType.general.+ assert "the completion criterion wins" not in goal+++async def test_execute_task_v3_states_criteria_precedence_only_for_validation(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Both criteria can describe one page at once, and the two engines resolved that differently+ (SKY-16193). v1 shows the terminate criterion to a decision-maker on validation tasks only, so+ saying which wins has no measured meaning anywhere else."""+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["click"])+ _step, _task, loop_mock, _post = await _run_execute_task_v3(+ monkeypatch,+ outcome,+ navigation_goal="Confirm the store was added",+ complete_criterion="no error message is present",+ terminate_criterion="an error message is present, or the pop up is not available",+ task_type=TaskType.validation,+ data_extraction_goal=None,+ extracted_information_schema=None,+ )+ assert "the completion criterion wins" in loop_mock.await_args.kwargs["goal"]@pytest.mark.asynciodiff --git a/tests/unit/test_taskv3_goal_composition.py b/tests/unit/test_taskv3_goal_composition.pyindex f813644f9..2c84af396 100644--- a/tests/unit/test_taskv3_goal_composition.py+++ b/tests/unit/test_taskv3_goal_composition.py@@ -70,6 +70,7 @@ _DIRECTIVE_VALUES: dict[str, tuple[Any, Any]] = {"complete_criterion": (None, "the confirmation page is showing"),"terminate_criterion": (None, "the form says the role is closed"),"error_code_mapping": (None, {"ROLE_CLOSED": "the posting is no longer accepting submissions"}),+ "criteria_precedence": (False, True),"framing": ("", "This is one block of a larger workflow."),"block_context_section": ("", "<workflow_context>\nblocks: one, two\n</workflow_context>"),}@@ -80,7 +81,7 @@ _DIRECTIVE_VALUES: dict[str, tuple[Any, Any]] = {def test_compose_goal_is_byte_identical_to_the_inline_patching_it_replaced(navigation_goal: str, mask: tuple[int, ...]) -> None:- # Every on/off combination of the seven directives, against both an empty and a non-empty+ # Every on/off combination of the eight directives, against both an empty and a non-empty# navigation goal. The empty one matters: the first directive's .strip() is what decides# whether the goal opens with a blank line, and that only shows up when the base is "".chosen = {name: options[bit] for (name, options), bit in zip(_DIRECTIVE_VALUES.items(), mask)}@@ -96,12 +97,14 @@ def _goal_as_agent_py_built_it(extracted_information_schema: Any,complete_criterion: str | None,terminate_criterion: str | None,+ criteria_precedence: bool,error_code_mapping: dict[str, str] | None,framing: str,block_context_section: str,) -> str:- """The goal-patching expressions exactly as `ForgeAgent._execute_task_v3` inlined them before- `compose_goal` existed, frozen here as the oracle for the extraction.+ """The goal-patching expressions as `ForgeAgent._execute_task_v3` inlined them before `compose_goal`+ existed, frozen here as the oracle for the extraction, plus any directive a later commit deliberately+ reworded -- the overlap-precedence sentence is one such, so this is no longer a pure historical record.This is deliberately a duplicate of production wording: it is the only thing that can catch asingle dropped space or reordered clause in a refactor whose entire acceptance bar is that the@@ -127,6 +130,15 @@ def _goal_as_agent_py_built_it(goal = (f"{goal}\n\nIf this becomes true, stop and finish with status=terminated: {terminate_criterion}").strip()+ if complete_criterion and terminate_criterion and criteria_precedence:+ # SKY-16193: the one deliberate divergence from what agent.py inlined. Criteria that can hold+ # at once had no stated precedence, and v3 resolved that the opposite way to the engine they+ # were authored against. Updated here in the same commit as the production wording, which is+ # what this oracle's docstring asks of a change that means to reword a directive.+ goal = (+ f"{goal}\n\nIf the completion criterion and the termination criterion both hold at once, "+ "the completion criterion wins: finish with status=completed."+ ).strip()if error_code_mapping:goal = (f"{goal}\n\nThe user defined these business outcomes and their descriptions:\n"@@ -163,3 +175,93 @@ def test_the_split_modules_stay_one_way_dependent_on_goal_composition() -> None:assert not any("goal_composition" in ref for ref in imported), (f"{module} imports goal_composition; the responsibilities are re-conflating")+++def test_two_criteria_that_can_hold_at_once_get_an_explicit_precedence() -> None:+ """A block whose criteria overlap has no correct answer without a precedence rule, and the two+ engines frame that choice differently (SKY-16193).++ The measured case: complete was "if no error message is present" and terminate ended "...or if+ pop up message is not available". On a page with neither an error nor a popup BOTH hold, and+ nothing told the model which to apply.++ Effect size, arm-resolved, over runs that REACHED the block: v3 terminated there 22/37 = 59.5%+ of the time against v1's 14/119 = 11.8%. That establishes something costly happens here; it+ does not establish what. What does is the composition. v1's `decisive-criterion-validate.j2`+ asks for "only one action" and resolves both criteria inside a single enum rule that names+ COMPLETE first (`:11`, `:15`); v3 renders the terminate criterion as its own standing interrupt+ -- "If this becomes true, stop and finish with status=terminated". The criteria were authored+ against v1, so v3 is the one that has to say which wins.++ And v1 fails this block too, 14 times in three days. Any mechanism proposed for it has to+ explain a 5x ratio, not a v3-only defect.+ """+ both = compose_goal(+ "Add the store.",+ GoalDirectives(+ complete_criterion="no error message is present",+ terminate_criterion="an error message is present, or the pop up is not available",+ criteria_precedence=True,+ ),+ )+ assert "the completion criterion wins" in both+ # Precedence for the OVERLAP only. An earlier draft added "terminate only when the termination+ # criterion holds", which reads as a ban on every other use of the status -- and `terminated` is+ # also how this engine reports being blocked, while v1 terminates whenever a complete criterion is+ # provided and not met. A precedence rule must not narrow the status it mentions.+ # The flag must add EXACTLY this paragraph and nothing else. An earlier guard asserted only that+ # ONE paragraph was added containing ONE full stop, which constrains punctuation rather than+ # content: round 1's prohibition walks straight through it joined by a semicolon, the naming fix+ # reverts to a demonstrative untouched, and even inverting the prescribed status passes. Asserting+ # the text verbatim is what makes the intent readable at the one place a reviewer looks.+ #+ # What equality buys, precisely: accidental defeat becomes impossible, and a DELIBERATE edit+ # becomes visible in the diff, because changing the sentence means retyping it here. It does NOT+ # make a lockstep edit impossible -- the oracle's docstring still invites one, and someone who+ # updates production, the oracle and this line together will pass. Naming that boundary is the+ # point; a fourth guard claiming to close it would repeat the error one level up.+ #+ # And it cannot see a prohibition appended to a DIFFERENT directive: the delta is between the+ # gated and ungated renders, so text added to both cancels. That is a property of the oracle+ # beside it rather than of this flag.+ #+ # AND IT IS THE SOLE CATCH FOR A LOCKSTEP EDIT. A lockstep change defeats the byte-identity mask+ # BY CONSTRUCTION -- production and the oracle agree again -- so all 512 mask cases stay GREEN and+ # only the assertion below reds. Three defeats are demonstrated against it: a semicolon for the+ # full stop, the prescribed status inverted to `terminated`, and a demonstrative restored in the+ # condition. Weaken, move or lose this one assertion in a refactor and all three go invisible+ # with every other test in the file still green.+ ungated_for_delta = compose_goal(+ "Add the store.",+ GoalDirectives(+ complete_criterion="no error message is present",+ terminate_criterion="an error message is present, or the pop up is not available",+ ),+ )+ added = [p for p in both.split("\n\n") if p not in ungated_for_delta.split("\n\n")]+ assert added == [+ "If the completion criterion and the termination criterion both hold at once, "+ "the completion criterion wins: finish with status=completed."+ ], added+ # Named rather than implied: the model must be able to tell which of the two it is applying.+ assert both.index("Consider the goal complete") < both.index("the completion criterion wins")++ # One criterion alone cannot conflict with anything, so it gets no precedence sentence.+ complete_only = compose_goal(+ "Add the store.", GoalDirectives(complete_criterion="the store is listed", criteria_precedence=True)+ )+ terminate_only = compose_goal(+ "Add the store.", GoalDirectives(terminate_criterion="the posting closed", criteria_precedence=True)+ )+ assert "the completion criterion wins" not in complete_only+ assert "the completion criterion wins" not in terminate_only++ # And it is OFF unless the caller asks: v1 shows the terminate criterion to a decision-maker only on+ # validation tasks, so a rule about which criterion wins has no measured meaning anywhere else.+ ungated = compose_goal(+ "Add the store.",+ GoalDirectives(complete_criterion="no error message is present", terminate_criterion="the pop up is missing"),+ )+ assert "the completion criterion wins" not in ungated+ # Worded without "the page": a page-free validation block is told it has no browser tools at all.+ assert "the page satisfies" not in both