Skip to content

Automate browser based workflows with AI

Say which criterion wins when both describe the page (SKY-16193) (#8522)

pedrohsdbbb59c6ea4ff1
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index e36ce121a..238c9d796 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -1872,6 +1872,9 @@ class ForgeAgent:
None if context and context.complete_criterion_is_untrusted else task.complete_criterion
),
terminate_criterion=task.terminate_criterion,
+ # Validation only: that is the task type whose criteria a decision-maker weighs against
+ # each other in both engines, and the only one this was measured on (SKY-16193).
+ criteria_precedence=task.task_type == TaskType.validation,
error_code_mapping=task.error_code_mapping if offer_error_codes else None,
framing=framing,
block_context_section=block_context_section,
diff --git a/skyvern/forge/taskv3/goal_composition.py b/skyvern/forge/taskv3/goal_composition.py
index 95d69a2b8..3f554de11 100644
--- a/skyvern/forge/taskv3/goal_composition.py
+++ b/skyvern/forge/taskv3/goal_composition.py
@@ -39,6 +39,11 @@ class GoalDirectives:
complete_criterion: str | None = None
terminate_criterion: str | None = None
error_code_mapping: dict[str, str] | None = None
+ # Whether to state which criterion wins when both hold. Scoped by the caller to the task type the
+ # measurement covers: on v1's general path the terminate criterion reaches no decision-maker at
+ # all, so "which wins" is not the difference there and a rule written for one population would be
+ # shipping ahead of its evidence.
+ criteria_precedence: bool = False
framing: str = ""
block_context_section: str = ""
@@ -70,6 +75,32 @@ def compose_goal(navigation_goal: str, directives: GoalDirectives) -> str:
goal = (
f"{goal}\n\nIf this becomes true, stop and finish with status=terminated: {directives.terminate_criterion}"
).strip()
+ if directives.complete_criterion and directives.terminate_criterion and directives.criteria_precedence:
+ # Two criteria can describe one page at once -- "no error is present" and "...or the pop up is
+ # not available" both hold on a page with neither. v1's validation prompt offers ONE forced choice
+ # and names completion first in its own rule; v3 read the termination clause as a standing
+ # interrupt and terminated, on criteria that were authored against v1 (SKY-16193). Neither
+ # engine evaluates the two in sequence -- the difference is how the choice is framed.
+ #
+ # Both criteria are NAMED rather than pointed at. A customer's terminate text routinely ends in
+ # a coordinated pair ("...or a dual-eligible flag", "...or shows a login form") sitting thousands
+ # of characters closer to this sentence than the two criteria are, so a demonstrative binds to
+ # the wrong pair -- and on one real workflow it would land directly after an instruction saying
+ # not to let anything override that guard.
+ #
+ # Worded without "the page": a page-free validation block is told it has no browser tools, so a
+ # rule conditioned on what the page shows is dead text on exactly the task type this targets.
+ #
+ # Scoped to the OVERLAP and nothing else. An earlier draft added "terminate only when the
+ # termination criterion holds", which reads as a ban on every other use of the status -- and
+ # `terminated` is also how this engine reports being blocked or the goal being impossible
+ # (engine.py's system prompt, loop.py's stuck options). v1 is broader still: its validation
+ # prompt terminates "when the terminate criterion is met, OR when a complete criterion is
+ # provided but not met". A precedence rule must not quietly narrow the status it mentions.
+ goal = (
+ f"{goal}\n\nIf the completion criterion and the termination criterion both hold at once, "
+ "the completion criterion wins: finish with status=completed."
+ ).strip()
if directives.error_code_mapping:
# v1 shows the model these codes in-loop (see the error_code_mapping_str prompt sites), so
# a v1 terminal verdict names its own code. v3 did not, and the codes were instead matched
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index bf81e74d0..fb92eb9db 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -444,6 +444,31 @@ async def test_execute_task_v3_surfaces_criteria_in_goal(monkeypatch: pytest.Mon
goal = loop_mock.await_args.kwargs["goal"]
assert "the confirmation page is shown" in goal
assert "the posting is closed" in goal
+ # The precedence rule is gated to validation tasks, and this seam is the only place that can catch
+ # the gate regressing: compose_goal's own tests exercise its default, not the policy the caller
+ # sets. `make_task` defaults to TaskType.general.
+ assert "the completion criterion wins" not in goal
+
+
+async def test_execute_task_v3_states_criteria_precedence_only_for_validation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Both criteria can describe one page at once, and the two engines resolved that differently
+ (SKY-16193). v1 shows the terminate criterion to a decision-maker on validation tasks only, so
+ saying which wins has no measured meaning anywhere else."""
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["click"])
+ _step, _task, loop_mock, _post = await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ navigation_goal="Confirm the store was added",
+ complete_criterion="no error message is present",
+ terminate_criterion="an error message is present, or the pop up is not available",
+ task_type=TaskType.validation,
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ assert "the completion criterion wins" in loop_mock.await_args.kwargs["goal"]
@pytest.mark.asyncio
diff --git a/tests/unit/test_taskv3_goal_composition.py b/tests/unit/test_taskv3_goal_composition.py
index f813644f9..2c84af396 100644
--- a/tests/unit/test_taskv3_goal_composition.py
+++ b/tests/unit/test_taskv3_goal_composition.py
@@ -70,6 +70,7 @@ _DIRECTIVE_VALUES: dict[str, tuple[Any, Any]] = {
"complete_criterion": (None, "the confirmation page is showing"),
"terminate_criterion": (None, "the form says the role is closed"),
"error_code_mapping": (None, {"ROLE_CLOSED": "the posting is no longer accepting submissions"}),
+ "criteria_precedence": (False, True),
"framing": ("", "This is one block of a larger workflow."),
"block_context_section": ("", "<workflow_context>\nblocks: one, two\n</workflow_context>"),
}
@@ -80,7 +81,7 @@ _DIRECTIVE_VALUES: dict[str, tuple[Any, Any]] = {
def test_compose_goal_is_byte_identical_to_the_inline_patching_it_replaced(
navigation_goal: str, mask: tuple[int, ...]
) -> None:
- # Every on/off combination of the seven directives, against both an empty and a non-empty
+ # Every on/off combination of the eight directives, against both an empty and a non-empty
# navigation goal. The empty one matters: the first directive's .strip() is what decides
# whether the goal opens with a blank line, and that only shows up when the base is "".
chosen = {name: options[bit] for (name, options), bit in zip(_DIRECTIVE_VALUES.items(), mask)}
@@ -96,12 +97,14 @@ def _goal_as_agent_py_built_it(
extracted_information_schema: Any,
complete_criterion: str | None,
terminate_criterion: str | None,
+ criteria_precedence: bool,
error_code_mapping: dict[str, str] | None,
framing: str,
block_context_section: str,
) -> str:
- """The goal-patching expressions exactly as `ForgeAgent._execute_task_v3` inlined them before
- `compose_goal` existed, frozen here as the oracle for the extraction.
+ """The goal-patching expressions as `ForgeAgent._execute_task_v3` inlined them before `compose_goal`
+ existed, frozen here as the oracle for the extraction, plus any directive a later commit deliberately
+ reworded -- the overlap-precedence sentence is one such, so this is no longer a pure historical record.
This is deliberately a duplicate of production wording: it is the only thing that can catch a
single dropped space or reordered clause in a refactor whose entire acceptance bar is that the
@@ -127,6 +130,15 @@ def _goal_as_agent_py_built_it(
goal = (
f"{goal}\n\nIf this becomes true, stop and finish with status=terminated: {terminate_criterion}"
).strip()
+ if complete_criterion and terminate_criterion and criteria_precedence:
+ # SKY-16193: the one deliberate divergence from what agent.py inlined. Criteria that can hold
+ # at once had no stated precedence, and v3 resolved that the opposite way to the engine they
+ # were authored against. Updated here in the same commit as the production wording, which is
+ # what this oracle's docstring asks of a change that means to reword a directive.
+ goal = (
+ f"{goal}\n\nIf the completion criterion and the termination criterion both hold at once, "
+ "the completion criterion wins: finish with status=completed."
+ ).strip()
if error_code_mapping:
goal = (
f"{goal}\n\nThe user defined these business outcomes and their descriptions:\n"
@@ -163,3 +175,93 @@ def test_the_split_modules_stay_one_way_dependent_on_goal_composition() -> None:
assert not any("goal_composition" in ref for ref in imported), (
f"{module} imports goal_composition; the responsibilities are re-conflating"
)
+
+
+def test_two_criteria_that_can_hold_at_once_get_an_explicit_precedence() -> None:
+ """A block whose criteria overlap has no correct answer without a precedence rule, and the two
+ engines frame that choice differently (SKY-16193).
+
+ The measured case: complete was "if no error message is present" and terminate ended "...or if
+ pop up message is not available". On a page with neither an error nor a popup BOTH hold, and
+ nothing told the model which to apply.
+
+ Effect size, arm-resolved, over runs that REACHED the block: v3 terminated there 22/37 = 59.5%
+ of the time against v1's 14/119 = 11.8%. That establishes something costly happens here; it
+ does not establish what. What does is the composition. v1's `decisive-criterion-validate.j2`
+ asks for "only one action" and resolves both criteria inside a single enum rule that names
+ COMPLETE first (`:11`, `:15`); v3 renders the terminate criterion as its own standing interrupt
+ -- "If this becomes true, stop and finish with status=terminated". The criteria were authored
+ against v1, so v3 is the one that has to say which wins.
+
+ And v1 fails this block too, 14 times in three days. Any mechanism proposed for it has to
+ explain a 5x ratio, not a v3-only defect.
+ """
+ both = compose_goal(
+ "Add the store.",
+ GoalDirectives(
+ complete_criterion="no error message is present",
+ terminate_criterion="an error message is present, or the pop up is not available",
+ criteria_precedence=True,
+ ),
+ )
+ assert "the completion criterion wins" in both
+ # Precedence for the OVERLAP only. An earlier draft added "terminate only when the termination
+ # criterion holds", which reads as a ban on every other use of the status -- and `terminated` is
+ # also how this engine reports being blocked, while v1 terminates whenever a complete criterion is
+ # provided and not met. A precedence rule must not narrow the status it mentions.
+ # The flag must add EXACTLY this paragraph and nothing else. An earlier guard asserted only that
+ # ONE paragraph was added containing ONE full stop, which constrains punctuation rather than
+ # content: round 1's prohibition walks straight through it joined by a semicolon, the naming fix
+ # reverts to a demonstrative untouched, and even inverting the prescribed status passes. Asserting
+ # the text verbatim is what makes the intent readable at the one place a reviewer looks.
+ #
+ # What equality buys, precisely: accidental defeat becomes impossible, and a DELIBERATE edit
+ # becomes visible in the diff, because changing the sentence means retyping it here. It does NOT
+ # make a lockstep edit impossible -- the oracle's docstring still invites one, and someone who
+ # updates production, the oracle and this line together will pass. Naming that boundary is the
+ # point; a fourth guard claiming to close it would repeat the error one level up.
+ #
+ # And it cannot see a prohibition appended to a DIFFERENT directive: the delta is between the
+ # gated and ungated renders, so text added to both cancels. That is a property of the oracle
+ # beside it rather than of this flag.
+ #
+ # AND IT IS THE SOLE CATCH FOR A LOCKSTEP EDIT. A lockstep change defeats the byte-identity mask
+ # BY CONSTRUCTION -- production and the oracle agree again -- so all 512 mask cases stay GREEN and
+ # only the assertion below reds. Three defeats are demonstrated against it: a semicolon for the
+ # full stop, the prescribed status inverted to `terminated`, and a demonstrative restored in the
+ # condition. Weaken, move or lose this one assertion in a refactor and all three go invisible
+ # with every other test in the file still green.
+ ungated_for_delta = compose_goal(
+ "Add the store.",
+ GoalDirectives(
+ complete_criterion="no error message is present",
+ terminate_criterion="an error message is present, or the pop up is not available",
+ ),
+ )
+ added = [p for p in both.split("\n\n") if p not in ungated_for_delta.split("\n\n")]
+ assert added == [
+ "If the completion criterion and the termination criterion both hold at once, "
+ "the completion criterion wins: finish with status=completed."
+ ], added
+ # Named rather than implied: the model must be able to tell which of the two it is applying.
+ assert both.index("Consider the goal complete") < both.index("the completion criterion wins")
+
+ # One criterion alone cannot conflict with anything, so it gets no precedence sentence.
+ complete_only = compose_goal(
+ "Add the store.", GoalDirectives(complete_criterion="the store is listed", criteria_precedence=True)
+ )
+ terminate_only = compose_goal(
+ "Add the store.", GoalDirectives(terminate_criterion="the posting closed", criteria_precedence=True)
+ )
+ assert "the completion criterion wins" not in complete_only
+ assert "the completion criterion wins" not in terminate_only
+
+ # And it is OFF unless the caller asks: v1 shows the terminate criterion to a decision-maker only on
+ # validation tasks, so a rule about which criterion wins has no measured meaning anywhere else.
+ ungated = compose_goal(
+ "Add the store.",
+ GoalDirectives(complete_criterion="no error message is present", terminate_criterion="the pop up is missing"),
+ )
+ assert "the completion criterion wins" not in ungated
+ # Worded without "the page": a page-free validation block is told it has no browser tools at all.
+ assert "the page satisfies" not in both