Skip to content

Automate browser based workflows with AI

Ask the goal judge before a single-action block completes on its step cap (#8811)

pedrohsdb5c623a878bbb
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index c8a6cf367..5226895ac 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -188,6 +188,7 @@ from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter, Workf
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatus
from skyvern.forge.sdk.workflow.page_derived_templates import NO_RENDER_RECORD, UNVERIFIED_ROOT_CLASSES
from skyvern.forge.taskv3.goal_check import (
+ BLOCK_COMPLETION_CHECK_PROMPT_NAME,
GOAL_CHECK_PROMPT_NAME,
PRE_JUDGE_SKIP_REASONS,
GoalJudge,
@@ -1421,6 +1422,7 @@ def _build_task_v3_goal_judge(
browser_state: BrowserState,
peek_page: Callable[[], Awaitable[Any]],
shot_holder: list[bytes],
+ prompt_name: str = GOAL_CHECK_PROMPT_NAME,
) -> GoalJudge:
handler = LLMAPIHandlerFactory.get_llm_api_handler(judge_key)
@@ -1439,12 +1441,14 @@ def _build_task_v3_goal_judge(
engine_selection=browser_state.engine_selection,
)
except Exception:
- LOG.warning("taskv3 goal check screenshot failed", task_id=task.task_id, exc_info=True)
+ LOG.warning(
+ "taskv3 goal check screenshot failed", task_id=task.task_id, prompt_name=prompt_name, exc_info=True
+ )
return None
shot_holder.append(shot)
response = await handler(
prompt=prompt,
- prompt_name=GOAL_CHECK_PROMPT_NAME,
+ prompt_name=prompt_name,
step=step,
screenshots=[shot],
)
@@ -3110,6 +3114,39 @@ class ForgeAgent:
peek_page=_fingerprint_page,
shot_holder=goal_judge_shot,
)
+ # A block that completes on a download is not done by its one action.
+ single_action_block = isinstance(task_block, ActionBlock) and not task_block.complete_on_download
+ block_completion_judge: GoalJudge | None = None
+ if single_action_block:
+ # The run's own model, outside the goal-check arm's payload; its non-flex twin, as flex queueing
+ # outlasts the judge's timeout. Without a judge the block's step-cap completion is never offered.
+ # The registry key: an OpenRouter caller rewrites llm_key to the bare model id.
+ run_key = llm_caller.original_llm_key
+ twin_key = app.AGENT_FUNCTION.get_standard_tier_twin_llm_key(run_key)
+ block_judge_key = twin_key if twin_key and LLMConfigRegistry.is_registered(twin_key) else run_key
+ block_judge_skip = (
+ "screenshots_disabled"
+ if context is not None and not context.llm_screenshots_enabled_for_prompt()
+ else _goal_judge_key_skip_reason(block_judge_key)
+ )
+ if block_judge_skip is not None:
+ LOG.info(
+ "taskv3 block completion judge skipped",
+ task_id=task.task_id,
+ reason=block_judge_skip,
+ judge_key=block_judge_key,
+ )
+ else:
+ assert block_judge_key is not None
+ block_completion_judge = _build_task_v3_goal_judge(
+ judge_key=block_judge_key,
+ task=task,
+ step=step,
+ browser_state=browser_state,
+ peek_page=_fingerprint_page,
+ shot_holder=[],
+ prompt_name=BLOCK_COMPLETION_CHECK_PROMPT_NAME,
+ )
outcome = await run_task_v3_agent_loop(
page_provider=_page_provider,
resolve_typed_text=resolve_typed_text,
@@ -3145,7 +3182,9 @@ class ForgeAgent:
),
goal_check_redactor=(
(lambda: _task_v3_goal_check_redactor(task, context))
- if goal_judge is not None or unlisted_reask_criteria is not None
+ if goal_judge is not None
+ or block_completion_judge is not None
+ or unlisted_reask_criteria is not None
else None
),
unlisted_reask_criteria=unlisted_reask_criteria,
@@ -3199,8 +3238,8 @@ class ForgeAgent:
caller_known_urls=verdict_known_urls,
label_secret_values=_label_secret_values,
login_identifier_tokens=_login_identifier_tokens,
- # A block that completes on a download is not done by its one action.
- single_action_block=isinstance(task_block, ActionBlock) and not task_block.complete_on_download,
+ single_action_block=single_action_block,
+ block_completion_judge=block_completion_judge,
code_typed_values=recovery_code_progress.typed_values if recovery_code_progress else (),
)
finally:
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index a7c354214..c357bdbed 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -306,6 +306,8 @@ async def run_task_v3_agent_loop(
# The workflow system prompt reads a page-derived value, so the re-ask shows it as untrusted data.
unlisted_reask_instructions_untrusted: bool = False,
single_action_block: bool = False,
+ # Asked before a single-action block completes on its step cap; outside the goal-check arm, never counted in it.
+ block_completion_judge: GoalJudge | None = None,
# Appended to the goal, whose Code outline section is last, only as far as the request has room for them.
code_typed_values: tuple[CodeTypedValue, ...] = (),
) -> LoopOutcome:
@@ -437,28 +439,28 @@ async def run_task_v3_agent_loop(
verification_blocker = None
refuse_input_entry = block_type == BlockType.EXTRACTION
deadline_at = time.monotonic() + deadline_seconds if deadline_seconds is not None else None
- goal_check_on = goal_judge is not None and goal_check_eligible(
+ # A block with its own completion verifier is left to it, by every judge.
+ judge_eligible = goal_check_eligible(
page_free=page_free,
completion_blocker_present=completion_blocker is not None,
extraction_requested=extraction_requested,
)
- # Same eligibility as the goal check: a block with its own completion verifier is left to it.
- reask_on = unlisted_reask_criteria is not None and goal_check_eligible(
- page_free=page_free,
- completion_blocker_present=completion_blocker is not None,
- extraction_requested=extraction_requested,
+ goal_check_on = goal_judge is not None and judge_eligible
+ reask_on = unlisted_reask_criteria is not None and judge_eligible
+ block_check_on = single_action_block and block_completion_judge is not None and judge_eligible
+ tool_trail = (
+ ToolTrail(secret_entered=secret_on_page_at_start) if goal_check_on or reask_on or block_check_on else None
)
- tool_trail = ToolTrail(secret_entered=secret_on_page_at_start) if goal_check_on or reask_on else None
goal_verdicts: list[GoalVerdict] = []
reasks: list[UnlistedReask] = []
- async def _goal_check() -> GoalVerdict:
- assert goal_judge is not None and tool_trail is not None
+ async def _judge_goal(judge: GoalJudge, *, secret_entered: bool, failure_log: str) -> GoalVerdict:
+ assert tool_trail is not None
timeout = GOAL_CHECK_TIMEOUT_SECONDS
if deadline_at is not None:
timeout = min(timeout, deadline_at - time.monotonic() - GOAL_CHECK_DEADLINE_MARGIN_SECONDS)
redact = goal_check_redactor() if goal_check_redactor is not None else None
- if tool_trail.secret_entered:
+ if secret_entered:
verdict = GoalVerdict("achieved", "", "", "secret_entered", 0.0)
elif timeout <= 0:
verdict = GoalVerdict("achieved", "", "", "deadline", 0.0)
@@ -470,14 +472,32 @@ async def run_task_v3_agent_loop(
verdict = await run_goal_check(
goal=model_goal,
trail=tool_trail,
- judge=goal_judge,
+ judge=judge,
timeout_seconds=timeout,
instructions=goal_instructions,
redact=redact,
+ failure_log=failure_log,
)
+ return verdict
+
+ async def _goal_check() -> GoalVerdict:
+ assert goal_judge is not None
+ assert tool_trail is not None
+ verdict = await _judge_goal(
+ goal_judge, secret_entered=tool_trail.secret_entered, failure_log="taskv3 goal check judge failed"
+ )
goal_verdicts.append(verdict)
return verdict
+ async def _block_completion_check() -> GoalVerdict:
+ assert block_completion_judge is not None and tool_trail is not None
+ # A secret typed before this block does not skip it: every block after a sign-in would then be refused.
+ return await _judge_goal(
+ block_completion_judge,
+ secret_entered=tool_trail.secret_entered_in_loop,
+ failure_log="taskv3 block completion judge failed",
+ )
+
async def _reask_judge(reask_caller: LLMCaller, prompt: str) -> dict[str, Any] | None:
# The run's own model, on its non-flex twin when one exists: flex queueing outlasts the 20s limit.
# No message history, so the loop's transcript is untouched.
@@ -627,6 +647,7 @@ async def run_task_v3_agent_loop(
refuse_input_entry=refuse_input_entry,
tool_trail=tool_trail,
single_action_block=single_action_block,
+ block_completion_check=_block_completion_check if block_check_on else None,
)
finally:
# The context outlives this run; a signal raised as the loop was cancelled must not fire
diff --git a/skyvern/forge/taskv3/goal_check.py b/skyvern/forge/taskv3/goal_check.py
index 137e82713..656509fe6 100644
--- a/skyvern/forge/taskv3/goal_check.py
+++ b/skyvern/forge/taskv3/goal_check.py
@@ -22,6 +22,9 @@ from skyvern.utils.strings import escape_code_fences, neutralize_untrusted_web_p
LOG = structlog.get_logger()
GOAL_CHECK_PROMPT_NAME = "taskv3-goal-check"
+# The same prompt, asked on a step-cap block completion outside the goal-check arm; its own name keeps the arm's calls
+# countable on their own.
+BLOCK_COMPLETION_CHECK_PROMPT_NAME = "taskv3-block-completion-check"
GOAL_CHECK_TIMEOUT_SECONDS = 20.0
TRAIL_SIZE = 8
RESULT_MAX_CHARS = 2000
@@ -79,9 +82,11 @@ class ToolTrail:
self.calls_since_page_read = 0
self.page_changes_since_page_read = 0
# Sticky: once a secret reached the page, a finish-time screenshot can show it (v3 tools apply
- # no visual secret mask), so the judge is not called for the rest of the run. Seeded when one
+ # no visual secret mask), so the goal-check arm's judge is not called for the rest of the run. Seeded when one
# may already be on the page before this loop starts.
self.secret_entered = secret_entered
+ # Only this loop's own calls: the typed secret can still be on screen, possibly in the run's first screenshot.
+ self.secret_entered_in_loop = False
# Unbounded, unlike `entries`: a value typed early in the run is still one this block entered.
self.entered_values: set[str] = set()
self.last_page_change: TrailEntry | None = None
@@ -110,6 +115,7 @@ class ToolTrail:
if url is not None:
self._urls_seen.add(url)
self.secret_entered = self.secret_entered or entry.secret_entered
+ self.secret_entered_in_loop = self.secret_entered_in_loop or entry.secret_entered
if entry.status == "ok" and not entry.secret_entered:
self.entered_values.update(entry.entered)
if entry.perception and entry.status == "ok":
@@ -330,6 +336,7 @@ async def run_goal_check(
timeout_seconds: float,
instructions: str = "",
redact: Redactor | None = None,
+ failure_log: str = "taskv3 goal check judge failed",
) -> GoalVerdict:
"""Every failure to reach a grounded verdict accepts the completion (verdict "achieved") and says why."""
started = time.monotonic()
@@ -340,9 +347,9 @@ async def run_goal_check(
try:
prompt, evidence = render_goal_check_prompt(goal, trail, instructions, redact)
except Exception:
- LOG.warning("taskv3 goal check judge failed", exc_info=True)
+ LOG.warning(failure_log, exc_info=True)
return _fail_open("judge_error")
- response, skipped_reason = await _ask(prompt, judge, timeout_seconds, "taskv3 goal check judge failed")
+ response, skipped_reason = await _ask(prompt, judge, timeout_seconds, failure_log)
if response is None:
return _fail_open(skipped_reason or "judge_declined")
verdict = response.get("verdict")
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index 1df0ba6ff..276457ecb 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -3291,6 +3291,8 @@ async def run_agent_tool_loop(
# A block whose contract is one action: once one succeeded, a follow-up the step cap refuses offers
# finish(completed) instead of failing the block, as the step engine completes it after that step.
single_action_block: bool = False,
+ # The goal judge's verdict on that completion; the completion is offered only on its grounded "achieved".
+ block_completion_check: Callable[[], Awaitable[GoalVerdict]] | None = None,
) -> LoopOutcome:
tool_by_name = {tool.name: tool for tool in tools}
st = LoopState(
@@ -4302,15 +4304,31 @@ async def run_agent_tool_loop(
and not st.block_completion_offered
):
st.block_completion_offered = True
+ # A successful action that moved nothing (a no-op click, a username typed before the
+ # login submits) is no sign the block is done, and a URL change can land on the wrong page
+ # or the next step of the same form. Every failure to reach a grounded "achieved" fails closed.
+ block_verdict: GoalVerdict | None = None
+ judge_skipped: str | None = None
+ if st.block_action_transitioned:
+ if block_completion_check is None:
+ judge_skipped = "no_judge"
+ else:
+ try:
+ block_verdict = await block_completion_check()
+ judge_skipped = block_verdict.skipped_reason
+ except Exception:
+ LOG.warning("taskv3 block completion judge raised", exc_info=True)
+ judge_skipped = "judge_error"
+ judged = block_verdict if judge_skipped is None else None
LOG.info(
"taskv3 block completion evidence",
url_changed=st.block_action_transitioned,
billable_actions=st.billable_actions,
+ judge_verdict=judged.verdict if judged is not None else None,
+ judge_skipped_reason=judge_skipped,
+ judge_latency_s=block_verdict.latency_s if block_verdict is not None else None,
)
- # A successful action that moved nothing (a no-op click, a username typed before the
- # login submits) is no sign the block is done; the finish gate's goal check, under enforce,
- # still vetoes a URL change that landed on the wrong page.
- if st.block_action_transitioned:
+ if judged is not None and judged.verdict == "achieved":
block_reason = (
f"performed the block's action ({st.billable_actions[0]}); "
"a further action was past the block's step limit"
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index 0ddca7218..9af3c36ff 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -67,6 +67,7 @@ from skyvern.forge.taskv3 import engine as taskv3_engine
from skyvern.forge.taskv3 import tools as taskv3_tools
from skyvern.forge.taskv3.auth_tools import VerificationFailure, VerificationState
from skyvern.forge.taskv3.engine import DEFAULT_MAX_SETTLE_DEFERRALS, MIN_ACTION_STEPS, run_task_v3_agent_loop
+from skyvern.forge.taskv3.goal_check import BLOCK_COMPLETION_CHECK_PROMPT_NAME
from skyvern.forge.taskv3.goal_composition import CodeProgressRecord, CodeTypedValue
from skyvern.forge.taskv3.handoff_redaction import pin_caller_authored_block_urls
from skyvern.forge.taskv3.loop import (
@@ -149,6 +150,7 @@ async def _run_execute_task_v3(
goal_judge_prompt: str | None = None,
# Leave the credential-TOTP candidate gate reading the real workflow-run context.
real_credential_totp_candidate: bool = False,
+ llm_caller_factory: Any = None,
**task_overrides: Any,
) -> tuple[Step, Any, AsyncMock, AsyncMock]:
agent = ForgeAgent()
@@ -213,7 +215,7 @@ async def _run_execute_task_v3(
loop_mock = AsyncMock(side_effect=_loop)
loop_mock.browser_state = browser_state
monkeypatch.setattr("skyvern.forge.taskv3.engine.run_task_v3_agent_loop", loop_mock)
- monkeypatch.setattr("skyvern.forge.agent.LLMCaller", MagicMock())
+ monkeypatch.setattr("skyvern.forge.agent.LLMCaller", llm_caller_factory or MagicMock())
monkeypatch.setattr("skyvern.forge.sdk.api.files.resolve_run_download_id", lambda *_a, **_k: "download-1")
monkeypatch.setattr("skyvern.forge.sdk.api.files.get_download_dir", lambda *_a, **_k: "/tmp/taskv3-test")
monkeypatch.setattr(
@@ -6408,6 +6410,59 @@ def _arm_goal_judge(
return judge_handler, get_handler
+ ("caller_key", "registry_key", "twin_key", "context_overrides", "judge_key"),
+ [
+ (_JUDGE_KEY, _JUDGE_KEY, None, {}, _JUDGE_KEY),
+ # An OpenRouter caller rewrites llm_key to the bare model id; the registry name is what resolves.
+ ("vendor/bare-model-id", _JUDGE_KEY, None, {}, _JUDGE_KEY),
+ ("RUN_FLEX_KEY", "RUN_FLEX_KEY", _JUDGE_KEY, {}, _JUDGE_KEY),
+ (_JUDGE_KEY, _JUDGE_KEY, None, {"enrich_tree_mode": EnrichTreeMode.ENRICHED_TREE_NO_IMAGES}, None),
+ (_BYO_JUDGE_KEY, _BYO_JUDGE_KEY, None, {}, None),
+ ],
+ ids=["run_key", "caller_rewrites_key", "standard_tier_twin", "screenshots_disabled", "byo_key"],
+)
+async def test_block_completion_judge_runs_on_the_runs_own_key_outside_the_goal_check_arm(
+ monkeypatch: pytest.MonkeyPatch,
+ caller_key: str,
+ registry_key: str,
+ twin_key: str | None,
+ context_overrides: dict[str, Any],
+ judge_key: str | None,
+) -> None:
+ judge_handler, get_handler = _arm_goal_judge(monkeypatch, forced=False)
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.AGENT_FUNCTION.get_standard_tier_twin_llm_key", MagicMock(return_value=twin_key)
+ )
+ caller = MagicMock(llm_key=caller_key, original_llm_key=registry_key)
+
+ with capture_logs() as logs:
+ step, _task, loop_mock, _post = await _run_execute_task_v3(
+ monkeypatch,
+ LoopOutcome(status="completed", reason="done", billable_actions=[]),
+ task_block=_make_block(ActionBlock),
+ context_overrides=context_overrides,
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ llm_caller_factory=MagicMock(return_value=caller),
+ )
+
+ assert loop_mock.call_args.kwargs["goal_judge"] is None
+ block_judge = loop_mock.call_args.kwargs["block_completion_judge"]
+ if judge_key is None:
+ assert block_judge is None
+ assert [log for log in logs if log["event"] == "taskv3 block completion judge skipped"]
+ get_handler.assert_not_called()
+ return
+ loop_mock.browser_state.must_get_working_page.return_value.is_closed = MagicMock(return_value=False)
+ await block_judge("judge prompt")
+ get_handler.assert_called_once_with(judge_key)
+ kwargs = judge_handler.await_args.kwargs
+ assert kwargs["prompt_name"] == BLOCK_COMPLETION_CHECK_PROMPT_NAME
+ assert kwargs["step"] is step
+
+
def _resolved_goal_check_flags(logs: list[dict[str, Any]]) -> set[str]:
return {
log["flag"] for log in logs if log["event"] == "Resolved Task V3 run arm" and log["flag"] in _GOAL_CHECK_FLAGS
diff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.py
index 7f9c30b29..72486b45c 100644
--- a/tests/unit/test_taskv3_engine.py
+++ b/tests/unit/test_taskv3_engine.py
@@ -258,9 +258,94 @@ async def test_engine_forwards_single_action_block_to_the_loop(
llm_caller=_ScriptedCaller([]),
goal="x",
single_action_block=single_action_block,
+ block_completion_judge=_achieving_judge([]),
)
assert captured["single_action_block"] is single_action_block
+ assert (captured["block_completion_check"] is not None) is single_action_block
+
+
+def _achieving_judge(prompts: list[str]):
+ async def judge(prompt: str) -> dict[str, object]:
+ prompts.append(prompt)
+ return {"verdict": "achieved", "quote": "", "missing": ""}
+
+ return judge
+
+
+async def test_engine_block_completion_judge_stays_out_of_the_goal_check_arm(monkeypatch: pytest.MonkeyPatch) -> None:
+ # A secret on the page skips the arm's judge but not the block's, which runs on the loop's own model; neither
+ # judge's verdict is counted as the other's.
+ from skyvern.forge.taskv3 import engine as engine_mod
+ from skyvern.forge.taskv3.loop import LoopOutcome
+
+ arm_prompts: list[str] = []
+ block_prompts: list[str] = []
+
+ async def _capture(**kwargs: Any) -> LoopOutcome:
+ verdict = await kwargs["block_completion_check"]()
+ assert verdict.verdict == "achieved" and verdict.skipped_reason is None
+ assert (await finish_kwargs["goal_check"]()).skipped_reason == "secret_entered"
+ return LoopOutcome(status="completed", reason="ok")
+
+ finish_kwargs: dict[str, Any] = {}
+ real_make_finish_tool = engine_mod.make_finish_tool
+
+ def _capture_finish(**kwargs: Any) -> Any:
+ finish_kwargs.update(kwargs)
+ return real_make_finish_tool(**kwargs)
+
+ monkeypatch.setattr(engine_mod, "make_finish_tool", _capture_finish)
+ monkeypatch.setattr(engine_mod, "run_agent_tool_loop", _capture)
+ outcome = await run_task_v3_agent_loop(
+ page_provider=_fixed_page_provider(_FakePage()),
+ llm_caller=_ScriptedCaller([]),
+ goal="x",
+ single_action_block=True,
+ goal_judge=_achieving_judge(arm_prompts),
+ block_completion_judge=_achieving_judge(block_prompts),
+ secret_on_page_at_start=True,
+ )
+
+ assert len(block_prompts) == 1 and not arm_prompts
+ assert outcome.goal_check is not None and outcome.goal_check["checks"] == 1
+ assert outcome.goal_check["judged"] == 0
+
+
+async def test_engine_block_completion_judge_skips_after_a_secret_typed_in_this_block(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # The secret's field can still be on screen and pixels cannot be redacted; a secret from an earlier block
+ # (secret_on_page_at_start) does not skip it.
+ from skyvern.forge.taskv3 import engine as engine_mod
+ from skyvern.forge.taskv3.goal_check import TrailEntry
+ from skyvern.forge.taskv3.loop import LoopOutcome
+
+ prompts: list[str] = []
+ verdicts: list[Any] = []
+
+ async def _capture(**kwargs: Any) -> LoopOutcome:
+ kwargs["tool_trail"].record(
+ TrailEntry(
+ tool="type", status="ok", content="typed", perception=False, page_changing=True, secret_entered=True
+ )
+ )
+ verdicts.append(await kwargs["block_completion_check"]())
+ return LoopOutcome(status="completed", reason="ok")
+
+ monkeypatch.setattr(engine_mod, "run_agent_tool_loop", _capture)
+ await run_task_v3_agent_loop(
+ page_provider=_fixed_page_provider(_FakePage()),
+ llm_caller=_ScriptedCaller([]),
+ goal="x",
+ single_action_block=True,
+ block_completion_judge=_achieving_judge(prompts),
+ )
+
+ [verdict] = verdicts
+ assert verdict.skipped_reason == "secret_entered" and not prompts
def test_runaway_backstops_scale_with_action_step_budget() -> None:
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index 33fc95b88..975048016 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -33,7 +33,7 @@ from skyvern.forge.taskv3 import loop as loop_module
from skyvern.forge.taskv3 import tools as taskv3_tools_module
from skyvern.forge.taskv3.auth_tools import _COMPLETION_BLOCKED, VerificationFailure, VerificationState
from skyvern.forge.taskv3.engine import MAX_TOKENS_CEILING, MAX_TOKENS_PER_ACTION_STEP, taskv3_runaway_backstops
-from skyvern.forge.taskv3.goal_check import GoalVerdict
+from skyvern.forge.taskv3.goal_check import GoalVerdict, ToolTrail, run_goal_check
from skyvern.forge.taskv3.handoff_redaction import ELIDED_URL_PATH, caller_known_published_urls
from skyvern.forge.taskv3.loop import (
ACTION_BUDGET_EXTENDED_EVENT,
@@ -1008,6 +1008,10 @@ async def test_spent_grant_caught_at_the_step_gate_reports_the_granting_cap() ->
_REACHED_TARGET = {"page_transitioned": True}
+async def _achieved_block_check() -> GoalVerdict:
+ return GoalVerdict("achieved", "", "", None, 0.1)
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("single_action_block", [True, False], ids=["single_action", "multi_action"])
@pytest.mark.parametrize("cap", ["step_cap", "runaway"])
@@ -1025,7 +1029,13 @@ async def test_single_action_block_completes_when_its_step_cap_refuses_a_follow_
)
caps: dict[str, Any] = {"max_action_steps": 1} if cap == "step_cap" else {"max_turns": 1}
script = [[("click", {})], [("click", {})], [("click", {})]]
- outcome, _ = await _run(script, [click, make_finish_tool()], single_action_block=single_action_block, **caps)
+ outcome, _ = await _run(
+ script,
+ [click, make_finish_tool()],
+ single_action_block=single_action_block,
+ block_completion_check=_achieved_block_check,
+ **caps,
+ )
if single_action_block and cap == "step_cap" and acted:
assert outcome.status == "completed"
@@ -1052,7 +1062,13 @@ async def test_single_action_block_completes_on_the_step_cap_only_when_its_actio
click_calls: list[tuple[str, dict[str, Any]]] = []
click = _recording_tool("click", click_calls, billable=True, ok_data=data)
script = [[("click", {})], [("click", {})], [("click", {})]]
- outcome, _ = await _run(script, [click, make_finish_tool()], single_action_block=True, max_action_steps=1)
+ outcome, _ = await _run(
+ script,
+ [click, make_finish_tool()],
+ single_action_block=True,
+ block_completion_check=_achieved_block_check,
+ max_action_steps=1,
+ )
assert outcome.status == ("completed" if url_changed else "budget_exhausted")
if not url_changed:
@@ -1067,7 +1083,13 @@ async def test_a_navigation_that_landed_on_an_error_page_is_no_evidence_for_the_
nav_calls: list[tuple[str, dict[str, Any]]] = []
nav = _recording_tool("navigate", nav_calls, billable=True, ok_data=dead_end)
script = [[("navigate", {})], [("navigate", {})], [("navigate", {})]]
- outcome, _ = await _run(script, [nav, make_finish_tool()], single_action_block=True, max_action_steps=1)
+ outcome, _ = await _run(
+ script,
+ [nav, make_finish_tool()],
+ single_action_block=True,
+ block_completion_check=_achieved_block_check,
+ max_action_steps=1,
+ )
assert outcome.status == "budget_exhausted"
@@ -1085,6 +1107,7 @@ async def test_a_contradicting_goal_check_vetoes_the_step_cap_completion_only_un
script,
[click, make_finish_tool(goal_check=goal_check, goal_check_enforce=enforce)],
single_action_block=True,
+ block_completion_check=_achieved_block_check,
max_action_steps=1,
)
@@ -1108,6 +1131,7 @@ async def test_single_action_block_completion_refused_by_a_finish_guard_stays_bu
script,
[click, make_finish_tool(verification_blocker=blocker)],
single_action_block=True,
+ block_completion_check=_achieved_block_check,
max_action_steps=1,
)
@@ -1135,6 +1159,7 @@ async def test_single_action_block_completion_is_offered_once_even_when_its_guar
script,
[click, make_finish_tool(verification_blocker=blocker)],
single_action_block=True,
+ block_completion_check=_achieved_block_check,
max_action_steps=1,
)
@@ -1143,6 +1168,104 @@ async def test_single_action_block_completion_is_offered_once_even_when_its_guar
assert len(offers) == 1
+_LANDED_ON_PASSWORD_STEP = "clicked button 'Next' — now at https://example.com/login/password\nEnter your password"
+_LANDED_ON_OTHER_LINK = "clicked link 'Report 2023' — now at https://example.com/reports/2023\nReport 2023"
+_LANDED_ON_TARGET = "clicked link 'Report 2024' — now at https://example.com/reports/2024\nReport 2024"
+
+
+ ("landed", "verdict", "quote", "completes"),
+ [
+ (_LANDED_ON_PASSWORD_STEP, "not_achieved", "Enter your password", False),
+ (_LANDED_ON_OTHER_LINK, "not_achieved", "Report 2023", False),
+ # The goal-check arm accepts an ungrounded contradiction; this conversion does not.
+ (_LANDED_ON_OTHER_LINK, "not_achieved", "Report 1999", False),
+ (_LANDED_ON_TARGET, "achieved", "", True),
+ ],
+ ids=["stopped_after_username_step", "clicked_a_different_link", "ungrounded_contradiction", "reached_target"],
+)
+async def test_step_cap_block_completion_converts_only_on_the_goal_judges_grounded_achieved(
+ landed: str, verdict: str, quote: str, completes: bool
+) -> None:
+ # A URL change shows the action took effect, not that the block reached its goal: the next step of the same
+ # login and a different link both move the URL.
+ trail = ToolTrail()
+ prompts: list[str] = []
+
+ async def judge(prompt: str) -> dict[str, Any]:
+ prompts.append(prompt)
+ return {"verdict": verdict, "quote": quote, "missing": "the page is not the goal's page"}
+
+ async def check() -> GoalVerdict:
+ return await run_goal_check(goal="open the block's target", trail=trail, judge=judge, timeout_seconds=5)
+
+ async def click(args: dict[str, Any]) -> ToolResult:
+ return ToolResult.ok(landed, data=_REACHED_TARGET)
+
+ tool = ToolSpec(name="click", description="click", parameters={"type": "object"}, handler=click, billable=True)
+ script = [[("click", {})], [("click", {})], [("click", {})]]
+ outcome, caller = await _run(
+ script,
+ [tool, make_finish_tool()],
+ single_action_block=True,
+ max_action_steps=1,
+ tool_trail=trail,
+ block_completion_check=check,
+ )
+
+ assert len(prompts) == 1 and landed.splitlines()[0] in prompts[0]
+ assert outcome.status == ("completed" if completes else "budget_exhausted")
+ # The judge read a screenshot that can show a typed secret: its text never reaches the model.
+ assert all("the page is not the goal's page" not in str(m.get("content")) for m in caller.message_history)
+
+
[email protected]("failure", ["no_judge", "raises", "timeout", "judge_error", "secret_entered"])
+async def test_step_cap_block_completion_fails_closed_without_a_judged_verdict(failure: str) -> None:
+ async def check() -> GoalVerdict:
+ if failure == "raises":
+ raise RuntimeError("judge down")
+ # run_goal_check's shape for a check that reached no verdict: "achieved", with the reason it was skipped.
+ return GoalVerdict("achieved", "", "", failure, 0.1)
+
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ click = _recording_tool("click", click_calls, billable=True, ok_data=_REACHED_TARGET)
+ script = [[("click", {})], [("click", {})], [("click", {})]]
+ outcome, _ = await _run(
+ script,
+ [click, make_finish_tool()],
+ single_action_block=True,
+ max_action_steps=1,
+ block_completion_check=None if failure == "no_judge" else check,
+ )
+
+ assert outcome.status == "budget_exhausted"
+ assert outcome.cap_trip == "Reached the maximum steps (1)"
+
+
+async def test_step_cap_block_completion_evidence_log_carries_the_verdict_not_the_judges_text() -> None:
+ async def check() -> GoalVerdict:
+ return GoalVerdict("not_achieved", "code 482913 entered", "482913 is still shown", None, 0.1)
+
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ click = _recording_tool("click", click_calls, billable=True, ok_data=_REACHED_TARGET)
+ script = [[("click", {})], [("click", {})], [("click", {})]]
+ with capture_logs() as logs:
+ await _run(
+ script,
+ [click, make_finish_tool()],
+ single_action_block=True,
+ max_action_steps=1,
+ block_completion_check=check,
+ )
+
+ [evidence] = [entry for entry in logs if entry["event"] == "taskv3 block completion evidence"]
+ assert evidence["judge_verdict"] == "not_achieved"
+ assert "482913" not in repr(evidence)
+
+
@pytest.mark.asyncio
async def test_step_gate_on_the_granted_turn_salvages_a_staged_finish_output() -> None:
# The granted turn batches an over-cap action AND a finish: the refused action voids the