Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Ask the goal judge before a single-action block completes on its step cap (#8811)
pedrohsdb5c623a878bbb
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex c8a6cf367..5226895ac 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -188,6 +188,7 @@ from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter, Workffrom skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatusfrom skyvern.forge.sdk.workflow.page_derived_templates import NO_RENDER_RECORD, UNVERIFIED_ROOT_CLASSESfrom skyvern.forge.taskv3.goal_check import (+ BLOCK_COMPLETION_CHECK_PROMPT_NAME,GOAL_CHECK_PROMPT_NAME,PRE_JUDGE_SKIP_REASONS,GoalJudge,@@ -1421,6 +1422,7 @@ def _build_task_v3_goal_judge(browser_state: BrowserState,peek_page: Callable[[], Awaitable[Any]],shot_holder: list[bytes],+ prompt_name: str = GOAL_CHECK_PROMPT_NAME,) -> GoalJudge:handler = LLMAPIHandlerFactory.get_llm_api_handler(judge_key)@@ -1439,12 +1441,14 @@ def _build_task_v3_goal_judge(engine_selection=browser_state.engine_selection,)except Exception:- LOG.warning("taskv3 goal check screenshot failed", task_id=task.task_id, exc_info=True)+ LOG.warning(+ "taskv3 goal check screenshot failed", task_id=task.task_id, prompt_name=prompt_name, exc_info=True+ )return Noneshot_holder.append(shot)response = await handler(prompt=prompt,- prompt_name=GOAL_CHECK_PROMPT_NAME,+ prompt_name=prompt_name,step=step,screenshots=[shot],)@@ -3110,6 +3114,39 @@ class ForgeAgent:peek_page=_fingerprint_page,shot_holder=goal_judge_shot,)+ # A block that completes on a download is not done by its one action.+ single_action_block = isinstance(task_block, ActionBlock) and not task_block.complete_on_download+ block_completion_judge: GoalJudge | None = None+ if single_action_block:+ # The run's own model, outside the goal-check arm's payload; its non-flex twin, as flex queueing+ # outlasts the judge's timeout. Without a judge the block's step-cap completion is never offered.+ # The registry key: an OpenRouter caller rewrites llm_key to the bare model id.+ run_key = llm_caller.original_llm_key+ twin_key = app.AGENT_FUNCTION.get_standard_tier_twin_llm_key(run_key)+ block_judge_key = twin_key if twin_key and LLMConfigRegistry.is_registered(twin_key) else run_key+ block_judge_skip = (+ "screenshots_disabled"+ if context is not None and not context.llm_screenshots_enabled_for_prompt()+ else _goal_judge_key_skip_reason(block_judge_key)+ )+ if block_judge_skip is not None:+ LOG.info(+ "taskv3 block completion judge skipped",+ task_id=task.task_id,+ reason=block_judge_skip,+ judge_key=block_judge_key,+ )+ else:+ assert block_judge_key is not None+ block_completion_judge = _build_task_v3_goal_judge(+ judge_key=block_judge_key,+ task=task,+ step=step,+ browser_state=browser_state,+ peek_page=_fingerprint_page,+ shot_holder=[],+ prompt_name=BLOCK_COMPLETION_CHECK_PROMPT_NAME,+ )outcome = await run_task_v3_agent_loop(page_provider=_page_provider,resolve_typed_text=resolve_typed_text,@@ -3145,7 +3182,9 @@ class ForgeAgent:),goal_check_redactor=((lambda: _task_v3_goal_check_redactor(task, context))- if goal_judge is not None or unlisted_reask_criteria is not None+ if goal_judge is not None+ or block_completion_judge is not None+ or unlisted_reask_criteria is not Noneelse None),unlisted_reask_criteria=unlisted_reask_criteria,@@ -3199,8 +3238,8 @@ class ForgeAgent:caller_known_urls=verdict_known_urls,label_secret_values=_label_secret_values,login_identifier_tokens=_login_identifier_tokens,- # A block that completes on a download is not done by its one action.- single_action_block=isinstance(task_block, ActionBlock) and not task_block.complete_on_download,+ single_action_block=single_action_block,+ block_completion_judge=block_completion_judge,code_typed_values=recovery_code_progress.typed_values if recovery_code_progress else (),)finally:diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.pyindex a7c354214..c357bdbed 100644--- a/skyvern/forge/taskv3/engine.py+++ b/skyvern/forge/taskv3/engine.py@@ -306,6 +306,8 @@ async def run_task_v3_agent_loop(# The workflow system prompt reads a page-derived value, so the re-ask shows it as untrusted data.unlisted_reask_instructions_untrusted: bool = False,single_action_block: bool = False,+ # Asked before a single-action block completes on its step cap; outside the goal-check arm, never counted in it.+ block_completion_judge: GoalJudge | None = None,# Appended to the goal, whose Code outline section is last, only as far as the request has room for them.code_typed_values: tuple[CodeTypedValue, ...] = (),) -> LoopOutcome:@@ -437,28 +439,28 @@ async def run_task_v3_agent_loop(verification_blocker = Nonerefuse_input_entry = block_type == BlockType.EXTRACTIONdeadline_at = time.monotonic() + deadline_seconds if deadline_seconds is not None else None- goal_check_on = goal_judge is not None and goal_check_eligible(+ # A block with its own completion verifier is left to it, by every judge.+ judge_eligible = goal_check_eligible(page_free=page_free,completion_blocker_present=completion_blocker is not None,extraction_requested=extraction_requested,)- # Same eligibility as the goal check: a block with its own completion verifier is left to it.- reask_on = unlisted_reask_criteria is not None and goal_check_eligible(- page_free=page_free,- completion_blocker_present=completion_blocker is not None,- extraction_requested=extraction_requested,+ goal_check_on = goal_judge is not None and judge_eligible+ reask_on = unlisted_reask_criteria is not None and judge_eligible+ block_check_on = single_action_block and block_completion_judge is not None and judge_eligible+ tool_trail = (+ ToolTrail(secret_entered=secret_on_page_at_start) if goal_check_on or reask_on or block_check_on else None)- tool_trail = ToolTrail(secret_entered=secret_on_page_at_start) if goal_check_on or reask_on else Nonegoal_verdicts: list[GoalVerdict] = []reasks: list[UnlistedReask] = []- async def _goal_check() -> GoalVerdict:- assert goal_judge is not None and tool_trail is not None+ async def _judge_goal(judge: GoalJudge, *, secret_entered: bool, failure_log: str) -> GoalVerdict:+ assert tool_trail is not Nonetimeout = GOAL_CHECK_TIMEOUT_SECONDSif deadline_at is not None:timeout = min(timeout, deadline_at - time.monotonic() - GOAL_CHECK_DEADLINE_MARGIN_SECONDS)redact = goal_check_redactor() if goal_check_redactor is not None else None- if tool_trail.secret_entered:+ if secret_entered:verdict = GoalVerdict("achieved", "", "", "secret_entered", 0.0)elif timeout <= 0:verdict = GoalVerdict("achieved", "", "", "deadline", 0.0)@@ -470,14 +472,32 @@ async def run_task_v3_agent_loop(verdict = await run_goal_check(goal=model_goal,trail=tool_trail,- judge=goal_judge,+ judge=judge,timeout_seconds=timeout,instructions=goal_instructions,redact=redact,+ failure_log=failure_log,)+ return verdict++ async def _goal_check() -> GoalVerdict:+ assert goal_judge is not None+ assert tool_trail is not None+ verdict = await _judge_goal(+ goal_judge, secret_entered=tool_trail.secret_entered, failure_log="taskv3 goal check judge failed"+ )goal_verdicts.append(verdict)return verdict+ async def _block_completion_check() -> GoalVerdict:+ assert block_completion_judge is not None and tool_trail is not None+ # A secret typed before this block does not skip it: every block after a sign-in would then be refused.+ return await _judge_goal(+ block_completion_judge,+ secret_entered=tool_trail.secret_entered_in_loop,+ failure_log="taskv3 block completion judge failed",+ )+async def _reask_judge(reask_caller: LLMCaller, prompt: str) -> dict[str, Any] | None:# The run's own model, on its non-flex twin when one exists: flex queueing outlasts the 20s limit.# No message history, so the loop's transcript is untouched.@@ -627,6 +647,7 @@ async def run_task_v3_agent_loop(refuse_input_entry=refuse_input_entry,tool_trail=tool_trail,single_action_block=single_action_block,+ block_completion_check=_block_completion_check if block_check_on else None,)finally:# The context outlives this run; a signal raised as the loop was cancelled must not firediff --git a/skyvern/forge/taskv3/goal_check.py b/skyvern/forge/taskv3/goal_check.pyindex 137e82713..656509fe6 100644--- a/skyvern/forge/taskv3/goal_check.py+++ b/skyvern/forge/taskv3/goal_check.py@@ -22,6 +22,9 @@ from skyvern.utils.strings import escape_code_fences, neutralize_untrusted_web_pLOG = structlog.get_logger()GOAL_CHECK_PROMPT_NAME = "taskv3-goal-check"+# The same prompt, asked on a step-cap block completion outside the goal-check arm; its own name keeps the arm's calls+# countable on their own.+BLOCK_COMPLETION_CHECK_PROMPT_NAME = "taskv3-block-completion-check"GOAL_CHECK_TIMEOUT_SECONDS = 20.0TRAIL_SIZE = 8RESULT_MAX_CHARS = 2000@@ -79,9 +82,11 @@ class ToolTrail:self.calls_since_page_read = 0self.page_changes_since_page_read = 0# Sticky: once a secret reached the page, a finish-time screenshot can show it (v3 tools apply- # no visual secret mask), so the judge is not called for the rest of the run. Seeded when one+ # no visual secret mask), so the goal-check arm's judge is not called for the rest of the run. Seeded when one# may already be on the page before this loop starts.self.secret_entered = secret_entered+ # Only this loop's own calls: the typed secret can still be on screen, possibly in the run's first screenshot.+ self.secret_entered_in_loop = False# Unbounded, unlike `entries`: a value typed early in the run is still one this block entered.self.entered_values: set[str] = set()self.last_page_change: TrailEntry | None = None@@ -110,6 +115,7 @@ class ToolTrail:if url is not None:self._urls_seen.add(url)self.secret_entered = self.secret_entered or entry.secret_entered+ self.secret_entered_in_loop = self.secret_entered_in_loop or entry.secret_enteredif entry.status == "ok" and not entry.secret_entered:self.entered_values.update(entry.entered)if entry.perception and entry.status == "ok":@@ -330,6 +336,7 @@ async def run_goal_check(timeout_seconds: float,instructions: str = "",redact: Redactor | None = None,+ failure_log: str = "taskv3 goal check judge failed",) -> GoalVerdict:"""Every failure to reach a grounded verdict accepts the completion (verdict "achieved") and says why."""started = time.monotonic()@@ -340,9 +347,9 @@ async def run_goal_check(try:prompt, evidence = render_goal_check_prompt(goal, trail, instructions, redact)except Exception:- LOG.warning("taskv3 goal check judge failed", exc_info=True)+ LOG.warning(failure_log, exc_info=True)return _fail_open("judge_error")- response, skipped_reason = await _ask(prompt, judge, timeout_seconds, "taskv3 goal check judge failed")+ response, skipped_reason = await _ask(prompt, judge, timeout_seconds, failure_log)if response is None:return _fail_open(skipped_reason or "judge_declined")verdict = response.get("verdict")diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex 1df0ba6ff..276457ecb 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -3291,6 +3291,8 @@ async def run_agent_tool_loop(# A block whose contract is one action: once one succeeded, a follow-up the step cap refuses offers# finish(completed) instead of failing the block, as the step engine completes it after that step.single_action_block: bool = False,+ # The goal judge's verdict on that completion; the completion is offered only on its grounded "achieved".+ block_completion_check: Callable[[], Awaitable[GoalVerdict]] | None = None,) -> LoopOutcome:tool_by_name = {tool.name: tool for tool in tools}st = LoopState(@@ -4302,15 +4304,31 @@ async def run_agent_tool_loop(and not st.block_completion_offered):st.block_completion_offered = True+ # A successful action that moved nothing (a no-op click, a username typed before the+ # login submits) is no sign the block is done, and a URL change can land on the wrong page+ # or the next step of the same form. Every failure to reach a grounded "achieved" fails closed.+ block_verdict: GoalVerdict | None = None+ judge_skipped: str | None = None+ if st.block_action_transitioned:+ if block_completion_check is None:+ judge_skipped = "no_judge"+ else:+ try:+ block_verdict = await block_completion_check()+ judge_skipped = block_verdict.skipped_reason+ except Exception:+ LOG.warning("taskv3 block completion judge raised", exc_info=True)+ judge_skipped = "judge_error"+ judged = block_verdict if judge_skipped is None else NoneLOG.info("taskv3 block completion evidence",url_changed=st.block_action_transitioned,billable_actions=st.billable_actions,+ judge_verdict=judged.verdict if judged is not None else None,+ judge_skipped_reason=judge_skipped,+ judge_latency_s=block_verdict.latency_s if block_verdict is not None else None,)- # A successful action that moved nothing (a no-op click, a username typed before the- # login submits) is no sign the block is done; the finish gate's goal check, under enforce,- # still vetoes a URL change that landed on the wrong page.- if st.block_action_transitioned:+ if judged is not None and judged.verdict == "achieved":block_reason = (f"performed the block's action ({st.billable_actions[0]}); ""a further action was past the block's step limit"diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex 0ddca7218..9af3c36ff 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -67,6 +67,7 @@ from skyvern.forge.taskv3 import engine as taskv3_enginefrom skyvern.forge.taskv3 import tools as taskv3_toolsfrom skyvern.forge.taskv3.auth_tools import VerificationFailure, VerificationStatefrom skyvern.forge.taskv3.engine import DEFAULT_MAX_SETTLE_DEFERRALS, MIN_ACTION_STEPS, run_task_v3_agent_loop+from skyvern.forge.taskv3.goal_check import BLOCK_COMPLETION_CHECK_PROMPT_NAMEfrom skyvern.forge.taskv3.goal_composition import CodeProgressRecord, CodeTypedValuefrom skyvern.forge.taskv3.handoff_redaction import pin_caller_authored_block_urlsfrom skyvern.forge.taskv3.loop import (@@ -149,6 +150,7 @@ async def _run_execute_task_v3(goal_judge_prompt: str | None = None,# Leave the credential-TOTP candidate gate reading the real workflow-run context.real_credential_totp_candidate: bool = False,+ llm_caller_factory: Any = None,**task_overrides: Any,) -> tuple[Step, Any, AsyncMock, AsyncMock]:agent = ForgeAgent()@@ -213,7 +215,7 @@ async def _run_execute_task_v3(loop_mock = AsyncMock(side_effect=_loop)loop_mock.browser_state = browser_statemonkeypatch.setattr("skyvern.forge.taskv3.engine.run_task_v3_agent_loop", loop_mock)- monkeypatch.setattr("skyvern.forge.agent.LLMCaller", MagicMock())+ monkeypatch.setattr("skyvern.forge.agent.LLMCaller", llm_caller_factory or MagicMock())monkeypatch.setattr("skyvern.forge.sdk.api.files.resolve_run_download_id", lambda *_a, **_k: "download-1")monkeypatch.setattr("skyvern.forge.sdk.api.files.get_download_dir", lambda *_a, **_k: "/tmp/taskv3-test")monkeypatch.setattr(@@ -6408,6 +6410,59 @@ def _arm_goal_judge(return judge_handler, get_handler+ ("caller_key", "registry_key", "twin_key", "context_overrides", "judge_key"),+ [+ (_JUDGE_KEY, _JUDGE_KEY, None, {}, _JUDGE_KEY),+ # An OpenRouter caller rewrites llm_key to the bare model id; the registry name is what resolves.+ ("vendor/bare-model-id", _JUDGE_KEY, None, {}, _JUDGE_KEY),+ ("RUN_FLEX_KEY", "RUN_FLEX_KEY", _JUDGE_KEY, {}, _JUDGE_KEY),+ (_JUDGE_KEY, _JUDGE_KEY, None, {"enrich_tree_mode": EnrichTreeMode.ENRICHED_TREE_NO_IMAGES}, None),+ (_BYO_JUDGE_KEY, _BYO_JUDGE_KEY, None, {}, None),+ ],+ ids=["run_key", "caller_rewrites_key", "standard_tier_twin", "screenshots_disabled", "byo_key"],+)+async def test_block_completion_judge_runs_on_the_runs_own_key_outside_the_goal_check_arm(+ monkeypatch: pytest.MonkeyPatch,+ caller_key: str,+ registry_key: str,+ twin_key: str | None,+ context_overrides: dict[str, Any],+ judge_key: str | None,+) -> None:+ judge_handler, get_handler = _arm_goal_judge(monkeypatch, forced=False)+ monkeypatch.setattr(+ "skyvern.forge.agent.app.AGENT_FUNCTION.get_standard_tier_twin_llm_key", MagicMock(return_value=twin_key)+ )+ caller = MagicMock(llm_key=caller_key, original_llm_key=registry_key)++ with capture_logs() as logs:+ step, _task, loop_mock, _post = await _run_execute_task_v3(+ monkeypatch,+ LoopOutcome(status="completed", reason="done", billable_actions=[]),+ task_block=_make_block(ActionBlock),+ context_overrides=context_overrides,+ data_extraction_goal=None,+ extracted_information_schema=None,+ llm_caller_factory=MagicMock(return_value=caller),+ )++ assert loop_mock.call_args.kwargs["goal_judge"] is None+ block_judge = loop_mock.call_args.kwargs["block_completion_judge"]+ if judge_key is None:+ assert block_judge is None+ assert [log for log in logs if log["event"] == "taskv3 block completion judge skipped"]+ get_handler.assert_not_called()+ return+ loop_mock.browser_state.must_get_working_page.return_value.is_closed = MagicMock(return_value=False)+ await block_judge("judge prompt")+ get_handler.assert_called_once_with(judge_key)+ kwargs = judge_handler.await_args.kwargs+ assert kwargs["prompt_name"] == BLOCK_COMPLETION_CHECK_PROMPT_NAME+ assert kwargs["step"] is step++def _resolved_goal_check_flags(logs: list[dict[str, Any]]) -> set[str]:return {log["flag"] for log in logs if log["event"] == "Resolved Task V3 run arm" and log["flag"] in _GOAL_CHECK_FLAGSdiff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.pyindex 7f9c30b29..72486b45c 100644--- a/tests/unit/test_taskv3_engine.py+++ b/tests/unit/test_taskv3_engine.py@@ -258,9 +258,94 @@ async def test_engine_forwards_single_action_block_to_the_loop(llm_caller=_ScriptedCaller([]),goal="x",single_action_block=single_action_block,+ block_completion_judge=_achieving_judge([]),)assert captured["single_action_block"] is single_action_block+ assert (captured["block_completion_check"] is not None) is single_action_block+++def _achieving_judge(prompts: list[str]):+ async def judge(prompt: str) -> dict[str, object]:+ prompts.append(prompt)+ return {"verdict": "achieved", "quote": "", "missing": ""}++ return judge+++async def test_engine_block_completion_judge_stays_out_of_the_goal_check_arm(monkeypatch: pytest.MonkeyPatch) -> None:+ # A secret on the page skips the arm's judge but not the block's, which runs on the loop's own model; neither+ # judge's verdict is counted as the other's.+ from skyvern.forge.taskv3 import engine as engine_mod+ from skyvern.forge.taskv3.loop import LoopOutcome++ arm_prompts: list[str] = []+ block_prompts: list[str] = []++ async def _capture(**kwargs: Any) -> LoopOutcome:+ verdict = await kwargs["block_completion_check"]()+ assert verdict.verdict == "achieved" and verdict.skipped_reason is None+ assert (await finish_kwargs["goal_check"]()).skipped_reason == "secret_entered"+ return LoopOutcome(status="completed", reason="ok")++ finish_kwargs: dict[str, Any] = {}+ real_make_finish_tool = engine_mod.make_finish_tool++ def _capture_finish(**kwargs: Any) -> Any:+ finish_kwargs.update(kwargs)+ return real_make_finish_tool(**kwargs)++ monkeypatch.setattr(engine_mod, "make_finish_tool", _capture_finish)+ monkeypatch.setattr(engine_mod, "run_agent_tool_loop", _capture)+ outcome = await run_task_v3_agent_loop(+ page_provider=_fixed_page_provider(_FakePage()),+ llm_caller=_ScriptedCaller([]),+ goal="x",+ single_action_block=True,+ goal_judge=_achieving_judge(arm_prompts),+ block_completion_judge=_achieving_judge(block_prompts),+ secret_on_page_at_start=True,+ )++ assert len(block_prompts) == 1 and not arm_prompts+ assert outcome.goal_check is not None and outcome.goal_check["checks"] == 1+ assert outcome.goal_check["judged"] == 0+++async def test_engine_block_completion_judge_skips_after_a_secret_typed_in_this_block(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # The secret's field can still be on screen and pixels cannot be redacted; a secret from an earlier block+ # (secret_on_page_at_start) does not skip it.+ from skyvern.forge.taskv3 import engine as engine_mod+ from skyvern.forge.taskv3.goal_check import TrailEntry+ from skyvern.forge.taskv3.loop import LoopOutcome++ prompts: list[str] = []+ verdicts: list[Any] = []++ async def _capture(**kwargs: Any) -> LoopOutcome:+ kwargs["tool_trail"].record(+ TrailEntry(+ tool="type", status="ok", content="typed", perception=False, page_changing=True, secret_entered=True+ )+ )+ verdicts.append(await kwargs["block_completion_check"]())+ return LoopOutcome(status="completed", reason="ok")++ monkeypatch.setattr(engine_mod, "run_agent_tool_loop", _capture)+ await run_task_v3_agent_loop(+ page_provider=_fixed_page_provider(_FakePage()),+ llm_caller=_ScriptedCaller([]),+ goal="x",+ single_action_block=True,+ block_completion_judge=_achieving_judge(prompts),+ )++ [verdict] = verdicts+ assert verdict.skipped_reason == "secret_entered" and not promptsdef test_runaway_backstops_scale_with_action_step_budget() -> None:diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex 33fc95b88..975048016 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -33,7 +33,7 @@ from skyvern.forge.taskv3 import loop as loop_modulefrom skyvern.forge.taskv3 import tools as taskv3_tools_modulefrom skyvern.forge.taskv3.auth_tools import _COMPLETION_BLOCKED, VerificationFailure, VerificationStatefrom skyvern.forge.taskv3.engine import MAX_TOKENS_CEILING, MAX_TOKENS_PER_ACTION_STEP, taskv3_runaway_backstops-from skyvern.forge.taskv3.goal_check import GoalVerdict+from skyvern.forge.taskv3.goal_check import GoalVerdict, ToolTrail, run_goal_checkfrom skyvern.forge.taskv3.handoff_redaction import ELIDED_URL_PATH, caller_known_published_urlsfrom skyvern.forge.taskv3.loop import (ACTION_BUDGET_EXTENDED_EVENT,@@ -1008,6 +1008,10 @@ async def test_spent_grant_caught_at_the_step_gate_reports_the_granting_cap() ->_REACHED_TARGET = {"page_transitioned": True}+async def _achieved_block_check() -> GoalVerdict:+ return GoalVerdict("achieved", "", "", None, 0.1)++@pytest.mark.asyncio@pytest.mark.parametrize("single_action_block", [True, False], ids=["single_action", "multi_action"])@pytest.mark.parametrize("cap", ["step_cap", "runaway"])@@ -1025,7 +1029,13 @@ async def test_single_action_block_completes_when_its_step_cap_refuses_a_follow_)caps: dict[str, Any] = {"max_action_steps": 1} if cap == "step_cap" else {"max_turns": 1}script = [[("click", {})], [("click", {})], [("click", {})]]- outcome, _ = await _run(script, [click, make_finish_tool()], single_action_block=single_action_block, **caps)+ outcome, _ = await _run(+ script,+ [click, make_finish_tool()],+ single_action_block=single_action_block,+ block_completion_check=_achieved_block_check,+ **caps,+ )if single_action_block and cap == "step_cap" and acted:assert outcome.status == "completed"@@ -1052,7 +1062,13 @@ async def test_single_action_block_completes_on_the_step_cap_only_when_its_actioclick_calls: list[tuple[str, dict[str, Any]]] = []click = _recording_tool("click", click_calls, billable=True, ok_data=data)script = [[("click", {})], [("click", {})], [("click", {})]]- outcome, _ = await _run(script, [click, make_finish_tool()], single_action_block=True, max_action_steps=1)+ outcome, _ = await _run(+ script,+ [click, make_finish_tool()],+ single_action_block=True,+ block_completion_check=_achieved_block_check,+ max_action_steps=1,+ )assert outcome.status == ("completed" if url_changed else "budget_exhausted")if not url_changed:@@ -1067,7 +1083,13 @@ async def test_a_navigation_that_landed_on_an_error_page_is_no_evidence_for_the_nav_calls: list[tuple[str, dict[str, Any]]] = []nav = _recording_tool("navigate", nav_calls, billable=True, ok_data=dead_end)script = [[("navigate", {})], [("navigate", {})], [("navigate", {})]]- outcome, _ = await _run(script, [nav, make_finish_tool()], single_action_block=True, max_action_steps=1)+ outcome, _ = await _run(+ script,+ [nav, make_finish_tool()],+ single_action_block=True,+ block_completion_check=_achieved_block_check,+ max_action_steps=1,+ )assert outcome.status == "budget_exhausted"@@ -1085,6 +1107,7 @@ async def test_a_contradicting_goal_check_vetoes_the_step_cap_completion_only_unscript,[click, make_finish_tool(goal_check=goal_check, goal_check_enforce=enforce)],single_action_block=True,+ block_completion_check=_achieved_block_check,max_action_steps=1,)@@ -1108,6 +1131,7 @@ async def test_single_action_block_completion_refused_by_a_finish_guard_stays_buscript,[click, make_finish_tool(verification_blocker=blocker)],single_action_block=True,+ block_completion_check=_achieved_block_check,max_action_steps=1,)@@ -1135,6 +1159,7 @@ async def test_single_action_block_completion_is_offered_once_even_when_its_guarscript,[click, make_finish_tool(verification_blocker=blocker)],single_action_block=True,+ block_completion_check=_achieved_block_check,max_action_steps=1,)@@ -1143,6 +1168,104 @@ async def test_single_action_block_completion_is_offered_once_even_when_its_guarassert len(offers) == 1+_LANDED_ON_PASSWORD_STEP = "clicked button 'Next' — now at https://example.com/login/password\nEnter your password"+_LANDED_ON_OTHER_LINK = "clicked link 'Report 2023' — now at https://example.com/reports/2023\nReport 2023"+_LANDED_ON_TARGET = "clicked link 'Report 2024' — now at https://example.com/reports/2024\nReport 2024"+++ ("landed", "verdict", "quote", "completes"),+ [+ (_LANDED_ON_PASSWORD_STEP, "not_achieved", "Enter your password", False),+ (_LANDED_ON_OTHER_LINK, "not_achieved", "Report 2023", False),+ # The goal-check arm accepts an ungrounded contradiction; this conversion does not.+ (_LANDED_ON_OTHER_LINK, "not_achieved", "Report 1999", False),+ (_LANDED_ON_TARGET, "achieved", "", True),+ ],+ ids=["stopped_after_username_step", "clicked_a_different_link", "ungrounded_contradiction", "reached_target"],+)+async def test_step_cap_block_completion_converts_only_on_the_goal_judges_grounded_achieved(+ landed: str, verdict: str, quote: str, completes: bool+) -> None:+ # A URL change shows the action took effect, not that the block reached its goal: the next step of the same+ # login and a different link both move the URL.+ trail = ToolTrail()+ prompts: list[str] = []++ async def judge(prompt: str) -> dict[str, Any]:+ prompts.append(prompt)+ return {"verdict": verdict, "quote": quote, "missing": "the page is not the goal's page"}++ async def check() -> GoalVerdict:+ return await run_goal_check(goal="open the block's target", trail=trail, judge=judge, timeout_seconds=5)++ async def click(args: dict[str, Any]) -> ToolResult:+ return ToolResult.ok(landed, data=_REACHED_TARGET)++ tool = ToolSpec(name="click", description="click", parameters={"type": "object"}, handler=click, billable=True)+ script = [[("click", {})], [("click", {})], [("click", {})]]+ outcome, caller = await _run(+ script,+ [tool, make_finish_tool()],+ single_action_block=True,+ max_action_steps=1,+ tool_trail=trail,+ block_completion_check=check,+ )++ assert len(prompts) == 1 and landed.splitlines()[0] in prompts[0]+ assert outcome.status == ("completed" if completes else "budget_exhausted")+ # The judge read a screenshot that can show a typed secret: its text never reaches the model.+ assert all("the page is not the goal's page" not in str(m.get("content")) for m in caller.message_history)++[email protected]("failure", ["no_judge", "raises", "timeout", "judge_error", "secret_entered"])+async def test_step_cap_block_completion_fails_closed_without_a_judged_verdict(failure: str) -> None:+ async def check() -> GoalVerdict:+ if failure == "raises":+ raise RuntimeError("judge down")+ # run_goal_check's shape for a check that reached no verdict: "achieved", with the reason it was skipped.+ return GoalVerdict("achieved", "", "", failure, 0.1)++ click_calls: list[tuple[str, dict[str, Any]]] = []+ click = _recording_tool("click", click_calls, billable=True, ok_data=_REACHED_TARGET)+ script = [[("click", {})], [("click", {})], [("click", {})]]+ outcome, _ = await _run(+ script,+ [click, make_finish_tool()],+ single_action_block=True,+ max_action_steps=1,+ block_completion_check=None if failure == "no_judge" else check,+ )++ assert outcome.status == "budget_exhausted"+ assert outcome.cap_trip == "Reached the maximum steps (1)"+++async def test_step_cap_block_completion_evidence_log_carries_the_verdict_not_the_judges_text() -> None:+ async def check() -> GoalVerdict:+ return GoalVerdict("not_achieved", "code 482913 entered", "482913 is still shown", None, 0.1)++ click_calls: list[tuple[str, dict[str, Any]]] = []+ click = _recording_tool("click", click_calls, billable=True, ok_data=_REACHED_TARGET)+ script = [[("click", {})], [("click", {})], [("click", {})]]+ with capture_logs() as logs:+ await _run(+ script,+ [click, make_finish_tool()],+ single_action_block=True,+ max_action_steps=1,+ block_completion_check=check,+ )++ [evidence] = [entry for entry in logs if entry["event"] == "taskv3 block completion evidence"]+ assert evidence["judge_verdict"] == "not_achieved"+ assert "482913" not in repr(evidence)++@pytest.mark.asyncioasync def test_step_gate_on_the_granted_turn_salvages_a_staged_finish_output() -> None:# The granted turn batches an over-cap action AND a finish: the refused action voids the