Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Tell every Task V3 extraction-only block to report what the page shows, and remove its experiment flag (#8771)
pedrohsdbd962f9fa13a2
diff --git a/skyvern/config.py b/skyvern/config.pyindex 47edff2f2..e495f8e5a 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -556,10 +556,6 @@ class Settings(BaseSettings):TASK_V3_GOAL_CHECK_ENFORCE: bool = False# The judge's model. Unset means the judge never runs; there is no fallback to another model.TASK_V3_GOAL_CHECK_LLM_KEY: str | None = None- # Tell a block with no navigation goal that it reports what the page shows -- absent fields as null,- # finished completed -- as v1's single extract action does (SKY-16398). Force-on term only: runs are- # randomized per run by the flag of the same name, read through run_arm_enabled().- TASK_V3_EXTRACTION_REPORTS: bool = False# Which browser surface the v3 loop offers: today's action tools ("off"), those plus a code# tool ("add"), or the code tool instead of them ("replace"). Three states rather than a boolean# because the benchmark separated add from replace on speed alone, not on success. The code tooldiff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 9e5a43462..70af3a208 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -196,7 +196,6 @@ from skyvern.forge.taskv3.loop import LoopOutcome, RoundActionfrom skyvern.forge.taskv3.pre_submit_capture import PreSubmitCaptureRing, is_run_sampled, pre_submit_screenshotfrom skyvern.forge.taskv3.run_arms import (DATE_SEGMENT_AIM_FLAG,- EXTRACTION_REPORTS_FLAG,GOAL_CHECK_ENFORCE_FLAG,GOAL_CHECK_FLAG,resolve_run_arm,@@ -2317,13 +2316,6 @@ class ForgeAgent:organization_id=task.organization_id,forced=settings.TASK_V3_DATE_SEGMENT_AIM,)- await resolve_run_arm(- context,- EXTRACTION_REPORTS_FLAG,- distinct_id=task.workflow_run_id or task.task_id,- organization_id=task.organization_id,- forced=settings.TASK_V3_EXTRACTION_REPORTS,- )# The judge's finish-time screenshot, reused as the decision screenshot of an accepted completion.goal_judge_shot: list[bytes] = []page_free_validation = bool(@@ -2376,7 +2368,6 @@ class ForgeAgent:handoff_enabled=handoff_enabled,previous_block=previous_block,selected_block_labels=context.run_block_labels if context else None,- extraction_reports=run_arm_enabled(EXTRACTION_REPORTS_FLAG, settings.TASK_V3_EXTRACTION_REPORTS),)# Surface the customer's completion/termination criteria (trusted task config, like the navigation# goal). Withhold a complete_criterion flagged untrusted (LLM-derived from page content) so it can't bediff --git a/skyvern/forge/taskv3/goal_composition.py b/skyvern/forge/taskv3/goal_composition.pyindex 62d76244d..83b987729 100644--- a/skyvern/forge/taskv3/goal_composition.py+++ b/skyvern/forge/taskv3/goal_composition.py@@ -245,7 +245,6 @@ def render_block_context(handoff_enabled: bool = False,previous_block: PreviousBlockHandoff | None = None,selected_block_labels: list[str] | None = None,- extraction_reports: bool = False,) -> tuple[str, str]:"""Return ``(framing, section)`` for a block task; both are ``""`` for a bare task.@@ -297,8 +296,7 @@ def render_block_context(elif task_block is not None and task.task_type == TaskType.action:pieces.append("This is a single, focused action: perform it and finish.")elif (- extraction_reports- and task_block is not None+ task_block is not Noneand task_block.block_type == BlockType.EXTRACTIONand not task_block.is_internal_evaluationand not task.navigation_goaldiff --git a/skyvern/forge/taskv3/run_arms.py b/skyvern/forge/taskv3/run_arms.pyindex 86ff96cea..29abc3e39 100644--- a/skyvern/forge/taskv3/run_arms.py+++ b/skyvern/forge/taskv3/run_arms.py@@ -17,7 +17,6 @@ from skyvern.forge.sdk.experimentation.providers import NoOpExperimentationProviLOG = structlog.get_logger()DATE_SEGMENT_AIM_FLAG = "TASK_V3_DATE_SEGMENT_AIM"-EXTRACTION_REPORTS_FLAG = "TASK_V3_EXTRACTION_REPORTS"GOAL_CHECK_FLAG = "TASK_V3_GOAL_CHECK"GOAL_CHECK_ENFORCE_FLAG = "TASK_V3_GOAL_CHECK_ENFORCE"diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex ea6d013f5..e35c49122 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -80,7 +80,6 @@ from skyvern.forge.taskv3.loop import ()from skyvern.forge.taskv3.run_arms import (DATE_SEGMENT_AIM_FLAG,- EXTRACTION_REPORTS_FLAG,run_arm_enabled,)from skyvern.forge.taskv3.tools import PageProvider, _record_frame_work@@ -493,18 +492,10 @@ async def test_execute_task_v3_passes_the_workflow_system_prompt_to_a_page_free_@pytest.mark.asyncio[email protected](("variant", "framed"), [("treatment", True), ("control", False)])-async def test_execute_task_v3_tells_an_extraction_block_to_report_only_in_the_treatment_arm(- monkeypatch: pytest.MonkeyPatch, variant: str, framed: bool-) -> None:- # The arm must be RESOLVED before the goal is composed: a deleted resolve call, or a render keyed on the- # wrong flag, leaves every run on control with the composition tests still green (SKY-16398).- monkeypatch.setattr(settings, "TASK_V3_EXTRACTION_REPORTS", False)- provider = AsyncMock(return_value=variant)- monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", provider)-+async def test_execute_task_v3_tells_an_extraction_block_it_only_reports(monkeypatch: pytest.MonkeyPatch) -> None:+ # The framing must reach the goal the loop is given, not only render_block_context (SKY-16398).outcome = LoopOutcome(status="completed", reason="done", billable_actions=[], extracted_output={"status": None})- _step, task, loop_mock, _post = await _run_execute_task_v3(+ _step, _task, loop_mock, _post = await _run_execute_task_v3(monkeypatch,outcome,task_block=_make_block(ExtractionBlock, data_extraction_goal="the license status"),@@ -513,8 +504,7 @@ async def test_execute_task_v3_tells_an_extraction_block_to_report_only_in_the_tdata_extraction_goal="the license status",)- assert ("This block only reads the page" in loop_mock.await_args.kwargs["goal"]) is framed- assert loop_mock.context.run_arms[EXTRACTION_REPORTS_FLAG] == (task.workflow_run_id, variant)+ assert "This block only reads the page" in loop_mock.await_args.kwargs["goal"]@pytest.mark.asynciodiff --git a/tests/unit/test_taskv3_goal_composition.py b/tests/unit/test_taskv3_goal_composition.pyindex 308cb7dab..2fde48860 100644--- a/tests/unit/test_taskv3_goal_composition.py+++ b/tests/unit/test_taskv3_goal_composition.py@@ -265,7 +265,7 @@ def test_an_extraction_only_block_is_told_to_report_absent_data_as_completed_nul# v1 runs a block with no navigation goal as ONE extract action that returns nulls for whatever the page does# not show and completes; workflows branch on those nulls. v3 was told neither what such a block is for nor# what finishing it means, so it failed the block whenever an earlier block had not reached the data- # (SKY-16398). Present on exactly v1's predicate, only in the treatment arm.+ # (SKY-16398). Present on exactly v1's predicate.now = datetime.now(UTC)org = make_organization(now)extraction_only = make_task(now, org, navigation_goal=None, data_extraction_goal="the license status")@@ -273,10 +273,9 @@ def test_an_extraction_only_block_is_told_to_report_absent_data_as_completed_nullabel="blk", output_parameter=output_param("blk"), data_extraction_goal="the license status")- treated, _ = render_block_context(extraction_only, block, None, extraction_reports=True)- control, _ = render_block_context(extraction_only, block, None, extraction_reports=False)+ framing, _ = render_block_context(extraction_only, block, None)- added = [p for p in treated.split("\n\n") if p not in control.split("\n\n")]+ added = [p for p in framing.split("\n\n") if p.startswith("This block only reads the page")]assert len(added) == 1, addedassert "null" in added[0]assert "status=completed" in added[0]@@ -286,8 +285,6 @@ def test_an_extraction_only_block_is_told_to_report_absent_data_as_completed_nulassert "read from the page" in added[0]assert "never invent" in added[0]assert "current date" in added[0]- # The control render is what shipped before the arm existed.- assert control == render_block_context(extraction_only, block, None)[0]out_of_predicate = [make_task(now, org, navigation_goal="Search for the record", data_extraction_goal="the license status"),@@ -295,15 +292,11 @@ def test_an_extraction_only_block_is_told_to_report_absent_data_as_completed_nulmake_task(now, org, navigation_goal=None, data_extraction_goal="x", task_type=TaskType.validation),]for task in out_of_predicate:- assert render_block_context(task, block, None, extraction_reports=True) == render_block_context(- task, block, None- )+ assert "This block only reads the page" not in render_block_context(task, block, None)[0]# A task block carrying only an extraction goal keeps its fill tools, so it is not told it only reads.- assert render_block_context(extraction_only, _make_block("blk"), None, extraction_reports=True) == (- render_block_context(extraction_only, _make_block("blk"), None)- )+ assert "This block only reads the page" not in render_block_context(extraction_only, _make_block("blk"), None)[0]# A bare task has no workflow to route its nulls, so it gets no block framing at all.- assert render_block_context(extraction_only, None, None, extraction_reports=True) == ("", "")+ assert render_block_context(extraction_only, None, None) == ("", "")def test_a_page_value_cannot_close_its_own_span() -> None:diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.pyindex 837a54f03..31979fa69 100644--- a/tests/unit/test_workflow_block_engine.py+++ b/tests/unit/test_workflow_block_engine.py@@ -869,8 +869,8 @@ async def test_branch_eval_synthetic_block_honors_v3_override(scoped_context: Sk@pytest.mark.asyncioasync def test_branch_eval_synthetic_block_gets_no_extraction_report_framing(scoped_context: SkyvernContext) -> None:- """The branch evaluator's block follows the run's arm and has no navigation goal, so without its marker the- TASK_V3_EXTRACTION_REPORTS framing ("absent fields are null, finish completed") would reach it. Its own prompt+ """The branch evaluator's block has no navigation goal, so without its marker the extraction-report+ framing ("absent fields are null, finish completed") would reach it. Its own promptjudges conditions from their text, and a null result parses as False: a silently wrong branch (SKY-16398)."""scoped_context.workflow_block_engine_resolved_run_id = "wr_branch_eval_framing"scoped_context.workflow_block_engine_override = RunEngine.skyvern_v3@@ -902,7 +902,7 @@ async def test_branch_eval_synthetic_block_gets_no_extraction_report_framing(sco(block,) = capturednow = datetime.now(UTC)task = make_task(now, make_organization(now), navigation_goal=None, data_extraction_goal=block.data_extraction_goal)- assert render_block_context(task, block, None, extraction_reports=True) == render_block_context(task, block, None)+ assert "This block only reads the page" not in render_block_context(task, block, None)[0]@pytest.mark.asyncio