Skip to content

Automate browser based workflows with AI

Tell every Task V3 extraction-only block to report what the page shows, and remove its experiment flag (#8771)

pedrohsdbd962f9fa13a2
diff --git a/skyvern/config.py b/skyvern/config.py
index 47edff2f2..e495f8e5a 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -556,10 +556,6 @@ class Settings(BaseSettings):
TASK_V3_GOAL_CHECK_ENFORCE: bool = False
# The judge's model. Unset means the judge never runs; there is no fallback to another model.
TASK_V3_GOAL_CHECK_LLM_KEY: str | None = None
- # Tell a block with no navigation goal that it reports what the page shows -- absent fields as null,
- # finished completed -- as v1's single extract action does (SKY-16398). Force-on term only: runs are
- # randomized per run by the flag of the same name, read through run_arm_enabled().
- TASK_V3_EXTRACTION_REPORTS: bool = False
# Which browser surface the v3 loop offers: today's action tools ("off"), those plus a code
# tool ("add"), or the code tool instead of them ("replace"). Three states rather than a boolean
# because the benchmark separated add from replace on speed alone, not on success. The code tool
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 9e5a43462..70af3a208 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -196,7 +196,6 @@ from skyvern.forge.taskv3.loop import LoopOutcome, RoundAction
from skyvern.forge.taskv3.pre_submit_capture import PreSubmitCaptureRing, is_run_sampled, pre_submit_screenshot
from skyvern.forge.taskv3.run_arms import (
DATE_SEGMENT_AIM_FLAG,
- EXTRACTION_REPORTS_FLAG,
GOAL_CHECK_ENFORCE_FLAG,
GOAL_CHECK_FLAG,
resolve_run_arm,
@@ -2317,13 +2316,6 @@ class ForgeAgent:
organization_id=task.organization_id,
forced=settings.TASK_V3_DATE_SEGMENT_AIM,
)
- await resolve_run_arm(
- context,
- EXTRACTION_REPORTS_FLAG,
- distinct_id=task.workflow_run_id or task.task_id,
- organization_id=task.organization_id,
- forced=settings.TASK_V3_EXTRACTION_REPORTS,
- )
# The judge's finish-time screenshot, reused as the decision screenshot of an accepted completion.
goal_judge_shot: list[bytes] = []
page_free_validation = bool(
@@ -2376,7 +2368,6 @@ class ForgeAgent:
handoff_enabled=handoff_enabled,
previous_block=previous_block,
selected_block_labels=context.run_block_labels if context else None,
- extraction_reports=run_arm_enabled(EXTRACTION_REPORTS_FLAG, settings.TASK_V3_EXTRACTION_REPORTS),
)
# Surface the customer's completion/termination criteria (trusted task config, like the navigation
# goal). Withhold a complete_criterion flagged untrusted (LLM-derived from page content) so it can't be
diff --git a/skyvern/forge/taskv3/goal_composition.py b/skyvern/forge/taskv3/goal_composition.py
index 62d76244d..83b987729 100644
--- a/skyvern/forge/taskv3/goal_composition.py
+++ b/skyvern/forge/taskv3/goal_composition.py
@@ -245,7 +245,6 @@ def render_block_context(
handoff_enabled: bool = False,
previous_block: PreviousBlockHandoff | None = None,
selected_block_labels: list[str] | None = None,
- extraction_reports: bool = False,
) -> tuple[str, str]:
"""Return ``(framing, section)`` for a block task; both are ``""`` for a bare task.
@@ -297,8 +296,7 @@ def render_block_context(
elif task_block is not None and task.task_type == TaskType.action:
pieces.append("This is a single, focused action: perform it and finish.")
elif (
- extraction_reports
- and task_block is not None
+ task_block is not None
and task_block.block_type == BlockType.EXTRACTION
and not task_block.is_internal_evaluation
and not task.navigation_goal
diff --git a/skyvern/forge/taskv3/run_arms.py b/skyvern/forge/taskv3/run_arms.py
index 86ff96cea..29abc3e39 100644
--- a/skyvern/forge/taskv3/run_arms.py
+++ b/skyvern/forge/taskv3/run_arms.py
@@ -17,7 +17,6 @@ from skyvern.forge.sdk.experimentation.providers import NoOpExperimentationProvi
LOG = structlog.get_logger()
DATE_SEGMENT_AIM_FLAG = "TASK_V3_DATE_SEGMENT_AIM"
-EXTRACTION_REPORTS_FLAG = "TASK_V3_EXTRACTION_REPORTS"
GOAL_CHECK_FLAG = "TASK_V3_GOAL_CHECK"
GOAL_CHECK_ENFORCE_FLAG = "TASK_V3_GOAL_CHECK_ENFORCE"
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index ea6d013f5..e35c49122 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -80,7 +80,6 @@ from skyvern.forge.taskv3.loop import (
)
from skyvern.forge.taskv3.run_arms import (
DATE_SEGMENT_AIM_FLAG,
- EXTRACTION_REPORTS_FLAG,
run_arm_enabled,
)
from skyvern.forge.taskv3.tools import PageProvider, _record_frame_work
@@ -493,18 +492,10 @@ async def test_execute_task_v3_passes_the_workflow_system_prompt_to_a_page_free_
@pytest.mark.asyncio
[email protected](("variant", "framed"), [("treatment", True), ("control", False)])
-async def test_execute_task_v3_tells_an_extraction_block_to_report_only_in_the_treatment_arm(
- monkeypatch: pytest.MonkeyPatch, variant: str, framed: bool
-) -> None:
- # The arm must be RESOLVED before the goal is composed: a deleted resolve call, or a render keyed on the
- # wrong flag, leaves every run on control with the composition tests still green (SKY-16398).
- monkeypatch.setattr(settings, "TASK_V3_EXTRACTION_REPORTS", False)
- provider = AsyncMock(return_value=variant)
- monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", provider)
-
+async def test_execute_task_v3_tells_an_extraction_block_it_only_reports(monkeypatch: pytest.MonkeyPatch) -> None:
+ # The framing must reach the goal the loop is given, not only render_block_context (SKY-16398).
outcome = LoopOutcome(status="completed", reason="done", billable_actions=[], extracted_output={"status": None})
- _step, task, loop_mock, _post = await _run_execute_task_v3(
+ _step, _task, loop_mock, _post = await _run_execute_task_v3(
monkeypatch,
outcome,
task_block=_make_block(ExtractionBlock, data_extraction_goal="the license status"),
@@ -513,8 +504,7 @@ async def test_execute_task_v3_tells_an_extraction_block_to_report_only_in_the_t
data_extraction_goal="the license status",
)
- assert ("This block only reads the page" in loop_mock.await_args.kwargs["goal"]) is framed
- assert loop_mock.context.run_arms[EXTRACTION_REPORTS_FLAG] == (task.workflow_run_id, variant)
+ assert "This block only reads the page" in loop_mock.await_args.kwargs["goal"]
@pytest.mark.asyncio
diff --git a/tests/unit/test_taskv3_goal_composition.py b/tests/unit/test_taskv3_goal_composition.py
index 308cb7dab..2fde48860 100644
--- a/tests/unit/test_taskv3_goal_composition.py
+++ b/tests/unit/test_taskv3_goal_composition.py
@@ -265,7 +265,7 @@ def test_an_extraction_only_block_is_told_to_report_absent_data_as_completed_nul
# v1 runs a block with no navigation goal as ONE extract action that returns nulls for whatever the page does
# not show and completes; workflows branch on those nulls. v3 was told neither what such a block is for nor
# what finishing it means, so it failed the block whenever an earlier block had not reached the data
- # (SKY-16398). Present on exactly v1's predicate, only in the treatment arm.
+ # (SKY-16398). Present on exactly v1's predicate.
now = datetime.now(UTC)
org = make_organization(now)
extraction_only = make_task(now, org, navigation_goal=None, data_extraction_goal="the license status")
@@ -273,10 +273,9 @@ def test_an_extraction_only_block_is_told_to_report_absent_data_as_completed_nul
label="blk", output_parameter=output_param("blk"), data_extraction_goal="the license status"
)
- treated, _ = render_block_context(extraction_only, block, None, extraction_reports=True)
- control, _ = render_block_context(extraction_only, block, None, extraction_reports=False)
+ framing, _ = render_block_context(extraction_only, block, None)
- added = [p for p in treated.split("\n\n") if p not in control.split("\n\n")]
+ added = [p for p in framing.split("\n\n") if p.startswith("This block only reads the page")]
assert len(added) == 1, added
assert "null" in added[0]
assert "status=completed" in added[0]
@@ -286,8 +285,6 @@ def test_an_extraction_only_block_is_told_to_report_absent_data_as_completed_nul
assert "read from the page" in added[0]
assert "never invent" in added[0]
assert "current date" in added[0]
- # The control render is what shipped before the arm existed.
- assert control == render_block_context(extraction_only, block, None)[0]
out_of_predicate = [
make_task(now, org, navigation_goal="Search for the record", data_extraction_goal="the license status"),
@@ -295,15 +292,11 @@ def test_an_extraction_only_block_is_told_to_report_absent_data_as_completed_nul
make_task(now, org, navigation_goal=None, data_extraction_goal="x", task_type=TaskType.validation),
]
for task in out_of_predicate:
- assert render_block_context(task, block, None, extraction_reports=True) == render_block_context(
- task, block, None
- )
+ assert "This block only reads the page" not in render_block_context(task, block, None)[0]
# A task block carrying only an extraction goal keeps its fill tools, so it is not told it only reads.
- assert render_block_context(extraction_only, _make_block("blk"), None, extraction_reports=True) == (
- render_block_context(extraction_only, _make_block("blk"), None)
- )
+ assert "This block only reads the page" not in render_block_context(extraction_only, _make_block("blk"), None)[0]
# A bare task has no workflow to route its nulls, so it gets no block framing at all.
- assert render_block_context(extraction_only, None, None, extraction_reports=True) == ("", "")
+ assert render_block_context(extraction_only, None, None) == ("", "")
def test_a_page_value_cannot_close_its_own_span() -> None:
diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.py
index 837a54f03..31979fa69 100644
--- a/tests/unit/test_workflow_block_engine.py
+++ b/tests/unit/test_workflow_block_engine.py
@@ -869,8 +869,8 @@ async def test_branch_eval_synthetic_block_honors_v3_override(scoped_context: Sk
@pytest.mark.asyncio
async def test_branch_eval_synthetic_block_gets_no_extraction_report_framing(scoped_context: SkyvernContext) -> None:
- """The branch evaluator's block follows the run's arm and has no navigation goal, so without its marker the
- TASK_V3_EXTRACTION_REPORTS framing ("absent fields are null, finish completed") would reach it. Its own prompt
+ """The branch evaluator's block has no navigation goal, so without its marker the extraction-report
+ framing ("absent fields are null, finish completed") would reach it. Its own prompt
judges conditions from their text, and a null result parses as False: a silently wrong branch (SKY-16398)."""
scoped_context.workflow_block_engine_resolved_run_id = "wr_branch_eval_framing"
scoped_context.workflow_block_engine_override = RunEngine.skyvern_v3
@@ -902,7 +902,7 @@ async def test_branch_eval_synthetic_block_gets_no_extraction_report_framing(sco
(block,) = captured
now = datetime.now(UTC)
task = make_task(now, make_organization(now), navigation_goal=None, data_extraction_goal=block.data_extraction_goal)
- assert render_block_context(task, block, None, extraction_reports=True) == render_block_context(task, block, None)
+ assert "This block only reads the page" not in render_block_context(task, block, None)[0]
@pytest.mark.asyncio