Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Re-read the run's engines after the actions before generating a script (SKY-18016 follow-up) (#8861)
pedrohsdb7de63c02e652
diff --git a/skyvern/core/script_generations/transform_workflow_run.py b/skyvern/core/script_generations/transform_workflow_run.pyindex 413933898..67084071b 100644--- a/skyvern/core/script_generations/transform_workflow_run.py+++ b/skyvern/core/script_generations/transform_workflow_run.py@@ -263,8 +263,8 @@ async def transform_workflow_run_to_code_gen_input(workflow_run_id: str, organizworkflow_run_id=workflow_run_id, organization_id=organization_id)workflow_run_blocks.sort(key=lambda x: x.created_at)- # Checked on the same snapshot the actions are read through: a block row records its engine before any of- # its actions exist, so an in-flight mint cannot read a v3 task's actions without also reading its v3 row.+ # Both this check and the re-read after the actions are needed: actions are fetched for the task ids read+ # here, and a retry can move a v3 row to a new task and relabel it v1 before the re-read.if any(block.engine == RunEngine.skyvern_v3 for block in workflow_run_blocks):raise TaskV3RunNotScriptableError(workflow_run_id)@@ -307,6 +307,13 @@ async def transform_workflow_run_to_code_gen_input(workflow_run_id: str, organizaction_count=len(all_actions),)+ # Read after the actions: a block row is labeled v3 before any v3 action exists (the cached-script fallback+ # stamps an existing row), so an in-flight mint that read a v3 action always sees the label here.+ if await app.DATABASE.observer.workflow_run_has_block_on_engine(+ workflow_run_id=workflow_run_id, engine=RunEngine.skyvern_v3, organization_id=organization_id+ ):+ raise TaskV3RunNotScriptableError(workflow_run_id)+workflow_block_dump = []actions_by_task: dict[str, list[dict[str, Any]]] = {}task_v2_child_blocks = {}diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 41d79f804..38317aac8 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -3768,9 +3768,14 @@ class ForgeAgent:# The block row was labeled v3 before dispatch, and script generation skips a run with a# v3 row, so a fallback left labeled v3 would lose a script the run was asked to generate.try:- await app.DATABASE.observer.set_workflow_run_block_engine_by_task_id(+ if not await app.DATABASE.observer.set_workflow_run_block_engine_by_task_id(task.task_id, RunEngine.skyvern_v1, organization_id=task.organization_id- )+ ):+ LOG.warning(+ "No block row matched the step-engine fallback relabel",+ task_id=task.task_id,+ workflow_run_id=task.workflow_run_id,+ )except Exception:LOG.warning("Could not record the step-engine fallback on the block row",diff --git a/skyvern/forge/sdk/db/repositories/observer.py b/skyvern/forge/sdk/db/repositories/observer.pyindex b4dcbaaf4..f8c976118 100644--- a/skyvern/forge/sdk/db/repositories/observer.py+++ b/skyvern/forge/sdk/db/repositories/observer.py@@ -719,15 +719,16 @@ class ObserverRepository(BaseRepository):task_id: str,engine: RunEngine,organization_id: str | None = None,- ) -> None:+ ) -> bool:async with self.Session() as session:- await session.execute(+ result = await session.execute(update(WorkflowRunBlockModel).where(WorkflowRunBlockModel.task_id == task_id).where(WorkflowRunBlockModel.organization_id == organization_id).values(engine=engine.value))await session.commit()+ return result.rowcount > 0@db_operation("workflow_run_has_block_on_engine")async def workflow_run_has_block_on_engine(diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 5b153271d..7b70b349e 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -2079,10 +2079,10 @@ class BaseTaskBlock(Block):"""The engine this block dispatches to, after the per-run A/B.Both the persisted workflow_run_blocks.engine and the execute_step dispatch read this; when the- dispatch falls back to the step engine, execute_step corrects the row to the engine that ran. A block pinned to a non-default- engine is honored as-authored, and a block the eligibility check never saw is left alone;- neither is ever rerouted. An unset engine routes like skyvern_v1, except in a run that honors the- chosen engine or on a block whose skyvern_v1 a person pinned.+ dispatch falls back to the step engine, execute_step corrects the row to the engine that ran. A+ block pinned to a non-default engine is honored as-authored, and a block the eligibility check+ never saw is left alone; neither is ever rerouted. An unset engine routes like skyvern_v1, except+ in a run that honors the chosen engine or on a block whose skyvern_v1 a person pinned."""declared = self.engine or RunEngine.skyvern_v1if (diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex 1bfc23249..625378441 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -14576,7 +14576,14 @@ def build_browser_tools(skin_probe = None# Every toggle answer carries `radio`. The browser engine's own enabled check decides (Playwright reads# aria-disabled only on roles that take it), so the label click below is never looser than a plain click.- if not (isinstance(skin_probe, dict) and "radio" in skin_probe) or await _engine_enabled(page, selector):+ if not (isinstance(skin_probe, dict) and "radio" in skin_probe):+ break+ if await _engine_enabled(page, selector):+ # A page that enables a control can move it in the same step, so the label point is measured after+ # the enabled read, never before it.+ pre_click_arg = await _probe_arg(page, selector)+ with contextlib.suppress(Exception):+ skin_probe = await _probe_evaluate(page, _SKINNED_CHECKBOX_PROBE_JS, selector, pre_click_arg)breakif time.monotonic() >= enable_deadline:return ToolResult.error(diff --git a/tests/unit/test_batch_action_queries.py b/tests/unit/test_batch_action_queries.pyindex c01df6562..8f1631d9e 100644--- a/tests/unit/test_batch_action_queries.py+++ b/tests/unit/test_batch_action_queries.py@@ -110,6 +110,7 @@ async def test_batch_actions_preserve_per_task_ordering() -> None:mock_get_wfr.return_value = mock_workflow_run_respmock_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=mock_workflow)mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(return_value=[run_block_a, run_block_b])+ mock_app.DATABASE.observer.workflow_run_has_block_on_engine = AsyncMock(return_value=False)mock_app.DATABASE.tasks.get_tasks_by_ids = AsyncMock(return_value=[mock_task_a, mock_task_b])mock_app.DATABASE.tasks.get_tasks_actions = AsyncMock(return_value=all_actions_descending)@@ -188,6 +189,7 @@ async def test_batch_actions_without_reverse_would_be_wrong() -> None:mock_get_wfr.return_value = mock_workflow_run_respmock_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=mock_workflow)mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(return_value=[run_block])+ mock_app.DATABASE.observer.workflow_run_has_block_on_engine = AsyncMock(return_value=False)mock_app.DATABASE.tasks.get_tasks_by_ids = AsyncMock(return_value=[mock_task])mock_app.DATABASE.tasks.get_tasks_actions = AsyncMock(return_value=actions_descending)@@ -251,6 +253,7 @@ async def test_batch_actions_preserve_none_element_id() -> None:mock_get_wfr.return_value = mock_workflow_run_respmock_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=mock_workflow)mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(return_value=[run_block])+ mock_app.DATABASE.observer.workflow_run_has_block_on_engine = AsyncMock(return_value=False)mock_app.DATABASE.tasks.get_tasks_by_ids = AsyncMock(return_value=[mock_task])mock_app.DATABASE.tasks.get_tasks_actions = AsyncMock(return_value=[action_extract])diff --git a/tests/unit/test_forloop_script_generation.py b/tests/unit/test_forloop_script_generation.pyindex 111bccaff..69e520734 100644--- a/tests/unit/test_forloop_script_generation.py+++ b/tests/unit/test_forloop_script_generation.py@@ -337,6 +337,7 @@ async def test_transform_forloop_block_with_mocked_db() -> None:):mock_get_wfr.return_value = mock_workflow_run_respmock_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=mock_workflow)+ mock_app.DATABASE.observer.workflow_run_has_block_on_engine = AsyncMock(return_value=False)mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(return_value=[mock_forloop_run_block,diff --git a/tests/unit/test_pending_script_promotion.py b/tests/unit/test_pending_script_promotion.pyindex 26371ec96..b9b40fc26 100644--- a/tests/unit/test_pending_script_promotion.py+++ b/tests/unit/test_pending_script_promotion.py@@ -14,6 +14,7 @@ import astimport asynciofrom datetime import UTC, datetimefrom types import SimpleNamespace+from typing import Anyfrom unittest.mock import AsyncMock, MagicMockimport libcst as cst@@ -398,8 +399,9 @@ class BlockRowsByTask:async def set_workflow_run_block_engine_by_task_id(self, task_id: str, engine: RunEngine, organization_id: str | None = None- ) -> None:+ ) -> bool:self.engines[task_id] = engine+ return Trueasync def workflow_run_has_block_on_engine(self, *, workflow_run_id: str, engine: RunEngine, organization_id: str | None@@ -580,15 +582,28 @@ class TestNoScriptFromATaskV3Run:assert step_engine_mock.await_args.kwargs["engine"] == RunEngine.openai_cua@pytest.mark.parametrize(- ("second_block_engine", "mints"), [(RunEngine.skyvern_v1, True), (RunEngine.skyvern_v3, False)]+ ("engine_at_block_read", "engine_after_action_read", "engine_read_fails", "mints"),+ [+ (None, RunEngine.skyvern_v1, False, True),+ # The cached-script fallback stamps the row after the block read, as its v3 actions land.+ (None, RunEngine.skyvern_v3, False, False),+ # A retry moves the v3 row to a new task and the kill switch relabels it v1 before the re-read.+ (RunEngine.skyvern_v3, RunEngine.skyvern_v1, False, False),+ (None, RunEngine.skyvern_v1, True, False),+ ],+ ids=["stays_v1", "stamped_v3_after_block_read", "relabeled_v1_after_block_read", "engine_read_fails_closed"],)@pytest.mark.asyncio- async def test_an_in_flight_pending_mint_whose_snapshot_holds_v3_actions_writes_nothing(- self, monkeypatch: pytest.MonkeyPatch, second_block_engine: RunEngine, mints: bool+ async def test_an_in_flight_pending_mint_never_writes_a_v3_tasks_actions(+ self,+ monkeypatch: pytest.MonkeyPatch,+ engine_at_block_read: RunEngine | None,+ engine_after_action_read: RunEngine,+ engine_read_fails: bool,+ mints: bool,) -> None:- # The mint passed its v3 check before the second block's row existed; by the snapshot that block's- # actions are stored, so only the snapshot itself can refuse them.- def run_block(label: str, task_id: str, engine: RunEngine, created_at: int) -> SimpleNamespace:+ # The mint passed its v3 check before the second block ran; the row changes between its two reads.+ def run_block(label: str, task_id: str, engine: RunEngine | None, created_at: int) -> SimpleNamespace:return SimpleNamespace(workflow_run_block_id=f"wrb_{label}",parent_workflow_run_block_id=None,@@ -620,18 +635,28 @@ class TestNoScriptFromATaskV3Run:actions = [ClickAction(action_id=f"a_{t}", task_id=t, element_id=f"el_{t}") for t in ("tsk_pinned", "tsk_cached")]+ rows = [+ run_block("cached", "tsk_cached", RunEngine.skyvern_v1, 1),+ run_block("pinned", "tsk_pinned", engine_at_block_read, 2),+ ]++ async def read_blocks(**_: Any) -> list[SimpleNamespace]:+ return [SimpleNamespace(**vars(row)) for row in rows]++ async def read_actions(**_: Any) -> list[ClickAction]:+ rows[1].engine = engine_after_action_read+ return list(actions)++ async def has_block_on_engine(*, workflow_run_id: str, engine: RunEngine, organization_id: str) -> bool:+ if engine_read_fails:+ raise RuntimeError("database unavailable")+ return any(row.engine == engine for row in rows)+database = SimpleNamespace(observer=SimpleNamespace(- get_workflow_run_blocks=AsyncMock(- return_value=[- run_block("cached", "tsk_cached", RunEngine.skyvern_v1, 1),- run_block("pinned", "tsk_pinned", second_block_engine, 2),- ]- )- ),- tasks=SimpleNamespace(- get_tasks_by_ids=AsyncMock(return_value=tasks), get_tasks_actions=AsyncMock(return_value=actions)+ get_workflow_run_blocks=read_blocks, workflow_run_has_block_on_engine=has_block_on_engine),+ tasks=SimpleNamespace(get_tasks_by_ids=AsyncMock(return_value=tasks), get_tasks_actions=read_actions),)monkeypatch.setattr(app, "DATABASE", database, raising=False)monkeypatch.setattr(diff --git a/tests/unit/test_while_loop_script_generation.py b/tests/unit/test_while_loop_script_generation.pyindex 3bfba28f4..ecc153782 100644--- a/tests/unit/test_while_loop_script_generation.py+++ b/tests/unit/test_while_loop_script_generation.py@@ -182,6 +182,7 @@ async def test_transform_whileloop_block_with_mocked_db() -> None:mock_get.return_value = mock_workflow_run_respmock_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=mock_workflow)mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(return_value=[mock_while_run, mock_child])+ mock_app.DATABASE.observer.workflow_run_has_block_on_engine = AsyncMock(return_value=False)mock_app.DATABASE.tasks.get_tasks_by_ids = AsyncMock(return_value=[mock_task])mock_app.DATABASE.tasks.get_tasks_actions = AsyncMock(return_value=[mock_action])