Skip to content

Automate browser based workflows with AI

Re-read the run's engines after the actions before generating a script (SKY-18016 follow-up) (#8861)

pedrohsdb7de63c02e652
diff --git a/skyvern/core/script_generations/transform_workflow_run.py b/skyvern/core/script_generations/transform_workflow_run.py
index 413933898..67084071b 100644
--- a/skyvern/core/script_generations/transform_workflow_run.py
+++ b/skyvern/core/script_generations/transform_workflow_run.py
@@ -263,8 +263,8 @@ async def transform_workflow_run_to_code_gen_input(workflow_run_id: str, organiz
workflow_run_id=workflow_run_id, organization_id=organization_id
)
workflow_run_blocks.sort(key=lambda x: x.created_at)
- # Checked on the same snapshot the actions are read through: a block row records its engine before any of
- # its actions exist, so an in-flight mint cannot read a v3 task's actions without also reading its v3 row.
+ # Both this check and the re-read after the actions are needed: actions are fetched for the task ids read
+ # here, and a retry can move a v3 row to a new task and relabel it v1 before the re-read.
if any(block.engine == RunEngine.skyvern_v3 for block in workflow_run_blocks):
raise TaskV3RunNotScriptableError(workflow_run_id)
@@ -307,6 +307,13 @@ async def transform_workflow_run_to_code_gen_input(workflow_run_id: str, organiz
action_count=len(all_actions),
)
+ # Read after the actions: a block row is labeled v3 before any v3 action exists (the cached-script fallback
+ # stamps an existing row), so an in-flight mint that read a v3 action always sees the label here.
+ if await app.DATABASE.observer.workflow_run_has_block_on_engine(
+ workflow_run_id=workflow_run_id, engine=RunEngine.skyvern_v3, organization_id=organization_id
+ ):
+ raise TaskV3RunNotScriptableError(workflow_run_id)
+
workflow_block_dump = []
actions_by_task: dict[str, list[dict[str, Any]]] = {}
task_v2_child_blocks = {}
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 41d79f804..38317aac8 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -3768,9 +3768,14 @@ class ForgeAgent:
# The block row was labeled v3 before dispatch, and script generation skips a run with a
# v3 row, so a fallback left labeled v3 would lose a script the run was asked to generate.
try:
- await app.DATABASE.observer.set_workflow_run_block_engine_by_task_id(
+ if not await app.DATABASE.observer.set_workflow_run_block_engine_by_task_id(
task.task_id, RunEngine.skyvern_v1, organization_id=task.organization_id
- )
+ ):
+ LOG.warning(
+ "No block row matched the step-engine fallback relabel",
+ task_id=task.task_id,
+ workflow_run_id=task.workflow_run_id,
+ )
except Exception:
LOG.warning(
"Could not record the step-engine fallback on the block row",
diff --git a/skyvern/forge/sdk/db/repositories/observer.py b/skyvern/forge/sdk/db/repositories/observer.py
index b4dcbaaf4..f8c976118 100644
--- a/skyvern/forge/sdk/db/repositories/observer.py
+++ b/skyvern/forge/sdk/db/repositories/observer.py
@@ -719,15 +719,16 @@ class ObserverRepository(BaseRepository):
task_id: str,
engine: RunEngine,
organization_id: str | None = None,
- ) -> None:
+ ) -> bool:
async with self.Session() as session:
- await session.execute(
+ result = await session.execute(
update(WorkflowRunBlockModel)
.where(WorkflowRunBlockModel.task_id == task_id)
.where(WorkflowRunBlockModel.organization_id == organization_id)
.values(engine=engine.value)
)
await session.commit()
+ return result.rowcount > 0
@db_operation("workflow_run_has_block_on_engine")
async def workflow_run_has_block_on_engine(
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 5b153271d..7b70b349e 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -2079,10 +2079,10 @@ class BaseTaskBlock(Block):
"""The engine this block dispatches to, after the per-run A/B.
Both the persisted workflow_run_blocks.engine and the execute_step dispatch read this; when the
- dispatch falls back to the step engine, execute_step corrects the row to the engine that ran. A block pinned to a non-default
- engine is honored as-authored, and a block the eligibility check never saw is left alone;
- neither is ever rerouted. An unset engine routes like skyvern_v1, except in a run that honors the
- chosen engine or on a block whose skyvern_v1 a person pinned.
+ dispatch falls back to the step engine, execute_step corrects the row to the engine that ran. A
+ block pinned to a non-default engine is honored as-authored, and a block the eligibility check
+ never saw is left alone; neither is ever rerouted. An unset engine routes like skyvern_v1, except
+ in a run that honors the chosen engine or on a block whose skyvern_v1 a person pinned.
"""
declared = self.engine or RunEngine.skyvern_v1
if (
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 1bfc23249..625378441 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -14576,7 +14576,14 @@ def build_browser_tools(
skin_probe = None
# Every toggle answer carries `radio`. The browser engine's own enabled check decides (Playwright reads
# aria-disabled only on roles that take it), so the label click below is never looser than a plain click.
- if not (isinstance(skin_probe, dict) and "radio" in skin_probe) or await _engine_enabled(page, selector):
+ if not (isinstance(skin_probe, dict) and "radio" in skin_probe):
+ break
+ if await _engine_enabled(page, selector):
+ # A page that enables a control can move it in the same step, so the label point is measured after
+ # the enabled read, never before it.
+ pre_click_arg = await _probe_arg(page, selector)
+ with contextlib.suppress(Exception):
+ skin_probe = await _probe_evaluate(page, _SKINNED_CHECKBOX_PROBE_JS, selector, pre_click_arg)
break
if time.monotonic() >= enable_deadline:
return ToolResult.error(
diff --git a/tests/unit/test_batch_action_queries.py b/tests/unit/test_batch_action_queries.py
index c01df6562..8f1631d9e 100644
--- a/tests/unit/test_batch_action_queries.py
+++ b/tests/unit/test_batch_action_queries.py
@@ -110,6 +110,7 @@ async def test_batch_actions_preserve_per_task_ordering() -> None:
mock_get_wfr.return_value = mock_workflow_run_resp
mock_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=mock_workflow)
mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(return_value=[run_block_a, run_block_b])
+ mock_app.DATABASE.observer.workflow_run_has_block_on_engine = AsyncMock(return_value=False)
mock_app.DATABASE.tasks.get_tasks_by_ids = AsyncMock(return_value=[mock_task_a, mock_task_b])
mock_app.DATABASE.tasks.get_tasks_actions = AsyncMock(return_value=all_actions_descending)
@@ -188,6 +189,7 @@ async def test_batch_actions_without_reverse_would_be_wrong() -> None:
mock_get_wfr.return_value = mock_workflow_run_resp
mock_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=mock_workflow)
mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(return_value=[run_block])
+ mock_app.DATABASE.observer.workflow_run_has_block_on_engine = AsyncMock(return_value=False)
mock_app.DATABASE.tasks.get_tasks_by_ids = AsyncMock(return_value=[mock_task])
mock_app.DATABASE.tasks.get_tasks_actions = AsyncMock(return_value=actions_descending)
@@ -251,6 +253,7 @@ async def test_batch_actions_preserve_none_element_id() -> None:
mock_get_wfr.return_value = mock_workflow_run_resp
mock_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=mock_workflow)
mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(return_value=[run_block])
+ mock_app.DATABASE.observer.workflow_run_has_block_on_engine = AsyncMock(return_value=False)
mock_app.DATABASE.tasks.get_tasks_by_ids = AsyncMock(return_value=[mock_task])
mock_app.DATABASE.tasks.get_tasks_actions = AsyncMock(return_value=[action_extract])
diff --git a/tests/unit/test_forloop_script_generation.py b/tests/unit/test_forloop_script_generation.py
index 111bccaff..69e520734 100644
--- a/tests/unit/test_forloop_script_generation.py
+++ b/tests/unit/test_forloop_script_generation.py
@@ -337,6 +337,7 @@ async def test_transform_forloop_block_with_mocked_db() -> None:
):
mock_get_wfr.return_value = mock_workflow_run_resp
mock_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=mock_workflow)
+ mock_app.DATABASE.observer.workflow_run_has_block_on_engine = AsyncMock(return_value=False)
mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(
return_value=[
mock_forloop_run_block,
diff --git a/tests/unit/test_pending_script_promotion.py b/tests/unit/test_pending_script_promotion.py
index 26371ec96..b9b40fc26 100644
--- a/tests/unit/test_pending_script_promotion.py
+++ b/tests/unit/test_pending_script_promotion.py
@@ -14,6 +14,7 @@ import ast
import asyncio
from datetime import UTC, datetime
from types import SimpleNamespace
+from typing import Any
from unittest.mock import AsyncMock, MagicMock
import libcst as cst
@@ -398,8 +399,9 @@ class BlockRowsByTask:
async def set_workflow_run_block_engine_by_task_id(
self, task_id: str, engine: RunEngine, organization_id: str | None = None
- ) -> None:
+ ) -> bool:
self.engines[task_id] = engine
+ return True
async def workflow_run_has_block_on_engine(
self, *, workflow_run_id: str, engine: RunEngine, organization_id: str | None
@@ -580,15 +582,28 @@ class TestNoScriptFromATaskV3Run:
assert step_engine_mock.await_args.kwargs["engine"] == RunEngine.openai_cua
@pytest.mark.parametrize(
- ("second_block_engine", "mints"), [(RunEngine.skyvern_v1, True), (RunEngine.skyvern_v3, False)]
+ ("engine_at_block_read", "engine_after_action_read", "engine_read_fails", "mints"),
+ [
+ (None, RunEngine.skyvern_v1, False, True),
+ # The cached-script fallback stamps the row after the block read, as its v3 actions land.
+ (None, RunEngine.skyvern_v3, False, False),
+ # A retry moves the v3 row to a new task and the kill switch relabels it v1 before the re-read.
+ (RunEngine.skyvern_v3, RunEngine.skyvern_v1, False, False),
+ (None, RunEngine.skyvern_v1, True, False),
+ ],
+ ids=["stays_v1", "stamped_v3_after_block_read", "relabeled_v1_after_block_read", "engine_read_fails_closed"],
)
@pytest.mark.asyncio
- async def test_an_in_flight_pending_mint_whose_snapshot_holds_v3_actions_writes_nothing(
- self, monkeypatch: pytest.MonkeyPatch, second_block_engine: RunEngine, mints: bool
+ async def test_an_in_flight_pending_mint_never_writes_a_v3_tasks_actions(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ engine_at_block_read: RunEngine | None,
+ engine_after_action_read: RunEngine,
+ engine_read_fails: bool,
+ mints: bool,
) -> None:
- # The mint passed its v3 check before the second block's row existed; by the snapshot that block's
- # actions are stored, so only the snapshot itself can refuse them.
- def run_block(label: str, task_id: str, engine: RunEngine, created_at: int) -> SimpleNamespace:
+ # The mint passed its v3 check before the second block ran; the row changes between its two reads.
+ def run_block(label: str, task_id: str, engine: RunEngine | None, created_at: int) -> SimpleNamespace:
return SimpleNamespace(
workflow_run_block_id=f"wrb_{label}",
parent_workflow_run_block_id=None,
@@ -620,18 +635,28 @@ class TestNoScriptFromATaskV3Run:
actions = [
ClickAction(action_id=f"a_{t}", task_id=t, element_id=f"el_{t}") for t in ("tsk_pinned", "tsk_cached")
]
+ rows = [
+ run_block("cached", "tsk_cached", RunEngine.skyvern_v1, 1),
+ run_block("pinned", "tsk_pinned", engine_at_block_read, 2),
+ ]
+
+ async def read_blocks(**_: Any) -> list[SimpleNamespace]:
+ return [SimpleNamespace(**vars(row)) for row in rows]
+
+ async def read_actions(**_: Any) -> list[ClickAction]:
+ rows[1].engine = engine_after_action_read
+ return list(actions)
+
+ async def has_block_on_engine(*, workflow_run_id: str, engine: RunEngine, organization_id: str) -> bool:
+ if engine_read_fails:
+ raise RuntimeError("database unavailable")
+ return any(row.engine == engine for row in rows)
+
database = SimpleNamespace(
observer=SimpleNamespace(
- get_workflow_run_blocks=AsyncMock(
- return_value=[
- run_block("cached", "tsk_cached", RunEngine.skyvern_v1, 1),
- run_block("pinned", "tsk_pinned", second_block_engine, 2),
- ]
- )
- ),
- tasks=SimpleNamespace(
- get_tasks_by_ids=AsyncMock(return_value=tasks), get_tasks_actions=AsyncMock(return_value=actions)
+ get_workflow_run_blocks=read_blocks, workflow_run_has_block_on_engine=has_block_on_engine
),
+ tasks=SimpleNamespace(get_tasks_by_ids=AsyncMock(return_value=tasks), get_tasks_actions=read_actions),
)
monkeypatch.setattr(app, "DATABASE", database, raising=False)
monkeypatch.setattr(
diff --git a/tests/unit/test_while_loop_script_generation.py b/tests/unit/test_while_loop_script_generation.py
index 3bfba28f4..ecc153782 100644
--- a/tests/unit/test_while_loop_script_generation.py
+++ b/tests/unit/test_while_loop_script_generation.py
@@ -182,6 +182,7 @@ async def test_transform_whileloop_block_with_mocked_db() -> None:
mock_get.return_value = mock_workflow_run_resp
mock_app.WORKFLOW_SERVICE.get_workflow_by_permanent_id = AsyncMock(return_value=mock_workflow)
mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(return_value=[mock_while_run, mock_child])
+ mock_app.DATABASE.observer.workflow_run_has_block_on_engine = AsyncMock(return_value=False)
mock_app.DATABASE.tasks.get_tasks_by_ids = AsyncMock(return_value=[mock_task])
mock_app.DATABASE.tasks.get_tasks_actions = AsyncMock(return_value=[mock_action])