Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Log and alert when a saved workflow version adds a skyvern-1.0 engine pin (#9019)
pedrohsdbe8f1028d4738
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.pyindex 3f5e8c381..680116cce 100644--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py@@ -3223,6 +3223,18 @@ class WorkflowRunsRepository(BaseRepository):for run, title in workflow_runs]+ @db_operation("count_recent_workflow_runs")+ async def count_recent_workflow_runs(self, workflow_permanent_id: str, organization_id: str, days: int) -> int:+ async with self.Session() as session:+ count_query = (+ select(func.count())+ .select_from(WorkflowRunModel)+ .filter(WorkflowRunModel.workflow_permanent_id == workflow_permanent_id)+ .filter(WorkflowRunModel.organization_id == organization_id)+ .filter(WorkflowRunModel.created_at >= naive_utc_now() - timedelta(days=days))+ )+ return (await session.execute(count_query)).scalar_one()+@db_operation("get_workflow_runs_count")async def get_workflow_runs_count(self,diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 6c021d206..6434c7738 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -10976,12 +10976,13 @@ class WorkflowService:workflow_id: str | None = None,encrypt_secrets: bool = True,) -> Workflow:+ pinned_v1_blocks = self._pinned_v1_blocks_to_check(workflow_definition)try:self._validate_code_block_templates(workflow_definition)self._validate_block_engines(workflow_definition)if encrypt_secrets:await encrypt_workflow_definition_secrets(workflow_definition, organization_id)- return await app.DATABASE.workflows.create_workflow(+ created_workflow = await app.DATABASE.workflows.create_workflow(title=title,workflow_definition=workflow_definition.model_dump(mode="json"),organization_id=organization_id,@@ -11025,6 +11026,19 @@ class WorkflowService:if "uc_org_permanent_id_version" in str(e) and workflow_permanent_id:raise WorkflowVersionConflict(workflow_permanent_id) from eraise+ if pinned_v1_blocks:+ await asyncio.create_task(self._log_if_v1_pin_added(None, created_workflow, pinned_v1_blocks))+ return created_workflow++ @staticmethod+ def _pinned_v1_blocks_to_check(workflow_definition: WorkflowDefinition | None) -> int:+ pinned_v1_blocks = (+ pinned_v1_block_count(get_all_blocks(workflow_definition.blocks)) if workflow_definition is not None else 0+ )+ # Off on SQLite: an in-memory database pools one connection, so the check's reads would share the save's.+ if pinned_v1_blocks and app.DATABASE.engine.dialect.name == "sqlite":+ return 0+ return pinned_v1_blocks@staticmethodasync def _insert_next_workflow_version(@@ -11630,6 +11644,10 @@ class WorkflowService:preserve_redacted_credential_references_from: Workflow | None = None,preserved_credential_references: set[tuple[str, str, str]] | None = None,) -> Workflow:+ pinned_v1_blocks = self._pinned_v1_blocks_to_check(workflow_definition)+ overwritten_workflow = (+ await self._read_overwritten_workflow(workflow_id, organization_id) if pinned_v1_blocks else None+ )if workflow_definition is not None:if validate_code_block_templates:self._validate_code_block_templates(workflow_definition)@@ -11719,6 +11737,12 @@ class WorkflowService:refuse_if_pinned_by_run_group=True,)+ if pinned_v1_blocks:+ # A task gets its own connection; a failed read on the save's bound transaction would abort the save.+ await asyncio.create_task(+ self._log_if_v1_pin_added(overwritten_workflow, updated_workflow, pinned_v1_blocks)+ )+if notify_workflow_saved:self._schedule_workflow_saved_hook_best_effort(organization_id=updated_workflow.organization_id,@@ -11733,6 +11757,61 @@ class WorkflowService:return updated_workflow+ @staticmethod+ async def _read_overwritten_workflow(workflow_id: str, organization_id: str | None) -> Workflow | None:+ """None for a row the save's own transaction has not committed yet, which a new version's row is."""+ try:+ return await asyncio.create_task(+ app.DATABASE.workflows.get_workflow(workflow_id=workflow_id, organization_id=organization_id)+ )+ except Exception:+ LOG.warning("Failed to read the workflow row a save overwrites", workflow_id=workflow_id, exc_info=True)+ return None++ @staticmethod+ async def _log_if_v1_pin_added(+ overwritten_workflow: Workflow | None, saved_workflow: Workflow, pinned_v1_blocks: int+ ) -> None:+ """One new skyvern-1.0 pin takes every run of the workflow off the v3 A/B, so a save that adds one is logged."""+ if saved_workflow.created_by == COPILOT_TEST_WORKFLOW_CREATOR:+ return+ try:+ baseline = overwritten_workflow+ # A new version's row is inserted empty before its definition is written; the version it follows is the+ # one its runs routed on until now.+ if baseline is None or not baseline.workflow_definition.blocks:+ baseline = await app.DATABASE.workflows.get_workflow_by_permanent_id(+ workflow_permanent_id=saved_workflow.workflow_permanent_id,+ organization_id=saved_workflow.organization_id,+ ignore_version=saved_workflow.version,+ )+ pinned_before = (+ pinned_v1_block_count(get_all_blocks(baseline.workflow_definition.blocks)) if baseline else 0+ )+ if pinned_v1_blocks <= pinned_before:+ return+ LOG.info(+ "Workflow version adds a skyvern-1.0 engine pin",+ organization_id=saved_workflow.organization_id,+ workflow_permanent_id=saved_workflow.workflow_permanent_id,+ version=saved_workflow.version,+ previous_version=baseline.version if baseline else None,+ pinned_v1_blocks_before=pinned_before,+ pinned_v1_blocks_after=pinned_v1_blocks,+ build_source=saved_workflow.build_source,+ recent_runs_7d=await app.DATABASE.workflow_runs.count_recent_workflow_runs(+ workflow_permanent_id=saved_workflow.workflow_permanent_id,+ organization_id=saved_workflow.organization_id,+ days=7,+ ),+ )+ except Exception:+ LOG.warning(+ "Failed to check a saved workflow version for an added skyvern-1.0 engine pin",+ workflow_permanent_id=saved_workflow.workflow_permanent_id,+ exc_info=True,+ )+async def maybe_delete_cached_code(self,workflow: Workflow,diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.pyindex 799a4bb3c..df31f13a4 100644--- a/tests/unit/test_workflow_block_engine.py+++ b/tests/unit/test_workflow_block_engine.py@@ -5,9 +5,12 @@ workflow_run_blocks and the dispatched engine come from the same resolution.from __future__ import annotations+import asyncio+import jsonimport timefrom collections.abc import Callable, Iteratorfrom datetime import UTC, datetime, timedelta, timezone+from pathlib import Pathfrom types import SimpleNamespacefrom typing import Anyfrom unittest.mock import AsyncMock, MagicMock, patch@@ -19,7 +22,9 @@ from skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.db.agent_db import AgentDBfrom skyvern.forge.sdk.db.enums import WorkflowRunTriggerType+from skyvern.forge.sdk.db.models import Basefrom skyvern.forge.sdk.experimentation.billing_tier import BILLING_TIER_PROPERTY, BillingTierfrom skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvider, NoOpExperimentationProviderfrom skyvern.forge.sdk.experimentation.workflow_block_engine import (@@ -57,11 +62,12 @@ from skyvern.forge.sdk.workflow.models.block import (WhileLoopBlock,_evaluate_prompt_branch_conditions_batch,get_all_blocks,+ pinned_v1_block_count,run_is_eligible_for_v3_ab,takes_default_engine,v3_ab_ineligibility_reason,)-from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition+from skyvern.forge.sdk.workflow.models.workflow import COPILOT_TEST_WORKFLOW_CREATOR, Workflow, WorkflowDefinitionfrom skyvern.forge.sdk.workflow.service import WorkflowService, workflow_definitions_differfrom skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definitionfrom skyvern.forge.taskv3.goal_composition import render_block_context@@ -70,6 +76,7 @@ from skyvern.schemas.workflows import (BlockResult,BlockType,NavigationBlockYAML,+ WorkflowCreateYAMLRequest,WorkflowDefinitionYAML,WorkflowStatus,)@@ -1902,6 +1909,249 @@ def test_a_pinned_skyvern_v1_survives_the_stored_definition_and_counts_as_a_chanassert workflow_definitions_differ(unmarked, pinned)+_PIN_ADDED_EVENT = "Workflow version adds a skyvern-1.0 engine pin"+_WAREHOUSE_PIN_PATTERN = '"engine":"skyvern-1.0","engine_pinned":true'+++def _nav(label: str, engine: str | None = None, pinned: bool = False) -> dict[str, Any]:+ block: dict[str, Any] = {+ "block_type": "navigation",+ "label": label,+ "url": "https://e.com",+ "navigation_goal": "Go",+ }+ if engine is not None:+ block["engine"] = engine+ if pinned:+ block["engine_pinned"] = True+ return block+++def _v1_pin(label: str) -> dict[str, Any]:+ return _nav(label, "skyvern-1.0", pinned=True)+++def _for_loop(*inner: dict[str, Any]) -> dict[str, Any]:+ return {"block_type": "for_loop", "label": "loop", "loop_variable_reference": "{{ items }}", "loop_blocks": inner}+++def _definition(blocks: list[dict[str, Any]]) -> WorkflowDefinition:+ return convert_workflow_definition(+ WorkflowDefinitionYAML.model_validate({"parameters": [], "blocks": blocks}), "w_test"+ )+++def _workflow_row(blocks: list[dict[str, Any]], version: int, created_by: str | None = None) -> Workflow:+ now = datetime.now(UTC)+ return Workflow(+ workflow_id=f"w_v{version}",+ organization_id="o_test",+ workflow_permanent_id="wpid_test",+ title="test",+ version=version,+ is_saved_task=False,+ workflow_definition=_definition(blocks),+ created_at=now,+ modified_at=now,+ created_by=created_by,+ )+++async def _save_definition(+ monkeypatch: pytest.MonkeyPatch,+ new_blocks: list[dict[str, Any]],+ *,+ overwritten: Workflow | None,+ get_previous_version: AsyncMock,+ created_by: str | None = None,+ get_overwritten: AsyncMock | None = None,+ dialect: str | None = "postgresql",+) -> tuple[Workflow, Workflow, MagicMock]:+ saved = _workflow_row(new_blocks, version=3, created_by=created_by)+ engine = SimpleNamespace() if dialect is None else SimpleNamespace(dialect=SimpleNamespace(name=dialect))+ monkeypatch.setattr(app.DATABASE, "engine", engine)+ monkeypatch.setattr(app.DATABASE.workflows, "get_workflow", get_overwritten or AsyncMock(return_value=overwritten))+ monkeypatch.setattr(app.DATABASE.workflows, "get_workflow_by_permanent_id", get_previous_version)+ monkeypatch.setattr(+ app.DATABASE.workflows, "update_workflow_and_reconcile_definition_params", AsyncMock(return_value=saved)+ )+ monkeypatch.setattr(app.DATABASE.organizations, "get_organization", AsyncMock(return_value=None))+ monkeypatch.setattr(app.DATABASE.workflow_runs, "count_recent_workflow_runs", AsyncMock(return_value=327))+ monkeypatch.setattr("skyvern.forge.sdk.workflow.service.encrypt_workflow_definition_secrets", AsyncMock())+ with patch("skyvern.forge.sdk.workflow.service.LOG") as mock_log:+ returned = await WorkflowService().update_workflow_definition(+ workflow_id=saved.workflow_id,+ organization_id="o_test",+ workflow_definition=saved.workflow_definition,+ notify_workflow_saved=False,+ )+ return returned, saved, mock_log+++def _pin_added_logs(mock_log: MagicMock) -> list[dict[str, Any]]:+ return [call.kwargs for call in mock_log.info.call_args_list if call.args and call.args[0] == _PIN_ADDED_EVENT]+++ ("previous_blocks", "overwritten_blocks", "new_blocks", "created_by", "expected"),+ [+ pytest.param([_nav("a", "skyvern-1.0")], [], [_v1_pin("a")], None, (0, 1, 2), id="new_version_adds_pin"),+ pytest.param([_v1_pin("a")], [], [_v1_pin("a")], None, None, id="new_version_keeps_pin"),+ pytest.param([_v1_pin("a"), _v1_pin("b")], [], [_v1_pin("a")], None, None, id="new_version_unpins"),+ pytest.param(+ [_v1_pin("a"), _nav("b")], [], [_nav("b"), _v1_pin("a")], None, None, id="new_version_reorders_pin"+ ),+ pytest.param(+ [_v1_pin("a")], [], [_v1_pin("a"), _for_loop(_v1_pin("inner"))], None, (1, 2, 2), id="nested_second_pin"+ ),+ pytest.param(None, [], [_v1_pin("a")], None, (0, 1, None), id="first_version_with_pin"),+ pytest.param([_nav("a")], [_v1_pin("a")], [_v1_pin("a")], None, None, id="in_place_edit_keeps_pin"),+ pytest.param([_nav("a")], [], [_v1_pin("a")], COPILOT_TEST_WORKFLOW_CREATOR, None, id="copilot_test_row"),+ ],+)+async def test_a_saved_version_logs_only_when_it_adds_a_skyvern_v1_pin(+ monkeypatch: pytest.MonkeyPatch,+ previous_blocks: list[dict[str, Any]] | None,+ overwritten_blocks: list[dict[str, Any]],+ new_blocks: list[dict[str, Any]],+ created_by: str | None,+ expected: tuple[int, int, int | None] | None,+) -> None:+ previous = None if previous_blocks is None else _workflow_row(previous_blocks, version=2)+ returned, saved, mock_log = await _save_definition(+ monkeypatch,+ new_blocks,+ overwritten=_workflow_row(overwritten_blocks, version=3),+ get_previous_version=AsyncMock(return_value=previous),+ created_by=created_by,+ )++ assert returned is saved+ logs = _pin_added_logs(mock_log)+ if expected is None:+ assert logs == []+ return+ before, after, previous_version = expected+ [log] = logs+ assert log["workflow_permanent_id"] == "wpid_test"+ assert log["version"] == 3+ assert log["previous_version"] == previous_version+ assert log["pinned_v1_blocks_before"] == before+ assert log["pinned_v1_blocks_after"] == after+ assert log["recent_runs_7d"] == 327++[email protected]("idempotent", [pytest.param(True, id="idempotent_create"), pytest.param(False, id="create")])+async def test_a_first_version_created_with_a_v1_pin_logs(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, idempotent: bool+) -> None:+ database = AgentDB(f"sqlite+aiosqlite:///{tmp_path / 'pin.db'}")+ async with database.engine.begin() as connection:+ await connection.run_sync(Base.metadata.create_all)+ organization = await database.organizations.create_organization(organization_name="Test", organization_id="o_test")+ monkeypatch.setattr(app, "DATABASE", database)+ monkeypatch.setattr(app, "WORKFLOW_SERVICE", WorkflowService())+ # A file database has its own connection per task, unlike the in-memory one the dialect gate exists for.+ monkeypatch.setattr(+ WorkflowService, "_pinned_v1_blocks_to_check", staticmethod(lambda d: pinned_v1_block_count(d.blocks))+ )+ request = WorkflowCreateYAMLRequest.model_validate(+ {"title": "test", "workflow_definition": {"parameters": [], "blocks": [_v1_pin("a")]}}+ )+ try:+ with patch("skyvern.forge.sdk.workflow.service.LOG") as mock_log:+ saved = await app.WORKFLOW_SERVICE.create_workflow_from_request(+ organization=organization,+ request=request,+ new_workflow_permanent_id="wpid_test" if idempotent else None,+ )+ finally:+ await database.engine.dispose()++ [log] = _pin_added_logs(mock_log)+ assert (log["workflow_permanent_id"], log["version"]) == (saved.workflow_permanent_id, 1)+ assert (log["pinned_v1_blocks_before"], log["pinned_v1_blocks_after"]) == (0, 1)+++ ("blocks", "dialect"),+ [+ # Without a v1 pin the save must not touch the engine; some test doubles of the database have none.+ pytest.param([_nav("a", "skyvern-1.0"), _nav("b", "skyvern-2.0", pinned=True)], None, id="no_v1_pin"),+ # An in-memory SQLite database pools one connection, so a read on another task would roll the save back.+ pytest.param([_v1_pin("a")], "sqlite", id="v1_pin_on_sqlite"),+ ],+)+async def test_a_save_reads_no_prior_row_without_a_v1_pin_or_on_sqlite(+ monkeypatch: pytest.MonkeyPatch, blocks: list[dict[str, Any]], dialect: str | None+) -> None:+ reads: list[dict[str, Any]] = []++ async def prior_row_read(**kwargs: Any) -> None:+ reads.append(kwargs)++ returned, saved, mock_log = await _save_definition(+ monkeypatch,+ blocks,+ overwritten=None,+ get_previous_version=AsyncMock(side_effect=prior_row_read),+ get_overwritten=AsyncMock(side_effect=prior_row_read),+ dialect=dialect,+ )++ assert returned is saved+ assert reads == []+ assert _pin_added_logs(mock_log) == []+++async def test_a_failing_pin_read_never_runs_on_the_save_task_or_fails_the_save(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ save_task = asyncio.current_task()+ read_tasks: list[asyncio.Task[Any] | None] = []++ async def failing_read(**_: Any) -> Workflow:+ read_tasks.append(asyncio.current_task())+ raise RuntimeError("db down")++ returned, saved, mock_log = await _save_definition(+ monkeypatch,+ [_v1_pin("a")],+ overwritten=None,+ get_previous_version=AsyncMock(side_effect=failing_read),+ get_overwritten=AsyncMock(side_effect=failing_read),+ )++ assert returned is saved+ assert len(read_tasks) == 2 and save_task not in read_tasks+ assert _pin_added_logs(mock_log) == []+ assert any(call.kwargs.get("workflow_permanent_id") == "wpid_test" for call in mock_log.warning.call_args_list)+++ ("blocks", "expected_pins"),+ [+ pytest.param([_v1_pin("a")], 1, id="pinned_v1"),+ pytest.param([_nav("a", "skyvern-1.0")], 0, id="unpinned_v1"),+ pytest.param([_nav("a", "skyvern-3.0", pinned=True)], 0, id="pinned_v3"),+ pytest.param([_for_loop(_v1_pin("inner"))], 1, id="pinned_v1_in_for_loop"),+ pytest.param([_v1_pin("a"), _nav("b"), _v1_pin("c")], 2, id="two_pins"),+ ],+)+def test_the_warehouse_pin_regex_counts_what_the_service_counts(+ blocks: list[dict[str, Any]], expected_pins: int+) -> None:+ definition = _definition(blocks)+ stored = json.dumps(definition.model_dump(mode="json"), separators=(",", ":"), sort_keys=True)++ assert pinned_v1_block_count(get_all_blocks(definition.blocks)) == expected_pins+ assert stored.count(_WAREHOUSE_PIN_PATTERN) == expected_pins++@pytest.mark.parametrize(("flags", "expected_reason", "displaced"),[