Skip to content

Automate browser based workflows with AI

Log and alert when a saved workflow version adds a skyvern-1.0 engine pin (#9019)

pedrohsdbe8f1028d4738
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index 3f5e8c381..680116cce 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -3223,6 +3223,18 @@ class WorkflowRunsRepository(BaseRepository):
for run, title in workflow_runs
]
+ @db_operation("count_recent_workflow_runs")
+ async def count_recent_workflow_runs(self, workflow_permanent_id: str, organization_id: str, days: int) -> int:
+ async with self.Session() as session:
+ count_query = (
+ select(func.count())
+ .select_from(WorkflowRunModel)
+ .filter(WorkflowRunModel.workflow_permanent_id == workflow_permanent_id)
+ .filter(WorkflowRunModel.organization_id == organization_id)
+ .filter(WorkflowRunModel.created_at >= naive_utc_now() - timedelta(days=days))
+ )
+ return (await session.execute(count_query)).scalar_one()
+
@db_operation("get_workflow_runs_count")
async def get_workflow_runs_count(
self,
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 6c021d206..6434c7738 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -10976,12 +10976,13 @@ class WorkflowService:
workflow_id: str | None = None,
encrypt_secrets: bool = True,
) -> Workflow:
+ pinned_v1_blocks = self._pinned_v1_blocks_to_check(workflow_definition)
try:
self._validate_code_block_templates(workflow_definition)
self._validate_block_engines(workflow_definition)
if encrypt_secrets:
await encrypt_workflow_definition_secrets(workflow_definition, organization_id)
- return await app.DATABASE.workflows.create_workflow(
+ created_workflow = await app.DATABASE.workflows.create_workflow(
title=title,
workflow_definition=workflow_definition.model_dump(mode="json"),
organization_id=organization_id,
@@ -11025,6 +11026,19 @@ class WorkflowService:
if "uc_org_permanent_id_version" in str(e) and workflow_permanent_id:
raise WorkflowVersionConflict(workflow_permanent_id) from e
raise
+ if pinned_v1_blocks:
+ await asyncio.create_task(self._log_if_v1_pin_added(None, created_workflow, pinned_v1_blocks))
+ return created_workflow
+
+ @staticmethod
+ def _pinned_v1_blocks_to_check(workflow_definition: WorkflowDefinition | None) -> int:
+ pinned_v1_blocks = (
+ pinned_v1_block_count(get_all_blocks(workflow_definition.blocks)) if workflow_definition is not None else 0
+ )
+ # Off on SQLite: an in-memory database pools one connection, so the check's reads would share the save's.
+ if pinned_v1_blocks and app.DATABASE.engine.dialect.name == "sqlite":
+ return 0
+ return pinned_v1_blocks
@staticmethod
async def _insert_next_workflow_version(
@@ -11630,6 +11644,10 @@ class WorkflowService:
preserve_redacted_credential_references_from: Workflow | None = None,
preserved_credential_references: set[tuple[str, str, str]] | None = None,
) -> Workflow:
+ pinned_v1_blocks = self._pinned_v1_blocks_to_check(workflow_definition)
+ overwritten_workflow = (
+ await self._read_overwritten_workflow(workflow_id, organization_id) if pinned_v1_blocks else None
+ )
if workflow_definition is not None:
if validate_code_block_templates:
self._validate_code_block_templates(workflow_definition)
@@ -11719,6 +11737,12 @@ class WorkflowService:
refuse_if_pinned_by_run_group=True,
)
+ if pinned_v1_blocks:
+ # A task gets its own connection; a failed read on the save's bound transaction would abort the save.
+ await asyncio.create_task(
+ self._log_if_v1_pin_added(overwritten_workflow, updated_workflow, pinned_v1_blocks)
+ )
+
if notify_workflow_saved:
self._schedule_workflow_saved_hook_best_effort(
organization_id=updated_workflow.organization_id,
@@ -11733,6 +11757,61 @@ class WorkflowService:
return updated_workflow
+ @staticmethod
+ async def _read_overwritten_workflow(workflow_id: str, organization_id: str | None) -> Workflow | None:
+ """None for a row the save's own transaction has not committed yet, which a new version's row is."""
+ try:
+ return await asyncio.create_task(
+ app.DATABASE.workflows.get_workflow(workflow_id=workflow_id, organization_id=organization_id)
+ )
+ except Exception:
+ LOG.warning("Failed to read the workflow row a save overwrites", workflow_id=workflow_id, exc_info=True)
+ return None
+
+ @staticmethod
+ async def _log_if_v1_pin_added(
+ overwritten_workflow: Workflow | None, saved_workflow: Workflow, pinned_v1_blocks: int
+ ) -> None:
+ """One new skyvern-1.0 pin takes every run of the workflow off the v3 A/B, so a save that adds one is logged."""
+ if saved_workflow.created_by == COPILOT_TEST_WORKFLOW_CREATOR:
+ return
+ try:
+ baseline = overwritten_workflow
+ # A new version's row is inserted empty before its definition is written; the version it follows is the
+ # one its runs routed on until now.
+ if baseline is None or not baseline.workflow_definition.blocks:
+ baseline = await app.DATABASE.workflows.get_workflow_by_permanent_id(
+ workflow_permanent_id=saved_workflow.workflow_permanent_id,
+ organization_id=saved_workflow.organization_id,
+ ignore_version=saved_workflow.version,
+ )
+ pinned_before = (
+ pinned_v1_block_count(get_all_blocks(baseline.workflow_definition.blocks)) if baseline else 0
+ )
+ if pinned_v1_blocks <= pinned_before:
+ return
+ LOG.info(
+ "Workflow version adds a skyvern-1.0 engine pin",
+ organization_id=saved_workflow.organization_id,
+ workflow_permanent_id=saved_workflow.workflow_permanent_id,
+ version=saved_workflow.version,
+ previous_version=baseline.version if baseline else None,
+ pinned_v1_blocks_before=pinned_before,
+ pinned_v1_blocks_after=pinned_v1_blocks,
+ build_source=saved_workflow.build_source,
+ recent_runs_7d=await app.DATABASE.workflow_runs.count_recent_workflow_runs(
+ workflow_permanent_id=saved_workflow.workflow_permanent_id,
+ organization_id=saved_workflow.organization_id,
+ days=7,
+ ),
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to check a saved workflow version for an added skyvern-1.0 engine pin",
+ workflow_permanent_id=saved_workflow.workflow_permanent_id,
+ exc_info=True,
+ )
+
async def maybe_delete_cached_code(
self,
workflow: Workflow,
diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.py
index 799a4bb3c..df31f13a4 100644
--- a/tests/unit/test_workflow_block_engine.py
+++ b/tests/unit/test_workflow_block_engine.py
@@ -5,9 +5,12 @@ workflow_run_blocks and the dispatched engine come from the same resolution.
from __future__ import annotations
+import asyncio
+import json
import time
from collections.abc import Callable, Iterator
from datetime import UTC, datetime, timedelta, timezone
+from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
@@ -19,7 +22,9 @@ from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.db.agent_db import AgentDB
from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType
+from skyvern.forge.sdk.db.models import Base
from skyvern.forge.sdk.experimentation.billing_tier import BILLING_TIER_PROPERTY, BillingTier
from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvider, NoOpExperimentationProvider
from skyvern.forge.sdk.experimentation.workflow_block_engine import (
@@ -57,11 +62,12 @@ from skyvern.forge.sdk.workflow.models.block import (
WhileLoopBlock,
_evaluate_prompt_branch_conditions_batch,
get_all_blocks,
+ pinned_v1_block_count,
run_is_eligible_for_v3_ab,
takes_default_engine,
v3_ab_ineligibility_reason,
)
-from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition
+from skyvern.forge.sdk.workflow.models.workflow import COPILOT_TEST_WORKFLOW_CREATOR, Workflow, WorkflowDefinition
from skyvern.forge.sdk.workflow.service import WorkflowService, workflow_definitions_differ
from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definition
from skyvern.forge.taskv3.goal_composition import render_block_context
@@ -70,6 +76,7 @@ from skyvern.schemas.workflows import (
BlockResult,
BlockType,
NavigationBlockYAML,
+ WorkflowCreateYAMLRequest,
WorkflowDefinitionYAML,
WorkflowStatus,
)
@@ -1902,6 +1909,249 @@ def test_a_pinned_skyvern_v1_survives_the_stored_definition_and_counts_as_a_chan
assert workflow_definitions_differ(unmarked, pinned)
+_PIN_ADDED_EVENT = "Workflow version adds a skyvern-1.0 engine pin"
+_WAREHOUSE_PIN_PATTERN = '"engine":"skyvern-1.0","engine_pinned":true'
+
+
+def _nav(label: str, engine: str | None = None, pinned: bool = False) -> dict[str, Any]:
+ block: dict[str, Any] = {
+ "block_type": "navigation",
+ "label": label,
+ "url": "https://e.com",
+ "navigation_goal": "Go",
+ }
+ if engine is not None:
+ block["engine"] = engine
+ if pinned:
+ block["engine_pinned"] = True
+ return block
+
+
+def _v1_pin(label: str) -> dict[str, Any]:
+ return _nav(label, "skyvern-1.0", pinned=True)
+
+
+def _for_loop(*inner: dict[str, Any]) -> dict[str, Any]:
+ return {"block_type": "for_loop", "label": "loop", "loop_variable_reference": "{{ items }}", "loop_blocks": inner}
+
+
+def _definition(blocks: list[dict[str, Any]]) -> WorkflowDefinition:
+ return convert_workflow_definition(
+ WorkflowDefinitionYAML.model_validate({"parameters": [], "blocks": blocks}), "w_test"
+ )
+
+
+def _workflow_row(blocks: list[dict[str, Any]], version: int, created_by: str | None = None) -> Workflow:
+ now = datetime.now(UTC)
+ return Workflow(
+ workflow_id=f"w_v{version}",
+ organization_id="o_test",
+ workflow_permanent_id="wpid_test",
+ title="test",
+ version=version,
+ is_saved_task=False,
+ workflow_definition=_definition(blocks),
+ created_at=now,
+ modified_at=now,
+ created_by=created_by,
+ )
+
+
+async def _save_definition(
+ monkeypatch: pytest.MonkeyPatch,
+ new_blocks: list[dict[str, Any]],
+ *,
+ overwritten: Workflow | None,
+ get_previous_version: AsyncMock,
+ created_by: str | None = None,
+ get_overwritten: AsyncMock | None = None,
+ dialect: str | None = "postgresql",
+) -> tuple[Workflow, Workflow, MagicMock]:
+ saved = _workflow_row(new_blocks, version=3, created_by=created_by)
+ engine = SimpleNamespace() if dialect is None else SimpleNamespace(dialect=SimpleNamespace(name=dialect))
+ monkeypatch.setattr(app.DATABASE, "engine", engine)
+ monkeypatch.setattr(app.DATABASE.workflows, "get_workflow", get_overwritten or AsyncMock(return_value=overwritten))
+ monkeypatch.setattr(app.DATABASE.workflows, "get_workflow_by_permanent_id", get_previous_version)
+ monkeypatch.setattr(
+ app.DATABASE.workflows, "update_workflow_and_reconcile_definition_params", AsyncMock(return_value=saved)
+ )
+ monkeypatch.setattr(app.DATABASE.organizations, "get_organization", AsyncMock(return_value=None))
+ monkeypatch.setattr(app.DATABASE.workflow_runs, "count_recent_workflow_runs", AsyncMock(return_value=327))
+ monkeypatch.setattr("skyvern.forge.sdk.workflow.service.encrypt_workflow_definition_secrets", AsyncMock())
+ with patch("skyvern.forge.sdk.workflow.service.LOG") as mock_log:
+ returned = await WorkflowService().update_workflow_definition(
+ workflow_id=saved.workflow_id,
+ organization_id="o_test",
+ workflow_definition=saved.workflow_definition,
+ notify_workflow_saved=False,
+ )
+ return returned, saved, mock_log
+
+
+def _pin_added_logs(mock_log: MagicMock) -> list[dict[str, Any]]:
+ return [call.kwargs for call in mock_log.info.call_args_list if call.args and call.args[0] == _PIN_ADDED_EVENT]
+
+
+ ("previous_blocks", "overwritten_blocks", "new_blocks", "created_by", "expected"),
+ [
+ pytest.param([_nav("a", "skyvern-1.0")], [], [_v1_pin("a")], None, (0, 1, 2), id="new_version_adds_pin"),
+ pytest.param([_v1_pin("a")], [], [_v1_pin("a")], None, None, id="new_version_keeps_pin"),
+ pytest.param([_v1_pin("a"), _v1_pin("b")], [], [_v1_pin("a")], None, None, id="new_version_unpins"),
+ pytest.param(
+ [_v1_pin("a"), _nav("b")], [], [_nav("b"), _v1_pin("a")], None, None, id="new_version_reorders_pin"
+ ),
+ pytest.param(
+ [_v1_pin("a")], [], [_v1_pin("a"), _for_loop(_v1_pin("inner"))], None, (1, 2, 2), id="nested_second_pin"
+ ),
+ pytest.param(None, [], [_v1_pin("a")], None, (0, 1, None), id="first_version_with_pin"),
+ pytest.param([_nav("a")], [_v1_pin("a")], [_v1_pin("a")], None, None, id="in_place_edit_keeps_pin"),
+ pytest.param([_nav("a")], [], [_v1_pin("a")], COPILOT_TEST_WORKFLOW_CREATOR, None, id="copilot_test_row"),
+ ],
+)
+async def test_a_saved_version_logs_only_when_it_adds_a_skyvern_v1_pin(
+ monkeypatch: pytest.MonkeyPatch,
+ previous_blocks: list[dict[str, Any]] | None,
+ overwritten_blocks: list[dict[str, Any]],
+ new_blocks: list[dict[str, Any]],
+ created_by: str | None,
+ expected: tuple[int, int, int | None] | None,
+) -> None:
+ previous = None if previous_blocks is None else _workflow_row(previous_blocks, version=2)
+ returned, saved, mock_log = await _save_definition(
+ monkeypatch,
+ new_blocks,
+ overwritten=_workflow_row(overwritten_blocks, version=3),
+ get_previous_version=AsyncMock(return_value=previous),
+ created_by=created_by,
+ )
+
+ assert returned is saved
+ logs = _pin_added_logs(mock_log)
+ if expected is None:
+ assert logs == []
+ return
+ before, after, previous_version = expected
+ [log] = logs
+ assert log["workflow_permanent_id"] == "wpid_test"
+ assert log["version"] == 3
+ assert log["previous_version"] == previous_version
+ assert log["pinned_v1_blocks_before"] == before
+ assert log["pinned_v1_blocks_after"] == after
+ assert log["recent_runs_7d"] == 327
+
+
[email protected]("idempotent", [pytest.param(True, id="idempotent_create"), pytest.param(False, id="create")])
+async def test_a_first_version_created_with_a_v1_pin_logs(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, idempotent: bool
+) -> None:
+ database = AgentDB(f"sqlite+aiosqlite:///{tmp_path / 'pin.db'}")
+ async with database.engine.begin() as connection:
+ await connection.run_sync(Base.metadata.create_all)
+ organization = await database.organizations.create_organization(organization_name="Test", organization_id="o_test")
+ monkeypatch.setattr(app, "DATABASE", database)
+ monkeypatch.setattr(app, "WORKFLOW_SERVICE", WorkflowService())
+ # A file database has its own connection per task, unlike the in-memory one the dialect gate exists for.
+ monkeypatch.setattr(
+ WorkflowService, "_pinned_v1_blocks_to_check", staticmethod(lambda d: pinned_v1_block_count(d.blocks))
+ )
+ request = WorkflowCreateYAMLRequest.model_validate(
+ {"title": "test", "workflow_definition": {"parameters": [], "blocks": [_v1_pin("a")]}}
+ )
+ try:
+ with patch("skyvern.forge.sdk.workflow.service.LOG") as mock_log:
+ saved = await app.WORKFLOW_SERVICE.create_workflow_from_request(
+ organization=organization,
+ request=request,
+ new_workflow_permanent_id="wpid_test" if idempotent else None,
+ )
+ finally:
+ await database.engine.dispose()
+
+ [log] = _pin_added_logs(mock_log)
+ assert (log["workflow_permanent_id"], log["version"]) == (saved.workflow_permanent_id, 1)
+ assert (log["pinned_v1_blocks_before"], log["pinned_v1_blocks_after"]) == (0, 1)
+
+
+ ("blocks", "dialect"),
+ [
+ # Without a v1 pin the save must not touch the engine; some test doubles of the database have none.
+ pytest.param([_nav("a", "skyvern-1.0"), _nav("b", "skyvern-2.0", pinned=True)], None, id="no_v1_pin"),
+ # An in-memory SQLite database pools one connection, so a read on another task would roll the save back.
+ pytest.param([_v1_pin("a")], "sqlite", id="v1_pin_on_sqlite"),
+ ],
+)
+async def test_a_save_reads_no_prior_row_without_a_v1_pin_or_on_sqlite(
+ monkeypatch: pytest.MonkeyPatch, blocks: list[dict[str, Any]], dialect: str | None
+) -> None:
+ reads: list[dict[str, Any]] = []
+
+ async def prior_row_read(**kwargs: Any) -> None:
+ reads.append(kwargs)
+
+ returned, saved, mock_log = await _save_definition(
+ monkeypatch,
+ blocks,
+ overwritten=None,
+ get_previous_version=AsyncMock(side_effect=prior_row_read),
+ get_overwritten=AsyncMock(side_effect=prior_row_read),
+ dialect=dialect,
+ )
+
+ assert returned is saved
+ assert reads == []
+ assert _pin_added_logs(mock_log) == []
+
+
+async def test_a_failing_pin_read_never_runs_on_the_save_task_or_fails_the_save(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ save_task = asyncio.current_task()
+ read_tasks: list[asyncio.Task[Any] | None] = []
+
+ async def failing_read(**_: Any) -> Workflow:
+ read_tasks.append(asyncio.current_task())
+ raise RuntimeError("db down")
+
+ returned, saved, mock_log = await _save_definition(
+ monkeypatch,
+ [_v1_pin("a")],
+ overwritten=None,
+ get_previous_version=AsyncMock(side_effect=failing_read),
+ get_overwritten=AsyncMock(side_effect=failing_read),
+ )
+
+ assert returned is saved
+ assert len(read_tasks) == 2 and save_task not in read_tasks
+ assert _pin_added_logs(mock_log) == []
+ assert any(call.kwargs.get("workflow_permanent_id") == "wpid_test" for call in mock_log.warning.call_args_list)
+
+
+ ("blocks", "expected_pins"),
+ [
+ pytest.param([_v1_pin("a")], 1, id="pinned_v1"),
+ pytest.param([_nav("a", "skyvern-1.0")], 0, id="unpinned_v1"),
+ pytest.param([_nav("a", "skyvern-3.0", pinned=True)], 0, id="pinned_v3"),
+ pytest.param([_for_loop(_v1_pin("inner"))], 1, id="pinned_v1_in_for_loop"),
+ pytest.param([_v1_pin("a"), _nav("b"), _v1_pin("c")], 2, id="two_pins"),
+ ],
+)
+def test_the_warehouse_pin_regex_counts_what_the_service_counts(
+ blocks: list[dict[str, Any]], expected_pins: int
+) -> None:
+ definition = _definition(blocks)
+ stored = json.dumps(definition.model_dump(mode="json"), separators=(",", ":"), sort_keys=True)
+
+ assert pinned_v1_block_count(get_all_blocks(definition.blocks)) == expected_pins
+ assert stored.count(_WAREHOUSE_PIN_PATTERN) == expected_pins
+
+
@pytest.mark.parametrize(
("flags", "expected_reason", "displaced"),
[