Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix: speed up workflow save cache invalidation (#6881)
Suchintan412c6320dac6
diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.pyindex f19aa1109..0ed7db692 100644--- a/skyvern/forge/sdk/db/agent_db.py+++ b/skyvern/forge/sdk/db/agent_db.py@@ -873,6 +873,9 @@ class AgentDB(BaseAlchemyDB):async def update_script_block(self, *args: Any, **kwargs: Any) -> Any:return await self.scripts.update_script_block(*args, **kwargs)+ async def clear_script_block_run_signatures(self, *args: Any, **kwargs: Any) -> Any:+ return await self.scripts.clear_script_block_run_signatures(*args, **kwargs)+async def get_script_files(self, *args: Any, **kwargs: Any) -> Any:return await self.scripts.get_script_files(*args, **kwargs)diff --git a/skyvern/forge/sdk/db/repositories/scripts.py b/skyvern/forge/sdk/db/repositories/scripts.pyindex 80ba32da0..2af8dafe9 100644--- a/skyvern/forge/sdk/db/repositories/scripts.py+++ b/skyvern/forge/sdk/db/repositories/scripts.py@@ -1,5 +1,6 @@from __future__ import annotations+from collections.abc import Sequencefrom dataclasses import dataclassfrom datetime import datetime, timedeltafrom enum import StrEnum@@ -555,6 +556,34 @@ class ScriptsRepository(BaseRepository):else:raise NotFoundError("Script block not found")+ @db_operation("clear_script_block_run_signatures")+ async def clear_script_block_run_signatures(+ self,+ *,+ organization_id: str,+ script_block_ids: Sequence[str],+ ) -> int:+ chunks = _dedup_into_chunks(list(script_block_ids))+ if not chunks:+ return 0++ total_updated = 0+ async with self.Session() as session:+ for chunk in chunks:+ stmt = (+ update(ScriptBlockModel)+ .where(ScriptBlockModel.organization_id == organization_id)+ .where(ScriptBlockModel.script_block_id.in_(chunk))+ .where(ScriptBlockModel.run_signature.isnot(None))+ .values(run_signature=None, modified_at=naive_utc_now())+ )+ result = await session.execute(stmt)+ total_updated += result.rowcount or 0++ await session.commit()++ return total_updated+@db_operation("get_script_files")async def get_script_files(self, script_revision_id: str, organization_id: str) -> list[ScriptFile]:async with self.Session() as session:diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 2793b8568..1f737522b 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -621,6 +621,7 @@ class WorkflowService:cached_groups: list[CachedScriptBlocks] = []published_groups: list[CachedScriptBlocks] = []target_labels = set(block_labels_to_disable)+ seen_group_keys: set[tuple[ScriptStatus | str, str, tuple[str, ...]]] = set()scripts_by_id = await app.DATABASE.scripts.get_latest_scripts_by_ids(organization_id=organization_id,@@ -643,6 +644,15 @@ class WorkflowService:if not blocks_to_clear:continue+ group_key = (+ candidate.status,+ script.script_revision_id,+ tuple(block.script_block_id for block in blocks_to_clear),+ )+ if group_key in seen_group_keys:+ continue+ seen_group_keys.add(group_key)+group = CachedScriptBlocks(workflow_script=candidate, script=script, blocks_to_clear=blocks_to_clear)if candidate.status == ScriptStatus.published:published_groups.append(group)@@ -661,30 +671,33 @@ class WorkflowService:groups: Sequence[CachedScriptBlocks],) -> None:"""Remove cached run signatures for the supplied block groups to force regeneration."""- for group in groups:- for block in group.blocks_to_clear:- await app.DATABASE.scripts.update_script_block(- script_block_id=block.script_block_id,- organization_id=organization_id,- clear_run_signature=True,- )+ blocks_by_id = {block.script_block_id: block for group in groups for block in group.blocks_to_clear}+ if not blocks_by_id:+ return- LOG.info(- "Cleared cached script blocks after workflow block change",- workflow_id=workflow.workflow_id,- workflow_permanent_id=previous_workflow.workflow_permanent_id,- organization_id=organization_id,- previous_version=previous_workflow.version,- new_version=workflow.version,- invalidate_reason=plan.reason,- invalidate_label=plan.label,- invalidate_index_prev=plan.previous_index,- invalidate_index_new=plan.new_index,- script_id=group.script.script_id,- script_revision_id=group.script.script_revision_id,- cleared_block_labels=[block.script_block_label for block in group.blocks_to_clear],- cleared_block_count=len(group.blocks_to_clear),- )+ cleared_count = await app.DATABASE.scripts.clear_script_block_run_signatures(+ organization_id=organization_id,+ script_block_ids=list(blocks_by_id),+ )++ LOG.info(+ "Cleared cached script blocks after workflow block change",+ workflow_id=workflow.workflow_id,+ workflow_permanent_id=previous_workflow.workflow_permanent_id,+ organization_id=organization_id,+ previous_version=previous_workflow.version,+ new_version=workflow.version,+ invalidate_reason=plan.reason,+ invalidate_label=plan.label,+ invalidate_index_prev=plan.previous_index,+ invalidate_index_new=plan.new_index,+ cached_group_count=len(groups),+ script_count=len({group.script.script_id for group in groups}),+ script_revision_count=len({group.script.script_revision_id for group in groups}),+ cleared_block_labels=list(dict.fromkeys(block.script_block_label for block in blocks_by_id.values())),+ cleared_block_count=cleared_count,+ deduped_block_count=len(blocks_by_id),+ )@staticmethoddef _collect_extracted_information(value: Any) -> list[Any]:diff --git a/tests/unit/test_partition_cached_blocks_batching.py b/tests/unit/test_partition_cached_blocks_batching.pyindex 7b6904f1d..23c940ae6 100644--- a/tests/unit/test_partition_cached_blocks_batching.py+++ b/tests/unit/test_partition_cached_blocks_batching.py@@ -14,6 +14,7 @@ Saving a large workflow timed out because cache invalidation walked every cachedfrom __future__ import annotationsfrom datetime import datetime+from types import SimpleNamespaceimport pytest@@ -35,6 +36,8 @@ class FakeScriptsDB:self._scripts_by_id = scripts_by_idself._blocks_by_revision = blocks_by_revisionself.call_log: list[str] = []+ self.cleared_script_block_ids: list[str] = []+ self.update_script_block_calls: list[str] = []async def get_script(self, script_id: str, organization_id: str, version: int | None = None) -> Script | None:self.call_log.append("get_script")@@ -61,6 +64,26 @@ class FakeScriptsDB:if self._blocks_by_revision.get(rid)}+ async def update_script_block(+ self,+ script_block_id: str,+ organization_id: str,+ clear_run_signature: bool = False,+ ) -> ScriptBlock | None:+ self.call_log.append("update_script_block")+ self.update_script_block_calls.append(script_block_id)+ return None++ async def clear_script_block_run_signatures(+ self,+ *,+ organization_id: str,+ script_block_ids: list[str],+ ) -> int:+ self.call_log.append("clear_script_block_run_signatures")+ self.cleared_script_block_ids.extend(script_block_ids)+ return len(script_block_ids)+def _now() -> datetime:return datetime(2026, 6, 18, 0, 0, 0)@@ -181,6 +204,83 @@ async def test_partition_cached_blocks_uses_constant_query_count(monkeypatch: pyassert len(fake.call_log) <= 2, f"expected constant query budget, got {fake.call_log}"+async def test_partition_cached_blocks_dedupes_duplicate_candidates(monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.forge.sdk.workflow.service import WorkflowService++ _, fake = _build_fixture()+ candidates = [+ _candidate("s1", ScriptStatus.published),+ _candidate("s1", ScriptStatus.published),+ _candidate("s3", ScriptStatus.pending),+ _candidate("s3", ScriptStatus.pending),+ ]+ monkeypatch.setattr(app.DATABASE, "scripts", fake)++ svc = WorkflowService()+ cached_groups, published_groups = await svc._partition_cached_blocks(+ organization_id=ORG_ID,+ candidates=candidates,+ block_labels_to_disable=["block_a", "block_b"],+ )++ assert [group.workflow_script.script_id for group in published_groups] == ["s1"]+ assert [group.workflow_script.script_id for group in cached_groups] == ["s3"]+ assert len(fake.call_log) <= 2, f"expected constant query budget, got {fake.call_log}"+++async def test_clear_cached_block_groups_bulk_clears_deduped_blocks(monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.forge.sdk.workflow.service import CachedScriptBlocks, CacheInvalidationPlan, WorkflowService++ _, fake = _build_fixture()+ monkeypatch.setattr(app.DATABASE, "scripts", fake)++ script = fake._scripts_by_id["s1"]+ block = fake._blocks_by_revision["r1"][0]+ groups = [+ CachedScriptBlocks(+ workflow_script=_candidate("s1", ScriptStatus.published),+ script=script,+ blocks_to_clear=[block, block],+ ),+ CachedScriptBlocks(+ workflow_script=_candidate("s1", ScriptStatus.published),+ script=script,+ blocks_to_clear=[block],+ ),+ ]++ svc = WorkflowService()+ await svc._clear_cached_block_groups(+ organization_id=ORG_ID,+ workflow=SimpleNamespace(+ workflow_id="wf_new",+ workflow_permanent_id=WPID,+ organization_id=ORG_ID,+ version=2,+ ),+ previous_workflow=SimpleNamespace(+ workflow_id="wf_previous",+ workflow_permanent_id=WPID,+ organization_id=ORG_ID,+ version=1,+ ),+ plan=CacheInvalidationPlan(+ reason="updated_block",+ label="block_a",+ previous_index=0,+ new_index=0,+ block_labels_to_disable=["block_a"],+ ),+ groups=groups,+ )++ assert fake.call_log == ["clear_script_block_run_signatures"]+ assert fake.cleared_script_block_ids == [block.script_block_id]+ assert fake.update_script_block_calls == []++def test_dedup_into_chunks_preserves_order_and_dedups() -> None:from skyvern.forge.sdk.db.repositories.scripts import _dedup_into_chunks