Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-10799): gate sequential runs on queue order, not a single dependency edge (#6611)
Celal Zamanoğlu4fd53c6c6d41
diff --git a/alembic/versions/2026_06_16_0803-cfc2318acdf9_add_sequential_key_lookup_index_on_workflow_runs.py b/alembic/versions/2026_06_16_0803-cfc2318acdf9_add_sequential_key_lookup_index_on_workflow_runs.pynew file mode 100644index 000000000..cf7b44631--- /dev/null+++ b/alembic/versions/2026_06_16_0803-cfc2318acdf9_add_sequential_key_lookup_index_on_workflow_runs.py@@ -0,0 +1,33 @@+"""add sequential_key lookup index on workflow_runs++Revision ID: cfc2318acdf9+Revises: 218048b68412+Create Date: 2026-06-16T08:03:01.419039+00:00++"""++from typing import Sequence, Union++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "cfc2318acdf9"+down_revision: Union[str, None] = "218048b68412"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ with op.get_context().autocommit_block():+ op.execute("SET statement_timeout = '24h';")+ op.execute("""+ CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_workflow_runs_sequential_key_lookup+ ON workflow_runs (workflow_permanent_id, sequential_key, queued_at)+ WHERE status IN ('queued', 'running', 'paused') AND browser_session_id IS NULL+ """)+ op.execute("RESET statement_timeout;")+++def downgrade() -> None:+ with op.get_context().autocommit_block():+ op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_workflow_runs_sequential_key_lookup")diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex be16e0963..68cb9fc61 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -574,6 +574,13 @@ class WorkflowRunModel(Base):"created_at",postgresql_where=text("status IN ('created', 'queued', 'running', 'paused')"),),+ Index(+ "ix_workflow_runs_sequential_key_lookup",+ "workflow_permanent_id",+ "sequential_key",+ "queued_at",+ postgresql_where=text("status IN ('queued', 'running', 'paused') AND browser_session_id IS NULL"),+ ),)workflow_run_id = Column(String, primary_key=True, default=generate_workflow_run_id)diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.pyindex fe2cc0d89..515f92452 100644--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py@@ -715,6 +715,71 @@ class WorkflowRunsRepository(BaseRepository):workflow_run = (await session.scalars(query)).first()return convert_to_workflow_run(workflow_run) if workflow_run else None+ @db_operation("get_blocking_sequential_workflow_run")+ async def get_blocking_sequential_workflow_run(self, workflow_run_id: str) -> WorkflowRun | None:+ # Sequential-execution gate: returns the earliest-queued run that shares this run's+ # sequential identity and is still in flight (queued/running/paused) and was queued+ # before it, or None when this run is clear to start. A direct scan over all+ # same-identity runs — not a walk of one depends_on chain — so it holds when the+ # dependency graph fans out (forest) or a queued predecessor is canceled.+ # Ordering uses queued_at, not created_at: it is stamped at enqueue (before Temporal+ # submission) so it reflects submit order. The no-double-start guarantee rests on the+ # strict total order over (queued_at, workflow_run_id) below, not on any lock.+ async with self.Session() as session:+ run = (await session.scalars(select(WorkflowRunModel).filter_by(workflow_run_id=workflow_run_id))).first()+ if run is None:+ return None++ # Lane resolution mirrors enqueue priority: browser_session_id > browser_address+ # > sequential_key > whole workflow. A debug-session run carries a browser_session_id+ # but is excluded from the session lane (as at enqueue), so it serializes on its key.+ query = select(WorkflowRunModel).filter_by(organization_id=run.organization_id)+ if run.browser_session_id and not run.debug_session_id:+ query = query.filter_by(browser_session_id=run.browser_session_id)+ elif run.browser_address:+ query = query.filter_by(browser_address=run.browser_address)+ elif run.sequential_key:+ query = query.filter_by(+ workflow_permanent_id=run.workflow_permanent_id,+ sequential_key=run.sequential_key,+ ).filter(WorkflowRunModel.browser_session_id.is_(None))+ else:+ query = query.filter_by(workflow_permanent_id=run.workflow_permanent_id).filter(+ WorkflowRunModel.browser_session_id.is_(None)+ )++ # Sequential runs are stamped queued_at before Temporal submission; the fallback+ # only guards hand-created rows (e.g. tests) from comparing against None.+ self_queued_at = run.queued_at if run.queued_at is not None else run.created_at++ query = query.filter(+ WorkflowRunModel.status.in_(+ [+ WorkflowRunStatus.queued,+ WorkflowRunStatus.running,+ WorkflowRunStatus.paused,+ ]+ )+ )+ # Safe because a sequential run is always stamped queued_at (passes through queued)+ # before it can reach running/paused, so this filter never hides a real blocker.+ query = query.filter(WorkflowRunModel.queued_at.isnot(None))+ query = query.filter(+ or_(+ WorkflowRunModel.queued_at < self_queued_at,+ and_(+ WorkflowRunModel.queued_at == self_queued_at,+ WorkflowRunModel.workflow_run_id < run.workflow_run_id,+ ),+ )+ )+ query = query.order_by(+ WorkflowRunModel.queued_at.asc(),+ WorkflowRunModel.workflow_run_id.asc(),+ )+ blocker = (await session.scalars(query)).first()+ return convert_to_workflow_run(blocker) if blocker else None+async def _get_last_workflow_run_by_filter(self,organization_id: str | None = None,diff --git a/tests/unit/forge/sdk/db/test_workflow_runs_repository.py b/tests/unit/forge/sdk/db/test_workflow_runs_repository.pyindex 5cd6d5c8a..20a6d5706 100644--- a/tests/unit/forge/sdk/db/test_workflow_runs_repository.py+++ b/tests/unit/forge/sdk/db/test_workflow_runs_repository.py@@ -43,6 +43,14 @@ class _SessionContext:return False+class _Result:+ def __init__(self, value: Any) -> None:+ self._value = value++ def first(self) -> Any:+ return self._value++class _EmptyExecuteResult:def mappings(self) -> _EmptyExecuteResult:return self@@ -464,3 +472,127 @@ async def test_get_workflow_runs_for_browser_session_filters_and_excludes() -> Nassert "ORDER BY workflow_runs.created_at DESC" in renderedassert "LIMIT 5" in renderedassert "OFFSET 5" in rendered+++async def test_get_blocking_sequential_workflow_run_scans_earlier_active_same_key() -> None:+ """SKY-10799: the sequential gate scans ALL earlier-queued same-key runs still in flight+ (queued/running/paused) — not a single depends_on edge — so it holds under a forest-shaped+ graph or a canceled predecessor. Earlier = (queued_at, id) strictly before self; queued_at+ is stamped under the submit lock, so it is the true queue order even when creation order+ diverges from submission order."""+ fake_run = MagicMock()+ fake_run.workflow_run_id = "wr_self"+ fake_run.organization_id = "o_test"+ fake_run.workflow_permanent_id = "wpid_test"+ fake_run.sequential_key = "cred_test-sequential-key"+ fake_run.browser_session_id = None+ fake_run.browser_address = None+ fake_run.created_at = datetime(2026, 6, 8, 18, 53, 48, tzinfo=timezone.utc)+ fake_run.queued_at = datetime(2026, 6, 8, 18, 53, 50, tzinfo=timezone.utc)++ calls: list[Any] = []++ async def _scalars(query: Any) -> Any:+ calls.append(query)+ # 1st query loads the run itself; 2nd is the gate scan we assert on.+ return _Result(fake_run) if len(calls) == 1 else _Result(None)++ session = MagicMock()+ session.scalars = AsyncMock(side_effect=_scalars)++ repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)++ blocker = await repo.get_blocking_sequential_workflow_run("wr_self")+ assert blocker is None++ gate_query = calls[1]+ where_clause = _where_clause_sql(gate_query)+ rendered = str(gate_query.compile(compile_kwargs={"literal_binds": True}))++ # scoped to the same wpid + key, ignoring browser-session runs+ assert "workflow_runs.workflow_permanent_id = 'wpid_test'" in where_clause+ assert "workflow_runs.sequential_key = 'cred_test-sequential-key'" in where_clause+ assert "workflow_runs.browser_session_id IS NULL" in where_clause+ assert "workflow_runs.organization_id = 'o_test'" in where_clause+ # only genuinely in-flight runs block; `created` and terminal statuses do not+ assert "'queued'" in where_clause and "'running'" in where_clause and "'paused'" in where_clause+ assert "'created'" not in where_clause and "'completed'" not in where_clause+ # earlier-queued by (queued_at, id), never created_at, and FIFO order so the+ # earliest blocker surfaces; unqueued rows can't block+ assert "workflow_runs.queued_at IS NOT NULL" in where_clause+ assert "workflow_runs.queued_at <" in where_clause+ assert "workflow_runs.created_at" not in where_clause+ assert "workflow_runs.workflow_run_id < 'wr_self'" in where_clause+ assert "ORDER BY workflow_runs.queued_at ASC" in rendered+++async def test_get_blocking_sequential_workflow_run_prefers_browser_session_lane() -> None:+ """The gate's lane resolution must mirror enqueue priority (browser_session_id >+ browser_address > sequential_key): a non-debug run carrying both a browser session and a+ sequential_key chains on the session lane at enqueue, so the gate must scan that+ same lane or it can miss its actual blocker."""+ fake_run = MagicMock()+ fake_run.workflow_run_id = "wr_self"+ fake_run.organization_id = "o_test"+ fake_run.workflow_permanent_id = "wpid_test"+ fake_run.sequential_key = "cred_test-sequential-key"+ fake_run.browser_session_id = "pbs_test"+ fake_run.debug_session_id = None+ fake_run.browser_address = None+ fake_run.created_at = datetime(2026, 6, 8, 18, 53, 48, tzinfo=timezone.utc)+ fake_run.queued_at = datetime(2026, 6, 8, 18, 53, 50, tzinfo=timezone.utc)++ calls: list[Any] = []++ async def _scalars(query: Any) -> Any:+ calls.append(query)+ return _Result(fake_run) if len(calls) == 1 else _Result(None)++ session = MagicMock()+ session.scalars = AsyncMock(side_effect=_scalars)++ repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)++ await repo.get_blocking_sequential_workflow_run("wr_self")++ where_clause = _where_clause_sql(calls[1])+ assert "workflow_runs.browser_session_id = 'pbs_test'" in where_clause+ assert "sequential_key" not in where_clause+++async def test_get_blocking_sequential_workflow_run_debug_session_uses_key_lane() -> None:+ """A debug-session run carries a browser_session_id but enqueue keeps it out of the+ browser-session lane (is_browser_session_workflow requires not debug_session_id). The gate+ must do the same, or the debug run scans only its own session and misses an earlier same-key+ run — the SKY-10799 regression."""+ fake_run = MagicMock()+ fake_run.workflow_run_id = "wr_self"+ fake_run.organization_id = "o_test"+ fake_run.workflow_permanent_id = "wpid_test"+ fake_run.sequential_key = "cred_test-sequential-key"+ fake_run.browser_session_id = "pbs_test"+ fake_run.debug_session_id = "dbg_test"+ fake_run.browser_address = None+ fake_run.created_at = datetime(2026, 6, 8, 18, 53, 48, tzinfo=timezone.utc)+ fake_run.queued_at = datetime(2026, 6, 8, 18, 53, 50, tzinfo=timezone.utc)++ calls: list[Any] = []++ async def _scalars(query: Any) -> Any:+ calls.append(query)+ return _Result(fake_run) if len(calls) == 1 else _Result(None)++ session = MagicMock()+ session.scalars = AsyncMock(side_effect=_scalars)++ repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)++ await repo.get_blocking_sequential_workflow_run("wr_self")++ where_clause = _where_clause_sql(calls[1])+ assert "workflow_runs.sequential_key = 'cred_test-sequential-key'" in where_clause+ assert "workflow_runs.browser_session_id IS NULL" in where_clause+ assert "workflow_runs.browser_session_id = 'pbs_test'" not in where_clause