Skip to content

Automate browser based workflows with AI

fix(SKY-10799): gate sequential runs on queue order, not a single dependency edge (#6611)

Celal Zamanoğlu4fd53c6c6d41
diff --git a/alembic/versions/2026_06_16_0803-cfc2318acdf9_add_sequential_key_lookup_index_on_workflow_runs.py b/alembic/versions/2026_06_16_0803-cfc2318acdf9_add_sequential_key_lookup_index_on_workflow_runs.py
new file mode 100644
index 000000000..cf7b44631
--- /dev/null
+++ b/alembic/versions/2026_06_16_0803-cfc2318acdf9_add_sequential_key_lookup_index_on_workflow_runs.py
@@ -0,0 +1,33 @@
+"""add sequential_key lookup index on workflow_runs
+
+Revision ID: cfc2318acdf9
+Revises: 218048b68412
+Create Date: 2026-06-16T08:03:01.419039+00:00
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "cfc2318acdf9"
+down_revision: Union[str, None] = "218048b68412"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ with op.get_context().autocommit_block():
+ op.execute("SET statement_timeout = '24h';")
+ op.execute("""
+ CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_workflow_runs_sequential_key_lookup
+ ON workflow_runs (workflow_permanent_id, sequential_key, queued_at)
+ WHERE status IN ('queued', 'running', 'paused') AND browser_session_id IS NULL
+ """)
+ op.execute("RESET statement_timeout;")
+
+
+def downgrade() -> None:
+ with op.get_context().autocommit_block():
+ op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_workflow_runs_sequential_key_lookup")
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index be16e0963..68cb9fc61 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -574,6 +574,13 @@ class WorkflowRunModel(Base):
"created_at",
postgresql_where=text("status IN ('created', 'queued', 'running', 'paused')"),
),
+ Index(
+ "ix_workflow_runs_sequential_key_lookup",
+ "workflow_permanent_id",
+ "sequential_key",
+ "queued_at",
+ postgresql_where=text("status IN ('queued', 'running', 'paused') AND browser_session_id IS NULL"),
+ ),
)
workflow_run_id = Column(String, primary_key=True, default=generate_workflow_run_id)
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index fe2cc0d89..515f92452 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -715,6 +715,71 @@ class WorkflowRunsRepository(BaseRepository):
workflow_run = (await session.scalars(query)).first()
return convert_to_workflow_run(workflow_run) if workflow_run else None
+ @db_operation("get_blocking_sequential_workflow_run")
+ async def get_blocking_sequential_workflow_run(self, workflow_run_id: str) -> WorkflowRun | None:
+ # Sequential-execution gate: returns the earliest-queued run that shares this run's
+ # sequential identity and is still in flight (queued/running/paused) and was queued
+ # before it, or None when this run is clear to start. A direct scan over all
+ # same-identity runs — not a walk of one depends_on chain — so it holds when the
+ # dependency graph fans out (forest) or a queued predecessor is canceled.
+ # Ordering uses queued_at, not created_at: it is stamped at enqueue (before Temporal
+ # submission) so it reflects submit order. The no-double-start guarantee rests on the
+ # strict total order over (queued_at, workflow_run_id) below, not on any lock.
+ async with self.Session() as session:
+ run = (await session.scalars(select(WorkflowRunModel).filter_by(workflow_run_id=workflow_run_id))).first()
+ if run is None:
+ return None
+
+ # Lane resolution mirrors enqueue priority: browser_session_id > browser_address
+ # > sequential_key > whole workflow. A debug-session run carries a browser_session_id
+ # but is excluded from the session lane (as at enqueue), so it serializes on its key.
+ query = select(WorkflowRunModel).filter_by(organization_id=run.organization_id)
+ if run.browser_session_id and not run.debug_session_id:
+ query = query.filter_by(browser_session_id=run.browser_session_id)
+ elif run.browser_address:
+ query = query.filter_by(browser_address=run.browser_address)
+ elif run.sequential_key:
+ query = query.filter_by(
+ workflow_permanent_id=run.workflow_permanent_id,
+ sequential_key=run.sequential_key,
+ ).filter(WorkflowRunModel.browser_session_id.is_(None))
+ else:
+ query = query.filter_by(workflow_permanent_id=run.workflow_permanent_id).filter(
+ WorkflowRunModel.browser_session_id.is_(None)
+ )
+
+ # Sequential runs are stamped queued_at before Temporal submission; the fallback
+ # only guards hand-created rows (e.g. tests) from comparing against None.
+ self_queued_at = run.queued_at if run.queued_at is not None else run.created_at
+
+ query = query.filter(
+ WorkflowRunModel.status.in_(
+ [
+ WorkflowRunStatus.queued,
+ WorkflowRunStatus.running,
+ WorkflowRunStatus.paused,
+ ]
+ )
+ )
+ # Safe because a sequential run is always stamped queued_at (passes through queued)
+ # before it can reach running/paused, so this filter never hides a real blocker.
+ query = query.filter(WorkflowRunModel.queued_at.isnot(None))
+ query = query.filter(
+ or_(
+ WorkflowRunModel.queued_at < self_queued_at,
+ and_(
+ WorkflowRunModel.queued_at == self_queued_at,
+ WorkflowRunModel.workflow_run_id < run.workflow_run_id,
+ ),
+ )
+ )
+ query = query.order_by(
+ WorkflowRunModel.queued_at.asc(),
+ WorkflowRunModel.workflow_run_id.asc(),
+ )
+ blocker = (await session.scalars(query)).first()
+ return convert_to_workflow_run(blocker) if blocker else None
+
async def _get_last_workflow_run_by_filter(
self,
organization_id: str | None = None,
diff --git a/tests/unit/forge/sdk/db/test_workflow_runs_repository.py b/tests/unit/forge/sdk/db/test_workflow_runs_repository.py
index 5cd6d5c8a..20a6d5706 100644
--- a/tests/unit/forge/sdk/db/test_workflow_runs_repository.py
+++ b/tests/unit/forge/sdk/db/test_workflow_runs_repository.py
@@ -43,6 +43,14 @@ class _SessionContext:
return False
+class _Result:
+ def __init__(self, value: Any) -> None:
+ self._value = value
+
+ def first(self) -> Any:
+ return self._value
+
+
class _EmptyExecuteResult:
def mappings(self) -> _EmptyExecuteResult:
return self
@@ -464,3 +472,127 @@ async def test_get_workflow_runs_for_browser_session_filters_and_excludes() -> N
assert "ORDER BY workflow_runs.created_at DESC" in rendered
assert "LIMIT 5" in rendered
assert "OFFSET 5" in rendered
+
+
+async def test_get_blocking_sequential_workflow_run_scans_earlier_active_same_key() -> None:
+ """SKY-10799: the sequential gate scans ALL earlier-queued same-key runs still in flight
+ (queued/running/paused) — not a single depends_on edge — so it holds under a forest-shaped
+ graph or a canceled predecessor. Earlier = (queued_at, id) strictly before self; queued_at
+ is stamped under the submit lock, so it is the true queue order even when creation order
+ diverges from submission order."""
+ fake_run = MagicMock()
+ fake_run.workflow_run_id = "wr_self"
+ fake_run.organization_id = "o_test"
+ fake_run.workflow_permanent_id = "wpid_test"
+ fake_run.sequential_key = "cred_test-sequential-key"
+ fake_run.browser_session_id = None
+ fake_run.browser_address = None
+ fake_run.created_at = datetime(2026, 6, 8, 18, 53, 48, tzinfo=timezone.utc)
+ fake_run.queued_at = datetime(2026, 6, 8, 18, 53, 50, tzinfo=timezone.utc)
+
+ calls: list[Any] = []
+
+ async def _scalars(query: Any) -> Any:
+ calls.append(query)
+ # 1st query loads the run itself; 2nd is the gate scan we assert on.
+ return _Result(fake_run) if len(calls) == 1 else _Result(None)
+
+ session = MagicMock()
+ session.scalars = AsyncMock(side_effect=_scalars)
+
+ repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)
+
+ blocker = await repo.get_blocking_sequential_workflow_run("wr_self")
+ assert blocker is None
+
+ gate_query = calls[1]
+ where_clause = _where_clause_sql(gate_query)
+ rendered = str(gate_query.compile(compile_kwargs={"literal_binds": True}))
+
+ # scoped to the same wpid + key, ignoring browser-session runs
+ assert "workflow_runs.workflow_permanent_id = 'wpid_test'" in where_clause
+ assert "workflow_runs.sequential_key = 'cred_test-sequential-key'" in where_clause
+ assert "workflow_runs.browser_session_id IS NULL" in where_clause
+ assert "workflow_runs.organization_id = 'o_test'" in where_clause
+ # only genuinely in-flight runs block; `created` and terminal statuses do not
+ assert "'queued'" in where_clause and "'running'" in where_clause and "'paused'" in where_clause
+ assert "'created'" not in where_clause and "'completed'" not in where_clause
+ # earlier-queued by (queued_at, id), never created_at, and FIFO order so the
+ # earliest blocker surfaces; unqueued rows can't block
+ assert "workflow_runs.queued_at IS NOT NULL" in where_clause
+ assert "workflow_runs.queued_at <" in where_clause
+ assert "workflow_runs.created_at" not in where_clause
+ assert "workflow_runs.workflow_run_id < 'wr_self'" in where_clause
+ assert "ORDER BY workflow_runs.queued_at ASC" in rendered
+
+
+async def test_get_blocking_sequential_workflow_run_prefers_browser_session_lane() -> None:
+ """The gate's lane resolution must mirror enqueue priority (browser_session_id >
+ browser_address > sequential_key): a non-debug run carrying both a browser session and a
+ sequential_key chains on the session lane at enqueue, so the gate must scan that
+ same lane or it can miss its actual blocker."""
+ fake_run = MagicMock()
+ fake_run.workflow_run_id = "wr_self"
+ fake_run.organization_id = "o_test"
+ fake_run.workflow_permanent_id = "wpid_test"
+ fake_run.sequential_key = "cred_test-sequential-key"
+ fake_run.browser_session_id = "pbs_test"
+ fake_run.debug_session_id = None
+ fake_run.browser_address = None
+ fake_run.created_at = datetime(2026, 6, 8, 18, 53, 48, tzinfo=timezone.utc)
+ fake_run.queued_at = datetime(2026, 6, 8, 18, 53, 50, tzinfo=timezone.utc)
+
+ calls: list[Any] = []
+
+ async def _scalars(query: Any) -> Any:
+ calls.append(query)
+ return _Result(fake_run) if len(calls) == 1 else _Result(None)
+
+ session = MagicMock()
+ session.scalars = AsyncMock(side_effect=_scalars)
+
+ repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)
+
+ await repo.get_blocking_sequential_workflow_run("wr_self")
+
+ where_clause = _where_clause_sql(calls[1])
+ assert "workflow_runs.browser_session_id = 'pbs_test'" in where_clause
+ assert "sequential_key" not in where_clause
+
+
+async def test_get_blocking_sequential_workflow_run_debug_session_uses_key_lane() -> None:
+ """A debug-session run carries a browser_session_id but enqueue keeps it out of the
+ browser-session lane (is_browser_session_workflow requires not debug_session_id). The gate
+ must do the same, or the debug run scans only its own session and misses an earlier same-key
+ run — the SKY-10799 regression."""
+ fake_run = MagicMock()
+ fake_run.workflow_run_id = "wr_self"
+ fake_run.organization_id = "o_test"
+ fake_run.workflow_permanent_id = "wpid_test"
+ fake_run.sequential_key = "cred_test-sequential-key"
+ fake_run.browser_session_id = "pbs_test"
+ fake_run.debug_session_id = "dbg_test"
+ fake_run.browser_address = None
+ fake_run.created_at = datetime(2026, 6, 8, 18, 53, 48, tzinfo=timezone.utc)
+ fake_run.queued_at = datetime(2026, 6, 8, 18, 53, 50, tzinfo=timezone.utc)
+
+ calls: list[Any] = []
+
+ async def _scalars(query: Any) -> Any:
+ calls.append(query)
+ return _Result(fake_run) if len(calls) == 1 else _Result(None)
+
+ session = MagicMock()
+ session.scalars = AsyncMock(side_effect=_scalars)
+
+ repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)
+
+ await repo.get_blocking_sequential_workflow_run("wr_self")
+
+ where_clause = _where_clause_sql(calls[1])
+ assert "workflow_runs.sequential_key = 'cred_test-sequential-key'" in where_clause
+ assert "workflow_runs.browser_session_id IS NULL" in where_clause
+ assert "workflow_runs.browser_session_id = 'pbs_test'" not in where_clause