Skip to content

Automate browser based workflows with AI

Add serial workflow run groups that submit, recover, and report up to 25 runs as one operation (SKY-17274) (#8773)

Andrew Neilson0ca8f2419b62
diff --git a/alembic/versions/2026_10_02_1934-6fbee6664b86_add_workflow_run_groups.py b/alembic/versions/2026_10_02_1934-6fbee6664b86_add_workflow_run_groups.py
new file mode 100644
index 000000000..0c0a1ce1b
--- /dev/null
+++ b/alembic/versions/2026_10_02_1934-6fbee6664b86_add_workflow_run_groups.py
@@ -0,0 +1,73 @@
+"""add workflow run groups
+
+Revision ID: 6fbee6664b86
+Revises: 13139662a50b
+Create Date: 2026-10-02T19:34:31.499073+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "6fbee6664b86"
+down_revision: Union[str, None] = "13139662a50b"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "workflow_run_groups",
+ sa.Column("workflow_run_group_id", sa.String(), nullable=False),
+ sa.Column("organization_id", sa.String(), nullable=False),
+ sa.Column("workflow_permanent_id", sa.String(), nullable=False),
+ sa.Column("requested_version", sa.Integer(), nullable=True),
+ sa.Column("workflow_id", sa.String(), nullable=False),
+ sa.Column("submission_key", sa.String(), nullable=False),
+ sa.Column("input_fingerprint", sa.String(), nullable=False),
+ sa.Column("status", sa.String(), nullable=False),
+ sa.Column("created_at", sa.DateTime(), nullable=False),
+ sa.Column("modified_at", sa.DateTime(), nullable=False),
+ sa.Column("finished_at", sa.DateTime(), nullable=True),
+ sa.PrimaryKeyConstraint("workflow_run_group_id"),
+ sa.UniqueConstraint("organization_id", "submission_key", name="uq_workflow_run_groups_org_submission_key"),
+ )
+ op.create_index(
+ "idx_workflow_run_groups_unfinished_modified_at",
+ "workflow_run_groups",
+ ["modified_at"],
+ unique=False,
+ postgresql_where=sa.text("status IN ('active', 'cancel_requested')"),
+ )
+ op.create_table(
+ "workflow_run_group_items",
+ sa.Column("workflow_run_group_id", sa.String(), nullable=False),
+ sa.Column("position", sa.Integer(), nullable=False),
+ sa.Column("item_key", sa.String(), nullable=False),
+ sa.Column("parameters", sa.JSON(), nullable=False),
+ sa.Column("workflow_run_id", sa.String(), nullable=False),
+ sa.Column("state", sa.String(), nullable=False),
+ sa.Column("dispatch_token", sa.String(), nullable=True),
+ sa.Column("claimed_at", sa.DateTime(), nullable=True),
+ sa.Column("dispatch_attempts", sa.Integer(), nullable=False),
+ sa.Column("failure_reason", sa.String(), nullable=True),
+ sa.Column("created_at", sa.DateTime(), nullable=False),
+ sa.Column("modified_at", sa.DateTime(), nullable=False),
+ sa.PrimaryKeyConstraint("workflow_run_group_id", "position"),
+ sa.UniqueConstraint("workflow_run_group_id", "item_key", name="uq_workflow_run_group_items_group_item_key"),
+ sa.UniqueConstraint("workflow_run_id", name="uq_workflow_run_group_items_workflow_run_id"),
+ )
+
+
+def downgrade() -> None:
+ op.drop_table("workflow_run_group_items")
+ op.drop_index(
+ "idx_workflow_run_groups_unfinished_modified_at",
+ table_name="workflow_run_groups",
+ postgresql_where=sa.text("status IN ('active', 'cancel_requested')"),
+ )
+ op.drop_table("workflow_run_groups")
diff --git a/docs/changelog.mdx b/docs/changelog.mdx
index f6966622d..739734ceb 100644
--- a/docs/changelog.mdx
+++ b/docs/changelog.mdx
@@ -113,6 +113,16 @@ keywords:
- **Simplified Onboarding Emails** — Added a Day-1 email introducing Copilot, simplified the Welcome email to two steps, and stopped sending "no successful run yet" nudges to team accounts.
- **Welcome Questionnaire for New Sign-Ups** — New sign-ups now land directly on the welcome questionnaire instead of briefly landing on Home first.
- **Clearer Insurance Quote Example** — The Home page's insurance quote example now reads as plain text instead of a JSON blob.
+- **Clearer Extraction Results** — Skyvern 3.0 extraction-only blocks now always report exactly what's on the page, marking any missing field as null. ([#8771](https://github.com/Skyvern-AI/skyvern/pull/8771))
+- **Instant Welcome Email** — New signups now get the Welcome email right after their first app load, instead of waiting up to an hour.
+- **Clearer Copilot Loop Debugging** — Workflow Copilot now shows which input a for_loop block actually used, with recorded row counts, making a failed loop easier to debug.
+- **Readable Copilot Browser-Code Steps** — Copilot's browser-code tool calls now list the steps the code performed, like "Opened a page → clicked next → read ×10," instead of just "Ran browser code."
+- **Clearer Blocked Code-Block Errors** — Copilot now explains which value caused a blocked code block and how earlier outputs reach code, instead of a generic error.
+- **Visible Page-Load Failures in Code Blocks** — A CodeBlock wait-timeout error now shows the page load failure that caused it, instead of just the timeout.
+- **More Complete Copilot Repairs** — Copilot's automatic repairs now see icon-only controls, like expanders and chevrons, even when their selector isn't unique on the page.
+- **Starter-Prompt Onboarding Email** — New signups now get an email with three starter prompts — collecting website data, downloading invoices, and filling out a form — to help them get going.
+- **Readable Copilot Tables** — Copilot replies now render Markdown tables, headings, quotes, and dividers properly instead of as unstyled text.
+- **Search Runs by More Identifiers** — Run History and run search can now find runs by browser profile ID, browser session ID, credential ID, or webhook callback URL, not just run ID or inputs.
## Bug fixes
@@ -168,6 +178,16 @@ keywords:
- Fixed the code block AI fallback running during Copilot build tests and editor block runs, hiding real code regressions; run details now show which block it rescued and why the code failed.
- Fixed a Terminate block's custom error code or reason being able to leak a referenced secret's value.
- Fixed task runs occasionally being marked failed or terminated right after a successful form submission.
+- Fixed Skyvern 3.0 refusing to finish a task when the page had a live countdown, clock, or rotating headline.
+- Fixed Skyvern 3.0 losing its view of the page entirely on sites that define their own `toJSON`, instead of reading the page correctly. ([#8770](https://github.com/Skyvern-AI/skyvern/pull/8770))
+- Fixed Skyvern 3.0 occasionally picking the wrong row from a long dropdown or menu due to a shifted or guessed row number. ([#8767](https://github.com/Skyvern-AI/skyvern/pull/8767))
+- Fixed a single-action step sometimes being marked complete before its action actually took effect, which could carry the run onto the wrong next page.
+- Fixed Skyvern 3.0 missing icon-only close buttons on overlays and incorrectly reporting that no controls were found. ([#8766](https://github.com/Skyvern-AI/skyvern/pull/8766))
+- Fixed Copilot's automatic repair discarding the notice that a full-screen overlay, like a consent banner, was blocking the page.
+- Fixed Skyvern 3.0 on slow-loading pages misjudging whether the page had actually finished loading.
+- Fixed Skyvern 3.0 incorrectly reporting a search-and-select field as empty when a single result committed automatically.
+- Fixed Skyvern 3.0 clicking the wrong control when its screen-reader label differed from its visible text.
+- Fixed Copilot failing before sending any request on certain Anthropic and Bedrock models.
</Update>
diff --git a/skyvern/config.py b/skyvern/config.py
index e495f8e5a..4089c49ed 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -986,6 +986,9 @@ class Settings(BaseSettings):
"""Maximum number of scheduled workflow runs dispatched concurrently by one OSS server process."""
RETRY_DISPATCH_GRACE_SECONDS: int = Field(default=600, ge=600)
"""OSS dispatch claim grace; the executor also enforces the retry lease takeover minimum."""
+ WORKFLOW_RUN_GROUPS_SUBMIT_ENABLED: bool = True
+ """Accept new serial workflow run groups. Turning it off stops submission only; reads, cancels and
+ dispatch of already-submitted groups continue."""
# OpenTelemetry Settings
OTEL_ENABLED: bool = False
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 7e2ad2eb5..ebd136e96 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -141,6 +141,23 @@ class SkyvernHTTPException(SkyvernException):
super().__init__(message)
+class WorkflowPinnedByRunGroup(SkyvernHTTPException):
+ def __init__(self, workflow_id: str) -> None:
+ super().__init__(
+ f"Workflow version {workflow_id} is running as a workflow run group, so it cannot be edited in place "
+ "until that group finishes. Save the change as a new version instead.",
+ status_code=HTTPStatus.CONFLICT,
+ )
+
+
+class WorkflowChangedSinceReview(SkyvernHTTPException):
+ def __init__(self, workflow_id: str) -> None:
+ super().__init__(
+ f"Workflow version {workflow_id} changed after it was reviewed, so the group was not submitted.",
+ status_code=HTTPStatus.CONFLICT,
+ )
+
+
_BROWSER_CONNECTION_GUIDANCE = "Please try re-running. If this continues, contact [email protected]."
# Patterns that indicate a browser session connection failure (e.g. CDP WebSocket errors).
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 09f3fa98f..67309adfc 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -86,6 +86,7 @@ from skyvern.forge.sdk.workflow.retry_policy import WORKFLOW_WEBHOOK_HTTP_TIMEOU
from skyvern.schemas.run_enums import RunEngine, RunType
from skyvern.schemas.workflows import BlockResult, FileStorageType, FileUploadDestination
from skyvern.services.otp_email import EmailOTPSearchError, EmailOTPVerificationContext, build_email_otp_sources
+from skyvern.services.workflow_run_group_service import schedule_advance_after_terminal
from skyvern.utils.email_validation import normalize_identifier_if_email
from skyvern.utils.url_validators import pinned_ip_client
from skyvern.webeye.actions.actions import Action
@@ -3068,6 +3069,9 @@ class AgentFunction:
"""Fired after a workflow run reaches a final status. The run may be supplied to avoid a fallback read."""
return
+ def schedule_workflow_run_group_advance(self, workflow_run: WorkflowRun) -> None:
+ schedule_advance_after_terminal(workflow_run)
+
async def on_task_completed(
self,
*,
diff --git a/skyvern/forge/api_app.py b/skyvern/forge/api_app.py
index 63c6178b6..4bb06df5f 100644
--- a/skyvern/forge/api_app.py
+++ b/skyvern/forge/api_app.py
@@ -45,6 +45,7 @@ from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.agent_db import AgentDB
from skyvern.forge.sdk.db.exceptions import DatabaseConnectionUnavailableError, NotFoundError, is_connection_failure
from skyvern.forge.sdk.db.models import Base
+from skyvern.forge.sdk.executor.background_task_executor import BackgroundTaskExecutor
from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
from skyvern.forge.sdk.routes import internal_auth, internal_llms
from skyvern.forge.sdk.routes.google_oauth import google_oauth_router
@@ -67,6 +68,7 @@ from skyvern.services.cleanup_service import (
stop_cleanup_scheduler,
stop_temp_artifact_sweep,
)
+from skyvern.services.workflow_run_group_service import run_workflow_run_group_recovery_loop
from skyvern.services.workflow_schedule_service import (
start_workflow_schedule_scheduler,
stop_workflow_schedule_scheduler,
@@ -408,6 +410,12 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:
# task group which is required for handling MCP requests.
mcp_app = getattr(fastapi_app.state, "mcp_starlette_app", None)
retry_recovery_task = asyncio.create_task(_recover_pending_retries(), name="workflow-retry-recovery")
+ # Cloud runs this sweep from its own periodic task loop.
+ group_recovery_task = (
+ asyncio.create_task(run_workflow_run_group_recovery_loop(), name="workflow-run-group-recovery")
+ if isinstance(AsyncExecutorFactory.get_executor(), BackgroundTaskExecutor)
+ else None
+ )
try:
if mcp_app:
async with mcp_app.lifespan(mcp_app):
@@ -418,7 +426,11 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:
yield
finally:
retry_recovery_task.cancel()
- await asyncio.gather(retry_recovery_task, return_exceptions=True)
+ if group_recovery_task is not None:
+ group_recovery_task.cancel()
+ await asyncio.gather(
+ retry_recovery_task, *([group_recovery_task] if group_recovery_task else []), return_exceptions=True
+ )
# The initial pass starts the periodic sweep even when cancelled; stop it so the loop can drain.
await AsyncExecutorFactory.get_executor().stop_retry_recovery()
diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.py
index 9f213697a..cf2ed680c 100644
--- a/skyvern/forge/sdk/db/agent_db.py
+++ b/skyvern/forge/sdk/db/agent_db.py
@@ -49,6 +49,7 @@ from skyvern.forge.sdk.db.repositories.workflow_run_attempts import WorkflowRunA
from skyvern.forge.sdk.db.repositories.workflow_run_credential_selections import (
WorkflowRunCredentialSelectionsRepository,
)
+from skyvern.forge.sdk.db.repositories.workflow_run_groups import WorkflowRunGroupsRepository
from skyvern.forge.sdk.db.repositories.workflow_runs import WorkflowRunsRepository
from skyvern.forge.sdk.db.repositories.workflows import WorkflowsRepository
from skyvern.forge.sdk.db.utils import (
@@ -410,6 +411,7 @@ class AgentDB(BaseAlchemyDB):
self.workflow_run_credential_selections = WorkflowRunCredentialSelectionsRepository(
self.Session, debug_enabled, self.is_retryable_error
)
+ self.workflow_run_groups = WorkflowRunGroupsRepository(self.Session, debug_enabled, self.is_retryable_error)
self.credentials = CredentialRepository(self.Session, debug_enabled, self.is_retryable_error)
self.credential_folders = CredentialFoldersRepository(self.Session, debug_enabled, self.is_retryable_error)
self.otp = OTPRepository(self.Session, debug_enabled, self.is_retryable_error)
diff --git a/skyvern/forge/sdk/db/id.py b/skyvern/forge/sdk/db/id.py
index 833a34231..8ba05feb7 100644
--- a/skyvern/forge/sdk/db/id.py
+++ b/skyvern/forge/sdk/db/id.py
@@ -84,6 +84,7 @@ SCRIPT_FALLBACK_EPISODE_PREFIX = "sfe"
HEAL_EPISODE_PREFIX = "he"
HEAL_PROPOSAL_PREFIX = "hp"
WORKFLOW_SCHEDULE_PREFIX = "wfs"
+WORKFLOW_RUN_GROUP_PREFIX = "wrg"
TAG_EVENT_PREFIX = "tge"
RUN_TAG_EVENT_PREFIX = "rtge"
TAG_KEY_PREFIX = "tkey"
@@ -392,6 +393,11 @@ def generate_workflow_schedule_id() -> str:
return f"{WORKFLOW_SCHEDULE_PREFIX}_{int_id}"
+def generate_workflow_run_group_id() -> str:
+ int_id = generate_id()
+ return f"{WORKFLOW_RUN_GROUP_PREFIX}_{int_id}"
+
+
############# Helper functions below ##############
def generate_id() -> int:
"""
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index 9df0d0a68..636ab01a5 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -80,6 +80,7 @@ from skyvern.forge.sdk.db.id import (
generate_workflow_permanent_id,
generate_workflow_run_block_id,
generate_workflow_run_credential_selection_id,
+ generate_workflow_run_group_id,
generate_workflow_run_id,
generate_workflow_schedule_id,
generate_workflow_script_id,
@@ -820,6 +821,61 @@ class WorkflowScheduleModel(Base):
deleted_at = Column(DateTime, nullable=True)
+class WorkflowRunGroupModel(Base):
+ __tablename__ = "workflow_run_groups"
+ __table_args__ = (
+ UniqueConstraint("organization_id", "submission_key", name="uq_workflow_run_groups_org_submission_key"),
+ Index(
+ "idx_workflow_run_groups_unfinished_modified_at",
+ "modified_at",
+ postgresql_where=text("status IN ('active', 'cancel_requested')"),
+ ),
+ )
+
+ workflow_run_group_id = Column(String, primary_key=True, default=generate_workflow_run_group_id)
+ organization_id = Column(String, nullable=False)
+ workflow_permanent_id = Column(String, nullable=False)
+ requested_version = Column(Integer, nullable=True)
+ workflow_id = Column(String, nullable=False)
+ submission_key = Column(String, nullable=False)
+ input_fingerprint = Column(String, nullable=False)
+ status = Column(String, nullable=False, default="active")
+ created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
+ modified_at = Column(
+ DateTime,
+ default=datetime.datetime.utcnow,
+ onupdate=datetime.datetime.utcnow,
+ nullable=False,
+ )
+ finished_at = Column(DateTime, nullable=True)
+
+
+class WorkflowRunGroupItemModel(Base):
+ __tablename__ = "workflow_run_group_items"
+ __table_args__ = (
+ UniqueConstraint("workflow_run_group_id", "item_key", name="uq_workflow_run_group_items_group_item_key"),
+ UniqueConstraint("workflow_run_id", name="uq_workflow_run_group_items_workflow_run_id"),
+ )
+
+ workflow_run_group_id = Column(String, primary_key=True)
+ position = Column(Integer, primary_key=True)
+ item_key = Column(String, nullable=False)
+ parameters = Column(JSON, nullable=False)
+ workflow_run_id = Column(String, nullable=False)
+ state = Column(String, nullable=False, default="pending")
+ dispatch_token = Column(String, nullable=True)
+ claimed_at = Column(DateTime, nullable=True)
+ dispatch_attempts = Column(Integer, nullable=False, default=0)
+ failure_reason = Column(String, nullable=True)
+ created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
+ modified_at = Column(
+ DateTime,
+ default=datetime.datetime.utcnow,
+ onupdate=datetime.datetime.utcnow,
+ nullable=False,
+ )
+
+
class WorkflowTemplateModel(Base):
"""
Tracks which workflows are marked as templates.
diff --git a/skyvern/forge/sdk/db/repositories/workflow_run_groups.py b/skyvern/forge/sdk/db/repositories/workflow_run_groups.py
new file mode 100644
index 000000000..135612ec3
--- /dev/null
+++ b/skyvern/forge/sdk/db/repositories/workflow_run_groups.py
@@ -0,0 +1,389 @@
+from __future__ import annotations
+
+import hmac
+from collections.abc import Sequence
+from datetime import datetime
+from hashlib import sha256
+from typing import Any, Literal
+
+from sqlalchemy import String, cast, func, select, update
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from skyvern.config import settings
+from skyvern.constants import SCRUBBED_VALUE
+from skyvern.exceptions import WorkflowChangedSinceReview
+from skyvern.forge.sdk.db._error_handling import db_operation
+from skyvern.forge.sdk.db.base_repository import BaseRepository
+from skyvern.forge.sdk.db.datetime_utils import naive_utc_now
+from skyvern.forge.sdk.db.id import generate_workflow_run_id
+from skyvern.forge.sdk.db.models import (
+ StepModel,
+ TaskModel,
+ WorkflowModel,
+ WorkflowRunGroupItemModel,
+ WorkflowRunGroupModel,
+)
+from skyvern.schemas.workflow_run_groups import (
+ IN_FLIGHT_ITEM_STATES,
+ SCRUBBED_SUBMISSION_KEY_PREFIX,
+ WorkflowRunGroup,
+ WorkflowRunGroupItem,
+ WorkflowRunGroupItemState,
+ WorkflowRunGroupStatus,
+)
+
+FlipResult = Literal["dispatched", "canceled", "lost"]
+_SCRUBBED_KEY_DOMAIN = b"skyvern.workflow_run_group_scrubbed_submission_key.v1\0"
+
+
+def scrubbed_submission_key(submission_key: str) -> str:
+ # Keyed so a scrubbed row cannot be matched to a guessed account name, yet a replay of the raw key still finds it.
+ digest = hmac.new(settings.SECRET_KEY.encode(), _SCRUBBED_KEY_DOMAIN + submission_key.encode(), sha256).hexdigest()
+ return f"{SCRUBBED_SUBMISSION_KEY_PREFIX}{digest}"
+
+
+class WorkflowRunGroupsRepository(BaseRepository):
+ # Item parameters are cleared on every transition out of pending/dispatching; the child run holds its own copy.
+ # Every state transition first touches the group row, so transitions of one group serialize on its row lock.
+ async def _lock_group(self, session: AsyncSession, workflow_run_group_id: str) -> WorkflowRunGroupStatus | None:
+ status = await session.scalar(
+ update(WorkflowRunGroupModel)
+ .where(WorkflowRunGroupModel.workflow_run_group_id == workflow_run_group_id)
+ .values(modified_at=naive_utc_now())
+ .returning(WorkflowRunGroupModel.status)
+ )
+ return WorkflowRunGroupStatus(status) if status is not None else None
+
+ @db_operation("create_workflow_run_group", expected_errors=(IntegrityError, WorkflowChangedSinceReview))
+ async def create_group(
+ self,
+ *,
+ organization_id: str,
+ workflow_permanent_id: str,
+ requested_version: int | None,
+ workflow_id: str,
+ submission_key: str,
+ input_fingerprint: str,
+ items: Sequence[tuple[str, dict[str, Any]]],
+ expected_workflow_modified_at: datetime | None = None,
+ ) -> WorkflowRunGroup:
+ async with self.Session() as session:
+ # Locks the row in-place edits lock, so an edit either lands before this check or sees this group.
+ modified_at = await session.scalar(
+ select(WorkflowModel.modified_at).where(WorkflowModel.workflow_id == workflow_id).with_for_update()
+ )
+ if expected_workflow_modified_at is not None and modified_at != expected_workflow_modified_at:
+ raise WorkflowChangedSinceReview(workflow_id)
+ group = WorkflowRunGroupModel(
+ organization_id=organization_id,
+ workflow_permanent_id=workflow_permanent_id,
+ requested_version=requested_version,
+ workflow_id=workflow_id,
+ submission_key=submission_key,
+ input_fingerprint=input_fingerprint,
+ status=WorkflowRunGroupStatus.active.value,
+ )
+ session.add(group)
+ await session.flush()
+ for position, (item_key, parameters) in enumerate(items):
+ session.add(
+ WorkflowRunGroupItemModel(
+ workflow_run_group_id=group.workflow_run_group_id,
+ position=position,
+ item_key=item_key,
+ parameters=parameters,
+ workflow_run_id=generate_workflow_run_id(),
+ state=WorkflowRunGroupItemState.pending.value,
+ dispatch_attempts=0,
+ )
+ )
+ await session.flush()
+ created = WorkflowRunGroup.model_validate(group)
+ await session.commit()
+ return created
+
+ @db_operation("get_workflow_run_group")
+ async def get_group(
+ self, workflow_run_group_id: str, organization_id: str | None = None
+ ) -> WorkflowRunGroup | None:
+ query = select(WorkflowRunGroupModel).where(
+ WorkflowRunGroupModel.workflow_run_group_id == workflow_run_group_id
+ )
+ if organization_id is not None:
+ query = query.where(WorkflowRunGroupModel.organization_id == organization_id)
+ async with self.Session() as session:
+ group = (await session.scalars(query)).first()
+ return WorkflowRunGroup.model_validate(group) if group else None
+
+ @db_operation("get_workflow_run_group_by_submission_key")
+ async def get_group_by_submission_key(self, organization_id: str, submission_key: str) -> WorkflowRunGroup | None:
+ async with self.Session() as session:
+ group = (
+ await session.scalars(
+ select(WorkflowRunGroupModel)
+ .where(WorkflowRunGroupModel.organization_id == organization_id)
+ .where(WorkflowRunGroupModel.submission_key == submission_key)
+ )
+ ).first()
+ return WorkflowRunGroup.model_validate(group) if group else None
+
+ @db_operation("scrub_workflow_run_group_keys")
+ async def scrub_keys(self, organization_id: str, workflow_run_group_ids: Sequence[str]) -> int:
+ """Replace the caller's submission and item keys; returns the groups scrubbed.
+
+ Item parameters are left alone: a pending or dispatching item still needs them to run, and every other state
+ has already cleared them.
+ """
+ async with self.Session() as session:
+ groups = (
+ await session.scalars(
+ select(WorkflowRunGroupModel)
+ .where(WorkflowRunGroupModel.organization_id == organization_id)
+ .where(WorkflowRunGroupModel.workflow_run_group_id.in_(workflow_run_group_ids))
+ .where(~WorkflowRunGroupModel.submission_key.startswith(SCRUBBED_SUBMISSION_KEY_PREFIX))
+ )
+ ).all()
+ if not groups:
+ return 0
+ for group in groups:
+ group.submission_key = scrubbed_submission_key(group.submission_key)
+ await session.execute(
+ update(WorkflowRunGroupItemModel)
+ .where(WorkflowRunGroupItemModel.workflow_run_group_id.in_([g.workflow_run_group_id for g in groups]))
+ .values(item_key=f"{SCRUBBED_VALUE}-" + cast(WorkflowRunGroupItemModel.position, String))
+ )
+ await session.commit()
+ return len(groups)
+
+ @db_operation("get_workflow_run_group_items")
+ async def get_items(self, workflow_run_group_id: str) -> list[WorkflowRunGroupItem]:
+ async with self.Session() as session:
+ items = (
+ await session.scalars(
+ select(WorkflowRunGroupItemModel)
+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)
+ .order_by(WorkflowRunGroupItemModel.position)
+ )
+ ).all()
+ return [WorkflowRunGroupItem.model_validate(item) for item in items]
+
+ @db_operation("get_workflow_run_group_item_by_workflow_run_id")
+ async def get_item_by_workflow_run_id(self, workflow_run_id: str) -> WorkflowRunGroupItem | None:
+ async with self.Session() as session:
+ item = (
+ await session.scalars(
+ select(WorkflowRunGroupItemModel).where(
+ WorkflowRunGroupItemModel.workflow_run_id == workflow_run_id
+ )
+ )
+ ).first()
+ return WorkflowRunGroupItem.model_validate(item) if item else None
+
+ @db_operation("list_unfinished_workflow_run_groups")
+ async def list_unfinished_groups(self, limit: int) -> list[tuple[str, str]]:
+ async with self.Session() as session:
+ rows = (
+ await session.execute(
+ select(WorkflowRunGroupModel.workflow_run_group_id, WorkflowRunGroupModel.organization_id)
+ .where(
+ WorkflowRunGroupModel.status.in_(
+ [WorkflowRunGroupStatus.active.value, WorkflowRunGroupStatus.cancel_requested.value]
+ )
+ )
+ .order_by(WorkflowRunGroupModel.modified_at)
+ .limit(limit)
+ )
+ ).all()
+ return [(group_id, organization_id) for group_id, organization_id in rows]
+
+ @db_operation("claim_next_workflow_run_group_item")
+ async def claim_next_item(self, workflow_run_group_id: str, dispatch_token: str) -> WorkflowRunGroupItem | None:
+ async with self.Session() as session:
+ if await self._lock_group(session, workflow_run_group_id) != WorkflowRunGroupStatus.active:
+ await session.commit()
+ return None
+ in_flight = await session.scalar(
+ select(func.count())
+ .select_from(WorkflowRunGroupItemModel)
+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)
+ .where(WorkflowRunGroupItemModel.state.in_([state.value for state in IN_FLIGHT_ITEM_STATES]))
+ )
+ next_position = None
+ if not in_flight:
+ next_position = await session.scalar(
+ select(func.min(WorkflowRunGroupItemModel.position))
+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)
+ .where(WorkflowRunGroupItemModel.state == WorkflowRunGroupItemState.pending.value)
+ )
+ if next_position is None:
+ await session.commit()
+ return None
+ claimed = await session.scalar(
+ update(WorkflowRunGroupItemModel)
+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)
+ .where(WorkflowRunGroupItemModel.position == next_position)
+ .where(WorkflowRunGroupItemModel.state == WorkflowRunGroupItemState.pending.value)
+ .values(
+ state=WorkflowRunGroupItemState.dispatching.value,
+ dispatch_token=dispatch_token,
+ claimed_at=naive_utc_now(),
+ dispatch_attempts=WorkflowRunGroupItemModel.dispatch_attempts + 1,
+ )
+ .returning(WorkflowRunGroupItemModel)
+ )
+ result = WorkflowRunGroupItem.model_validate(claimed) if claimed else None
+ await session.commit()
+ return result
+
+ @db_operation("reclaim_stale_workflow_run_group_item")
+ async def reclaim_stale_item(
+ self,
+ workflow_run_group_id: str,
+ position: int,
+ *,
+ dispatch_token: str,
+ stale_before: datetime,
+ ) -> WorkflowRunGroupItem | None:
+ async with self.Session() as session:
+ await self._lock_group(session, workflow_run_group_id)
+ reclaimed = await session.scalar(
+ update(WorkflowRunGroupItemModel)
+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)
+ .where(WorkflowRunGroupItemModel.position == position)
+ .where(WorkflowRunGroupItemModel.state == WorkflowRunGroupItemState.dispatching.value)
+ .where(WorkflowRunGroupItemModel.claimed_at < stale_before)
+ .values(
+ dispatch_token=dispatch_token,
+ claimed_at=naive_utc_now(),
+ dispatch_attempts=WorkflowRunGroupItemModel.dispatch_attempts + 1,
+ )
+ .returning(WorkflowRunGroupItemModel)
+ )
+ result = WorkflowRunGroupItem.model_validate(reclaimed) if reclaimed else None
+ await session.commit()
+ return result
+
+ @db_operation("flip_workflow_run_group_item_to_dispatched")
+ async def flip_to_dispatched(self, workflow_run_group_id: str, position: int, dispatch_token: str) -> FlipResult:
+ async with self.Session() as session:
+ group_status = await self._lock_group(session, workflow_run_group_id)
+ item = (
+ await session.scalars(
+ select(WorkflowRunGroupItemModel)
+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)
+ .where(WorkflowRunGroupItemModel.position == position)
+ )
+ ).first()
+ if (
+ item is None
+ or item.state != WorkflowRunGroupItemState.dispatching.value
+ or item.dispatch_token != dispatch_token
+ ):
+ await session.commit()
+ return "lost"
+ result: FlipResult = "dispatched"
+ item.parameters = {}
+ if group_status != WorkflowRunGroupStatus.active:
+ result = "canceled"
+ item.state = WorkflowRunGroupItemState.canceled.value
+ else:
+ item.state = WorkflowRunGroupItemState.dispatched.value
+ item.claimed_at = naive_utc_now()
+ await session.commit()
+ return result
+
+ @db_operation("finish_workflow_run_group_item")
+ async def finish_item(
+ self,
+ workflow_run_group_id: str,
+ position: int,
+ *,
+ state: WorkflowRunGroupItemState,
+ from_states: Sequence[WorkflowRunGroupItemState],
+ dispatch_token: str | None = None,
+ failure_reason: str | None = None,
+ ) -> bool:
+ async with self.Session() as session:
+ await self._lock_group(session, workflow_run_group_id)
+ statement = (
+ update(WorkflowRunGroupItemModel)
+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)
+ .where(WorkflowRunGroupItemModel.position == position)
+ .where(WorkflowRunGroupItemModel.state.in_([from_state.value for from_state in from_states]))
+ )
+ if dispatch_token is not None:
+ statement = statement.where(WorkflowRunGroupItemModel.dispatch_token == dispatch_token)
+ values: dict[str, str | dict[str, Any]] = {"state": state.value, "parameters": {}}
+ if failure_reason is not None:
+ values["failure_reason"] = failure_reason
+ updated = await session.scalar(statement.values(**values).returning(WorkflowRunGroupItemModel.position))
+ await session.commit()
+ return updated is not None
+
+ @db_operation("request_workflow_run_group_cancel")
+ async def request_cancel(self, workflow_run_group_id: str, organization_id: str) -> WorkflowRunGroup | None:
+ async with self.Session() as session:
+ group = (
+ await session.scalars(
+ update(WorkflowRunGroupModel)
+ .where(WorkflowRunGroupModel.workflow_run_group_id == workflow_run_group_id)
+ .where(WorkflowRunGroupModel.organization_id == organization_id)
+ .values(modified_at=naive_utc_now())
+ .returning(WorkflowRunGroupModel)
+ )
+ ).first()
+ if group is None:
+ await session.commit()
+ return None
+ if group.status == WorkflowRunGroupStatus.active.value:
+ group.status = WorkflowRunGroupStatus.cancel_requested.value
+ await session.execute(
+ update(WorkflowRunGroupItemModel)
+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)
+ .where(WorkflowRunGroupItemModel.state == WorkflowRunGroupItemState.pending.value)
+ .values(state=WorkflowRunGroupItemState.canceled.value, parameters={})
+ )
+ await session.flush()
+ result = WorkflowRunGroup.model_validate(group)
+ await session.commit()
+ return result
+
+ @db_operation("finish_workflow_run_group_if_complete")
+ async def finish_group_if_complete(self, workflow_run_group_id: str) -> bool:
+ async with self.Session() as session:
+ group_status = await self._lock_group(session, workflow_run_group_id)
+ if group_status in (None, WorkflowRunGroupStatus.finished):
+ await session.commit()
+ return False
+ unfinished = await session.scalar(
+ select(func.count())
+ .select_from(WorkflowRunGroupItemModel)
+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)
+ .where(
+ WorkflowRunGroupItemModel.state.not_in(
+ [state.value for state in WorkflowRunGroupItemState if state.is_final()]
+ )
+ )
+ )
+ if unfinished:
+ await session.commit()
+ return False
+ now = naive_utc_now()
+ await session.execute(
+ update(WorkflowRunGroupModel)
+ .where(WorkflowRunGroupModel.workflow_run_group_id == workflow_run_group_id)
+ .values(status=WorkflowRunGroupStatus.finished.value, finished_at=now, modified_at=now)
+ )
+ await session.commit()
+ return True
+
+ @db_operation("count_workflow_run_steps")
+ async def count_steps(self, workflow_run_id: str) -> int:
+ async with self.Session() as session:
+ count = await session.scalar(
+ select(func.count(StepModel.step_id))
+ .join(TaskModel, TaskModel.task_id == StepModel.task_id)
+ .where(TaskModel.workflow_run_id == workflow_run_id)
+ )
+ return int(count or 0)
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index b62517434..f4053355e 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -365,6 +365,23 @@ async def _has_serialized_publication_identity(
return serialized_publication
+async def _first_queued_at(
+ session: AsyncSession, workflow_run: WorkflowRunModel, *, sequential_credential_id: str | None = None
+) -> datetime:
+ """A serialized run takes the next org-wide queue ticket; the caller must hold its publication-lane locks."""
+ if await _has_serialized_publication_identity(
+ session,
+ workflow_run,
+ browser_session_id=workflow_run.browser_session_id,
+ browser_address=workflow_run.browser_address,
+ sequential_credential_id=sequential_credential_id,
+ ):
+ return await _allocate_serialized_queue_ticket(
+ session, organization_id=workflow_run.organization_id, workflow_run_id=workflow_run.workflow_run_id
+ )
+ return naive_utc_now()
+
+
class WorkflowRunsRepository(BaseRepository):
"""Database operations for workflow runs."""
@@ -1005,25 +1022,9 @@ class WorkflowRunsRepository(BaseRepository):
if status:
workflow_run.status = status
if status and status == WorkflowRunStatus.queued and workflow_run.queued_at is None:
- serialized_publication = await _has_serialized_publication_identity(
- session,
- workflow_run,
- browser_session_id=workflow_run.browser_session_id,
- browser_address=workflow_run.browser_address,
- sequential_credential_id=sequential_credential_id,
- )
- if serialized_publication:
- # The caller holds every composed publication-lane lock until this transaction
- # commits. Advance beyond every active serialized ticket in the organization,
- # so all composed lanes share one comparable clock even when database transaction
- # time or an application host clock moved backwards.
- workflow_run.queued_at = await _allocate_serialized_queue_ticket(
- session,
- organization_id=workflow_run.organization_id,
- workflow_run_id=workflow_run_id,
- )
- else:
- workflow_run.queued_at = naive_utc_now()
+ workflow_run.queued_at = await _first_queued_at(
+ session, workflow_run, sequential_credential_id=sequential_credential_id
+ )
if status and status == WorkflowRunStatus.running and workflow_run.started_at is None:
workflow_run.started_at = naive_utc_now()
if status and status.is_final() and workflow_run.finished_at is None:
@@ -1216,19 +1217,16 @@ class WorkflowRunsRepository(BaseRepository):
run_with: str | None = None,
ai_fallback: bool | None = None,
failure_category: list[dict[str, Any]] | None = None,
+ only_from: Sequence[WorkflowRunStatus] | None = None,
+ job_id: str | None = None,
+ depends_on_workflow_run_id: str | None = None,
) -> WorkflowRun | None:
- """Transition a workflow run to ``status`` only if it is not already in a
- terminal state. Returns the updated row, or ``None`` when the row was
- already terminal (or missing). Implemented as a single conditional
- ``UPDATE ... WHERE status IN (<non-terminal>)`` so a concurrent
- finalization write cannot be clobbered by a late cancel.
-
- Mirrors the timestamp side effects of :meth:`update_workflow_run`:
- ``finished_at`` is stamped on terminal transitions and ``started_at``
- is stamped on the first ``running`` transition (preserving any
- existing value via ``COALESCE``).
- """
- non_terminal = [s.value for s in WorkflowRunStatus if not s.is_final()]
+ """One conditional UPDATE to ``status`` from a non-terminal state (``only_from`` narrows it), so a late
+ cancel cannot clobber a finalization; None when the row was terminal or missing. Timestamps follow
+ :meth:`update_workflow_run`."""
+ non_terminal = [
+ s.value for s in WorkflowRunStatus if not s.is_final() and (only_from is None or s in only_from)
+ ]
now = naive_utc_now()
values: dict[str, Any] = {"status": status}
if status.is_final():
@@ -1243,6 +1241,10 @@ class WorkflowRunsRepository(BaseRepository):
values["ai_fallback"] = ai_fallback
if failure_category is not None:
values["failure_category"] = failure_category
+ if job_id:
+ values["job_id"] = job_id
+ if depends_on_workflow_run_id:
+ values["depends_on_workflow_run_id"] = depends_on_workflow_run_id
# The reopen/reset path clears attribution when it returns the row to `created`, so a
# later terminal transition starts from SQL NULL and this COALESCE fills the freshly
# derived document; on any row that already carries one it preserves the first writer.
@@ -1260,6 +1262,14 @@ class WorkflowRunsRepository(BaseRepository):
)
async with self.Session() as session:
+ if status == WorkflowRunStatus.queued:
+ workflow_run = (
+ await session.scalars(select(WorkflowRunModel).filter_by(workflow_run_id=workflow_run_id))
+ ).first()
+ if workflow_run is not None and workflow_run.queued_at is None:
+ values["queued_at"] = func.coalesce(
+ WorkflowRunModel.queued_at, await _first_queued_at(session, workflow_run)
+ )
result = await session.execute(
update(WorkflowRunModel)
.where(
@@ -1966,21 +1976,7 @@ class WorkflowRunsRepository(BaseRepository):
if workflow_run is None:
return False
if workflow_run.status == WorkflowRunStatus.created:
- queued_at = workflow_run.queued_at
- if queued_at is None:
- serialized_publication = await _has_serialized_publication_identity(
- session,
- workflow_run,
- browser_session_id=workflow_run.browser_session_id,
- browser_address=workflow_run.browser_address,
- )
- queued_at = (
- await _allocate_serialized_queue_ticket(
- session, organization_id=workflow_run.organization_id, workflow_run_id=workflow_run_id
- )
- if serialized_publication
- else now
- )
+ queued_at = workflow_run.queued_at or await _first_queued_at(session, workflow_run)
# A cancel committed since the caller's read must win: the status predicate re-evaluates
# against the latest committed row.
moved = await session.execute(
diff --git a/skyvern/forge/sdk/db/repositories/workflows.py b/skyvern/forge/sdk/db/repositories/workflows.py
index 7fcd90de8..db7955d1c 100644
--- a/skyvern/forge/sdk/db/repositories/workflows.py
+++ b/skyvern/forge/sdk/db/repositories/workflows.py
@@ -10,10 +10,11 @@ from typing import TYPE_CHECKING, Any, cast
import structlog
from sqlalchemy import exists, func, or_, select, text, update
from sqlalchemy.exc import SQLAlchemyError
-from sqlalchemy.ext.asyncio import AsyncEngine
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession
from sqlalchemy.orm import aliased
from skyvern.constants import DEFAULT_SCRIPT_RUN_ID, DEFAULT_WORKFLOW_TITLES
+from skyvern.exceptions import WorkflowPinnedByRunGroup
from skyvern.forge.sdk.browser_action_policy import BrowserActionPolicy, declare_policy
from skyvern.forge.sdk.db._error_handling import db_operation
from skyvern.forge.sdk.db._sentinels import _UNSET
@@ -33,6 +34,7 @@ from skyvern.forge.sdk.db.models import (
OutputParameterModel,
WorkflowModel,
WorkflowParameterModel,
+ WorkflowRunGroupModel,
WorkflowRunModel,
WorkflowScheduleModel,
WorkflowTemplateModel,
@@ -52,6 +54,7 @@ from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
from skyvern.forge.sdk.workflow.models.workflow import COPILOT_TEST_WORKFLOW_CREATOR, Workflow, WorkflowDefinition
from skyvern.forge.sdk.workflow.runtime_completion import carried_contract, with_contract
from skyvern.schemas.runs import ProxyLocationInput
+from skyvern.schemas.workflow_run_groups import WorkflowRunGroupStatus
from skyvern.schemas.workflows import WorkflowStatus
if TYPE_CHECKING:
@@ -107,6 +110,21 @@ def _align_block_output_parameters(workflow_definition: WorkflowDefinition) -> N
_visit(workflow_definition.blocks)
+async def _refuse_if_pinned_by_run_group(session: AsyncSession, workflow_id: str) -> None:
+ pinned = await session.scalar(
+ select(WorkflowRunGroupModel.workflow_run_group_id)
+ .where(WorkflowRunGroupModel.workflow_id == workflow_id)
+ .where(
+ WorkflowRunGroupModel.status.in_(
+ [WorkflowRunGroupStatus.active.value, WorkflowRunGroupStatus.cancel_requested.value]
+ )
+ )
+ .limit(1)
+ )
+ if pinned is not None:
+ raise WorkflowPinnedByRunGroup(workflow_id)
+
+
class WorkflowsRepository(BaseRepository):
"""Database operations for workflow management."""
@@ -941,7 +959,7 @@ class WorkflowsRepository(BaseRepository):
await session.commit()
return result.rowcount > 0
- @db_operation("update_workflow")
+ @db_operation("update_workflow", expected_errors=(WorkflowPinnedByRunGroup,))
async def update_workflow(
self,
workflow_id: str,
@@ -977,6 +995,7 @@ class WorkflowsRepository(BaseRepository):
sequential_key: str | None | object = _UNSET,
created_by: str | None | object = _UNSET,
edited_by: str | None | object = _UNSET,
+ refuse_if_pinned_by_run_group: bool = False,
) -> Workflow:
async with self.Session() as session:
get_workflow_query = exclude_deleted(
@@ -984,7 +1003,11 @@ class WorkflowsRepository(BaseRepository):
)
if organization_id:
get_workflow_query = get_workflow_query.filter_by(organization_id=organization_id)
+ if refuse_if_pinned_by_run_group:
+ get_workflow_query = get_workflow_query.with_for_update()
if workflow := (await session.scalars(get_workflow_query)).first():
+ if refuse_if_pinned_by_run_group:
+ await _refuse_if_pinned_by_run_group(session, workflow_id)
if title is not None:
workflow.title = title
if description is not None:
@@ -1274,7 +1297,7 @@ class WorkflowsRepository(BaseRepository):
is_template=is_template,
)
- @db_operation("update_workflow_and_reconcile_definition_params")
+ @db_operation("update_workflow_and_reconcile_definition_params", expected_errors=(WorkflowPinnedByRunGroup,))
async def update_workflow_and_reconcile_definition_params(
self,
workflow_id: str,
@@ -1311,6 +1334,7 @@ class WorkflowsRepository(BaseRepository):
created_by: str | None | object = _UNSET,
edited_by: str | None | object = _UNSET,
preserve_completion_contract: bool = True,
+ refuse_if_pinned_by_run_group: bool = False,
) -> Workflow:
"""One-session, one-commit update of the workflow row + definition-parameter rows.
@@ -1343,9 +1367,13 @@ class WorkflowsRepository(BaseRepository):
)
if organization_id:
get_workflow_query = get_workflow_query.filter_by(organization_id=organization_id)
+ if refuse_if_pinned_by_run_group:
+ get_workflow_query = get_workflow_query.with_for_update()
workflow = (await session.scalars(get_workflow_query)).first()
if not workflow:
raise NotFoundError("Workflow not found")
+ if refuse_if_pinned_by_run_group:
+ await _refuse_if_pinned_by_run_group(session, workflow_id)
if title is not None:
workflow.title = title
diff --git a/skyvern/forge/sdk/routes/__init__.py b/skyvern/forge/sdk/routes/__init__.py
index 87d2dfcfd..e3c373c39 100644
--- a/skyvern/forge/sdk/routes/__init__.py
+++ b/skyvern/forge/sdk/routes/__init__.py
@@ -19,6 +19,7 @@ from skyvern.forge.sdk.routes import self_heal # noqa: F401
from skyvern.forge.sdk.routes import streaming # noqa: F401
from skyvern.forge.sdk.routes import ui_session # noqa: F401
from skyvern.forge.sdk.routes import webhooks # noqa: F401
+from skyvern.forge.sdk.routes import workflow_run_groups # noqa: F401
from skyvern.forge.sdk.routes import workflow_schedules # noqa: F401
from skyvern.forge.sdk.routes.streaming import cdp_input # noqa: F401
from skyvern.forge.sdk.routes.streaming import messages # noqa: F401
diff --git a/skyvern/forge/sdk/routes/workflow_run_groups.py b/skyvern/forge/sdk/routes/workflow_run_groups.py
new file mode 100644
index 000000000..4a871ee3b
--- /dev/null
+++ b/skyvern/forge/sdk/routes/workflow_run_groups.py
@@ -0,0 +1,44 @@
+from fastapi import Depends
+
+from skyvern.forge.sdk.core.permissions.permission_checker_factory import PermissionCheckerFactory
+from skyvern.forge.sdk.routes.routers import base_router, legacy_base_router
+from skyvern.forge.sdk.schemas.organizations import Organization
+from skyvern.forge.sdk.services import org_auth_service
+from skyvern.schemas.workflow_run_groups import WorkflowRunGroupCreateRequest, WorkflowRunGroupResponse
+from skyvern.services import workflow_run_group_service
+
+
+@legacy_base_router.post("/workflow_run_groups", include_in_schema=False)
+@base_router.post("/workflow_run_groups", response_model=WorkflowRunGroupResponse, include_in_schema=False)
+async def submit_workflow_run_group(
+ body: WorkflowRunGroupCreateRequest,
+ organization: Organization = Depends(org_auth_service.get_current_org),
+) -> WorkflowRunGroupResponse:
+ await PermissionCheckerFactory.get_instance().check(organization)
+ return await workflow_run_group_service.submit_workflow_run_group(organization, body)
+
+
+@legacy_base_router.get("/workflow_run_groups/{workflow_run_group_id}", include_in_schema=False)
+@base_router.get(
+ "/workflow_run_groups/{workflow_run_group_id}", response_model=WorkflowRunGroupResponse, include_in_schema=False
+)
+async def get_workflow_run_group(
+ workflow_run_group_id: str,
+ organization: Organization = Depends(org_auth_service.get_current_org),
+) -> WorkflowRunGroupResponse:
+ return await workflow_run_group_service.get_workflow_run_group(workflow_run_group_id, organization.organization_id)
+
+
+@legacy_base_router.post("/workflow_run_groups/{workflow_run_group_id}/cancel", include_in_schema=False)
+@base_router.post(
+ "/workflow_run_groups/{workflow_run_group_id}/cancel",
+ response_model=WorkflowRunGroupResponse,
+ include_in_schema=False,
+)
+async def cancel_workflow_run_group(
+ workflow_run_group_id: str,
+ organization: Organization = Depends(org_auth_service.get_current_org),
+) -> WorkflowRunGroupResponse:
+ return await workflow_run_group_service.cancel_workflow_run_group(
+ workflow_run_group_id, organization.organization_id
+ )
diff --git a/skyvern/forge/sdk/services/route_authorization.py b/skyvern/forge/sdk/services/route_authorization.py
index 0eae0fbd8..b10d47d65 100644
--- a/skyvern/forge/sdk/services/route_authorization.py
+++ b/skyvern/forge/sdk/services/route_authorization.py
@@ -260,6 +260,9 @@ ROUTE_ACTIONS: Mapping[str, str] = {
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_message_feedback": "workflow_copilot.manage",
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_question_response": "workflow_copilot.manage",
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_suggest_goal": "workflow_copilot.suggest",
+ "skyvern.forge.sdk.routes.workflow_run_groups.cancel_workflow_run_group": "workflow_run.cancel",
+ "skyvern.forge.sdk.routes.workflow_run_groups.get_workflow_run_group": "workflow_run.read",
+ "skyvern.forge.sdk.routes.workflow_run_groups.submit_workflow_run_group": "workflow.run",
"skyvern.forge.sdk.routes.workflow_schedules.cancel_workflow_schedule": "workflow_schedule.cancel",
"skyvern.forge.sdk.routes.workflow_schedules.create_workflow_schedule": "workflow_schedule.create",
"skyvern.forge.sdk.routes.workflow_schedules.delete_workflow_schedule_route": "workflow_schedule.delete",
diff --git a/skyvern/forge/sdk/workflow/credential_fallback.py b/skyvern/forge/sdk/workflow/credential_fallback.py
index 41103c407..09d23b23e 100644
--- a/skyvern/forge/sdk/workflow/credential_fallback.py
+++ b/skyvern/forge/sdk/workflow/credential_fallback.py
@@ -128,6 +128,13 @@ async def _maybe_start_credential_fallback_retry(workflow_run: WorkflowRun, orga
if not await _retry_enabled_for_organization(organization_id, workflow_run):
return None
+ # A run group moves to its next item when this child fails, so a fallback run would act beside it under
+ # another login. A lookup error raises, and the wrapper then skips the retry.
+ if workflow_run.start_fresh_browser and await app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id(
+ workflow_run.workflow_run_id
+ ):
+ return None
+
is_block_scoped_run = await app.AGENT_FUNCTION.is_block_scoped_workflow_run(workflow_run)
if not is_block_scoped_run:
is_block_scoped_run = await app.DATABASE.debug.has_block_run_for_workflow_run(
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index fca870a84..8ecb82cfb 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -18571,18 +18571,11 @@ class WorkflowTriggerBlock(Block):
) -> list[PARAMETER_TYPE]:
return self.parameters
- async def _check_trigger_depth(self, workflow_run_id: str) -> int:
- """Check the nesting depth of workflow triggers to prevent infinite recursion.
-
- Note: This depth guard walks the parent_workflow_run_id chain, which is only
- populated for synchronous triggers. For async (fire-and-forget) dispatch, the
- parent may have already completed before the child runs, so circular async
- chains (A->B->A) are only blocked while A is still running. A full
- visited-workflow guard would require persistent state and is left as a future
- enhancement.
- """
+ async def _check_trigger_depth(self, workflow_run_id: str) -> str:
+ """Return the root run of the parent_workflow_run_id chain, raising past MAX_TRIGGER_DEPTH. An async child's
+ parent may finish first, so a circular async chain (A->B->A) is blocked only while A is still running."""
depth = 0
- current_run_id: str | None = workflow_run_id
+ current_run_id = workflow_run_id
while current_run_id:
if depth >= self.MAX_TRIGGER_DEPTH:
raise InvalidWorkflowDefinition(
@@ -18594,7 +18587,7 @@ class WorkflowTriggerBlock(Block):
break
current_run_id = run.parent_workflow_run_id
depth += 1
- return depth
+ return current_run_id
def _render_template_value(
self,
@@ -18817,9 +18810,16 @@ class WorkflowTriggerBlock(Block):
# 2. Check recursion depth
try:
- await self._check_trigger_depth(workflow_run_id)
+ root_workflow_run_id = await self._check_trigger_depth(workflow_run_id)
except InvalidWorkflowDefinition as e:
return await _fail(str(e))
+ if (
+ not self.wait_for_completion or self.browser_session_id
+ ) and await app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id(root_workflow_run_id):
+ return await _fail(
+ "A workflow run group child cannot trigger a workflow without waiting for it"
+ " or into a supplied browser session"
+ )
# 3. Get the organization
if not organization_id:
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 68a4695e2..ef376c8fa 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -11160,6 +11160,7 @@ class WorkflowService:
created_by=created_by,
edited_by=edited_by,
preserve_completion_contract=preserve_completion_contract,
+ refuse_if_pinned_by_run_group=True,
)
else:
updated_workflow = await app.DATABASE.workflows.update_workflow(
@@ -11193,6 +11194,7 @@ class WorkflowService:
sequential_key=sequential_key,
created_by=created_by,
edited_by=edited_by,
+ refuse_if_pinned_by_run_group=True,
)
if notify_workflow_saved:
@@ -12056,6 +12058,10 @@ class WorkflowService:
task = asyncio.create_task(hook_coroutine)
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
+ try:
+ app.AGENT_FUNCTION.schedule_workflow_run_group_advance(workflow_run)
+ except Exception:
+ LOG.warning("Failed to schedule workflow run group advance", workflow_run_id=workflow_run_id, exc_info=True)
async def _run_workflow_run_terminal_hooks(
self,
@@ -12869,6 +12875,7 @@ class WorkflowService:
failure_category: list[dict] | None = None,
finalized_by: str | None = None,
run_minutes_recorded_through: datetime | None = None,
+ only_from: Sequence[WorkflowRunStatus] | None = None,
) -> WorkflowRun | None:
""":meth:`_update_workflow_run_status` for writers that must lose a race against the
run's own finalizer. Returns ``None`` when the row was already terminal."""
@@ -12878,6 +12885,7 @@ class WorkflowService:
failure_reason=failure_reason,
run_with=run_with,
failure_category=failure_category,
+ only_from=only_from,
)
if workflow_run is None:
return None
@@ -13482,6 +13490,7 @@ class WorkflowService:
failure_reason: str | None,
failure_category: list[dict] | None = None,
cascade_children: bool = False,
+ only_from: Sequence[WorkflowRunStatus] | None = None,
) -> WorkflowRun | None:
"""Conditional failure finalize that no-ops when the run has already reached a terminal
state. Out-of-band finalizers (the interrupted-activity backstop) run concurrently with
@@ -13500,6 +13509,7 @@ class WorkflowService:
status=WorkflowRunStatus.failed,
failure_reason=failure_reason,
failure_category=failure_category,
+ only_from=only_from,
)
if workflow_run is None:
return None
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 502654bc6..5279b5bce 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -6946,7 +6946,8 @@ _FIND_MENU_JS = (
const tag = el.tagName;
const box = rowBox(el);
if (!box) {
- if (arg.saltUnlisted && vis(el.getBoundingClientRect()) && (el.innerText || '').trim() && ownClickable(el)) unreadLong.push(el);
+ // innerText walks the subtree, so it runs after the cheaper checks that reject most containers.
+ if (arg.saltUnlisted && vis(el.getBoundingClientRect()) && ownClickable(el) && (el.innerText || '').trim()) unreadLong.push(el);
continue;
}
const { r, txt } = box;
diff --git a/skyvern/schemas/workflow_run_groups.py b/skyvern/schemas/workflow_run_groups.py
new file mode 100644
index 000000000..08654e035
--- /dev/null
+++ b/skyvern/schemas/workflow_run_groups.py
@@ -0,0 +1,127 @@
+from datetime import datetime
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from skyvern.constants import SCRUBBED_VALUE
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
+
+MAX_WORKFLOW_RUN_GROUP_ITEMS = 25
+SCRUBBED_SUBMISSION_KEY_PREFIX = "scrubbed:"
+
+
+class WorkflowRunGroupStatus(StrEnum):
+ active = "active"
+ cancel_requested = "cancel_requested"
+ finished = "finished"
+
+
+class WorkflowRunGroupItemState(StrEnum):
+ pending = "pending"
+ dispatching = "dispatching"
+ dispatched = "dispatched"
+ done = "done"
+ canceled = "canceled"
+ failed_to_start = "failed_to_start"
+
+ def is_final(self) -> bool:
+ return self in (
+ WorkflowRunGroupItemState.done,
+ WorkflowRunGroupItemState.canceled,
+ WorkflowRunGroupItemState.failed_to_start,
+ )
+
+
+IN_FLIGHT_ITEM_STATES = (WorkflowRunGroupItemState.dispatching, WorkflowRunGroupItemState.dispatched)
+
+
+class WorkflowRunGroupItemOutcome(StrEnum):
+ pending = "pending"
+ in_progress = "in_progress"
+ completed = "completed"
+ failed = "failed"
+ unknown = "unknown"
+ canceled = "canceled"
+
+
+class WorkflowRunGroup(BaseModel):
+ model_config = ConfigDict(from_attributes=True)
+
+ workflow_run_group_id: str
+ organization_id: str
+ workflow_permanent_id: str
+ requested_version: int | None
+ workflow_id: str
+ submission_key: str
+ input_fingerprint: str
+ status: WorkflowRunGroupStatus
+ created_at: datetime
+ modified_at: datetime
+ finished_at: datetime | None
+
+
+class WorkflowRunGroupItem(BaseModel):
+ model_config = ConfigDict(from_attributes=True)
+
+ workflow_run_group_id: str
+ position: int
+ item_key: str
+ parameters: dict[str, Any]
+ workflow_run_id: str
+ state: WorkflowRunGroupItemState
+ dispatch_token: str | None
+ claimed_at: datetime | None
+ dispatch_attempts: int
+ failure_reason: str | None
+
+
+class WorkflowRunGroupItemRequest(BaseModel):
+ # Forbidding extra fields rejects per-item browser session, address or profile inputs.
+ model_config = ConfigDict(extra="forbid")
+
+ key: str = Field(min_length=1, max_length=255)
+ parameters: dict[str, Any] = Field(default_factory=dict)
+
+
+class WorkflowRunGroupCreateRequest(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ workflow_id: str = Field(description="The workflow permanent id (wpid_...) every item runs against.")
+ version: int | None = Field(default=None, ge=1)
+ submission_key: str = Field(min_length=1, max_length=255)
+ items: list[WorkflowRunGroupItemRequest] = Field(min_length=1, max_length=MAX_WORKFLOW_RUN_GROUP_ITEMS)
+
+ @model_validator(mode="after")
+ def _validate_keys(self) -> "WorkflowRunGroupCreateRequest":
+ keys = [item.key for item in self.items]
+ if len(set(keys)) != len(keys):
+ raise ValueError("item keys must be unique within a group")
+ if self.submission_key.startswith(SCRUBBED_SUBMISSION_KEY_PREFIX):
+ raise ValueError(f"submission_key cannot start with {SCRUBBED_SUBMISSION_KEY_PREFIX!r}")
+ # The retention scrub renames item keys into this space, and a caller key already there would collide.
+ if any(key.startswith(SCRUBBED_VALUE) for key in keys):
+ raise ValueError(f"item keys cannot start with {SCRUBBED_VALUE!r}")
+ return self
+
+
+class WorkflowRunGroupItemResponse(BaseModel):
+ key: str
+ position: int
+ workflow_run_id: str
+ state: WorkflowRunGroupItemState
+ run_status: WorkflowRunStatus | None
+ outcome: WorkflowRunGroupItemOutcome
+ failure_reason: str | None
+
+
+class WorkflowRunGroupResponse(BaseModel):
+ workflow_run_group_id: str
+ workflow_permanent_id: str
+ workflow_id: str
+ requested_version: int | None
+ submission_key: str
+ status: WorkflowRunGroupStatus
+ created_at: datetime
+ finished_at: datetime | None
+ items: list[WorkflowRunGroupItemResponse]
diff --git a/skyvern/services/workflow_run_group_service.py b/skyvern/services/workflow_run_group_service.py
new file mode 100644
index 000000000..4ecbb271d
--- /dev/null
+++ b/skyvern/services/workflow_run_group_service.py
@@ -0,0 +1,649 @@
+from __future__ import annotations
+
+import asyncio
+import contextvars
+import hmac
+import json
+import uuid
+from collections.abc import Coroutine
+from datetime import datetime, timedelta
+from hashlib import sha256
+from http import HTTPStatus
+from typing import Any
+
+import structlog
+from fastapi import HTTPException
+from sqlalchemy.exc import IntegrityError
+
+from skyvern.config import settings
+from skyvern.exceptions import SkyvernHTTPException, WorkflowHasNoBlocks
+from skyvern.forge import app
+from skyvern.forge.sdk.core.permissions.permission_checker_factory import PermissionCheckerFactory
+from skyvern.forge.sdk.db.datetime_utils import naive_utc_now, to_naive_utc
+from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType
+from skyvern.forge.sdk.db.repositories.workflow_run_groups import scrubbed_submission_key
+from skyvern.forge.sdk.executor import factory as executor_factory
+from skyvern.forge.sdk.schemas.organizations import Organization
+from skyvern.forge.sdk.workflow.models.block import (
+ BaseTaskBlock,
+ HumanInteractionBlock,
+ WorkflowTriggerBlock,
+ get_all_blocks,
+)
+from skyvern.forge.sdk.workflow.models.workflow import (
+ Workflow,
+ WorkflowRequestBody,
+ WorkflowRun,
+ WorkflowRunStatus,
+)
+from skyvern.forge.sdk.workflow.retry_policy import resolve_workflow_retry_policy
+from skyvern.schemas.workflow_run_groups import (
+ IN_FLIGHT_ITEM_STATES,
+ WorkflowRunGroup,
+ WorkflowRunGroupCreateRequest,
+ WorkflowRunGroupItem,
+ WorkflowRunGroupItemOutcome,
+ WorkflowRunGroupItemResponse,
+ WorkflowRunGroupItemState,
+ WorkflowRunGroupResponse,
+ WorkflowRunGroupStatus,
+)
+from skyvern.services import run_service, workflow_service
+from skyvern.services.organization_log_scope import organization_log_scope
+
+LOG = structlog.get_logger()
+
+DISPATCH_GRACE = timedelta(minutes=10)
+MAX_DISPATCH_ATTEMPTS = 3
+# ponytail: a fixed settle window, because a group child records no "worker stopped" signal (it has no attempt row).
+# It outlasts the longest step that ignores a cancel (a code block); a worker slower than that can still overlap.
+CHILD_STOP_SETTLE = timedelta(seconds=settings.CODE_BLOCK_EXECUTION_TIMEOUT_SECONDS + 60)
+# An outside writer (a run cancel, the stuck-run reaper) can set these while the child's worker is still running.
+_STATUSES_SET_BESIDE_A_LIVE_WORKER = (WorkflowRunStatus.canceled, WorkflowRunStatus.timed_out)
+RECOVERY_BATCH_SIZE = 200
+RECOVERY_INTERVAL_SECONDS = 60
+_FINGERPRINT_DOMAIN = b"skyvern.workflow_run_group_submission.v1\0"
+_background_tasks: set[asyncio.Task[None]] = set()
+
+
+def submission_fingerprint(request: WorkflowRunGroupCreateRequest) -> str:
+ payload = {
+ "workflow_id": request.workflow_id,
+ "version": request.version,
+ "items": [{"key": item.key, "parameters": item.parameters} for item in request.items],
+ }
+ serialized = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+ return hmac.new(settings.SECRET_KEY.encode(), _FINGERPRINT_DOMAIN + serialized.encode(), sha256).hexdigest()
+
+
+def non_fresh_browser_reason(workflow: Workflow) -> str | None:
+ if resolve_workflow_retry_policy(workflow) is not None:
+ return "a retry policy, which could replay a child whose effects already happened"
+ if workflow.persist_browser_session:
+ return "persist_browser_session"
+ if workflow.browser_profile_id or workflow.browser_profile_key:
+ return "a browser profile"
+ if workflow.sequential_key:
+ return "a sequential_key"
+ if workflow.workflow_definition.finally_block_label:
+ return "a finally block, which runs after the run first reports a final status"
+ for block in get_all_blocks(workflow.workflow_definition.blocks):
+ if not isinstance(block, WorkflowTriggerBlock):
+ continue
+ if block.browser_session_id:
+ return f"block {block.label} triggering a workflow into a supplied browser_session_id"
+ if not block.wait_for_completion:
+ return (
+ f"block {block.label} triggering a workflow without waiting for it, so that run could outlive its item"
+ )
+ return None
+
+
+async def _existing_group_or_conflict(
+ group: WorkflowRunGroup, fingerprint: str, organization_id: str
+) -> WorkflowRunGroupResponse:
+ if not hmac.compare_digest(group.input_fingerprint, fingerprint):
+ raise SkyvernHTTPException(
+ "submission_key was already used with different input",
+ status_code=HTTPStatus.CONFLICT,
+ )
+ return await get_workflow_run_group(group.workflow_run_group_id, organization_id)
+
+
+async def submit_workflow_run_group(
+ organization: Organization,
+ request: WorkflowRunGroupCreateRequest,
+ *,
+ expected_workflow_modified_at: datetime | None = None,
+) -> WorkflowRunGroupResponse:
+ organization_id = organization.organization_id
+ fingerprint = submission_fingerprint(request)
+ existing = await app.DATABASE.workflow_run_groups.get_group_by_submission_key(
+ organization_id, request.submission_key
+ )
+ if existing is not None:
+ return await _existing_group_or_conflict(existing, fingerprint, organization_id)
+ if await app.DATABASE.workflow_run_groups.get_group_by_submission_key(
+ organization_id, scrubbed_submission_key(request.submission_key)
+ ):
+ raise SkyvernHTTPException(
+ "submission_key belongs to a group whose records were removed by the organization's retention policy",
+ status_code=HTTPStatus.CONFLICT,
+ )
+ if not settings.WORKFLOW_RUN_GROUPS_SUBMIT_ENABLED:
+ raise SkyvernHTTPException(
+ "Workflow run group submission is disabled", status_code=HTTPStatus.SERVICE_UNAVAILABLE
+ )
+ await app.RATE_LIMITER.rate_limit_submit_run(organization_id)
+
+ workflow = await app.WORKFLOW_SERVICE.get_workflow_by_permanent_id(
+ workflow_permanent_id=request.workflow_id,
+ organization_id=organization_id,
+ version=request.version,
+ )
+ if not workflow.workflow_definition.blocks:
+ raise WorkflowHasNoBlocks(workflow_permanent_id=workflow.workflow_permanent_id)
+ if reason := non_fresh_browser_reason(workflow):
+ raise SkyvernHTTPException(
+ f"Workflow version {workflow.version} cannot run as a group because it has {reason}",
+ status_code=HTTPStatus.BAD_REQUEST,
+ )
+ input_parameters = {
+ parameter.key: parameter
+ for parameter in await app.WORKFLOW_SERVICE.get_workflow_parameters(workflow_id=workflow.workflow_id)
+ }
+ for item in request.items:
+ await app.WORKFLOW_SERVICE.validate_schedule_parameters(workflow, organization, item.parameters)
+ # Values are otherwise converted only when a child is created, so a mistyped one would fail mid-group.
+ for key, value in item.parameters.items():
+ if value is not None and key in input_parameters:
+ app.WORKFLOW_SERVICE._serialize_workflow_run_parameter_value(input_parameters[key], value)
+
+ try:
+ group = await app.DATABASE.workflow_run_groups.create_group(
+ organization_id=organization_id,
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ requested_version=request.version,
+ workflow_id=workflow.workflow_id,
+ submission_key=request.submission_key,
+ input_fingerprint=fingerprint,
+ items=[(item.key, item.parameters) for item in request.items],
+ # Without a reviewed timestamp, bind to the row validation read so an edit landing after it is refused.
+ expected_workflow_modified_at=expected_workflow_modified_at or workflow.modified_at,
+ )
+ except IntegrityError:
+ raced = await app.DATABASE.workflow_run_groups.get_group_by_submission_key(
+ organization_id, request.submission_key
+ )
+ if raced is None:
+ raise
+ return await _existing_group_or_conflict(raced, fingerprint, organization_id)
+
+ LOG.info(
+ "Workflow run group submitted",
+ workflow_run_group_id=group.workflow_run_group_id,
+ workflow_id=group.workflow_id,
+ item_count=len(request.items),
+ )
+ _spawn(_advance_in_org_scope(group.workflow_run_group_id, organization_id))
+ return await get_workflow_run_group(group.workflow_run_group_id, organization_id)
+
+
+def _is_task_family_only(workflow: Workflow | None) -> bool:
+ if workflow is None or not workflow.workflow_definition.blocks:
+ return False
+ # Task-family blocks act only through recorded steps; a human interaction block sends email without one.
+ return all(
+ isinstance(block, BaseTaskBlock) and not isinstance(block, HumanInteractionBlock)
+ for block in workflow.workflow_definition.blocks
+ )
+
+
+def classify_item_outcome(
+ item_state: WorkflowRunGroupItemState,
+ child: WorkflowRun | None,
+ *,
+ task_family_only: bool,
+ step_count: int | None,
+) -> WorkflowRunGroupItemOutcome:
+ if item_state == WorkflowRunGroupItemState.failed_to_start:
+ # A child can already have run to a final status before its item was failed; its effects are then unknown.
+ if child is not None and child.started_at is not None:
+ return WorkflowRunGroupItemOutcome.unknown
+ return WorkflowRunGroupItemOutcome.failed
+ if item_state == WorkflowRunGroupItemState.pending:
+ return WorkflowRunGroupItemOutcome.pending
+ if child is None:
+ if item_state == WorkflowRunGroupItemState.canceled:
+ return WorkflowRunGroupItemOutcome.canceled
+ return WorkflowRunGroupItemOutcome.in_progress
+ if not child.status.is_final():
+ if item_state == WorkflowRunGroupItemState.canceled:
+ return WorkflowRunGroupItemOutcome.canceled
+ return WorkflowRunGroupItemOutcome.in_progress
+ if child.status == WorkflowRunStatus.completed:
+ return WorkflowRunGroupItemOutcome.completed
+ if child.status == WorkflowRunStatus.canceled and child.started_at is None:
+ return WorkflowRunGroupItemOutcome.canceled
+ if (
+ child.status in (WorkflowRunStatus.failed, WorkflowRunStatus.terminated)
+ and task_family_only
+ and step_count == 0
+ and child.script_run is None
+ ):
+ return WorkflowRunGroupItemOutcome.failed
+ return WorkflowRunGroupItemOutcome.unknown
+
+
+async def get_workflow_run_group(workflow_run_group_id: str, organization_id: str) -> WorkflowRunGroupResponse:
+ group = await app.DATABASE.workflow_run_groups.get_group(workflow_run_group_id, organization_id)
+ if group is None:
+ raise SkyvernHTTPException(
+ f"Workflow run group {workflow_run_group_id} not found", status_code=HTTPStatus.NOT_FOUND
+ )
+ items = await app.DATABASE.workflow_run_groups.get_items(workflow_run_group_id)
+ children = {
+ run.workflow_run_id: run
+ for run in await app.DATABASE.workflow_runs.get_workflow_runs_by_ids(
+ [item.workflow_run_id for item in items], organization_id=organization_id
+ )
+ }
+ task_family_only: bool | None = None
+ item_responses = []
+ for item in items:
+ child = children.get(item.workflow_run_id)
+ step_count = None
+ if child is not None and child.status in (WorkflowRunStatus.failed, WorkflowRunStatus.terminated):
+ if task_family_only is None:
+ task_family_only = _is_task_family_only(
+ await app.DATABASE.workflows.get_workflow(group.workflow_id, organization_id)
+ )
+ if task_family_only:
+ step_count = await app.DATABASE.workflow_run_groups.count_steps(item.workflow_run_id)
+ item_responses.append(
+ WorkflowRunGroupItemResponse(
+ key=item.item_key,
+ position=item.position,
+ workflow_run_id=item.workflow_run_id,
+ state=item.state,
+ run_status=child.status if child else None,
+ outcome=classify_item_outcome(
+ item.state, child, task_family_only=bool(task_family_only), step_count=step_count
+ ),
+ failure_reason=item.failure_reason or (child.failure_reason if child else None),
+ )
+ )
+ return WorkflowRunGroupResponse(
+ workflow_run_group_id=group.workflow_run_group_id,
+ workflow_permanent_id=group.workflow_permanent_id,
+ workflow_id=group.workflow_id,
+ requested_version=group.requested_version,
+ submission_key=group.submission_key,
+ status=group.status,
+ created_at=group.created_at,
+ finished_at=group.finished_at,
+ items=item_responses,
+ )
+
+
+async def cancel_workflow_run_group(workflow_run_group_id: str, organization_id: str) -> WorkflowRunGroupResponse:
+ group = await app.DATABASE.workflow_run_groups.request_cancel(workflow_run_group_id, organization_id)
+ if group is None:
+ raise SkyvernHTTPException(
+ f"Workflow run group {workflow_run_group_id} not found", status_code=HTTPStatus.NOT_FOUND
+ )
+ await advance_workflow_run_group(workflow_run_group_id)
+ return await get_workflow_run_group(workflow_run_group_id, organization_id)
+
+
+async def _cancel_child(workflow_run_id: str, organization_id: str) -> None:
+ try:
+ child = await app.DATABASE.workflow_runs.get_workflow_run(workflow_run_id, organization_id=organization_id)
+ # On a final run, cancel_workflow_run still resends the webhook, deletes attached files and revokes the attempt.
+ if child is None or child.status.is_final():
+ return
+ await run_service.cancel_workflow_run(workflow_run_id, organization_id)
+ except Exception:
+ LOG.warning("Failed to cancel workflow run group child", workflow_run_id=workflow_run_id, exc_info=True)
+
+
+async def _fail_unstarted_child(workflow_run_id: str, failure_reason: str) -> bool:
+ """Fail a child that is still `created`, meaning no executor ever took it. False when it moved on or is missing."""
+ failed = await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(
+ workflow_run_id=workflow_run_id,
+ failure_reason=failure_reason,
+ only_from=(WorkflowRunStatus.created,),
+ )
+ return failed is not None
+
+
+async def _fail_item_to_start(
+ item: WorkflowRunGroupItem,
+ dispatch_token: str,
+ failure_reason: str,
+ organization_id: str,
+ from_state: WorkflowRunGroupItemState = WorkflowRunGroupItemState.dispatching,
+) -> None:
+ # Holding the item in flight first fences out a stale owner whose token was reclaimed, and keeps the next item
+ # from starting while an executor that took this child in the meantime may still be running it.
+ if not await app.DATABASE.workflow_run_groups.finish_item(
+ item.workflow_run_group_id,
+ item.position,
+ state=WorkflowRunGroupItemState.dispatched,
+ from_states=(from_state,),
+ dispatch_token=dispatch_token,
+ ):
+ return
+ if not await _fail_unstarted_child(item.workflow_run_id, failure_reason):
+ child = await app.DATABASE.workflow_runs.get_workflow_run(item.workflow_run_id, organization_id=organization_id)
+ if child is not None and not child.status.is_final():
+ # Reconcile releases the item once the canceled child has stopped and settled.
+ await _cancel_child(item.workflow_run_id, organization_id)
+ return
+ if child is not None and _worker_may_still_be_stopping(child):
+ return
+ # The failed child's terminal hook may already have marked the item done.
+ await app.DATABASE.workflow_run_groups.finish_item(
+ item.workflow_run_group_id,
+ item.position,
+ state=WorkflowRunGroupItemState.failed_to_start,
+ from_states=(WorkflowRunGroupItemState.dispatched, WorkflowRunGroupItemState.done),
+ dispatch_token=dispatch_token,
+ failure_reason=failure_reason,
+ )
+
+
+def _failure_reason(message: str, error: Exception) -> str:
+ # Exception text can echo item parameters such as credential ids, and this reason is stored and returned.
+ return f"{message} ({type(error).__name__})"
+
+
+async def _dispatch_item(group: WorkflowRunGroup, item: WorkflowRunGroupItem, dispatch_token: str) -> bool:
+ """Prepare and execute one claimed item. Returns True while the item stays in flight."""
+ # setup_workflow_run replaces the calling task's SkyvernContext and inherits its run ids, so a dispatch that shared
+ # its caller's context would hand the previous child's ids to this one.
+ return await asyncio.create_task(
+ _prepare_and_execute_item_in_org_scope(group, item, dispatch_token), context=contextvars.Context()
+ )
+
+
+async def _prepare_and_execute_item_in_org_scope(
+ group: WorkflowRunGroup, item: WorkflowRunGroupItem, dispatch_token: str
+) -> bool:
+ async with organization_log_scope(group.organization_id):
+ return await _prepare_and_execute_item(group, item, dispatch_token)
+
+
+async def _prepare_and_execute_item(group: WorkflowRunGroup, item: WorkflowRunGroupItem, dispatch_token: str) -> bool:
+ organization = await app.DATABASE.organizations.get_organization(group.organization_id)
+ if organization is None:
+ await _fail_item_to_start(item, dispatch_token, "Organization not found", group.organization_id)
+ return False
+ try:
+ await PermissionCheckerFactory.get_instance().check(organization)
+ workflow_run = await workflow_service.prepare_workflow(
+ workflow_id=group.workflow_permanent_id,
+ organization=organization,
+ workflow_request=WorkflowRequestBody(
+ data=item.parameters, start_fresh_browser=True, reuse_browser_session=False
+ ),
+ trigger_type=WorkflowRunTriggerType.api,
+ workflow_run_id=item.workflow_run_id,
+ resolved_workflow_id=group.workflow_id,
+ )
+ except IntegrityError:
+ # The task_run row is written last, so its presence means an earlier preparation completed.
+ if await app.DATABASE.tasks.get_run(item.workflow_run_id, organization_id=group.organization_id) is None:
+ await _fail_item_to_start(
+ item, dispatch_token, "Workflow run preparation was interrupted", group.organization_id
+ )
+ return False
+ existing_run = await app.DATABASE.workflow_runs.get_workflow_run(
+ item.workflow_run_id, organization_id=group.organization_id
+ )
+ if existing_run is None:
+ await _fail_item_to_start(
+ item, dispatch_token, "Workflow run preparation was interrupted", group.organization_id
+ )
+ return False
+ workflow_run = existing_run
+ except SkyvernHTTPException as e:
+ LOG.warning(
+ "Workflow run group child could not be prepared",
+ workflow_run_group_id=group.workflow_run_group_id,
+ workflow_run_id=item.workflow_run_id,
+ exc_info=True,
+ )
+ await _fail_item_to_start(
+ item, dispatch_token, _failure_reason("Workflow run could not be prepared", e), group.organization_id
+ )
+ return False
+ except HTTPException as e:
+ LOG.warning(
+ "Organization cannot start workflow run group child",
+ workflow_run_group_id=group.workflow_run_group_id,
+ workflow_run_id=item.workflow_run_id,
+ exc_info=True,
+ )
+ await _fail_item_to_start(
+ item, dispatch_token, _failure_reason("Organization cannot start runs", e), group.organization_id
+ )
+ return False
+ except Exception:
+ LOG.exception(
+ "Failed to prepare workflow run group child",
+ workflow_run_group_id=group.workflow_run_group_id,
+ workflow_run_id=item.workflow_run_id,
+ dispatch_attempts=item.dispatch_attempts,
+ )
+ if item.dispatch_attempts >= MAX_DISPATCH_ATTEMPTS:
+ await _fail_item_to_start(
+ item, dispatch_token, "Workflow run preparation kept failing", group.organization_id
+ )
+ return False
+ return True
+
+ if workflow_run.status.is_final():
+ await app.DATABASE.workflow_run_groups.finish_item(
+ group.workflow_run_group_id,
+ item.position,
+ state=WorkflowRunGroupItemState.done,
+ from_states=(WorkflowRunGroupItemState.dispatching,),
+ dispatch_token=dispatch_token,
+ )
+ return False
+
+ flip = await app.DATABASE.workflow_run_groups.flip_to_dispatched(
+ group.workflow_run_group_id, item.position, dispatch_token
+ )
+ if flip == "lost":
+ return True
+ if flip == "canceled":
+ await _cancel_child(item.workflow_run_id, group.organization_id)
+ return False
+
+ LOG.info(
+ "Dispatching workflow run group child",
+ workflow_run_group_id=group.workflow_run_group_id,
+ workflow_run_id=item.workflow_run_id,
+ position=item.position,
+ )
+ try:
+ await executor_factory.AsyncExecutorFactory.get_executor().execute_workflow(
+ request=None,
+ background_tasks=None,
+ organization=organization,
+ workflow_id=workflow_run.workflow_id,
+ workflow_run_id=workflow_run.workflow_run_id,
+ workflow_permanent_id=workflow_run.workflow_permanent_id,
+ max_steps_override=None,
+ api_key=None,
+ browser_session_id=workflow_run.browser_session_id,
+ block_labels=None,
+ block_outputs=None,
+ )
+ except Exception:
+ LOG.exception(
+ "Failed to execute workflow run group child",
+ workflow_run_group_id=group.workflow_run_group_id,
+ workflow_run_id=item.workflow_run_id,
+ )
+ return True
+
+
+def _worker_may_still_be_stopping(child: WorkflowRun) -> bool:
+ if child.status not in _STATUSES_SET_BESIDE_A_LIVE_WORKER:
+ return False
+ # The reaper's bulk write sets the status without finished_at, so fall back to the row's last write.
+ ended_at = to_naive_utc(child.finished_at or child.modified_at)
+ return ended_at is None or ended_at > naive_utc_now() - CHILD_STOP_SETTLE
+
+
+async def _reconcile_in_flight(group: WorkflowRunGroup, items: list[WorkflowRunGroupItem]) -> None:
+ in_flight = [item for item in items if item.state in IN_FLIGHT_ITEM_STATES]
+ canceling = group.status != WorkflowRunGroupStatus.active
+ if canceling:
+ in_flight += [item for item in items if item.state == WorkflowRunGroupItemState.canceled]
+ if not in_flight:
+ return
+ children = {
+ run.workflow_run_id: run
+ for run in await app.DATABASE.workflow_runs.get_workflow_runs_by_ids(
+ [item.workflow_run_id for item in in_flight], organization_id=group.organization_id
+ )
+ }
+ stale_before = naive_utc_now() - DISPATCH_GRACE
+ for item in in_flight:
+ child = children.get(item.workflow_run_id)
+ claimed_at = to_naive_utc(item.claimed_at)
+ is_stale = claimed_at is not None and claimed_at < stale_before
+ if child is not None and not child.status.is_final() and canceling:
+ await _cancel_child(item.workflow_run_id, group.organization_id)
+ child = await app.DATABASE.workflow_runs.get_workflow_run(item.workflow_run_id)
+ if item.state == WorkflowRunGroupItemState.canceled:
+ continue
+ if child is not None and child.status.is_final():
+ # A canceling group starts nothing after this item, so only an active one has to wait out the settle.
+ if not canceling and _worker_may_still_be_stopping(child):
+ continue
+ await app.DATABASE.workflow_run_groups.finish_item(
+ group.workflow_run_group_id,
+ item.position,
+ state=WorkflowRunGroupItemState.done,
+ from_states=IN_FLIGHT_ITEM_STATES,
+ )
+ continue
+ if not is_stale:
+ continue
+ if item.state == WorkflowRunGroupItemState.dispatched:
+ # Only a child no executor ever took is failed; a queued or running one is left to the stuck-run reaper.
+ # A dispatched item with no child row was held by a failure that stopped before releasing it.
+ unstarted = child is None or child.status == WorkflowRunStatus.created
+ if unstarted and item.dispatch_token is not None:
+ await _fail_item_to_start(
+ item,
+ item.dispatch_token,
+ "Workflow run dispatch was interrupted",
+ group.organization_id,
+ from_state=WorkflowRunGroupItemState.dispatched,
+ )
+ continue
+ dispatch_token = uuid.uuid4().hex
+ reclaimed = await app.DATABASE.workflow_run_groups.reclaim_stale_item(
+ group.workflow_run_group_id,
+ item.position,
+ dispatch_token=dispatch_token,
+ stale_before=stale_before,
+ )
+ if reclaimed is None:
+ continue
+ if reclaimed.dispatch_attempts > MAX_DISPATCH_ATTEMPTS:
+ await _fail_item_to_start(
+ reclaimed, dispatch_token, "Workflow run dispatch kept stalling", group.organization_id
+ )
+ continue
+ if canceling:
+ await _cancel_child(item.workflow_run_id, group.organization_id)
+ await app.DATABASE.workflow_run_groups.finish_item(
+ group.workflow_run_group_id,
+ item.position,
+ state=WorkflowRunGroupItemState.canceled,
+ from_states=(WorkflowRunGroupItemState.dispatching,),
+ dispatch_token=dispatch_token,
+ )
+ continue
+ await _dispatch_item(group, reclaimed, dispatch_token)
+
+
+async def advance_workflow_run_group(workflow_run_group_id: str) -> None:
+ group = await app.DATABASE.workflow_run_groups.get_group(workflow_run_group_id)
+ if group is None or group.status == WorkflowRunGroupStatus.finished:
+ return
+ await _reconcile_in_flight(group, await app.DATABASE.workflow_run_groups.get_items(workflow_run_group_id))
+ while True:
+ dispatch_token = uuid.uuid4().hex
+ item = await app.DATABASE.workflow_run_groups.claim_next_item(workflow_run_group_id, dispatch_token)
+ if item is None or await _dispatch_item(group, item, dispatch_token):
+ break
+ await app.DATABASE.workflow_run_groups.finish_group_if_complete(workflow_run_group_id)
+
+
+async def _advance_in_org_scope(workflow_run_group_id: str, organization_id: str) -> None:
+ async with organization_log_scope(organization_id):
+ try:
+ await advance_workflow_run_group(workflow_run_group_id)
+ except Exception:
+ LOG.exception("Failed to advance workflow run group", workflow_run_group_id=workflow_run_group_id)
+
+
+async def _advance_after_child_terminal(workflow_run_id: str, organization_id: str) -> None:
+ try:
+ item = await app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id(workflow_run_id)
+ except Exception:
+ LOG.warning("Failed to look up workflow run group item", workflow_run_id=workflow_run_id, exc_info=True)
+ return
+ if item is not None:
+ await _advance_in_org_scope(item.workflow_run_group_id, organization_id)
+
+
+def _spawn(coroutine: Coroutine[Any, Any, None]) -> None:
+ # An empty context keeps the finishing run's SkyvernContext out of the next child's preparation.
+ task = asyncio.create_task(coroutine, context=contextvars.Context())
+ _background_tasks.add(task)
+ task.add_done_callback(_background_tasks.discard)
+
+
+def schedule_advance_after_terminal(workflow_run: WorkflowRun) -> None:
+ if not workflow_run.start_fresh_browser or workflow_run.parent_workflow_run_id is not None:
+ return
+ _spawn(_advance_after_child_terminal(workflow_run.workflow_run_id, workflow_run.organization_id))
+
+
+async def is_workflow_run_group_child(workflow_run: WorkflowRun) -> bool:
+ if workflow_run.start_fresh_browser is not True:
+ return False
+ try:
+ item = await app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id(workflow_run.workflow_run_id)
+ except Exception:
+ LOG.warning(
+ "Failed to look up workflow run group item", workflow_run_id=workflow_run.workflow_run_id, exc_info=True
+ )
+ # Fail closed: a group child must not reuse a rotated browser, even though this moves a non-group run off its
+ # configured browser addresses for this dispatch.
+ return True
+ return item is not None
+
+
+async def recover_workflow_run_groups() -> None:
+ groups = await app.DATABASE.workflow_run_groups.list_unfinished_groups(RECOVERY_BATCH_SIZE)
+ for workflow_run_group_id, organization_id in groups:
+ await _advance_in_org_scope(workflow_run_group_id, organization_id)
+
+
+async def run_workflow_run_group_recovery_loop() -> None:
+ while True:
+ try:
+ await recover_workflow_run_groups()
+ except Exception:
+ LOG.exception("Workflow run group recovery sweep failed")
+ await asyncio.sleep(RECOVERY_INTERVAL_SECONDS)
diff --git a/tests/unit/force_stub_app.py b/tests/unit/force_stub_app.py
index db11a77ab..ce9f33668 100644
--- a/tests/unit/force_stub_app.py
+++ b/tests/unit/force_stub_app.py
@@ -88,6 +88,7 @@ def create_forge_stub_app() -> ForgeApp:
fake_app_module.AGENT_FUNCTION.is_backup_queue_organization = base_agent_function.is_backup_queue_organization
fake_app_module.AGENT_FUNCTION.resolve_mcp_oauth_org_lookups = MagicMock(return_value=None)
fake_app_module.AGENT_FUNCTION.get_mcp_request_organization_id = MagicMock(return_value=None)
+ fake_app_module.AGENT_FUNCTION.schedule_workflow_run_group_advance = MagicMock(return_value=None)
# Sync method returning a key or None — _LazyNamespace would auto-mock it as a truthy
# AsyncMock and hijack the TextPromptBlock llm_key. Match the OSS no-op.
fake_app_module.AGENT_FUNCTION.get_fallback_llm_key = MagicMock(return_value=None)
diff --git a/tests/unit/test_credential_fallback.py b/tests/unit/test_credential_fallback.py
index 64a1a650f..2b16f4d29 100644
--- a/tests/unit/test_credential_fallback.py
+++ b/tests/unit/test_credential_fallback.py
@@ -65,6 +65,7 @@ def _workflow_run(
extra_http_headers: dict[str, str] | None = None,
workflow_schedule_id: str | None = None,
copilot_session_id: str | None = None,
+ start_fresh_browser: bool | None = None,
) -> WorkflowRun:
now = datetime.now(timezone.utc)
return WorkflowRun(
@@ -86,6 +87,7 @@ def _workflow_run(
extra_http_headers=extra_http_headers,
workflow_schedule_id=workflow_schedule_id,
copilot_session_id=copilot_session_id,
+ start_fresh_browser=start_fresh_browser,
trigger_type=WorkflowRunTriggerType.api,
created_at=now,
modified_at=now,
@@ -269,6 +271,7 @@ async def _run_fallback_retry(
flag_enabled: bool = True,
flag_error: Exception | None = None,
run_tags: dict[str, str] | None = None,
+ run_group_item: object | None = None,
) -> tuple[str | None, AsyncMock, MagicMock]:
captured: dict[str, object] = {}
@@ -297,6 +300,7 @@ async def _run_fallback_retry(
mock_app.AGENT_FUNCTION.is_block_scoped_workflow_run = AsyncMock(return_value=block_scoped)
mock_app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers = _strip_marker_header
mock_app.DATABASE.debug.has_block_run_for_workflow_run = AsyncMock(return_value=False)
+ mock_app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id = AsyncMock(return_value=run_group_item)
mock_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(return_value=_workflow(parameters))
if retried_by_results is not None:
mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by = AsyncMock(side_effect=retried_by_results)
@@ -411,6 +415,24 @@ async def test_mixed_triggers_advance_each_parameter_from_its_own_prior_selectio
assert kwargs["workflow_request"].data["backup_cred"] == "cred_b_fb1"
[email protected]("is_group_item", [False, True])
+async def test_a_run_group_child_never_starts_a_fallback_run(is_group_item: bool) -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}], start_fresh_browser=True)
+
+ result, run_workflow_mock, _ = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[_credential_parameter(fallback_credential_ids=["cred_fb1"])],
+ run_group_item=SimpleNamespace(workflow_run_group_id="wrg_1") if is_group_item else None,
+ )
+
+ if is_group_item:
+ assert result is None
+ run_workflow_mock.assert_not_awaited()
+ else:
+ assert result == "wr_retry"
+
+
@pytest.mark.asyncio
async def test_fallback_exhaustion_creates_no_retry_run() -> None:
workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}], fallback_attempt=2)
diff --git a/tests/unit/test_workflow_run_groups.py b/tests/unit/test_workflow_run_groups.py
new file mode 100644
index 000000000..733e6625d
--- /dev/null
+++ b/tests/unit/test_workflow_run_groups.py
@@ -0,0 +1,1218 @@
+import asyncio
+from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Mapping, Sequence
+from dataclasses import dataclass, field
+from datetime import timedelta
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+import pytest_asyncio
+from fastapi import HTTPException
+from pydantic import ValidationError
+from sqlalchemy import func, select, update
+from sqlalchemy.ext.asyncio import AsyncEngine
+
+from skyvern.config import settings
+from skyvern.exceptions import SkyvernHTTPException, WorkflowChangedSinceReview, WorkflowPinnedByRunGroup
+from skyvern.forge import app
+from skyvern.forge.agent_functions import AgentFunction
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.permissions.permission_checkers import PermissionChecker
+from skyvern.forge.sdk.db.agent_db import AgentDB
+from skyvern.forge.sdk.db.datetime_utils import naive_utc_now
+from skyvern.forge.sdk.db.models import (
+ Base,
+ CredentialModel,
+ WorkflowModel,
+ WorkflowRunGroupItemModel,
+ WorkflowRunGroupModel,
+ WorkflowRunModel,
+)
+from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
+from skyvern.forge.sdk.schemas.organizations import Organization
+from skyvern.forge.sdk.schemas.runs import Run
+from skyvern.forge.sdk.workflow.models.block import (
+ BlockTypeVar,
+ ForLoopBlock,
+ HttpRequestBlock,
+ TaskBlock,
+ WorkflowTriggerBlock,
+)
+from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameterType
+from skyvern.forge.sdk.workflow.models.workflow import (
+ COPILOT_TEST_WORKFLOW_CREATOR,
+ Workflow,
+ WorkflowDefinition,
+ WorkflowRun,
+ WorkflowRunStatus,
+)
+from skyvern.forge.sdk.workflow.service import WorkflowService
+from skyvern.schemas.workflow_run_groups import (
+ WorkflowRunGroupCreateRequest,
+ WorkflowRunGroupItemOutcome,
+ WorkflowRunGroupItemState,
+ WorkflowRunGroupStatus,
+)
+from skyvern.services import workflow_run_group_service as group_service
+from tests.unit.conftest import make_block_output_parameter
+
+ORG = "o_test"
+OTHER_ORG = "o_other"
+WPID = "wpid_test"
+
+
+@dataclass
+class FakeExecutor:
+ database: AgentDB
+ executed: list[str] = field(default_factory=list)
+ submitted: list[str] = field(default_factory=list)
+ before_queue: Callable[[str], Awaitable[None]] | None = None
+
+ async def execute_workflow(self, *, workflow_run_id: str, **_: object) -> None:
+ self.executed.append(workflow_run_id)
+ if self.before_queue is not None:
+ await self.before_queue(workflow_run_id)
+ if await self.database.workflow_runs.update_workflow_run_if_not_final(
+ workflow_run_id, WorkflowRunStatus.queued
+ ):
+ self.submitted.append(workflow_run_id)
+
+
+@dataclass
+class RecordingRateLimiter:
+ calls: list[str] = field(default_factory=list)
+
+ async def rate_limit_submit_run(self, organization_id: str) -> None:
+ self.calls.append(organization_id)
+
+
+@dataclass
+class GroupEnv:
+ database: AgentDB
+ executor: FakeExecutor
+ organization: Organization
+ spawned: list[Coroutine[Any, Any, None]]
+ limiter: RecordingRateLimiter
+
+
+def _definition(*blocks: BlockTypeVar) -> dict[str, Any]:
+ return WorkflowDefinition(parameters=[], blocks=list(blocks)).model_dump(mode="json")
+
+
+def _task_block() -> TaskBlock:
+ return TaskBlock(label="login", url="https://example.com", output_parameter=make_block_output_parameter("login"))
+
+
+def _looped_trigger(browser_session_id: str | None = None, wait_for_completion: bool = True) -> ForLoopBlock:
+ trigger = WorkflowTriggerBlock(
+ label="trigger",
+ workflow_permanent_id="wpid_child",
+ browser_session_id=browser_session_id,
+ wait_for_completion=wait_for_completion,
+ output_parameter=make_block_output_parameter("trigger"),
+ )
+ return ForLoopBlock(label="loop", loop_blocks=[trigger], output_parameter=make_block_output_parameter("loop"))
+
+
+def _http_block() -> HttpRequestBlock:
+ return HttpRequestBlock(
+ label="post", url="https://example.com", method="POST", output_parameter=make_block_output_parameter("post")
+ )
+
+
+@pytest_asyncio.fixture
+async def env(monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine) -> AsyncIterator[GroupEnv]:
+ database = AgentDB("sqlite+aiosqlite://", db_engine=sqlite_engine)
+ organization = await database.organizations.create_organization("Test", organization_id=ORG)
+ await database.organizations.create_organization("Other", organization_id=OTHER_ORG)
+ async with database.Session() as session:
+ session.add(
+ WorkflowModel(
+ workflow_id="wf_1",
+ workflow_permanent_id=WPID,
+ organization_id=ORG,
+ title="Workflow",
+ version=1,
+ workflow_definition=_definition(_task_block()),
+ )
+ )
+ session.add_all(
+ CredentialModel(
+ credential_id=credential_id,
+ organization_id=org_id,
+ name="Login",
+ credential_type="password",
+ item_id=f"item_{credential_id}",
+ )
+ for credential_id, org_id in (("cred_1", ORG), ("cred_2", ORG), ("cred_foreign", OTHER_ORG))
+ )
+ await session.commit()
+ await database.workflow_params.create_workflow_parameter(
+ workflow_id="wf_1", workflow_parameter_type=WorkflowParameterType.CREDENTIAL_ID, key="login", default_value=None
+ )
+ service = WorkflowService()
+ executor = FakeExecutor(database)
+ spawned: list[Coroutine[Any, Any, None]] = []
+ limiter = RecordingRateLimiter()
+ monkeypatch.setattr(app, "DATABASE", database)
+ monkeypatch.setattr(object.__getattribute__(app, "_inst"), "RATE_LIMITER", limiter, raising=False)
+ monkeypatch.setattr(app, "WORKFLOW_SERVICE", service)
+ monkeypatch.setattr(service, "_resolve_managed_browser_profile_for_run_request", AsyncMock(return_value=None))
+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", AsyncMock(return_value=False))
+ monkeypatch.setattr(app.AGENT_FUNCTION, "is_block_scoped_workflow_run", AsyncMock(return_value=False))
+ monkeypatch.setattr(AsyncExecutorFactory, "get_executor", lambda: executor)
+ monkeypatch.setattr(group_service, "_spawn", spawned.append)
+ monkeypatch.setattr(service, "_schedule_workflow_run_terminal_hooks", lambda **_: None)
+ yield GroupEnv(database, executor, organization, spawned, limiter)
+ for coroutine in spawned:
+ coroutine.close()
+ await asyncio.gather(*app.WORKFLOW_SERVICE._background_tasks, return_exceptions=True)
+
+
+def _request(
+ key: str = "sub-1", count: int = 3, items: Sequence[Mapping[str, object]] | None = None
+) -> WorkflowRunGroupCreateRequest:
+ if items is None:
+ items = [{"key": f"acct-{i}", "parameters": {"login": "cred_1" if i % 2 else "cred_2"}} for i in range(count)]
+ return WorkflowRunGroupCreateRequest.model_validate({"workflow_id": WPID, "submission_key": key, "items": items})
+
+
+async def _count(env: GroupEnv, model: type[Base]) -> int:
+ async with env.database.Session() as session:
+ return int(await session.scalar(select(func.count()).select_from(model)) or 0)
+
+
+async def _set_child_status(env: GroupEnv, workflow_run_id: str, status: WorkflowRunStatus) -> None:
+ await env.database.workflow_runs.update_workflow_run(workflow_run_id, status=status)
+
+
+async def _start_nested_run(env: GroupEnv, parent_workflow_run_id: str) -> str:
+ nested = await env.database.workflow_runs.create_workflow_run(
+ workflow_permanent_id=WPID,
+ workflow_id="wf_1",
+ organization_id=ORG,
+ parent_workflow_run_id=parent_workflow_run_id,
+ )
+ await _set_child_status(env, nested.workflow_run_id, WorkflowRunStatus.running)
+ return nested.workflow_run_id
+
+
+async def _status(env: GroupEnv, workflow_run_id: str) -> WorkflowRunStatus | None:
+ run = await env.database.workflow_runs.get_workflow_run(workflow_run_id)
+ return run.status if run else None
+
+
+async def _age_claims(env: GroupEnv) -> None:
+ async with env.database.Session() as session:
+ await session.execute(
+ update(WorkflowRunGroupItemModel).values(claimed_at=naive_utc_now() - timedelta(minutes=30))
+ )
+ await session.commit()
+
+
+async def _submit(env: GroupEnv, request: WorkflowRunGroupCreateRequest | None = None) -> str:
+ response = await group_service.submit_workflow_run_group(env.organization, request or _request())
+ return response.workflow_run_group_id
+
+
+async def test_submit_preallocates_ids_without_child_rows_then_dispatches_one(env: GroupEnv) -> None:
+ items = [{"key": f"acct-{i}", "parameters": {"login": "cred_1"}} for i in range(25)]
+ response = await group_service.submit_workflow_run_group(env.organization, _request(items=items))
+
+ run_ids = [item.workflow_run_id for item in response.items]
+ assert len(set(run_ids)) == 25
+ assert response.workflow_id == "wf_1"
+ assert await _count(env, WorkflowRunModel) == 0
+
+ await group_service.advance_workflow_run_group(response.workflow_run_group_id)
+
+ assert env.executor.executed == [run_ids[0]]
+ child = await env.database.workflow_runs.get_workflow_run(run_ids[0])
+ assert child is not None and child.workflow_id == "wf_1" and child.start_fresh_browser is True
+ assert await _count(env, WorkflowRunModel) == 1
+ group = await group_service.get_workflow_run_group(response.workflow_run_group_id, ORG)
+ assert [item.workflow_run_id for item in group.items] == run_ids
+
+
+async def _run_ids(env: GroupEnv, group_id: str) -> list[str]:
+ return [item.workflow_run_id for item in await env.database.workflow_run_groups.get_items(group_id)]
+
+
+async def _states(env: GroupEnv, group_id: str) -> list[WorkflowRunGroupItemState]:
+ return [item.state for item in await env.database.workflow_run_groups.get_items(group_id)]
+
+
+async def test_next_child_waits_until_the_running_one_is_final(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ await group_service.advance_workflow_run_group(group_id)
+
+ await asyncio.gather(*(group_service.advance_workflow_run_group(group_id) for _ in range(3)))
+ await group_service.recover_workflow_run_groups()
+
+ assert env.executor.executed == [run_ids[0]]
+ assert (await _states(env, group_id))[1:] == [WorkflowRunGroupItemState.pending] * 2
+
+ await _set_child_status(env, run_ids[0], WorkflowRunStatus.completed)
+ await group_service.advance_workflow_run_group(group_id)
+ assert env.executor.executed == run_ids[:2]
+
+
[email protected]("outside_status", [WorkflowRunStatus.canceled, WorkflowRunStatus.timed_out])
+async def test_an_outside_cancel_or_timeout_holds_the_next_child_until_the_worker_can_stop(
+ env: GroupEnv, outside_status: WorkflowRunStatus
+) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ await group_service.advance_workflow_run_group(group_id)
+ await env.database.workflow_runs.update_workflow_run_if_not_final(run_ids[0], outside_status)
+
+ await group_service.advance_workflow_run_group(group_id)
+ assert env.executor.executed == run_ids[:1]
+
+ async with env.database.Session() as session:
+ await session.execute(
+ update(WorkflowRunModel)
+ .where(WorkflowRunModel.workflow_run_id == run_ids[0])
+ .values(finished_at=naive_utc_now() - group_service.CHILD_STOP_SETTLE - timedelta(seconds=1))
+ )
+ await session.commit()
+ await group_service.advance_workflow_run_group(group_id)
+ assert env.executor.executed == run_ids[:2]
+
+
[email protected]("bad_credential", ["cred_foreign", "cred_missing"])
+async def test_one_bad_credential_rejects_the_whole_group(env: GroupEnv, bad_credential: str) -> None:
+ items = [
+ {"key": "a", "parameters": {"login": "cred_1"}},
+ {"key": "b", "parameters": {"login": bad_credential}},
+ {"key": "c", "parameters": {"login": "cred_2"}},
+ ]
+ with pytest.raises(SkyvernHTTPException) as exc_info:
+ await group_service.submit_workflow_run_group(env.organization, _request(items=items))
+
+ assert 400 <= exc_info.value.status_code < 500
+ assert await _count(env, WorkflowRunGroupModel) == 0
+ assert await _count(env, WorkflowRunGroupItemModel) == 0
+ assert await _count(env, WorkflowRunModel) == 0
+ assert env.spawned == [] and env.executor.executed == []
+
+
[email protected]("count", ["3", "abc"])
+async def test_a_mistyped_value_in_any_item_rejects_the_whole_group(env: GroupEnv, count: str) -> None:
+ await env.database.workflow_params.create_workflow_parameter(
+ workflow_id="wf_1", workflow_parameter_type=WorkflowParameterType.INTEGER, key="count", default_value=None
+ )
+ items = [
+ {"key": "a", "parameters": {"login": "cred_1", "count": "1"}},
+ {"key": "b", "parameters": {"login": "cred_2", "count": count}},
+ ]
+
+ if count.isdigit():
+ await group_service.submit_workflow_run_group(env.organization, _request(items=items))
+ assert await _count(env, WorkflowRunGroupItemModel) == 2
+ return
+ with pytest.raises(SkyvernHTTPException) as exc_info:
+ await group_service.submit_workflow_run_group(env.organization, _request(items=items))
+ assert exc_info.value.status_code == 400
+ assert await _count(env, WorkflowRunGroupModel) == 0
+ assert await _count(env, WorkflowRunGroupItemModel) == 0
+
+
+async def test_identical_retry_after_workflow_saves_returns_the_same_group(env: GroupEnv) -> None:
+ async with env.database.Session() as session:
+ session.add(
+ WorkflowModel(
+ workflow_id="wf_copilot_test",
+ workflow_permanent_id=WPID,
+ organization_id=ORG,
+ title="Workflow",
+ version=2,
+ created_by=COPILOT_TEST_WORKFLOW_CREATOR,
+ workflow_definition=_definition(_task_block()),
+ )
+ )
+ await session.commit()
+ first = await group_service.submit_workflow_run_group(env.organization, _request())
+ assert first.workflow_id == "wf_1"
+
+ async with env.database.Session() as session:
+ session.add(
+ WorkflowModel(
+ workflow_id="wf_3",
+ workflow_permanent_id=WPID,
+ organization_id=ORG,
+ title="Workflow",
+ version=3,
+ workflow_definition=_definition(),
+ )
+ )
+ await session.commit()
+ retry = await group_service.submit_workflow_run_group(env.organization, _request())
+
+ assert retry.workflow_run_group_id == first.workflow_run_group_id
+ assert retry.workflow_id == "wf_1"
+ assert [item.workflow_run_id for item in retry.items] == [item.workflow_run_id for item in first.items]
+ assert await _count(env, WorkflowRunGroupModel) == 1
+
+
+async def test_identical_retry_with_submission_disabled_returns_the_same_group(
+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ first = await group_service.submit_workflow_run_group(env.organization, _request())
+ monkeypatch.setattr(settings, "WORKFLOW_RUN_GROUPS_SUBMIT_ENABLED", False)
+
+ retry = await group_service.submit_workflow_run_group(env.organization, _request())
+ with pytest.raises(SkyvernHTTPException) as exc_info:
+ await group_service.submit_workflow_run_group(env.organization, _request(key="sub-2"))
+
+ assert retry.workflow_run_group_id == first.workflow_run_group_id
+ assert exc_info.value.status_code == 503
+ assert await _count(env, WorkflowRunGroupModel) == 1
+
+
+ "changed",
+ [
+ _request(items=[{"key": "acct-0", "parameters": {"login": "cred_2"}}]),
+ _request().model_copy(update={"version": 1}),
+ ],
+ ids=["items", "version"],
+)
+async def test_conflicting_input_for_a_submission_key_is_rejected(
+ env: GroupEnv, changed: WorkflowRunGroupCreateRequest
+) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+
+ with pytest.raises(SkyvernHTTPException) as exc_info:
+ await group_service.submit_workflow_run_group(env.organization, changed)
+
+ assert exc_info.value.status_code == 409
+ assert await _count(env, WorkflowRunGroupModel) == 1
+ assert await _run_ids(env, group_id) == run_ids
+ assert len(env.spawned) == 1
+
+
+async def test_concurrent_identical_submissions_share_one_group(env: GroupEnv) -> None:
+ responses = await asyncio.gather(
+ *(group_service.submit_workflow_run_group(env.organization, _request()) for _ in range(3))
+ )
+
+ assert len({response.workflow_run_group_id for response in responses}) == 1
+ assert len({tuple(item.workflow_run_id for item in response.items) for response in responses}) == 1
+ assert await _count(env, WorkflowRunGroupModel) == 1
+ assert await _count(env, WorkflowRunGroupItemModel) == 3
+
+
+async def test_concurrent_recoveries_of_a_stale_claim_execute_once(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ assert await env.database.workflow_run_groups.claim_next_item(group_id, "dead-owner") is not None
+ await _age_claims(env)
+
+ await asyncio.gather(group_service.recover_workflow_run_groups(), group_service.recover_workflow_run_groups())
+
+ assert env.executor.executed == [run_ids[0]]
+ assert await _count(env, WorkflowRunModel) == 1
+
+
+async def test_slow_owner_cannot_execute_after_recovery_reclaims_its_item(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ group = await env.database.workflow_run_groups.get_group(group_id)
+ slow_item = await env.database.workflow_run_groups.claim_next_item(group_id, "slow-owner")
+ assert group is not None and slow_item is not None
+ await _age_claims(env)
+ reclaimed = await env.database.workflow_run_groups.reclaim_stale_item(
+ group_id, 0, dispatch_token="new-owner", stale_before=naive_utc_now() - group_service.DISPATCH_GRACE
+ )
+ assert reclaimed is not None
+
+ assert await group_service._dispatch_item(group, slow_item, "slow-owner") is True
+ assert env.executor.executed == []
+
+ assert await group_service._dispatch_item(group, reclaimed, "new-owner") is True
+ assert env.executor.executed == [run_ids[0]]
+ assert await _count(env, WorkflowRunModel) == 1
+
+
+async def test_stale_owner_interrupted_prep_leaves_the_new_owners_child_alone(
+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ group = await env.database.workflow_run_groups.get_group(group_id)
+ stale_item = await env.database.workflow_run_groups.claim_next_item(group_id, "stale-owner")
+ assert group is not None and stale_item is not None
+ await _age_claims(env)
+ reclaimed = await env.database.workflow_run_groups.reclaim_stale_item(
+ group_id, 0, dispatch_token="new-owner", stale_before=naive_utc_now() - group_service.DISPATCH_GRACE
+ )
+ assert reclaimed is not None
+ create_task_run = env.database.tasks.create_task_run
+ status_after_stale_owner: list[WorkflowRunStatus | None] = []
+
+ async def stale_owner_runs_mid_preparation(**kwargs: Any) -> Run:
+ if not status_after_stale_owner:
+ assert await group_service._dispatch_item(group, stale_item, "stale-owner") is False
+ status_after_stale_owner.append(await _status(env, run_ids[0]))
+ return await create_task_run(**kwargs)
+
+ monkeypatch.setattr(env.database.tasks, "create_task_run", stale_owner_runs_mid_preparation)
+
+ assert await group_service._dispatch_item(group, reclaimed, "new-owner") is True
+
+ assert status_after_stale_owner == [WorkflowRunStatus.created]
+ assert env.executor.executed == [run_ids[0]]
+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.dispatched
+ assert await _count(env, WorkflowRunModel) == 1
+
+
+async def test_crash_between_child_row_and_task_run_fails_that_item_and_runs_the_next(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ await _leave_item_zero_unstarted(env, group_id, run_ids[0], "dispatching")
+
+ await group_service.recover_workflow_run_groups()
+
+ assert env.executor.executed == [run_ids[1]]
+ child = await env.database.workflow_runs.get_workflow_run(run_ids[0])
+ assert child is not None and child.status == WorkflowRunStatus.failed
+ group = await group_service.get_workflow_run_group(group_id, ORG)
+ assert group.items[0].state == WorkflowRunGroupItemState.failed_to_start
+ assert group.items[0].outcome == WorkflowRunGroupItemOutcome.failed
+ assert await _count(env, WorkflowRunModel) == 2
+
+
+class RefusingPermissionChecker(PermissionChecker):
+ async def check(self, organization: Organization, browser_session_id: str | None = None) -> None:
+ raise HTTPException(status_code=402, detail="Marketplace subscription is not active")
+
+
+async def test_org_refused_after_submit_starts_no_remaining_children(
+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ group_id = await _submit(env)
+ monkeypatch.setattr(group_service.PermissionCheckerFactory, "get_instance", RefusingPermissionChecker)
+
+ await group_service.advance_workflow_run_group(group_id)
+
+ assert env.executor.executed == []
+ assert await _states(env, group_id) == [WorkflowRunGroupItemState.failed_to_start] * 3
+ assert await _count(env, WorkflowRunModel) == 0
+
+
+async def test_failing_an_item_whose_child_was_just_canceled_holds_the_next_item_until_it_settles(
+ env: GroupEnv,
+) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ claimed = await env.database.workflow_run_groups.claim_next_item(group_id, "owner")
+ group = await env.database.workflow_run_groups.get_group(group_id)
+ assert claimed is not None and group is not None
+ async with env.database.Session() as session:
+ session.add(
+ WorkflowRunModel(
+ workflow_run_id=run_ids[0],
+ workflow_id="wf_1",
+ workflow_permanent_id=WPID,
+ organization_id=ORG,
+ status=WorkflowRunStatus.canceled.value,
+ finished_at=naive_utc_now(),
+ start_fresh_browser=True,
+ )
+ )
+ await session.commit()
+
+ assert await group_service._dispatch_item(group, claimed, "owner") is False
+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.dispatched
+ await group_service.advance_workflow_run_group(group_id)
+ assert env.executor.executed == []
+
+ async with env.database.Session() as session:
+ await session.execute(
+ update(WorkflowRunModel)
+ .where(WorkflowRunModel.workflow_run_id == run_ids[0])
+ .values(finished_at=naive_utc_now() - group_service.CHILD_STOP_SETTLE - timedelta(seconds=1))
+ )
+ await session.commit()
+ await group_service.advance_workflow_run_group(group_id)
+
+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.done
+ assert env.executor.executed == run_ids[1:2]
+
+
+async def test_an_item_held_before_its_child_existed_is_released_by_the_sweep(
+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ group_id = await _submit(env)
+ monkeypatch.setattr(group_service.PermissionCheckerFactory, "get_instance", RefusingPermissionChecker)
+ finish_item = env.database.workflow_run_groups.finish_item
+ crashed: list[bool] = []
+
+ async def crash_once_before_the_release(*args: Any, **kwargs: Any) -> bool:
+ if kwargs["state"] == WorkflowRunGroupItemState.failed_to_start and not crashed:
+ crashed.append(True)
+ raise RuntimeError("process died")
+ return await finish_item(*args, **kwargs)
+
+ monkeypatch.setattr(env.database.workflow_run_groups, "finish_item", crash_once_before_the_release)
+ with pytest.raises(RuntimeError):
+ await group_service.advance_workflow_run_group(group_id)
+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.dispatched
+
+ await _age_claims(env)
+ await group_service.recover_workflow_run_groups()
+
+ assert await _states(env, group_id) == [WorkflowRunGroupItemState.failed_to_start] * 3
+ group = await env.database.workflow_run_groups.get_group(group_id)
+ assert group is not None and group.status == WorkflowRunGroupStatus.finished
+
+
+async def test_child_cancel_between_flip_and_queued_write(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+
+ async def cancel_group(_: str) -> None:
+ await group_service.cancel_workflow_run_group(group_id, ORG)
+
+ env.executor.before_queue = cancel_group
+ await group_service.advance_workflow_run_group(group_id)
+
+ assert env.executor.executed == [run_ids[0]]
+ assert env.executor.submitted == []
+ assert await _status(env, run_ids[0]) == WorkflowRunStatus.canceled
+ group = await group_service.get_workflow_run_group(group_id, ORG)
+ assert group.status == WorkflowRunGroupStatus.finished
+ assert [item.outcome for item in group.items] == [WorkflowRunGroupItemOutcome.canceled] * 3
+ assert await _count(env, WorkflowRunModel) == 1
+
+
+async def test_cancel_between_claim_and_dispatch_never_executes(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+ group = await env.database.workflow_run_groups.get_group(group_id)
+ claimed = await env.database.workflow_run_groups.claim_next_item(group_id, "owner")
+ assert group is not None and claimed is not None
+
+ await group_service.cancel_workflow_run_group(group_id, ORG)
+ still_in_flight = await group_service._dispatch_item(group, claimed, "owner")
+
+ assert still_in_flight is False
+ assert env.executor.executed == []
+ child = await env.database.workflow_runs.get_workflow_run(claimed.workflow_run_id)
+ assert child is not None and child.status == WorkflowRunStatus.canceled
+ assert await _states(env, group_id) == [WorkflowRunGroupItemState.canceled] * 3
+
+
+async def test_in_place_edit_of_a_pinned_version_is_refused_until_the_group_finishes(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+
+ for edit in (
+ {"title": "Edited"},
+ {"workflow_definition": WorkflowDefinition(parameters=[], blocks=[_task_block()])},
+ ):
+ with pytest.raises(WorkflowPinnedByRunGroup) as refused:
+ await app.WORKFLOW_SERVICE.update_workflow_definition(workflow_id="wf_1", organization_id=ORG, **edit)
+ assert refused.value.status_code == 409
+ saved = await env.database.workflows.get_workflow(workflow_id="wf_1", organization_id=ORG)
+ assert saved is not None and saved.title == "Workflow"
+
+ await group_service.cancel_workflow_run_group(group_id, ORG)
+ await group_service.recover_workflow_run_groups()
+ assert (await group_service.get_workflow_run_group(group_id, ORG)).status == WorkflowRunGroupStatus.finished
+ edited = await app.WORKFLOW_SERVICE.update_workflow_definition(
+ workflow_id="wf_1", organization_id=ORG, title="Edited"
+ )
+ assert edited.title == "Edited"
+
+
+async def test_submit_bound_to_a_reviewed_version_refuses_an_edit_that_landed_after_review(env: GroupEnv) -> None:
+ reviewed = await env.database.workflows.get_workflow(workflow_id="wf_1", organization_id=ORG)
+ assert reviewed is not None
+ await app.WORKFLOW_SERVICE.update_workflow_definition(workflow_id="wf_1", organization_id=ORG, title="Edited")
+
+ with pytest.raises(WorkflowChangedSinceReview) as refused:
+ await group_service.submit_workflow_run_group(
+ env.organization, _request(), expected_workflow_modified_at=reviewed.modified_at
+ )
+
+ assert refused.value.status_code == 409
+ assert await _count(env, WorkflowRunGroupModel) == 0
+ assert env.spawned == []
+ current = await env.database.workflows.get_workflow(workflow_id="wf_1", organization_id=ORG)
+ assert current is not None
+ response = await group_service.submit_workflow_run_group(
+ env.organization, _request(), expected_workflow_modified_at=current.modified_at
+ )
+ assert response.workflow_id == "wf_1"
+
+
+async def test_cancel_while_a_child_runs_gives_every_item_an_outcome(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ await group_service.advance_workflow_run_group(group_id)
+ await env.database.workflow_runs.update_workflow_run(run_ids[0], status=WorkflowRunStatus.running)
+
+ await group_service.cancel_workflow_run_group(group_id, ORG)
+ await group_service.recover_workflow_run_groups()
+
+ assert env.executor.executed == [run_ids[0]]
+ group = await group_service.get_workflow_run_group(group_id, ORG)
+ assert group.status == WorkflowRunGroupStatus.finished
+ assert [item.run_status for item in group.items] == [WorkflowRunStatus.canceled, None, None]
+ assert [item.outcome for item in group.items] == [
+ WorkflowRunGroupItemOutcome.unknown,
+ WorkflowRunGroupItemOutcome.canceled,
+ WorkflowRunGroupItemOutcome.canceled,
+ ]
+
+
+async def test_cancel_cascades_to_a_running_childs_nested_runs_and_keeps_final_statuses(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ await group_service.advance_workflow_run_group(group_id)
+ finished_nested = await _start_nested_run(env, run_ids[0])
+ await _set_child_status(env, finished_nested, WorkflowRunStatus.completed)
+ await _set_child_status(env, run_ids[0], WorkflowRunStatus.completed)
+ await group_service.advance_workflow_run_group(group_id)
+ await _set_child_status(env, run_ids[1], WorkflowRunStatus.running)
+ running_nested = await _start_nested_run(env, run_ids[1])
+ completed_nested = await _start_nested_run(env, run_ids[1])
+ await _set_child_status(env, completed_nested, WorkflowRunStatus.completed)
+
+ await group_service.cancel_workflow_run_group(group_id, ORG)
+
+ assert await _status(env, run_ids[0]) == WorkflowRunStatus.completed
+ assert await _status(env, finished_nested) == WorkflowRunStatus.completed
+ assert await _status(env, run_ids[1]) == WorkflowRunStatus.canceled
+ assert await _status(env, running_nested) == WorkflowRunStatus.canceled
+ assert await _status(env, completed_nested) == WorkflowRunStatus.completed
+
+
+async def test_cancel_runs_terminal_side_effects_for_non_final_children_only(
+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ webhooks: list[str] = []
+
+ async def record_webhook(workflow_run: WorkflowRun, **_: object) -> None:
+ webhooks.append(workflow_run.workflow_run_id)
+
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_webhook", record_webhook)
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ await group_service.advance_workflow_run_group(group_id)
+ await _set_child_status(env, run_ids[0], WorkflowRunStatus.completed)
+ await group_service.advance_workflow_run_group(group_id)
+ await _set_child_status(env, run_ids[1], WorkflowRunStatus.running)
+
+ await group_service.cancel_workflow_run_group(group_id, ORG)
+ await group_service.recover_workflow_run_groups()
+ await group_service._cancel_child(run_ids[0], ORG)
+
+ assert webhooks == [run_ids[1]]
+ assert await _status(env, run_ids[0]) == WorkflowRunStatus.completed
+ assert await _status(env, run_ids[1]) == WorkflowRunStatus.canceled
+
+
+async def test_failed_child_keeps_earlier_results_and_later_items_run(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ for run_id, status in zip(
+ run_ids, (WorkflowRunStatus.completed, WorkflowRunStatus.failed, WorkflowRunStatus.completed), strict=True
+ ):
+ await group_service.advance_workflow_run_group(group_id)
+ await _set_child_status(env, run_id, status)
+ await group_service.advance_workflow_run_group(group_id)
+
+ group = await group_service.get_workflow_run_group(group_id, ORG)
+ assert env.executor.executed == run_ids
+ assert group.status == WorkflowRunGroupStatus.finished
+ assert [item.outcome for item in group.items] == [
+ WorkflowRunGroupItemOutcome.completed,
+ WorkflowRunGroupItemOutcome.failed,
+ WorkflowRunGroupItemOutcome.completed,
+ ]
+
+
[email protected]("evidence", ["http_request_block", "recorded_step", "script_run"])
+async def test_failure_after_possible_side_effects_is_unknown_and_not_replayed(env: GroupEnv, evidence: str) -> None:
+ if evidence == "http_request_block":
+ async with env.database.Session() as session:
+ await session.execute(
+ update(WorkflowModel).values(workflow_definition=_definition(_task_block(), _http_block()))
+ )
+ await session.commit()
+ group_id = await _submit(env, _request(count=1))
+ run_ids = await _run_ids(env, group_id)
+ await group_service.advance_workflow_run_group(group_id)
+ if evidence == "recorded_step":
+ task = await env.database.tasks.create_task(
+ url="https://example.com",
+ title="login",
+ navigation_goal=None,
+ data_extraction_goal=None,
+ navigation_payload=None,
+ organization_id=ORG,
+ workflow_run_id=run_ids[0],
+ )
+ await env.database.tasks.create_step(task.task_id, order=0, retry_index=0, organization_id=ORG)
+ if evidence == "script_run":
+ await env.database.workflow_runs.update_workflow_run(run_ids[0], script_id="s_1")
+ await _set_child_status(env, run_ids[0], WorkflowRunStatus.failed)
+
+ await group_service.advance_workflow_run_group(group_id)
+ await group_service.recover_workflow_run_groups()
+
+ group = await group_service.get_workflow_run_group(group_id, ORG)
+ assert group.items[0].outcome == WorkflowRunGroupItemOutcome.unknown
+ assert env.executor.executed == run_ids
+ assert await _count(env, WorkflowRunModel) == 1
+
+
+async def test_terminal_child_advances_the_group_through_the_run_terminal_hook(
+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ service = WorkflowService()
+ monkeypatch.setattr(app, "WORKFLOW_SERVICE", service)
+ monkeypatch.setattr(service, "_resolve_managed_browser_profile_for_run_request", AsyncMock(return_value=None))
+ monkeypatch.setattr(
+ app.AGENT_FUNCTION, "schedule_workflow_run_group_advance", AgentFunction().schedule_workflow_run_group_advance
+ )
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ await group_service.advance_workflow_run_group(group_id)
+ while env.spawned:
+ env.spawned.pop().close()
+
+ await service.mark_workflow_run_as_failed_if_not_final(workflow_run_id=run_ids[0], failure_reason="boom")
+ while env.spawned:
+ await env.spawned.pop()
+
+ assert env.executor.executed == run_ids[:2]
+
+
+def test_browser_session_inputs_are_rejected() -> None:
+ base = {"workflow_id": WPID, "submission_key": "k", "items": [{"key": "a", "parameters": {}}]}
+ for field_name in ("browser_session_id", "browser_address", "browser_profile_id"):
+ with pytest.raises(ValidationError):
+ WorkflowRunGroupCreateRequest.model_validate({**base, field_name: "x"})
+ with pytest.raises(ValidationError):
+ WorkflowRunGroupCreateRequest.model_validate({**base, "items": [{"key": "a", field_name: "x"}]})
+
+
+ "workflow_values",
+ [
+ {"workflow_definition": {**_definition(_task_block()), "retry_policy": {"retry_on": [{"status": "failed"}]}}},
+ {"persist_browser_session": True},
+ {"browser_profile_id": "bp_1"},
+ {"sequential_key": "{{ login }}"},
+ {"workflow_definition": _definition()},
+ {"browser_profile_key": "profile-{{ login }}"},
+ {"workflow_definition": _definition(_task_block(), _looped_trigger(browser_session_id="{{ session_id }}"))},
+ {"workflow_definition": _definition(_task_block(), _looped_trigger(wait_for_completion=False))},
+ {"workflow_definition": {**_definition(_task_block()), "finally_block_label": "login"}},
+ ],
+)
+async def test_versions_a_group_cannot_run_are_rejected(env: GroupEnv, workflow_values: dict[str, object]) -> None:
+ async with env.database.Session() as session:
+ await session.execute(update(WorkflowModel).values(**workflow_values))
+ await session.commit()
+
+ with pytest.raises(SkyvernHTTPException) as exc_info:
+ await _submit(env)
+
+ assert exc_info.value.status_code == 400
+ assert await _count(env, WorkflowRunGroupModel) == 0
+ assert await _count(env, WorkflowRunGroupItemModel) == 0
+
+
+async def test_sweep_with_a_stale_read_cannot_reclaim_an_item_another_sweep_just_reclaimed(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+ assert await env.database.workflow_run_groups.claim_next_item(group_id, "dead-owner") is not None
+ await _age_claims(env)
+ group = await env.database.workflow_run_groups.get_group(group_id)
+ stale_snapshot = await env.database.workflow_run_groups.get_items(group_id)
+ assert group is not None
+ assert await env.database.workflow_run_groups.reclaim_stale_item(
+ group_id, 0, dispatch_token="sweep-a", stale_before=naive_utc_now() - group_service.DISPATCH_GRACE
+ )
+
+ await group_service._reconcile_in_flight(group, stale_snapshot)
+
+ assert env.executor.executed == []
+ assert (await env.database.workflow_run_groups.get_items(group_id))[0].dispatch_token == "sweep-a"
+
+
[email protected]("executor_took_the_child", [False, True])
+async def test_sweep_fails_a_dispatched_child_no_executor_took_and_leaves_a_queued_one(
+ env: GroupEnv, executor_took_the_child: bool
+) -> None:
+ async def executor_crashes(_: str) -> None:
+ raise RuntimeError("executor unavailable")
+
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ env.executor.before_queue = None if executor_took_the_child else executor_crashes
+ await group_service.advance_workflow_run_group(group_id)
+ env.executor.before_queue = None
+ await _age_claims(env)
+
+ await group_service.recover_workflow_run_groups()
+
+ if executor_took_the_child:
+ assert await _status(env, run_ids[0]) == WorkflowRunStatus.queued
+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.dispatched
+ assert env.executor.executed == run_ids[:1]
+ else:
+ assert await _status(env, run_ids[0]) == WorkflowRunStatus.failed
+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.failed_to_start
+ assert env.executor.executed == run_ids[:2]
+
+
+async def _leave_item_zero_unstarted(env: GroupEnv, group_id: str, run_id: str, item_state: str) -> None:
+ if item_state == "dispatched":
+
+ async def executor_crashes(_: str) -> None:
+ raise RuntimeError("executor unavailable")
+
+ env.executor.before_queue = executor_crashes
+ await group_service.advance_workflow_run_group(group_id)
+ env.executor.before_queue = None
+ else:
+ assert await env.database.workflow_run_groups.claim_next_item(group_id, "crashed-owner") is not None
+ async with env.database.Session() as session:
+ session.add(
+ WorkflowRunModel(
+ workflow_run_id=run_id,
+ workflow_id="wf_1",
+ workflow_permanent_id=WPID,
+ organization_id=ORG,
+ status=WorkflowRunStatus.created.value,
+ start_fresh_browser=True,
+ )
+ )
+ await session.commit()
+ await _age_claims(env)
+
+
[email protected]("item_state", ["dispatching", "dispatched"])
+async def test_an_item_that_never_started_ends_failed_to_start_when_failing_its_child_advances_the_group(
+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch, item_state: str
+) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ await _leave_item_zero_unstarted(env, group_id, run_ids[0], item_state)
+ fail_run = app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final
+
+ async def fail_then_run_terminal_hook(**kwargs: Any) -> WorkflowRun | None:
+ failed = await fail_run(**kwargs)
+ await group_service.advance_workflow_run_group(group_id)
+ return failed
+
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", fail_then_run_terminal_hook)
+
+ await group_service.recover_workflow_run_groups()
+
+ assert await _status(env, run_ids[0]) == WorkflowRunStatus.failed
+ assert (await _states(env, group_id))[:2] == [
+ WorkflowRunGroupItemState.failed_to_start,
+ WorkflowRunGroupItemState.dispatched,
+ ]
+
+
+ ("executor_status", "outcome"),
+ [
+ (WorkflowRunStatus.queued, WorkflowRunGroupItemOutcome.canceled),
+ (WorkflowRunStatus.running, WorkflowRunGroupItemOutcome.unknown),
+ ],
+)
+async def test_a_child_an_executor_took_while_the_sweep_failed_it_holds_the_next_item_until_it_stops(
+ env: GroupEnv,
+ monkeypatch: pytest.MonkeyPatch,
+ executor_status: WorkflowRunStatus,
+ outcome: WorkflowRunGroupItemOutcome,
+) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ await _leave_item_zero_unstarted(env, group_id, run_ids[0], "dispatched")
+ finish_item = env.database.workflow_run_groups.finish_item
+
+ async def executor_takes_the_child_once_the_sweep_holds_it(*args: Any, **kwargs: Any) -> bool:
+ finished = await finish_item(*args, **kwargs)
+ if finished and kwargs["state"] == WorkflowRunGroupItemState.dispatched:
+ await _set_child_status(env, run_ids[0], executor_status)
+ return finished
+
+ monkeypatch.setattr(
+ env.database.workflow_run_groups, "finish_item", executor_takes_the_child_once_the_sweep_holds_it
+ )
+
+ await group_service.recover_workflow_run_groups()
+
+ assert await _status(env, run_ids[0]) == WorkflowRunStatus.canceled
+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.dispatched
+ assert env.executor.executed == run_ids[:1]
+
+ async with env.database.Session() as session:
+ await session.execute(
+ update(WorkflowRunModel)
+ .where(WorkflowRunModel.workflow_run_id == run_ids[0])
+ .values(finished_at=naive_utc_now() - group_service.CHILD_STOP_SETTLE - timedelta(seconds=1))
+ )
+ await session.commit()
+ await group_service.recover_workflow_run_groups()
+
+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.done
+ assert env.executor.executed == run_ids[:2]
+ assert (await group_service.get_workflow_run_group(group_id, ORG)).items[0].outcome == outcome
+
+
+ "prior_attempts", [group_service.MAX_DISPATCH_ATTEMPTS - 1, group_service.MAX_DISPATCH_ATTEMPTS]
+)
+async def test_a_dispatch_that_keeps_dying_fails_its_item_at_the_attempt_cap(
+ env: GroupEnv, prior_attempts: int
+) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ assert await env.database.workflow_run_groups.claim_next_item(group_id, "dead-owner") is not None
+ async with env.database.Session() as session:
+ await session.execute(update(WorkflowRunGroupItemModel).values(dispatch_attempts=prior_attempts))
+ await session.commit()
+ await _age_claims(env)
+
+ await group_service.recover_workflow_run_groups()
+
+ if prior_attempts < group_service.MAX_DISPATCH_ATTEMPTS:
+ assert env.executor.executed == run_ids[:1]
+ else:
+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.failed_to_start
+ assert env.executor.executed == run_ids[1:2]
+
+
+async def test_the_child_after_a_failed_preparation_prepares_in_its_own_context(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+ await env.database.workflow_params.create_workflow_parameter(
+ workflow_id="wf_1", workflow_parameter_type=WorkflowParameterType.INTEGER, key="count", default_value="2"
+ )
+ async with env.database.Session() as session:
+ await session.execute(
+ update(WorkflowRunGroupItemModel)
+ .where(WorkflowRunGroupItemModel.position == 0)
+ .values(parameters={"login": "cred_1", "count": "abc"})
+ )
+ await session.commit()
+ contexts: dict[str, tuple[str | None, str | None]] = {}
+
+ async def record_context(workflow_run_id: str) -> None:
+ context = skyvern_context.current()
+ contexts[workflow_run_id] = (context.run_id, context.root_workflow_run_id) if context else (None, None)
+
+ env.executor.before_queue = record_context
+
+ await group_service.advance_workflow_run_group(group_id)
+
+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.failed_to_start
+ assert env.executor.executed == [run_ids[1]]
+ assert contexts[run_ids[1]] == (run_ids[1], run_ids[1])
+
+
+async def test_scrubbed_group_keeps_no_caller_keys_and_a_replay_of_its_key_conflicts(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+ run_ids = await _run_ids(env, group_id)
+
+ assert await env.database.workflow_run_groups.scrub_keys(ORG, [group_id]) == 1
+
+ async with env.database.Session() as session:
+ stored = [
+ *(await session.scalars(select(WorkflowRunGroupModel.submission_key))).all(),
+ *(await session.scalars(select(WorkflowRunGroupItemModel.item_key))).all(),
+ ]
+ assert not any(raw in value for value in stored for raw in ("sub-1", "acct-"))
+ # A group still running when it passes the cap must keep running its pending items with their own inputs.
+ await group_service.advance_workflow_run_group(group_id)
+ child_parameters = await env.database.workflow_runs.get_workflow_run_parameters(run_ids[0])
+ assert [parameter.value for _, parameter in child_parameters] == ["cred_2"]
+ with pytest.raises(SkyvernHTTPException) as exc_info:
+ await _submit(env)
+ assert exc_info.value.status_code == 409
+ assert await _count(env, WorkflowRunGroupModel) == 1
+ with pytest.raises(ValidationError):
+ _request(key=stored[0])
+ with pytest.raises(ValidationError):
+ _request(items=[{"key": stored[1], "parameters": {"login": "cred_1"}}])
+
+
+async def test_an_edit_landing_between_validation_and_insert_refuses_the_submit(
+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ validate = app.WORKFLOW_SERVICE.validate_schedule_parameters
+
+ async def edit_after_validation(
+ workflow: Workflow, organization: Organization, request_data: dict[str, Any] | None
+ ) -> None:
+ await validate(workflow, organization, request_data)
+ async with env.database.Session() as session:
+ await session.execute(update(WorkflowModel).values(modified_at=naive_utc_now() + timedelta(seconds=1)))
+ await session.commit()
+
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "validate_schedule_parameters", edit_after_validation)
+
+ with pytest.raises(SkyvernHTTPException) as exc_info:
+ await _submit(env)
+
+ assert exc_info.value.status_code == 409
+ assert await _count(env, WorkflowRunGroupModel) == 0
+
+
+async def test_another_organization_cannot_read_or_cancel_a_group(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+
+ for call in (group_service.get_workflow_run_group, group_service.cancel_workflow_run_group):
+ with pytest.raises(SkyvernHTTPException) as exc_info:
+ await call(group_id, OTHER_ORG)
+ assert exc_info.value.status_code == 404
+
+ group = await env.database.workflow_run_groups.get_group(group_id)
+ assert group is not None and group.status == WorkflowRunGroupStatus.active
+
+
+async def test_a_new_group_consumes_a_submit_token_and_a_replay_does_not(env: GroupEnv) -> None:
+ first = await _submit(env)
+ replay = await _submit(env)
+
+ assert replay == first
+ assert env.limiter.calls == [ORG]
+
+
+async def test_item_parameters_are_cleared_once_the_item_leaves_dispatch(env: GroupEnv) -> None:
+ group_id = await _submit(env)
+ assert await env.database.workflow_run_groups.claim_next_item(group_id, "owner") is not None
+ assert await env.database.workflow_run_groups.finish_item(
+ group_id,
+ 0,
+ state=WorkflowRunGroupItemState.failed_to_start,
+ from_states=(WorkflowRunGroupItemState.dispatching,),
+ dispatch_token="owner",
+ )
+ await group_service.advance_workflow_run_group(group_id)
+ items = await env.database.workflow_run_groups.get_items(group_id)
+ assert [item.state for item in items[:2]] == [
+ WorkflowRunGroupItemState.failed_to_start,
+ WorkflowRunGroupItemState.dispatched,
+ ]
+ assert [item.parameters for item in items] == [{}, {}, {"login": "cred_2"}]
+
+ await group_service.cancel_workflow_run_group(group_id, ORG)
+
+ assert [item.parameters for item in await env.database.workflow_run_groups.get_items(group_id)] == [{}, {}, {}]
+
+
+async def test_prepare_error_text_never_reaches_the_item_or_the_group_read(env: GroupEnv) -> None:
+ sentinel = "SENTINEL-acct-7731"
+ group_id = await _submit(env)
+ await env.database.workflow_params.create_workflow_parameter(
+ workflow_id="wf_1", workflow_parameter_type=WorkflowParameterType.INTEGER, key="count", default_value=None
+ )
+ async with env.database.Session() as session:
+ await session.execute(
+ update(WorkflowRunGroupItemModel).values(parameters={"login": "cred_1", "count": sentinel})
+ )
+ await session.commit()
+
+ await group_service.advance_workflow_run_group(group_id)
+
+ item = (await env.database.workflow_run_groups.get_items(group_id))[0]
+ assert item.state == WorkflowRunGroupItemState.failed_to_start
+ assert item.failure_reason == "Workflow run could not be prepared (InvalidWorkflowParameter)"
+ group = await group_service.get_workflow_run_group(group_id, ORG)
+ assert sentinel not in group.model_dump_json()
+
+
[email protected]("nesting", [0, 1, 2])
+ ("wait_for_completion", "supplies_session"),
+ [(False, False), (True, True)],
+ ids=["detached", "sync_into_supplied_session"],
+)
+async def test_unfresh_trigger_under_a_group_child_fails_before_creating_a_run(
+ env: GroupEnv,
+ monkeypatch: pytest.MonkeyPatch,
+ nesting: int,
+ wait_for_completion: bool,
+ supplies_session: bool,
+) -> None:
+ group_id = await _submit(env)
+ await group_service.advance_workflow_run_group(group_id)
+ trigger_run_id = (await _run_ids(env, group_id))[0]
+ for _ in range(nesting):
+ trigger_run_id = await _start_nested_run(env, trigger_run_id)
+ browser_session_id = (
+ (
+ await env.database.browser_sessions.create_persistent_browser_session(organization_id=ORG)
+ ).persistent_browser_session_id
+ if supplies_session
+ else None
+ )
+ runs_before = await _count(env, WorkflowRunModel)
+ block = WorkflowTriggerBlock(
+ label="detached",
+ workflow_permanent_id=WPID,
+ payload={"login": "cred_1"},
+ wait_for_completion=wait_for_completion,
+ browser_session_id=browser_session_id,
+ output_parameter=make_block_output_parameter("detached"),
+ )
+ result = AsyncMock(return_value=MagicMock())
+ monkeypatch.setattr(WorkflowTriggerBlock, "get_workflow_run_context", lambda self, workflow_run_id: MagicMock())
+ monkeypatch.setattr(WorkflowTriggerBlock, "record_output_parameter_value", AsyncMock())
+ monkeypatch.setattr(WorkflowTriggerBlock, "build_block_result", result)
+
+ await block.execute(workflow_run_id=trigger_run_id, workflow_run_block_id="wrb_detached", organization_id=ORG)
+
+ assert result.await_args is not None and result.await_args.kwargs["success"] is False
+ assert await _count(env, WorkflowRunModel) == runs_before
diff --git a/tests/unit/workflow/test_workflow_trigger_block.py b/tests/unit/workflow/test_workflow_trigger_block.py
index 5f6402909..9022ab76d 100644
--- a/tests/unit/workflow/test_workflow_trigger_block.py
+++ b/tests/unit/workflow/test_workflow_trigger_block.py
@@ -374,17 +374,17 @@ class TestCheckTriggerDepth:
"""Test _check_trigger_depth: boundary conditions at/above/below MAX_TRIGGER_DEPTH."""
@pytest.mark.asyncio
- async def test_no_parent_returns_zero(self) -> None:
+ async def test_no_parent_returns_the_run_itself(self) -> None:
block = _make_block()
mock_run = MagicMock()
mock_run.parent_workflow_run_id = None
with patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app:
mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=mock_run)
- depth = await block._check_trigger_depth("wr_current")
- assert depth == 0
+ root = await block._check_trigger_depth("wr_current")
+ assert root == "wr_current"
@pytest.mark.asyncio
- async def test_single_parent_returns_one(self) -> None:
+ async def test_single_parent_returns_the_parent(self) -> None:
block = _make_block()
run_with_parent = MagicMock()
run_with_parent.parent_workflow_run_id = "wr_parent"
@@ -393,8 +393,8 @@ class TestCheckTriggerDepth:
with patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app:
mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(side_effect=[run_with_parent, run_no_parent])
- depth = await block._check_trigger_depth("wr_current")
- assert depth == 1
+ root = await block._check_trigger_depth("wr_current")
+ assert root == "wr_parent"
@pytest.mark.asyncio
async def test_depth_at_max_raises(self) -> None:
@@ -421,16 +421,16 @@ class TestCheckTriggerDepth:
with patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app:
mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(side_effect=runs)
- depth = await block._check_trigger_depth("wr_current")
- assert depth == block.MAX_TRIGGER_DEPTH - 1
+ root = await block._check_trigger_depth("wr_current")
+ assert root == f"wr_parent_{block.MAX_TRIGGER_DEPTH - 2}"
@pytest.mark.asyncio
- async def test_run_not_found_returns_zero(self) -> None:
+ async def test_run_not_found_returns_the_run_id(self) -> None:
block = _make_block()
with patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app:
mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=None)
- depth = await block._check_trigger_depth("wr_nonexistent")
- assert depth == 0
+ root = await block._check_trigger_depth("wr_nonexistent")
+ assert root == "wr_nonexistent"
@pytest.mark.asyncio
@@ -488,7 +488,7 @@ async def test_sync_trigger_preserves_parent_feature_flag_summary(monkeypatch: p
monkeypatch.setattr(WorkflowTriggerBlock, "get_workflow_run_context", lambda self, workflow_run_id: MagicMock())
monkeypatch.setattr(WorkflowTriggerBlock, "format_potential_template_parameters", lambda self, ctx: None)
- monkeypatch.setattr(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value=0))
+ monkeypatch.setattr(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value="wr_parent"))
monkeypatch.setattr(WorkflowTriggerBlock, "record_output_parameter_value", AsyncMock())
monkeypatch.setattr(WorkflowTriggerBlock, "build_block_result", AsyncMock(return_value=MagicMock()))
@@ -557,11 +557,12 @@ async def test_sync_trigger_fails_closed_when_child_resolves_sequential_credenti
patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,
patch.object(WorkflowTriggerBlock, "get_workflow_run_context", lambda self, workflow_run_id: MagicMock()),
patch.object(WorkflowTriggerBlock, "format_potential_template_parameters", lambda self, ctx: None),
- patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value=0)),
+ patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value="wr_parent")),
patch.object(WorkflowTriggerBlock, "record_output_parameter_value", AsyncMock()),
patch.object(WorkflowTriggerBlock, "build_block_result", AsyncMock(side_effect=_build_result)),
):
mock_app.DATABASE.organizations.get_organization = AsyncMock(return_value=organization)
+ mock_app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id = AsyncMock(return_value=None)
mock_app.WORKFLOW_SERVICE.setup_workflow_run = AsyncMock(return_value=child_run)
mock_app.WORKFLOW_SERVICE.execute_workflow = AsyncMock()
mock_app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final = AsyncMock()
@@ -607,11 +608,12 @@ async def _run_sync_trigger_fence(
patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,
patch.object(WorkflowTriggerBlock, "get_workflow_run_context", lambda self, workflow_run_id: MagicMock()),
patch.object(WorkflowTriggerBlock, "format_potential_template_parameters", lambda self, ctx: None),
- patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value=0)),
+ patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value="wr_parent")),
patch.object(WorkflowTriggerBlock, "record_output_parameter_value", AsyncMock()),
patch.object(WorkflowTriggerBlock, "build_block_result", AsyncMock(side_effect=_build_result)),
):
mock_app.DATABASE.organizations.get_organization = AsyncMock(return_value=organization)
+ mock_app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id = AsyncMock(return_value=None)
mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=MagicMock(proxy_location=None))
created_session = MagicMock()
created_session.persistent_browser_session_id = created_session_id
@@ -752,11 +754,12 @@ async def _run_async_trigger(run_workflow_stub: Any) -> tuple[dict[str, Any], di
patch("skyvern.services.workflow_service.run_workflow", side_effect=run_workflow_stub),
patch.object(WorkflowTriggerBlock, "get_workflow_run_context", lambda self, workflow_run_id: MagicMock()),
patch.object(WorkflowTriggerBlock, "format_potential_template_parameters", lambda self, ctx: None),
- patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value=0)),
+ patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value="wr_parent")),
patch.object(WorkflowTriggerBlock, "record_output_parameter_value", AsyncMock(side_effect=_capture_record)),
patch.object(WorkflowTriggerBlock, "build_block_result", AsyncMock(side_effect=_build_result)),
):
mock_app.DATABASE.organizations.get_organization = AsyncMock(return_value=organization)
+ mock_app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id = AsyncMock(return_value=None)
await block.execute(
workflow_run_id="wr_parent",
workflow_run_block_id="wrb_parent",