Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Add serial workflow run groups that submit, recover, and report up to 25 runs as one operation (SKY-17274) (#8773)
Andrew Neilson0ca8f2419b62
diff --git a/alembic/versions/2026_10_02_1934-6fbee6664b86_add_workflow_run_groups.py b/alembic/versions/2026_10_02_1934-6fbee6664b86_add_workflow_run_groups.pynew file mode 100644index 000000000..0c0a1ce1b--- /dev/null+++ b/alembic/versions/2026_10_02_1934-6fbee6664b86_add_workflow_run_groups.py@@ -0,0 +1,73 @@+"""add workflow run groups++Revision ID: 6fbee6664b86+Revises: 13139662a50b+Create Date: 2026-10-02T19:34:31.499073+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "6fbee6664b86"+down_revision: Union[str, None] = "13139662a50b"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ op.create_table(+ "workflow_run_groups",+ sa.Column("workflow_run_group_id", sa.String(), nullable=False),+ sa.Column("organization_id", sa.String(), nullable=False),+ sa.Column("workflow_permanent_id", sa.String(), nullable=False),+ sa.Column("requested_version", sa.Integer(), nullable=True),+ sa.Column("workflow_id", sa.String(), nullable=False),+ sa.Column("submission_key", sa.String(), nullable=False),+ sa.Column("input_fingerprint", sa.String(), nullable=False),+ sa.Column("status", sa.String(), nullable=False),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("modified_at", sa.DateTime(), nullable=False),+ sa.Column("finished_at", sa.DateTime(), nullable=True),+ sa.PrimaryKeyConstraint("workflow_run_group_id"),+ sa.UniqueConstraint("organization_id", "submission_key", name="uq_workflow_run_groups_org_submission_key"),+ )+ op.create_index(+ "idx_workflow_run_groups_unfinished_modified_at",+ "workflow_run_groups",+ ["modified_at"],+ unique=False,+ postgresql_where=sa.text("status IN ('active', 'cancel_requested')"),+ )+ op.create_table(+ "workflow_run_group_items",+ sa.Column("workflow_run_group_id", sa.String(), nullable=False),+ sa.Column("position", sa.Integer(), nullable=False),+ sa.Column("item_key", sa.String(), nullable=False),+ sa.Column("parameters", sa.JSON(), nullable=False),+ sa.Column("workflow_run_id", sa.String(), nullable=False),+ sa.Column("state", sa.String(), nullable=False),+ sa.Column("dispatch_token", sa.String(), nullable=True),+ sa.Column("claimed_at", sa.DateTime(), nullable=True),+ sa.Column("dispatch_attempts", sa.Integer(), nullable=False),+ sa.Column("failure_reason", sa.String(), nullable=True),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("modified_at", sa.DateTime(), nullable=False),+ sa.PrimaryKeyConstraint("workflow_run_group_id", "position"),+ sa.UniqueConstraint("workflow_run_group_id", "item_key", name="uq_workflow_run_group_items_group_item_key"),+ sa.UniqueConstraint("workflow_run_id", name="uq_workflow_run_group_items_workflow_run_id"),+ )+++def downgrade() -> None:+ op.drop_table("workflow_run_group_items")+ op.drop_index(+ "idx_workflow_run_groups_unfinished_modified_at",+ table_name="workflow_run_groups",+ postgresql_where=sa.text("status IN ('active', 'cancel_requested')"),+ )+ op.drop_table("workflow_run_groups")diff --git a/docs/changelog.mdx b/docs/changelog.mdxindex f6966622d..739734ceb 100644--- a/docs/changelog.mdx+++ b/docs/changelog.mdx@@ -113,6 +113,16 @@ keywords:- **Simplified Onboarding Emails** — Added a Day-1 email introducing Copilot, simplified the Welcome email to two steps, and stopped sending "no successful run yet" nudges to team accounts.- **Welcome Questionnaire for New Sign-Ups** — New sign-ups now land directly on the welcome questionnaire instead of briefly landing on Home first.- **Clearer Insurance Quote Example** — The Home page's insurance quote example now reads as plain text instead of a JSON blob.+- **Clearer Extraction Results** — Skyvern 3.0 extraction-only blocks now always report exactly what's on the page, marking any missing field as null. ([#8771](https://github.com/Skyvern-AI/skyvern/pull/8771))+- **Instant Welcome Email** — New signups now get the Welcome email right after their first app load, instead of waiting up to an hour.+- **Clearer Copilot Loop Debugging** — Workflow Copilot now shows which input a for_loop block actually used, with recorded row counts, making a failed loop easier to debug.+- **Readable Copilot Browser-Code Steps** — Copilot's browser-code tool calls now list the steps the code performed, like "Opened a page → clicked next → read ×10," instead of just "Ran browser code."+- **Clearer Blocked Code-Block Errors** — Copilot now explains which value caused a blocked code block and how earlier outputs reach code, instead of a generic error.+- **Visible Page-Load Failures in Code Blocks** — A CodeBlock wait-timeout error now shows the page load failure that caused it, instead of just the timeout.+- **More Complete Copilot Repairs** — Copilot's automatic repairs now see icon-only controls, like expanders and chevrons, even when their selector isn't unique on the page.+- **Starter-Prompt Onboarding Email** — New signups now get an email with three starter prompts — collecting website data, downloading invoices, and filling out a form — to help them get going.+- **Readable Copilot Tables** — Copilot replies now render Markdown tables, headings, quotes, and dividers properly instead of as unstyled text.+- **Search Runs by More Identifiers** — Run History and run search can now find runs by browser profile ID, browser session ID, credential ID, or webhook callback URL, not just run ID or inputs.## Bug fixes@@ -168,6 +178,16 @@ keywords:- Fixed the code block AI fallback running during Copilot build tests and editor block runs, hiding real code regressions; run details now show which block it rescued and why the code failed.- Fixed a Terminate block's custom error code or reason being able to leak a referenced secret's value.- Fixed task runs occasionally being marked failed or terminated right after a successful form submission.+- Fixed Skyvern 3.0 refusing to finish a task when the page had a live countdown, clock, or rotating headline.+- Fixed Skyvern 3.0 losing its view of the page entirely on sites that define their own `toJSON`, instead of reading the page correctly. ([#8770](https://github.com/Skyvern-AI/skyvern/pull/8770))+- Fixed Skyvern 3.0 occasionally picking the wrong row from a long dropdown or menu due to a shifted or guessed row number. ([#8767](https://github.com/Skyvern-AI/skyvern/pull/8767))+- Fixed a single-action step sometimes being marked complete before its action actually took effect, which could carry the run onto the wrong next page.+- Fixed Skyvern 3.0 missing icon-only close buttons on overlays and incorrectly reporting that no controls were found. ([#8766](https://github.com/Skyvern-AI/skyvern/pull/8766))+- Fixed Copilot's automatic repair discarding the notice that a full-screen overlay, like a consent banner, was blocking the page.+- Fixed Skyvern 3.0 on slow-loading pages misjudging whether the page had actually finished loading.+- Fixed Skyvern 3.0 incorrectly reporting a search-and-select field as empty when a single result committed automatically.+- Fixed Skyvern 3.0 clicking the wrong control when its screen-reader label differed from its visible text.+- Fixed Copilot failing before sending any request on certain Anthropic and Bedrock models.</Update>diff --git a/skyvern/config.py b/skyvern/config.pyindex e495f8e5a..4089c49ed 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -986,6 +986,9 @@ class Settings(BaseSettings):"""Maximum number of scheduled workflow runs dispatched concurrently by one OSS server process."""RETRY_DISPATCH_GRACE_SECONDS: int = Field(default=600, ge=600)"""OSS dispatch claim grace; the executor also enforces the retry lease takeover minimum."""+ WORKFLOW_RUN_GROUPS_SUBMIT_ENABLED: bool = True+ """Accept new serial workflow run groups. Turning it off stops submission only; reads, cancels and+ dispatch of already-submitted groups continue."""# OpenTelemetry SettingsOTEL_ENABLED: bool = Falsediff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex 7e2ad2eb5..ebd136e96 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -141,6 +141,23 @@ class SkyvernHTTPException(SkyvernException):super().__init__(message)+class WorkflowPinnedByRunGroup(SkyvernHTTPException):+ def __init__(self, workflow_id: str) -> None:+ super().__init__(+ f"Workflow version {workflow_id} is running as a workflow run group, so it cannot be edited in place "+ "until that group finishes. Save the change as a new version instead.",+ status_code=HTTPStatus.CONFLICT,+ )+++class WorkflowChangedSinceReview(SkyvernHTTPException):+ def __init__(self, workflow_id: str) -> None:+ super().__init__(+ f"Workflow version {workflow_id} changed after it was reviewed, so the group was not submitted.",+ status_code=HTTPStatus.CONFLICT,+ )++_BROWSER_CONNECTION_GUIDANCE = "Please try re-running. If this continues, contact [email protected]."# Patterns that indicate a browser session connection failure (e.g. CDP WebSocket errors).diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 09f3fa98f..67309adfc 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -86,6 +86,7 @@ from skyvern.forge.sdk.workflow.retry_policy import WORKFLOW_WEBHOOK_HTTP_TIMEOUfrom skyvern.schemas.run_enums import RunEngine, RunTypefrom skyvern.schemas.workflows import BlockResult, FileStorageType, FileUploadDestinationfrom skyvern.services.otp_email import EmailOTPSearchError, EmailOTPVerificationContext, build_email_otp_sources+from skyvern.services.workflow_run_group_service import schedule_advance_after_terminalfrom skyvern.utils.email_validation import normalize_identifier_if_emailfrom skyvern.utils.url_validators import pinned_ip_clientfrom skyvern.webeye.actions.actions import Action@@ -3068,6 +3069,9 @@ class AgentFunction:"""Fired after a workflow run reaches a final status. The run may be supplied to avoid a fallback read."""return+ def schedule_workflow_run_group_advance(self, workflow_run: WorkflowRun) -> None:+ schedule_advance_after_terminal(workflow_run)+async def on_task_completed(self,*,diff --git a/skyvern/forge/api_app.py b/skyvern/forge/api_app.pyindex 63c6178b6..4bb06df5f 100644--- a/skyvern/forge/api_app.py+++ b/skyvern/forge/api_app.py@@ -45,6 +45,7 @@ from skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.db.agent_db import AgentDBfrom skyvern.forge.sdk.db.exceptions import DatabaseConnectionUnavailableError, NotFoundError, is_connection_failurefrom skyvern.forge.sdk.db.models import Base+from skyvern.forge.sdk.executor.background_task_executor import BackgroundTaskExecutorfrom skyvern.forge.sdk.executor.factory import AsyncExecutorFactoryfrom skyvern.forge.sdk.routes import internal_auth, internal_llmsfrom skyvern.forge.sdk.routes.google_oauth import google_oauth_router@@ -67,6 +68,7 @@ from skyvern.services.cleanup_service import (stop_cleanup_scheduler,stop_temp_artifact_sweep,)+from skyvern.services.workflow_run_group_service import run_workflow_run_group_recovery_loopfrom skyvern.services.workflow_schedule_service import (start_workflow_schedule_scheduler,stop_workflow_schedule_scheduler,@@ -408,6 +410,12 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:# task group which is required for handling MCP requests.mcp_app = getattr(fastapi_app.state, "mcp_starlette_app", None)retry_recovery_task = asyncio.create_task(_recover_pending_retries(), name="workflow-retry-recovery")+ # Cloud runs this sweep from its own periodic task loop.+ group_recovery_task = (+ asyncio.create_task(run_workflow_run_group_recovery_loop(), name="workflow-run-group-recovery")+ if isinstance(AsyncExecutorFactory.get_executor(), BackgroundTaskExecutor)+ else None+ )try:if mcp_app:async with mcp_app.lifespan(mcp_app):@@ -418,7 +426,11 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:yieldfinally:retry_recovery_task.cancel()- await asyncio.gather(retry_recovery_task, return_exceptions=True)+ if group_recovery_task is not None:+ group_recovery_task.cancel()+ await asyncio.gather(+ retry_recovery_task, *([group_recovery_task] if group_recovery_task else []), return_exceptions=True+ )# The initial pass starts the periodic sweep even when cancelled; stop it so the loop can drain.await AsyncExecutorFactory.get_executor().stop_retry_recovery()diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.pyindex 9f213697a..cf2ed680c 100644--- a/skyvern/forge/sdk/db/agent_db.py+++ b/skyvern/forge/sdk/db/agent_db.py@@ -49,6 +49,7 @@ from skyvern.forge.sdk.db.repositories.workflow_run_attempts import WorkflowRunAfrom skyvern.forge.sdk.db.repositories.workflow_run_credential_selections import (WorkflowRunCredentialSelectionsRepository,)+from skyvern.forge.sdk.db.repositories.workflow_run_groups import WorkflowRunGroupsRepositoryfrom skyvern.forge.sdk.db.repositories.workflow_runs import WorkflowRunsRepositoryfrom skyvern.forge.sdk.db.repositories.workflows import WorkflowsRepositoryfrom skyvern.forge.sdk.db.utils import (@@ -410,6 +411,7 @@ class AgentDB(BaseAlchemyDB):self.workflow_run_credential_selections = WorkflowRunCredentialSelectionsRepository(self.Session, debug_enabled, self.is_retryable_error)+ self.workflow_run_groups = WorkflowRunGroupsRepository(self.Session, debug_enabled, self.is_retryable_error)self.credentials = CredentialRepository(self.Session, debug_enabled, self.is_retryable_error)self.credential_folders = CredentialFoldersRepository(self.Session, debug_enabled, self.is_retryable_error)self.otp = OTPRepository(self.Session, debug_enabled, self.is_retryable_error)diff --git a/skyvern/forge/sdk/db/id.py b/skyvern/forge/sdk/db/id.pyindex 833a34231..8ba05feb7 100644--- a/skyvern/forge/sdk/db/id.py+++ b/skyvern/forge/sdk/db/id.py@@ -84,6 +84,7 @@ SCRIPT_FALLBACK_EPISODE_PREFIX = "sfe"HEAL_EPISODE_PREFIX = "he"HEAL_PROPOSAL_PREFIX = "hp"WORKFLOW_SCHEDULE_PREFIX = "wfs"+WORKFLOW_RUN_GROUP_PREFIX = "wrg"TAG_EVENT_PREFIX = "tge"RUN_TAG_EVENT_PREFIX = "rtge"TAG_KEY_PREFIX = "tkey"@@ -392,6 +393,11 @@ def generate_workflow_schedule_id() -> str:return f"{WORKFLOW_SCHEDULE_PREFIX}_{int_id}"+def generate_workflow_run_group_id() -> str:+ int_id = generate_id()+ return f"{WORKFLOW_RUN_GROUP_PREFIX}_{int_id}"++############# Helper functions below ##############def generate_id() -> int:"""diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex 9df0d0a68..636ab01a5 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -80,6 +80,7 @@ from skyvern.forge.sdk.db.id import (generate_workflow_permanent_id,generate_workflow_run_block_id,generate_workflow_run_credential_selection_id,+ generate_workflow_run_group_id,generate_workflow_run_id,generate_workflow_schedule_id,generate_workflow_script_id,@@ -820,6 +821,61 @@ class WorkflowScheduleModel(Base):deleted_at = Column(DateTime, nullable=True)+class WorkflowRunGroupModel(Base):+ __tablename__ = "workflow_run_groups"+ __table_args__ = (+ UniqueConstraint("organization_id", "submission_key", name="uq_workflow_run_groups_org_submission_key"),+ Index(+ "idx_workflow_run_groups_unfinished_modified_at",+ "modified_at",+ postgresql_where=text("status IN ('active', 'cancel_requested')"),+ ),+ )++ workflow_run_group_id = Column(String, primary_key=True, default=generate_workflow_run_group_id)+ organization_id = Column(String, nullable=False)+ workflow_permanent_id = Column(String, nullable=False)+ requested_version = Column(Integer, nullable=True)+ workflow_id = Column(String, nullable=False)+ submission_key = Column(String, nullable=False)+ input_fingerprint = Column(String, nullable=False)+ status = Column(String, nullable=False, default="active")+ created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)+ modified_at = Column(+ DateTime,+ default=datetime.datetime.utcnow,+ onupdate=datetime.datetime.utcnow,+ nullable=False,+ )+ finished_at = Column(DateTime, nullable=True)+++class WorkflowRunGroupItemModel(Base):+ __tablename__ = "workflow_run_group_items"+ __table_args__ = (+ UniqueConstraint("workflow_run_group_id", "item_key", name="uq_workflow_run_group_items_group_item_key"),+ UniqueConstraint("workflow_run_id", name="uq_workflow_run_group_items_workflow_run_id"),+ )++ workflow_run_group_id = Column(String, primary_key=True)+ position = Column(Integer, primary_key=True)+ item_key = Column(String, nullable=False)+ parameters = Column(JSON, nullable=False)+ workflow_run_id = Column(String, nullable=False)+ state = Column(String, nullable=False, default="pending")+ dispatch_token = Column(String, nullable=True)+ claimed_at = Column(DateTime, nullable=True)+ dispatch_attempts = Column(Integer, nullable=False, default=0)+ failure_reason = Column(String, nullable=True)+ created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)+ modified_at = Column(+ DateTime,+ default=datetime.datetime.utcnow,+ onupdate=datetime.datetime.utcnow,+ nullable=False,+ )++class WorkflowTemplateModel(Base):"""Tracks which workflows are marked as templates.diff --git a/skyvern/forge/sdk/db/repositories/workflow_run_groups.py b/skyvern/forge/sdk/db/repositories/workflow_run_groups.pynew file mode 100644index 000000000..135612ec3--- /dev/null+++ b/skyvern/forge/sdk/db/repositories/workflow_run_groups.py@@ -0,0 +1,389 @@+from __future__ import annotations++import hmac+from collections.abc import Sequence+from datetime import datetime+from hashlib import sha256+from typing import Any, Literal++from sqlalchemy import String, cast, func, select, update+from sqlalchemy.exc import IntegrityError+from sqlalchemy.ext.asyncio import AsyncSession++from skyvern.config import settings+from skyvern.constants import SCRUBBED_VALUE+from skyvern.exceptions import WorkflowChangedSinceReview+from skyvern.forge.sdk.db._error_handling import db_operation+from skyvern.forge.sdk.db.base_repository import BaseRepository+from skyvern.forge.sdk.db.datetime_utils import naive_utc_now+from skyvern.forge.sdk.db.id import generate_workflow_run_id+from skyvern.forge.sdk.db.models import (+ StepModel,+ TaskModel,+ WorkflowModel,+ WorkflowRunGroupItemModel,+ WorkflowRunGroupModel,+)+from skyvern.schemas.workflow_run_groups import (+ IN_FLIGHT_ITEM_STATES,+ SCRUBBED_SUBMISSION_KEY_PREFIX,+ WorkflowRunGroup,+ WorkflowRunGroupItem,+ WorkflowRunGroupItemState,+ WorkflowRunGroupStatus,+)++FlipResult = Literal["dispatched", "canceled", "lost"]+_SCRUBBED_KEY_DOMAIN = b"skyvern.workflow_run_group_scrubbed_submission_key.v1\0"+++def scrubbed_submission_key(submission_key: str) -> str:+ # Keyed so a scrubbed row cannot be matched to a guessed account name, yet a replay of the raw key still finds it.+ digest = hmac.new(settings.SECRET_KEY.encode(), _SCRUBBED_KEY_DOMAIN + submission_key.encode(), sha256).hexdigest()+ return f"{SCRUBBED_SUBMISSION_KEY_PREFIX}{digest}"+++class WorkflowRunGroupsRepository(BaseRepository):+ # Item parameters are cleared on every transition out of pending/dispatching; the child run holds its own copy.+ # Every state transition first touches the group row, so transitions of one group serialize on its row lock.+ async def _lock_group(self, session: AsyncSession, workflow_run_group_id: str) -> WorkflowRunGroupStatus | None:+ status = await session.scalar(+ update(WorkflowRunGroupModel)+ .where(WorkflowRunGroupModel.workflow_run_group_id == workflow_run_group_id)+ .values(modified_at=naive_utc_now())+ .returning(WorkflowRunGroupModel.status)+ )+ return WorkflowRunGroupStatus(status) if status is not None else None++ @db_operation("create_workflow_run_group", expected_errors=(IntegrityError, WorkflowChangedSinceReview))+ async def create_group(+ self,+ *,+ organization_id: str,+ workflow_permanent_id: str,+ requested_version: int | None,+ workflow_id: str,+ submission_key: str,+ input_fingerprint: str,+ items: Sequence[tuple[str, dict[str, Any]]],+ expected_workflow_modified_at: datetime | None = None,+ ) -> WorkflowRunGroup:+ async with self.Session() as session:+ # Locks the row in-place edits lock, so an edit either lands before this check or sees this group.+ modified_at = await session.scalar(+ select(WorkflowModel.modified_at).where(WorkflowModel.workflow_id == workflow_id).with_for_update()+ )+ if expected_workflow_modified_at is not None and modified_at != expected_workflow_modified_at:+ raise WorkflowChangedSinceReview(workflow_id)+ group = WorkflowRunGroupModel(+ organization_id=organization_id,+ workflow_permanent_id=workflow_permanent_id,+ requested_version=requested_version,+ workflow_id=workflow_id,+ submission_key=submission_key,+ input_fingerprint=input_fingerprint,+ status=WorkflowRunGroupStatus.active.value,+ )+ session.add(group)+ await session.flush()+ for position, (item_key, parameters) in enumerate(items):+ session.add(+ WorkflowRunGroupItemModel(+ workflow_run_group_id=group.workflow_run_group_id,+ position=position,+ item_key=item_key,+ parameters=parameters,+ workflow_run_id=generate_workflow_run_id(),+ state=WorkflowRunGroupItemState.pending.value,+ dispatch_attempts=0,+ )+ )+ await session.flush()+ created = WorkflowRunGroup.model_validate(group)+ await session.commit()+ return created++ @db_operation("get_workflow_run_group")+ async def get_group(+ self, workflow_run_group_id: str, organization_id: str | None = None+ ) -> WorkflowRunGroup | None:+ query = select(WorkflowRunGroupModel).where(+ WorkflowRunGroupModel.workflow_run_group_id == workflow_run_group_id+ )+ if organization_id is not None:+ query = query.where(WorkflowRunGroupModel.organization_id == organization_id)+ async with self.Session() as session:+ group = (await session.scalars(query)).first()+ return WorkflowRunGroup.model_validate(group) if group else None++ @db_operation("get_workflow_run_group_by_submission_key")+ async def get_group_by_submission_key(self, organization_id: str, submission_key: str) -> WorkflowRunGroup | None:+ async with self.Session() as session:+ group = (+ await session.scalars(+ select(WorkflowRunGroupModel)+ .where(WorkflowRunGroupModel.organization_id == organization_id)+ .where(WorkflowRunGroupModel.submission_key == submission_key)+ )+ ).first()+ return WorkflowRunGroup.model_validate(group) if group else None++ @db_operation("scrub_workflow_run_group_keys")+ async def scrub_keys(self, organization_id: str, workflow_run_group_ids: Sequence[str]) -> int:+ """Replace the caller's submission and item keys; returns the groups scrubbed.++ Item parameters are left alone: a pending or dispatching item still needs them to run, and every other state+ has already cleared them.+ """+ async with self.Session() as session:+ groups = (+ await session.scalars(+ select(WorkflowRunGroupModel)+ .where(WorkflowRunGroupModel.organization_id == organization_id)+ .where(WorkflowRunGroupModel.workflow_run_group_id.in_(workflow_run_group_ids))+ .where(~WorkflowRunGroupModel.submission_key.startswith(SCRUBBED_SUBMISSION_KEY_PREFIX))+ )+ ).all()+ if not groups:+ return 0+ for group in groups:+ group.submission_key = scrubbed_submission_key(group.submission_key)+ await session.execute(+ update(WorkflowRunGroupItemModel)+ .where(WorkflowRunGroupItemModel.workflow_run_group_id.in_([g.workflow_run_group_id for g in groups]))+ .values(item_key=f"{SCRUBBED_VALUE}-" + cast(WorkflowRunGroupItemModel.position, String))+ )+ await session.commit()+ return len(groups)++ @db_operation("get_workflow_run_group_items")+ async def get_items(self, workflow_run_group_id: str) -> list[WorkflowRunGroupItem]:+ async with self.Session() as session:+ items = (+ await session.scalars(+ select(WorkflowRunGroupItemModel)+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)+ .order_by(WorkflowRunGroupItemModel.position)+ )+ ).all()+ return [WorkflowRunGroupItem.model_validate(item) for item in items]++ @db_operation("get_workflow_run_group_item_by_workflow_run_id")+ async def get_item_by_workflow_run_id(self, workflow_run_id: str) -> WorkflowRunGroupItem | None:+ async with self.Session() as session:+ item = (+ await session.scalars(+ select(WorkflowRunGroupItemModel).where(+ WorkflowRunGroupItemModel.workflow_run_id == workflow_run_id+ )+ )+ ).first()+ return WorkflowRunGroupItem.model_validate(item) if item else None++ @db_operation("list_unfinished_workflow_run_groups")+ async def list_unfinished_groups(self, limit: int) -> list[tuple[str, str]]:+ async with self.Session() as session:+ rows = (+ await session.execute(+ select(WorkflowRunGroupModel.workflow_run_group_id, WorkflowRunGroupModel.organization_id)+ .where(+ WorkflowRunGroupModel.status.in_(+ [WorkflowRunGroupStatus.active.value, WorkflowRunGroupStatus.cancel_requested.value]+ )+ )+ .order_by(WorkflowRunGroupModel.modified_at)+ .limit(limit)+ )+ ).all()+ return [(group_id, organization_id) for group_id, organization_id in rows]++ @db_operation("claim_next_workflow_run_group_item")+ async def claim_next_item(self, workflow_run_group_id: str, dispatch_token: str) -> WorkflowRunGroupItem | None:+ async with self.Session() as session:+ if await self._lock_group(session, workflow_run_group_id) != WorkflowRunGroupStatus.active:+ await session.commit()+ return None+ in_flight = await session.scalar(+ select(func.count())+ .select_from(WorkflowRunGroupItemModel)+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)+ .where(WorkflowRunGroupItemModel.state.in_([state.value for state in IN_FLIGHT_ITEM_STATES]))+ )+ next_position = None+ if not in_flight:+ next_position = await session.scalar(+ select(func.min(WorkflowRunGroupItemModel.position))+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)+ .where(WorkflowRunGroupItemModel.state == WorkflowRunGroupItemState.pending.value)+ )+ if next_position is None:+ await session.commit()+ return None+ claimed = await session.scalar(+ update(WorkflowRunGroupItemModel)+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)+ .where(WorkflowRunGroupItemModel.position == next_position)+ .where(WorkflowRunGroupItemModel.state == WorkflowRunGroupItemState.pending.value)+ .values(+ state=WorkflowRunGroupItemState.dispatching.value,+ dispatch_token=dispatch_token,+ claimed_at=naive_utc_now(),+ dispatch_attempts=WorkflowRunGroupItemModel.dispatch_attempts + 1,+ )+ .returning(WorkflowRunGroupItemModel)+ )+ result = WorkflowRunGroupItem.model_validate(claimed) if claimed else None+ await session.commit()+ return result++ @db_operation("reclaim_stale_workflow_run_group_item")+ async def reclaim_stale_item(+ self,+ workflow_run_group_id: str,+ position: int,+ *,+ dispatch_token: str,+ stale_before: datetime,+ ) -> WorkflowRunGroupItem | None:+ async with self.Session() as session:+ await self._lock_group(session, workflow_run_group_id)+ reclaimed = await session.scalar(+ update(WorkflowRunGroupItemModel)+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)+ .where(WorkflowRunGroupItemModel.position == position)+ .where(WorkflowRunGroupItemModel.state == WorkflowRunGroupItemState.dispatching.value)+ .where(WorkflowRunGroupItemModel.claimed_at < stale_before)+ .values(+ dispatch_token=dispatch_token,+ claimed_at=naive_utc_now(),+ dispatch_attempts=WorkflowRunGroupItemModel.dispatch_attempts + 1,+ )+ .returning(WorkflowRunGroupItemModel)+ )+ result = WorkflowRunGroupItem.model_validate(reclaimed) if reclaimed else None+ await session.commit()+ return result++ @db_operation("flip_workflow_run_group_item_to_dispatched")+ async def flip_to_dispatched(self, workflow_run_group_id: str, position: int, dispatch_token: str) -> FlipResult:+ async with self.Session() as session:+ group_status = await self._lock_group(session, workflow_run_group_id)+ item = (+ await session.scalars(+ select(WorkflowRunGroupItemModel)+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)+ .where(WorkflowRunGroupItemModel.position == position)+ )+ ).first()+ if (+ item is None+ or item.state != WorkflowRunGroupItemState.dispatching.value+ or item.dispatch_token != dispatch_token+ ):+ await session.commit()+ return "lost"+ result: FlipResult = "dispatched"+ item.parameters = {}+ if group_status != WorkflowRunGroupStatus.active:+ result = "canceled"+ item.state = WorkflowRunGroupItemState.canceled.value+ else:+ item.state = WorkflowRunGroupItemState.dispatched.value+ item.claimed_at = naive_utc_now()+ await session.commit()+ return result++ @db_operation("finish_workflow_run_group_item")+ async def finish_item(+ self,+ workflow_run_group_id: str,+ position: int,+ *,+ state: WorkflowRunGroupItemState,+ from_states: Sequence[WorkflowRunGroupItemState],+ dispatch_token: str | None = None,+ failure_reason: str | None = None,+ ) -> bool:+ async with self.Session() as session:+ await self._lock_group(session, workflow_run_group_id)+ statement = (+ update(WorkflowRunGroupItemModel)+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)+ .where(WorkflowRunGroupItemModel.position == position)+ .where(WorkflowRunGroupItemModel.state.in_([from_state.value for from_state in from_states]))+ )+ if dispatch_token is not None:+ statement = statement.where(WorkflowRunGroupItemModel.dispatch_token == dispatch_token)+ values: dict[str, str | dict[str, Any]] = {"state": state.value, "parameters": {}}+ if failure_reason is not None:+ values["failure_reason"] = failure_reason+ updated = await session.scalar(statement.values(**values).returning(WorkflowRunGroupItemModel.position))+ await session.commit()+ return updated is not None++ @db_operation("request_workflow_run_group_cancel")+ async def request_cancel(self, workflow_run_group_id: str, organization_id: str) -> WorkflowRunGroup | None:+ async with self.Session() as session:+ group = (+ await session.scalars(+ update(WorkflowRunGroupModel)+ .where(WorkflowRunGroupModel.workflow_run_group_id == workflow_run_group_id)+ .where(WorkflowRunGroupModel.organization_id == organization_id)+ .values(modified_at=naive_utc_now())+ .returning(WorkflowRunGroupModel)+ )+ ).first()+ if group is None:+ await session.commit()+ return None+ if group.status == WorkflowRunGroupStatus.active.value:+ group.status = WorkflowRunGroupStatus.cancel_requested.value+ await session.execute(+ update(WorkflowRunGroupItemModel)+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)+ .where(WorkflowRunGroupItemModel.state == WorkflowRunGroupItemState.pending.value)+ .values(state=WorkflowRunGroupItemState.canceled.value, parameters={})+ )+ await session.flush()+ result = WorkflowRunGroup.model_validate(group)+ await session.commit()+ return result++ @db_operation("finish_workflow_run_group_if_complete")+ async def finish_group_if_complete(self, workflow_run_group_id: str) -> bool:+ async with self.Session() as session:+ group_status = await self._lock_group(session, workflow_run_group_id)+ if group_status in (None, WorkflowRunGroupStatus.finished):+ await session.commit()+ return False+ unfinished = await session.scalar(+ select(func.count())+ .select_from(WorkflowRunGroupItemModel)+ .where(WorkflowRunGroupItemModel.workflow_run_group_id == workflow_run_group_id)+ .where(+ WorkflowRunGroupItemModel.state.not_in(+ [state.value for state in WorkflowRunGroupItemState if state.is_final()]+ )+ )+ )+ if unfinished:+ await session.commit()+ return False+ now = naive_utc_now()+ await session.execute(+ update(WorkflowRunGroupModel)+ .where(WorkflowRunGroupModel.workflow_run_group_id == workflow_run_group_id)+ .values(status=WorkflowRunGroupStatus.finished.value, finished_at=now, modified_at=now)+ )+ await session.commit()+ return True++ @db_operation("count_workflow_run_steps")+ async def count_steps(self, workflow_run_id: str) -> int:+ async with self.Session() as session:+ count = await session.scalar(+ select(func.count(StepModel.step_id))+ .join(TaskModel, TaskModel.task_id == StepModel.task_id)+ .where(TaskModel.workflow_run_id == workflow_run_id)+ )+ return int(count or 0)diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.pyindex b62517434..f4053355e 100644--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py@@ -365,6 +365,23 @@ async def _has_serialized_publication_identity(return serialized_publication+async def _first_queued_at(+ session: AsyncSession, workflow_run: WorkflowRunModel, *, sequential_credential_id: str | None = None+) -> datetime:+ """A serialized run takes the next org-wide queue ticket; the caller must hold its publication-lane locks."""+ if await _has_serialized_publication_identity(+ session,+ workflow_run,+ browser_session_id=workflow_run.browser_session_id,+ browser_address=workflow_run.browser_address,+ sequential_credential_id=sequential_credential_id,+ ):+ return await _allocate_serialized_queue_ticket(+ session, organization_id=workflow_run.organization_id, workflow_run_id=workflow_run.workflow_run_id+ )+ return naive_utc_now()++class WorkflowRunsRepository(BaseRepository):"""Database operations for workflow runs."""@@ -1005,25 +1022,9 @@ class WorkflowRunsRepository(BaseRepository):if status:workflow_run.status = statusif status and status == WorkflowRunStatus.queued and workflow_run.queued_at is None:- serialized_publication = await _has_serialized_publication_identity(- session,- workflow_run,- browser_session_id=workflow_run.browser_session_id,- browser_address=workflow_run.browser_address,- sequential_credential_id=sequential_credential_id,- )- if serialized_publication:- # The caller holds every composed publication-lane lock until this transaction- # commits. Advance beyond every active serialized ticket in the organization,- # so all composed lanes share one comparable clock even when database transaction- # time or an application host clock moved backwards.- workflow_run.queued_at = await _allocate_serialized_queue_ticket(- session,- organization_id=workflow_run.organization_id,- workflow_run_id=workflow_run_id,- )- else:- workflow_run.queued_at = naive_utc_now()+ workflow_run.queued_at = await _first_queued_at(+ session, workflow_run, sequential_credential_id=sequential_credential_id+ )if status and status == WorkflowRunStatus.running and workflow_run.started_at is None:workflow_run.started_at = naive_utc_now()if status and status.is_final() and workflow_run.finished_at is None:@@ -1216,19 +1217,16 @@ class WorkflowRunsRepository(BaseRepository):run_with: str | None = None,ai_fallback: bool | None = None,failure_category: list[dict[str, Any]] | None = None,+ only_from: Sequence[WorkflowRunStatus] | None = None,+ job_id: str | None = None,+ depends_on_workflow_run_id: str | None = None,) -> WorkflowRun | None:- """Transition a workflow run to ``status`` only if it is not already in a- terminal state. Returns the updated row, or ``None`` when the row was- already terminal (or missing). Implemented as a single conditional- ``UPDATE ... WHERE status IN (<non-terminal>)`` so a concurrent- finalization write cannot be clobbered by a late cancel.-- Mirrors the timestamp side effects of :meth:`update_workflow_run`:- ``finished_at`` is stamped on terminal transitions and ``started_at``- is stamped on the first ``running`` transition (preserving any- existing value via ``COALESCE``).- """- non_terminal = [s.value for s in WorkflowRunStatus if not s.is_final()]+ """One conditional UPDATE to ``status`` from a non-terminal state (``only_from`` narrows it), so a late+ cancel cannot clobber a finalization; None when the row was terminal or missing. Timestamps follow+ :meth:`update_workflow_run`."""+ non_terminal = [+ s.value for s in WorkflowRunStatus if not s.is_final() and (only_from is None or s in only_from)+ ]now = naive_utc_now()values: dict[str, Any] = {"status": status}if status.is_final():@@ -1243,6 +1241,10 @@ class WorkflowRunsRepository(BaseRepository):values["ai_fallback"] = ai_fallbackif failure_category is not None:values["failure_category"] = failure_category+ if job_id:+ values["job_id"] = job_id+ if depends_on_workflow_run_id:+ values["depends_on_workflow_run_id"] = depends_on_workflow_run_id# The reopen/reset path clears attribution when it returns the row to `created`, so a# later terminal transition starts from SQL NULL and this COALESCE fills the freshly# derived document; on any row that already carries one it preserves the first writer.@@ -1260,6 +1262,14 @@ class WorkflowRunsRepository(BaseRepository):)async with self.Session() as session:+ if status == WorkflowRunStatus.queued:+ workflow_run = (+ await session.scalars(select(WorkflowRunModel).filter_by(workflow_run_id=workflow_run_id))+ ).first()+ if workflow_run is not None and workflow_run.queued_at is None:+ values["queued_at"] = func.coalesce(+ WorkflowRunModel.queued_at, await _first_queued_at(session, workflow_run)+ )result = await session.execute(update(WorkflowRunModel).where(@@ -1966,21 +1976,7 @@ class WorkflowRunsRepository(BaseRepository):if workflow_run is None:return Falseif workflow_run.status == WorkflowRunStatus.created:- queued_at = workflow_run.queued_at- if queued_at is None:- serialized_publication = await _has_serialized_publication_identity(- session,- workflow_run,- browser_session_id=workflow_run.browser_session_id,- browser_address=workflow_run.browser_address,- )- queued_at = (- await _allocate_serialized_queue_ticket(- session, organization_id=workflow_run.organization_id, workflow_run_id=workflow_run_id- )- if serialized_publication- else now- )+ queued_at = workflow_run.queued_at or await _first_queued_at(session, workflow_run)# A cancel committed since the caller's read must win: the status predicate re-evaluates# against the latest committed row.moved = await session.execute(diff --git a/skyvern/forge/sdk/db/repositories/workflows.py b/skyvern/forge/sdk/db/repositories/workflows.pyindex 7fcd90de8..db7955d1c 100644--- a/skyvern/forge/sdk/db/repositories/workflows.py+++ b/skyvern/forge/sdk/db/repositories/workflows.py@@ -10,10 +10,11 @@ from typing import TYPE_CHECKING, Any, castimport structlogfrom sqlalchemy import exists, func, or_, select, text, updatefrom sqlalchemy.exc import SQLAlchemyError-from sqlalchemy.ext.asyncio import AsyncEngine+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSessionfrom sqlalchemy.orm import aliasedfrom skyvern.constants import DEFAULT_SCRIPT_RUN_ID, DEFAULT_WORKFLOW_TITLES+from skyvern.exceptions import WorkflowPinnedByRunGroupfrom skyvern.forge.sdk.browser_action_policy import BrowserActionPolicy, declare_policyfrom skyvern.forge.sdk.db._error_handling import db_operationfrom skyvern.forge.sdk.db._sentinels import _UNSET@@ -33,6 +34,7 @@ from skyvern.forge.sdk.db.models import (OutputParameterModel,WorkflowModel,WorkflowParameterModel,+ WorkflowRunGroupModel,WorkflowRunModel,WorkflowScheduleModel,WorkflowTemplateModel,@@ -52,6 +54,7 @@ from skyvern.forge.sdk.workflow.models.parameter import OutputParameterfrom skyvern.forge.sdk.workflow.models.workflow import COPILOT_TEST_WORKFLOW_CREATOR, Workflow, WorkflowDefinitionfrom skyvern.forge.sdk.workflow.runtime_completion import carried_contract, with_contractfrom skyvern.schemas.runs import ProxyLocationInput+from skyvern.schemas.workflow_run_groups import WorkflowRunGroupStatusfrom skyvern.schemas.workflows import WorkflowStatusif TYPE_CHECKING:@@ -107,6 +110,21 @@ def _align_block_output_parameters(workflow_definition: WorkflowDefinition) -> N_visit(workflow_definition.blocks)+async def _refuse_if_pinned_by_run_group(session: AsyncSession, workflow_id: str) -> None:+ pinned = await session.scalar(+ select(WorkflowRunGroupModel.workflow_run_group_id)+ .where(WorkflowRunGroupModel.workflow_id == workflow_id)+ .where(+ WorkflowRunGroupModel.status.in_(+ [WorkflowRunGroupStatus.active.value, WorkflowRunGroupStatus.cancel_requested.value]+ )+ )+ .limit(1)+ )+ if pinned is not None:+ raise WorkflowPinnedByRunGroup(workflow_id)++class WorkflowsRepository(BaseRepository):"""Database operations for workflow management."""@@ -941,7 +959,7 @@ class WorkflowsRepository(BaseRepository):await session.commit()return result.rowcount > 0- @db_operation("update_workflow")+ @db_operation("update_workflow", expected_errors=(WorkflowPinnedByRunGroup,))async def update_workflow(self,workflow_id: str,@@ -977,6 +995,7 @@ class WorkflowsRepository(BaseRepository):sequential_key: str | None | object = _UNSET,created_by: str | None | object = _UNSET,edited_by: str | None | object = _UNSET,+ refuse_if_pinned_by_run_group: bool = False,) -> Workflow:async with self.Session() as session:get_workflow_query = exclude_deleted(@@ -984,7 +1003,11 @@ class WorkflowsRepository(BaseRepository):)if organization_id:get_workflow_query = get_workflow_query.filter_by(organization_id=organization_id)+ if refuse_if_pinned_by_run_group:+ get_workflow_query = get_workflow_query.with_for_update()if workflow := (await session.scalars(get_workflow_query)).first():+ if refuse_if_pinned_by_run_group:+ await _refuse_if_pinned_by_run_group(session, workflow_id)if title is not None:workflow.title = titleif description is not None:@@ -1274,7 +1297,7 @@ class WorkflowsRepository(BaseRepository):is_template=is_template,)- @db_operation("update_workflow_and_reconcile_definition_params")+ @db_operation("update_workflow_and_reconcile_definition_params", expected_errors=(WorkflowPinnedByRunGroup,))async def update_workflow_and_reconcile_definition_params(self,workflow_id: str,@@ -1311,6 +1334,7 @@ class WorkflowsRepository(BaseRepository):created_by: str | None | object = _UNSET,edited_by: str | None | object = _UNSET,preserve_completion_contract: bool = True,+ refuse_if_pinned_by_run_group: bool = False,) -> Workflow:"""One-session, one-commit update of the workflow row + definition-parameter rows.@@ -1343,9 +1367,13 @@ class WorkflowsRepository(BaseRepository):)if organization_id:get_workflow_query = get_workflow_query.filter_by(organization_id=organization_id)+ if refuse_if_pinned_by_run_group:+ get_workflow_query = get_workflow_query.with_for_update()workflow = (await session.scalars(get_workflow_query)).first()if not workflow:raise NotFoundError("Workflow not found")+ if refuse_if_pinned_by_run_group:+ await _refuse_if_pinned_by_run_group(session, workflow_id)if title is not None:workflow.title = titlediff --git a/skyvern/forge/sdk/routes/__init__.py b/skyvern/forge/sdk/routes/__init__.pyindex 87d2dfcfd..e3c373c39 100644--- a/skyvern/forge/sdk/routes/__init__.py+++ b/skyvern/forge/sdk/routes/__init__.py@@ -19,6 +19,7 @@ from skyvern.forge.sdk.routes import self_heal # noqa: F401from skyvern.forge.sdk.routes import streaming # noqa: F401from skyvern.forge.sdk.routes import ui_session # noqa: F401from skyvern.forge.sdk.routes import webhooks # noqa: F401+from skyvern.forge.sdk.routes import workflow_run_groups # noqa: F401from skyvern.forge.sdk.routes import workflow_schedules # noqa: F401from skyvern.forge.sdk.routes.streaming import cdp_input # noqa: F401from skyvern.forge.sdk.routes.streaming import messages # noqa: F401diff --git a/skyvern/forge/sdk/routes/workflow_run_groups.py b/skyvern/forge/sdk/routes/workflow_run_groups.pynew file mode 100644index 000000000..4a871ee3b--- /dev/null+++ b/skyvern/forge/sdk/routes/workflow_run_groups.py@@ -0,0 +1,44 @@+from fastapi import Depends++from skyvern.forge.sdk.core.permissions.permission_checker_factory import PermissionCheckerFactory+from skyvern.forge.sdk.routes.routers import base_router, legacy_base_router+from skyvern.forge.sdk.schemas.organizations import Organization+from skyvern.forge.sdk.services import org_auth_service+from skyvern.schemas.workflow_run_groups import WorkflowRunGroupCreateRequest, WorkflowRunGroupResponse+from skyvern.services import workflow_run_group_service+++@legacy_base_router.post("/workflow_run_groups", include_in_schema=False)+@base_router.post("/workflow_run_groups", response_model=WorkflowRunGroupResponse, include_in_schema=False)+async def submit_workflow_run_group(+ body: WorkflowRunGroupCreateRequest,+ organization: Organization = Depends(org_auth_service.get_current_org),+) -> WorkflowRunGroupResponse:+ await PermissionCheckerFactory.get_instance().check(organization)+ return await workflow_run_group_service.submit_workflow_run_group(organization, body)+++@legacy_base_router.get("/workflow_run_groups/{workflow_run_group_id}", include_in_schema=False)+@base_router.get(+ "/workflow_run_groups/{workflow_run_group_id}", response_model=WorkflowRunGroupResponse, include_in_schema=False+)+async def get_workflow_run_group(+ workflow_run_group_id: str,+ organization: Organization = Depends(org_auth_service.get_current_org),+) -> WorkflowRunGroupResponse:+ return await workflow_run_group_service.get_workflow_run_group(workflow_run_group_id, organization.organization_id)+++@legacy_base_router.post("/workflow_run_groups/{workflow_run_group_id}/cancel", include_in_schema=False)+@base_router.post(+ "/workflow_run_groups/{workflow_run_group_id}/cancel",+ response_model=WorkflowRunGroupResponse,+ include_in_schema=False,+)+async def cancel_workflow_run_group(+ workflow_run_group_id: str,+ organization: Organization = Depends(org_auth_service.get_current_org),+) -> WorkflowRunGroupResponse:+ return await workflow_run_group_service.cancel_workflow_run_group(+ workflow_run_group_id, organization.organization_id+ )diff --git a/skyvern/forge/sdk/services/route_authorization.py b/skyvern/forge/sdk/services/route_authorization.pyindex 0eae0fbd8..b10d47d65 100644--- a/skyvern/forge/sdk/services/route_authorization.py+++ b/skyvern/forge/sdk/services/route_authorization.py@@ -260,6 +260,9 @@ ROUTE_ACTIONS: Mapping[str, str] = {"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_message_feedback": "workflow_copilot.manage","skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_question_response": "workflow_copilot.manage","skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_suggest_goal": "workflow_copilot.suggest",+ "skyvern.forge.sdk.routes.workflow_run_groups.cancel_workflow_run_group": "workflow_run.cancel",+ "skyvern.forge.sdk.routes.workflow_run_groups.get_workflow_run_group": "workflow_run.read",+ "skyvern.forge.sdk.routes.workflow_run_groups.submit_workflow_run_group": "workflow.run","skyvern.forge.sdk.routes.workflow_schedules.cancel_workflow_schedule": "workflow_schedule.cancel","skyvern.forge.sdk.routes.workflow_schedules.create_workflow_schedule": "workflow_schedule.create","skyvern.forge.sdk.routes.workflow_schedules.delete_workflow_schedule_route": "workflow_schedule.delete",diff --git a/skyvern/forge/sdk/workflow/credential_fallback.py b/skyvern/forge/sdk/workflow/credential_fallback.pyindex 41103c407..09d23b23e 100644--- a/skyvern/forge/sdk/workflow/credential_fallback.py+++ b/skyvern/forge/sdk/workflow/credential_fallback.py@@ -128,6 +128,13 @@ async def _maybe_start_credential_fallback_retry(workflow_run: WorkflowRun, orgaif not await _retry_enabled_for_organization(organization_id, workflow_run):return None+ # A run group moves to its next item when this child fails, so a fallback run would act beside it under+ # another login. A lookup error raises, and the wrapper then skips the retry.+ if workflow_run.start_fresh_browser and await app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id(+ workflow_run.workflow_run_id+ ):+ return None+is_block_scoped_run = await app.AGENT_FUNCTION.is_block_scoped_workflow_run(workflow_run)if not is_block_scoped_run:is_block_scoped_run = await app.DATABASE.debug.has_block_run_for_workflow_run(diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex fca870a84..8ecb82cfb 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -18571,18 +18571,11 @@ class WorkflowTriggerBlock(Block):) -> list[PARAMETER_TYPE]:return self.parameters- async def _check_trigger_depth(self, workflow_run_id: str) -> int:- """Check the nesting depth of workflow triggers to prevent infinite recursion.-- Note: This depth guard walks the parent_workflow_run_id chain, which is only- populated for synchronous triggers. For async (fire-and-forget) dispatch, the- parent may have already completed before the child runs, so circular async- chains (A->B->A) are only blocked while A is still running. A full- visited-workflow guard would require persistent state and is left as a future- enhancement.- """+ async def _check_trigger_depth(self, workflow_run_id: str) -> str:+ """Return the root run of the parent_workflow_run_id chain, raising past MAX_TRIGGER_DEPTH. An async child's+ parent may finish first, so a circular async chain (A->B->A) is blocked only while A is still running."""depth = 0- current_run_id: str | None = workflow_run_id+ current_run_id = workflow_run_idwhile current_run_id:if depth >= self.MAX_TRIGGER_DEPTH:raise InvalidWorkflowDefinition(@@ -18594,7 +18587,7 @@ class WorkflowTriggerBlock(Block):breakcurrent_run_id = run.parent_workflow_run_iddepth += 1- return depth+ return current_run_iddef _render_template_value(self,@@ -18817,9 +18810,16 @@ class WorkflowTriggerBlock(Block):# 2. Check recursion depthtry:- await self._check_trigger_depth(workflow_run_id)+ root_workflow_run_id = await self._check_trigger_depth(workflow_run_id)except InvalidWorkflowDefinition as e:return await _fail(str(e))+ if (+ not self.wait_for_completion or self.browser_session_id+ ) and await app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id(root_workflow_run_id):+ return await _fail(+ "A workflow run group child cannot trigger a workflow without waiting for it"+ " or into a supplied browser session"+ )# 3. Get the organizationif not organization_id:diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 68a4695e2..ef376c8fa 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -11160,6 +11160,7 @@ class WorkflowService:created_by=created_by,edited_by=edited_by,preserve_completion_contract=preserve_completion_contract,+ refuse_if_pinned_by_run_group=True,)else:updated_workflow = await app.DATABASE.workflows.update_workflow(@@ -11193,6 +11194,7 @@ class WorkflowService:sequential_key=sequential_key,created_by=created_by,edited_by=edited_by,+ refuse_if_pinned_by_run_group=True,)if notify_workflow_saved:@@ -12056,6 +12058,10 @@ class WorkflowService:task = asyncio.create_task(hook_coroutine)self._background_tasks.add(task)task.add_done_callback(self._background_tasks.discard)+ try:+ app.AGENT_FUNCTION.schedule_workflow_run_group_advance(workflow_run)+ except Exception:+ LOG.warning("Failed to schedule workflow run group advance", workflow_run_id=workflow_run_id, exc_info=True)async def _run_workflow_run_terminal_hooks(self,@@ -12869,6 +12875,7 @@ class WorkflowService:failure_category: list[dict] | None = None,finalized_by: str | None = None,run_minutes_recorded_through: datetime | None = None,+ only_from: Sequence[WorkflowRunStatus] | None = None,) -> WorkflowRun | None:""":meth:`_update_workflow_run_status` for writers that must lose a race against therun's own finalizer. Returns ``None`` when the row was already terminal."""@@ -12878,6 +12885,7 @@ class WorkflowService:failure_reason=failure_reason,run_with=run_with,failure_category=failure_category,+ only_from=only_from,)if workflow_run is None:return None@@ -13482,6 +13490,7 @@ class WorkflowService:failure_reason: str | None,failure_category: list[dict] | None = None,cascade_children: bool = False,+ only_from: Sequence[WorkflowRunStatus] | None = None,) -> WorkflowRun | None:"""Conditional failure finalize that no-ops when the run has already reached a terminalstate. Out-of-band finalizers (the interrupted-activity backstop) run concurrently with@@ -13500,6 +13509,7 @@ class WorkflowService:status=WorkflowRunStatus.failed,failure_reason=failure_reason,failure_category=failure_category,+ only_from=only_from,)if workflow_run is None:return Nonediff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex 502654bc6..5279b5bce 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -6946,7 +6946,8 @@ _FIND_MENU_JS = (const tag = el.tagName;const box = rowBox(el);if (!box) {- if (arg.saltUnlisted && vis(el.getBoundingClientRect()) && (el.innerText || '').trim() && ownClickable(el)) unreadLong.push(el);+ // innerText walks the subtree, so it runs after the cheaper checks that reject most containers.+ if (arg.saltUnlisted && vis(el.getBoundingClientRect()) && ownClickable(el) && (el.innerText || '').trim()) unreadLong.push(el);continue;}const { r, txt } = box;diff --git a/skyvern/schemas/workflow_run_groups.py b/skyvern/schemas/workflow_run_groups.pynew file mode 100644index 000000000..08654e035--- /dev/null+++ b/skyvern/schemas/workflow_run_groups.py@@ -0,0 +1,127 @@+from datetime import datetime+from enum import StrEnum+from typing import Any++from pydantic import BaseModel, ConfigDict, Field, model_validator++from skyvern.constants import SCRUBBED_VALUE+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus++MAX_WORKFLOW_RUN_GROUP_ITEMS = 25+SCRUBBED_SUBMISSION_KEY_PREFIX = "scrubbed:"+++class WorkflowRunGroupStatus(StrEnum):+ active = "active"+ cancel_requested = "cancel_requested"+ finished = "finished"+++class WorkflowRunGroupItemState(StrEnum):+ pending = "pending"+ dispatching = "dispatching"+ dispatched = "dispatched"+ done = "done"+ canceled = "canceled"+ failed_to_start = "failed_to_start"++ def is_final(self) -> bool:+ return self in (+ WorkflowRunGroupItemState.done,+ WorkflowRunGroupItemState.canceled,+ WorkflowRunGroupItemState.failed_to_start,+ )+++IN_FLIGHT_ITEM_STATES = (WorkflowRunGroupItemState.dispatching, WorkflowRunGroupItemState.dispatched)+++class WorkflowRunGroupItemOutcome(StrEnum):+ pending = "pending"+ in_progress = "in_progress"+ completed = "completed"+ failed = "failed"+ unknown = "unknown"+ canceled = "canceled"+++class WorkflowRunGroup(BaseModel):+ model_config = ConfigDict(from_attributes=True)++ workflow_run_group_id: str+ organization_id: str+ workflow_permanent_id: str+ requested_version: int | None+ workflow_id: str+ submission_key: str+ input_fingerprint: str+ status: WorkflowRunGroupStatus+ created_at: datetime+ modified_at: datetime+ finished_at: datetime | None+++class WorkflowRunGroupItem(BaseModel):+ model_config = ConfigDict(from_attributes=True)++ workflow_run_group_id: str+ position: int+ item_key: str+ parameters: dict[str, Any]+ workflow_run_id: str+ state: WorkflowRunGroupItemState+ dispatch_token: str | None+ claimed_at: datetime | None+ dispatch_attempts: int+ failure_reason: str | None+++class WorkflowRunGroupItemRequest(BaseModel):+ # Forbidding extra fields rejects per-item browser session, address or profile inputs.+ model_config = ConfigDict(extra="forbid")++ key: str = Field(min_length=1, max_length=255)+ parameters: dict[str, Any] = Field(default_factory=dict)+++class WorkflowRunGroupCreateRequest(BaseModel):+ model_config = ConfigDict(extra="forbid")++ workflow_id: str = Field(description="The workflow permanent id (wpid_...) every item runs against.")+ version: int | None = Field(default=None, ge=1)+ submission_key: str = Field(min_length=1, max_length=255)+ items: list[WorkflowRunGroupItemRequest] = Field(min_length=1, max_length=MAX_WORKFLOW_RUN_GROUP_ITEMS)++ @model_validator(mode="after")+ def _validate_keys(self) -> "WorkflowRunGroupCreateRequest":+ keys = [item.key for item in self.items]+ if len(set(keys)) != len(keys):+ raise ValueError("item keys must be unique within a group")+ if self.submission_key.startswith(SCRUBBED_SUBMISSION_KEY_PREFIX):+ raise ValueError(f"submission_key cannot start with {SCRUBBED_SUBMISSION_KEY_PREFIX!r}")+ # The retention scrub renames item keys into this space, and a caller key already there would collide.+ if any(key.startswith(SCRUBBED_VALUE) for key in keys):+ raise ValueError(f"item keys cannot start with {SCRUBBED_VALUE!r}")+ return self+++class WorkflowRunGroupItemResponse(BaseModel):+ key: str+ position: int+ workflow_run_id: str+ state: WorkflowRunGroupItemState+ run_status: WorkflowRunStatus | None+ outcome: WorkflowRunGroupItemOutcome+ failure_reason: str | None+++class WorkflowRunGroupResponse(BaseModel):+ workflow_run_group_id: str+ workflow_permanent_id: str+ workflow_id: str+ requested_version: int | None+ submission_key: str+ status: WorkflowRunGroupStatus+ created_at: datetime+ finished_at: datetime | None+ items: list[WorkflowRunGroupItemResponse]diff --git a/skyvern/services/workflow_run_group_service.py b/skyvern/services/workflow_run_group_service.pynew file mode 100644index 000000000..4ecbb271d--- /dev/null+++ b/skyvern/services/workflow_run_group_service.py@@ -0,0 +1,649 @@+from __future__ import annotations++import asyncio+import contextvars+import hmac+import json+import uuid+from collections.abc import Coroutine+from datetime import datetime, timedelta+from hashlib import sha256+from http import HTTPStatus+from typing import Any++import structlog+from fastapi import HTTPException+from sqlalchemy.exc import IntegrityError++from skyvern.config import settings+from skyvern.exceptions import SkyvernHTTPException, WorkflowHasNoBlocks+from skyvern.forge import app+from skyvern.forge.sdk.core.permissions.permission_checker_factory import PermissionCheckerFactory+from skyvern.forge.sdk.db.datetime_utils import naive_utc_now, to_naive_utc+from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType+from skyvern.forge.sdk.db.repositories.workflow_run_groups import scrubbed_submission_key+from skyvern.forge.sdk.executor import factory as executor_factory+from skyvern.forge.sdk.schemas.organizations import Organization+from skyvern.forge.sdk.workflow.models.block import (+ BaseTaskBlock,+ HumanInteractionBlock,+ WorkflowTriggerBlock,+ get_all_blocks,+)+from skyvern.forge.sdk.workflow.models.workflow import (+ Workflow,+ WorkflowRequestBody,+ WorkflowRun,+ WorkflowRunStatus,+)+from skyvern.forge.sdk.workflow.retry_policy import resolve_workflow_retry_policy+from skyvern.schemas.workflow_run_groups import (+ IN_FLIGHT_ITEM_STATES,+ WorkflowRunGroup,+ WorkflowRunGroupCreateRequest,+ WorkflowRunGroupItem,+ WorkflowRunGroupItemOutcome,+ WorkflowRunGroupItemResponse,+ WorkflowRunGroupItemState,+ WorkflowRunGroupResponse,+ WorkflowRunGroupStatus,+)+from skyvern.services import run_service, workflow_service+from skyvern.services.organization_log_scope import organization_log_scope++LOG = structlog.get_logger()++DISPATCH_GRACE = timedelta(minutes=10)+MAX_DISPATCH_ATTEMPTS = 3+# ponytail: a fixed settle window, because a group child records no "worker stopped" signal (it has no attempt row).+# It outlasts the longest step that ignores a cancel (a code block); a worker slower than that can still overlap.+CHILD_STOP_SETTLE = timedelta(seconds=settings.CODE_BLOCK_EXECUTION_TIMEOUT_SECONDS + 60)+# An outside writer (a run cancel, the stuck-run reaper) can set these while the child's worker is still running.+_STATUSES_SET_BESIDE_A_LIVE_WORKER = (WorkflowRunStatus.canceled, WorkflowRunStatus.timed_out)+RECOVERY_BATCH_SIZE = 200+RECOVERY_INTERVAL_SECONDS = 60+_FINGERPRINT_DOMAIN = b"skyvern.workflow_run_group_submission.v1\0"+_background_tasks: set[asyncio.Task[None]] = set()+++def submission_fingerprint(request: WorkflowRunGroupCreateRequest) -> str:+ payload = {+ "workflow_id": request.workflow_id,+ "version": request.version,+ "items": [{"key": item.key, "parameters": item.parameters} for item in request.items],+ }+ serialized = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)+ return hmac.new(settings.SECRET_KEY.encode(), _FINGERPRINT_DOMAIN + serialized.encode(), sha256).hexdigest()+++def non_fresh_browser_reason(workflow: Workflow) -> str | None:+ if resolve_workflow_retry_policy(workflow) is not None:+ return "a retry policy, which could replay a child whose effects already happened"+ if workflow.persist_browser_session:+ return "persist_browser_session"+ if workflow.browser_profile_id or workflow.browser_profile_key:+ return "a browser profile"+ if workflow.sequential_key:+ return "a sequential_key"+ if workflow.workflow_definition.finally_block_label:+ return "a finally block, which runs after the run first reports a final status"+ for block in get_all_blocks(workflow.workflow_definition.blocks):+ if not isinstance(block, WorkflowTriggerBlock):+ continue+ if block.browser_session_id:+ return f"block {block.label} triggering a workflow into a supplied browser_session_id"+ if not block.wait_for_completion:+ return (+ f"block {block.label} triggering a workflow without waiting for it, so that run could outlive its item"+ )+ return None+++async def _existing_group_or_conflict(+ group: WorkflowRunGroup, fingerprint: str, organization_id: str+) -> WorkflowRunGroupResponse:+ if not hmac.compare_digest(group.input_fingerprint, fingerprint):+ raise SkyvernHTTPException(+ "submission_key was already used with different input",+ status_code=HTTPStatus.CONFLICT,+ )+ return await get_workflow_run_group(group.workflow_run_group_id, organization_id)+++async def submit_workflow_run_group(+ organization: Organization,+ request: WorkflowRunGroupCreateRequest,+ *,+ expected_workflow_modified_at: datetime | None = None,+) -> WorkflowRunGroupResponse:+ organization_id = organization.organization_id+ fingerprint = submission_fingerprint(request)+ existing = await app.DATABASE.workflow_run_groups.get_group_by_submission_key(+ organization_id, request.submission_key+ )+ if existing is not None:+ return await _existing_group_or_conflict(existing, fingerprint, organization_id)+ if await app.DATABASE.workflow_run_groups.get_group_by_submission_key(+ organization_id, scrubbed_submission_key(request.submission_key)+ ):+ raise SkyvernHTTPException(+ "submission_key belongs to a group whose records were removed by the organization's retention policy",+ status_code=HTTPStatus.CONFLICT,+ )+ if not settings.WORKFLOW_RUN_GROUPS_SUBMIT_ENABLED:+ raise SkyvernHTTPException(+ "Workflow run group submission is disabled", status_code=HTTPStatus.SERVICE_UNAVAILABLE+ )+ await app.RATE_LIMITER.rate_limit_submit_run(organization_id)++ workflow = await app.WORKFLOW_SERVICE.get_workflow_by_permanent_id(+ workflow_permanent_id=request.workflow_id,+ organization_id=organization_id,+ version=request.version,+ )+ if not workflow.workflow_definition.blocks:+ raise WorkflowHasNoBlocks(workflow_permanent_id=workflow.workflow_permanent_id)+ if reason := non_fresh_browser_reason(workflow):+ raise SkyvernHTTPException(+ f"Workflow version {workflow.version} cannot run as a group because it has {reason}",+ status_code=HTTPStatus.BAD_REQUEST,+ )+ input_parameters = {+ parameter.key: parameter+ for parameter in await app.WORKFLOW_SERVICE.get_workflow_parameters(workflow_id=workflow.workflow_id)+ }+ for item in request.items:+ await app.WORKFLOW_SERVICE.validate_schedule_parameters(workflow, organization, item.parameters)+ # Values are otherwise converted only when a child is created, so a mistyped one would fail mid-group.+ for key, value in item.parameters.items():+ if value is not None and key in input_parameters:+ app.WORKFLOW_SERVICE._serialize_workflow_run_parameter_value(input_parameters[key], value)++ try:+ group = await app.DATABASE.workflow_run_groups.create_group(+ organization_id=organization_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ requested_version=request.version,+ workflow_id=workflow.workflow_id,+ submission_key=request.submission_key,+ input_fingerprint=fingerprint,+ items=[(item.key, item.parameters) for item in request.items],+ # Without a reviewed timestamp, bind to the row validation read so an edit landing after it is refused.+ expected_workflow_modified_at=expected_workflow_modified_at or workflow.modified_at,+ )+ except IntegrityError:+ raced = await app.DATABASE.workflow_run_groups.get_group_by_submission_key(+ organization_id, request.submission_key+ )+ if raced is None:+ raise+ return await _existing_group_or_conflict(raced, fingerprint, organization_id)++ LOG.info(+ "Workflow run group submitted",+ workflow_run_group_id=group.workflow_run_group_id,+ workflow_id=group.workflow_id,+ item_count=len(request.items),+ )+ _spawn(_advance_in_org_scope(group.workflow_run_group_id, organization_id))+ return await get_workflow_run_group(group.workflow_run_group_id, organization_id)+++def _is_task_family_only(workflow: Workflow | None) -> bool:+ if workflow is None or not workflow.workflow_definition.blocks:+ return False+ # Task-family blocks act only through recorded steps; a human interaction block sends email without one.+ return all(+ isinstance(block, BaseTaskBlock) and not isinstance(block, HumanInteractionBlock)+ for block in workflow.workflow_definition.blocks+ )+++def classify_item_outcome(+ item_state: WorkflowRunGroupItemState,+ child: WorkflowRun | None,+ *,+ task_family_only: bool,+ step_count: int | None,+) -> WorkflowRunGroupItemOutcome:+ if item_state == WorkflowRunGroupItemState.failed_to_start:+ # A child can already have run to a final status before its item was failed; its effects are then unknown.+ if child is not None and child.started_at is not None:+ return WorkflowRunGroupItemOutcome.unknown+ return WorkflowRunGroupItemOutcome.failed+ if item_state == WorkflowRunGroupItemState.pending:+ return WorkflowRunGroupItemOutcome.pending+ if child is None:+ if item_state == WorkflowRunGroupItemState.canceled:+ return WorkflowRunGroupItemOutcome.canceled+ return WorkflowRunGroupItemOutcome.in_progress+ if not child.status.is_final():+ if item_state == WorkflowRunGroupItemState.canceled:+ return WorkflowRunGroupItemOutcome.canceled+ return WorkflowRunGroupItemOutcome.in_progress+ if child.status == WorkflowRunStatus.completed:+ return WorkflowRunGroupItemOutcome.completed+ if child.status == WorkflowRunStatus.canceled and child.started_at is None:+ return WorkflowRunGroupItemOutcome.canceled+ if (+ child.status in (WorkflowRunStatus.failed, WorkflowRunStatus.terminated)+ and task_family_only+ and step_count == 0+ and child.script_run is None+ ):+ return WorkflowRunGroupItemOutcome.failed+ return WorkflowRunGroupItemOutcome.unknown+++async def get_workflow_run_group(workflow_run_group_id: str, organization_id: str) -> WorkflowRunGroupResponse:+ group = await app.DATABASE.workflow_run_groups.get_group(workflow_run_group_id, organization_id)+ if group is None:+ raise SkyvernHTTPException(+ f"Workflow run group {workflow_run_group_id} not found", status_code=HTTPStatus.NOT_FOUND+ )+ items = await app.DATABASE.workflow_run_groups.get_items(workflow_run_group_id)+ children = {+ run.workflow_run_id: run+ for run in await app.DATABASE.workflow_runs.get_workflow_runs_by_ids(+ [item.workflow_run_id for item in items], organization_id=organization_id+ )+ }+ task_family_only: bool | None = None+ item_responses = []+ for item in items:+ child = children.get(item.workflow_run_id)+ step_count = None+ if child is not None and child.status in (WorkflowRunStatus.failed, WorkflowRunStatus.terminated):+ if task_family_only is None:+ task_family_only = _is_task_family_only(+ await app.DATABASE.workflows.get_workflow(group.workflow_id, organization_id)+ )+ if task_family_only:+ step_count = await app.DATABASE.workflow_run_groups.count_steps(item.workflow_run_id)+ item_responses.append(+ WorkflowRunGroupItemResponse(+ key=item.item_key,+ position=item.position,+ workflow_run_id=item.workflow_run_id,+ state=item.state,+ run_status=child.status if child else None,+ outcome=classify_item_outcome(+ item.state, child, task_family_only=bool(task_family_only), step_count=step_count+ ),+ failure_reason=item.failure_reason or (child.failure_reason if child else None),+ )+ )+ return WorkflowRunGroupResponse(+ workflow_run_group_id=group.workflow_run_group_id,+ workflow_permanent_id=group.workflow_permanent_id,+ workflow_id=group.workflow_id,+ requested_version=group.requested_version,+ submission_key=group.submission_key,+ status=group.status,+ created_at=group.created_at,+ finished_at=group.finished_at,+ items=item_responses,+ )+++async def cancel_workflow_run_group(workflow_run_group_id: str, organization_id: str) -> WorkflowRunGroupResponse:+ group = await app.DATABASE.workflow_run_groups.request_cancel(workflow_run_group_id, organization_id)+ if group is None:+ raise SkyvernHTTPException(+ f"Workflow run group {workflow_run_group_id} not found", status_code=HTTPStatus.NOT_FOUND+ )+ await advance_workflow_run_group(workflow_run_group_id)+ return await get_workflow_run_group(workflow_run_group_id, organization_id)+++async def _cancel_child(workflow_run_id: str, organization_id: str) -> None:+ try:+ child = await app.DATABASE.workflow_runs.get_workflow_run(workflow_run_id, organization_id=organization_id)+ # On a final run, cancel_workflow_run still resends the webhook, deletes attached files and revokes the attempt.+ if child is None or child.status.is_final():+ return+ await run_service.cancel_workflow_run(workflow_run_id, organization_id)+ except Exception:+ LOG.warning("Failed to cancel workflow run group child", workflow_run_id=workflow_run_id, exc_info=True)+++async def _fail_unstarted_child(workflow_run_id: str, failure_reason: str) -> bool:+ """Fail a child that is still `created`, meaning no executor ever took it. False when it moved on or is missing."""+ failed = await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(+ workflow_run_id=workflow_run_id,+ failure_reason=failure_reason,+ only_from=(WorkflowRunStatus.created,),+ )+ return failed is not None+++async def _fail_item_to_start(+ item: WorkflowRunGroupItem,+ dispatch_token: str,+ failure_reason: str,+ organization_id: str,+ from_state: WorkflowRunGroupItemState = WorkflowRunGroupItemState.dispatching,+) -> None:+ # Holding the item in flight first fences out a stale owner whose token was reclaimed, and keeps the next item+ # from starting while an executor that took this child in the meantime may still be running it.+ if not await app.DATABASE.workflow_run_groups.finish_item(+ item.workflow_run_group_id,+ item.position,+ state=WorkflowRunGroupItemState.dispatched,+ from_states=(from_state,),+ dispatch_token=dispatch_token,+ ):+ return+ if not await _fail_unstarted_child(item.workflow_run_id, failure_reason):+ child = await app.DATABASE.workflow_runs.get_workflow_run(item.workflow_run_id, organization_id=organization_id)+ if child is not None and not child.status.is_final():+ # Reconcile releases the item once the canceled child has stopped and settled.+ await _cancel_child(item.workflow_run_id, organization_id)+ return+ if child is not None and _worker_may_still_be_stopping(child):+ return+ # The failed child's terminal hook may already have marked the item done.+ await app.DATABASE.workflow_run_groups.finish_item(+ item.workflow_run_group_id,+ item.position,+ state=WorkflowRunGroupItemState.failed_to_start,+ from_states=(WorkflowRunGroupItemState.dispatched, WorkflowRunGroupItemState.done),+ dispatch_token=dispatch_token,+ failure_reason=failure_reason,+ )+++def _failure_reason(message: str, error: Exception) -> str:+ # Exception text can echo item parameters such as credential ids, and this reason is stored and returned.+ return f"{message} ({type(error).__name__})"+++async def _dispatch_item(group: WorkflowRunGroup, item: WorkflowRunGroupItem, dispatch_token: str) -> bool:+ """Prepare and execute one claimed item. Returns True while the item stays in flight."""+ # setup_workflow_run replaces the calling task's SkyvernContext and inherits its run ids, so a dispatch that shared+ # its caller's context would hand the previous child's ids to this one.+ return await asyncio.create_task(+ _prepare_and_execute_item_in_org_scope(group, item, dispatch_token), context=contextvars.Context()+ )+++async def _prepare_and_execute_item_in_org_scope(+ group: WorkflowRunGroup, item: WorkflowRunGroupItem, dispatch_token: str+) -> bool:+ async with organization_log_scope(group.organization_id):+ return await _prepare_and_execute_item(group, item, dispatch_token)+++async def _prepare_and_execute_item(group: WorkflowRunGroup, item: WorkflowRunGroupItem, dispatch_token: str) -> bool:+ organization = await app.DATABASE.organizations.get_organization(group.organization_id)+ if organization is None:+ await _fail_item_to_start(item, dispatch_token, "Organization not found", group.organization_id)+ return False+ try:+ await PermissionCheckerFactory.get_instance().check(organization)+ workflow_run = await workflow_service.prepare_workflow(+ workflow_id=group.workflow_permanent_id,+ organization=organization,+ workflow_request=WorkflowRequestBody(+ data=item.parameters, start_fresh_browser=True, reuse_browser_session=False+ ),+ trigger_type=WorkflowRunTriggerType.api,+ workflow_run_id=item.workflow_run_id,+ resolved_workflow_id=group.workflow_id,+ )+ except IntegrityError:+ # The task_run row is written last, so its presence means an earlier preparation completed.+ if await app.DATABASE.tasks.get_run(item.workflow_run_id, organization_id=group.organization_id) is None:+ await _fail_item_to_start(+ item, dispatch_token, "Workflow run preparation was interrupted", group.organization_id+ )+ return False+ existing_run = await app.DATABASE.workflow_runs.get_workflow_run(+ item.workflow_run_id, organization_id=group.organization_id+ )+ if existing_run is None:+ await _fail_item_to_start(+ item, dispatch_token, "Workflow run preparation was interrupted", group.organization_id+ )+ return False+ workflow_run = existing_run+ except SkyvernHTTPException as e:+ LOG.warning(+ "Workflow run group child could not be prepared",+ workflow_run_group_id=group.workflow_run_group_id,+ workflow_run_id=item.workflow_run_id,+ exc_info=True,+ )+ await _fail_item_to_start(+ item, dispatch_token, _failure_reason("Workflow run could not be prepared", e), group.organization_id+ )+ return False+ except HTTPException as e:+ LOG.warning(+ "Organization cannot start workflow run group child",+ workflow_run_group_id=group.workflow_run_group_id,+ workflow_run_id=item.workflow_run_id,+ exc_info=True,+ )+ await _fail_item_to_start(+ item, dispatch_token, _failure_reason("Organization cannot start runs", e), group.organization_id+ )+ return False+ except Exception:+ LOG.exception(+ "Failed to prepare workflow run group child",+ workflow_run_group_id=group.workflow_run_group_id,+ workflow_run_id=item.workflow_run_id,+ dispatch_attempts=item.dispatch_attempts,+ )+ if item.dispatch_attempts >= MAX_DISPATCH_ATTEMPTS:+ await _fail_item_to_start(+ item, dispatch_token, "Workflow run preparation kept failing", group.organization_id+ )+ return False+ return True++ if workflow_run.status.is_final():+ await app.DATABASE.workflow_run_groups.finish_item(+ group.workflow_run_group_id,+ item.position,+ state=WorkflowRunGroupItemState.done,+ from_states=(WorkflowRunGroupItemState.dispatching,),+ dispatch_token=dispatch_token,+ )+ return False++ flip = await app.DATABASE.workflow_run_groups.flip_to_dispatched(+ group.workflow_run_group_id, item.position, dispatch_token+ )+ if flip == "lost":+ return True+ if flip == "canceled":+ await _cancel_child(item.workflow_run_id, group.organization_id)+ return False++ LOG.info(+ "Dispatching workflow run group child",+ workflow_run_group_id=group.workflow_run_group_id,+ workflow_run_id=item.workflow_run_id,+ position=item.position,+ )+ try:+ await executor_factory.AsyncExecutorFactory.get_executor().execute_workflow(+ request=None,+ background_tasks=None,+ organization=organization,+ workflow_id=workflow_run.workflow_id,+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_permanent_id=workflow_run.workflow_permanent_id,+ max_steps_override=None,+ api_key=None,+ browser_session_id=workflow_run.browser_session_id,+ block_labels=None,+ block_outputs=None,+ )+ except Exception:+ LOG.exception(+ "Failed to execute workflow run group child",+ workflow_run_group_id=group.workflow_run_group_id,+ workflow_run_id=item.workflow_run_id,+ )+ return True+++def _worker_may_still_be_stopping(child: WorkflowRun) -> bool:+ if child.status not in _STATUSES_SET_BESIDE_A_LIVE_WORKER:+ return False+ # The reaper's bulk write sets the status without finished_at, so fall back to the row's last write.+ ended_at = to_naive_utc(child.finished_at or child.modified_at)+ return ended_at is None or ended_at > naive_utc_now() - CHILD_STOP_SETTLE+++async def _reconcile_in_flight(group: WorkflowRunGroup, items: list[WorkflowRunGroupItem]) -> None:+ in_flight = [item for item in items if item.state in IN_FLIGHT_ITEM_STATES]+ canceling = group.status != WorkflowRunGroupStatus.active+ if canceling:+ in_flight += [item for item in items if item.state == WorkflowRunGroupItemState.canceled]+ if not in_flight:+ return+ children = {+ run.workflow_run_id: run+ for run in await app.DATABASE.workflow_runs.get_workflow_runs_by_ids(+ [item.workflow_run_id for item in in_flight], organization_id=group.organization_id+ )+ }+ stale_before = naive_utc_now() - DISPATCH_GRACE+ for item in in_flight:+ child = children.get(item.workflow_run_id)+ claimed_at = to_naive_utc(item.claimed_at)+ is_stale = claimed_at is not None and claimed_at < stale_before+ if child is not None and not child.status.is_final() and canceling:+ await _cancel_child(item.workflow_run_id, group.organization_id)+ child = await app.DATABASE.workflow_runs.get_workflow_run(item.workflow_run_id)+ if item.state == WorkflowRunGroupItemState.canceled:+ continue+ if child is not None and child.status.is_final():+ # A canceling group starts nothing after this item, so only an active one has to wait out the settle.+ if not canceling and _worker_may_still_be_stopping(child):+ continue+ await app.DATABASE.workflow_run_groups.finish_item(+ group.workflow_run_group_id,+ item.position,+ state=WorkflowRunGroupItemState.done,+ from_states=IN_FLIGHT_ITEM_STATES,+ )+ continue+ if not is_stale:+ continue+ if item.state == WorkflowRunGroupItemState.dispatched:+ # Only a child no executor ever took is failed; a queued or running one is left to the stuck-run reaper.+ # A dispatched item with no child row was held by a failure that stopped before releasing it.+ unstarted = child is None or child.status == WorkflowRunStatus.created+ if unstarted and item.dispatch_token is not None:+ await _fail_item_to_start(+ item,+ item.dispatch_token,+ "Workflow run dispatch was interrupted",+ group.organization_id,+ from_state=WorkflowRunGroupItemState.dispatched,+ )+ continue+ dispatch_token = uuid.uuid4().hex+ reclaimed = await app.DATABASE.workflow_run_groups.reclaim_stale_item(+ group.workflow_run_group_id,+ item.position,+ dispatch_token=dispatch_token,+ stale_before=stale_before,+ )+ if reclaimed is None:+ continue+ if reclaimed.dispatch_attempts > MAX_DISPATCH_ATTEMPTS:+ await _fail_item_to_start(+ reclaimed, dispatch_token, "Workflow run dispatch kept stalling", group.organization_id+ )+ continue+ if canceling:+ await _cancel_child(item.workflow_run_id, group.organization_id)+ await app.DATABASE.workflow_run_groups.finish_item(+ group.workflow_run_group_id,+ item.position,+ state=WorkflowRunGroupItemState.canceled,+ from_states=(WorkflowRunGroupItemState.dispatching,),+ dispatch_token=dispatch_token,+ )+ continue+ await _dispatch_item(group, reclaimed, dispatch_token)+++async def advance_workflow_run_group(workflow_run_group_id: str) -> None:+ group = await app.DATABASE.workflow_run_groups.get_group(workflow_run_group_id)+ if group is None or group.status == WorkflowRunGroupStatus.finished:+ return+ await _reconcile_in_flight(group, await app.DATABASE.workflow_run_groups.get_items(workflow_run_group_id))+ while True:+ dispatch_token = uuid.uuid4().hex+ item = await app.DATABASE.workflow_run_groups.claim_next_item(workflow_run_group_id, dispatch_token)+ if item is None or await _dispatch_item(group, item, dispatch_token):+ break+ await app.DATABASE.workflow_run_groups.finish_group_if_complete(workflow_run_group_id)+++async def _advance_in_org_scope(workflow_run_group_id: str, organization_id: str) -> None:+ async with organization_log_scope(organization_id):+ try:+ await advance_workflow_run_group(workflow_run_group_id)+ except Exception:+ LOG.exception("Failed to advance workflow run group", workflow_run_group_id=workflow_run_group_id)+++async def _advance_after_child_terminal(workflow_run_id: str, organization_id: str) -> None:+ try:+ item = await app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id(workflow_run_id)+ except Exception:+ LOG.warning("Failed to look up workflow run group item", workflow_run_id=workflow_run_id, exc_info=True)+ return+ if item is not None:+ await _advance_in_org_scope(item.workflow_run_group_id, organization_id)+++def _spawn(coroutine: Coroutine[Any, Any, None]) -> None:+ # An empty context keeps the finishing run's SkyvernContext out of the next child's preparation.+ task = asyncio.create_task(coroutine, context=contextvars.Context())+ _background_tasks.add(task)+ task.add_done_callback(_background_tasks.discard)+++def schedule_advance_after_terminal(workflow_run: WorkflowRun) -> None:+ if not workflow_run.start_fresh_browser or workflow_run.parent_workflow_run_id is not None:+ return+ _spawn(_advance_after_child_terminal(workflow_run.workflow_run_id, workflow_run.organization_id))+++async def is_workflow_run_group_child(workflow_run: WorkflowRun) -> bool:+ if workflow_run.start_fresh_browser is not True:+ return False+ try:+ item = await app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id(workflow_run.workflow_run_id)+ except Exception:+ LOG.warning(+ "Failed to look up workflow run group item", workflow_run_id=workflow_run.workflow_run_id, exc_info=True+ )+ # Fail closed: a group child must not reuse a rotated browser, even though this moves a non-group run off its+ # configured browser addresses for this dispatch.+ return True+ return item is not None+++async def recover_workflow_run_groups() -> None:+ groups = await app.DATABASE.workflow_run_groups.list_unfinished_groups(RECOVERY_BATCH_SIZE)+ for workflow_run_group_id, organization_id in groups:+ await _advance_in_org_scope(workflow_run_group_id, organization_id)+++async def run_workflow_run_group_recovery_loop() -> None:+ while True:+ try:+ await recover_workflow_run_groups()+ except Exception:+ LOG.exception("Workflow run group recovery sweep failed")+ await asyncio.sleep(RECOVERY_INTERVAL_SECONDS)diff --git a/tests/unit/force_stub_app.py b/tests/unit/force_stub_app.pyindex db11a77ab..ce9f33668 100644--- a/tests/unit/force_stub_app.py+++ b/tests/unit/force_stub_app.py@@ -88,6 +88,7 @@ def create_forge_stub_app() -> ForgeApp:fake_app_module.AGENT_FUNCTION.is_backup_queue_organization = base_agent_function.is_backup_queue_organizationfake_app_module.AGENT_FUNCTION.resolve_mcp_oauth_org_lookups = MagicMock(return_value=None)fake_app_module.AGENT_FUNCTION.get_mcp_request_organization_id = MagicMock(return_value=None)+ fake_app_module.AGENT_FUNCTION.schedule_workflow_run_group_advance = MagicMock(return_value=None)# Sync method returning a key or None — _LazyNamespace would auto-mock it as a truthy# AsyncMock and hijack the TextPromptBlock llm_key. Match the OSS no-op.fake_app_module.AGENT_FUNCTION.get_fallback_llm_key = MagicMock(return_value=None)diff --git a/tests/unit/test_credential_fallback.py b/tests/unit/test_credential_fallback.pyindex 64a1a650f..2b16f4d29 100644--- a/tests/unit/test_credential_fallback.py+++ b/tests/unit/test_credential_fallback.py@@ -65,6 +65,7 @@ def _workflow_run(extra_http_headers: dict[str, str] | None = None,workflow_schedule_id: str | None = None,copilot_session_id: str | None = None,+ start_fresh_browser: bool | None = None,) -> WorkflowRun:now = datetime.now(timezone.utc)return WorkflowRun(@@ -86,6 +87,7 @@ def _workflow_run(extra_http_headers=extra_http_headers,workflow_schedule_id=workflow_schedule_id,copilot_session_id=copilot_session_id,+ start_fresh_browser=start_fresh_browser,trigger_type=WorkflowRunTriggerType.api,created_at=now,modified_at=now,@@ -269,6 +271,7 @@ async def _run_fallback_retry(flag_enabled: bool = True,flag_error: Exception | None = None,run_tags: dict[str, str] | None = None,+ run_group_item: object | None = None,) -> tuple[str | None, AsyncMock, MagicMock]:captured: dict[str, object] = {}@@ -297,6 +300,7 @@ async def _run_fallback_retry(mock_app.AGENT_FUNCTION.is_block_scoped_workflow_run = AsyncMock(return_value=block_scoped)mock_app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers = _strip_marker_headermock_app.DATABASE.debug.has_block_run_for_workflow_run = AsyncMock(return_value=False)+ mock_app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id = AsyncMock(return_value=run_group_item)mock_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(return_value=_workflow(parameters))if retried_by_results is not None:mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by = AsyncMock(side_effect=retried_by_results)@@ -411,6 +415,24 @@ async def test_mixed_triggers_advance_each_parameter_from_its_own_prior_selectioassert kwargs["workflow_request"].data["backup_cred"] == "cred_b_fb1"[email protected]("is_group_item", [False, True])+async def test_a_run_group_child_never_starts_a_fallback_run(is_group_item: bool) -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}], start_fresh_browser=True)++ result, run_workflow_mock, _ = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[_credential_parameter(fallback_credential_ids=["cred_fb1"])],+ run_group_item=SimpleNamespace(workflow_run_group_id="wrg_1") if is_group_item else None,+ )++ if is_group_item:+ assert result is None+ run_workflow_mock.assert_not_awaited()+ else:+ assert result == "wr_retry"++@pytest.mark.asyncioasync def test_fallback_exhaustion_creates_no_retry_run() -> None:workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}], fallback_attempt=2)diff --git a/tests/unit/test_workflow_run_groups.py b/tests/unit/test_workflow_run_groups.pynew file mode 100644index 000000000..733e6625d--- /dev/null+++ b/tests/unit/test_workflow_run_groups.py@@ -0,0 +1,1218 @@+import asyncio+from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Mapping, Sequence+from dataclasses import dataclass, field+from datetime import timedelta+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest+import pytest_asyncio+from fastapi import HTTPException+from pydantic import ValidationError+from sqlalchemy import func, select, update+from sqlalchemy.ext.asyncio import AsyncEngine++from skyvern.config import settings+from skyvern.exceptions import SkyvernHTTPException, WorkflowChangedSinceReview, WorkflowPinnedByRunGroup+from skyvern.forge import app+from skyvern.forge.agent_functions import AgentFunction+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.permissions.permission_checkers import PermissionChecker+from skyvern.forge.sdk.db.agent_db import AgentDB+from skyvern.forge.sdk.db.datetime_utils import naive_utc_now+from skyvern.forge.sdk.db.models import (+ Base,+ CredentialModel,+ WorkflowModel,+ WorkflowRunGroupItemModel,+ WorkflowRunGroupModel,+ WorkflowRunModel,+)+from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory+from skyvern.forge.sdk.schemas.organizations import Organization+from skyvern.forge.sdk.schemas.runs import Run+from skyvern.forge.sdk.workflow.models.block import (+ BlockTypeVar,+ ForLoopBlock,+ HttpRequestBlock,+ TaskBlock,+ WorkflowTriggerBlock,+)+from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameterType+from skyvern.forge.sdk.workflow.models.workflow import (+ COPILOT_TEST_WORKFLOW_CREATOR,+ Workflow,+ WorkflowDefinition,+ WorkflowRun,+ WorkflowRunStatus,+)+from skyvern.forge.sdk.workflow.service import WorkflowService+from skyvern.schemas.workflow_run_groups import (+ WorkflowRunGroupCreateRequest,+ WorkflowRunGroupItemOutcome,+ WorkflowRunGroupItemState,+ WorkflowRunGroupStatus,+)+from skyvern.services import workflow_run_group_service as group_service+from tests.unit.conftest import make_block_output_parameter++ORG = "o_test"+OTHER_ORG = "o_other"+WPID = "wpid_test"+++@dataclass+class FakeExecutor:+ database: AgentDB+ executed: list[str] = field(default_factory=list)+ submitted: list[str] = field(default_factory=list)+ before_queue: Callable[[str], Awaitable[None]] | None = None++ async def execute_workflow(self, *, workflow_run_id: str, **_: object) -> None:+ self.executed.append(workflow_run_id)+ if self.before_queue is not None:+ await self.before_queue(workflow_run_id)+ if await self.database.workflow_runs.update_workflow_run_if_not_final(+ workflow_run_id, WorkflowRunStatus.queued+ ):+ self.submitted.append(workflow_run_id)+++@dataclass+class RecordingRateLimiter:+ calls: list[str] = field(default_factory=list)++ async def rate_limit_submit_run(self, organization_id: str) -> None:+ self.calls.append(organization_id)+++@dataclass+class GroupEnv:+ database: AgentDB+ executor: FakeExecutor+ organization: Organization+ spawned: list[Coroutine[Any, Any, None]]+ limiter: RecordingRateLimiter+++def _definition(*blocks: BlockTypeVar) -> dict[str, Any]:+ return WorkflowDefinition(parameters=[], blocks=list(blocks)).model_dump(mode="json")+++def _task_block() -> TaskBlock:+ return TaskBlock(label="login", url="https://example.com", output_parameter=make_block_output_parameter("login"))+++def _looped_trigger(browser_session_id: str | None = None, wait_for_completion: bool = True) -> ForLoopBlock:+ trigger = WorkflowTriggerBlock(+ label="trigger",+ workflow_permanent_id="wpid_child",+ browser_session_id=browser_session_id,+ wait_for_completion=wait_for_completion,+ output_parameter=make_block_output_parameter("trigger"),+ )+ return ForLoopBlock(label="loop", loop_blocks=[trigger], output_parameter=make_block_output_parameter("loop"))+++def _http_block() -> HttpRequestBlock:+ return HttpRequestBlock(+ label="post", url="https://example.com", method="POST", output_parameter=make_block_output_parameter("post")+ )+++@pytest_asyncio.fixture+async def env(monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine) -> AsyncIterator[GroupEnv]:+ database = AgentDB("sqlite+aiosqlite://", db_engine=sqlite_engine)+ organization = await database.organizations.create_organization("Test", organization_id=ORG)+ await database.organizations.create_organization("Other", organization_id=OTHER_ORG)+ async with database.Session() as session:+ session.add(+ WorkflowModel(+ workflow_id="wf_1",+ workflow_permanent_id=WPID,+ organization_id=ORG,+ title="Workflow",+ version=1,+ workflow_definition=_definition(_task_block()),+ )+ )+ session.add_all(+ CredentialModel(+ credential_id=credential_id,+ organization_id=org_id,+ name="Login",+ credential_type="password",+ item_id=f"item_{credential_id}",+ )+ for credential_id, org_id in (("cred_1", ORG), ("cred_2", ORG), ("cred_foreign", OTHER_ORG))+ )+ await session.commit()+ await database.workflow_params.create_workflow_parameter(+ workflow_id="wf_1", workflow_parameter_type=WorkflowParameterType.CREDENTIAL_ID, key="login", default_value=None+ )+ service = WorkflowService()+ executor = FakeExecutor(database)+ spawned: list[Coroutine[Any, Any, None]] = []+ limiter = RecordingRateLimiter()+ monkeypatch.setattr(app, "DATABASE", database)+ monkeypatch.setattr(object.__getattribute__(app, "_inst"), "RATE_LIMITER", limiter, raising=False)+ monkeypatch.setattr(app, "WORKFLOW_SERVICE", service)+ monkeypatch.setattr(service, "_resolve_managed_browser_profile_for_run_request", AsyncMock(return_value=None))+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", AsyncMock(return_value=False))+ monkeypatch.setattr(app.AGENT_FUNCTION, "is_block_scoped_workflow_run", AsyncMock(return_value=False))+ monkeypatch.setattr(AsyncExecutorFactory, "get_executor", lambda: executor)+ monkeypatch.setattr(group_service, "_spawn", spawned.append)+ monkeypatch.setattr(service, "_schedule_workflow_run_terminal_hooks", lambda **_: None)+ yield GroupEnv(database, executor, organization, spawned, limiter)+ for coroutine in spawned:+ coroutine.close()+ await asyncio.gather(*app.WORKFLOW_SERVICE._background_tasks, return_exceptions=True)+++def _request(+ key: str = "sub-1", count: int = 3, items: Sequence[Mapping[str, object]] | None = None+) -> WorkflowRunGroupCreateRequest:+ if items is None:+ items = [{"key": f"acct-{i}", "parameters": {"login": "cred_1" if i % 2 else "cred_2"}} for i in range(count)]+ return WorkflowRunGroupCreateRequest.model_validate({"workflow_id": WPID, "submission_key": key, "items": items})+++async def _count(env: GroupEnv, model: type[Base]) -> int:+ async with env.database.Session() as session:+ return int(await session.scalar(select(func.count()).select_from(model)) or 0)+++async def _set_child_status(env: GroupEnv, workflow_run_id: str, status: WorkflowRunStatus) -> None:+ await env.database.workflow_runs.update_workflow_run(workflow_run_id, status=status)+++async def _start_nested_run(env: GroupEnv, parent_workflow_run_id: str) -> str:+ nested = await env.database.workflow_runs.create_workflow_run(+ workflow_permanent_id=WPID,+ workflow_id="wf_1",+ organization_id=ORG,+ parent_workflow_run_id=parent_workflow_run_id,+ )+ await _set_child_status(env, nested.workflow_run_id, WorkflowRunStatus.running)+ return nested.workflow_run_id+++async def _status(env: GroupEnv, workflow_run_id: str) -> WorkflowRunStatus | None:+ run = await env.database.workflow_runs.get_workflow_run(workflow_run_id)+ return run.status if run else None+++async def _age_claims(env: GroupEnv) -> None:+ async with env.database.Session() as session:+ await session.execute(+ update(WorkflowRunGroupItemModel).values(claimed_at=naive_utc_now() - timedelta(minutes=30))+ )+ await session.commit()+++async def _submit(env: GroupEnv, request: WorkflowRunGroupCreateRequest | None = None) -> str:+ response = await group_service.submit_workflow_run_group(env.organization, request or _request())+ return response.workflow_run_group_id+++async def test_submit_preallocates_ids_without_child_rows_then_dispatches_one(env: GroupEnv) -> None:+ items = [{"key": f"acct-{i}", "parameters": {"login": "cred_1"}} for i in range(25)]+ response = await group_service.submit_workflow_run_group(env.organization, _request(items=items))++ run_ids = [item.workflow_run_id for item in response.items]+ assert len(set(run_ids)) == 25+ assert response.workflow_id == "wf_1"+ assert await _count(env, WorkflowRunModel) == 0++ await group_service.advance_workflow_run_group(response.workflow_run_group_id)++ assert env.executor.executed == [run_ids[0]]+ child = await env.database.workflow_runs.get_workflow_run(run_ids[0])+ assert child is not None and child.workflow_id == "wf_1" and child.start_fresh_browser is True+ assert await _count(env, WorkflowRunModel) == 1+ group = await group_service.get_workflow_run_group(response.workflow_run_group_id, ORG)+ assert [item.workflow_run_id for item in group.items] == run_ids+++async def _run_ids(env: GroupEnv, group_id: str) -> list[str]:+ return [item.workflow_run_id for item in await env.database.workflow_run_groups.get_items(group_id)]+++async def _states(env: GroupEnv, group_id: str) -> list[WorkflowRunGroupItemState]:+ return [item.state for item in await env.database.workflow_run_groups.get_items(group_id)]+++async def test_next_child_waits_until_the_running_one_is_final(env: GroupEnv) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ await group_service.advance_workflow_run_group(group_id)++ await asyncio.gather(*(group_service.advance_workflow_run_group(group_id) for _ in range(3)))+ await group_service.recover_workflow_run_groups()++ assert env.executor.executed == [run_ids[0]]+ assert (await _states(env, group_id))[1:] == [WorkflowRunGroupItemState.pending] * 2++ await _set_child_status(env, run_ids[0], WorkflowRunStatus.completed)+ await group_service.advance_workflow_run_group(group_id)+ assert env.executor.executed == run_ids[:2]++[email protected]("outside_status", [WorkflowRunStatus.canceled, WorkflowRunStatus.timed_out])+async def test_an_outside_cancel_or_timeout_holds_the_next_child_until_the_worker_can_stop(+ env: GroupEnv, outside_status: WorkflowRunStatus+) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ await group_service.advance_workflow_run_group(group_id)+ await env.database.workflow_runs.update_workflow_run_if_not_final(run_ids[0], outside_status)++ await group_service.advance_workflow_run_group(group_id)+ assert env.executor.executed == run_ids[:1]++ async with env.database.Session() as session:+ await session.execute(+ update(WorkflowRunModel)+ .where(WorkflowRunModel.workflow_run_id == run_ids[0])+ .values(finished_at=naive_utc_now() - group_service.CHILD_STOP_SETTLE - timedelta(seconds=1))+ )+ await session.commit()+ await group_service.advance_workflow_run_group(group_id)+ assert env.executor.executed == run_ids[:2]++[email protected]("bad_credential", ["cred_foreign", "cred_missing"])+async def test_one_bad_credential_rejects_the_whole_group(env: GroupEnv, bad_credential: str) -> None:+ items = [+ {"key": "a", "parameters": {"login": "cred_1"}},+ {"key": "b", "parameters": {"login": bad_credential}},+ {"key": "c", "parameters": {"login": "cred_2"}},+ ]+ with pytest.raises(SkyvernHTTPException) as exc_info:+ await group_service.submit_workflow_run_group(env.organization, _request(items=items))++ assert 400 <= exc_info.value.status_code < 500+ assert await _count(env, WorkflowRunGroupModel) == 0+ assert await _count(env, WorkflowRunGroupItemModel) == 0+ assert await _count(env, WorkflowRunModel) == 0+ assert env.spawned == [] and env.executor.executed == []++[email protected]("count", ["3", "abc"])+async def test_a_mistyped_value_in_any_item_rejects_the_whole_group(env: GroupEnv, count: str) -> None:+ await env.database.workflow_params.create_workflow_parameter(+ workflow_id="wf_1", workflow_parameter_type=WorkflowParameterType.INTEGER, key="count", default_value=None+ )+ items = [+ {"key": "a", "parameters": {"login": "cred_1", "count": "1"}},+ {"key": "b", "parameters": {"login": "cred_2", "count": count}},+ ]++ if count.isdigit():+ await group_service.submit_workflow_run_group(env.organization, _request(items=items))+ assert await _count(env, WorkflowRunGroupItemModel) == 2+ return+ with pytest.raises(SkyvernHTTPException) as exc_info:+ await group_service.submit_workflow_run_group(env.organization, _request(items=items))+ assert exc_info.value.status_code == 400+ assert await _count(env, WorkflowRunGroupModel) == 0+ assert await _count(env, WorkflowRunGroupItemModel) == 0+++async def test_identical_retry_after_workflow_saves_returns_the_same_group(env: GroupEnv) -> None:+ async with env.database.Session() as session:+ session.add(+ WorkflowModel(+ workflow_id="wf_copilot_test",+ workflow_permanent_id=WPID,+ organization_id=ORG,+ title="Workflow",+ version=2,+ created_by=COPILOT_TEST_WORKFLOW_CREATOR,+ workflow_definition=_definition(_task_block()),+ )+ )+ await session.commit()+ first = await group_service.submit_workflow_run_group(env.organization, _request())+ assert first.workflow_id == "wf_1"++ async with env.database.Session() as session:+ session.add(+ WorkflowModel(+ workflow_id="wf_3",+ workflow_permanent_id=WPID,+ organization_id=ORG,+ title="Workflow",+ version=3,+ workflow_definition=_definition(),+ )+ )+ await session.commit()+ retry = await group_service.submit_workflow_run_group(env.organization, _request())++ assert retry.workflow_run_group_id == first.workflow_run_group_id+ assert retry.workflow_id == "wf_1"+ assert [item.workflow_run_id for item in retry.items] == [item.workflow_run_id for item in first.items]+ assert await _count(env, WorkflowRunGroupModel) == 1+++async def test_identical_retry_with_submission_disabled_returns_the_same_group(+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch+) -> None:+ first = await group_service.submit_workflow_run_group(env.organization, _request())+ monkeypatch.setattr(settings, "WORKFLOW_RUN_GROUPS_SUBMIT_ENABLED", False)++ retry = await group_service.submit_workflow_run_group(env.organization, _request())+ with pytest.raises(SkyvernHTTPException) as exc_info:+ await group_service.submit_workflow_run_group(env.organization, _request(key="sub-2"))++ assert retry.workflow_run_group_id == first.workflow_run_group_id+ assert exc_info.value.status_code == 503+ assert await _count(env, WorkflowRunGroupModel) == 1+++ "changed",+ [+ _request(items=[{"key": "acct-0", "parameters": {"login": "cred_2"}}]),+ _request().model_copy(update={"version": 1}),+ ],+ ids=["items", "version"],+)+async def test_conflicting_input_for_a_submission_key_is_rejected(+ env: GroupEnv, changed: WorkflowRunGroupCreateRequest+) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)++ with pytest.raises(SkyvernHTTPException) as exc_info:+ await group_service.submit_workflow_run_group(env.organization, changed)++ assert exc_info.value.status_code == 409+ assert await _count(env, WorkflowRunGroupModel) == 1+ assert await _run_ids(env, group_id) == run_ids+ assert len(env.spawned) == 1+++async def test_concurrent_identical_submissions_share_one_group(env: GroupEnv) -> None:+ responses = await asyncio.gather(+ *(group_service.submit_workflow_run_group(env.organization, _request()) for _ in range(3))+ )++ assert len({response.workflow_run_group_id for response in responses}) == 1+ assert len({tuple(item.workflow_run_id for item in response.items) for response in responses}) == 1+ assert await _count(env, WorkflowRunGroupModel) == 1+ assert await _count(env, WorkflowRunGroupItemModel) == 3+++async def test_concurrent_recoveries_of_a_stale_claim_execute_once(env: GroupEnv) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ assert await env.database.workflow_run_groups.claim_next_item(group_id, "dead-owner") is not None+ await _age_claims(env)++ await asyncio.gather(group_service.recover_workflow_run_groups(), group_service.recover_workflow_run_groups())++ assert env.executor.executed == [run_ids[0]]+ assert await _count(env, WorkflowRunModel) == 1+++async def test_slow_owner_cannot_execute_after_recovery_reclaims_its_item(env: GroupEnv) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ group = await env.database.workflow_run_groups.get_group(group_id)+ slow_item = await env.database.workflow_run_groups.claim_next_item(group_id, "slow-owner")+ assert group is not None and slow_item is not None+ await _age_claims(env)+ reclaimed = await env.database.workflow_run_groups.reclaim_stale_item(+ group_id, 0, dispatch_token="new-owner", stale_before=naive_utc_now() - group_service.DISPATCH_GRACE+ )+ assert reclaimed is not None++ assert await group_service._dispatch_item(group, slow_item, "slow-owner") is True+ assert env.executor.executed == []++ assert await group_service._dispatch_item(group, reclaimed, "new-owner") is True+ assert env.executor.executed == [run_ids[0]]+ assert await _count(env, WorkflowRunModel) == 1+++async def test_stale_owner_interrupted_prep_leaves_the_new_owners_child_alone(+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch+) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ group = await env.database.workflow_run_groups.get_group(group_id)+ stale_item = await env.database.workflow_run_groups.claim_next_item(group_id, "stale-owner")+ assert group is not None and stale_item is not None+ await _age_claims(env)+ reclaimed = await env.database.workflow_run_groups.reclaim_stale_item(+ group_id, 0, dispatch_token="new-owner", stale_before=naive_utc_now() - group_service.DISPATCH_GRACE+ )+ assert reclaimed is not None+ create_task_run = env.database.tasks.create_task_run+ status_after_stale_owner: list[WorkflowRunStatus | None] = []++ async def stale_owner_runs_mid_preparation(**kwargs: Any) -> Run:+ if not status_after_stale_owner:+ assert await group_service._dispatch_item(group, stale_item, "stale-owner") is False+ status_after_stale_owner.append(await _status(env, run_ids[0]))+ return await create_task_run(**kwargs)++ monkeypatch.setattr(env.database.tasks, "create_task_run", stale_owner_runs_mid_preparation)++ assert await group_service._dispatch_item(group, reclaimed, "new-owner") is True++ assert status_after_stale_owner == [WorkflowRunStatus.created]+ assert env.executor.executed == [run_ids[0]]+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.dispatched+ assert await _count(env, WorkflowRunModel) == 1+++async def test_crash_between_child_row_and_task_run_fails_that_item_and_runs_the_next(env: GroupEnv) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ await _leave_item_zero_unstarted(env, group_id, run_ids[0], "dispatching")++ await group_service.recover_workflow_run_groups()++ assert env.executor.executed == [run_ids[1]]+ child = await env.database.workflow_runs.get_workflow_run(run_ids[0])+ assert child is not None and child.status == WorkflowRunStatus.failed+ group = await group_service.get_workflow_run_group(group_id, ORG)+ assert group.items[0].state == WorkflowRunGroupItemState.failed_to_start+ assert group.items[0].outcome == WorkflowRunGroupItemOutcome.failed+ assert await _count(env, WorkflowRunModel) == 2+++class RefusingPermissionChecker(PermissionChecker):+ async def check(self, organization: Organization, browser_session_id: str | None = None) -> None:+ raise HTTPException(status_code=402, detail="Marketplace subscription is not active")+++async def test_org_refused_after_submit_starts_no_remaining_children(+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch+) -> None:+ group_id = await _submit(env)+ monkeypatch.setattr(group_service.PermissionCheckerFactory, "get_instance", RefusingPermissionChecker)++ await group_service.advance_workflow_run_group(group_id)++ assert env.executor.executed == []+ assert await _states(env, group_id) == [WorkflowRunGroupItemState.failed_to_start] * 3+ assert await _count(env, WorkflowRunModel) == 0+++async def test_failing_an_item_whose_child_was_just_canceled_holds_the_next_item_until_it_settles(+ env: GroupEnv,+) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ claimed = await env.database.workflow_run_groups.claim_next_item(group_id, "owner")+ group = await env.database.workflow_run_groups.get_group(group_id)+ assert claimed is not None and group is not None+ async with env.database.Session() as session:+ session.add(+ WorkflowRunModel(+ workflow_run_id=run_ids[0],+ workflow_id="wf_1",+ workflow_permanent_id=WPID,+ organization_id=ORG,+ status=WorkflowRunStatus.canceled.value,+ finished_at=naive_utc_now(),+ start_fresh_browser=True,+ )+ )+ await session.commit()++ assert await group_service._dispatch_item(group, claimed, "owner") is False+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.dispatched+ await group_service.advance_workflow_run_group(group_id)+ assert env.executor.executed == []++ async with env.database.Session() as session:+ await session.execute(+ update(WorkflowRunModel)+ .where(WorkflowRunModel.workflow_run_id == run_ids[0])+ .values(finished_at=naive_utc_now() - group_service.CHILD_STOP_SETTLE - timedelta(seconds=1))+ )+ await session.commit()+ await group_service.advance_workflow_run_group(group_id)++ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.done+ assert env.executor.executed == run_ids[1:2]+++async def test_an_item_held_before_its_child_existed_is_released_by_the_sweep(+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch+) -> None:+ group_id = await _submit(env)+ monkeypatch.setattr(group_service.PermissionCheckerFactory, "get_instance", RefusingPermissionChecker)+ finish_item = env.database.workflow_run_groups.finish_item+ crashed: list[bool] = []++ async def crash_once_before_the_release(*args: Any, **kwargs: Any) -> bool:+ if kwargs["state"] == WorkflowRunGroupItemState.failed_to_start and not crashed:+ crashed.append(True)+ raise RuntimeError("process died")+ return await finish_item(*args, **kwargs)++ monkeypatch.setattr(env.database.workflow_run_groups, "finish_item", crash_once_before_the_release)+ with pytest.raises(RuntimeError):+ await group_service.advance_workflow_run_group(group_id)+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.dispatched++ await _age_claims(env)+ await group_service.recover_workflow_run_groups()++ assert await _states(env, group_id) == [WorkflowRunGroupItemState.failed_to_start] * 3+ group = await env.database.workflow_run_groups.get_group(group_id)+ assert group is not None and group.status == WorkflowRunGroupStatus.finished+++async def test_child_cancel_between_flip_and_queued_write(env: GroupEnv) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)++ async def cancel_group(_: str) -> None:+ await group_service.cancel_workflow_run_group(group_id, ORG)++ env.executor.before_queue = cancel_group+ await group_service.advance_workflow_run_group(group_id)++ assert env.executor.executed == [run_ids[0]]+ assert env.executor.submitted == []+ assert await _status(env, run_ids[0]) == WorkflowRunStatus.canceled+ group = await group_service.get_workflow_run_group(group_id, ORG)+ assert group.status == WorkflowRunGroupStatus.finished+ assert [item.outcome for item in group.items] == [WorkflowRunGroupItemOutcome.canceled] * 3+ assert await _count(env, WorkflowRunModel) == 1+++async def test_cancel_between_claim_and_dispatch_never_executes(env: GroupEnv) -> None:+ group_id = await _submit(env)+ group = await env.database.workflow_run_groups.get_group(group_id)+ claimed = await env.database.workflow_run_groups.claim_next_item(group_id, "owner")+ assert group is not None and claimed is not None++ await group_service.cancel_workflow_run_group(group_id, ORG)+ still_in_flight = await group_service._dispatch_item(group, claimed, "owner")++ assert still_in_flight is False+ assert env.executor.executed == []+ child = await env.database.workflow_runs.get_workflow_run(claimed.workflow_run_id)+ assert child is not None and child.status == WorkflowRunStatus.canceled+ assert await _states(env, group_id) == [WorkflowRunGroupItemState.canceled] * 3+++async def test_in_place_edit_of_a_pinned_version_is_refused_until_the_group_finishes(env: GroupEnv) -> None:+ group_id = await _submit(env)++ for edit in (+ {"title": "Edited"},+ {"workflow_definition": WorkflowDefinition(parameters=[], blocks=[_task_block()])},+ ):+ with pytest.raises(WorkflowPinnedByRunGroup) as refused:+ await app.WORKFLOW_SERVICE.update_workflow_definition(workflow_id="wf_1", organization_id=ORG, **edit)+ assert refused.value.status_code == 409+ saved = await env.database.workflows.get_workflow(workflow_id="wf_1", organization_id=ORG)+ assert saved is not None and saved.title == "Workflow"++ await group_service.cancel_workflow_run_group(group_id, ORG)+ await group_service.recover_workflow_run_groups()+ assert (await group_service.get_workflow_run_group(group_id, ORG)).status == WorkflowRunGroupStatus.finished+ edited = await app.WORKFLOW_SERVICE.update_workflow_definition(+ workflow_id="wf_1", organization_id=ORG, title="Edited"+ )+ assert edited.title == "Edited"+++async def test_submit_bound_to_a_reviewed_version_refuses_an_edit_that_landed_after_review(env: GroupEnv) -> None:+ reviewed = await env.database.workflows.get_workflow(workflow_id="wf_1", organization_id=ORG)+ assert reviewed is not None+ await app.WORKFLOW_SERVICE.update_workflow_definition(workflow_id="wf_1", organization_id=ORG, title="Edited")++ with pytest.raises(WorkflowChangedSinceReview) as refused:+ await group_service.submit_workflow_run_group(+ env.organization, _request(), expected_workflow_modified_at=reviewed.modified_at+ )++ assert refused.value.status_code == 409+ assert await _count(env, WorkflowRunGroupModel) == 0+ assert env.spawned == []+ current = await env.database.workflows.get_workflow(workflow_id="wf_1", organization_id=ORG)+ assert current is not None+ response = await group_service.submit_workflow_run_group(+ env.organization, _request(), expected_workflow_modified_at=current.modified_at+ )+ assert response.workflow_id == "wf_1"+++async def test_cancel_while_a_child_runs_gives_every_item_an_outcome(env: GroupEnv) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ await group_service.advance_workflow_run_group(group_id)+ await env.database.workflow_runs.update_workflow_run(run_ids[0], status=WorkflowRunStatus.running)++ await group_service.cancel_workflow_run_group(group_id, ORG)+ await group_service.recover_workflow_run_groups()++ assert env.executor.executed == [run_ids[0]]+ group = await group_service.get_workflow_run_group(group_id, ORG)+ assert group.status == WorkflowRunGroupStatus.finished+ assert [item.run_status for item in group.items] == [WorkflowRunStatus.canceled, None, None]+ assert [item.outcome for item in group.items] == [+ WorkflowRunGroupItemOutcome.unknown,+ WorkflowRunGroupItemOutcome.canceled,+ WorkflowRunGroupItemOutcome.canceled,+ ]+++async def test_cancel_cascades_to_a_running_childs_nested_runs_and_keeps_final_statuses(env: GroupEnv) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ await group_service.advance_workflow_run_group(group_id)+ finished_nested = await _start_nested_run(env, run_ids[0])+ await _set_child_status(env, finished_nested, WorkflowRunStatus.completed)+ await _set_child_status(env, run_ids[0], WorkflowRunStatus.completed)+ await group_service.advance_workflow_run_group(group_id)+ await _set_child_status(env, run_ids[1], WorkflowRunStatus.running)+ running_nested = await _start_nested_run(env, run_ids[1])+ completed_nested = await _start_nested_run(env, run_ids[1])+ await _set_child_status(env, completed_nested, WorkflowRunStatus.completed)++ await group_service.cancel_workflow_run_group(group_id, ORG)++ assert await _status(env, run_ids[0]) == WorkflowRunStatus.completed+ assert await _status(env, finished_nested) == WorkflowRunStatus.completed+ assert await _status(env, run_ids[1]) == WorkflowRunStatus.canceled+ assert await _status(env, running_nested) == WorkflowRunStatus.canceled+ assert await _status(env, completed_nested) == WorkflowRunStatus.completed+++async def test_cancel_runs_terminal_side_effects_for_non_final_children_only(+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch+) -> None:+ webhooks: list[str] = []++ async def record_webhook(workflow_run: WorkflowRun, **_: object) -> None:+ webhooks.append(workflow_run.workflow_run_id)++ monkeypatch.setattr(app.WORKFLOW_SERVICE, "execute_workflow_webhook", record_webhook)+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ await group_service.advance_workflow_run_group(group_id)+ await _set_child_status(env, run_ids[0], WorkflowRunStatus.completed)+ await group_service.advance_workflow_run_group(group_id)+ await _set_child_status(env, run_ids[1], WorkflowRunStatus.running)++ await group_service.cancel_workflow_run_group(group_id, ORG)+ await group_service.recover_workflow_run_groups()+ await group_service._cancel_child(run_ids[0], ORG)++ assert webhooks == [run_ids[1]]+ assert await _status(env, run_ids[0]) == WorkflowRunStatus.completed+ assert await _status(env, run_ids[1]) == WorkflowRunStatus.canceled+++async def test_failed_child_keeps_earlier_results_and_later_items_run(env: GroupEnv) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ for run_id, status in zip(+ run_ids, (WorkflowRunStatus.completed, WorkflowRunStatus.failed, WorkflowRunStatus.completed), strict=True+ ):+ await group_service.advance_workflow_run_group(group_id)+ await _set_child_status(env, run_id, status)+ await group_service.advance_workflow_run_group(group_id)++ group = await group_service.get_workflow_run_group(group_id, ORG)+ assert env.executor.executed == run_ids+ assert group.status == WorkflowRunGroupStatus.finished+ assert [item.outcome for item in group.items] == [+ WorkflowRunGroupItemOutcome.completed,+ WorkflowRunGroupItemOutcome.failed,+ WorkflowRunGroupItemOutcome.completed,+ ]++[email protected]("evidence", ["http_request_block", "recorded_step", "script_run"])+async def test_failure_after_possible_side_effects_is_unknown_and_not_replayed(env: GroupEnv, evidence: str) -> None:+ if evidence == "http_request_block":+ async with env.database.Session() as session:+ await session.execute(+ update(WorkflowModel).values(workflow_definition=_definition(_task_block(), _http_block()))+ )+ await session.commit()+ group_id = await _submit(env, _request(count=1))+ run_ids = await _run_ids(env, group_id)+ await group_service.advance_workflow_run_group(group_id)+ if evidence == "recorded_step":+ task = await env.database.tasks.create_task(+ url="https://example.com",+ title="login",+ navigation_goal=None,+ data_extraction_goal=None,+ navigation_payload=None,+ organization_id=ORG,+ workflow_run_id=run_ids[0],+ )+ await env.database.tasks.create_step(task.task_id, order=0, retry_index=0, organization_id=ORG)+ if evidence == "script_run":+ await env.database.workflow_runs.update_workflow_run(run_ids[0], script_id="s_1")+ await _set_child_status(env, run_ids[0], WorkflowRunStatus.failed)++ await group_service.advance_workflow_run_group(group_id)+ await group_service.recover_workflow_run_groups()++ group = await group_service.get_workflow_run_group(group_id, ORG)+ assert group.items[0].outcome == WorkflowRunGroupItemOutcome.unknown+ assert env.executor.executed == run_ids+ assert await _count(env, WorkflowRunModel) == 1+++async def test_terminal_child_advances_the_group_through_the_run_terminal_hook(+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch+) -> None:+ service = WorkflowService()+ monkeypatch.setattr(app, "WORKFLOW_SERVICE", service)+ monkeypatch.setattr(service, "_resolve_managed_browser_profile_for_run_request", AsyncMock(return_value=None))+ monkeypatch.setattr(+ app.AGENT_FUNCTION, "schedule_workflow_run_group_advance", AgentFunction().schedule_workflow_run_group_advance+ )+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ await group_service.advance_workflow_run_group(group_id)+ while env.spawned:+ env.spawned.pop().close()++ await service.mark_workflow_run_as_failed_if_not_final(workflow_run_id=run_ids[0], failure_reason="boom")+ while env.spawned:+ await env.spawned.pop()++ assert env.executor.executed == run_ids[:2]+++def test_browser_session_inputs_are_rejected() -> None:+ base = {"workflow_id": WPID, "submission_key": "k", "items": [{"key": "a", "parameters": {}}]}+ for field_name in ("browser_session_id", "browser_address", "browser_profile_id"):+ with pytest.raises(ValidationError):+ WorkflowRunGroupCreateRequest.model_validate({**base, field_name: "x"})+ with pytest.raises(ValidationError):+ WorkflowRunGroupCreateRequest.model_validate({**base, "items": [{"key": "a", field_name: "x"}]})+++ "workflow_values",+ [+ {"workflow_definition": {**_definition(_task_block()), "retry_policy": {"retry_on": [{"status": "failed"}]}}},+ {"persist_browser_session": True},+ {"browser_profile_id": "bp_1"},+ {"sequential_key": "{{ login }}"},+ {"workflow_definition": _definition()},+ {"browser_profile_key": "profile-{{ login }}"},+ {"workflow_definition": _definition(_task_block(), _looped_trigger(browser_session_id="{{ session_id }}"))},+ {"workflow_definition": _definition(_task_block(), _looped_trigger(wait_for_completion=False))},+ {"workflow_definition": {**_definition(_task_block()), "finally_block_label": "login"}},+ ],+)+async def test_versions_a_group_cannot_run_are_rejected(env: GroupEnv, workflow_values: dict[str, object]) -> None:+ async with env.database.Session() as session:+ await session.execute(update(WorkflowModel).values(**workflow_values))+ await session.commit()++ with pytest.raises(SkyvernHTTPException) as exc_info:+ await _submit(env)++ assert exc_info.value.status_code == 400+ assert await _count(env, WorkflowRunGroupModel) == 0+ assert await _count(env, WorkflowRunGroupItemModel) == 0+++async def test_sweep_with_a_stale_read_cannot_reclaim_an_item_another_sweep_just_reclaimed(env: GroupEnv) -> None:+ group_id = await _submit(env)+ assert await env.database.workflow_run_groups.claim_next_item(group_id, "dead-owner") is not None+ await _age_claims(env)+ group = await env.database.workflow_run_groups.get_group(group_id)+ stale_snapshot = await env.database.workflow_run_groups.get_items(group_id)+ assert group is not None+ assert await env.database.workflow_run_groups.reclaim_stale_item(+ group_id, 0, dispatch_token="sweep-a", stale_before=naive_utc_now() - group_service.DISPATCH_GRACE+ )++ await group_service._reconcile_in_flight(group, stale_snapshot)++ assert env.executor.executed == []+ assert (await env.database.workflow_run_groups.get_items(group_id))[0].dispatch_token == "sweep-a"++[email protected]("executor_took_the_child", [False, True])+async def test_sweep_fails_a_dispatched_child_no_executor_took_and_leaves_a_queued_one(+ env: GroupEnv, executor_took_the_child: bool+) -> None:+ async def executor_crashes(_: str) -> None:+ raise RuntimeError("executor unavailable")++ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ env.executor.before_queue = None if executor_took_the_child else executor_crashes+ await group_service.advance_workflow_run_group(group_id)+ env.executor.before_queue = None+ await _age_claims(env)++ await group_service.recover_workflow_run_groups()++ if executor_took_the_child:+ assert await _status(env, run_ids[0]) == WorkflowRunStatus.queued+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.dispatched+ assert env.executor.executed == run_ids[:1]+ else:+ assert await _status(env, run_ids[0]) == WorkflowRunStatus.failed+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.failed_to_start+ assert env.executor.executed == run_ids[:2]+++async def _leave_item_zero_unstarted(env: GroupEnv, group_id: str, run_id: str, item_state: str) -> None:+ if item_state == "dispatched":++ async def executor_crashes(_: str) -> None:+ raise RuntimeError("executor unavailable")++ env.executor.before_queue = executor_crashes+ await group_service.advance_workflow_run_group(group_id)+ env.executor.before_queue = None+ else:+ assert await env.database.workflow_run_groups.claim_next_item(group_id, "crashed-owner") is not None+ async with env.database.Session() as session:+ session.add(+ WorkflowRunModel(+ workflow_run_id=run_id,+ workflow_id="wf_1",+ workflow_permanent_id=WPID,+ organization_id=ORG,+ status=WorkflowRunStatus.created.value,+ start_fresh_browser=True,+ )+ )+ await session.commit()+ await _age_claims(env)++[email protected]("item_state", ["dispatching", "dispatched"])+async def test_an_item_that_never_started_ends_failed_to_start_when_failing_its_child_advances_the_group(+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch, item_state: str+) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ await _leave_item_zero_unstarted(env, group_id, run_ids[0], item_state)+ fail_run = app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final++ async def fail_then_run_terminal_hook(**kwargs: Any) -> WorkflowRun | None:+ failed = await fail_run(**kwargs)+ await group_service.advance_workflow_run_group(group_id)+ return failed++ monkeypatch.setattr(app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", fail_then_run_terminal_hook)++ await group_service.recover_workflow_run_groups()++ assert await _status(env, run_ids[0]) == WorkflowRunStatus.failed+ assert (await _states(env, group_id))[:2] == [+ WorkflowRunGroupItemState.failed_to_start,+ WorkflowRunGroupItemState.dispatched,+ ]+++ ("executor_status", "outcome"),+ [+ (WorkflowRunStatus.queued, WorkflowRunGroupItemOutcome.canceled),+ (WorkflowRunStatus.running, WorkflowRunGroupItemOutcome.unknown),+ ],+)+async def test_a_child_an_executor_took_while_the_sweep_failed_it_holds_the_next_item_until_it_stops(+ env: GroupEnv,+ monkeypatch: pytest.MonkeyPatch,+ executor_status: WorkflowRunStatus,+ outcome: WorkflowRunGroupItemOutcome,+) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ await _leave_item_zero_unstarted(env, group_id, run_ids[0], "dispatched")+ finish_item = env.database.workflow_run_groups.finish_item++ async def executor_takes_the_child_once_the_sweep_holds_it(*args: Any, **kwargs: Any) -> bool:+ finished = await finish_item(*args, **kwargs)+ if finished and kwargs["state"] == WorkflowRunGroupItemState.dispatched:+ await _set_child_status(env, run_ids[0], executor_status)+ return finished++ monkeypatch.setattr(+ env.database.workflow_run_groups, "finish_item", executor_takes_the_child_once_the_sweep_holds_it+ )++ await group_service.recover_workflow_run_groups()++ assert await _status(env, run_ids[0]) == WorkflowRunStatus.canceled+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.dispatched+ assert env.executor.executed == run_ids[:1]++ async with env.database.Session() as session:+ await session.execute(+ update(WorkflowRunModel)+ .where(WorkflowRunModel.workflow_run_id == run_ids[0])+ .values(finished_at=naive_utc_now() - group_service.CHILD_STOP_SETTLE - timedelta(seconds=1))+ )+ await session.commit()+ await group_service.recover_workflow_run_groups()++ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.done+ assert env.executor.executed == run_ids[:2]+ assert (await group_service.get_workflow_run_group(group_id, ORG)).items[0].outcome == outcome+++ "prior_attempts", [group_service.MAX_DISPATCH_ATTEMPTS - 1, group_service.MAX_DISPATCH_ATTEMPTS]+)+async def test_a_dispatch_that_keeps_dying_fails_its_item_at_the_attempt_cap(+ env: GroupEnv, prior_attempts: int+) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ assert await env.database.workflow_run_groups.claim_next_item(group_id, "dead-owner") is not None+ async with env.database.Session() as session:+ await session.execute(update(WorkflowRunGroupItemModel).values(dispatch_attempts=prior_attempts))+ await session.commit()+ await _age_claims(env)++ await group_service.recover_workflow_run_groups()++ if prior_attempts < group_service.MAX_DISPATCH_ATTEMPTS:+ assert env.executor.executed == run_ids[:1]+ else:+ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.failed_to_start+ assert env.executor.executed == run_ids[1:2]+++async def test_the_child_after_a_failed_preparation_prepares_in_its_own_context(env: GroupEnv) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)+ await env.database.workflow_params.create_workflow_parameter(+ workflow_id="wf_1", workflow_parameter_type=WorkflowParameterType.INTEGER, key="count", default_value="2"+ )+ async with env.database.Session() as session:+ await session.execute(+ update(WorkflowRunGroupItemModel)+ .where(WorkflowRunGroupItemModel.position == 0)+ .values(parameters={"login": "cred_1", "count": "abc"})+ )+ await session.commit()+ contexts: dict[str, tuple[str | None, str | None]] = {}++ async def record_context(workflow_run_id: str) -> None:+ context = skyvern_context.current()+ contexts[workflow_run_id] = (context.run_id, context.root_workflow_run_id) if context else (None, None)++ env.executor.before_queue = record_context++ await group_service.advance_workflow_run_group(group_id)++ assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.failed_to_start+ assert env.executor.executed == [run_ids[1]]+ assert contexts[run_ids[1]] == (run_ids[1], run_ids[1])+++async def test_scrubbed_group_keeps_no_caller_keys_and_a_replay_of_its_key_conflicts(env: GroupEnv) -> None:+ group_id = await _submit(env)+ run_ids = await _run_ids(env, group_id)++ assert await env.database.workflow_run_groups.scrub_keys(ORG, [group_id]) == 1++ async with env.database.Session() as session:+ stored = [+ *(await session.scalars(select(WorkflowRunGroupModel.submission_key))).all(),+ *(await session.scalars(select(WorkflowRunGroupItemModel.item_key))).all(),+ ]+ assert not any(raw in value for value in stored for raw in ("sub-1", "acct-"))+ # A group still running when it passes the cap must keep running its pending items with their own inputs.+ await group_service.advance_workflow_run_group(group_id)+ child_parameters = await env.database.workflow_runs.get_workflow_run_parameters(run_ids[0])+ assert [parameter.value for _, parameter in child_parameters] == ["cred_2"]+ with pytest.raises(SkyvernHTTPException) as exc_info:+ await _submit(env)+ assert exc_info.value.status_code == 409+ assert await _count(env, WorkflowRunGroupModel) == 1+ with pytest.raises(ValidationError):+ _request(key=stored[0])+ with pytest.raises(ValidationError):+ _request(items=[{"key": stored[1], "parameters": {"login": "cred_1"}}])+++async def test_an_edit_landing_between_validation_and_insert_refuses_the_submit(+ env: GroupEnv, monkeypatch: pytest.MonkeyPatch+) -> None:+ validate = app.WORKFLOW_SERVICE.validate_schedule_parameters++ async def edit_after_validation(+ workflow: Workflow, organization: Organization, request_data: dict[str, Any] | None+ ) -> None:+ await validate(workflow, organization, request_data)+ async with env.database.Session() as session:+ await session.execute(update(WorkflowModel).values(modified_at=naive_utc_now() + timedelta(seconds=1)))+ await session.commit()++ monkeypatch.setattr(app.WORKFLOW_SERVICE, "validate_schedule_parameters", edit_after_validation)++ with pytest.raises(SkyvernHTTPException) as exc_info:+ await _submit(env)++ assert exc_info.value.status_code == 409+ assert await _count(env, WorkflowRunGroupModel) == 0+++async def test_another_organization_cannot_read_or_cancel_a_group(env: GroupEnv) -> None:+ group_id = await _submit(env)++ for call in (group_service.get_workflow_run_group, group_service.cancel_workflow_run_group):+ with pytest.raises(SkyvernHTTPException) as exc_info:+ await call(group_id, OTHER_ORG)+ assert exc_info.value.status_code == 404++ group = await env.database.workflow_run_groups.get_group(group_id)+ assert group is not None and group.status == WorkflowRunGroupStatus.active+++async def test_a_new_group_consumes_a_submit_token_and_a_replay_does_not(env: GroupEnv) -> None:+ first = await _submit(env)+ replay = await _submit(env)++ assert replay == first+ assert env.limiter.calls == [ORG]+++async def test_item_parameters_are_cleared_once_the_item_leaves_dispatch(env: GroupEnv) -> None:+ group_id = await _submit(env)+ assert await env.database.workflow_run_groups.claim_next_item(group_id, "owner") is not None+ assert await env.database.workflow_run_groups.finish_item(+ group_id,+ 0,+ state=WorkflowRunGroupItemState.failed_to_start,+ from_states=(WorkflowRunGroupItemState.dispatching,),+ dispatch_token="owner",+ )+ await group_service.advance_workflow_run_group(group_id)+ items = await env.database.workflow_run_groups.get_items(group_id)+ assert [item.state for item in items[:2]] == [+ WorkflowRunGroupItemState.failed_to_start,+ WorkflowRunGroupItemState.dispatched,+ ]+ assert [item.parameters for item in items] == [{}, {}, {"login": "cred_2"}]++ await group_service.cancel_workflow_run_group(group_id, ORG)++ assert [item.parameters for item in await env.database.workflow_run_groups.get_items(group_id)] == [{}, {}, {}]+++async def test_prepare_error_text_never_reaches_the_item_or_the_group_read(env: GroupEnv) -> None:+ sentinel = "SENTINEL-acct-7731"+ group_id = await _submit(env)+ await env.database.workflow_params.create_workflow_parameter(+ workflow_id="wf_1", workflow_parameter_type=WorkflowParameterType.INTEGER, key="count", default_value=None+ )+ async with env.database.Session() as session:+ await session.execute(+ update(WorkflowRunGroupItemModel).values(parameters={"login": "cred_1", "count": sentinel})+ )+ await session.commit()++ await group_service.advance_workflow_run_group(group_id)++ item = (await env.database.workflow_run_groups.get_items(group_id))[0]+ assert item.state == WorkflowRunGroupItemState.failed_to_start+ assert item.failure_reason == "Workflow run could not be prepared (InvalidWorkflowParameter)"+ group = await group_service.get_workflow_run_group(group_id, ORG)+ assert sentinel not in group.model_dump_json()++[email protected]("nesting", [0, 1, 2])+ ("wait_for_completion", "supplies_session"),+ [(False, False), (True, True)],+ ids=["detached", "sync_into_supplied_session"],+)+async def test_unfresh_trigger_under_a_group_child_fails_before_creating_a_run(+ env: GroupEnv,+ monkeypatch: pytest.MonkeyPatch,+ nesting: int,+ wait_for_completion: bool,+ supplies_session: bool,+) -> None:+ group_id = await _submit(env)+ await group_service.advance_workflow_run_group(group_id)+ trigger_run_id = (await _run_ids(env, group_id))[0]+ for _ in range(nesting):+ trigger_run_id = await _start_nested_run(env, trigger_run_id)+ browser_session_id = (+ (+ await env.database.browser_sessions.create_persistent_browser_session(organization_id=ORG)+ ).persistent_browser_session_id+ if supplies_session+ else None+ )+ runs_before = await _count(env, WorkflowRunModel)+ block = WorkflowTriggerBlock(+ label="detached",+ workflow_permanent_id=WPID,+ payload={"login": "cred_1"},+ wait_for_completion=wait_for_completion,+ browser_session_id=browser_session_id,+ output_parameter=make_block_output_parameter("detached"),+ )+ result = AsyncMock(return_value=MagicMock())+ monkeypatch.setattr(WorkflowTriggerBlock, "get_workflow_run_context", lambda self, workflow_run_id: MagicMock())+ monkeypatch.setattr(WorkflowTriggerBlock, "record_output_parameter_value", AsyncMock())+ monkeypatch.setattr(WorkflowTriggerBlock, "build_block_result", result)++ await block.execute(workflow_run_id=trigger_run_id, workflow_run_block_id="wrb_detached", organization_id=ORG)++ assert result.await_args is not None and result.await_args.kwargs["success"] is False+ assert await _count(env, WorkflowRunModel) == runs_beforediff --git a/tests/unit/workflow/test_workflow_trigger_block.py b/tests/unit/workflow/test_workflow_trigger_block.pyindex 5f6402909..9022ab76d 100644--- a/tests/unit/workflow/test_workflow_trigger_block.py+++ b/tests/unit/workflow/test_workflow_trigger_block.py@@ -374,17 +374,17 @@ class TestCheckTriggerDepth:"""Test _check_trigger_depth: boundary conditions at/above/below MAX_TRIGGER_DEPTH."""@pytest.mark.asyncio- async def test_no_parent_returns_zero(self) -> None:+ async def test_no_parent_returns_the_run_itself(self) -> None:block = _make_block()mock_run = MagicMock()mock_run.parent_workflow_run_id = Nonewith patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app:mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=mock_run)- depth = await block._check_trigger_depth("wr_current")- assert depth == 0+ root = await block._check_trigger_depth("wr_current")+ assert root == "wr_current"@pytest.mark.asyncio- async def test_single_parent_returns_one(self) -> None:+ async def test_single_parent_returns_the_parent(self) -> None:block = _make_block()run_with_parent = MagicMock()run_with_parent.parent_workflow_run_id = "wr_parent"@@ -393,8 +393,8 @@ class TestCheckTriggerDepth:with patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app:mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(side_effect=[run_with_parent, run_no_parent])- depth = await block._check_trigger_depth("wr_current")- assert depth == 1+ root = await block._check_trigger_depth("wr_current")+ assert root == "wr_parent"@pytest.mark.asyncioasync def test_depth_at_max_raises(self) -> None:@@ -421,16 +421,16 @@ class TestCheckTriggerDepth:with patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app:mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(side_effect=runs)- depth = await block._check_trigger_depth("wr_current")- assert depth == block.MAX_TRIGGER_DEPTH - 1+ root = await block._check_trigger_depth("wr_current")+ assert root == f"wr_parent_{block.MAX_TRIGGER_DEPTH - 2}"@pytest.mark.asyncio- async def test_run_not_found_returns_zero(self) -> None:+ async def test_run_not_found_returns_the_run_id(self) -> None:block = _make_block()with patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app:mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=None)- depth = await block._check_trigger_depth("wr_nonexistent")- assert depth == 0+ root = await block._check_trigger_depth("wr_nonexistent")+ assert root == "wr_nonexistent"@pytest.mark.asyncio@@ -488,7 +488,7 @@ async def test_sync_trigger_preserves_parent_feature_flag_summary(monkeypatch: pmonkeypatch.setattr(WorkflowTriggerBlock, "get_workflow_run_context", lambda self, workflow_run_id: MagicMock())monkeypatch.setattr(WorkflowTriggerBlock, "format_potential_template_parameters", lambda self, ctx: None)- monkeypatch.setattr(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value=0))+ monkeypatch.setattr(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value="wr_parent"))monkeypatch.setattr(WorkflowTriggerBlock, "record_output_parameter_value", AsyncMock())monkeypatch.setattr(WorkflowTriggerBlock, "build_block_result", AsyncMock(return_value=MagicMock()))@@ -557,11 +557,12 @@ async def test_sync_trigger_fails_closed_when_child_resolves_sequential_credentipatch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,patch.object(WorkflowTriggerBlock, "get_workflow_run_context", lambda self, workflow_run_id: MagicMock()),patch.object(WorkflowTriggerBlock, "format_potential_template_parameters", lambda self, ctx: None),- patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value=0)),+ patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value="wr_parent")),patch.object(WorkflowTriggerBlock, "record_output_parameter_value", AsyncMock()),patch.object(WorkflowTriggerBlock, "build_block_result", AsyncMock(side_effect=_build_result)),):mock_app.DATABASE.organizations.get_organization = AsyncMock(return_value=organization)+ mock_app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id = AsyncMock(return_value=None)mock_app.WORKFLOW_SERVICE.setup_workflow_run = AsyncMock(return_value=child_run)mock_app.WORKFLOW_SERVICE.execute_workflow = AsyncMock()mock_app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final = AsyncMock()@@ -607,11 +608,12 @@ async def _run_sync_trigger_fence(patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,patch.object(WorkflowTriggerBlock, "get_workflow_run_context", lambda self, workflow_run_id: MagicMock()),patch.object(WorkflowTriggerBlock, "format_potential_template_parameters", lambda self, ctx: None),- patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value=0)),+ patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value="wr_parent")),patch.object(WorkflowTriggerBlock, "record_output_parameter_value", AsyncMock()),patch.object(WorkflowTriggerBlock, "build_block_result", AsyncMock(side_effect=_build_result)),):mock_app.DATABASE.organizations.get_organization = AsyncMock(return_value=organization)+ mock_app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id = AsyncMock(return_value=None)mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=MagicMock(proxy_location=None))created_session = MagicMock()created_session.persistent_browser_session_id = created_session_id@@ -752,11 +754,12 @@ async def _run_async_trigger(run_workflow_stub: Any) -> tuple[dict[str, Any], dipatch("skyvern.services.workflow_service.run_workflow", side_effect=run_workflow_stub),patch.object(WorkflowTriggerBlock, "get_workflow_run_context", lambda self, workflow_run_id: MagicMock()),patch.object(WorkflowTriggerBlock, "format_potential_template_parameters", lambda self, ctx: None),- patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value=0)),+ patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value="wr_parent")),patch.object(WorkflowTriggerBlock, "record_output_parameter_value", AsyncMock(side_effect=_capture_record)),patch.object(WorkflowTriggerBlock, "build_block_result", AsyncMock(side_effect=_build_result)),):mock_app.DATABASE.organizations.get_organization = AsyncMock(return_value=organization)+ mock_app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id = AsyncMock(return_value=None)await block.execute(workflow_run_id="wr_parent",workflow_run_block_id="wrb_parent",