Skip to content

Automate browser based workflows with AI

feat(SKY-12845): serialize workflow runs by credential (core, non-scheduled) (#7865)

LawyZheng7df5badc1ade
diff --git a/alembic/versions/2026_07_30_0957-58c683692601_add_credential_run_serialization.py b/alembic/versions/2026_07_30_0957-58c683692601_add_credential_run_serialization.py
new file mode 100644
index 000000000..85979273a
--- /dev/null
+++ b/alembic/versions/2026_07_30_0957-58c683692601_add_credential_run_serialization.py
@@ -0,0 +1,110 @@
+"""add credential run serialization
+
+Revision ID: 58c683692601
+Revises: 1fe4efc6b752
+Create Date: 2026-07-30T09:57:46.639367+00:00
+
+"""
+
+import random
+import time
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from sqlalchemy.exc import DBAPIError
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "58c683692601"
+down_revision: Union[str, None] = "1fe4efc6b752"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+_STATEMENT_TIMEOUT = "5s"
+_LOCK_TIMEOUT = "250ms"
+_LOCK_NOT_AVAILABLE_SQLSTATE = "55P03"
+_MIGRATION_RETRY_SECONDS = 20 * 60
+_BACKOFF_BASE_SECONDS = 0.25
+_BACKOFF_JITTER_SECONDS = 0.75
+_ADDS = (
+ ("credentials", "run_sequentially", "BOOLEAN NOT NULL DEFAULT false"),
+ ("workflow_runs", "sequential_credential_id", "VARCHAR"),
+)
+
+
+def _is_lock_not_available(exc: DBAPIError) -> bool:
+ orig = getattr(exc, "orig", None)
+ return (
+ getattr(orig, "sqlstate", None) == _LOCK_NOT_AVAILABLE_SQLSTATE
+ or getattr(orig, "pgcode", None) == _LOCK_NOT_AVAILABLE_SQLSTATE
+ )
+
+
+def _column_exists(table: str, column: str) -> bool:
+ result = op.get_bind().execute(
+ sa.text(
+ """
+ SELECT 1
+ FROM information_schema.columns
+ WHERE table_schema = current_schema()
+ AND table_name = :table_name
+ AND column_name = :column_name
+ """
+ ),
+ {"table_name": table, "column_name": column},
+ )
+ return result.first() is not None
+
+
+def _execute_transactional_schema_change(table: str, statement: str) -> None:
+ try:
+ op.execute("BEGIN")
+ op.execute(f"SET LOCAL statement_timeout = '{_STATEMENT_TIMEOUT}'")
+ op.execute(f"SET LOCAL lock_timeout = '{_LOCK_TIMEOUT}'")
+ op.execute(f'LOCK TABLE "{table}" IN ACCESS EXCLUSIVE MODE')
+ op.execute(statement)
+ op.execute("COMMIT")
+ except Exception as exc:
+ try:
+ op.execute("ROLLBACK")
+ except Exception as rollback_exc:
+ raise RuntimeError(
+ f"rollback failed after schema change error; aborting migration: {rollback_exc}"
+ ) from exc
+ raise
+
+
+def _execute_with_retry(table: str, statement: str, deadline: float) -> None:
+ while True:
+ try:
+ _execute_transactional_schema_change(table, statement)
+ return
+ except DBAPIError as exc:
+ if not _is_lock_not_available(exc) or time.monotonic() >= deadline:
+ raise
+ time.sleep(_BACKOFF_BASE_SECONDS + random.random() * _BACKOFF_JITTER_SECONDS)
+
+
+def upgrade() -> None:
+ deadline = time.monotonic() + _MIGRATION_RETRY_SECONDS
+ with op.get_context().autocommit_block():
+ for table, column, definition in _ADDS:
+ if not _column_exists(table, column):
+ _execute_with_retry(
+ table,
+ f'ALTER TABLE "{table}" ADD COLUMN IF NOT EXISTS {column} {definition}',
+ deadline,
+ )
+
+
+def downgrade() -> None:
+ deadline = time.monotonic() + _MIGRATION_RETRY_SECONDS
+ with op.get_context().autocommit_block():
+ for table, column, _definition in reversed(_ADDS):
+ if _column_exists(table, column):
+ _execute_with_retry(
+ table,
+ f'ALTER TABLE "{table}" DROP COLUMN IF EXISTS {column}',
+ deadline,
+ )
diff --git a/alembic/versions/2026_07_30_0957-64cbada89956_add_credential_gate_indexes.py b/alembic/versions/2026_07_30_0957-64cbada89956_add_credential_gate_indexes.py
new file mode 100644
index 000000000..375e07d35
--- /dev/null
+++ b/alembic/versions/2026_07_30_0957-64cbada89956_add_credential_gate_indexes.py
@@ -0,0 +1,40 @@
+"""add credential gate indexes
+
+Revision ID: 64cbada89956
+Revises: 58c683692601
+Create Date: 2026-07-30T09:57:46.639871+00:00
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "64cbada89956"
+down_revision: Union[str, None] = "58c683692601"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ with op.get_context().autocommit_block():
+ op.execute("SET statement_timeout = '3h';")
+ op.execute("""
+ CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_workflow_runs_sequential_credential_gate
+ ON workflow_runs (organization_id, sequential_credential_id, queued_at)
+ WHERE sequential_credential_id IS NOT NULL
+ AND status IN ('queued', 'running', 'paused');
+ """)
+ op.execute("""
+ CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_workflow_runs_serialized_ticket
+ ON workflow_runs (organization_id, queued_at DESC)
+ WHERE status IN ('queued', 'running', 'paused');
+ """)
+ op.execute("RESET statement_timeout;")
+
+
+def downgrade() -> None:
+ with op.get_context().autocommit_block():
+ op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_workflow_runs_serialized_ticket;")
+ op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_workflow_runs_sequential_credential_gate;")
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index 59cc70096..fa14d9c75 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -11745,6 +11745,11 @@
"title": "Save Browser Session Intent",
"description": "Whether the user intends to save a browser session, regardless of test outcome"
},
+ "run_sequentially": {
+ "type": "boolean",
+ "title": "Run Sequentially",
+ "default": false
+ },
"folder_id": {
"anyOf": [
{
@@ -28218,6 +28223,17 @@
],
"title": "Sequential Key"
},
+ "sequential_credential_id": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Sequential Credential Id"
+ },
"ai_fallback": {
"anyOf": [
{
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 0029f38b6..af27571d2 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -463,6 +463,78 @@ class InvalidCredentialId(SkyvernHTTPException):
)
+class SequentialCredentialLimitExceeded(SkyvernHTTPException):
+ def __init__(self, credential_ids: list[str]) -> None:
+ sanitized = ", ".join(sanitize_credential_for_error(credential_id) for credential_id in credential_ids)
+ super().__init__(
+ f"This workflow run resolves to {len(credential_ids)} credentials marked run_sequentially "
+ f"({sanitized}), but a run may use at most one sequential credential. Mark only one of them "
+ "run_sequentially, or split the workflow so each run resolves to a single sequential credential.",
+ status_code=HTTPStatus.BAD_REQUEST,
+ )
+
+
+class SequentialCredentialWorkerUnavailable(SkyvernHTTPException):
+ def __init__(self, credential_id: str) -> None:
+ super().__init__(
+ f"This run resolves to a sequential credential ({sanitize_credential_for_error(credential_id)}), whose "
+ "org-wide serialization gate lives only in the Temporal V2 worker, but workers are globally disabled "
+ "(ENABLE_WORKER). Routing it to the legacy engine would run it unserialized, so the run fails closed. "
+ "Re-enable workers, or clear run_sequentially on the credential to run it without the sequential lane.",
+ status_code=HTTPStatus.SERVICE_UNAVAILABLE,
+ )
+
+
+class RuntimeSequentialCredentialUnsupported(SkyvernException):
+ def __init__(self, workflow_run_id: str) -> None:
+ super().__init__(
+ f"Workflow run {workflow_run_id} resolved a sequential credential only at runtime, after publication "
+ "without that credential's serialization lane. The run fails closed before loading the credential "
+ "rather than using it outside the gate."
+ )
+
+
+class BackgroundSequentialCredentialUnsupported(SkyvernException):
+ def __init__(self, workflow_run_id: str) -> None:
+ super().__init__(
+ f"Workflow run {workflow_run_id} resolves to a sequential credential, but the background executor "
+ "has no org-wide serialization gate. The run fails closed before execution rather than using the "
+ "credential outside its lane."
+ )
+
+
+class ScheduledSequentialCredentialUnsupported(SkyvernException):
+ def __init__(self, workflow_run_id: str) -> None:
+ super().__init__(
+ f"Scheduled run {workflow_run_id} resolves to a sequential credential, but scheduled credential "
+ "serialization is not yet supported. The run fails closed rather than publishing an ungated row that "
+ "would race on the credential. Every subsequent fire of this schedule fails the same way until scheduled "
+ "parity ships or run_sequentially is cleared on the credential."
+ )
+
+
+class SyncTriggeredSequentialCredentialUnsupported(SkyvernException):
+ def __init__(self, workflow_run_id: str) -> None:
+ super().__init__(
+ f"Triggered run {workflow_run_id} resolves to a sequential credential, but a synchronous "
+ "workflow_trigger child runs inline and never reaches the Temporal V2 serialization gate, so "
+ "it cannot be serialized against other runs using the same credential. The child fails closed "
+ "rather than running unserialized. Clear run_sequentially on the credential, or run the child "
+ "as an async (fire-and-forget) trigger so it publishes through the gate."
+ )
+
+
+class CopilotInlineSequentialCredentialUnsupported(SkyvernException):
+ def __init__(self, workflow_run_id: str) -> None:
+ super().__init__(
+ f"Copilot run {workflow_run_id} resolves to a sequential credential, but the copilot inline "
+ "block-run path executes in-process and never reaches the Temporal V2 serialization gate, so it "
+ "cannot be serialized against other runs using the same credential. The run fails closed rather "
+ "than running unserialized. Clear run_sequentially on the credential, or enable the copilot "
+ "dispatch flag so the run publishes through the gate."
+ )
+
+
class WorkflowParameterNotFound(SkyvernHTTPException):
def __init__(self, workflow_parameter_id: str) -> None:
super().__init__(
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 3c58a810f..ede95b13f 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -780,6 +780,10 @@ class AgentFunction:
workflow_schedules_use_local_scheduler: bool = settings.ENABLE_WORKFLOW_SCHEDULES
"""Whether the API process should run the built-in local scheduler loop."""
+ def supports_sequential_credentials(self) -> bool:
+ """Whether this deployment can execute credentials marked run_sequentially."""
+ return False
+
def is_wait_time_optimization_enabled(self) -> bool:
return False
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index bfff8fb15..44ee5d34d 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -15,6 +15,7 @@ from urllib.parse import urlparse
import structlog
+from skyvern.exceptions import CopilotInlineSequentialCredentialUnsupported
from skyvern.forge import app
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
from skyvern.forge.sdk.copilot.blocker_signal import (
@@ -851,6 +852,40 @@ def _runtime_code_security_failure_for_selected_labels(
}
+def _inline_sequential_credential_fence_failure(
+ *,
+ workflow_run_id: str,
+ sequential_credential_id: str | None,
+ dispatch_to_worker: bool,
+ block_labels: list[str],
+ labels_to_execute: list[str],
+ frontier_start_label: str | None,
+) -> dict[str, Any] | None:
+ # The copilot inline path runs execute_workflow in-process: the run never queues, never gets a
+ # queued_at, and never reaches the Temporal V2 serialization gate, yet setup stamped its
+ # sequential_credential_id. The gate filters queued_at IS NOT NULL, so a concurrent run sharing the
+ # credential would neither see this run nor be waited on by it. Fail closed before it uses the
+ # credential — the same fence the scheduled and sync-trigger paths apply. The dispatch path enqueues
+ # through the executor (stamped queued_at, gated), so it is exempt.
+ if dispatch_to_worker or not sequential_credential_id:
+ return None
+ failure_reason = str(CopilotInlineSequentialCredentialUnsupported(workflow_run_id))
+ return {
+ "ok": False,
+ "error": failure_reason,
+ "data": {
+ "workflow_run_id": workflow_run_id,
+ "overall_status": "failed",
+ "failure_reason": failure_reason,
+ "requested_block_labels": list(block_labels),
+ "executed_block_labels": [],
+ "planned_block_labels": list(labels_to_execute),
+ "frontier_start_label": frontier_start_label,
+ "blocks": [],
+ },
+ }
+
+
def _workflow_definition_blocks_for_code_security(workflow_definition: Any) -> list[Any]:
if isinstance(workflow_definition, Mapping):
blocks = workflow_definition.get("blocks")
@@ -1737,6 +1772,21 @@ async def _run_blocks_and_collect_debug(
if inline_run_context is not None:
inline_run_context.copilot_inline_execution = True
+ inline_fence_failure = _inline_sequential_credential_fence_failure(
+ workflow_run_id=workflow_run.workflow_run_id,
+ sequential_credential_id=workflow_run.sequential_credential_id,
+ dispatch_to_worker=dispatch_to_worker,
+ block_labels=block_labels,
+ labels_to_execute=labels_to_execute,
+ frontier_start_label=frontier_start_label,
+ )
+ if inline_fence_failure is not None:
+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(
+ workflow_run_id=workflow_run.workflow_run_id,
+ failure_reason=inline_fence_failure["error"],
+ )
+ return inline_fence_failure
+
await initialize_skyvern_state_file(
workflow_run_id=workflow_run.workflow_run_id,
organization_id=ctx.organization_id,
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index a1610b396..9aed4999e 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -701,6 +701,19 @@ class WorkflowRunModel(Base):
"queued_at",
postgresql_where=text("status IN ('queued', 'running', 'paused') AND browser_session_id IS NULL"),
),
+ Index(
+ "ix_workflow_runs_sequential_credential_gate",
+ "organization_id",
+ "sequential_credential_id",
+ "queued_at",
+ postgresql_where=text("sequential_credential_id IS NOT NULL AND status IN ('queued', 'running', 'paused')"),
+ ),
+ Index(
+ "ix_workflow_runs_serialized_ticket",
+ "organization_id",
+ text("queued_at DESC"),
+ postgresql_where=text("status IN ('queued', 'running', 'paused')"),
+ ),
Index(
"ix_workflow_runs_retried_from_workflow_run_id",
"retried_from_workflow_run_id",
@@ -737,6 +750,7 @@ class WorkflowRunModel(Base):
job_id = Column(String, nullable=True, index=True)
depends_on_workflow_run_id = Column(String, nullable=True, index=True)
sequential_key = Column(String, nullable=True)
+ sequential_credential_id = Column(String, nullable=True)
run_with = Column(String, nullable=True) # 'agent' or 'code'
debug_session_id: Column = Column(String, nullable=True)
trigger_type = Column(String, nullable=True)
@@ -1534,6 +1548,7 @@ class CredentialModel(Base):
user_context = Column(String(1000), nullable=True)
save_browser_session_intent = Column(Boolean, nullable=True, default=False)
pin_saved_session_ip = Column(Boolean, nullable=False, default=False, server_default=sqlalchemy.false())
+ run_sequentially = Column(Boolean, nullable=False, default=False, server_default=sqlalchemy.false())
proxy_location = Column(String, nullable=True)
proxy_session_id = Column(String, nullable=True)
folder_id = Column(String, ForeignKey("credential_folders.folder_id", ondelete="SET NULL"), nullable=True)
diff --git a/skyvern/forge/sdk/db/repositories/credentials.py b/skyvern/forge/sdk/db/repositories/credentials.py
index fde49eb70..f31645358 100644
--- a/skyvern/forge/sdk/db/repositories/credentials.py
+++ b/skyvern/forge/sdk/db/repositories/credentials.py
@@ -203,6 +203,7 @@ class CredentialRepository(BaseRepository):
tested_url: str | None = None,
user_context: str | None = None,
save_browser_session_intent: bool | None = None,
+ run_sequentially: bool | None = None,
proxy_location: ProxyLocationInput | object = _UNSET,
proxy_session_id: str | None | object = _UNSET,
rotate_proxy_session_id: bool = False,
@@ -232,6 +233,8 @@ class CredentialRepository(BaseRepository):
credential.user_context = user_context
if save_browser_session_intent is not None:
credential.save_browser_session_intent = save_browser_session_intent
+ if run_sequentially is not None:
+ credential.run_sequentially = run_sequentially
apply_proxy_pin_to_model(
credential,
entity_id=credential_id,
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index 924d725a2..569a5da35 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -9,6 +9,7 @@ import structlog
from sqlalchemy import (
ColumnElement,
Label,
+ Select,
Text,
and_,
cast,
@@ -71,6 +72,22 @@ from skyvern.schemas.runs import MAX_SEARCH_FETCH_LIMIT, ProxyLocationInput, Run
LOG = structlog.get_logger()
+def _noncredential_lane_variants(base: Select) -> tuple[Select, Select]:
+ """Split the widened non-session lane into two index-eligible candidates whose union equals
+ ``browser_session_id IS NULL OR sequential_credential_id IS NOT NULL``.
+
+ A single OR of those two predicates cannot use ``ix_workflow_runs_sequential_key_lookup`` (partial
+ on ``browser_session_id IS NULL``), so it de-indexes the pre-existing sequential-key/whole-workflow
+ lookups on the high-write ``workflow_runs`` table. Kept apart, the legacy lane stays on that partial
+ index and the rare credential-composed lane rides the ``workflow_permanent_id`` index. Callers run
+ both and pick the winner by the caller's own ordering — identical result set, index-eligible shape.
+ """
+ return (
+ base.filter(WorkflowRunModel.browser_session_id.is_(None)),
+ base.filter(WorkflowRunModel.sequential_credential_id.isnot(None)),
+ )
+
+
def _merge_script_run(
existing: dict | None,
ai_fallback_triggered: bool | None,
@@ -201,6 +218,7 @@ class WorkflowRunsRepository(BaseRepository):
cdp_connect_headers: dict[str, str] | None = None,
browser_address: str | None = None,
sequential_key: str | None = None,
+ sequential_credential_id: str | None = None,
run_with: str | None = None,
debug_session_id: str | None = None,
ai_fallback: bool | None = None,
@@ -237,6 +255,7 @@ class WorkflowRunsRepository(BaseRepository):
cdp_connect_headers=cdp_connect_headers,
browser_address=browser_address,
sequential_key=sequential_key,
+ sequential_credential_id=sequential_credential_id,
run_with=run_with,
debug_session_id=debug_session_id,
ai_fallback=ai_fallback,
@@ -279,6 +298,7 @@ class WorkflowRunsRepository(BaseRepository):
job_id: str | None = None,
run_with: str | None = None,
sequential_key: str | None = None,
+ sequential_credential_id: str | None = None,
ai_fallback: bool | None = None,
depends_on_workflow_run_id: str | None = None,
browser_session_id: str | None = None,
@@ -305,7 +325,51 @@ class WorkflowRunsRepository(BaseRepository):
if status:
workflow_run.status = status
if status and status == WorkflowRunStatus.queued and workflow_run.queued_at is None:
- workflow_run.queued_at = naive_utc_now()
+ credential_id = sequential_credential_id or workflow_run.sequential_credential_id
+ serialized_publication = bool(
+ credential_id
+ or (workflow_run.browser_session_id and not workflow_run.debug_session_id)
+ or workflow_run.browser_address
+ or workflow_run.sequential_key
+ )
+ if not serialized_publication and workflow_run.workflow_id:
+ workflow = await session.get(WorkflowModel, workflow_run.workflow_id)
+ serialized_publication = bool(workflow and workflow.run_sequentially)
+ if serialized_publication:
+ # The caller holds every composed publication-lane lock until this transaction
+ # commits. Advance beyond every active serialized ticket in the organization,
+ # so all composed lanes share one comparable clock even when database transaction
+ # time or an application host clock moved backwards.
+ latest_ticket = await session.scalar(
+ select(func.max(WorkflowRunModel.queued_at))
+ .where(WorkflowRunModel.organization_id == workflow_run.organization_id)
+ .where(WorkflowRunModel.workflow_run_id != workflow_run_id)
+ .where(
+ WorkflowRunModel.status.in_(
+ [
+ WorkflowRunStatus.queued,
+ WorkflowRunStatus.running,
+ WorkflowRunStatus.paused,
+ ]
+ )
+ )
+ )
+ database_now = await session.scalar(select(func.now()))
+ if database_now is None:
+ raise RuntimeError("Database did not return a credential publication timestamp")
+ database_now = to_naive_utc(database_now)
+ assert database_now is not None
+ if latest_ticket is not None:
+ latest_ticket_utc = to_naive_utc(latest_ticket)
+ assert latest_ticket_utc is not None
+ workflow_run.queued_at = max(
+ database_now,
+ latest_ticket_utc + timedelta(microseconds=1),
+ )
+ else:
+ workflow_run.queued_at = database_now
+ else:
+ workflow_run.queued_at = naive_utc_now()
if status and status == WorkflowRunStatus.running and workflow_run.started_at is None:
workflow_run.started_at = naive_utc_now()
if status and status.is_final() and workflow_run.finished_at is None:
@@ -327,6 +391,8 @@ class WorkflowRunsRepository(BaseRepository):
workflow_run.run_with = run_with
if sequential_key:
workflow_run.sequential_key = sequential_key
+ if sequential_credential_id is not None:
+ workflow_run.sequential_credential_id = sequential_credential_id
if ai_fallback is not None:
workflow_run.ai_fallback = ai_fallback
if depends_on_workflow_run_id:
@@ -859,19 +925,40 @@ class WorkflowRunsRepository(BaseRepository):
organization_id: str | None = None,
sequential_key: str | None = None,
include_browser_session_rows: bool = False,
+ include_credential_composed_rows: bool = True,
) -> WorkflowRun | None:
async with self.Session() as session:
query = select(WorkflowRunModel).filter_by(workflow_permanent_id=workflow_permanent_id)
- if not include_browser_session_rows:
- query = query.filter(WorkflowRunModel.browser_session_id.is_(None))
if organization_id:
query = query.filter_by(organization_id=organization_id)
query = query.filter_by(status=WorkflowRunStatus.queued)
if sequential_key:
query = query.filter_by(sequential_key=sequential_key)
- query = query.order_by(WorkflowRunModel.modified_at.desc())
- workflow_run = (await session.scalars(query)).first()
- return convert_to_workflow_run(workflow_run) if workflow_run else None
+ query = query.order_by(WorkflowRunModel.modified_at.desc()).limit(1)
+ if include_browser_session_rows:
+ workflow_run = (await session.scalars(query)).first()
+ return convert_to_workflow_run(workflow_run) if workflow_run else None
+ # Credential-composed rows occupy the manual-key/whole-workflow lane by the gate contract
+ # even though they carry a browser_session_id, so a later non-credential run in that lane
+ # must still see them; plain (non-credential) session rows stay excluded. Run the two
+ # index-eligible lane candidates separately (not one de-indexing OR) and keep the most
+ # recently modified — the same row the OR would have returned.
+ legacy_query, credential_query = _noncredential_lane_variants(query)
+ if not include_credential_composed_rows:
+ workflow_run = (await session.scalars(legacy_query)).first()
+ return convert_to_workflow_run(workflow_run) if workflow_run else None
+ candidates = [
+ candidate
+ for candidate in (
+ (await session.scalars(legacy_query)).first(),
+ (await session.scalars(credential_query)).first(),
+ )
+ if candidate is not None
+ ]
+ if not candidates:
+ return None
+ workflow_run = max(candidates, key=lambda run: run.modified_at)
+ return convert_to_workflow_run(workflow_run)
@db_operation("get_workflow_runs_by_ids")
async def get_workflow_runs_by_ids(
@@ -896,11 +983,10 @@ class WorkflowRunsRepository(BaseRepository):
organization_id: str | None = None,
sequential_key: str | None = None,
include_browser_session_rows: bool = False,
+ include_credential_composed_rows: bool = True,
) -> WorkflowRun | None:
async with self.Session() as session:
query = select(WorkflowRunModel).filter_by(workflow_permanent_id=workflow_permanent_id)
- if not include_browser_session_rows:
- query = query.filter(WorkflowRunModel.browser_session_id.is_(None))
if organization_id:
query = query.filter_by(organization_id=organization_id)
query = query.filter_by(status=WorkflowRunStatus.running)
@@ -909,9 +995,28 @@ class WorkflowRunsRepository(BaseRepository):
query = query.filter(
WorkflowRunModel.started_at.isnot(None)
) # filter out workflow runs that does not have a started_at timestamp
- query = query.order_by(WorkflowRunModel.started_at.desc())
- workflow_run = (await session.scalars(query)).first()
- return convert_to_workflow_run(workflow_run) if workflow_run else None
+ query = query.order_by(WorkflowRunModel.started_at.desc()).limit(1)
+ if include_browser_session_rows:
+ workflow_run = (await session.scalars(query)).first()
+ return convert_to_workflow_run(workflow_run) if workflow_run else None
+ # Same composed-lane admission as the queued lookup, split into two index-eligible lane
+ # candidates instead of one de-indexing OR; keep the most recently started.
+ legacy_query, credential_query = _noncredential_lane_variants(query)
+ if not include_credential_composed_rows:
+ workflow_run = (await session.scalars(legacy_query)).first()
+ return convert_to_workflow_run(workflow_run) if workflow_run else None
+ candidates = [
+ candidate
+ for candidate in (
+ (await session.scalars(legacy_query)).first(),
+ (await session.scalars(credential_query)).first(),
+ )
+ if candidate is not None and candidate.started_at is not None
+ ]
+ if not candidates:
+ return None
+ workflow_run = max(candidates, key=lambda run: run.started_at)
+ return convert_to_workflow_run(workflow_run)
@db_operation("get_blocking_sequential_workflow_run")
async def get_blocking_sequential_workflow_run(self, workflow_run_id: str) -> WorkflowRun | None:
@@ -952,56 +1057,105 @@ class WorkflowRunsRepository(BaseRepository):
exc_info=True,
)
- # Lane resolution mirrors enqueue priority: browser_session_id > browser_address
- # > sequential_key > whole workflow. Debug and forced-session runs carry a
- # browser_session_id but are excluded from the session lane as they are at enqueue.
+ # A credential run composes lanes: it blocks on any earlier active run sharing its
+ # single credential (exact equality), browser session, browser address, sequential_key,
+ # or a run_sequentially workflow — one bounded SQL query, not a scan + set intersection.
+ # A non-credential run keeps strict lane priority:
+ # browser_session_id > browser_address > sequential_key > whole workflow.
query = select(WorkflowRunModel).filter_by(organization_id=run.organization_id)
- if run.browser_session_id and not run.debug_session_id and not self_forced:
- query = query.filter_by(browser_session_id=run.browser_session_id)
+ credential_id = run.sequential_credential_id
+ # Each lane candidate is an independently index-eligible query; the earliest blocker across
+ # them is the run's blocker. The non-credential manual-key / whole-workflow lanes must admit
+ # credential-composed predecessors, but as two separate candidates (legacy browser_session_id
+ # IS NULL lane + credential lane) rather than one OR that de-indexes the sequential-key index.
+ lane_queries: list[Select]
+ if credential_id:
+ run_workflow = await session.get(WorkflowModel, run.workflow_id) if run.workflow_id else None
+ whole_workflow_sequential = bool(run_workflow and run_workflow.run_sequentially)
+ lanes = [WorkflowRunModel.sequential_credential_id == credential_id]
+ if not self_forced and run.browser_session_id and not run.debug_session_id:
+ lanes.append(WorkflowRunModel.browser_session_id == run.browser_session_id)
+ if run.browser_address:
+ lanes.append(WorkflowRunModel.browser_address == run.browser_address)
+ if whole_workflow_sequential:
+ lanes.append(
+ and_(
+ WorkflowRunModel.workflow_permanent_id == run.workflow_permanent_id,
+ or_(
+ WorkflowRunModel.browser_session_id.is_(None),
+ WorkflowRunModel.sequential_credential_id.isnot(None),
+ ),
+ )
+ )
+ elif run.sequential_key:
+ lanes.append(
+ and_(
+ WorkflowRunModel.workflow_permanent_id == run.workflow_permanent_id,
+ WorkflowRunModel.sequential_key == run.sequential_key,
+ or_(
+ WorkflowRunModel.browser_session_id.is_(None),
+ WorkflowRunModel.sequential_credential_id.isnot(None),
+ ),
+ )
+ )
+ lane_queries = [query.filter(or_(*lanes))]
+ elif run.browser_session_id and not run.debug_session_id and not self_forced:
+ lane_queries = [query.filter_by(browser_session_id=run.browser_session_id)]
elif run.browser_address:
- query = query.filter_by(browser_address=run.browser_address)
+ lane_queries = [query.filter_by(browser_address=run.browser_address)]
elif run.sequential_key:
- query = query.filter_by(
+ keyed = query.filter_by(
workflow_permanent_id=run.workflow_permanent_id,
sequential_key=run.sequential_key,
)
- if not self_forced:
- query = query.filter(WorkflowRunModel.browser_session_id.is_(None))
+ # self_forced runs already saw every keyed occupant (including plain session rows), so
+ # they keep the single unfiltered lane; a non-forced run splits into the two admission
+ # candidates so a credential-composed predecessor is seen while plain session rows stay
+ # excluded — index-eligible either way.
+ lane_queries = [keyed] if self_forced else list(_noncredential_lane_variants(keyed))
else:
- query = query.filter_by(workflow_permanent_id=run.workflow_permanent_id)
- if not self_forced:
- query = query.filter(WorkflowRunModel.browser_session_id.is_(None))
+ whole = query.filter_by(workflow_permanent_id=run.workflow_permanent_id)
+ lane_queries = [whole] if self_forced else list(_noncredential_lane_variants(whole))
# Sequential runs are stamped queued_at before Temporal submission; the fallback
# only guards hand-created rows (e.g. tests) from comparing against None.
self_queued_at = run.queued_at if run.queued_at is not None else run.created_at
- query = query.filter(
- WorkflowRunModel.status.in_(
- [
- WorkflowRunStatus.queued,
- WorkflowRunStatus.running,
- WorkflowRunStatus.paused,
- ]
+ def _apply_gate_tail(lane_query: Select) -> Select:
+ lane_query = lane_query.filter(
+ WorkflowRunModel.status.in_(
+ [
+ WorkflowRunStatus.queued,
+ WorkflowRunStatus.running,
+ WorkflowRunStatus.paused,
+ ]
+ )
)
- )
- # Safe because a sequential run is always stamped queued_at (passes through queued)
- # before it can reach running/paused, so this filter never hides a real blocker.
- query = query.filter(WorkflowRunModel.queued_at.isnot(None))
- query = query.filter(
- or_(
- WorkflowRunModel.queued_at < self_queued_at,
- and_(
- WorkflowRunModel.queued_at == self_queued_at,
- WorkflowRunModel.workflow_run_id < run.workflow_run_id,
- ),
+ # Safe because a sequential run is always stamped queued_at (passes through queued)
+ # before it can reach running/paused, so this filter never hides a real blocker.
+ lane_query = lane_query.filter(WorkflowRunModel.queued_at.isnot(None))
+ lane_query = lane_query.filter(
+ or_(
+ WorkflowRunModel.queued_at < self_queued_at,
+ and_(
+ WorkflowRunModel.queued_at == self_queued_at,
+ WorkflowRunModel.workflow_run_id < run.workflow_run_id,
+ ),
+ )
)
- )
- query = query.order_by(
- WorkflowRunModel.queued_at.asc(),
- WorkflowRunModel.workflow_run_id.asc(),
- )
- blocker = (await session.scalars(query)).first()
+ return lane_query.order_by(
+ WorkflowRunModel.queued_at.asc(),
+ WorkflowRunModel.workflow_run_id.asc(),
+ )
+
+ blocker: WorkflowRunModel | None = None
+ for lane_query in lane_queries:
+ candidate = (await session.scalars(_apply_gate_tail(lane_query))).first()
+ if candidate is not None and (
+ blocker is None
+ or (candidate.queued_at, candidate.workflow_run_id) < (blocker.queued_at, blocker.workflow_run_id)
+ ):
+ blocker = candidate
return convert_to_workflow_run(blocker) if blocker else None
async def _get_last_workflow_run_by_filter(
diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py
index cf566f1ec..9fb580437 100644
--- a/skyvern/forge/sdk/db/utils.py
+++ b/skyvern/forge/sdk/db/utils.py
@@ -637,6 +637,7 @@ def convert_to_workflow_run(
job_id=workflow_run_model.job_id,
depends_on_workflow_run_id=workflow_run_model.depends_on_workflow_run_id,
sequential_key=workflow_run_model.sequential_key,
+ sequential_credential_id=workflow_run_model.sequential_credential_id,
script_run=ScriptRunResponse.model_validate(workflow_run_model.script_run)
if workflow_run_model.script_run
else None,
diff --git a/skyvern/forge/sdk/executor/background_task_executor.py b/skyvern/forge/sdk/executor/background_task_executor.py
index 70d9d2047..ae9538ea3 100644
--- a/skyvern/forge/sdk/executor/background_task_executor.py
+++ b/skyvern/forge/sdk/executor/background_task_executor.py
@@ -6,13 +6,14 @@ from typing import Any
import structlog
from fastapi import BackgroundTasks, Request
-from skyvern.exceptions import OrganizationNotFound
+from skyvern.exceptions import BackgroundSequentialCredentialUnsupported, OrganizationNotFound
from skyvern.forge import app
from skyvern.forge.sdk.api.llm.custom_llm_registry import load_custom_llm_configs_for_organization
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.executor.async_executor import AsyncExecutor
from skyvern.forge.sdk.schemas.organizations import Organization
+from skyvern.forge.sdk.schemas.persistent_browser_sessions import FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE
from skyvern.forge.sdk.schemas.task_v2 import TaskV2Status
from skyvern.forge.sdk.schemas.tasks import TaskStatus
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
@@ -145,6 +146,42 @@ class BackgroundTaskExecutor(AsyncExecutor):
workflow_run_id=workflow_run_id,
)
+ workflow_run = await app.DATABASE.workflow_runs.get_workflow_run(
+ workflow_run_id,
+ organization_id=organization.organization_id,
+ )
+ if workflow_run and workflow_run.sequential_credential_id:
+ if workflow_run.browser_session_id:
+ persistent_browser_session = await app.DATABASE.browser_sessions.get_persistent_browser_session(
+ session_id=workflow_run.browser_session_id,
+ organization_id=organization.organization_id,
+ )
+ if (
+ persistent_browser_session
+ and persistent_browser_session.runnable_type == FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE
+ ):
+ try:
+ await app.PERSISTENT_SESSIONS_MANAGER.close_session(
+ organization.organization_id,
+ workflow_run.browser_session_id,
+ )
+ except Exception:
+ LOG.exception(
+ "Failed to close forced browser session before rejecting sequential credential run",
+ organization_id=organization.organization_id,
+ workflow_run_id=workflow_run_id,
+ browser_session_id=workflow_run.browser_session_id,
+ )
+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(
+ workflow_run_id=workflow_run_id,
+ failure_reason=(
+ "Sequential credential execution is unavailable in the background executor; "
+ "the run failed closed before execution."
+ ),
+ cascade_children=True,
+ )
+ raise BackgroundSequentialCredentialUnsupported(workflow_run_id)
+
await initialize_skyvern_state_file(
workflow_run_id=workflow_run_id, organization_id=organization.organization_id
)
diff --git a/skyvern/forge/sdk/routes/credentials.py b/skyvern/forge/sdk/routes/credentials.py
index 36f12761b..ecb877eb5 100644
--- a/skyvern/forge/sdk/routes/credentials.py
+++ b/skyvern/forge/sdk/routes/credentials.py
@@ -876,6 +876,13 @@ async def rename_credential(
update_kwargs["user_context"] = data.user_context
if data.save_browser_session_intent is not None:
update_kwargs["save_browser_session_intent"] = data.save_browser_session_intent
+ if data.run_sequentially is True and not app.AGENT_FUNCTION.supports_sequential_credentials():
+ raise HTTPException(
+ status_code=400,
+ detail="Sequential credential execution is not supported by this deployment",
+ )
+ if data.run_sequentially is not None:
+ update_kwargs["run_sequentially"] = data.run_sequentially
_apply_proxy_pin_update(
update_kwargs,
proxy_location_was_set="proxy_location" in data.model_fields_set,
@@ -3212,6 +3219,7 @@ def _convert_to_response(credential: Credential) -> CredentialResponse:
tested_url=credential.tested_url,
user_context=credential.user_context,
save_browser_session_intent=credential.save_browser_session_intent,
+ run_sequentially=credential.run_sequentially,
folder_id=credential.folder_id,
proxy_location=credential.proxy_location,
proxy_session_id=credential.proxy_session_id,
@@ -3232,6 +3240,7 @@ def _convert_to_response(credential: Credential) -> CredentialResponse:
tested_url=credential.tested_url,
user_context=credential.user_context,
save_browser_session_intent=credential.save_browser_session_intent,
+ run_sequentially=credential.run_sequentially,
folder_id=credential.folder_id,
proxy_location=credential.proxy_location,
proxy_session_id=credential.proxy_session_id,
@@ -3249,6 +3258,7 @@ def _convert_to_response(credential: Credential) -> CredentialResponse:
tested_url=credential.tested_url,
user_context=credential.user_context,
save_browser_session_intent=credential.save_browser_session_intent,
+ run_sequentially=credential.run_sequentially,
folder_id=credential.folder_id,
proxy_location=credential.proxy_location,
proxy_session_id=credential.proxy_session_id,
diff --git a/skyvern/forge/sdk/schemas/credentials.py b/skyvern/forge/sdk/schemas/credentials.py
index 5c9b375a8..e395cb7e3 100644
--- a/skyvern/forge/sdk/schemas/credentials.py
+++ b/skyvern/forge/sdk/schemas/credentials.py
@@ -284,6 +284,7 @@ class CredentialResponse(BaseModel):
default=None,
description="Whether the user intends to save a browser session, regardless of test outcome",
)
+ run_sequentially: bool = False
folder_id: str | None = Field(
default=None,
description="ID of the credential folder this credential belongs to, if any",
@@ -383,6 +384,7 @@ class Credential(BaseModel):
default=False,
description="Whether the user intends to save a browser session, regardless of test outcome",
)
+ run_sequentially: bool = False
folder_id: str | None = Field(
default=None,
description="ID of the credential folder this credential belongs to, if any",
@@ -434,6 +436,15 @@ class UpdateCredentialRequest(BaseModel):
default=None,
description="Whether the user intends to save a browser session, regardless of test outcome",
)
+ run_sequentially: bool | None = Field(
+ default=None,
+ description=(
+ "Whether runs using this credential are serialized so at most one runs at a time. "
+ "Serialization applies only to runs created after this flag is enabled: runs already "
+ "queued or running when it is turned on are not stamped and are not serialized against "
+ "later runs. Drain in-flight runs on this credential before enabling for full protection."
+ ),
+ )
proxy_location: ProxyLocationInput = Field(
default=None,
description="Optional proxy location for this credential's pinned proxy identity.",
diff --git a/skyvern/forge/sdk/workflow/context_manager.py b/skyvern/forge/sdk/workflow/context_manager.py
index 1d639be8f..b1df64920 100644
--- a/skyvern/forge/sdk/workflow/context_manager.py
+++ b/skyvern/forge/sdk/workflow/context_manager.py
@@ -1,5 +1,6 @@
import copy
import re
+from collections.abc import Mapping
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, Awaitable, Callable, Self
@@ -16,10 +17,12 @@ from skyvern.exceptions import (
CredentialParameterNotFoundError,
CredentialVaultNotConfiguredError,
ImaginarySecretValue,
+ InvalidCredentialId,
OnePasswordGetItemError,
OnePasswordRateLimitError,
OnePasswordServiceUnavailableError,
OnePasswordSessionExpiredError,
+ RuntimeSequentialCredentialUnsupported,
SkyvernException,
WorkflowRunContextNotInitialized,
sanitize_credential_for_error,
@@ -83,6 +86,32 @@ _CREDENTIAL_PARAMETER_TYPES: tuple[type, ...] = (
_SECRET_FIELD_KEY_PATTERN = re.compile(r"[^A-Za-z0-9_]+")
+# Registered secrets are masked by substring across run outputs, so a low-entropy value corrupts
+# unrelated text — a registered "visa" blanks that word wherever it appears. Only fields the safe
+# credential API already returns belong here; billing fields are excluded from that API on purpose.
+NON_SECRET_CREDENTIAL_FIELDS = frozenset({"card_brand"})
+
+# Secrets shorter than this mask only on exact whole-string match: substring-replacing a short
+# value (a CVV, a 2-digit expiry) corrupts unrelated scalars such as timestamp milliseconds.
+_SECRET_SUBSTRING_MIN_LENGTH = 5
+
+
+def resolve_credential_parameter_binding(
+ parameter: CredentialParameter,
+ parameter_values: Mapping[str, Any],
+ selected_credential_id: str | None = None,
+) -> str:
+ credential_id = (
+ selected_credential_id
+ if selected_credential_id is not None
+ else parameter_values.get(parameter.credential_id, parameter.credential_id)
+ )
+ if not isinstance(credential_id, str):
+ raise InvalidCredentialId(f"<non-string value of type {type(credential_id).__name__}>")
+ if not credential_id:
+ raise InvalidCredentialId(credential_id)
+ return credential_id
+
class WorkflowRunContext:
@classmethod
@@ -487,10 +516,11 @@ class WorkflowRunContext:
def mask_secrets_in_data(self, data: Any, mask: str = "*****") -> Any:
"""
- Recursively replace any real secret values in data with a mask.
+ Recursively replace registered secret values in data with a mask.
Used to sanitize HttpRequestBlock output before storing.
- Only masks values that exist in self.secrets (registered credentials).
+ Values shorter than _SECRET_SUBSTRING_MIN_LENGTH mask only when they are the entire
+ string; a short secret embedded inside a longer scalar is knowingly left unmasked.
"""
if not self.secrets:
return data
@@ -502,9 +532,12 @@ class WorkflowRunContext:
return data
if isinstance(data, str):
+ if data in secret_values:
+ return mask
result = data
for secret in secret_values:
- result = result.replace(secret, mask)
+ if len(secret) >= _SECRET_SUBSTRING_MIN_LENGTH:
+ result = result.replace(secret, mask)
return result
elif isinstance(data, dict):
return {k: self.mask_secrets_in_data(v, mask) for k, v in data.items()}
@@ -644,6 +677,13 @@ class WorkflowRunContext:
)
if db_credential is None:
raise CredentialParameterNotFoundError(credential_id)
+ if db_credential.run_sequentially is True:
+ workflow_run = await app.DATABASE.workflow_runs.get_workflow_run(
+ self.workflow_run_id,
+ organization.organization_id,
+ )
+ if workflow_run is None or workflow_run.sequential_credential_id != credential_id:
+ raise RuntimeSequentialCredentialUnsupported(self.workflow_run_id)
self.resolved_credential_parameter_ids[parameter.key] = credential_id
@@ -673,6 +713,9 @@ class WorkflowRunContext:
continue
for field_key, field_value in self._flatten_credential_secret_field(key, value):
field_key = self._dedupe_secret_field_key(field_key, used_secret_field_keys)
+ if field_key in NON_SECRET_CREDENTIAL_FIELDS:
+ self.values[parameter.key][field_key] = field_value
+ continue
random_secret_id = self.generate_random_secret_id()
secret_id = f"{random_secret_id}_{field_key}"
self.secrets[secret_id] = field_value
@@ -721,19 +764,7 @@ class WorkflowRunContext:
) -> None:
LOG.info("Fetching credential parameter value", parameter_key=parameter.key)
- credential_id = None
- if parameter.credential_ids or parameter.fallback_credential_ids:
- credential_id = await self.resolve_credential_parameter_id(parameter, organization.organization_id)
- elif parameter.credential_id:
- if self.has_parameter(parameter.credential_id) and self.has_value(parameter.credential_id):
- credential_id = self.values[parameter.credential_id]
- else:
- credential_id = parameter.credential_id
-
- if credential_id is None:
- LOG.error("Credential ID not found", parameter_key=parameter.key)
- raise CredentialParameterNotFoundError(parameter.credential_id)
-
+ credential_id = await self.resolve_credential_parameter_id(parameter, organization.organization_id)
await self._register_credential_parameter_value(credential_id, parameter, organization)
async def resolve_credential_parameter_id(
@@ -744,26 +775,30 @@ class WorkflowRunContext:
cached = self.resolved_credential_parameter_ids.get(parameter.key)
if cached:
return cached
- if not parameter.credential_ids:
- credential_id = parameter.credential_id
- if parameter.fallback_credential_ids:
- selected = await app.DATABASE.workflow_run_credential_selections.get_selection(
- workflow_run_id=self.workflow_run_id,
- parameter_key=parameter.key,
- )
- if selected:
- credential_id = selected
- elif self.has_parameter(credential_id) and self.has_value(credential_id):
- credential_id = self.values[credential_id]
- self.resolved_credential_parameter_ids[parameter.key] = credential_id
- return credential_id
- credential_id = await select_credential_for_run(
- workflow_run_id=self.workflow_run_id,
- organization_id=organization_id,
- workflow_permanent_id=self.workflow_permanent_id,
- parameter_key=parameter.key,
- credential_ids=parameter.credential_ids,
- selection_strategy=parameter.selection_strategy,
+ selected_credential_id = None
+ if parameter.credential_ids:
+ selected_credential_id = await select_credential_for_run(
+ workflow_run_id=self.workflow_run_id,
+ organization_id=organization_id,
+ workflow_permanent_id=self.workflow_permanent_id,
+ parameter_key=parameter.key,
+ credential_ids=parameter.credential_ids,
+ selection_strategy=parameter.selection_strategy,
+ )
+ elif parameter.fallback_credential_ids:
+ selected_credential_id = await app.DATABASE.workflow_run_credential_selections.get_selection(
+ workflow_run_id=self.workflow_run_id,
+ parameter_key=parameter.key,
+ )
+ registered_parameter_values = {
+ key: self.resolved_credential_parameter_ids.get(key, self.values[key])
+ for key in self.parameters
+ if key in self.values
+ }
+ credential_id = resolve_credential_parameter_binding(
+ parameter,
+ registered_parameter_values,
+ selected_credential_id,
)
self.resolved_credential_parameter_ids[parameter.key] = credential_id
return credential_id
@@ -1322,6 +1357,9 @@ class WorkflowRunContext:
if not field_key:
continue
field_key = self._dedupe_secret_field_key(field_key, used_secret_field_keys)
+ if field_key in NON_SECRET_CREDENTIAL_FIELDS:
+ parameter_value[field_key] = credit_card_data[data_key]
+ continue
random_secret_id = self.generate_random_secret_id()
secret_id = f"{random_secret_id}_{field_key}"
self.secrets[secret_id] = credit_card_data[data_key]
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index a265ed501..fa0e40c26 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -71,6 +71,7 @@ from skyvern.exceptions import (
NoTOTPVerificationCodeFound,
PDFParsingError,
SkyvernException,
+ SyncTriggeredSequentialCredentialUnsupported,
TaskNotFound,
UnexpectedTaskStatus,
get_user_facing_exception_message,
@@ -12385,86 +12386,105 @@ class WorkflowTriggerBlock(Block):
trigger_type=inherited_trigger_type,
)
):
+ # A self-created fresh session must be closed on every exit from this block —
+ # including the setup-failure and sequential-credential fence early returns — so the
+ # cleanup lives in a finally rather than only on the normal execute path below.
+ triggered_run_id: str | None = None
try:
- triggered_workflow_run = await app.WORKFLOW_SERVICE.setup_workflow_run(
- request_id=None,
- workflow_request=workflow_request,
- workflow_permanent_id=resolved_workflow_permanent_id,
- organization=organization,
- parent_workflow_run_id=workflow_run_id,
- ignore_inherited_workflow_system_prompt=self.ignore_workflow_system_prompt,
- trigger_type=inherited_trigger_type,
- )
- except Exception as e:
- error_msg = get_user_facing_exception_message(e)
- return await _fail(f"Failed to setup triggered workflow run: {error_msg}")
-
- triggered_run_id = triggered_workflow_run.workflow_run_id
-
- LOG.info(
- "Triggered workflow run (sync)",
- parent_workflow_run_id=workflow_run_id,
- triggered_workflow_run_id=triggered_run_id,
- triggered_workflow_permanent_id=resolved_workflow_permanent_id,
- )
-
- try:
- # The opt-out flag is persisted on the child's workflow_run row at
- # spawn time (setup_workflow_run above), so execute_workflow reads
- # it from the DB. This works identically for sync and async triggers.
- final_run = await app.WORKFLOW_SERVICE.execute_workflow(
- workflow_run_id=triggered_run_id,
- api_key=None,
- organization=organization,
- browser_session_id=resolved_browser_session_id,
- )
- success = final_run.status == WorkflowRunStatus.completed
- output_data = {
- "workflow_run_id": triggered_run_id,
- "workflow_permanent_id": resolved_workflow_permanent_id,
- "status": str(final_run.status),
- "failure_reason": final_run.failure_reason,
- }
- # Include the child workflow's output parameters so downstream
- # blocks can reference them (e.g. block_3_output.outputs.block_2_output)
try:
- child_output_params = (
- await app.WORKFLOW_SERVICE.get_output_parameter_workflow_run_output_parameter_tuples(
- workflow_id=final_run.workflow_id,
- workflow_run_id=triggered_run_id,
- )
+ triggered_workflow_run = await app.WORKFLOW_SERVICE.setup_workflow_run(
+ request_id=None,
+ workflow_request=workflow_request,
+ workflow_permanent_id=resolved_workflow_permanent_id,
+ organization=organization,
+ parent_workflow_run_id=workflow_run_id,
+ ignore_inherited_workflow_system_prompt=self.ignore_workflow_system_prompt,
+ trigger_type=inherited_trigger_type,
)
- child_outputs: dict[str, Any] = {}
- for output_param, run_output_param in child_output_params:
- child_outputs[output_param.key] = run_output_param.value
- output_data["outputs"] = child_outputs
- except Exception:
- LOG.warning(
- "Failed to fetch child workflow outputs",
- triggered_workflow_run_id=triggered_run_id,
- exc_info=True,
+ except Exception as e:
+ error_msg = get_user_facing_exception_message(e)
+ return await _fail(f"Failed to setup triggered workflow run: {error_msg}")
+
+ triggered_run_id = triggered_workflow_run.workflow_run_id
+
+ # A synchronous child runs inline via execute_workflow below: it never queues, never
+ # gets a queued_at, and never reaches the Temporal V2 serialization gate. setup stamped
+ # its sequential_credential_id, but the gate filters queued_at IS NOT NULL, so a
+ # concurrent run sharing that credential would not see this child as a blocker. Fail
+ # closed before it uses the credential — the same fence the scheduled path applies.
+ if triggered_workflow_run.sequential_credential_id:
+ fence = SyncTriggeredSequentialCredentialUnsupported(triggered_run_id)
+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(
+ workflow_run_id=triggered_run_id,
+ failure_reason=str(fence),
)
- except Exception as e:
- error_msg = get_user_facing_exception_message(e)
- output_data = {
- "workflow_run_id": triggered_run_id,
- "workflow_permanent_id": resolved_workflow_permanent_id,
- "status": "failed",
- "failure_reason": f"Triggered workflow execution failed: {error_msg}",
- }
- success = False
- if created_fresh_session and resolved_browser_session_id:
+ return await _fail(str(fence))
+
+ LOG.info(
+ "Triggered workflow run (sync)",
+ parent_workflow_run_id=workflow_run_id,
+ triggered_workflow_run_id=triggered_run_id,
+ triggered_workflow_permanent_id=resolved_workflow_permanent_id,
+ )
+
try:
- await app.PERSISTENT_SESSIONS_MANAGER.close_session(
- organization_id, resolved_browser_session_id
- )
- except Exception:
- LOG.warning(
- "Failed to close child browser session",
- child_browser_session_id=resolved_browser_session_id,
- triggered_workflow_run_id=triggered_run_id,
- exc_info=True,
+ # The opt-out flag is persisted on the child's workflow_run row at
+ # spawn time (setup_workflow_run above), so execute_workflow reads
+ # it from the DB. This works identically for sync and async triggers.
+ final_run = await app.WORKFLOW_SERVICE.execute_workflow(
+ workflow_run_id=triggered_run_id,
+ api_key=None,
+ organization=organization,
+ browser_session_id=resolved_browser_session_id,
)
+ success = final_run.status == WorkflowRunStatus.completed
+ output_data = {
+ "workflow_run_id": triggered_run_id,
+ "workflow_permanent_id": resolved_workflow_permanent_id,
+ "status": str(final_run.status),
+ "failure_reason": final_run.failure_reason,
+ }
+ # Include the child workflow's output parameters so downstream
+ # blocks can reference them (e.g. block_3_output.outputs.block_2_output)
+ try:
+ child_output_params = (
+ await app.WORKFLOW_SERVICE.get_output_parameter_workflow_run_output_parameter_tuples(
+ workflow_id=final_run.workflow_id,
+ workflow_run_id=triggered_run_id,
+ )
+ )
+ child_outputs: dict[str, Any] = {}
+ for output_param, run_output_param in child_output_params:
+ child_outputs[output_param.key] = run_output_param.value
+ output_data["outputs"] = child_outputs
+ except Exception:
+ LOG.warning(
+ "Failed to fetch child workflow outputs",
+ triggered_workflow_run_id=triggered_run_id,
+ exc_info=True,
+ )
+ except Exception as e:
+ error_msg = get_user_facing_exception_message(e)
+ output_data = {
+ "workflow_run_id": triggered_run_id,
+ "workflow_permanent_id": resolved_workflow_permanent_id,
+ "status": "failed",
+ "failure_reason": f"Triggered workflow execution failed: {error_msg}",
+ }
+ success = False
+ finally:
+ if created_fresh_session and resolved_browser_session_id:
+ try:
+ await app.PERSISTENT_SESSIONS_MANAGER.close_session(
+ organization_id, resolved_browser_session_id
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to close child browser session",
+ child_browser_session_id=resolved_browser_session_id,
+ triggered_workflow_run_id=triggered_run_id,
+ exc_info=True,
+ )
else:
# Fire and forget: dispatch the child workflow via Temporal so it
# gets its own independent worker process. This ensures the child
diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.py
index 0e181a2cc..b8a9ac3a4 100644
--- a/skyvern/forge/sdk/workflow/models/workflow.py
+++ b/skyvern/forge/sdk/workflow/models/workflow.py
@@ -285,6 +285,7 @@ class WorkflowRun(BaseModel):
job_id: str | None = None
depends_on_workflow_run_id: str | None = None
sequential_key: str | None = None
+ sequential_credential_id: str | None = None
ai_fallback: bool | None = None
code_gen: bool | None = None
trigger_type: WorkflowRunTriggerType | None = None
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index b89417db2..e0ef8141b 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -49,6 +49,7 @@ from skyvern.exceptions import (
InvalidWorkflowParameter,
MissingValueForParameter,
ScriptTerminationException,
+ SequentialCredentialLimitExceeded,
SkyvernException,
SkyvernHTTPException,
WorkflowNotFound,
@@ -77,6 +78,7 @@ from skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_fo
from skyvern.forge.sdk.experimentation.transient_ui_capture import resolve_transient_ui_capture_arm
from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.forge.sdk.schemas.browser_profiles import BrowserProfile
+from skyvern.forge.sdk.schemas.credentials import Credential
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.persistent_browser_sessions import (
@@ -93,7 +95,7 @@ from skyvern.forge.sdk.workflow.browser_profile_key import (
build_workflow_browser_session_storage_key,
render_browser_profile_key,
)
-from skyvern.forge.sdk.workflow.context_manager import jinja_sandbox_env
+from skyvern.forge.sdk.workflow.context_manager import jinja_sandbox_env, resolve_credential_parameter_binding
from skyvern.forge.sdk.workflow.credential_fallback import (
VALID_FALLBACK_TRIGGERS,
maybe_start_credential_fallback_retry,
@@ -1200,9 +1202,9 @@ class WorkflowService:
)
return value
- async def _validate_credential_ids(self, credential_ids: list[str], organization: Organization) -> None:
+ async def _validate_credential_ids(self, credential_ids: list[str], organization: Organization) -> list[Credential]:
if not credential_ids:
- return
+ return []
unique_ids = list(dict.fromkeys(credential_ids))
existing = await app.DATABASE.credentials.get_credentials_by_ids(
unique_ids, organization_id=organization.organization_id
@@ -1211,6 +1213,65 @@ class WorkflowService:
missing = [credential_id for credential_id in unique_ids if credential_id not in found]
if missing:
raise InvalidCredentialId(", ".join(missing))
+ return existing
+
+ async def _resolve_sequential_credential_id(
+ self,
+ *,
+ workflow: Workflow,
+ workflow_run: WorkflowRun,
+ organization: Organization,
+ parameter_values: dict[str, Any],
+ credential_selections: dict[str, str],
+ ) -> str | None:
+ """Resolve the run's single sequential credential from the actually-selected credential of
+ every credential-bearing parameter. Persist the one id when present; leave it NULL (no write)
+ for a credential-less run; fail closed before publication when 2+ distinct opted-in credentials
+ resolve — the MVP serializes at most one credential per run. The id is a carried identity, never
+ a setup-completion sentinel."""
+ bound_credential_ids: list[str] = []
+ runtime_only_parameter_keys = {
+ parameter.key
+ for parameter in workflow.workflow_definition.parameters
+ if isinstance(parameter, (ContextParameter, OutputParameter))
+ }
+ for parameter in workflow.workflow_definition.parameters:
+ credential_id: Any
+ if isinstance(parameter, CredentialParameter):
+ if (
+ parameter.key not in credential_selections
+ and parameter.credential_id in runtime_only_parameter_keys
+ and parameter.credential_id not in parameter_values
+ ):
+ continue
+ credential_id = resolve_credential_parameter_binding(
+ parameter,
+ parameter_values,
+ credential_selections.get(parameter.key),
+ )
+ elif (
+ isinstance(parameter, WorkflowParameter)
+ and parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID
+ ):
+ credential_id = parameter_values.get(parameter.key)
+ else:
+ continue
+ if isinstance(credential_id, str) and credential_id:
+ bound_credential_ids.append(credential_id)
+ validated_bound_credentials = await self._validate_credential_ids(bound_credential_ids, organization)
+ sequential_ids = sorted(
+ {credential.credential_id for credential in validated_bound_credentials if credential.run_sequentially}
+ )
+ if not sequential_ids:
+ return None
+ if len(sequential_ids) > 1:
+ raise SequentialCredentialLimitExceeded(sequential_ids)
+ sequential_credential_id = sequential_ids[0]
+ await app.DATABASE.workflow_runs.update_workflow_run(
+ workflow_run_id=workflow_run.workflow_run_id,
+ sequential_credential_id=sequential_credential_id,
+ )
+ return sequential_credential_id
async def _validate_and_normalize_credential_rotation_parameters(
self,
@@ -1397,6 +1458,28 @@ class WorkflowService:
return overrides
+ async def _rotation_candidates_may_require_serialization(self, workflow: Workflow, organization_id: str) -> bool:
+ """Conservatively decide whether a failed rotating-credential selection could still have
+ resolved to a credential opted into sequential execution. Any candidate flagged
+ run_sequentially — or any candidate we cannot verify — fails closed so a serialized lane is
+ never silently skipped. A pool that is provably all non-sequential preserves the legacy
+ best-effort partial selection for keyless workflows outside the serialization feature."""
+ candidate_ids: set[str] = set()
+ for parameter in self._get_credential_parameters_with_configured_selection(workflow):
+ candidate_ids.update(parameter.credential_ids or [])
+ candidate_ids.update(parameter.fallback_credential_ids or [])
+ if not candidate_ids:
+ return False
+ organization = await app.DATABASE.organizations.get_organization(organization_id)
+ if organization is None:
+ return True
+ try:
+ credentials = await self._validate_credential_ids(sorted(candidate_ids), organization)
+ except Exception:
+ # Cannot verify the candidate pool (missing/invalid id or lookup error) — fail closed.
+ return True
+ return any(credential.run_sequentially for credential in credentials)
+
async def _select_rotating_credential_parameters_for_render(
self,
*,
@@ -1447,7 +1530,13 @@ class WorkflowService:
workflow_permanent_id=workflow.workflow_permanent_id,
exc_info=True,
)
- if workflow.browser_profile_key:
+ # Scope fail-closed to runs that could actually resolve to a sequential credential: a keyed
+ # workflow (browser_profile_key), or a rotation pool with any opted-in or unverifiable
+ # candidate. A keyless workflow whose pool is provably non-sequential keeps the legacy
+ # best-effort partial selection — the pre-feature behavior for runs outside serialization.
+ if workflow.browser_profile_key or await self._rotation_candidates_may_require_serialization(
+ workflow, organization_id
+ ):
raise
return selections
@@ -1818,6 +1907,13 @@ class WorkflowService:
parameter_values=parameter_values,
)
parameter_values.update(rotating_credential_selections)
+ workflow_run.sequential_credential_id = await self._resolve_sequential_credential_id(
+ workflow=workflow,
+ workflow_run=workflow_run,
+ organization=organization,
+ parameter_values=parameter_values,
+ credential_selections=rotating_credential_selections,
+ )
workflow_run = await self._resolve_and_stamp_run_seed(
workflow=workflow,
workflow_run=workflow_run,
@@ -1826,10 +1922,6 @@ class WorkflowService:
start_fresh=resolve_start_fresh(
workflow_request.start_fresh_browser, explicit_request_browser_profile_id
),
- # Keyless workflows keep best-effort rotation selection; keyed workflows re-raise above.
- allow_missing_browser_profile_key=(
- bool(self._get_rotating_credential_parameters(workflow)) and not rotating_credential_selections
- ),
)
if workflow_parameter_values:
diff --git a/tests/unit/forge/sdk/db/test_workflow_runs_repository.py b/tests/unit/forge/sdk/db/test_workflow_runs_repository.py
index e34a9ec8b..073de1278 100644
--- a/tests/unit/forge/sdk/db/test_workflow_runs_repository.py
+++ b/tests/unit/forge/sdk/db/test_workflow_runs_repository.py
@@ -2,17 +2,26 @@
from __future__ import annotations
+import asyncio
+import re
from datetime import datetime, timezone
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
import pytest_asyncio
+from sqlalchemy import event
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncEngine
from skyvern.forge.sdk.db.agent_db import AgentDB, _build_engine
-from skyvern.forge.sdk.db.models import Base, PersistentBrowserSessionModel, TaskRunModel, WorkflowRunModel
+from skyvern.forge.sdk.db.models import (
+ Base,
+ PersistentBrowserSessionModel,
+ TaskRunModel,
+ WorkflowModel,
+ WorkflowRunModel,
+)
from skyvern.forge.sdk.db.repositories.workflow_runs import WorkflowRunsRepository
from skyvern.forge.sdk.schemas.persistent_browser_sessions import FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE
from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter, WorkflowParameterType
@@ -100,24 +109,282 @@ def _workflow_run_model(
workflow_run_id: str,
queued_at: datetime,
browser_session_id: str | None = None,
+ debug_session_id: str | None = None,
sequential_key: str | None = None,
workflow_permanent_id: str = "wpid_test",
+ workflow_id: str = "wf_test",
status: str = WorkflowRunStatus.queued.value,
+ organization_id: str = "org_test",
+ sequential_credential_id: str | None = None,
) -> WorkflowRunModel:
return WorkflowRunModel(
workflow_run_id=workflow_run_id,
- workflow_id="wf_test",
+ workflow_id=workflow_id,
workflow_permanent_id=workflow_permanent_id,
- organization_id="org_test",
+ organization_id=organization_id,
browser_session_id=browser_session_id,
+ debug_session_id=debug_session_id,
status=status,
sequential_key=sequential_key,
+ sequential_credential_id=sequential_credential_id,
created_at=queued_at,
modified_at=queued_at,
queued_at=queued_at,
)
+def _credential_run(
+ workflow_run_id: str, queued_at: datetime, credential_id: str | None, **kwargs: Any
+) -> WorkflowRunModel:
+ kwargs.setdefault("browser_session_id", f"pbs_{workflow_run_id}")
+ return _workflow_run_model(
+ workflow_run_id=workflow_run_id,
+ queued_at=queued_at,
+ sequential_credential_id=credential_id,
+ **kwargs,
+ )
+
+
+async def test_credential_gate_crosses_workflows_and_respects_scope_and_status(sqlite_db: AgentDB) -> None:
+ # The three excluded peers (other-org / completed / disjoint-credential) are stamped strictly
+ # EARLIER than the legitimate blocker, so if the org-scope, non-terminal-status, or credential
+ # exact-equality predicate regressed, that peer would sort first and become the returned blocker.
+ # The `blocker == wr_a_overlap` assertion is therefore falsifiable for each predicate, not an
+ # artifact of the strict (queued_at, workflow_run_id) total order.
+ early_at = datetime(2026, 7, 22, 10, 0, tzinfo=timezone.utc)
+ mid_at = datetime(2026, 7, 22, 11, 0, tzinfo=timezone.utc)
+ probe_at = datetime(2026, 7, 22, 12, 0, tzinfo=timezone.utc)
+ async with sqlite_db.Session() as session:
+ # wf_seq is run_sequentially, so its runs serialize whole-workflow even with a disjoint cred.
+ session.add(
+ WorkflowModel(
+ workflow_id="wf_seq",
+ workflow_permanent_id="wpid_ws",
+ title="seq",
+ workflow_definition={},
+ run_sequentially=True,
+ )
+ )
+ session.add_all(
+ [
+ # Excluded peers, each strictly earlier than wr_a_overlap so a broken predicate surfaces here.
+ _credential_run(
+ "wr_other_org",
+ early_at,
+ "cred_a",
+ workflow_permanent_id="wpid_other",
+ organization_id="org_other",
+ ),
+ _credential_run(
+ "wr_completed",
+ early_at,
+ "cred_a",
+ workflow_permanent_id="wpid_other",
+ status=WorkflowRunStatus.completed.value,
+ ),
+ _credential_run("wr_disjoint", early_at, "cred_b", workflow_permanent_id="wpid_other"),
+ # The one legitimate cross-workflow same-credential blocker.
+ _credential_run("wr_a_overlap", mid_at, "cred_a", workflow_permanent_id="wpid_other"),
+ _credential_run("wr_b_self", probe_at, "cred_a"),
+ # Disjoint-credential peers that must still serialize via a shared sequential_key or a
+ # run_sequentially workflow: the credential lane composes with the legacy lanes.
+ _credential_run("wr_key_prior", mid_at, "cred_x", workflow_permanent_id="wpid_key", sequential_key="K"),
+ _workflow_run_model(
+ workflow_run_id="wr_key_plain_session",
+ queued_at=early_at,
+ workflow_permanent_id="wpid_key",
+ sequential_key="K",
+ browser_session_id="pbs_key_plain",
+ ),
+ _credential_run(
+ "wr_key_self", probe_at, "cred_y", workflow_permanent_id="wpid_key", sequential_key="K"
+ ),
+ # A plain browser-session occupant belongs only to its session lane and must not
+ # displace the real whole-workflow predecessor for a credential-composed probe.
+ _workflow_run_model(
+ workflow_run_id="wr_ws_plain_session",
+ queued_at=early_at,
+ workflow_permanent_id="wpid_ws",
+ workflow_id="wf_seq",
+ browser_session_id="pbs_plain",
+ ),
+ _credential_run("wr_ws_prior", mid_at, "cred_p", workflow_permanent_id="wpid_ws", workflow_id="wf_seq"),
+ _credential_run(
+ "wr_ws_self", probe_at, "cred_q", workflow_permanent_id="wpid_ws", workflow_id="wf_seq"
+ ),
+ ]
+ )
+ await session.commit()
+
+ # Same credential + same org + different workflow blocks; disjoint / other-org / completed do not,
+ # even though all three are strictly earlier than both the probe and the legitimate blocker.
+ blocker = await sqlite_db.workflow_runs.get_blocking_sequential_workflow_run("wr_b_self")
+ assert blocker is not None
+ assert blocker.workflow_run_id == "wr_a_overlap"
+ # A disjoint credential still blocks via a shared manual key ...
+ key_blocker = await sqlite_db.workflow_runs.get_blocking_sequential_workflow_run("wr_key_self")
+ assert key_blocker is not None and key_blocker.workflow_run_id == "wr_key_prior"
+ # ... and via a run_sequentially workflow.
+ ws_blocker = await sqlite_db.workflow_runs.get_blocking_sequential_workflow_run("wr_ws_self")
+ assert ws_blocker is not None and ws_blocker.workflow_run_id == "wr_ws_prior"
+
+ async with sqlite_db.Session() as session:
+ overlap = await session.get(WorkflowRunModel, "wr_a_overlap")
+ assert overlap is not None
+ overlap.status = WorkflowRunStatus.canceled.value
+ await session.commit()
+
+ # With the only legitimate blocker canceled, the strictly-earlier excluded peers still do not block.
+ assert await sqlite_db.workflow_runs.get_blocking_sequential_workflow_run("wr_b_self") is None
+
+
+ "peer_credential, peer_kwargs, guard",
+ [
+ ("cred_other", {"workflow_permanent_id": "wpid_other"}, "credential exact-equality"),
+ ("cred_self", {"organization_id": "org_other"}, "org scoping"),
+ ("cred_self", {"status": WorkflowRunStatus.completed.value}, "non-terminal status"),
+ ],
+)
+async def test_credential_gate_excludes_earlier_non_matching_peer(
+ sqlite_db: AgentDB, peer_credential: str, peer_kwargs: dict[str, Any], guard: str
+) -> None:
+ # A single earlier peer that shares the credential lane in every dimension but the one under test.
+ # Because it is strictly earlier than the probe, a regression in the org-scope, non-terminal-status,
+ # or credential exact-equality predicate would return it as the blocker — so `is None` here is a
+ # falsifiable proof that the guarding predicate fires, not a total-ordering artifact.
+ earlier = datetime(2026, 7, 22, 10, 0, tzinfo=timezone.utc)
+ later = datetime(2026, 7, 22, 12, 0, tzinfo=timezone.utc)
+ async with sqlite_db.Session() as session:
+ session.add(
+ _credential_run("wr_peer_earlier", earlier, peer_credential, browser_session_id=None, **peer_kwargs)
+ )
+ session.add(_credential_run("wr_probe_later", later, "cred_self", browser_session_id=None))
+ await session.commit()
+
+ assert await sqlite_db.workflow_runs.get_blocking_sequential_workflow_run("wr_probe_later") is None, guard
+
+
+ "debug_session_id, expected_blocker",
+ [
+ # A debug run is excluded from browser-session serialization at enqueue
+ # (is_browser_session_workflow is False when debug_session_id is set), so the credential branch
+ # must not re-add the browser-session lane for it — it gates only on its credential.
+ ("dbg_1", None),
+ # A non-debug credential run sharing the session still composes the browser-session lane.
+ (None, "wr_session_peer"),
+ ],
+)
+async def test_credential_gate_browser_session_lane_honors_debug_exclusion(
+ sqlite_db: AgentDB, debug_session_id: str | None, expected_blocker: str | None
+) -> None:
+ earlier = datetime(2026, 7, 22, 10, 0, tzinfo=timezone.utc)
+ later = datetime(2026, 7, 22, 12, 0, tzinfo=timezone.utc)
+ async with sqlite_db.Session() as session:
+ session.add(
+ _credential_run(
+ "wr_session_peer",
+ earlier,
+ "cred_other",
+ browser_session_id="pbs_shared",
+ workflow_permanent_id="wpid_other",
+ )
+ )
+ session.add(
+ _credential_run(
+ "wr_probe",
+ later,
+ "cred_self",
+ browser_session_id="pbs_shared",
+ debug_session_id=debug_session_id,
+ workflow_permanent_id="wpid_debug",
+ )
+ )
+ await session.commit()
+
+ blocker = await sqlite_db.workflow_runs.get_blocking_sequential_workflow_run("wr_probe")
+ assert (blocker.workflow_run_id if blocker else None) == expected_blocker
+
+
+async def test_get_workflow_run_preserves_sequential_credential_id(sqlite_db: AgentDB) -> None:
+ queued_at = datetime(2026, 7, 22, 12, 0, tzinfo=timezone.utc)
+ async with sqlite_db.Session() as session:
+ session.add(
+ _workflow_run_model(
+ workflow_run_id="wr_seq_snapshot",
+ queued_at=queued_at,
+ sequential_credential_id="cred_a",
+ )
+ )
+ await session.commit()
+
+ # The enqueue executor and scheduled-setup activity read this identity off the
+ # converted schema to arm credential serialization; it must survive conversion.
+ fetched = await sqlite_db.workflow_runs.get_workflow_run("wr_seq_snapshot", organization_id="org_test")
+ assert fetched is not None
+ assert fetched.sequential_credential_id == "cred_a"
+
+
+async def test_composed_credential_queued_at_advances_past_cross_lane_publication_despite_clock_skew(
+ sqlite_db: AgentDB, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ prior_ticket = datetime(2099, 1, 1, tzinfo=timezone.utc)
+ skewed_host_clock = datetime(2000, 1, 1, tzinfo=timezone.utc)
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.db.repositories.workflow_runs.naive_utc_now",
+ lambda: skewed_host_clock.replace(tzinfo=None),
+ )
+ async with sqlite_db.Session() as session:
+ session.add_all(
+ [
+ _workflow_run_model(
+ workflow_run_id="wr_first_publisher",
+ status=WorkflowRunStatus.created.value,
+ queued_at=None,
+ sequential_key="shared_lane",
+ ),
+ _workflow_run_model(
+ workflow_run_id="wr_second_publisher",
+ status=WorkflowRunStatus.created.value,
+ queued_at=None,
+ sequential_credential_id="cred_skew",
+ ),
+ ]
+ )
+ await session.commit()
+
+ publication_lock = asyncio.Lock()
+ first_published = asyncio.Event()
+
+ async def publish_first() -> None:
+ async with publication_lock:
+ await sqlite_db.workflow_runs.update_workflow_run(
+ "wr_first_publisher",
+ status=WorkflowRunStatus.queued,
+ queued_at=prior_ticket,
+ )
+ first_published.set()
+
+ async def publish_second() -> Any:
+ await first_published.wait()
+ async with publication_lock:
+ return await sqlite_db.workflow_runs.update_workflow_run(
+ "wr_second_publisher",
+ status=WorkflowRunStatus.queued,
+ )
+
+ _, second = await asyncio.gather(publish_first(), publish_second())
+
+ assert second.queued_at is not None
+ assert second.queued_at > prior_ticket.replace(tzinfo=None)
+
+
def _task_run_model(
*,
run_id: str,
@@ -984,6 +1251,7 @@ async def test_get_blocking_sequential_workflow_run_scans_earlier_active_same_ke
fake_run.workflow_run_id = "wr_self"
fake_run.organization_id = "o_test"
fake_run.workflow_permanent_id = "wpid_test"
+ fake_run.sequential_credential_id = None
fake_run.sequential_key = "cred_test-sequential-key"
fake_run.browser_session_id = None
fake_run.browser_address = None
@@ -1036,6 +1304,7 @@ async def test_get_blocking_sequential_workflow_run_prefers_browser_session_lane
fake_run.workflow_run_id = "wr_self"
fake_run.organization_id = "o_test"
fake_run.workflow_permanent_id = "wpid_test"
+ fake_run.sequential_credential_id = None
fake_run.sequential_key = "cred_test-sequential-key"
fake_run.browser_session_id = "pbs_test"
fake_run.debug_session_id = None
@@ -1077,6 +1346,7 @@ async def test_get_blocking_sequential_workflow_run_debug_session_uses_key_lane(
fake_run.workflow_run_id = "wr_self"
fake_run.organization_id = "o_test"
fake_run.workflow_permanent_id = "wpid_test"
+ fake_run.sequential_credential_id = None
fake_run.sequential_key = "cred_test-sequential-key"
fake_run.browser_session_id = "pbs_test"
fake_run.debug_session_id = "dbg_test"
@@ -1101,3 +1371,300 @@ async def test_get_blocking_sequential_workflow_run_debug_session_uses_key_lane(
assert "workflow_runs.sequential_key = 'cred_test-sequential-key'" in where_clause
assert "workflow_runs.browser_session_id IS NULL" in where_clause
assert "workflow_runs.browser_session_id = 'pbs_test'" not in where_clause
+
+
+async def test_get_last_queued_workflow_run_admits_credential_composed_session_row(sqlite_db: AgentDB) -> None:
+ # A credential-composed predecessor carries a browser_session_id but is a K-lane occupant by the
+ # PR's gate contract, so the legacy K lookup must admit it. A plain (non-credential) session+K row
+ # stays excluded even though it is later-modified — legacy session-lane priority is preserved.
+ early = datetime(2026, 7, 22, 10, 0, tzinfo=timezone.utc)
+ late = datetime(2026, 7, 22, 11, 0, tzinfo=timezone.utc)
+ async with sqlite_db.Session() as session:
+ session.add_all(
+ [
+ _workflow_run_model(
+ workflow_run_id="wr_cred_session",
+ browser_session_id="pbs_cred",
+ sequential_key="K",
+ sequential_credential_id="cred_a",
+ queued_at=early,
+ ),
+ _workflow_run_model(
+ workflow_run_id="wr_plain_session",
+ browser_session_id="pbs_plain",
+ sequential_key="K",
+ queued_at=late,
+ ),
+ ]
+ )
+ await session.commit()
+
+ result = await sqlite_db.workflow_runs.get_last_queued_workflow_run("wpid_test", "org_test", "K")
+
+ assert result is not None
+ assert result.workflow_run_id == "wr_cred_session"
+
+
+async def test_composed_lane_predecessor_queries_limit_each_index_candidate() -> None:
+ captured: list[Any] = []
+
+ async def _scalars(query: Any) -> Any:
+ captured.append(query)
+ return _Result(None)
+
+ session = MagicMock()
+ session.scalars = AsyncMock(side_effect=_scalars)
+ repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)
+
+ await repo.get_last_queued_workflow_run("wpid_test", "o_test", "K")
+ await repo.get_last_running_workflow_run("wpid_test", "o_test", "K")
+
+ assert len(captured) == 4
+ for query in captured:
+ rendered = str(query.compile(compile_kwargs={"literal_binds": True}))
+ assert "LIMIT 1" in rendered
+
+
+async def test_get_last_queued_workflow_run_can_keep_legacy_batch_predecessor(sqlite_db: AgentDB) -> None:
+ early = datetime(2026, 7, 22, 10, 0, tzinfo=timezone.utc)
+ late = datetime(2026, 7, 22, 11, 0, tzinfo=timezone.utc)
+ async with sqlite_db.Session() as session:
+ legacy = _workflow_run_model(
+ workflow_run_id="wr_batch_legacy",
+ sequential_key="K",
+ queued_at=early,
+ )
+ credential = _workflow_run_model(
+ workflow_run_id="wr_credential_newer",
+ browser_session_id="pbs_cred",
+ sequential_key="K",
+ sequential_credential_id="cred_a",
+ queued_at=late,
+ )
+ legacy.modified_at = early
+ credential.modified_at = late
+ session.add_all([legacy, credential])
+ await session.commit()
+
+ cross_lane = await sqlite_db.workflow_runs.get_last_queued_workflow_run("wpid_test", "org_test", "K")
+ legacy_only = await sqlite_db.workflow_runs.get_last_queued_workflow_run(
+ "wpid_test",
+ "org_test",
+ "K",
+ include_credential_composed_rows=False,
+ )
+
+ assert cross_lane is not None
+ assert cross_lane.workflow_run_id == "wr_credential_newer"
+ assert legacy_only is not None
+ assert legacy_only.workflow_run_id == "wr_batch_legacy"
+
+
+async def test_get_last_running_workflow_run_admits_credential_composed_session_row(sqlite_db: AgentDB) -> None:
+ early = datetime(2026, 7, 22, 10, 0, tzinfo=timezone.utc)
+ late = datetime(2026, 7, 22, 11, 0, tzinfo=timezone.utc)
+ async with sqlite_db.Session() as session:
+ cred = _workflow_run_model(
+ workflow_run_id="wr_cred_session",
+ browser_session_id="pbs_cred",
+ sequential_key="K",
+ sequential_credential_id="cred_a",
+ status=WorkflowRunStatus.running.value,
+ queued_at=early,
+ )
+ cred.started_at = early
+ plain = _workflow_run_model(
+ workflow_run_id="wr_plain_session",
+ browser_session_id="pbs_plain",
+ sequential_key="K",
+ status=WorkflowRunStatus.running.value,
+ queued_at=late,
+ )
+ plain.started_at = late
+ session.add_all([cred, plain])
+ await session.commit()
+
+ result = await sqlite_db.workflow_runs.get_last_running_workflow_run("wpid_test", "org_test", "K")
+
+ assert result is not None
+ assert result.workflow_run_id == "wr_cred_session"
+
+
[email protected]("pred_status", [WorkflowRunStatus.queued.value, WorkflowRunStatus.running.value])
+async def test_gate_noncredential_key_blocks_on_credential_composed_session_predecessor(
+ sqlite_db: AgentDB, pred_status: str
+) -> None:
+ # A later non-credential K run must see and wait on an earlier credential+session+K occupant,
+ # for both queued and running predecessors — the gate's non-credential K branch admits it.
+ early = datetime(2026, 7, 22, 10, 0, tzinfo=timezone.utc)
+ late = datetime(2026, 7, 22, 11, 0, tzinfo=timezone.utc)
+ async with sqlite_db.Session() as session:
+ session.add_all(
+ [
+ _workflow_run_model(
+ workflow_run_id="wr_pred",
+ browser_session_id="pbs_pred",
+ sequential_key="K",
+ sequential_credential_id="cred_a",
+ status=pred_status,
+ queued_at=early,
+ ),
+ _workflow_run_model(
+ workflow_run_id="wr_self",
+ sequential_key="K",
+ queued_at=late,
+ ),
+ ]
+ )
+ await session.commit()
+
+ blocker = await sqlite_db.workflow_runs.get_blocking_sequential_workflow_run("wr_self")
+
+ assert blocker is not None
+ assert blocker.workflow_run_id == "wr_pred"
+
+
+async def test_gate_noncredential_whole_workflow_blocks_on_credential_composed_session_predecessor(
+ sqlite_db: AgentDB,
+) -> None:
+ # A later non-credential whole-workflow run must see and wait on an earlier credential+session
+ # occupant of the same wpid — the gate's non-credential whole-workflow branch admits it.
+ early = datetime(2026, 7, 22, 10, 0, tzinfo=timezone.utc)
+ late = datetime(2026, 7, 22, 11, 0, tzinfo=timezone.utc)
+ async with sqlite_db.Session() as session:
+ session.add_all(
+ [
+ _workflow_run_model(
+ workflow_run_id="wr_pred",
+ browser_session_id="pbs_pred",
+ sequential_credential_id="cred_a",
+ queued_at=early,
+ ),
+ _workflow_run_model(
+ workflow_run_id="wr_self",
+ queued_at=late,
+ ),
+ ]
+ )
+ await session.commit()
+
+ blocker = await sqlite_db.workflow_runs.get_blocking_sequential_workflow_run("wr_self")
+
+ assert blocker is not None
+ assert blocker.workflow_run_id == "wr_pred"
+
+
+async def test_gate_noncredential_key_ignores_plain_session_predecessor(sqlite_db: AgentDB) -> None:
+ # Legacy-priority regression pin (stays green before and after): a plain non-credential session+K
+ # row must remain invisible to a later non-credential K run — the OR admits only credential rows.
+ early = datetime(2026, 7, 22, 10, 0, tzinfo=timezone.utc)
+ late = datetime(2026, 7, 22, 11, 0, tzinfo=timezone.utc)
+ async with sqlite_db.Session() as session:
+ session.add_all(
+ [
+ _workflow_run_model(
+ workflow_run_id="wr_plain_session",
+ browser_session_id="pbs_plain",
+ sequential_key="K",
+ queued_at=early,
+ ),
+ _workflow_run_model(
+ workflow_run_id="wr_self",
+ sequential_key="K",
+ queued_at=late,
+ ),
+ ]
+ )
+ await session.commit()
+
+ blocker = await sqlite_db.workflow_runs.get_blocking_sequential_workflow_run("wr_self")
+
+ assert blocker is None
+
+
+def _capture_sql(engine: AsyncEngine) -> tuple[list[str], Any]:
+ """Collect every SQL statement emitted on the engine so a test can assert query shape."""
+ statements: list[str] = []
+
+ def _on_execute(conn: Any, cursor: Any, statement: str, *args: Any) -> None: # noqa: ANN401
+ statements.append(" ".join(statement.split()))
+
+ event.listen(engine.sync_engine, "before_cursor_execute", _on_execute)
+ return statements, _on_execute
+
+
+# The de-indexing OR the legacy partial index (ix_workflow_runs_sequential_key_lookup, partial on
+# `browser_session_id IS NULL`) can no longer serve. A regex, not a substring, so column-order or
+# alias changes in the compiled SQL cannot silently let the OR slip back in.
+_DEINDEXING_OR = re.compile(
+ r"browser_session_id IS NULL\s+OR\s+\S*sequential_credential_id IS NOT NULL",
+ re.IGNORECASE,
+)
+
+
+async def test_get_last_queued_workflow_run_keeps_legacy_lane_index_eligible(
+ sqlite_db: AgentDB, sqlite_engine: AsyncEngine
+) -> None:
+ # The non-session lookup must scan the legacy lane with a standalone `browser_session_id IS
+ # NULL` (index-eligible via ix_workflow_runs_sequential_key_lookup), never ORed with
+ # `sequential_credential_id IS NOT NULL` — that OR de-indexes the high-write lookup.
+ statements, listener = _capture_sql(sqlite_engine)
+ try:
+ await sqlite_db.workflow_runs.get_last_queued_workflow_run("wpid_test", "org_test", "K")
+ finally:
+ event.remove(sqlite_engine.sync_engine, "before_cursor_execute", listener)
+
+ lookups = [s for s in statements if "FROM workflow_runs" in s and "browser_session_id IS NULL" in s]
+ assert lookups, "expected a legacy-lane lookup that filters browser_session_id IS NULL"
+ for statement in statements:
+ assert not _DEINDEXING_OR.search(statement), f"legacy lane de-indexed by widened OR: {statement}"
+
+
+async def test_get_last_running_workflow_run_keeps_legacy_lane_index_eligible(
+ sqlite_db: AgentDB, sqlite_engine: AsyncEngine
+) -> None:
+ statements, listener = _capture_sql(sqlite_engine)
+ try:
+ await sqlite_db.workflow_runs.get_last_running_workflow_run("wpid_test", "org_test", "K")
+ finally:
+ event.remove(sqlite_engine.sync_engine, "before_cursor_execute", listener)
+
+ lookups = [s for s in statements if "FROM workflow_runs" in s and "browser_session_id IS NULL" in s]
+ assert lookups, "expected a legacy-lane lookup that filters browser_session_id IS NULL"
+ for statement in statements:
+ assert not _DEINDEXING_OR.search(statement), f"legacy lane de-indexed by widened OR: {statement}"
+
+
+async def test_gate_noncredential_key_lane_keeps_index_eligible_shape(
+ sqlite_db: AgentDB, sqlite_engine: AsyncEngine
+) -> None:
+ # The gate's non-credential sequential_key branch admits credential-composed predecessors, but must
+ # do so without ORing the credential predicate into the keyed lane scan (which de-indexes it).
+ async with sqlite_db.Session() as session:
+ session.add(
+ _workflow_run_model(
+ workflow_run_id="wr_self",
+ sequential_key="K",
+ queued_at=datetime(2026, 7, 22, 11, 0, tzinfo=timezone.utc),
+ )
+ )
+ await session.commit()
+
+ statements, listener = _capture_sql(sqlite_engine)
+ try:
+ await sqlite_db.workflow_runs.get_blocking_sequential_workflow_run("wr_self")
+ finally:
+ event.remove(sqlite_engine.sync_engine, "before_cursor_execute", listener)
+
+ for statement in statements:
+ assert not _DEINDEXING_OR.search(statement), f"gate keyed lane de-indexed by widened OR: {statement}"
diff --git a/tests/unit/test_background_task_executor.py b/tests/unit/test_background_task_executor.py
index 443d04f0b..683ad2b38 100644
--- a/tests/unit/test_background_task_executor.py
+++ b/tests/unit/test_background_task_executor.py
@@ -1,11 +1,16 @@
import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
import pytest
from fastapi import BackgroundTasks
+from skyvern.exceptions import SkyvernException
+from skyvern.forge import app
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.executor.background_task_executor import BackgroundTaskExecutor
+from skyvern.forge.sdk.schemas.persistent_browser_sessions import FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE
@pytest.mark.asyncio
@@ -85,3 +90,127 @@ async def test_scheduled_task_is_retained_until_done() -> None:
await asyncio.sleep(0)
await asyncio.sleep(0)
assert executor._background_tasks == set()
+
+
+async def test_execute_workflow_fails_closed_for_stamped_sequential_credential(monkeypatch: pytest.MonkeyPatch) -> None:
+ workflow_run = SimpleNamespace(sequential_credential_id="cred_sequential", browser_session_id=None)
+ monkeypatch.setattr(
+ app.DATABASE.workflow_runs,
+ "get_workflow_run",
+ AsyncMock(return_value=workflow_run),
+ )
+ mark_failed = AsyncMock()
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", mark_failed)
+ executor = BackgroundTaskExecutor()
+ executor._schedule = MagicMock() # type: ignore[method-assign]
+
+ with pytest.raises(SkyvernException, match="background executor"):
+ await executor.execute_workflow(
+ request=None,
+ background_tasks=None,
+ organization=SimpleNamespace(organization_id="org_test"),
+ workflow_id="wf_test",
+ workflow_run_id="wr_test",
+ workflow_permanent_id="wpid_test",
+ max_steps_override=None,
+ api_key=None,
+ browser_session_id=None,
+ block_labels=None,
+ block_outputs=None,
+ )
+
+ executor._schedule.assert_not_called()
+ mark_failed.assert_awaited_once()
+ assert mark_failed.await_args.kwargs["workflow_run_id"] == "wr_test"
+ assert mark_failed.await_args.kwargs["cascade_children"] is True
+
+
+async def test_execute_workflow_closes_forced_session_before_credential_fence(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ workflow_run = SimpleNamespace(
+ sequential_credential_id="cred_sequential",
+ browser_session_id="pbs_forced",
+ )
+ monkeypatch.setattr(
+ app.DATABASE.workflow_runs,
+ "get_workflow_run",
+ AsyncMock(return_value=workflow_run),
+ )
+ monkeypatch.setattr(
+ app.DATABASE.browser_sessions,
+ "get_persistent_browser_session",
+ AsyncMock(return_value=SimpleNamespace(runnable_type=FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE)),
+ )
+ close_session = AsyncMock()
+ monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "close_session", close_session)
+ monkeypatch.setattr(
+ app.WORKFLOW_SERVICE,
+ "mark_workflow_run_as_failed_if_not_final",
+ AsyncMock(),
+ )
+ executor = BackgroundTaskExecutor()
+ executor._schedule = MagicMock() # type: ignore[method-assign]
+
+ with pytest.raises(SkyvernException, match="background executor"):
+ await executor.execute_workflow(
+ request=None,
+ background_tasks=None,
+ organization=SimpleNamespace(organization_id="org_test"),
+ workflow_id="wf_test",
+ workflow_run_id="wr_test",
+ workflow_permanent_id="wpid_test",
+ max_steps_override=None,
+ api_key=None,
+ browser_session_id="pbs_forced",
+ block_labels=None,
+ block_outputs=None,
+ )
+
+ close_session.assert_awaited_once_with("org_test", "pbs_forced")
+ executor._schedule.assert_not_called()
+
+
+async def test_execute_workflow_terminalizes_when_forced_session_cleanup_fails(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ workflow_run = SimpleNamespace(
+ sequential_credential_id="cred_sequential",
+ browser_session_id="pbs_forced",
+ )
+ monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run", AsyncMock(return_value=workflow_run))
+ monkeypatch.setattr(
+ app.DATABASE.browser_sessions,
+ "get_persistent_browser_session",
+ AsyncMock(return_value=SimpleNamespace(runnable_type=FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE)),
+ )
+ monkeypatch.setattr(
+ app.PERSISTENT_SESSIONS_MANAGER,
+ "close_session",
+ AsyncMock(side_effect=RuntimeError("cleanup failed")),
+ )
+ mark_failed = AsyncMock()
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", mark_failed)
+ executor = BackgroundTaskExecutor()
+ executor._schedule = MagicMock() # type: ignore[method-assign]
+
+ with pytest.raises(SkyvernException, match="background executor"):
+ await executor.execute_workflow(
+ request=None,
+ background_tasks=None,
+ organization=SimpleNamespace(organization_id="org_test"),
+ workflow_id="wf_test",
+ workflow_run_id="wr_test",
+ workflow_permanent_id="wpid_test",
+ max_steps_override=None,
+ api_key=None,
+ browser_session_id="pbs_forced",
+ block_labels=None,
+ block_outputs=None,
+ )
+
+ mark_failed.assert_awaited_once()
+ executor._schedule.assert_not_called()
diff --git a/tests/unit/test_credential_rotation.py b/tests/unit/test_credential_rotation.py
index 81bda9537..6e4eb5033 100644
--- a/tests/unit/test_credential_rotation.py
+++ b/tests/unit/test_credential_rotation.py
@@ -10,7 +10,12 @@ from sqlalchemy import func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncEngine
-from skyvern.exceptions import InvalidCredentialId, SkyvernHTTPException
+from skyvern.exceptions import (
+ InvalidCredentialId,
+ RuntimeSequentialCredentialUnsupported,
+ SequentialCredentialLimitExceeded,
+ SkyvernHTTPException,
+)
from skyvern.forge.sdk.copilot.output_policy import OutputPolicyReason, evaluate_output_policy
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.db.agent_db import AgentDB, _build_engine
@@ -19,8 +24,16 @@ from skyvern.forge.sdk.db.repositories.workflow_run_credential_selections import
WorkflowRunCredentialSelectionsRepository,
)
from skyvern.forge.sdk.workflow.browser_profile_key import build_browser_profile_key_digest
+from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.credential_selection import select_credential_for_run
-from skyvern.forge.sdk.workflow.models.parameter import CredentialParameter
+from skyvern.forge.sdk.workflow.models.parameter import (
+ ContextParameter,
+ CredentialParameter,
+ OutputParameter,
+ Parameter,
+ WorkflowParameter,
+ WorkflowParameterType,
+)
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRequestBody
from skyvern.forge.sdk.workflow.service import WorkflowService
from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definition
@@ -47,6 +60,21 @@ def _credential_parameter(
)
+def _workflow_parameter(
+ key: str,
+ workflow_parameter_type: WorkflowParameterType = WorkflowParameterType.STRING,
+) -> WorkflowParameter:
+ now = datetime.now(timezone.utc)
+ return WorkflowParameter(
+ key=key,
+ workflow_parameter_id=f"wp_{key}",
+ workflow_id="wf_test",
+ workflow_parameter_type=workflow_parameter_type,
+ created_at=now,
+ modified_at=now,
+ )
+
+
class _SelectionRepo:
def __init__(
self,
@@ -747,6 +775,11 @@ async def _attempt_setup_rotation_profile_run(
)
)
mock_app.DATABASE.workflow_runs.update_workflow_run = AsyncMock(return_value=updated_run)
+ mock_app.DATABASE.organizations.get_organization = AsyncMock(return_value=organization)
+ selected_credential_id = select_side_effect if isinstance(select_side_effect, str) else "cred_a"
+ mock_app.DATABASE.credentials.get_credentials_by_ids = AsyncMock(
+ return_value=[SimpleNamespace(credential_id=selected_credential_id, run_sequentially=True)]
+ )
result = None
caught = None
@@ -763,6 +796,270 @@ async def _attempt_setup_rotation_profile_run(
return result, mock_app, service, caught
+async def _setup_bound_credentials(
+ credential_parameters: list[Parameter],
+ *,
+ workflow_parameters: list[WorkflowParameter] | None = None,
+ request_data: dict[str, str] | None = None,
+ credentials: dict[str, bool],
+ selection_repo: _SelectionRepo | None = None,
+) -> SimpleNamespace:
+ service = WorkflowService()
+ workflow = _setup_workflow_with_rotating_credential(browser_profile_key=None)
+ workflow.persist_browser_session = False
+ workflow.workflow_definition.parameters = credential_parameters
+ workflow_run = _setup_workflow_run()
+ workflow_run.sequential_credential_id = None
+ organization = SimpleNamespace(organization_id="org_test", organization_name="Test Org")
+ repo = selection_repo or _SelectionRepo()
+
+ service.get_workflow_by_permanent_id = AsyncMock(return_value=workflow) # type: ignore[method-assign]
+ service.create_workflow_run = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]
+ service.get_workflow_parameters = AsyncMock(return_value=workflow_parameters or []) # type: ignore[method-assign]
+ service.create_workflow_run_parameters = AsyncMock(return_value=[]) # type: ignore[method-assign]
+ service.mark_workflow_run_as_failed = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]
+ # These tests assert credential identity, not browser-seed resolution; stub the seed step so they
+ # do not depend on the unrelated browser-memory machinery reached via _resolve_and_stamp_run_seed.
+ service._resolve_and_stamp_run_seed = AsyncMock( # type: ignore[method-assign]
+ side_effect=lambda **kwargs: kwargs["workflow_run"]
+ )
+
+ async def update_workflow_run(*, workflow_run_id: str, **values: object) -> SimpleNamespace:
+ assert workflow_run_id == workflow_run.workflow_run_id
+ for key, value in values.items():
+ setattr(workflow_run, key, value)
+ return workflow_run
+
+ async def get_credentials_by_ids(credential_ids: list[str], *, organization_id: str) -> list[SimpleNamespace]:
+ assert organization_id == "org_test"
+ return [
+ SimpleNamespace(credential_id=credential_id, run_sequentially=credentials[credential_id])
+ for credential_id in credential_ids
+ if credential_id in credentials
+ ]
+
+ with (
+ patch("skyvern.forge.sdk.workflow.service.app") as mock_app,
+ patch("skyvern.forge.sdk.workflow.credential_selection.app") as selection_app,
+ ):
+ mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
+ mock_app.AGENT_FUNCTION.should_use_flex_llm_routing = AsyncMock(return_value=False)
+ mock_app.DATABASE.workflows.get_browser_action_policy = AsyncMock(return_value=None)
+ mock_app.DATABASE.workflow_runs.update_workflow_run = AsyncMock(side_effect=update_workflow_run)
+ mock_app.DATABASE.credentials.get_credentials_by_ids = AsyncMock(side_effect=get_credentials_by_ids)
+ selection_app.DATABASE.workflow_run_credential_selections = repo
+ return await service.setup_workflow_run(
+ request_id="req_test",
+ workflow_request=WorkflowRequestBody(data=request_data or {}),
+ workflow_permanent_id="wpid_test",
+ organization=organization,
+ )
+
+
+def _runtime_context() -> WorkflowRunContext:
+ return WorkflowRunContext(
+ workflow_title="Workflow",
+ workflow_id="wf_test",
+ workflow_permanent_id="wpid_test",
+ workflow_run_id="wr_test",
+ aws_client=MagicMock(),
+ )
+
+
+async def test_direct_bindings_snapshot_only_opted_in_credentials_and_runtime_resolves_same_ids() -> None:
+ sequential = _credential_parameter(key="serial_login", credential_id="cred_serial")
+ parallel = _credential_parameter(key="parallel_login", credential_id="cred_parallel")
+
+ workflow_run = await _setup_bound_credentials(
+ [sequential, parallel],
+ credentials={"cred_serial": True, "cred_parallel": False},
+ )
+ sequential_context = _runtime_context()
+ sequential_context.values["cred_serial"] = "cred_unrelated_runtime_value"
+ parallel_context = _runtime_context()
+
+ assert workflow_run.sequential_credential_id == "cred_serial"
+ assert await sequential_context.resolve_credential_parameter_id(sequential, "org_test") == "cred_serial"
+ assert await parallel_context.resolve_credential_parameter_id(parallel, "org_test") == "cred_parallel"
+
+
+ "workflow_parameter_type",
+ [WorkflowParameterType.STRING, WorkflowParameterType.CREDENTIAL_ID],
+)
+async def test_plain_indirect_binding_snapshots_and_runtime_resolves_bound_parameter_value(
+ workflow_parameter_type: WorkflowParameterType,
+) -> None:
+ account_parameter = _workflow_parameter("account_param", workflow_parameter_type)
+ login = _credential_parameter(credential_id=account_parameter.key)
+
+ workflow_run = await _setup_bound_credentials(
+ [login],
+ workflow_parameters=[account_parameter],
+ request_data={account_parameter.key: "cred_runtime"},
+ credentials={"cred_runtime": True},
+ )
+ context = _runtime_context()
+ context.parameters[account_parameter.key] = account_parameter
+ if workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID:
+ context.values[account_parameter.key] = {"context": "credential placeholders"}
+ context.resolved_credential_parameter_ids[account_parameter.key] = "cred_runtime"
+ else:
+ context.values[account_parameter.key] = "cred_runtime"
+
+ assert workflow_run.sequential_credential_id == "cred_runtime"
+ assert await context.resolve_credential_parameter_id(login, "org_test") == "cred_runtime"
+ assert context.get_resolved_credential_parameter_id(login.key) == "cred_runtime"
+
+
+async def test_plain_indirect_binding_rejects_unknown_bound_credential_not_literal_key() -> None:
+ account_parameter = _workflow_parameter("account_param")
+ login = _credential_parameter(credential_id=account_parameter.key)
+
+ with pytest.raises(InvalidCredentialId, match="cred_missing"):
+ await _setup_bound_credentials(
+ [login],
+ workflow_parameters=[account_parameter],
+ request_data={account_parameter.key: "cred_missing"},
+ credentials={},
+ )
+
+
[email protected]("runtime_parameter_type", ["context", "output"])
+async def test_runtime_only_indirect_binding_defers_credential_validation_until_execution(
+ runtime_parameter_type: str,
+) -> None:
+ source = _workflow_parameter("source")
+ now = datetime.now(timezone.utc)
+ runtime_parameter = (
+ ContextParameter(key="runtime_credential", source=source)
+ if runtime_parameter_type == "context"
+ else OutputParameter(
+ key="runtime_credential",
+ output_parameter_id="op_runtime_credential",
+ workflow_id="wf_test",
+ created_at=now,
+ modified_at=now,
+ )
+ )
+ login = _credential_parameter(credential_id=runtime_parameter.key)
+
+ workflow_run = await _setup_bound_credentials(
+ [login, runtime_parameter],
+ credentials={},
+ )
+
+ assert workflow_run.sequential_credential_id is None
+
+
+async def test_runtime_only_indirect_binding_fails_closed_if_resolved_credential_is_sequential() -> None:
+ context = _runtime_context()
+ source = _workflow_parameter("source")
+ runtime_parameter = ContextParameter(key="runtime_credential", source=source)
+ login = _credential_parameter(credential_id=runtime_parameter.key)
+ context.parameters[runtime_parameter.key] = runtime_parameter
+ context.values[runtime_parameter.key] = "cred_runtime"
+ organization = SimpleNamespace(organization_id="org_test")
+
+ with patch("skyvern.forge.sdk.workflow.context_manager.app") as mock_app:
+ mock_app.DATABASE.credentials.get_credential = AsyncMock(
+ return_value=SimpleNamespace(credential_id="cred_runtime", run_sequentially=True)
+ )
+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(
+ return_value=SimpleNamespace(sequential_credential_id=None)
+ )
+
+ with pytest.raises(RuntimeSequentialCredentialUnsupported, match="wr_test"):
+ await context.register_credential_parameter_value(login, organization)
+
+ mock_app.CREDENTIAL_VAULT_SERVICES.get.assert_not_called()
+
+
+ ("run_sequentially", "stamped_credential_id"),
+ [
+ (True, "cred_runtime"),
+ (False, None),
+ ],
+)
+async def test_runtime_credential_registration_proceeds_when_lane_is_safe(
+ run_sequentially: bool,
+ stamped_credential_id: str | None,
+) -> None:
+ context = _runtime_context()
+ source = _workflow_parameter("source")
+ runtime_parameter = ContextParameter(key="runtime_credential", source=source)
+ login = _credential_parameter(credential_id=runtime_parameter.key)
+ context.parameters[runtime_parameter.key] = runtime_parameter
+ context.values[runtime_parameter.key] = "cred_runtime"
+ organization = SimpleNamespace(organization_id="org_test")
+ db_credential = SimpleNamespace(
+ credential_id="cred_runtime",
+ run_sequentially=run_sequentially,
+ vault_type=None,
+ totp_identifier=None,
+ )
+ credential = MagicMock()
+ credential.model_dump.return_value = {}
+ credential_service = MagicMock()
+ credential_service.get_credential_item = AsyncMock(return_value=SimpleNamespace(credential=credential))
+
+ with patch("skyvern.forge.sdk.workflow.context_manager.app") as mock_app:
+ mock_app.DATABASE.credentials.get_credential = AsyncMock(return_value=db_credential)
+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(
+ return_value=SimpleNamespace(sequential_credential_id=stamped_credential_id)
+ )
+ mock_app.CREDENTIAL_VAULT_SERVICES.get.return_value = credential_service
+
+ await context.register_credential_parameter_value(login, organization)
+
+ credential_service.get_credential_item.assert_awaited_once_with(db_credential)
+ assert context.get_resolved_credential_parameter_id(login.key) == "cred_runtime"
+ if run_sequentially:
+ mock_app.DATABASE.workflow_runs.get_workflow_run.assert_awaited_once_with("wr_test", "org_test")
+ else:
+ mock_app.DATABASE.workflow_runs.get_workflow_run.assert_not_awaited()
+
+
+async def test_keyed_rotation_snapshot_and_runtime_reuse_one_persisted_selection() -> None:
+ login = _credential_parameter(credential_ids=["cred_a", "cred_b"])
+ repo = _SelectionRepo()
+
+ workflow_run = await _setup_bound_credentials(
+ [login],
+ credentials={"cred_a": True, "cred_b": False},
+ selection_repo=repo,
+ )
+ context = _runtime_context()
+ with patch("skyvern.forge.sdk.workflow.credential_selection.app") as selection_app:
+ selection_app.DATABASE.workflow_run_credential_selections = repo
+ runtime_credential_id = await context.resolve_credential_parameter_id(login, "org_test")
+
+ assert workflow_run.sequential_credential_id == "cred_a"
+ assert runtime_credential_id == "cred_a"
+ assert repo.existing[("wr_test", "login_cred")] == "cred_a"
+ assert [created["credential_id"] for created in repo.created] == ["cred_a"]
+
+
+async def test_two_opted_in_credentials_fail_closed_before_publication() -> None:
+ first = _credential_parameter(key="first_login", credential_id="cred_first")
+ second = _credential_parameter(key="second_login", credential_id="cred_second")
+
+ with pytest.raises(SequentialCredentialLimitExceeded):
+ await _setup_bound_credentials(
+ [first, second],
+ credentials={"cred_first": True, "cred_second": True},
+ )
+
+
@pytest.mark.asyncio
async def test_setup_workflow_run_uses_selected_rotating_credential_for_profile_key() -> None:
result, mock_app = await _setup_rotation_profile_run(select_side_effect="cred_b", profile_id="bp_selected")
@@ -794,19 +1091,101 @@ async def test_keyed_setup_workflow_run_fails_when_rotation_selection_fails() ->
@pytest.mark.asyncio
-async def test_keyless_setup_workflow_run_falls_back_to_keyless_profile_when_rotation_selection_fails() -> None:
- result, mock_app, _, caught = await _attempt_setup_rotation_profile_run(
+async def test_keyless_setup_workflow_run_fails_closed_when_rotation_selection_fails() -> None:
+ result, mock_app, service, caught = await _attempt_setup_rotation_profile_run(
select_side_effect=RuntimeError("selection failed"),
profile_id="bp_keyless",
browser_profile_key=None,
)
- assert caught is None
- assert result is not None
- assert result.browser_profile_id == "bp_keyless"
- mock_app.DATABASE.browser_sessions.get_or_create_managed_browser_profile.assert_awaited_once_with(
- organization_id="org_test",
- workflow_permanent_id="wpid_test",
- browser_profile_key_digest="",
- name="Workflow (auto-saved session)",
+ assert result is None
+ assert isinstance(caught, RuntimeError)
+ assert str(caught) == "selection failed"
+ mock_app.DATABASE.browser_sessions.get_or_create_managed_browser_profile.assert_not_awaited()
+ service.mark_workflow_run_as_failed.assert_awaited_once()
+ assert service.mark_workflow_run_as_failed.await_args.kwargs["workflow_run_id"] == "wr_test"
+ assert service.mark_workflow_run_as_failed.await_args.kwargs["failure_reason"].startswith(
+ "Setup workflow failed. failure reason:"
)
+
+
+async def _select_render_with_failed_rotation(
+ *,
+ browser_profile_key: str | None,
+ candidate_credentials: list[SimpleNamespace] | Exception,
+) -> dict[str, str]:
+ service = WorkflowService()
+ workflow = _setup_workflow_with_rotating_credential(browser_profile_key=browser_profile_key)
+ select_mock = AsyncMock(side_effect=RuntimeError("selection failed"))
+ get_credentials = (
+ AsyncMock(side_effect=candidate_credentials)
+ if isinstance(candidate_credentials, Exception)
+ else AsyncMock(return_value=candidate_credentials)
+ )
+ with (
+ patch("skyvern.forge.sdk.workflow.service.app") as mock_app,
+ patch("skyvern.forge.sdk.workflow.service.select_credential_for_run", select_mock),
+ ):
+ mock_app.DATABASE.organizations.get_organization = AsyncMock(
+ return_value=SimpleNamespace(organization_id="org_test")
+ )
+ mock_app.DATABASE.credentials.get_credentials_by_ids = get_credentials
+ return await service._select_rotating_credential_parameters_for_render(
+ workflow=workflow,
+ workflow_run=_setup_workflow_run(),
+ organization_id="org_test",
+ )
+
+
+async def test_keyless_rotation_selection_failure_preserves_legacy_when_no_candidate_is_sequential() -> None:
+ # A keyless workflow whose rotation pool is provably non-sequential keeps the legacy
+ # best-effort partial selection when rotation selection fails — the pre-feature behavior for runs
+ # that cannot resolve to a sequential credential. It must NOT hard-fail setup.
+ selections = await _select_render_with_failed_rotation(
+ browser_profile_key=None,
+ candidate_credentials=[
+ SimpleNamespace(credential_id="cred_a", run_sequentially=False),
+ SimpleNamespace(credential_id="cred_b", run_sequentially=False),
+ ],
+ )
+ assert selections == {}
+
+
+async def test_keyless_rotation_selection_failure_fails_closed_when_a_candidate_is_sequential() -> None:
+ # A keyless workflow with any opted-in candidate must fail closed on selection failure: the run
+ # could have resolved to that sequential credential, and silently skipping it drops the lane.
+ with pytest.raises(RuntimeError, match="selection failed"):
+ await _select_render_with_failed_rotation(
+ browser_profile_key=None,
+ candidate_credentials=[
+ SimpleNamespace(credential_id="cred_a", run_sequentially=False),
+ SimpleNamespace(credential_id="cred_b", run_sequentially=True),
+ ],
+ )
+
+
+async def test_keyless_rotation_selection_failure_fails_closed_when_candidate_unverifiable() -> None:
+ # A candidate that cannot be verified (missing/invalid id -> InvalidCredentialId, or a lookup
+ # error) means we cannot prove the pool is non-sequential, so fail closed conservatively.
+ with pytest.raises(RuntimeError, match="selection failed"):
+ await _select_render_with_failed_rotation(
+ browser_profile_key=None,
+ candidate_credentials=[], # neither cred_a nor cred_b resolves -> InvalidCredentialId
+ )
+
+
+async def test_keyed_rotation_selection_failure_fails_closed_even_with_non_sequential_pool() -> None:
+ # A browser_profile_key must render a real value; a selection failure fails closed regardless of
+ # the pool so distinct accounts never collapse onto one keyless managed profile.
+ with pytest.raises(RuntimeError, match="selection failed"):
+ await _select_render_with_failed_rotation(
+ browser_profile_key="{{ login_cred }}",
+ candidate_credentials=[
+ SimpleNamespace(credential_id="cred_a", run_sequentially=False),
+ SimpleNamespace(credential_id="cred_b", run_sequentially=False),
+ ],
+ )
diff --git a/tests/unit/test_credential_update_params.py b/tests/unit/test_credential_update_params.py
index 86ae83292..6055e4e3d 100644
--- a/tests/unit/test_credential_update_params.py
+++ b/tests/unit/test_credential_update_params.py
@@ -9,6 +9,7 @@ import pytest
from fastapi import BackgroundTasks, HTTPException
from skyvern.forge import app as forge_app
+from skyvern.forge.agent_functions import AgentFunction
from skyvern.forge.sdk.db.repositories.browser_sessions import BrowserSessionsRepository
from skyvern.forge.sdk.db.repositories.credentials import CredentialRepository
from skyvern.forge.sdk.routes import credentials as credentials_routes
@@ -18,6 +19,7 @@ from skyvern.forge.sdk.schemas.credentials import (
CredentialType,
CredentialVaultType,
TotpType,
+ UpdateCredentialRequest,
)
from skyvern.forge.sdk.services.credential.credential_vault_service import CredentialVaultService
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
@@ -1499,3 +1501,64 @@ async def test_create_credential_profile_alone_does_not_reset_pin(monkeypatch: p
)
assert update_credential.await_args.kwargs["pin_saved_session_ip"] is None
+
+
+def test_update_run_sequentially_documents_drain_before_enable_window() -> None:
+ # The false->true toggle does not retroactively serialize runs that were already active when the
+ # flag flipped on (they were never stamped with the credential). That adoption-window limitation is
+ # documented on the field rather than enforced with a broad active-run scan; pin the warning so the
+ # contract is not silently dropped.
+ description = UpdateCredentialRequest.model_fields["run_sequentially"].description or ""
+ assert "after this flag is enabled" in description
+ assert "Drain" in description
+
+
+def test_oss_agent_function_does_not_support_sequential_credentials() -> None:
+ assert AgentFunction().supports_sequential_credentials() is False
+
+
+async def test_oss_route_rejects_enabling_sequential_credential(monkeypatch: pytest.MonkeyPatch) -> None:
+ existing = _make_password_credential(run_sequentially=False)
+ update_credential = AsyncMock(return_value=_make_password_credential(run_sequentially=True))
+ monkeypatch.setattr(forge_app.DATABASE.credentials, "get_credential", AsyncMock(return_value=existing))
+ monkeypatch.setattr(forge_app.DATABASE.credentials, "update_credential", update_credential)
+ monkeypatch.setattr(
+ forge_app,
+ "AGENT_FUNCTION",
+ SimpleNamespace(supports_sequential_credentials=lambda: False),
+ )
+
+ with pytest.raises(HTTPException) as exc:
+ await credentials_routes.rename_credential(
+ credential_id="cred_123",
+ data=UpdateCredentialRequest(run_sequentially=True),
+ current_org=SimpleNamespace(organization_id="org_123"),
+ )
+
+ assert exc.value.status_code == 400
+ assert exc.value.detail == "Sequential credential execution is not supported by this deployment"
+ update_credential.assert_not_awaited()
+
+
+async def test_oss_route_allows_disabling_sequential_credential(monkeypatch: pytest.MonkeyPatch) -> None:
+ existing = _make_password_credential(run_sequentially=True)
+ updated = _make_password_credential(run_sequentially=False)
+ update_credential = AsyncMock(return_value=updated)
+ monkeypatch.setattr(forge_app.DATABASE.credentials, "get_credential", AsyncMock(return_value=existing))
+ monkeypatch.setattr(forge_app.DATABASE.credentials, "update_credential", update_credential)
+ monkeypatch.setattr(
+ forge_app,
+ "AGENT_FUNCTION",
+ SimpleNamespace(supports_sequential_credentials=lambda: False),
+ )
+
+ response = await credentials_routes.rename_credential(
+ credential_id="cred_123",
+ data=UpdateCredentialRequest(run_sequentially=False),
+ current_org=SimpleNamespace(organization_id="org_123"),
+ )
+
+ assert response.run_sequentially is False
+ assert update_credential.await_args.kwargs["run_sequentially"] is False
diff --git a/tests/unit/test_totp_identifier_fallback.py b/tests/unit/test_totp_identifier_fallback.py
index ce4f44c6b..bc370d487 100644
--- a/tests/unit/test_totp_identifier_fallback.py
+++ b/tests/unit/test_totp_identifier_fallback.py
@@ -22,6 +22,7 @@ async def test_register_credential_parameter_uses_db_totp_identifier(monkeypatch
organization_id="org-1",
vault_type=CredentialVaultType.BITWARDEN,
totp_identifier="[email protected]",
+ run_sequentially=False,
)
class FakeCredential:
@@ -80,6 +81,7 @@ async def _register_with_credential(monkeypatch: pytest.MonkeyPatch, credential:
organization_id="org-1",
vault_type=CredentialVaultType.BITWARDEN,
totp_identifier=None,
+ run_sequentially=False,
)
class FakeCredentialItem:
diff --git a/tests/unit/test_workflow_browser_profile_id.py b/tests/unit/test_workflow_browser_profile_id.py
index bf10a3f9b..72a212b88 100644
--- a/tests/unit/test_workflow_browser_profile_id.py
+++ b/tests/unit/test_workflow_browser_profile_id.py
@@ -328,6 +328,13 @@ def _make_setup_service(workflow: SimpleNamespace) -> tuple[WorkflowService, Sim
return service, organization, workflow_run
+def _configure_setup_app_mocks(mock_app: Any) -> None:
+ mock_app.DATABASE.workflows.get_browser_action_policy = AsyncMock(return_value=None)
+ mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
+ mock_app.AGENT_FUNCTION.should_use_flex_llm_routing = AsyncMock(return_value=False)
+ mock_app.DATABASE.workflow_runs.update_workflow_run = AsyncMock()
+
+
def _make_workflow_stub(
browser_profile_id: str | None,
max_elapsed_time_minutes: int | None = None,
@@ -371,9 +378,7 @@ async def test_setup_workflow_run_falls_back_to_workflow_browser_profile_id() ->
assert request.browser_profile_id is None
with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:
- mock_app.DATABASE.workflows.get_browser_action_policy = AsyncMock(return_value=None)
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
- mock_app.AGENT_FUNCTION.should_use_flex_llm_routing = AsyncMock(return_value=False)
+ _configure_setup_app_mocks(mock_app)
await service.setup_workflow_run(
request_id="req_test",
@@ -394,9 +399,7 @@ async def test_setup_workflow_run_run_level_value_takes_precedence() -> None:
request = WorkflowRequestBody(data={}, browser_profile_id="bp_run_specific")
with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:
- mock_app.DATABASE.workflows.get_browser_action_policy = AsyncMock(return_value=None)
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
- mock_app.AGENT_FUNCTION.should_use_flex_llm_routing = AsyncMock(return_value=False)
+ _configure_setup_app_mocks(mock_app)
await service.setup_workflow_run(
request_id="req_test",
@@ -417,9 +420,7 @@ async def test_setup_workflow_run_no_default_no_request_stays_none() -> None:
request = WorkflowRequestBody(data={})
with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:
- mock_app.DATABASE.workflows.get_browser_action_policy = AsyncMock(return_value=None)
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
- mock_app.AGENT_FUNCTION.should_use_flex_llm_routing = AsyncMock(return_value=False)
+ _configure_setup_app_mocks(mock_app)
await service.setup_workflow_run(
request_id="req_test",
@@ -441,9 +442,7 @@ async def test_setup_workflow_run_session_present_skips_workflow_default() -> No
assert request.browser_profile_id is None
with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:
- mock_app.DATABASE.workflows.get_browser_action_policy = AsyncMock(return_value=None)
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
- mock_app.AGENT_FUNCTION.should_use_flex_llm_routing = AsyncMock(return_value=False)
+ _configure_setup_app_mocks(mock_app)
await service.setup_workflow_run(
request_id="req_test",
@@ -464,9 +463,7 @@ async def test_setup_workflow_run_prefers_request_max_elapsed_time_over_workflow
request = WorkflowRequestBody(data={}, max_elapsed_time_minutes=10)
with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:
- mock_app.DATABASE.workflows.get_browser_action_policy = AsyncMock(return_value=None)
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
- mock_app.AGENT_FUNCTION.should_use_flex_llm_routing = AsyncMock(return_value=False)
+ _configure_setup_app_mocks(mock_app)
await service.setup_workflow_run(
request_id="req_test",
@@ -480,3 +477,27 @@ async def test_setup_workflow_run_prefers_request_max_elapsed_time_over_workflow
create_workflow_run_mock.assert_awaited_once()
assert create_workflow_run_mock.await_args is not None
assert create_workflow_run_mock.await_args.kwargs["max_elapsed_time_minutes"] == 10
+
+
+async def test_setup_workflow_run_without_credentials_writes_no_sequential_credential() -> None:
+ """A credential-less run leaves sequential_credential_id NULL — the MVP never writes a completion
+ sentinel. Readiness is proven from queued status + queued task + parameter equality, so no credential
+ write happens and the column stays NULL for a run that resolves to no sequential credential."""
+ workflow_stub = _make_workflow_stub(browser_profile_id=None)
+ service, organization, _ = _make_setup_service(workflow_stub)
+
+ request = WorkflowRequestBody(data={})
+
+ with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:
+ _configure_setup_app_mocks(mock_app)
+
+ await service.setup_workflow_run(
+ request_id="req_test",
+ workflow_request=request,
+ workflow_permanent_id="wpid_test",
+ organization=organization,
+ )
+
+ update_mock = mock_app.DATABASE.workflow_runs.update_workflow_run
+ assert all("sequential_credential_id" not in call.kwargs for call in update_mock.await_args_list)
diff --git a/tests/unit/test_workflow_run_parameter_batching.py b/tests/unit/test_workflow_run_parameter_batching.py
index ba0bac20b..00f0a26a8 100644
--- a/tests/unit/test_workflow_run_parameter_batching.py
+++ b/tests/unit/test_workflow_run_parameter_batching.py
@@ -89,6 +89,7 @@ def _make_service_with_mocks(
service.get_workflow_by_permanent_id = AsyncMock(return_value=workflow) # type: ignore[method-assign]
service.create_workflow_run = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]
service.get_workflow_parameters = AsyncMock(return_value=workflow_parameters) # type: ignore[method-assign]
+ service._resolve_sequential_credential_id = AsyncMock(return_value=None) # type: ignore[method-assign]
if batch_side_effect:
service.create_workflow_run_parameters = AsyncMock(side_effect=batch_side_effect) # type: ignore[method-assign]
else:
diff --git a/tests/unit/workflow/test_workflow_trigger_block.py b/tests/unit/workflow/test_workflow_trigger_block.py
index 3e404956e..475501e9c 100644
--- a/tests/unit/workflow/test_workflow_trigger_block.py
+++ b/tests/unit/workflow/test_workflow_trigger_block.py
@@ -444,6 +444,7 @@ async def test_sync_trigger_preserves_parent_feature_flag_summary(monkeypatch: p
workflow_run = MagicMock()
workflow_run.workflow_run_id = "wr_child"
workflow_run.workflow_permanent_id = "wfp_child"
+ workflow_run.sequential_credential_id = None
return workflow_run
async def _execute_workflow(**_: Any) -> Any:
@@ -501,6 +502,149 @@ async def test_sync_trigger_preserves_parent_feature_flag_summary(monkeypatch: p
}
+async def test_sync_trigger_fails_closed_when_child_resolves_sequential_credential() -> None:
+ # A synchronous trigger child runs inline via execute_workflow — it never queues, never gets a
+ # queued_at, and never reaches the Temporal V2 serialization gate. setup_workflow_run still stamps
+ # its sequential_credential_id, so a concurrent run sharing that credential would not see it as a
+ # blocker (the gate filters queued_at IS NOT NULL). The child must fail closed before it uses the
+ # credential — mirroring the scheduled-run fence — instead of running unserialized.
+ block = _make_block(browser_session_id="pbs_child")
+
+ child_run = MagicMock()
+ child_run.workflow_run_id = "wr_child"
+ child_run.workflow_permanent_id = "wfp_child"
+ child_run.sequential_credential_id = "cred_a"
+
+ organization = MagicMock()
+ organization.organization_id = "org_parent"
+
+ captured: dict[str, Any] = {}
+
+ async def _build_result(**kwargs: Any) -> Any:
+ captured["success"] = kwargs.get("success")
+ captured["failure_reason"] = kwargs.get("failure_reason")
+ return MagicMock()
+
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,
+ patch.object(WorkflowTriggerBlock, "get_workflow_run_context", lambda self, workflow_run_id: MagicMock()),
+ patch.object(WorkflowTriggerBlock, "format_potential_template_parameters", lambda self, ctx: None),
+ patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value=0)),
+ patch.object(WorkflowTriggerBlock, "record_output_parameter_value", AsyncMock()),
+ patch.object(WorkflowTriggerBlock, "build_block_result", AsyncMock(side_effect=_build_result)),
+ ):
+ mock_app.DATABASE.organizations.get_organization = AsyncMock(return_value=organization)
+ mock_app.WORKFLOW_SERVICE.setup_workflow_run = AsyncMock(return_value=child_run)
+ mock_app.WORKFLOW_SERVICE.execute_workflow = AsyncMock()
+ mock_app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final = AsyncMock()
+
+ await block.execute(
+ workflow_run_id="wr_parent",
+ workflow_run_block_id="wrb_parent",
+ organization_id="org_parent",
+ browser_session_id=None,
+ )
+
+ mock_app.WORKFLOW_SERVICE.execute_workflow.assert_not_awaited()
+ mock_app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final.assert_awaited_once()
+ assert (
+ mock_app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final.await_args.kwargs["workflow_run_id"]
+ == "wr_child"
+ )
+ assert captured["success"] is False
+
+
+async def _run_sync_trigger_fence(
+ block: WorkflowTriggerBlock,
+ *,
+ created_session_id: str | None,
+ setup_raises: bool = False,
+) -> tuple[MagicMock, dict[str, Any]]:
+ child_run = MagicMock()
+ child_run.workflow_run_id = "wr_child"
+ child_run.workflow_permanent_id = "wfp_child"
+ child_run.sequential_credential_id = "cred_a"
+
+ organization = MagicMock()
+ organization.organization_id = "org_parent"
+
+ captured: dict[str, Any] = {}
+
+ async def _build_result(**kwargs: Any) -> Any:
+ captured["success"] = kwargs.get("success")
+ captured["failure_reason"] = kwargs.get("failure_reason")
+ return MagicMock()
+
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,
+ patch.object(WorkflowTriggerBlock, "get_workflow_run_context", lambda self, workflow_run_id: MagicMock()),
+ patch.object(WorkflowTriggerBlock, "format_potential_template_parameters", lambda self, ctx: None),
+ patch.object(WorkflowTriggerBlock, "_check_trigger_depth", AsyncMock(return_value=0)),
+ patch.object(WorkflowTriggerBlock, "record_output_parameter_value", AsyncMock()),
+ patch.object(WorkflowTriggerBlock, "build_block_result", AsyncMock(side_effect=_build_result)),
+ ):
+ mock_app.DATABASE.organizations.get_organization = AsyncMock(return_value=organization)
+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=MagicMock(proxy_location=None))
+ created_session = MagicMock()
+ created_session.persistent_browser_session_id = created_session_id
+ mock_app.PERSISTENT_SESSIONS_MANAGER.create_session = AsyncMock(return_value=created_session)
+ mock_app.PERSISTENT_SESSIONS_MANAGER.close_session = AsyncMock()
+ if setup_raises:
+ mock_app.WORKFLOW_SERVICE.setup_workflow_run = AsyncMock(side_effect=RuntimeError("setup boom"))
+ else:
+ mock_app.WORKFLOW_SERVICE.setup_workflow_run = AsyncMock(return_value=child_run)
+ mock_app.WORKFLOW_SERVICE.execute_workflow = AsyncMock()
+ mock_app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final = AsyncMock()
+
+ await block.execute(
+ workflow_run_id="wr_parent",
+ workflow_run_block_id="wrb_parent",
+ organization_id="org_parent",
+ browser_session_id=None,
+ )
+
+ return mock_app, captured
+
+
+async def test_sync_trigger_closes_fresh_session_when_fence_fires() -> None:
+ # A sync trigger with no configured/inherited session creates a fresh 30-minute persistent
+ # session before setup. When the child resolves a sequential credential, the fail-closed fence
+ # must still close that self-created session before returning; otherwise it strands until timeout.
+ block = _make_block()
+
+ mock_app, captured = await _run_sync_trigger_fence(block, created_session_id="pbs_fresh")
+
+ mock_app.WORKFLOW_SERVICE.execute_workflow.assert_not_awaited()
+ mock_app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final.assert_awaited_once()
+ assert captured["success"] is False
+ mock_app.PERSISTENT_SESSIONS_MANAGER.close_session.assert_awaited_once_with("org_parent", "pbs_fresh")
+
+
+async def test_sync_trigger_does_not_close_configured_session_on_fence() -> None:
+ # A block-configured session is owned by the caller, not created here: the fence must not close it.
+ block = _make_block(browser_session_id="pbs_child")
+
+ mock_app, captured = await _run_sync_trigger_fence(block, created_session_id="pbs_fresh")
+
+ mock_app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final.assert_awaited_once()
+ assert captured["success"] is False
+ mock_app.PERSISTENT_SESSIONS_MANAGER.close_session.assert_not_awaited()
+
+
+async def test_sync_trigger_closes_fresh_session_when_setup_raises() -> None:
+ # The pre-existing setup-failure early return shares the leak: cleanup must run there too.
+ block = _make_block()
+
+ mock_app, captured = await _run_sync_trigger_fence(block, created_session_id="pbs_fresh", setup_raises=True)
+
+ assert captured["success"] is False
+ mock_app.PERSISTENT_SESSIONS_MANAGER.close_session.assert_awaited_once_with("org_parent", "pbs_fresh")
+
+
class TestBlockMetadata:
"""Verify basic block properties."""