Skip to content

Automate browser based workflows with AI

feat(SKY-12138): pause copilot BUILD turns on a mid-loop credential ask (#7360)

Celal Zamanoğlu114441f4e25e
diff --git a/skyvern/config.py b/skyvern/config.py
index 937f2aab5..0673e2dd6 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -209,6 +209,13 @@ class Settings(BaseSettings):
# Off = standard block authoring. On = prefer code blocks for browser work.
WORKFLOW_COPILOT_CODE_BLOCK_MODE: bool = False
WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY: bool = False
+ # Pause a BUILD turn in place on a typed mid-loop credential ask instead of ending it;
+ # the FE resumes the same turn via a credential-connect card. Off = today's turn-terminal behavior.
+ # Requires app.CACHE to be a shared cache (Redis) -- a same-process-only cache can't
+ # coordinate the poller with a /credential-response POST that may land on another worker,
+ # so this is a guaranteed no-op behind app.CACHE.is_shared regardless of this flag.
+ WORKFLOW_COPILOT_CREDENTIAL_PAUSE_ENABLED: bool = False
+ WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS: int = 300
# Kill switch for the live codegen-progress SSE frame (drafted block labels while an authoring
# tool call streams). Off restores exact pre-change behavior; old frontends drop the frame either way.
WORKFLOW_COPILOT_CODEGEN_PROGRESS_ENABLED: bool = True
diff --git a/skyvern/forge/sdk/cache/base.py b/skyvern/forge/sdk/cache/base.py
index 0b8bc6509..8173b3fd8 100644
--- a/skyvern/forge/sdk/cache/base.py
+++ b/skyvern/forge/sdk/cache/base.py
@@ -45,6 +45,13 @@ class NoopLock:
class BaseCache(ABC):
+ #: True only for caches visible across separate worker processes (Redis).
+ #: A feature that writes in one request and reads in another (e.g. a
+ #: paused turn resumed by a POST that may land on a different worker)
+ #: must check this before relying on get/set alone -- a same-process-only
+ #: cache like LocalCache would silently hang, not error.
+ is_shared: bool = False
+
@abstractmethod
async def set(self, key: str, value: Any, ex: Union[int, timedelta, None] = CACHE_EXPIRE_TIME) -> None:
pass
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index e359da63f..2c847b1d3 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -1728,6 +1728,13 @@ def _make_agent_result(
reason = credential_prompt_reason(policy, kwargs.get("user_response"))
if reason:
payload_updates["credentialPrompt"] = {"reason": reason}
+ if ctx is not None and "credentialPause" not in narrative_payload:
+ pause_outcome = ctx.credential_pause_outcome
+ if pause_outcome:
+ pause_payload = {"outcome": pause_outcome}
+ if pause_outcome == "connected" and ctx.request_policy and ctx.request_policy.resolved_credentials:
+ pause_payload["credentialId"] = ctx.request_policy.resolved_credentials[-1].credential_id
+ payload_updates["credentialPause"] = pause_payload
if ctx is not None and "verifiedSuccess" not in narrative_payload:
payload_updates["verifiedSuccess"] = bool(verified_goal_claim_authorized(ctx))
if ctx is not None and "outcomeAdjudication" not in narrative_payload:
@@ -4557,6 +4564,7 @@ async def _run_copilot_turn_impl(
impose_synthesized_code_block=copilot_config.impose_synthesized_code_block,
copilot_config=copilot_config,
target_block_label=getattr(chat_request, "target_block_label", None),
+ client_supports_credential_pause=getattr(chat_request, "supports_credential_pause", False),
)
LOG.info(
"copilot_block_authoring_policy_resolved",
diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.py
index 61c385fef..4cfa0a910 100644
--- a/skyvern/forge/sdk/copilot/config.py
+++ b/skyvern/forge/sdk/copilot/config.py
@@ -356,6 +356,14 @@ def _default_fallback_llm_key() -> str | None:
return settings.SECONDARY_LLM_KEY
+def _default_credential_pause_enabled() -> bool:
+ return settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_ENABLED
+
+
+def _default_credential_pause_timeout_seconds() -> int:
+ return settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS
+
+
@dataclass(slots=True)
class CopilotConfig:
prompt_template: str = DEFAULT_PROMPT_TEMPLATE
@@ -367,6 +375,8 @@ class CopilotConfig:
block_authoring_policy: BlockAuthoringPolicy = BlockAuthoringPolicy.STANDARD
impose_synthesized_code_block: bool = False
requested_output_path_aliases: dict[str, str] = field(default_factory=dict)
+ credential_pause_enabled: bool = field(default_factory=_default_credential_pause_enabled)
+ credential_pause_timeout_seconds: int = field(default_factory=_default_credential_pause_timeout_seconds)
def nudge(self, key: str) -> str:
return self.enforcement_nudges.get(key, DEFAULT_ENFORCEMENT_NUDGES[key])
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index d8c7fb959..2dc2740eb 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -87,6 +87,9 @@ class TurnNarrativePayload(TypedDict):
outcomeAdjudication: NotRequired[NarrativeOutcomeAdjudication]
# {"reason": <credential_prompt_reason() token>}, set when this turn surfaces a credential need.
credentialPrompt: NotRequired[dict[str, str]]
+ # {"outcome": "connected"|"skipped"|"timeout", "credentialId": ...}, set when a mid-build
+ # credential pause (credential_pause.py) resolved during this turn.
+ credentialPause: NotRequired[dict[str, str]]
designStarted: bool
designEnded: bool
draft: NarrativeDraft | None
@@ -668,6 +671,15 @@ class CopilotContext(AgentContext):
blocked_reply_signatures: list[str] = field(default_factory=list)
requested_output_extraction_candidate: FrozenRequestedOutputExtractionCandidate | None = None
+ # Mid-build credential pause (credential_pause.py). last_run_skipped_unbound_credentials
+ # is set by tools/__init__.py's update_and_run_blocks skip branch; client_supports_credential_pause
+ # is set from the chat request at construction; the rest are owned by maybe_credential_pause.
+ last_run_skipped_unbound_credentials: bool = False
+ client_supports_credential_pause: bool = False
+ credential_pause_used: bool = False
+ copilot_credential_pause_seconds: float = 0.0
+ credential_pause_outcome: str | None = None
+
# Tool tracking
consecutive_tool_tracker: list[str] = field(default_factory=list)
tool_activity: list[dict[str, Any]] = field(default_factory=list)
diff --git a/skyvern/forge/sdk/copilot/credential_pause.py b/skyvern/forge/sdk/copilot/credential_pause.py
new file mode 100644
index 000000000..e6f6f85c0
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/credential_pause.py
@@ -0,0 +1,542 @@
+"""Mid-loop pause for copilot BUILD turns blocked on a missing credential.
+
+Parks between ``Runner.run_streamed`` iterations at the enforcement finalize
+seam (see ``run_with_enforcement``) so the SSE connection stays open while
+the frontend surfaces a credential-connect card. Resume transport is a
+per-turn Redis flag polled directly by the paused coroutine -- the inverse of
+the ``/workflow/copilot/cancel`` sidecar watcher, since here the paused loop
+itself is the poller rather than a task racing the handler.
+
+The resume path is not authorized by org auth + ``turn_id`` alone. Establishing
+the pause writes an *active-pause record*, keyed by (org, chat, turn), that
+carries a one-time ``resume_token`` delivered only in the ``credential_required``
+frame. ``resolve_credential_pause`` -- the only writer of the loop-facing
+response flag -- refuses to store a decision unless the caller presents that
+token against a still-pending record, and consumes the record on the first
+accepted response so a leaked or replayed ``turn_id`` can't resolve the pause.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import re
+import secrets
+import time
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Any, Literal
+
+import structlog
+
+from skyvern.config import settings
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot.config import CopilotConfig
+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import DiagnosisFailureType, RepairNextAction
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+from skyvern.forge.sdk.schemas.credentials import Credential
+from skyvern.forge.sdk.schemas.workflow_copilot import (
+ WorkflowCopilotCredentialRequiredUpdate,
+ WorkflowCopilotStreamMessageType,
+)
+
+if TYPE_CHECKING:
+ from agents.result import RunResultStreaming
+
+ # Importing the routes package at module scope pulls in workflow_copilot.py ->
+ # agent.py -> enforcement.py, which imports this module -> circular import.
+ from skyvern.forge.sdk.core.event_source_stream import EventSourceStream
+
+LOG = structlog.get_logger()
+
+CREDENTIAL_RESPONSE_POLL_SECONDS = 1.5
+
+# The active-pause record and loop-facing response flag outlive the wait window
+# by this grace so an accepted response is still readable if the resumed loop is
+# briefly slow to poll.
+CREDENTIAL_PAUSE_RECORD_TTL_GRACE_SECONDS = 300
+
+
+def credential_response_cache_key(organization_id: str, chat_id: str, turn_id: str) -> str:
+ return f"copilot_credential_response:{organization_id}:{chat_id}:{turn_id}"
+
+
+def credential_pause_active_key(organization_id: str, chat_id: str, turn_id: str) -> str:
+ return f"copilot_credential_pause:{organization_id}:{chat_id}:{turn_id}"
+
+
+def _credential_pause_lock_key(organization_id: str, chat_id: str, turn_id: str) -> str:
+ return f"copilot_credential_pause_lock:{organization_id}:{chat_id}:{turn_id}"
+
+
+def _credential_pause_record_ttl(timeout_seconds: int) -> timedelta:
+ return timedelta(seconds=timeout_seconds + CREDENTIAL_PAUSE_RECORD_TTL_GRACE_SECONDS)
+
+
+def _new_resume_token() -> str:
+ return secrets.token_urlsafe(32)
+
+
+class CredentialPauseRejection(Exception):
+ """Raised by ``resolve_credential_pause`` when a resume attempt is not authorized.
+
+ Carries the HTTP status the route should surface. Kept fastapi-free so this
+ module stays importable from the enforcement loop without pulling in routes.
+ """
+
+ def __init__(self, status_code: int, detail: str) -> None:
+ super().__init__(detail)
+ self.status_code = status_code
+ self.detail = detail
+
+
+@dataclass
+class _ActivePauseRecord:
+ resume_token: str
+ status: Literal["pending", "consumed"]
+ expires_at: datetime
+
+
+def _encode_active_pause(resume_token: str, expires_at: datetime, *, consumed: bool = False) -> str:
+ return json.dumps(
+ {
+ "resume_token": resume_token,
+ "status": "consumed" if consumed else "pending",
+ "expires_at": expires_at.isoformat(),
+ }
+ )
+
+
+def _decode_active_pause(raw: Any) -> _ActivePauseRecord | None:
+ try:
+ data = json.loads(raw)
+ except (TypeError, ValueError):
+ return None
+ if not isinstance(data, dict):
+ return None
+ token = data.get("resume_token")
+ record_status = data.get("status")
+ if not isinstance(token, str) or record_status not in ("pending", "consumed"):
+ return None
+ try:
+ expires_at = datetime.fromisoformat(str(data.get("expires_at")))
+ except ValueError:
+ return None
+ return _ActivePauseRecord(resume_token=token, status=record_status, expires_at=expires_at)
+
+
+def _validate_pending_pause(record: _ActivePauseRecord | None, resume_token: str) -> _ActivePauseRecord:
+ """Shared checks for a resume attempt: a pending, unexpired record bound to this token."""
+ if record is None:
+ raise CredentialPauseRejection(
+ status_code=HTTPStatus.NOT_FOUND,
+ detail="No active credential pause for this turn",
+ )
+ if record.status != "pending":
+ raise CredentialPauseRejection(
+ status_code=HTTPStatus.CONFLICT,
+ detail="Credential pause already resolved",
+ )
+ # The record's Redis TTL outlives the wait window by design (grace for a
+ # slow final poll) -- that's an infra buffer, not permission to accept a
+ # response after the waiter gave up and the turn already finalized without
+ # it. Enforce the actual deadline the frame told the client about.
+ if datetime.now(timezone.utc) >= record.expires_at:
+ raise CredentialPauseRejection(
+ status_code=HTTPStatus.GONE,
+ detail="Credential pause has expired",
+ )
+ if not resume_token or not secrets.compare_digest(str(resume_token), record.resume_token):
+ raise CredentialPauseRejection(
+ status_code=HTTPStatus.FORBIDDEN,
+ detail="Invalid credential resume token",
+ )
+ return record
+
+
+async def check_credential_pause_resumable(
+ cache: Any,
+ *,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ turn_id: str,
+ resume_token: str,
+) -> None:
+ """Read-only precheck: raises the same rejection a resolve would, without consuming anything.
+
+ Lets a caller (the route) validate the token BEFORE doing anything that
+ reveals org-scoped information (e.g. a credential-id existence lookup) to
+ an unauthenticated-for-this-turn caller who only has org auth.
+ """
+ active_key = credential_pause_active_key(organization_id, workflow_copilot_chat_id, turn_id)
+ record = _decode_active_pause(await cache.get(active_key))
+ _validate_pending_pause(record, resume_token)
+
+
+async def resolve_credential_pause(
+ cache: Any,
+ *,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ turn_id: str,
+ resume_token: str,
+ action: Literal["connected", "skip"],
+ credential_id: str | None,
+) -> None:
+ """Validate the resume token against the active pause, consume it, then store the decision.
+
+ The security boundary for resuming a paused turn: org auth alone is not
+ enough. The caller must present the one-time ``resume_token`` from the
+ ``credential_required`` frame, and (org, chat, turn) must name a still-pending
+ active-pause record. Under a per-turn lock, the first accepted response flips
+ the record to ``consumed`` before writing the loop-facing flag, so a leaked or
+ replayed ``turn_id`` can neither resolve a pause it never opened nor overwrite
+ a decision already made.
+ """
+ active_key = credential_pause_active_key(organization_id, workflow_copilot_chat_id, turn_id)
+ lock_key = _credential_pause_lock_key(organization_id, workflow_copilot_chat_id, turn_id)
+ async with cache.get_lock(lock_key):
+ record = _validate_pending_pause(_decode_active_pause(await cache.get(active_key)), resume_token)
+ ttl = _credential_pause_record_ttl(settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS)
+ await cache.set(active_key, _encode_active_pause(record.resume_token, record.expires_at, consumed=True), ex=ttl)
+ await cache.set(
+ credential_response_cache_key(organization_id, workflow_copilot_chat_id, turn_id),
+ encode_credential_response(action, credential_id),
+ ex=ttl,
+ )
+
+
+@dataclass
+class CredentialPauseResolution:
+ action: Literal["connected", "skip"]
+ credential: Credential | None = None
+
+
+def encode_credential_response(action: Literal["connected", "skip"], credential_id: str | None) -> str:
+ return json.dumps({"action": action, "credential_id": credential_id})
+
+
+def _decode_credential_response(raw: Any) -> tuple[str, str | None] | None:
+ try:
+ data = json.loads(raw)
+ except (TypeError, ValueError):
+ return None
+ if not isinstance(data, dict) or data.get("action") not in ("connected", "skip"):
+ return None
+ credential_id = data.get("credential_id")
+ return data["action"], credential_id if isinstance(credential_id, str) else None
+
+
+def credential_pause_reason(ctx: Any) -> str | None:
+ """Typed-signal-only detector for a mid-build credential ask.
+
+ Deliberately narrower than ``credential_prompt_reason`` (request_policy.py):
+ no text-marker tier, so a REPLY that merely mentions credentials in prose
+ can't trigger a pause -- see the SKY-11988 false-positive lesson.
+ """
+ policy = getattr(ctx, "request_policy", None)
+ skip_test = isinstance(policy, RequestPolicy) and policy.testing_intent == "skip_test"
+ if getattr(ctx, "last_run_skipped_unbound_credentials", False) and not skip_test:
+ # A skip_test turn can still land here: _request_policy_allows_update_and_skip_run
+ # (guardrails.py) skips the run for a skip_test-deferred credential, same as it
+ # does for a credential_deferred_draft -- pausing here would contradict the same
+ # explicit "don't verify this yet" request the deferred-draft branch below excludes.
+ return "workflow_credential_inputs_unbound"
+
+ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)
+ if (
+ contract is not None
+ and contract.repair_decision.next_action == RepairNextAction.ASK
+ and contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ # MISSING_CREDENTIAL_OR_INIT is a combined category -- diagnosis_repair_contract.py
+ # also assigns it to PARAMETER_BINDING_ERROR and org/workflow/browser-session lookup
+ # failures that have nothing to do with credentials. Require the specific category so
+ # those don't get preempted by a credential card that can't unblock them.
+ and "CREDENTIAL_ERROR" in contract.diagnosis_result.root_cause_identity.failure_categories
+ ):
+ return "missing_credential_run_failure"
+
+ if (
+ isinstance(policy, RequestPolicy)
+ and policy.credential_draft_deferred_explicitly
+ # An explicit skip-test credential draft (request_policy.py's
+ # _apply_explicit_code_block_credential_draft_policy) means the user already said
+ # not to run/verify this -- pausing to ask for a credential contradicts that ask.
+ and not skip_test
+ and getattr(ctx, "update_workflow_called", False)
+ ):
+ return "credential_deferred_draft"
+
+ return None
+
+
+def credential_pause_would_fire(ctx: Any, copilot_config: CopilotConfig | None) -> bool:
+ """Synchronous subset of maybe_credential_pause's guards.
+
+ Called from both here and enforcement.py's _check_enforcement so the two
+ can't drift: enforcement pre-empts hygiene nudges based on this same
+ prediction, and a client/kill-switch/latch mismatch between the two would
+ either suppress nudges for a pause that will never fire, or let a nudge
+ race a pause that will. Excludes the async-only checks (stream disconnect).
+ """
+ return (
+ copilot_config is not None
+ and copilot_config.credential_pause_enabled
+ and getattr(ctx, "client_supports_credential_pause", False)
+ and not getattr(ctx, "credential_pause_used", False)
+ # A same-process-only cache (LocalCache) can't coordinate the poller with a
+ # /credential-response POST that may land on a different worker -- gate on a
+ # cache that's explicitly known to be shared (Redis) rather than merely non-None.
+ and getattr(getattr(app, "CACHE", None), "is_shared", False)
+ and credential_pause_reason(ctx) is not None
+ )
+
+
+def _defang(text: str) -> str:
+ return re.sub(r"\s+", " ", text).replace('"', "").strip()[:200]
+
+
+def _apply_connected_credential_to_policy(ctx: Any, policy: RequestPolicy, credential: Credential) -> None:
+ """Grant the resumed loop run authority for ``credential``.
+
+ Load-bearing: without this, the resumed ``update_and_run_blocks`` call
+ re-skips via ``_request_policy_allows_update_and_skip_run`` (guardrails.py)
+ and the turn re-asks for the same credential.
+
+ Also un-latches ``test_after_update_done``: the skipped run before the pause
+ already set it True, so without this reset a reply that never re-runs the
+ blocks would still clear enforcement's post_update nudge and finalize an
+ untested draft instead of being forced back through update_and_run_blocks.
+ """
+ ctx.test_after_update_done = False
+ policy.resolved_credentials.append(credential)
+ policy.allow_run_blocks = True
+ policy.clarification_reason = "none"
+ policy.allow_missing_credentials_in_draft = False
+ policy.requires_user_clarification = False
+ # Otherwise credential_prompt_reason() still sees the deferred-draft flag on the
+ # terminal RESPONSE and stamps credentialPrompt right next to credentialPause:
+ # connected -- a contradictory "still need a credential" signal to the FE.
+ policy.credential_draft_deferred_explicitly = False
+
+
+def _assemble_resume_messages(ctx: Any, text: str) -> list[dict[str, Any]]:
+ # Local import avoids a module-load cycle: enforcement.py imports
+ # maybe_credential_pause at module scope.
+ from skyvern.forge.sdk.copilot.enforcement import _assemble_enforcement_messages, _consume_pending_screenshots
+
+ screenshot_msg = _consume_pending_screenshots(ctx)
+ return _assemble_enforcement_messages(screenshot_msg, text, None)
+
+
+def _connected_resume_text(credential: Credential) -> str:
+ safe_name = _defang(credential.name)
+ return (
+ f"The user connected saved credential {safe_name} ({credential.credential_id}) via the credential card. "
+ "Bind it as the credential parameter and continue the build; run the blocks that were skipped or failed "
+ "on the missing credential."
+ )
+
+
+_SKIP_RESUME_TEXT = (
+ "The user chose not to connect a credential now. Continue without it: keep the credential parameter "
+ "placeholder in the draft, do not ask for the credential again this turn, and finish. If you run a test, "
+ "it may stop at the login step."
+)
+
+
+async def _try_resolve_credential_response(
+ response_key: str, organization_id: str
+) -> CredentialPauseResolution | None | Literal["pending"]:
+ cache = app.CACHE
+ raw = await cache.get(response_key)
+ if not raw:
+ return "pending"
+ decoded = _decode_credential_response(raw)
+ if decoded is None:
+ return "pending"
+ action, credential_id = decoded
+ if action == "skip":
+ return CredentialPauseResolution(action="skip")
+ if not credential_id:
+ return None
+ existing = await app.DATABASE.credentials.get_credentials_by_ids([credential_id], organization_id=organization_id)
+ if not existing:
+ return None
+ return CredentialPauseResolution(action="connected", credential=existing[0])
+
+
+async def _wait_for_credential_response(
+ response_key: str,
+ ctx: Any,
+ stream: EventSourceStream,
+ timeout_seconds: int,
+) -> CredentialPauseResolution | None:
+ # Check once before the sleep loop so a card response posted in the brief
+ # window before the first poll doesn't cost a full extra poll interval.
+ first = await _try_resolve_credential_response(response_key, ctx.organization_id)
+ if first != "pending":
+ return first
+
+ deadline = time.monotonic() + timeout_seconds
+ while time.monotonic() < deadline:
+ await asyncio.sleep(CREDENTIAL_RESPONSE_POLL_SECONDS)
+ # Resolve before checking disconnect: an already-posted response must win
+ # over a disconnect that happened after the POST (e.g. connect-then-refresh),
+ # not get discarded as a timeout.
+ resolved = await _try_resolve_credential_response(response_key, ctx.organization_id)
+ if resolved != "pending":
+ return resolved
+ if await stream.is_disconnected():
+ return None
+ return None
+
+
+async def maybe_credential_pause(
+ ctx: Any,
+ result: RunResultStreaming,
+ stream: EventSourceStream,
+ copilot_config: CopilotConfig,
+) -> list[dict[str, Any]] | None:
+ """Pause a finalizing turn that hit a typed mid-build credential ask.
+
+ Returns the resume messages to re-enter the loop with, or None to let
+ the caller finalize normally (kill-switch off, client can't render the
+ frame, already paused once this turn, no cache configured or not shared
+ across workers, client gone, or no typed signal fired).
+ """
+ if not credential_pause_would_fire(ctx, copilot_config):
+ return None
+ # Predicted true (the sync guard chain passed) but bailing below anyway --
+ # latch it now so this iteration can't loop back into the same prediction,
+ # and tag the outcome "declined" (distinct from "timeout") so the caller
+ # knows no frame was ever sent and can fall back to a normal nudge instead
+ # of a premature finalize. credential_pause_would_fire's own docstring notes
+ # it excludes the async-only disconnect check, which is exactly the gap here.
+ ctx.credential_pause_used = True
+ cache = getattr(app, "CACHE", None)
+ if cache is None:
+ ctx.credential_pause_outcome = "declined"
+ return None
+ if await stream.is_disconnected():
+ ctx.credential_pause_outcome = "declined"
+ return None
+ reason = credential_pause_reason(ctx)
+ if reason is None:
+ ctx.credential_pause_outcome = "declined"
+ return None
+
+ policy = getattr(ctx, "request_policy", None)
+ login_page_urls = list(policy.login_page_urls) if isinstance(policy, RequestPolicy) else []
+ credential_refs = list(policy.credential_refs) if isinstance(policy, RequestPolicy) else []
+ timeout_seconds = copilot_config.credential_pause_timeout_seconds
+ now = datetime.now(timezone.utc)
+
+ organization_id = ctx.organization_id
+ chat_id = getattr(ctx, "workflow_copilot_chat_id", None) or ""
+ turn_id = getattr(ctx, "turn_id", None) or ""
+ resume_token = _new_resume_token()
+ expires_at = now + timedelta(seconds=timeout_seconds)
+ # Establish the active-pause record before the frame carries the token: the
+ # response endpoint refuses to resolve a turn that has no pending record.
+ # expires_at is the same deadline the frame tells the client -- the record's
+ # own TTL is a separate infra grace, not a resolve-after-timeout allowance.
+ await cache.set(
+ credential_pause_active_key(organization_id, chat_id, turn_id),
+ _encode_active_pause(resume_token, expires_at),
+ ex=_credential_pause_record_ttl(timeout_seconds),
+ )
+
+ # Local import: same module-load-cycle reason as _assemble_resume_messages.
+ from skyvern.forge.sdk.copilot.enforcement import _parse_normalized_final_response
+
+ parsed = _parse_normalized_final_response(result)
+ message = str((parsed or {}).get("user_response") or "")
+
+ await stream.send(
+ WorkflowCopilotCredentialRequiredUpdate(
+ type=WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED,
+ turn_id=turn_id,
+ workflow_copilot_chat_id=chat_id,
+ resume_token=resume_token,
+ reason=reason,
+ message=message,
+ login_page_urls=login_page_urls,
+ credential_refs=credential_refs,
+ timeout_seconds=timeout_seconds,
+ expires_at=expires_at,
+ timestamp=now,
+ )
+ )
+
+ async def _invalidate_active_pause_record() -> CredentialPauseResolution | None:
+ # The waiter can exit (disconnect, genuine timeout, cancellation, or an
+ # unexpected error) well before expires_at -- invalidate the record so a
+ # late POST (e.g. the tab reconnects) gets a clear 409 instead of silently
+ # writing a response nobody will read. Uses the SAME lock
+ # resolve_credential_pause takes, with one final response check under it:
+ # a POST that validated the record as still-pending an instant before this
+ # runs would otherwise get a clean 204 for a response the waiter -- already
+ # given up -- will never read. Rescue it if it raced in.
+ lock_key = _credential_pause_lock_key(organization_id, chat_id, turn_id)
+ async with cache.get_lock(lock_key):
+ raced_in = await _try_resolve_credential_response(response_key, organization_id)
+ if isinstance(raced_in, CredentialPauseResolution):
+ return raced_in
+ await cache.set(
+ credential_pause_active_key(organization_id, chat_id, turn_id),
+ _encode_active_pause(resume_token, expires_at, consumed=True),
+ ex=_credential_pause_record_ttl(timeout_seconds),
+ )
+ return None
+
+ response_key = credential_response_cache_key(organization_id, chat_id, turn_id)
+
+ if await stream.is_disconnected():
+ # send() can return True even when the client is already gone (its own
+ # protocol contract: "queued for delivery or dropped because the client
+ # is gone") -- a disconnect racing the send itself would otherwise wait
+ # out the full timeout for a card nobody ever saw. Re-check right after
+ # send and treat it the same as the pre-send disconnect guard above.
+ await _invalidate_active_pause_record()
+ ctx.credential_pause_outcome = "declined"
+ return None
+
+ start = time.monotonic()
+ try:
+ resolution = await _wait_for_credential_response(response_key, ctx, stream, timeout_seconds)
+ except BaseException:
+ # Covers CancelledError (a direct BaseException subclass, not Exception)
+ # alongside any unexpected failure in the wait loop itself -- the frame's
+ # resume token is already live client-side either way, so the record must
+ # not be left pending for the same reason as the disconnect/timeout case.
+ # Can't act on a rescued resolution mid-unwind, only avoid corrupting state.
+ await _invalidate_active_pause_record()
+ raise
+ finally:
+ ctx.copilot_credential_pause_seconds = getattr(ctx, "copilot_credential_pause_seconds", 0.0) + (
+ time.monotonic() - start
+ )
+
+ if resolution is None:
+ resolution = await _invalidate_active_pause_record()
+ if resolution is None:
+ ctx.credential_pause_outcome = "timeout"
+ return None
+ if resolution.action == "skip":
+ ctx.credential_pause_outcome = "skipped"
+ # A missing_credential_run_failure pause means the diagnosed run left
+ # last_test_ok=False; without clearing it, the resumed reply is intercepted
+ # by the generic failed-test nudge instead of honoring the skip decision.
+ ctx.last_test_ok = None
+ return _assemble_resume_messages(ctx, _SKIP_RESUME_TEXT)
+
+ credential = resolution.credential
+ if credential is None:
+ ctx.credential_pause_outcome = "timeout"
+ return None
+ if isinstance(policy, RequestPolicy):
+ _apply_connected_credential_to_policy(ctx, policy, credential)
+ ctx.credential_pause_outcome = "connected"
+ return _assemble_resume_messages(ctx, _connected_resume_text(credential))
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 6902ce8c4..aba3eade8 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -86,6 +86,7 @@ from skyvern.forge.sdk.copilot.config import (
normalize_block_authoring_policy,
)
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
+from skyvern.forge.sdk.copilot.credential_pause import credential_pause_would_fire, maybe_credential_pause
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisRepairContract,
RepairLoopState,
@@ -946,8 +947,24 @@ def _mark_copilot_total_timeout(ctx: Any) -> None:
ctx.copilot_total_timeout_exceeded = True
+def _elapsed_run_seconds(ctx: Any, start_time: float) -> float:
+ """Wall-clock elapsed since ``start_time``, minus time spent in a credential pause.
+
+ Keeps TOTAL_TIMEOUT_SECONDS a budget over actual agent work, not real
+ time, so a paused-and-resumed turn isn't penalized for pause time.
+
+ ``pause_seconds`` is coerced defensively: tests commonly pass a bare
+ ``MagicMock()`` as ``ctx``, whose ``getattr(..., default)`` returns a
+ fresh Mock instead of the default (Mock never raises AttributeError).
+ """
+ pause_seconds = getattr(ctx, "copilot_credential_pause_seconds", 0.0)
+ if not isinstance(pause_seconds, (int, float)):
+ pause_seconds = 0.0
+ return time.monotonic() - start_time - pause_seconds
+
+
def _mark_copilot_total_timeout_if_elapsed(ctx: Any, start_time: float) -> None:
- if time.monotonic() - start_time >= TOTAL_TIMEOUT_SECONDS:
+ if _elapsed_run_seconds(ctx, start_time) >= TOTAL_TIMEOUT_SECONDS:
_mark_copilot_total_timeout(ctx)
@@ -1516,6 +1533,14 @@ def _check_enforcement(
)
raise_if_turn_halt(ctx, verified=verified)
+ # A pending credential pause pre-empts every hygiene nudge below, not just
+ # the failed-test one: a credential-blocked update_and_run_blocks call
+ # satisfies post_update (test not run) and, when the diagnosis contract
+ # is the source, the generic failed-test nudge too. None of those nudges
+ # can be acted on without the credential the pause is about to ask for.
+ if credential_pause_would_fire(ctx, config):
+ return None
+
# A permanent navigation error (DNS / cert / SSL / invalid URL) cannot be
# resolved by observing a prior navigate or by testing an updated
# workflow against the same bad URL, so let it speak first.
@@ -2095,7 +2120,7 @@ async def _run_streamed_with_deadline(
``MIN_DEADLINE_REMAINING_SECONDS`` floors ``remaining`` so
``wait_for(timeout=0)`` never panics on an already-spent budget.
"""
- elapsed = time.monotonic() - start_time
+ elapsed = _elapsed_run_seconds(ctx, start_time)
remaining = max(MIN_DEADLINE_REMAINING_SECONDS, TOTAL_TIMEOUT_SECONDS - elapsed)
result = Runner.run_streamed(agent, input=current_input, context=ctx, session=session, **runner_kwargs)
try:
@@ -2919,7 +2944,7 @@ async def run_with_enforcement(
# SSE stream silently drops events once the browser is gone, but
# the agent keeps running so the reply can be persisted to the
# chat history on the server side (see SKY-8986).
- elapsed = time.monotonic() - start_time
+ elapsed = _elapsed_run_seconds(ctx, start_time)
if elapsed > TOTAL_TIMEOUT_SECONDS:
_mark_copilot_total_timeout(ctx)
raise CopilotTotalTimeoutError()
@@ -3073,14 +3098,41 @@ async def run_with_enforcement(
synthesized_msg = _maybe_synthesized_block_offer_msg(ctx)
spine_checkpoint_nudge = False
- if nudge is None and synthesized_msg is None:
- spine_nudge = _scouted_spine_turn_end_nudge(ctx)
- if spine_nudge is None:
- _consume_pending_screenshots(ctx)
- _maybe_raise_non_retriable_nav(ctx)
- return result
- nudge = spine_nudge
- spine_checkpoint_nudge = True
+ if nudge is None:
+ # Checked whenever there's no regular nudge, even if a synthesized
+ # offer is also pending: a credential-blocked run's diagnosis can
+ # coincide with a reopened synthesized-block offer, and the pause
+ # must win so the loop doesn't send the offer instead of the card.
+ pause_used_before_this_call = getattr(ctx, "credential_pause_used", False)
+ resume_msgs = await maybe_credential_pause(ctx, result, stream, copilot_config)
+ if resume_msgs is not None:
+ current_input = (
+ resume_msgs if session is not None else _prune_input_list(result.to_input_list()) + resume_msgs
+ )
+ iteration += 1
+ continue
+ if (
+ not pause_used_before_this_call
+ and getattr(ctx, "credential_pause_used", False)
+ and getattr(ctx, "credential_pause_outcome", None) == "declined"
+ ):
+ # The latch just flipped on THIS call with no frame ever sent
+ # (disconnect, cache gone, or the reason vanished under the
+ # async-only checks credential_pause_would_fire's docstring notes
+ # it excludes) -- fall back to whatever nudge this iteration would
+ # have gotten without the pre-empt, instead of silently finalizing
+ # an uncorrected reply. Gated on the latch's own transition (not
+ # just the outcome value) so a later iteration's unrelated
+ # nudge=None doesn't re-trigger this off a stale "declined".
+ nudge = _check_enforcement(ctx, result, copilot_config)
+ if nudge is None and synthesized_msg is None:
+ spine_nudge = _scouted_spine_turn_end_nudge(ctx)
+ if spine_nudge is None:
+ _consume_pending_screenshots(ctx)
+ _maybe_raise_non_retriable_nav(ctx)
+ return result
+ nudge = spine_nudge
+ spine_checkpoint_nudge = True
if nudge is not None and not spine_checkpoint_nudge and nudge == _nudge(copilot_config, "post_update"):
if ctx.post_update_nudge_count >= MAX_POST_UPDATE_NUDGES:
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 36a21c538..33a743cd3 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -615,6 +615,12 @@ async def update_and_run_blocks_tool(
"parameters": parameters or {},
}
skip_run_after_update = _request_policy_allows_update_and_skip_run(copilot_ctx, "update_and_run_blocks")
+ # Cleared unconditionally up front and only set True at the actual skip
+ # branch below — reflects "we skipped a run", not "the policy would have
+ # allowed a skip if we got that far". A stale True from an earlier call, or
+ # a premature True from a policy check ahead of an unrelated update_workflow
+ # failure, would misreport an authoring error as a credential ask.
+ copilot_ctx.last_run_skipped_unbound_credentials = False
authority_error = _authority_tool_error(
copilot_ctx,
"update_and_run_blocks",
@@ -721,6 +727,7 @@ async def update_and_run_blocks_tool(
return json.dumps(sanitized)
if skip_run_after_update:
+ copilot_ctx.last_run_skipped_unbound_credentials = True
skip_message = "Skipped test run: required credentials are not configured."
skip_result = {
"ok": True,
diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.py
index 378c04a9f..9094efaad 100644
--- a/skyvern/forge/sdk/copilot/tools/_shared.py
+++ b/skyvern/forge/sdk/copilot/tools/_shared.py
@@ -2,7 +2,6 @@ from __future__ import annotations
import asyncio
import json
-import time
from collections.abc import Mapping
from typing import Any
from urllib.parse import urlparse
@@ -26,7 +25,7 @@ from skyvern.forge.sdk.copilot.composition_browser_expressions import (
)
from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema, parse_composition_structured
from skyvern.forge.sdk.copilot.context import CopilotContext
-from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
+from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS, _elapsed_run_seconds
from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.task_output_envelope import (
_TASK_ENVELOPE_BLOCK_TYPES,
@@ -270,7 +269,7 @@ def _copilot_seconds_remaining(ctx: AgentContext) -> float | None:
started_at = getattr(ctx, "copilot_run_start_monotonic", None)
if not isinstance(started_at, int | float):
return None
- return TOTAL_TIMEOUT_SECONDS - (time.monotonic() - float(started_at))
+ return TOTAL_TIMEOUT_SECONDS - _elapsed_run_seconds(ctx, float(started_at))
def _same_page_ignoring_fragment(left: str | None, right: str | None) -> bool:
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 4d37e170a..9caadb4a1 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -42,6 +42,11 @@ from skyvern.forge.sdk.copilot.context import (
render_loaded_result_context_for_prompt,
sanitize_global_llm_context_for_prompt,
)
+from skyvern.forge.sdk.copilot.credential_pause import (
+ CredentialPauseRejection,
+ check_credential_pause_resumable,
+ resolve_credential_pause,
+)
from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failure
from skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handler
from skyvern.forge.sdk.copilot.output_utils import truncate_output
@@ -77,6 +82,7 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatSender,
WorkflowCopilotChatSummary,
WorkflowCopilotClearProposedWorkflowRequest,
+ WorkflowCopilotCredentialResponseRequest,
WorkflowCopilotProcessingUpdate,
WorkflowCopilotStreamErrorUpdate,
WorkflowCopilotStreamMessageType,
@@ -2352,6 +2358,72 @@ async def workflow_copilot_cancel(
)
+@base_router.post(
+ "/workflow/copilot/credential-response", include_in_schema=False, status_code=status.HTTP_204_NO_CONTENT
+)
+async def workflow_copilot_credential_response(
+ response_request: WorkflowCopilotCredentialResponseRequest,
+ organization: Organization = Depends(org_auth_service.get_current_org),
+) -> None:
+ """Resume a turn paused on ``credential_required`` with the user's card response.
+
+ The resume path is not authorized by org auth + ``turn_id`` alone: the caller
+ must echo back the one-time ``resume_token`` and ``workflow_copilot_chat_id``
+ from the frame, which ``resolve_credential_pause`` checks against a still-pending
+ active-pause record before it writes the flag the paused loop polls. Returns
+ 503 when ``app.CACHE`` is absent, 422 when ``action="connected"`` omits a
+ ``credential_id``, 404 when that ID doesn't resolve in this organization or no
+ active pause matches, 403 on a bad token, and 409 once the pause is consumed.
+ """
+ cache = getattr(app, "CACHE", None)
+ if cache is None:
+ raise HTTPException(
+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+ detail="Credential response not supported in this environment",
+ )
+
+ # Validate the resume token BEFORE the credential-id lookup below: org auth
+ # alone doesn't authorize this turn, and checking token validity first avoids
+ # using an unauthorized-for-this-turn caller's credential_id as a small
+ # authenticated existence oracle (a bogus token still gets a real 404/not-found).
+ try:
+ await check_credential_pause_resumable(
+ cache,
+ organization_id=organization.organization_id,
+ workflow_copilot_chat_id=response_request.workflow_copilot_chat_id,
+ turn_id=response_request.turn_id,
+ resume_token=response_request.resume_token,
+ )
+ except CredentialPauseRejection as rejection:
+ raise HTTPException(status_code=rejection.status_code, detail=rejection.detail)
+
+ credential_id = response_request.credential_id
+ if response_request.action == "connected":
+ if not credential_id:
+ raise HTTPException(
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+ detail="credential_id is required when action is 'connected'",
+ )
+ existing = await app.DATABASE.credentials.get_credentials_by_ids(
+ [credential_id], organization_id=organization.organization_id
+ )
+ if not existing:
+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Credential {credential_id} not found")
+
+ try:
+ await resolve_credential_pause(
+ cache,
+ organization_id=organization.organization_id,
+ workflow_copilot_chat_id=response_request.workflow_copilot_chat_id,
+ turn_id=response_request.turn_id,
+ resume_token=response_request.resume_token,
+ action=response_request.action,
+ credential_id=credential_id,
+ )
+ except CredentialPauseRejection as rejection:
+ raise HTTPException(status_code=rejection.status_code, detail=rejection.detail)
+
+
@base_router.post(
"/workflow/copilot/clear-proposed-workflow", include_in_schema=False, status_code=status.HTTP_204_NO_CONTENT
)
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index 3436c0b0b..89cc8ac9b 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -124,6 +124,14 @@ class WorkflowCopilotChatRequest(BaseModel):
"otherwise it is a no-op."
),
)
+ supports_credential_pause: bool = Field(
+ False,
+ description=(
+ "True when the client can render the credential_required frame. Older clients default to False "
+ "and silently drop unknown frame types, so the backend must not pause for them even if the "
+ "server-side flag is on."
+ ),
+ )
keep_pending_proposal: bool = Field(
False,
description=(
@@ -137,6 +145,14 @@ class WorkflowCopilotCancelRequest(BaseModel):
cancel_token: str = Field(..., description="The cancel_token sent on the original /chat-post request")
+class WorkflowCopilotCredentialResponseRequest(BaseModel):
+ turn_id: str = Field(..., description="turn_id from the matching credential_required frame")
+ workflow_copilot_chat_id: str = Field(..., description="chat ID from the matching credential_required frame")
+ resume_token: str = Field(..., description="One-time resume token from the matching credential_required frame")
+ action: Literal["connected", "skip"] = Field(..., description="The user's response to the credential card")
+ credential_id: str | None = Field(None, description="Saved credential ID; required when action is 'connected'")
+
+
class WorkflowCopilotClearProposedWorkflowRequest(BaseModel):
workflow_copilot_chat_id: str = Field(..., description="The chat ID to update")
auto_accept: bool = Field(..., description="Whether to auto-accept future workflow updates")
@@ -199,6 +215,7 @@ class WorkflowCopilotStreamMessageType(StrEnum):
DESIGN_START = "design_start"
DESIGN_END = "design_end"
WORKFLOW_DRAFT = "workflow_draft"
+ CREDENTIAL_REQUIRED = "credential_required"
CODEGEN_PROGRESS = "codegen_progress"
@@ -406,6 +423,26 @@ class WorkflowCopilotWorkflowDraftUpdate(BaseModel):
)
+class WorkflowCopilotCredentialRequiredUpdate(BaseModel):
+ type: WorkflowCopilotStreamMessageType = Field(
+ WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED, description="Message type"
+ )
+ turn_id: str = Field(..., description="UUID for the paused turn; correlates with credential-response POSTs")
+ workflow_copilot_chat_id: str = Field(..., description="The chat ID")
+ resume_token: str = Field(..., description="One-time token the credential-response POST must echo back to resume")
+ reason: Literal[
+ "workflow_credential_inputs_unbound",
+ "missing_credential_run_failure",
+ "credential_deferred_draft",
+ ] = Field(..., description="Typed signal that triggered the pause")
+ message: str = Field(..., description="The agent's explanatory text at the moment of pausing")
+ login_page_urls: list[str] = Field(default_factory=list, description="Candidate login page URLs, if known")
+ credential_refs: list[str] = Field(default_factory=list, description="Credential IDs or names referenced")
+ timeout_seconds: int = Field(..., description="How long the backend will wait before degrading to terminal")
+ expires_at: datetime = Field(..., description="Server time after which the pause degrades to terminal")
+ timestamp: datetime = Field(..., description="Server timestamp")
+
+
class WorkflowCopilotCodegenProgressUpdate(BaseModel):
"""Live-only drafting progress while the LLM streams authoring-tool arguments. Not persisted; the
workflow_draft frame supersedes it."""
diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.py
new file mode 100644
index 000000000..1a8719f71
--- /dev/null
+++ b/tests/unit/test_copilot_credential_pause.py
@@ -0,0 +1,1835 @@
+"""Tests for the copilot mid-build credential pause-and-wait (SKY-12138).
+
+Covers:
+
+- ``credential_pause_reason`` fires only on the three typed mid-build signals
+ and never on ``credential_prompt_reason``'s text-marker tier.
+- ``maybe_credential_pause``'s waiter: connect mutates ``RequestPolicy`` and
+ resolves, skip leaves the policy untouched, timeout/disconnect/no-cache
+ degrade to None without sending a frame, an invalid/foreign credential id
+ degrades rather than crashing, and CancelledError always propagates.
+- ``run_with_enforcement`` loop integration: a typed signal at finalize sends
+ exactly one ``credential_required`` frame and re-enters the loop with the
+ resume message, instead of returning on the first finalize.
+- Pause time is credited back against ``TOTAL_TIMEOUT_SECONDS`` so a slow
+ pause doesn't trip the total-timeout on the resumed iteration.
+- The one-pause-per-turn latch and kill-switch-off parity.
+- The ``/workflow/copilot/credential-response`` route's validation and TTL.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from agents import RunConfig
+from fastapi import HTTPException, status
+
+from skyvern.config import settings
+from skyvern.forge import app
+from skyvern.forge.sdk.cache.base import NoopLock
+from skyvern.forge.sdk.copilot import credential_pause as credential_pause_module
+from skyvern.forge.sdk.copilot import tools as tools_module
+from skyvern.forge.sdk.copilot.config import CopilotConfig
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.credential_pause import (
+ CredentialPauseRejection,
+ CredentialPauseResolution,
+ _encode_active_pause,
+ credential_pause_active_key,
+ credential_response_cache_key,
+ encode_credential_response,
+ maybe_credential_pause,
+ resolve_credential_pause,
+)
+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
+ DiagnosisFailureType,
+ DiagnosisInput,
+ DiagnosisRepairContract,
+ DiagnosisResult,
+ RepairDecision,
+ RepairNextAction,
+ RepairRootCauseIdentity,
+ VerificationResult,
+)
+from skyvern.forge.sdk.copilot.enforcement import (
+ NUDGE_SENTINEL,
+ _check_enforcement,
+ _elapsed_run_seconds,
+ run_with_enforcement,
+)
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, credential_prompt_reason
+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
+from skyvern.forge.sdk.routes.workflow_copilot import (
+ WorkflowCopilotCredentialResponseRequest,
+ workflow_copilot_credential_response,
+)
+from skyvern.forge.sdk.schemas.credentials import Credential, CredentialType, CredentialVaultType
+from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotStreamMessageType
+from tests.unit.conftest import make_copilot_context
+
+
+def _repair_contract(
+ next_action: Any,
+ failure_type: Any = DiagnosisFailureType.UNKNOWN,
+ categories: tuple[str, ...] = ("CREDENTIAL_ERROR",),
+) -> DiagnosisRepairContract:
+ return DiagnosisRepairContract(
+ diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),
+ diagnosis_result=DiagnosisResult(
+ suspected_failure_type=failure_type,
+ root_cause_identity=RepairRootCauseIdentity(failure_categories=categories),
+ ),
+ repair_decision=RepairDecision(next_action=next_action),
+ verification_result=VerificationResult(),
+ )
+
+
+def _make_credential(credential_id: str = "cred_1", name: str = "Example Login") -> Credential:
+ now = datetime(2026, 1, 1, tzinfo=timezone.utc)
+ return Credential(
+ credential_id=credential_id,
+ organization_id="org-1",
+ name=name,
+ vault_type=CredentialVaultType.SKYVERN,
+ item_id="item_1",
+ credential_type=CredentialType.PASSWORD,
+ username="[email protected]",
+ card_last4=None,
+ card_brand=None,
+ created_at=now,
+ modified_at=now,
+ )
+
+
+class _FakeCache:
+ """Minimal in-memory double of the ``get`` / ``set`` / ``get_lock`` surface of app.CACHE.
+
+ ``is_shared = True`` by default (stands in for Redis); flip an instance's
+ to False to simulate a LocalCache-shaped non-shared cache.
+ """
+
+ is_shared = True
+
+ def __init__(self) -> None:
+ self.store: dict[str, Any] = {}
+ self.set_calls: list[tuple[str, Any, Any]] = []
+
+ async def get(self, key: str) -> Any:
+ return self.store.get(key)
+
+ async def set(self, key: str, value: Any, ex: Any = None) -> None:
+ self.store[key] = value
+ self.set_calls.append((key, value, ex))
+
+ def get_lock(self, lock_name: str, blocking_timeout: int = 5, timeout: int = 10) -> NoopLock:
+ return NoopLock(lock_name, blocking_timeout, timeout)
+
+
+def _seed_active_pause(
+ cache: _FakeCache,
+ org: str,
+ chat: str,
+ turn: str,
+ token: str,
+ *,
+ expires_at: datetime | None = None,
+) -> None:
+ """Write a pending active-pause record the way maybe_credential_pause would."""
+ cache.store[credential_pause_active_key(org, chat, turn)] = _encode_active_pause(
+ token, expires_at or (datetime.now(timezone.utc) + timedelta(minutes=5))
+ )
+
+
+def _make_stream(*, disconnected: bool = False) -> MagicMock:
+ stream = MagicMock()
+ stream.send = AsyncMock(return_value=True)
+ stream.is_disconnected = AsyncMock(return_value=disconnected)
+ return stream
+
+
+def _fake_result() -> MagicMock:
+ result = MagicMock()
+ result.final_output = "Done."
+ result.new_items = []
+ result.to_input_list.return_value = []
+ return result
+
+
+# ---------------------------------------------------------------------------
+# 1 - credential_pause_reason detector
+# ---------------------------------------------------------------------------
+
+
+def test_reason_fires_on_skipped_unbound_credentials() -> None:
+ ctx = SimpleNamespace(last_run_skipped_unbound_credentials=True)
+ assert credential_pause_module.credential_pause_reason(ctx) == "workflow_credential_inputs_unbound"
+
+
+def test_reason_ignores_skipped_unbound_credentials_when_user_explicitly_skipped_testing() -> None:
+ """_request_policy_allows_update_and_skip_run (guardrails.py) skips the run for a
+ skip_test-deferred credential too -- pausing here would contradict the same explicit
+ request the credential_deferred_draft branch already excludes skip_test for."""
+ policy = RequestPolicy(testing_intent="skip_test")
+ ctx = SimpleNamespace(last_run_skipped_unbound_credentials=True, request_policy=policy)
+ assert credential_pause_module.credential_pause_reason(ctx) is None
+
+
+def test_reason_fires_on_missing_credential_run_failure() -> None:
+ ctx = SimpleNamespace(
+ last_run_skipped_unbound_credentials=False,
+ latest_diagnosis_repair_contract=_repair_contract(
+ RepairNextAction.ASK, DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ ),
+ )
+ assert credential_pause_module.credential_pause_reason(ctx) == "missing_credential_run_failure"
+
+
+def test_reason_does_not_fire_when_ask_is_for_a_different_failure_type() -> None:
+ ctx = SimpleNamespace(
+ last_run_skipped_unbound_credentials=False,
+ latest_diagnosis_repair_contract=_repair_contract(RepairNextAction.ASK, DiagnosisFailureType.UNKNOWN),
+ )
+ assert credential_pause_module.credential_pause_reason(ctx) is None
+
+
+def test_reason_ignores_missing_credential_or_init_when_not_categorized_as_credential_error() -> None:
+ """MISSING_CREDENTIAL_OR_INIT also covers param-binding/lookup failures; only the
+ CREDENTIAL_ERROR category should preempt with a card that can't unblock those."""
+ ctx = SimpleNamespace(
+ last_run_skipped_unbound_credentials=False,
+ latest_diagnosis_repair_contract=_repair_contract(
+ RepairNextAction.ASK,
+ DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT,
+ categories=("PARAMETER_BINDING_ERROR",),
+ ),
+ )
+ assert credential_pause_module.credential_pause_reason(ctx) is None
+
+
+def test_reason_fires_on_credential_deferred_draft() -> None:
+ policy = RequestPolicy(credential_draft_deferred_explicitly=True)
+ ctx = SimpleNamespace(
+ last_run_skipped_unbound_credentials=False,
+ latest_diagnosis_repair_contract=None,
+ request_policy=policy,
+ update_workflow_called=True,
+ )
+ assert credential_pause_module.credential_pause_reason(ctx) == "credential_deferred_draft"
+
+
+def test_reason_requires_update_workflow_called_for_deferred_draft() -> None:
+ policy = RequestPolicy(credential_draft_deferred_explicitly=True)
+ ctx = SimpleNamespace(
+ last_run_skipped_unbound_credentials=False,
+ latest_diagnosis_repair_contract=None,
+ request_policy=policy,
+ update_workflow_called=False,
+ )
+ assert credential_pause_module.credential_pause_reason(ctx) is None
+
+
+def test_reason_ignores_deferred_draft_when_user_explicitly_skipped_testing() -> None:
+ """skip_test means the user already said not to run/verify this -- pausing to
+ ask for a credential would contradict that explicit request."""
+ policy = RequestPolicy(credential_draft_deferred_explicitly=True, testing_intent="skip_test")
+ ctx = SimpleNamespace(
+ last_run_skipped_unbound_credentials=False,
+ latest_diagnosis_repair_contract=None,
+ request_policy=policy,
+ update_workflow_called=True,
+ )
+ assert credential_pause_module.credential_pause_reason(ctx) is None
+
+
+def test_reason_ignores_text_marker_tier_that_credential_prompt_reason_catches() -> None:
+ """Pins the SKY-11988 false-positive lesson: no text-marker fallback here."""
+ policy = RequestPolicy()
+ final_text = "I couldn't test this. Please add the credential via the Credentials UI."
+
+ # The sibling function DOES classify this via its text-marker tier.
+ assert credential_prompt_reason(policy, final_text) == "assistant_directed"
+
+ ctx = SimpleNamespace(
+ last_run_skipped_unbound_credentials=False,
+ latest_diagnosis_repair_contract=None,
+ request_policy=policy,
+ update_workflow_called=True,
+ )
+ assert credential_pause_module.credential_pause_reason(ctx) is None
+
+
+def test_reason_none_when_no_signals_present() -> None:
+ ctx = make_copilot_context()
+ assert credential_pause_module.credential_pause_reason(ctx) is None
+
+
+# ---------------------------------------------------------------------------
+# 2 - maybe_credential_pause waiter
+# ---------------------------------------------------------------------------
+
+
+async def test_connected_action_mutates_policy_and_resolves(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-1"
+ ctx.client_supports_credential_pause = True
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")] = encode_credential_response(
+ "connected", "cred_1"
+ )
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ credential = _make_credential()
+ monkeypatch.setattr(
+ credential_pause_module.app,
+ "DATABASE",
+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[credential]))),
+ )
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert resume_msgs is not None
+ assert ctx.credential_pause_outcome == "connected"
+ assert ctx.request_policy.resolved_credentials == [credential]
+ assert ctx.request_policy.allow_run_blocks is True
+ assert ctx.request_policy.clarification_reason == "none"
+ assert ctx.request_policy.allow_missing_credentials_in_draft is False
+ assert ctx.request_policy.requires_user_clarification is False
+ resume_text = resume_msgs[-1]["content"]
+ assert resume_text.startswith(NUDGE_SENTINEL)
+ assert "cred_1" in resume_text
+ sent_types = [call.args[0].type for call in stream.send.await_args_list]
+ assert sent_types == [WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED]
+
+
+async def test_connected_action_unlatches_test_after_update_done(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The skipped run before the pause already stamped test_after_update_done=True;
+ a connected resume must reset it so a reply that never re-runs the blocks still
+ gets forced back through update_and_run_blocks by the post_update nudge."""
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-1"
+ ctx.client_supports_credential_pause = True
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+ ctx.update_workflow_called = True
+ ctx.test_after_update_done = True
+
+ cache = _FakeCache()
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")] = encode_credential_response(
+ "connected", "cred_1"
+ )
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(
+ credential_pause_module.app,
+ "DATABASE",
+ SimpleNamespace(
+ credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[_make_credential()]))
+ ),
+ )
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+
+ resume_msgs = await maybe_credential_pause(
+ ctx,
+ _fake_result(),
+ _make_stream(),
+ CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5),
+ )
+
+ assert resume_msgs is not None
+ assert ctx.test_after_update_done is False
+
+
+async def test_skip_action_leaves_policy_untouched(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-2"
+ ctx.client_supports_credential_pause = True
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-2")] = encode_credential_response("skip", None)
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert resume_msgs is not None
+ assert ctx.credential_pause_outcome == "skipped"
+ assert ctx.request_policy.resolved_credentials == []
+ assert ctx.request_policy == RequestPolicy()
+ assert "chose not to connect a credential now" in resume_msgs[-1]["content"]
+
+
+async def test_skip_clears_stale_last_test_ok_from_the_diagnosed_failure(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A missing_credential_run_failure pause means the diagnosed run left
+ last_test_ok=False; skip must clear it or the resumed reply is intercepted
+ by the generic failed-test nudge instead of honoring the skip decision."""
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-skip-clears"
+ ctx.client_supports_credential_pause = True
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.last_test_ok = False
+ ctx.test_after_update_done = True
+ ctx.latest_diagnosis_repair_contract = _repair_contract(
+ RepairNextAction.ASK, DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ )
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-skip-clears")] = encode_credential_response(
+ "skip", None
+ )
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert resume_msgs is not None
+ assert ctx.last_test_ok is None
+ assert _check_enforcement(ctx, result=None, config=config) is None
+
+
+async def test_timeout_returns_none_and_marks_outcome(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = make_copilot_context()
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache() # never populated
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=0)
+
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert resume_msgs is None
+ assert ctx.credential_pause_outcome == "timeout"
+ assert ctx.copilot_credential_pause_seconds >= 0.0
+
+
+async def test_response_racing_in_at_the_timeout_instant_is_rescued(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Fails on old code: resolve_credential_pause writes under a per-turn lock,
+ but the timeout invalidate wrote the same active-pause key unlocked -- a
+ response that lands in the same instant the waiter gives up would validate
+ against the still-pending record (204 to the client) yet never get read,
+ since the waiter already returned None. The invalidate must re-check under
+ the same lock and use the response if one raced in, not silently drop it."""
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-race"
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+
+ async def fake_wait_races_in_a_response(response_key: str, ctx: Any, stream: Any, timeout_seconds: int) -> None:
+ # Simulates resolve_credential_pause's lock-protected write landing in
+ # the same instant the waiter's own poll loop concludes None.
+ cache.store[response_key] = encode_credential_response("skip", None)
+ return None
+
+ monkeypatch.setattr(credential_pause_module, "_wait_for_credential_response", fake_wait_races_in_a_response)
+
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert resume_msgs is not None
+ assert ctx.credential_pause_outcome == "skipped"
+
+
+async def test_client_disconnect_mid_wait_degrades_early(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = make_copilot_context()
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache() # never populated
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+
+ stream = MagicMock()
+ stream.send = AsyncMock(return_value=True)
+ # Connected for the initial guard and the post-send re-check, gone by the first poll.
+ stream.is_disconnected = AsyncMock(side_effect=[False, False, True])
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
+
+ start = time.monotonic()
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+ elapsed = time.monotonic() - start
+
+ assert resume_msgs is None
+ assert ctx.credential_pause_outcome == "timeout"
+ assert elapsed < 1.0 # degraded on the first poll, not the full 30s timeout
+
+
+async def test_disconnect_racing_the_send_itself_declines_without_waiting(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Fails on old code: send() returns True even when the client is already
+ gone (its own contract), so a disconnect that lands between the pre-send
+ guard and delivery would otherwise wait out the full timeout for a card
+ nobody ever saw, and land on 'timeout' instead of 'declined'."""
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-race-send"
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+
+ stream = MagicMock()
+ stream.send = AsyncMock(return_value=True)
+ # Connected for the pre-send guard, gone by the time send() returns.
+ stream.is_disconnected = AsyncMock(side_effect=[False, True])
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
+
+ start = time.monotonic()
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+ elapsed = time.monotonic() - start
+
+ assert resume_msgs is None
+ assert ctx.credential_pause_outcome == "declined"
+ assert elapsed < 1.0 # never entered the wait loop
+
+
+async def test_already_disconnected_before_send_declines_without_sending_a_frame(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The predictor (credential_pause_would_fire) excludes the async-only
+ disconnect check by design -- a client gone before the frame is even sent
+ must latch and tag 'declined' (not 'timeout') so the caller can fall back
+ to a normal nudge instead of treating this like a delivered-and-waited pause."""
+ ctx = make_copilot_context()
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+
+ stream = _make_stream(disconnected=True)
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
+
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert resume_msgs is None
+ assert ctx.credential_pause_outcome == "declined"
+ assert ctx.credential_pause_used is True
+ stream.send.assert_not_called()
+
+
+async def test_disconnect_invalidates_the_active_pause_record(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A disconnect degrades well before expires_at; a late POST after the tab
+ reconnects must be rejected instead of silently writing an unread response."""
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-disconnect"
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+
+ stream = MagicMock()
+ stream.send = AsyncMock(return_value=True)
+ stream.is_disconnected = AsyncMock(side_effect=[False, True])
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
+
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+ assert resume_msgs is None
+
+ frame = stream.send.await_args_list[0].args[0]
+ with pytest.raises(CredentialPauseRejection) as excinfo:
+ await resolve_credential_pause(
+ cache,
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ turn_id="turn-disconnect",
+ resume_token=frame.resume_token,
+ action="skip",
+ credential_id=None,
+ )
+ assert excinfo.value.status_code == status.HTTP_409_CONFLICT
+
+
+async def test_no_cache_returns_none_without_sending_a_frame(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(app._inst, "CACHE", None, raising=False)
+ ctx = make_copilot_context()
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True)
+
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert resume_msgs is None
+ stream.send.assert_not_awaited()
+
+
+async def test_cancelled_error_propagates_and_still_accumulates_pause_seconds(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-cancel"
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+
+ async def _raise_cancelled(*args: object, **kwargs: object) -> None:
+ raise asyncio.CancelledError()
+
+ monkeypatch.setattr(credential_pause_module.asyncio, "sleep", _raise_cancelled)
+
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
+
+ with pytest.raises(asyncio.CancelledError):
+ await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert ctx.copilot_credential_pause_seconds > 0.0
+ assert ctx.credential_pause_outcome is None # neither timeout nor resolved path ran
+
+ frame = stream.send.await_args_list[0].args[0]
+ with pytest.raises(CredentialPauseRejection) as excinfo:
+ await resolve_credential_pause(
+ cache,
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ turn_id="turn-cancel",
+ resume_token=frame.resume_token,
+ action="skip",
+ credential_id=None,
+ )
+ assert excinfo.value.status_code == status.HTTP_409_CONFLICT
+
+
+async def test_unexpected_error_in_waiter_also_invalidates_the_active_pause_record(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Not just CancelledError: any unexpected failure in the wait loop leaves the
+ frame's resume token live client-side, so the record must be invalidated too."""
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-crash"
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+
+ async def _raise_runtime_error(*args: object, **kwargs: object) -> None:
+ raise RuntimeError("cache connection reset")
+
+ monkeypatch.setattr(credential_pause_module.asyncio, "sleep", _raise_runtime_error)
+
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
+
+ with pytest.raises(RuntimeError):
+ await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ frame = stream.send.await_args_list[0].args[0]
+ with pytest.raises(CredentialPauseRejection) as excinfo:
+ await resolve_credential_pause(
+ cache,
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ turn_id="turn-crash",
+ resume_token=frame.resume_token,
+ action="skip",
+ credential_id=None,
+ )
+ assert excinfo.value.status_code == status.HTTP_409_CONFLICT
+
+
+async def test_invalid_credential_id_degrades_instead_of_crashing(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-3"
+ ctx.client_supports_credential_pause = True
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-3")] = encode_credential_response(
+ "connected", "cred_foreign"
+ )
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(
+ credential_pause_module.app,
+ "DATABASE",
+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[]))),
+ )
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert resume_msgs is None
+ assert ctx.credential_pause_outcome == "timeout"
+ assert ctx.request_policy.resolved_credentials == []
+
+
+async def test_full_round_trip_binds_frame_token_to_active_pause(monkeypatch: pytest.MonkeyPatch) -> None:
+ """End-to-end: establishing the pause writes an active record + one-time token,
+ the response endpoint resolves only with that token, and the resolved pause is
+ consumed so a replay conflicts.
+ """
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-rt"
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+ active_key = credential_pause_active_key("org-1", "chat-1", "turn-rt")
+
+ async def _respond_using_frame_token() -> None:
+ # Read the one-time token exactly as the FE would from the frame, then resolve.
+ for _ in range(2000):
+ raw = await cache.get(active_key)
+ record = json.loads(raw) if raw else None
+ if record and record.get("status") == "pending":
+ await resolve_credential_pause(
+ cache,
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ turn_id="turn-rt",
+ resume_token=record["resume_token"],
+ action="skip",
+ credential_id=None,
+ )
+ return
+ await asyncio.sleep(0.001)
+
+ responder = asyncio.ensure_future(_respond_using_frame_token())
+ try:
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+ finally:
+ await responder
+
+ assert resume_msgs is not None
+ assert ctx.credential_pause_outcome == "skipped"
+ frame = stream.send.await_args_list[0].args[0]
+ assert frame.resume_token
+
+ with pytest.raises(CredentialPauseRejection) as excinfo:
+ await resolve_credential_pause(
+ cache,
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ turn_id="turn-rt",
+ resume_token=frame.resume_token,
+ action="skip",
+ credential_id=None,
+ )
+ assert excinfo.value.status_code == status.HTTP_409_CONFLICT
+
+
+# ---------------------------------------------------------------------------
+# 5 - one-pause-per-turn latch / 6 - kill-switch-off parity
+# ---------------------------------------------------------------------------
+
+
+async def test_latch_prevents_second_pause_this_turn() -> None:
+ ctx = make_copilot_context()
+ ctx.credential_pause_used = True
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True)
+
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert resume_msgs is None
+ stream.send.assert_not_awaited()
+
+
+async def test_kill_switch_off_returns_none_without_touching_cache_or_stream() -> None:
+ ctx = make_copilot_context()
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=False)
+
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert resume_msgs is None
+ stream.send.assert_not_awaited()
+ stream.is_disconnected.assert_not_awaited()
+
+
+# ---------------------------------------------------------------------------
+# 3 - run_with_enforcement loop integration (regression pin)
+# ---------------------------------------------------------------------------
+
+
+async def test_loop_pauses_at_finalize_and_resumes_same_turn(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Fails on old code: without the pause hook this returns on the first finalize."""
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-loop"
+ ctx.client_supports_credential_pause = True
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-loop")] = encode_credential_response(
+ "skip", None
+ )
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+
+ stream = _make_stream()
+ fake_result = _fake_result()
+ calls: list[dict[str, Any]] = []
+
+ def fake_run_streamed(*args: Any, **kwargs: Any) -> Any:
+ calls.append(kwargs)
+ return fake_result
+
+ async def fake_stream_to_sse(result: Any, s: Any, c: Any) -> None:
+ return None
+
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", fake_run_streamed)
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse", fake_stream_to_sse)
+
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ returned = await run_with_enforcement(
+ agent=MagicMock(),
+ initial_input="hello",
+ ctx=ctx,
+ stream=stream,
+ run_config=RunConfig(),
+ copilot_config=config,
+ )
+
+ assert returned is fake_result
+ assert len(calls) == 2, "second Runner invocation must happen after the pause resumes"
+ second_call_input = calls[1]["input"]
+ assert any(NUDGE_SENTINEL in item.get("content", "") for item in second_call_input)
+ assert ctx.credential_pause_used is True
+ frame_types = [call.args[0].type for call in stream.send.await_args_list]
+ assert frame_types.count(WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED) == 1
+
+
+async def test_credential_pause_preempts_a_concurrent_synthesized_offer(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Fails on old code: the finalize branch only tried the pause when nudge AND
+ synthesized_msg were both None, so a reopened synthesized-block offer coinciding
+ with a credential diagnosis sent the offer instead of the credential_required frame.
+ """
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-offer-race"
+ ctx.client_supports_credential_pause = True
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-offer-race")] = encode_credential_response(
+ "skip", None
+ )
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+
+ stream = _make_stream()
+ fake_result = _fake_result()
+ calls: list[dict[str, Any]] = []
+
+ def fake_synthesized_offer(_ctx: Any) -> dict[str, Any] | None:
+ # _check_enforcement runs after the Runner call, so `calls` already has
+ # 1 entry on the pre-pause finalize check; gone by the post-resume check
+ # (2 entries), so the test isolates the preemption itself.
+ return {"role": "user", "content": "offer to add the reopened block"} if len(calls) <= 1 else None
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.enforcement._maybe_synthesized_block_offer_msg",
+ fake_synthesized_offer,
+ )
+
+ def fake_run_streamed(*args: Any, **kwargs: Any) -> Any:
+ calls.append(kwargs)
+ return fake_result
+
+ async def fake_stream_to_sse(result: Any, s: Any, c: Any) -> None:
+ return None
+
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", fake_run_streamed)
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse", fake_stream_to_sse)
+
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ await run_with_enforcement(
+ agent=MagicMock(),
+ initial_input="hello",
+ ctx=ctx,
+ stream=stream,
+ run_config=RunConfig(),
+ copilot_config=config,
+ )
+
+ frame_types = [call.args[0].type for call in stream.send.await_args_list]
+ assert frame_types.count(WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED) == 1
+ # On old code the offer nudge wins the first check (pause only fires once the
+ # offer stops recurring), costing an extra Runner call before the skip resume
+ # text appears -- this pins that the pause wins on the very first check.
+ assert len(calls) == 2
+ second_call_input = calls[1]["input"]
+ assert any("chose not to connect a credential now" in item.get("content", "") for item in second_call_input)
+
+
+async def test_declined_pause_falls_back_to_the_normal_nudge_instead_of_finalizing(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Fails on old code: an already-disconnected stream made maybe_credential_pause
+ return None with nothing sent, and run_with_enforcement finalized the turn as-is
+ instead of the post_update nudge this iteration would otherwise have gotten."""
+ ctx = make_copilot_context()
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.update_workflow_called = True
+ ctx.test_after_update_done = False
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+
+ stream = _make_stream(disconnected=True)
+ fake_result = _fake_result()
+ calls: list[dict[str, Any]] = []
+
+ def fake_run_streamed(*args: Any, **kwargs: Any) -> Any:
+ calls.append(kwargs)
+ return fake_result
+
+ async def fake_stream_to_sse(result: Any, s: Any, c: Any) -> None:
+ return None
+
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", fake_run_streamed)
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse", fake_stream_to_sse)
+
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
+
+ await run_with_enforcement(
+ agent=MagicMock(),
+ initial_input="hello",
+ ctx=ctx,
+ stream=stream,
+ run_config=RunConfig(),
+ copilot_config=config,
+ )
+
+ assert ctx.credential_pause_outcome == "declined"
+ stream.send.assert_not_called()
+ # On old code this returns after the first call (nudge=None, pause declined,
+ # synthesized_msg=None -> immediate finalize). The fallback must fire the
+ # post_update nudge instead; MAX_POST_UPDATE_NUDGES allows retries before the
+ # fake (which never "tests" anything) exhausts them, so >1 call is the claim.
+ assert len(calls) > 1, "the post_update nudge must fire instead of an immediate finalize"
+ second_call_input = calls[1]["input"]
+ assert any("did not test it" in item.get("content", "") for item in second_call_input)
+
+
+# ---------------------------------------------------------------------------
+# 4 - timeout credit
+# ---------------------------------------------------------------------------
+
+
+def test_elapsed_run_seconds_subtracts_pause_time() -> None:
+ ctx = SimpleNamespace(copilot_credential_pause_seconds=50.0)
+ start_time = time.monotonic() - 60.0
+
+ elapsed = _elapsed_run_seconds(ctx, start_time)
+
+ assert 9.0 < elapsed < 11.0
+
+
+async def test_paused_loop_does_not_trip_total_timeout_on_resume(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Fails on old code: a slow pause would consume the (tiny) real timeout budget."""
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 0.15)
+
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-credit"
+ ctx.client_supports_credential_pause = True
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.2)
+
+ async def _populate_after_first_poll() -> None:
+ await asyncio.sleep(0.25)
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-credit")] = encode_credential_response(
+ "skip", None
+ )
+
+ stream = _make_stream()
+ fake_result = _fake_result()
+ calls: list[dict[str, Any]] = []
+
+ def fake_run_streamed(*args: Any, **kwargs: Any) -> Any:
+ calls.append(kwargs)
+ return fake_result
+
+ async def fake_stream_to_sse(result: Any, s: Any, c: Any) -> None:
+ return None
+
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", fake_run_streamed)
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse", fake_stream_to_sse)
+
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ populate_task = asyncio.ensure_future(_populate_after_first_poll())
+ try:
+ returned = await run_with_enforcement(
+ agent=MagicMock(),
+ initial_input="hello",
+ ctx=ctx,
+ stream=stream,
+ run_config=RunConfig(),
+ copilot_config=config,
+ )
+ finally:
+ await populate_task
+
+ assert returned is fake_result
+ assert len(calls) == 2, "pause time must be credited so the resumed iteration isn't timed out"
+ assert ctx.copilot_total_timeout_exceeded is False
+
+
+# ---------------------------------------------------------------------------
+# 7 - /workflow/copilot/credential-response route
+# ---------------------------------------------------------------------------
+
+
+def _response_request(
+ *,
+ turn_id: str = "turn-1",
+ chat_id: str = "chat-1",
+ resume_token: str = "tok-1",
+ action: str = "skip",
+ credential_id: str | None = None,
+) -> WorkflowCopilotCredentialResponseRequest:
+ return WorkflowCopilotCredentialResponseRequest(
+ turn_id=turn_id,
+ workflow_copilot_chat_id=chat_id,
+ resume_token=resume_token,
+ action=action, # type: ignore[arg-type]
+ credential_id=credential_id,
+ )
+
+
+async def test_route_503_when_cache_missing(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(app._inst, "CACHE", None, raising=False)
+ organization = SimpleNamespace(organization_id="org-1")
+
+ with pytest.raises(HTTPException) as excinfo:
+ await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)
+ assert excinfo.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE
+
+
+async def test_route_204_writes_flag_for_skip_without_credential_id(monkeypatch: pytest.MonkeyPatch) -> None:
+ cache = _FakeCache()
+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ organization = SimpleNamespace(organization_id="org-1")
+
+ result = await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)
+
+ assert result is None
+ expected_key = credential_response_cache_key("org-1", "chat-1", "turn-1")
+ assert cache.store[expected_key] == encode_credential_response("skip", None)
+ response_set = next(call for call in cache.set_calls if call[0] == expected_key)
+ assert response_set[1] == encode_credential_response("skip", None)
+ assert response_set[2] == timedelta(seconds=settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS + 300)
+
+
+async def test_route_422_when_connected_without_credential_id(monkeypatch: pytest.MonkeyPatch) -> None:
+ cache = _FakeCache()
+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ organization = SimpleNamespace(organization_id="org-1")
+
+ with pytest.raises(HTTPException) as excinfo:
+ await workflow_copilot_credential_response(_response_request(action="connected"), organization=organization)
+ assert excinfo.value.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY
+ assert cache.set_calls == []
+
+
+async def test_route_404_when_credential_unknown_or_foreign_org(monkeypatch: pytest.MonkeyPatch) -> None:
+ cache = _FakeCache()
+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(
+ app,
+ "DATABASE",
+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[]))),
+ )
+ organization = SimpleNamespace(organization_id="org-1")
+
+ with pytest.raises(HTTPException) as excinfo:
+ await workflow_copilot_credential_response(
+ _response_request(action="connected", credential_id="cred_other_org"),
+ organization=organization,
+ )
+ assert excinfo.value.status_code == status.HTTP_404_NOT_FOUND
+ assert cache.set_calls == []
+
+
+async def test_route_204_writes_flag_for_connected_with_valid_credential(monkeypatch: pytest.MonkeyPatch) -> None:
+ cache = _FakeCache()
+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(
+ app,
+ "DATABASE",
+ SimpleNamespace(
+ credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[_make_credential()]))
+ ),
+ )
+ organization = SimpleNamespace(organization_id="org-1")
+
+ result = await workflow_copilot_credential_response(
+ _response_request(action="connected", credential_id="cred_1"),
+ organization=organization,
+ )
+
+ assert result is None
+ expected_key = credential_response_cache_key("org-1", "chat-1", "turn-1")
+ assert cache.store[expected_key] == encode_credential_response("connected", "cred_1")
+
+
+async def test_route_404_when_no_active_pause_record(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A valid-looking token can't resolve a turn that was never paused."""
+ cache = _FakeCache() # no active-pause record seeded
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ organization = SimpleNamespace(organization_id="org-1")
+
+ with pytest.raises(HTTPException) as excinfo:
+ await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)
+ assert excinfo.value.status_code == status.HTTP_404_NOT_FOUND
+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store
+
+
+async def test_route_403_when_resume_token_mismatches(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A leaked/guessed turn+chat id without the frame's token is rejected."""
+ cache = _FakeCache()
+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-real")
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ organization = SimpleNamespace(organization_id="org-1")
+
+ with pytest.raises(HTTPException) as excinfo:
+ await workflow_copilot_credential_response(
+ _response_request(action="skip", resume_token="tok-wrong"),
+ organization=organization,
+ )
+ assert excinfo.value.status_code == status.HTTP_403_FORBIDDEN
+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store
+
+
+async def test_route_rejects_bad_token_before_looking_up_credential_id(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A bad token must not reach the credential-id DB lookup -- org auth alone
+ can't authorize this turn, and doing the lookup first turns a wrong-token
+ response into a small authenticated credential-id existence oracle."""
+ cache = _FakeCache()
+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-real")
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ lookup = AsyncMock(return_value=[_make_credential(credential_id="cred_real")])
+ monkeypatch.setattr(app, "DATABASE", SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=lookup)))
+ organization = SimpleNamespace(organization_id="org-1")
+
+ with pytest.raises(HTTPException) as excinfo:
+ await workflow_copilot_credential_response(
+ _response_request(action="connected", credential_id="cred_real", resume_token="tok-wrong"),
+ organization=organization,
+ )
+ assert excinfo.value.status_code == status.HTTP_403_FORBIDDEN
+ lookup.assert_not_called()
+
+
+async def test_route_404_when_chat_id_does_not_match_the_pause(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The pause is keyed by chat+turn: a response with the right turn but a foreign chat misses it."""
+ cache = _FakeCache()
+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ organization = SimpleNamespace(organization_id="org-1")
+
+ with pytest.raises(HTTPException) as excinfo:
+ await workflow_copilot_credential_response(
+ _response_request(action="skip", chat_id="chat-foreign"),
+ organization=organization,
+ )
+ assert excinfo.value.status_code == status.HTTP_404_NOT_FOUND
+
+
+async def test_route_409_on_replay_after_first_accepted_response(monkeypatch: pytest.MonkeyPatch) -> None:
+ """First accepted response consumes the pause; a replay of the same POST is rejected."""
+ cache = _FakeCache()
+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ organization = SimpleNamespace(organization_id="org-1")
+
+ first = await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)
+ assert first is None
+
+ with pytest.raises(HTTPException) as excinfo:
+ await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)
+ assert excinfo.value.status_code == status.HTTP_409_CONFLICT
+
+
+async def test_route_410_when_response_arrives_after_the_frame_deadline(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The record's Redis TTL outlives the wait window for grace on a slow poll,
+ but a response after the deadline the frame told the client about must still
+ be rejected -- the waiter already gave up and the turn finalized without it."""
+ cache = _FakeCache()
+ _seed_active_pause(
+ cache, "org-1", "chat-1", "turn-1", "tok-1", expires_at=datetime.now(timezone.utc) - timedelta(seconds=1)
+ )
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ organization = SimpleNamespace(organization_id="org-1")
+
+ with pytest.raises(HTTPException) as excinfo:
+ await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)
+ assert excinfo.value.status_code == status.HTTP_410_GONE
+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store
+
+
+async def test_resolve_credential_pause_rejections_are_typed() -> None:
+ """resolve_credential_pause raises CredentialPauseRejection (not HTTPException) so it stays route-agnostic."""
+ cache = _FakeCache()
+
+ with pytest.raises(CredentialPauseRejection) as excinfo:
+ await resolve_credential_pause(
+ cache,
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ turn_id="turn-1",
+ resume_token="tok-1",
+ action="skip",
+ credential_id=None,
+ )
+ assert excinfo.value.status_code == status.HTTP_404_NOT_FOUND
+
+
+# ---------------------------------------------------------------------------
+# A diagnosed missing-credential run failure must pre-empt the generic
+# failed-test nudge, or the pause hook (gated on nudge is None) never runs on
+# the turn's first credential ask.
+# ---------------------------------------------------------------------------
+
+
+def _failed_test_ctx() -> CopilotContext:
+ """A real CopilotContext (all enforcement fields defaulted) with just the
+ failed-test + missing-credential-diagnosis fields set, so _check_enforcement
+ doesn't AttributeError on some unrelated field this hand-picked set omits.
+ """
+ ctx = make_copilot_context()
+ ctx.client_supports_credential_pause = True
+ ctx.last_test_ok = False
+ ctx.test_after_update_done = True
+ ctx.latest_diagnosis_repair_contract = _repair_contract(
+ RepairNextAction.ASK, DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ )
+ ctx.request_policy = RequestPolicy()
+ return ctx
+
+
+def test_missing_credential_failure_suppresses_generic_failed_test_nudge_when_pause_enabled() -> None:
+ """Fails on old code: the generic 'post_failed_test' nudge fires here instead."""
+ ctx = _failed_test_ctx()
+ config = CopilotConfig(credential_pause_enabled=True)
+
+ nudge = _check_enforcement(ctx, result=None, config=config)
+
+ assert nudge is None
+ assert ctx.failed_test_nudge_count == 0
+
+
+def test_missing_credential_failure_still_nudges_when_kill_switch_off() -> None:
+ """Flag-off parity: unrelated to the fix, the generic nudge still fires as before."""
+ ctx = _failed_test_ctx()
+ config = CopilotConfig(credential_pause_enabled=False)
+
+ nudge = _check_enforcement(ctx, result=None, config=config)
+
+ assert nudge is not None
+ assert ctx.failed_test_nudge_count == 1
+
+
+def test_missing_credential_failure_nudges_normally_once_pause_already_used() -> None:
+ ctx = _failed_test_ctx()
+ ctx.credential_pause_used = True
+ config = CopilotConfig(credential_pause_enabled=True)
+
+ nudge = _check_enforcement(ctx, result=None, config=config)
+
+ assert nudge is not None
+ assert ctx.failed_test_nudge_count == 1
+
+
+def _skipped_run_ctx() -> CopilotContext:
+ """Defensive fixture: even if test_after_update_done were False after a
+ skipped run (in the real runtime it isn't -- streaming_adapter's
+ _update_enforcement_from_tool sets it True unconditionally for
+ update_and_run_blocks/run_blocks_and_collect_debug regardless of skip;
+ see test_workflow_credential_inputs_unbound_skip_does_not_nudge_post_update
+ for the real-state pin), the pause must still pre-empt the post_update
+ nudge, a second nudge that can race the pause hook.
+ """
+ ctx = make_copilot_context()
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.update_workflow_called = True
+ ctx.test_after_update_done = False
+ ctx.request_policy = RequestPolicy()
+ return ctx
+
+
+def test_skipped_run_suppresses_post_update_nudge_when_pause_enabled() -> None:
+ """Fails on old code: the earlier 'post_update' nudge fires instead."""
+ ctx = _skipped_run_ctx()
+ config = CopilotConfig(credential_pause_enabled=True)
+
+ nudge = _check_enforcement(ctx, result=None, config=config)
+
+ assert nudge is None
+
+
+def test_skipped_run_still_nudges_post_update_when_kill_switch_off() -> None:
+ ctx = _skipped_run_ctx()
+ config = CopilotConfig(credential_pause_enabled=False)
+
+ nudge = _check_enforcement(ctx, result=None, config=config)
+
+ assert nudge is not None
+ assert "updated the workflow but did not test it" in nudge
+
+
+def test_workflow_credential_inputs_unbound_skip_does_not_nudge_post_update() -> None:
+ """Pins the real post-skip state (not the defensive _skipped_run_ctx fixture):
+ _update_enforcement_from_tool sets test_after_update_done=True unconditionally
+ for update_and_run_blocks, even on the credential-unbound skip branch, so the
+ post_update nudge condition (not test_after_update_done) is already False by
+ the time enforcement runs -- verified via a direct call to that function with
+ the real skip_result shape from tools/__init__.py's skip branch."""
+ ctx = make_copilot_context()
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.update_workflow_called = True
+ ctx.test_after_update_done = True
+ ctx.request_policy = RequestPolicy()
+ config = CopilotConfig(credential_pause_enabled=True)
+
+ assert _check_enforcement(ctx, result=None, config=config) is None
+
+ ctx.credential_pause_used = True
+ assert _check_enforcement(ctx, result=None, config=config) is None
+
+
+async def test_missing_credential_run_failure_pauses_the_loop(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Loop-level pin for the same fix, through the real finalize seam."""
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-nudge-race"
+ ctx.client_supports_credential_pause = True
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.last_test_ok = False
+ ctx.test_after_update_done = True
+ ctx.latest_diagnosis_repair_contract = _repair_contract(
+ RepairNextAction.ASK, DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
+ )
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-nudge-race")] = encode_credential_response(
+ "skip", None
+ )
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+
+ stream = _make_stream()
+ fake_result = _fake_result()
+ calls: list[dict[str, Any]] = []
+
+ def fake_run_streamed(*args: Any, **kwargs: Any) -> Any:
+ calls.append(kwargs)
+ return fake_result
+
+ async def fake_stream_to_sse(result: Any, s: Any, c: Any) -> None:
+ return None
+
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", fake_run_streamed)
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse", fake_stream_to_sse)
+
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ returned = await run_with_enforcement(
+ agent=MagicMock(),
+ initial_input="hello",
+ ctx=ctx,
+ stream=stream,
+ run_config=RunConfig(),
+ copilot_config=config,
+ )
+
+ assert returned is fake_result
+ # If maybe_credential_pause didn't clear the stale last_test_ok=False on
+ # skip, the resumed reply would be intercepted by the failed-test nudge
+ # for a 3rd Runner call instead of finalizing here.
+ assert len(calls) == 2
+ frame_types = [call.args[0].type for call in stream.send.await_args_list]
+ assert frame_types.count(WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED) == 1
+
+
+# ---------------------------------------------------------------------------
+# last_run_skipped_unbound_credentials must reflect the MOST RECENT
+# update_and_run_blocks call, not "ever skipped this turn" — otherwise a
+# later successful call leaves a stale True and pauses an already-passing turn.
+# ---------------------------------------------------------------------------
+
+
+async def test_last_run_skipped_flag_clears_on_a_later_successful_call(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Fails on old code: the flag stays True after the second, successful call."""
+ ctx = make_copilot_context()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ )
+ ctx.request_policy = RequestPolicy()
+
+ workflow_yaml = "workflow_definition:\n parameters: []\n blocks:\n - block_type: code\n label: step_one\n"
+
+ async def fake_prior_definition(update_ctx: Any) -> object:
+ return None
+
+ async def fake_update_workflow(payload: dict, update_ctx: Any, **kwargs: object) -> dict:
+ workflow = SimpleNamespace(workflow_definition={"blocks": [{"label": "step_one"}]})
+ update_ctx.last_workflow = workflow
+ update_ctx.last_update_block_count = 1
+ return {"ok": True, "_workflow": workflow, "data": {"block_count": 1}}
+
+ async def fake_run_blocks(params: dict, run_ctx: Any, **kwargs: object) -> dict:
+ return {"ok": True, "data": {"workflow_run_id": "wr-1", "overall_status": "completed", "blocks": []}}
+
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *a, **k: None)
+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", fake_prior_definition)
+ monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)
+ monkeypatch.setattr(tools_module, "_plan_frontier", lambda *args: (["step_one"], {}, "step_one"))
+ monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *args: None)
+ monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", fake_run_blocks)
+ monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "enqueue_screenshot_from_result", lambda *args, **kwargs: None)
+
+ call_args = json.dumps({"workflow_yaml": workflow_yaml, "block_labels": ["step_one"], "parameters": {}})
+
+ # Call 1: policy forces a skip (unbound credential).
+ monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: True)
+ result_1 = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"), call_args
+ )
+ assert json.loads(result_1)["data"]["skip_reason"] == "workflow_credential_inputs_unbound"
+ assert ctx.last_run_skipped_unbound_credentials is True
+
+ # Call 2: credential now bound, policy allows the real run.
+ monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: False)
+ result_2 = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"), call_args
+ )
+ assert json.loads(result_2)["ok"] is True
+ assert ctx.last_run_skipped_unbound_credentials is False
+ assert credential_pause_module.credential_pause_reason(ctx) is None
+
+
+async def test_last_run_skipped_flag_stays_false_when_update_workflow_fails_before_skip_branch(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Fails on old code: the flag was set True from the policy check alone,
+ before _update_workflow ever ran, so an unrelated authoring failure got
+ misreported as a credential ask.
+ """
+ ctx = make_copilot_context()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ )
+ ctx.request_policy = RequestPolicy()
+ workflow_yaml = "workflow_definition:\n parameters: []\n blocks:\n - block_type: code\n label: step_one\n"
+
+ async def fake_prior_definition(update_ctx: Any) -> object:
+ return None
+
+ async def failing_update_workflow(payload: dict, update_ctx: Any, **kwargs: object) -> dict:
+ return {"ok": False, "error": "workflow_yaml is not valid: bad block reference"}
+
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *a, **k: None)
+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", fake_prior_definition)
+ monkeypatch.setattr(tools_module, "_update_workflow", failing_update_workflow)
+ monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
+ # The policy would allow a skip if we got that far — but we never do.
+ monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: True)
+
+ result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
+ json.dumps({"workflow_yaml": workflow_yaml, "block_labels": ["step_one"], "parameters": {}}),
+ )
+
+ assert json.loads(result)["ok"] is False
+ assert ctx.last_run_skipped_unbound_credentials is False
+ assert credential_pause_module.credential_pause_reason(ctx) is None
+
+
+# ---------------------------------------------------------------------------
+# _copilot_seconds_remaining must credit pause time, or a long pause late in
+# the turn FORBIDS the very test the connected credential was for
+# (blockers.py's _late_block_running_call_signal).
+# ---------------------------------------------------------------------------
+
+
+def test_copilot_seconds_remaining_credits_pause_time() -> None:
+ from skyvern.forge.sdk.copilot.tools._shared import _copilot_seconds_remaining
+
+ ctx = SimpleNamespace(copilot_run_start_monotonic=time.monotonic() - 1000.0, copilot_credential_pause_seconds=300.0)
+
+ remaining = _copilot_seconds_remaining(ctx)
+
+ # Uncredited: 900 - 1000 = -100s (would forbid). Credited: 900 - 700 = 200s.
+ assert remaining is not None
+ assert remaining > 150.0
+
+
+def test_late_block_running_call_signal_allows_call_after_credited_pause() -> None:
+ """Fails on old code: a 300s pause 700s into the turn wrongly forbids the
+ resumed test run because the wall-clock check never saw the pause credit."""
+ from skyvern.forge.sdk.copilot.tools.blockers import _late_block_running_call_signal
+
+ ctx = SimpleNamespace(
+ copilot_run_start_monotonic=time.monotonic() - 1000.0,
+ copilot_credential_pause_seconds=300.0,
+ last_failed_workflow_yaml=None,
+ last_good_workflow_yaml=None,
+ )
+
+ assert _late_block_running_call_signal(ctx, "update_and_run_blocks") is None
+
+
+def test_late_block_running_call_signal_forbids_same_elapsed_without_pause_credit() -> None:
+ """Sanity pair for the test above: the identical wall-clock elapsed DOES
+ forbid the call when there was no pause to credit, proving the crediting
+ -- not something else -- is what keeps the resumed call allowed."""
+ from skyvern.forge.sdk.copilot.tools.blockers import _late_block_running_call_signal
+
+ ctx = SimpleNamespace(
+ copilot_run_start_monotonic=time.monotonic() - 1000.0,
+ copilot_credential_pause_seconds=0.0,
+ last_failed_workflow_yaml=None,
+ last_good_workflow_yaml=None,
+ )
+
+ assert _late_block_running_call_signal(ctx, "update_and_run_blocks") is not None
+
+
+# ---------------------------------------------------------------------------
+# A client that can't render credential_required must never be paused for,
+# or it stares at "Working..." until the timeout.
+# ---------------------------------------------------------------------------
+
+
+def test_would_fire_false_when_client_does_not_support_pause() -> None:
+ ctx = make_copilot_context()
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+ config = CopilotConfig(credential_pause_enabled=True)
+
+ assert credential_pause_module.credential_pause_would_fire(ctx, config) is False
+
+
+def test_would_fire_true_once_client_support_is_set() -> None:
+ ctx = make_copilot_context()
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+ config = CopilotConfig(credential_pause_enabled=True)
+
+ assert credential_pause_module.credential_pause_would_fire(ctx, config) is True
+
+
+async def test_unsupported_client_never_pauses_even_with_everything_else_true(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Fails on old code: no client-support guard existed, so a pre-12139 FE
+ build (or the env flag on with an unwired client) got paused and the FE
+ silently dropped the unknown frame -- a guaranteed ~timeout-length hang."""
+ ctx = make_copilot_context()
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+ # ctx.client_supports_credential_pause left at its False default.
+
+ cache = _FakeCache()
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert resume_msgs is None
+ stream.send.assert_not_awaited()
+
+
+# ---------------------------------------------------------------------------
+# A successful connect must clear credential_draft_deferred_explicitly, or
+# credential_prompt_reason() keeps stamping credentialPrompt right next to
+# credentialPause: connected.
+# ---------------------------------------------------------------------------
+
+
+async def test_connect_clears_credential_draft_deferred_explicitly(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Fails on old code: the flag survives a successful connect, so the FE
+ would see a contradictory connect-CTA right after the user connected."""
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-connect-clear"
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy(credential_draft_deferred_explicitly=True)
+
+ cache = _FakeCache()
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-connect-clear")] = encode_credential_response(
+ "connected", "cred_1"
+ )
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ credential = _make_credential()
+ monkeypatch.setattr(
+ credential_pause_module.app,
+ "DATABASE",
+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[credential]))),
+ )
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert ctx.request_policy.credential_draft_deferred_explicitly is False
+ assert credential_prompt_reason(ctx.request_policy, "any final text") is None
+
+
+async def test_skip_leaves_credential_draft_deferred_explicitly_untouched(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Skip/timeout correctly keep the flag -- only a successful connect clears it."""
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-skip-keep"
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy(credential_draft_deferred_explicitly=True)
+
+ cache = _FakeCache()
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-skip-keep")] = encode_credential_response(
+ "skip", None
+ )
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert ctx.request_policy.credential_draft_deferred_explicitly is True
+
+
+# ---------------------------------------------------------------------------
+# The poller must resolve before checking disconnect, or an already-arrived
+# response (connect-then-refresh) is discarded as a timeout instead of being
+# read.
+# ---------------------------------------------------------------------------
+
+
+async def test_poller_resolves_before_checking_disconnect(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Fails on old code: is_disconnected() was checked first in the poll loop,
+ so a response that arrives during the sleep window is discarded as a
+ timeout once the client has since disconnected."""
+ ctx = make_copilot_context()
+ ctx.organization_id = "org-1"
+ ctx.turn_id = "turn-race"
+ ctx.workflow_copilot_chat_id = "chat-1"
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.1)
+
+ async def _populate_during_sleep() -> None:
+ await asyncio.sleep(0.02)
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-race")] = encode_credential_response(
+ "skip", None
+ )
+
+ stream = MagicMock()
+ stream.send = AsyncMock(return_value=True)
+ # Connected for the top-level guard and the post-send re-check; "gone" for
+ # every poll-loop check after.
+ stream.is_disconnected = AsyncMock(side_effect=[False, False] + [True] * 10)
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ populate_task = asyncio.ensure_future(_populate_during_sleep())
+ try:
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+ finally:
+ await populate_task
+
+ assert resume_msgs is not None
+ assert ctx.credential_pause_outcome == "skipped"
+
+
+# ---------------------------------------------------------------------------
+# The pause must gate on a SHARED cache, not merely a non-None one -- a
+# same-process-only cache (LocalCache) guarantees a hang in a multi-worker
+# deployment since the poller and the POST can land on different workers.
+# ---------------------------------------------------------------------------
+
+
+def test_local_cache_is_not_shared() -> None:
+ from skyvern.forge.sdk.cache.local import LocalCache
+
+ assert LocalCache().is_shared is False
+
+
+async def test_non_shared_cache_never_pauses(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Fails on old code: `cache is None` was the only cache guard, so a
+ same-process-only cache (LocalCache in a multi-worker OSS deployment)
+ passed it and produced a guaranteed hang."""
+ ctx = make_copilot_context()
+ ctx.client_supports_credential_pause = True
+ ctx.last_run_skipped_unbound_credentials = True
+ ctx.request_policy = RequestPolicy()
+
+ cache = _FakeCache()
+ cache.is_shared = False
+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
+ stream = _make_stream()
+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+
+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
+
+ assert resume_msgs is None
+ stream.send.assert_not_awaited()
+
+
+# ---------------------------------------------------------------------------
+# CredentialPauseResolution sanity
+# ---------------------------------------------------------------------------
+
+
+def test_credential_pause_resolution_defaults_credential_to_none() -> None:
+ resolution = CredentialPauseResolution(action="skip")
+ assert resolution.credential is None