Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-12138): pause copilot BUILD turns on a mid-loop credential ask (#7360)
Celal Zamanoğlu114441f4e25e
diff --git a/skyvern/config.py b/skyvern/config.pyindex 937f2aab5..0673e2dd6 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -209,6 +209,13 @@ class Settings(BaseSettings):# Off = standard block authoring. On = prefer code blocks for browser work.WORKFLOW_COPILOT_CODE_BLOCK_MODE: bool = FalseWORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY: bool = False+ # Pause a BUILD turn in place on a typed mid-loop credential ask instead of ending it;+ # the FE resumes the same turn via a credential-connect card. Off = today's turn-terminal behavior.+ # Requires app.CACHE to be a shared cache (Redis) -- a same-process-only cache can't+ # coordinate the poller with a /credential-response POST that may land on another worker,+ # so this is a guaranteed no-op behind app.CACHE.is_shared regardless of this flag.+ WORKFLOW_COPILOT_CREDENTIAL_PAUSE_ENABLED: bool = False+ WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS: int = 300# Kill switch for the live codegen-progress SSE frame (drafted block labels while an authoring# tool call streams). Off restores exact pre-change behavior; old frontends drop the frame either way.WORKFLOW_COPILOT_CODEGEN_PROGRESS_ENABLED: bool = Truediff --git a/skyvern/forge/sdk/cache/base.py b/skyvern/forge/sdk/cache/base.pyindex 0b8bc6509..8173b3fd8 100644--- a/skyvern/forge/sdk/cache/base.py+++ b/skyvern/forge/sdk/cache/base.py@@ -45,6 +45,13 @@ class NoopLock:class BaseCache(ABC):+ #: True only for caches visible across separate worker processes (Redis).+ #: A feature that writes in one request and reads in another (e.g. a+ #: paused turn resumed by a POST that may land on a different worker)+ #: must check this before relying on get/set alone -- a same-process-only+ #: cache like LocalCache would silently hang, not error.+ is_shared: bool = False+@abstractmethodasync def set(self, key: str, value: Any, ex: Union[int, timedelta, None] = CACHE_EXPIRE_TIME) -> None:passdiff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex e359da63f..2c847b1d3 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -1728,6 +1728,13 @@ def _make_agent_result(reason = credential_prompt_reason(policy, kwargs.get("user_response"))if reason:payload_updates["credentialPrompt"] = {"reason": reason}+ if ctx is not None and "credentialPause" not in narrative_payload:+ pause_outcome = ctx.credential_pause_outcome+ if pause_outcome:+ pause_payload = {"outcome": pause_outcome}+ if pause_outcome == "connected" and ctx.request_policy and ctx.request_policy.resolved_credentials:+ pause_payload["credentialId"] = ctx.request_policy.resolved_credentials[-1].credential_id+ payload_updates["credentialPause"] = pause_payloadif ctx is not None and "verifiedSuccess" not in narrative_payload:payload_updates["verifiedSuccess"] = bool(verified_goal_claim_authorized(ctx))if ctx is not None and "outcomeAdjudication" not in narrative_payload:@@ -4557,6 +4564,7 @@ async def _run_copilot_turn_impl(impose_synthesized_code_block=copilot_config.impose_synthesized_code_block,copilot_config=copilot_config,target_block_label=getattr(chat_request, "target_block_label", None),+ client_supports_credential_pause=getattr(chat_request, "supports_credential_pause", False),)LOG.info("copilot_block_authoring_policy_resolved",diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.pyindex 61c385fef..4cfa0a910 100644--- a/skyvern/forge/sdk/copilot/config.py+++ b/skyvern/forge/sdk/copilot/config.py@@ -356,6 +356,14 @@ def _default_fallback_llm_key() -> str | None:return settings.SECONDARY_LLM_KEY+def _default_credential_pause_enabled() -> bool:+ return settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_ENABLED+++def _default_credential_pause_timeout_seconds() -> int:+ return settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS++@dataclass(slots=True)class CopilotConfig:prompt_template: str = DEFAULT_PROMPT_TEMPLATE@@ -367,6 +375,8 @@ class CopilotConfig:block_authoring_policy: BlockAuthoringPolicy = BlockAuthoringPolicy.STANDARDimpose_synthesized_code_block: bool = Falserequested_output_path_aliases: dict[str, str] = field(default_factory=dict)+ credential_pause_enabled: bool = field(default_factory=_default_credential_pause_enabled)+ credential_pause_timeout_seconds: int = field(default_factory=_default_credential_pause_timeout_seconds)def nudge(self, key: str) -> str:return self.enforcement_nudges.get(key, DEFAULT_ENFORCEMENT_NUDGES[key])diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex d8c7fb959..2dc2740eb 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -87,6 +87,9 @@ class TurnNarrativePayload(TypedDict):outcomeAdjudication: NotRequired[NarrativeOutcomeAdjudication]# {"reason": <credential_prompt_reason() token>}, set when this turn surfaces a credential need.credentialPrompt: NotRequired[dict[str, str]]+ # {"outcome": "connected"|"skipped"|"timeout", "credentialId": ...}, set when a mid-build+ # credential pause (credential_pause.py) resolved during this turn.+ credentialPause: NotRequired[dict[str, str]]designStarted: booldesignEnded: booldraft: NarrativeDraft | None@@ -668,6 +671,15 @@ class CopilotContext(AgentContext):blocked_reply_signatures: list[str] = field(default_factory=list)requested_output_extraction_candidate: FrozenRequestedOutputExtractionCandidate | None = None+ # Mid-build credential pause (credential_pause.py). last_run_skipped_unbound_credentials+ # is set by tools/__init__.py's update_and_run_blocks skip branch; client_supports_credential_pause+ # is set from the chat request at construction; the rest are owned by maybe_credential_pause.+ last_run_skipped_unbound_credentials: bool = False+ client_supports_credential_pause: bool = False+ credential_pause_used: bool = False+ copilot_credential_pause_seconds: float = 0.0+ credential_pause_outcome: str | None = None+# Tool trackingconsecutive_tool_tracker: list[str] = field(default_factory=list)tool_activity: list[dict[str, Any]] = field(default_factory=list)diff --git a/skyvern/forge/sdk/copilot/credential_pause.py b/skyvern/forge/sdk/copilot/credential_pause.pynew file mode 100644index 000000000..e6f6f85c0--- /dev/null+++ b/skyvern/forge/sdk/copilot/credential_pause.py@@ -0,0 +1,542 @@+"""Mid-loop pause for copilot BUILD turns blocked on a missing credential.++Parks between ``Runner.run_streamed`` iterations at the enforcement finalize+seam (see ``run_with_enforcement``) so the SSE connection stays open while+the frontend surfaces a credential-connect card. Resume transport is a+per-turn Redis flag polled directly by the paused coroutine -- the inverse of+the ``/workflow/copilot/cancel`` sidecar watcher, since here the paused loop+itself is the poller rather than a task racing the handler.++The resume path is not authorized by org auth + ``turn_id`` alone. Establishing+the pause writes an *active-pause record*, keyed by (org, chat, turn), that+carries a one-time ``resume_token`` delivered only in the ``credential_required``+frame. ``resolve_credential_pause`` -- the only writer of the loop-facing+response flag -- refuses to store a decision unless the caller presents that+token against a still-pending record, and consumes the record on the first+accepted response so a leaked or replayed ``turn_id`` can't resolve the pause.+"""++from __future__ import annotations++import asyncio+import json+import re+import secrets+import time+from dataclasses import dataclass+from datetime import datetime, timedelta, timezone+from http import HTTPStatus+from typing import TYPE_CHECKING, Any, Literal++import structlog++from skyvern.config import settings+from skyvern.forge import app+from skyvern.forge.sdk.copilot.config import CopilotConfig+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import DiagnosisFailureType, RepairNextAction+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy+from skyvern.forge.sdk.schemas.credentials import Credential+from skyvern.forge.sdk.schemas.workflow_copilot import (+ WorkflowCopilotCredentialRequiredUpdate,+ WorkflowCopilotStreamMessageType,+)++if TYPE_CHECKING:+ from agents.result import RunResultStreaming++ # Importing the routes package at module scope pulls in workflow_copilot.py ->+ # agent.py -> enforcement.py, which imports this module -> circular import.+ from skyvern.forge.sdk.core.event_source_stream import EventSourceStream++LOG = structlog.get_logger()++CREDENTIAL_RESPONSE_POLL_SECONDS = 1.5++# The active-pause record and loop-facing response flag outlive the wait window+# by this grace so an accepted response is still readable if the resumed loop is+# briefly slow to poll.+CREDENTIAL_PAUSE_RECORD_TTL_GRACE_SECONDS = 300+++def credential_response_cache_key(organization_id: str, chat_id: str, turn_id: str) -> str:+ return f"copilot_credential_response:{organization_id}:{chat_id}:{turn_id}"+++def credential_pause_active_key(organization_id: str, chat_id: str, turn_id: str) -> str:+ return f"copilot_credential_pause:{organization_id}:{chat_id}:{turn_id}"+++def _credential_pause_lock_key(organization_id: str, chat_id: str, turn_id: str) -> str:+ return f"copilot_credential_pause_lock:{organization_id}:{chat_id}:{turn_id}"+++def _credential_pause_record_ttl(timeout_seconds: int) -> timedelta:+ return timedelta(seconds=timeout_seconds + CREDENTIAL_PAUSE_RECORD_TTL_GRACE_SECONDS)+++def _new_resume_token() -> str:+ return secrets.token_urlsafe(32)+++class CredentialPauseRejection(Exception):+ """Raised by ``resolve_credential_pause`` when a resume attempt is not authorized.++ Carries the HTTP status the route should surface. Kept fastapi-free so this+ module stays importable from the enforcement loop without pulling in routes.+ """++ def __init__(self, status_code: int, detail: str) -> None:+ super().__init__(detail)+ self.status_code = status_code+ self.detail = detail+++@dataclass+class _ActivePauseRecord:+ resume_token: str+ status: Literal["pending", "consumed"]+ expires_at: datetime+++def _encode_active_pause(resume_token: str, expires_at: datetime, *, consumed: bool = False) -> str:+ return json.dumps(+ {+ "resume_token": resume_token,+ "status": "consumed" if consumed else "pending",+ "expires_at": expires_at.isoformat(),+ }+ )+++def _decode_active_pause(raw: Any) -> _ActivePauseRecord | None:+ try:+ data = json.loads(raw)+ except (TypeError, ValueError):+ return None+ if not isinstance(data, dict):+ return None+ token = data.get("resume_token")+ record_status = data.get("status")+ if not isinstance(token, str) or record_status not in ("pending", "consumed"):+ return None+ try:+ expires_at = datetime.fromisoformat(str(data.get("expires_at")))+ except ValueError:+ return None+ return _ActivePauseRecord(resume_token=token, status=record_status, expires_at=expires_at)+++def _validate_pending_pause(record: _ActivePauseRecord | None, resume_token: str) -> _ActivePauseRecord:+ """Shared checks for a resume attempt: a pending, unexpired record bound to this token."""+ if record is None:+ raise CredentialPauseRejection(+ status_code=HTTPStatus.NOT_FOUND,+ detail="No active credential pause for this turn",+ )+ if record.status != "pending":+ raise CredentialPauseRejection(+ status_code=HTTPStatus.CONFLICT,+ detail="Credential pause already resolved",+ )+ # The record's Redis TTL outlives the wait window by design (grace for a+ # slow final poll) -- that's an infra buffer, not permission to accept a+ # response after the waiter gave up and the turn already finalized without+ # it. Enforce the actual deadline the frame told the client about.+ if datetime.now(timezone.utc) >= record.expires_at:+ raise CredentialPauseRejection(+ status_code=HTTPStatus.GONE,+ detail="Credential pause has expired",+ )+ if not resume_token or not secrets.compare_digest(str(resume_token), record.resume_token):+ raise CredentialPauseRejection(+ status_code=HTTPStatus.FORBIDDEN,+ detail="Invalid credential resume token",+ )+ return record+++async def check_credential_pause_resumable(+ cache: Any,+ *,+ organization_id: str,+ workflow_copilot_chat_id: str,+ turn_id: str,+ resume_token: str,+) -> None:+ """Read-only precheck: raises the same rejection a resolve would, without consuming anything.++ Lets a caller (the route) validate the token BEFORE doing anything that+ reveals org-scoped information (e.g. a credential-id existence lookup) to+ an unauthenticated-for-this-turn caller who only has org auth.+ """+ active_key = credential_pause_active_key(organization_id, workflow_copilot_chat_id, turn_id)+ record = _decode_active_pause(await cache.get(active_key))+ _validate_pending_pause(record, resume_token)+++async def resolve_credential_pause(+ cache: Any,+ *,+ organization_id: str,+ workflow_copilot_chat_id: str,+ turn_id: str,+ resume_token: str,+ action: Literal["connected", "skip"],+ credential_id: str | None,+) -> None:+ """Validate the resume token against the active pause, consume it, then store the decision.++ The security boundary for resuming a paused turn: org auth alone is not+ enough. The caller must present the one-time ``resume_token`` from the+ ``credential_required`` frame, and (org, chat, turn) must name a still-pending+ active-pause record. Under a per-turn lock, the first accepted response flips+ the record to ``consumed`` before writing the loop-facing flag, so a leaked or+ replayed ``turn_id`` can neither resolve a pause it never opened nor overwrite+ a decision already made.+ """+ active_key = credential_pause_active_key(organization_id, workflow_copilot_chat_id, turn_id)+ lock_key = _credential_pause_lock_key(organization_id, workflow_copilot_chat_id, turn_id)+ async with cache.get_lock(lock_key):+ record = _validate_pending_pause(_decode_active_pause(await cache.get(active_key)), resume_token)+ ttl = _credential_pause_record_ttl(settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS)+ await cache.set(active_key, _encode_active_pause(record.resume_token, record.expires_at, consumed=True), ex=ttl)+ await cache.set(+ credential_response_cache_key(organization_id, workflow_copilot_chat_id, turn_id),+ encode_credential_response(action, credential_id),+ ex=ttl,+ )+++@dataclass+class CredentialPauseResolution:+ action: Literal["connected", "skip"]+ credential: Credential | None = None+++def encode_credential_response(action: Literal["connected", "skip"], credential_id: str | None) -> str:+ return json.dumps({"action": action, "credential_id": credential_id})+++def _decode_credential_response(raw: Any) -> tuple[str, str | None] | None:+ try:+ data = json.loads(raw)+ except (TypeError, ValueError):+ return None+ if not isinstance(data, dict) or data.get("action") not in ("connected", "skip"):+ return None+ credential_id = data.get("credential_id")+ return data["action"], credential_id if isinstance(credential_id, str) else None+++def credential_pause_reason(ctx: Any) -> str | None:+ """Typed-signal-only detector for a mid-build credential ask.++ Deliberately narrower than ``credential_prompt_reason`` (request_policy.py):+ no text-marker tier, so a REPLY that merely mentions credentials in prose+ can't trigger a pause -- see the SKY-11988 false-positive lesson.+ """+ policy = getattr(ctx, "request_policy", None)+ skip_test = isinstance(policy, RequestPolicy) and policy.testing_intent == "skip_test"+ if getattr(ctx, "last_run_skipped_unbound_credentials", False) and not skip_test:+ # A skip_test turn can still land here: _request_policy_allows_update_and_skip_run+ # (guardrails.py) skips the run for a skip_test-deferred credential, same as it+ # does for a credential_deferred_draft -- pausing here would contradict the same+ # explicit "don't verify this yet" request the deferred-draft branch below excludes.+ return "workflow_credential_inputs_unbound"++ contract = getattr(ctx, "latest_diagnosis_repair_contract", None)+ if (+ contract is not None+ and contract.repair_decision.next_action == RepairNextAction.ASK+ and contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT+ # MISSING_CREDENTIAL_OR_INIT is a combined category -- diagnosis_repair_contract.py+ # also assigns it to PARAMETER_BINDING_ERROR and org/workflow/browser-session lookup+ # failures that have nothing to do with credentials. Require the specific category so+ # those don't get preempted by a credential card that can't unblock them.+ and "CREDENTIAL_ERROR" in contract.diagnosis_result.root_cause_identity.failure_categories+ ):+ return "missing_credential_run_failure"++ if (+ isinstance(policy, RequestPolicy)+ and policy.credential_draft_deferred_explicitly+ # An explicit skip-test credential draft (request_policy.py's+ # _apply_explicit_code_block_credential_draft_policy) means the user already said+ # not to run/verify this -- pausing to ask for a credential contradicts that ask.+ and not skip_test+ and getattr(ctx, "update_workflow_called", False)+ ):+ return "credential_deferred_draft"++ return None+++def credential_pause_would_fire(ctx: Any, copilot_config: CopilotConfig | None) -> bool:+ """Synchronous subset of maybe_credential_pause's guards.++ Called from both here and enforcement.py's _check_enforcement so the two+ can't drift: enforcement pre-empts hygiene nudges based on this same+ prediction, and a client/kill-switch/latch mismatch between the two would+ either suppress nudges for a pause that will never fire, or let a nudge+ race a pause that will. Excludes the async-only checks (stream disconnect).+ """+ return (+ copilot_config is not None+ and copilot_config.credential_pause_enabled+ and getattr(ctx, "client_supports_credential_pause", False)+ and not getattr(ctx, "credential_pause_used", False)+ # A same-process-only cache (LocalCache) can't coordinate the poller with a+ # /credential-response POST that may land on a different worker -- gate on a+ # cache that's explicitly known to be shared (Redis) rather than merely non-None.+ and getattr(getattr(app, "CACHE", None), "is_shared", False)+ and credential_pause_reason(ctx) is not None+ )+++def _defang(text: str) -> str:+ return re.sub(r"\s+", " ", text).replace('"', "").strip()[:200]+++def _apply_connected_credential_to_policy(ctx: Any, policy: RequestPolicy, credential: Credential) -> None:+ """Grant the resumed loop run authority for ``credential``.++ Load-bearing: without this, the resumed ``update_and_run_blocks`` call+ re-skips via ``_request_policy_allows_update_and_skip_run`` (guardrails.py)+ and the turn re-asks for the same credential.++ Also un-latches ``test_after_update_done``: the skipped run before the pause+ already set it True, so without this reset a reply that never re-runs the+ blocks would still clear enforcement's post_update nudge and finalize an+ untested draft instead of being forced back through update_and_run_blocks.+ """+ ctx.test_after_update_done = False+ policy.resolved_credentials.append(credential)+ policy.allow_run_blocks = True+ policy.clarification_reason = "none"+ policy.allow_missing_credentials_in_draft = False+ policy.requires_user_clarification = False+ # Otherwise credential_prompt_reason() still sees the deferred-draft flag on the+ # terminal RESPONSE and stamps credentialPrompt right next to credentialPause:+ # connected -- a contradictory "still need a credential" signal to the FE.+ policy.credential_draft_deferred_explicitly = False+++def _assemble_resume_messages(ctx: Any, text: str) -> list[dict[str, Any]]:+ # Local import avoids a module-load cycle: enforcement.py imports+ # maybe_credential_pause at module scope.+ from skyvern.forge.sdk.copilot.enforcement import _assemble_enforcement_messages, _consume_pending_screenshots++ screenshot_msg = _consume_pending_screenshots(ctx)+ return _assemble_enforcement_messages(screenshot_msg, text, None)+++def _connected_resume_text(credential: Credential) -> str:+ safe_name = _defang(credential.name)+ return (+ f"The user connected saved credential {safe_name} ({credential.credential_id}) via the credential card. "+ "Bind it as the credential parameter and continue the build; run the blocks that were skipped or failed "+ "on the missing credential."+ )+++_SKIP_RESUME_TEXT = (+ "The user chose not to connect a credential now. Continue without it: keep the credential parameter "+ "placeholder in the draft, do not ask for the credential again this turn, and finish. If you run a test, "+ "it may stop at the login step."+)+++async def _try_resolve_credential_response(+ response_key: str, organization_id: str+) -> CredentialPauseResolution | None | Literal["pending"]:+ cache = app.CACHE+ raw = await cache.get(response_key)+ if not raw:+ return "pending"+ decoded = _decode_credential_response(raw)+ if decoded is None:+ return "pending"+ action, credential_id = decoded+ if action == "skip":+ return CredentialPauseResolution(action="skip")+ if not credential_id:+ return None+ existing = await app.DATABASE.credentials.get_credentials_by_ids([credential_id], organization_id=organization_id)+ if not existing:+ return None+ return CredentialPauseResolution(action="connected", credential=existing[0])+++async def _wait_for_credential_response(+ response_key: str,+ ctx: Any,+ stream: EventSourceStream,+ timeout_seconds: int,+) -> CredentialPauseResolution | None:+ # Check once before the sleep loop so a card response posted in the brief+ # window before the first poll doesn't cost a full extra poll interval.+ first = await _try_resolve_credential_response(response_key, ctx.organization_id)+ if first != "pending":+ return first++ deadline = time.monotonic() + timeout_seconds+ while time.monotonic() < deadline:+ await asyncio.sleep(CREDENTIAL_RESPONSE_POLL_SECONDS)+ # Resolve before checking disconnect: an already-posted response must win+ # over a disconnect that happened after the POST (e.g. connect-then-refresh),+ # not get discarded as a timeout.+ resolved = await _try_resolve_credential_response(response_key, ctx.organization_id)+ if resolved != "pending":+ return resolved+ if await stream.is_disconnected():+ return None+ return None+++async def maybe_credential_pause(+ ctx: Any,+ result: RunResultStreaming,+ stream: EventSourceStream,+ copilot_config: CopilotConfig,+) -> list[dict[str, Any]] | None:+ """Pause a finalizing turn that hit a typed mid-build credential ask.++ Returns the resume messages to re-enter the loop with, or None to let+ the caller finalize normally (kill-switch off, client can't render the+ frame, already paused once this turn, no cache configured or not shared+ across workers, client gone, or no typed signal fired).+ """+ if not credential_pause_would_fire(ctx, copilot_config):+ return None+ # Predicted true (the sync guard chain passed) but bailing below anyway --+ # latch it now so this iteration can't loop back into the same prediction,+ # and tag the outcome "declined" (distinct from "timeout") so the caller+ # knows no frame was ever sent and can fall back to a normal nudge instead+ # of a premature finalize. credential_pause_would_fire's own docstring notes+ # it excludes the async-only disconnect check, which is exactly the gap here.+ ctx.credential_pause_used = True+ cache = getattr(app, "CACHE", None)+ if cache is None:+ ctx.credential_pause_outcome = "declined"+ return None+ if await stream.is_disconnected():+ ctx.credential_pause_outcome = "declined"+ return None+ reason = credential_pause_reason(ctx)+ if reason is None:+ ctx.credential_pause_outcome = "declined"+ return None++ policy = getattr(ctx, "request_policy", None)+ login_page_urls = list(policy.login_page_urls) if isinstance(policy, RequestPolicy) else []+ credential_refs = list(policy.credential_refs) if isinstance(policy, RequestPolicy) else []+ timeout_seconds = copilot_config.credential_pause_timeout_seconds+ now = datetime.now(timezone.utc)++ organization_id = ctx.organization_id+ chat_id = getattr(ctx, "workflow_copilot_chat_id", None) or ""+ turn_id = getattr(ctx, "turn_id", None) or ""+ resume_token = _new_resume_token()+ expires_at = now + timedelta(seconds=timeout_seconds)+ # Establish the active-pause record before the frame carries the token: the+ # response endpoint refuses to resolve a turn that has no pending record.+ # expires_at is the same deadline the frame tells the client -- the record's+ # own TTL is a separate infra grace, not a resolve-after-timeout allowance.+ await cache.set(+ credential_pause_active_key(organization_id, chat_id, turn_id),+ _encode_active_pause(resume_token, expires_at),+ ex=_credential_pause_record_ttl(timeout_seconds),+ )++ # Local import: same module-load-cycle reason as _assemble_resume_messages.+ from skyvern.forge.sdk.copilot.enforcement import _parse_normalized_final_response++ parsed = _parse_normalized_final_response(result)+ message = str((parsed or {}).get("user_response") or "")++ await stream.send(+ WorkflowCopilotCredentialRequiredUpdate(+ type=WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED,+ turn_id=turn_id,+ workflow_copilot_chat_id=chat_id,+ resume_token=resume_token,+ reason=reason,+ message=message,+ login_page_urls=login_page_urls,+ credential_refs=credential_refs,+ timeout_seconds=timeout_seconds,+ expires_at=expires_at,+ timestamp=now,+ )+ )++ async def _invalidate_active_pause_record() -> CredentialPauseResolution | None:+ # The waiter can exit (disconnect, genuine timeout, cancellation, or an+ # unexpected error) well before expires_at -- invalidate the record so a+ # late POST (e.g. the tab reconnects) gets a clear 409 instead of silently+ # writing a response nobody will read. Uses the SAME lock+ # resolve_credential_pause takes, with one final response check under it:+ # a POST that validated the record as still-pending an instant before this+ # runs would otherwise get a clean 204 for a response the waiter -- already+ # given up -- will never read. Rescue it if it raced in.+ lock_key = _credential_pause_lock_key(organization_id, chat_id, turn_id)+ async with cache.get_lock(lock_key):+ raced_in = await _try_resolve_credential_response(response_key, organization_id)+ if isinstance(raced_in, CredentialPauseResolution):+ return raced_in+ await cache.set(+ credential_pause_active_key(organization_id, chat_id, turn_id),+ _encode_active_pause(resume_token, expires_at, consumed=True),+ ex=_credential_pause_record_ttl(timeout_seconds),+ )+ return None++ response_key = credential_response_cache_key(organization_id, chat_id, turn_id)++ if await stream.is_disconnected():+ # send() can return True even when the client is already gone (its own+ # protocol contract: "queued for delivery or dropped because the client+ # is gone") -- a disconnect racing the send itself would otherwise wait+ # out the full timeout for a card nobody ever saw. Re-check right after+ # send and treat it the same as the pre-send disconnect guard above.+ await _invalidate_active_pause_record()+ ctx.credential_pause_outcome = "declined"+ return None++ start = time.monotonic()+ try:+ resolution = await _wait_for_credential_response(response_key, ctx, stream, timeout_seconds)+ except BaseException:+ # Covers CancelledError (a direct BaseException subclass, not Exception)+ # alongside any unexpected failure in the wait loop itself -- the frame's+ # resume token is already live client-side either way, so the record must+ # not be left pending for the same reason as the disconnect/timeout case.+ # Can't act on a rescued resolution mid-unwind, only avoid corrupting state.+ await _invalidate_active_pause_record()+ raise+ finally:+ ctx.copilot_credential_pause_seconds = getattr(ctx, "copilot_credential_pause_seconds", 0.0) + (+ time.monotonic() - start+ )++ if resolution is None:+ resolution = await _invalidate_active_pause_record()+ if resolution is None:+ ctx.credential_pause_outcome = "timeout"+ return None+ if resolution.action == "skip":+ ctx.credential_pause_outcome = "skipped"+ # A missing_credential_run_failure pause means the diagnosed run left+ # last_test_ok=False; without clearing it, the resumed reply is intercepted+ # by the generic failed-test nudge instead of honoring the skip decision.+ ctx.last_test_ok = None+ return _assemble_resume_messages(ctx, _SKIP_RESUME_TEXT)++ credential = resolution.credential+ if credential is None:+ ctx.credential_pause_outcome = "timeout"+ return None+ if isinstance(policy, RequestPolicy):+ _apply_connected_credential_to_policy(ctx, policy, credential)+ ctx.credential_pause_outcome = "connected"+ return _assemble_resume_messages(ctx, _connected_resume_text(credential))diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 6902ce8c4..aba3eade8 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -86,6 +86,7 @@ from skyvern.forge.sdk.copilot.config import (normalize_block_authoring_policy,)from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext+from skyvern.forge.sdk.copilot.credential_pause import credential_pause_would_fire, maybe_credential_pausefrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (DiagnosisRepairContract,RepairLoopState,@@ -946,8 +947,24 @@ def _mark_copilot_total_timeout(ctx: Any) -> None:ctx.copilot_total_timeout_exceeded = True+def _elapsed_run_seconds(ctx: Any, start_time: float) -> float:+ """Wall-clock elapsed since ``start_time``, minus time spent in a credential pause.++ Keeps TOTAL_TIMEOUT_SECONDS a budget over actual agent work, not real+ time, so a paused-and-resumed turn isn't penalized for pause time.++ ``pause_seconds`` is coerced defensively: tests commonly pass a bare+ ``MagicMock()`` as ``ctx``, whose ``getattr(..., default)`` returns a+ fresh Mock instead of the default (Mock never raises AttributeError).+ """+ pause_seconds = getattr(ctx, "copilot_credential_pause_seconds", 0.0)+ if not isinstance(pause_seconds, (int, float)):+ pause_seconds = 0.0+ return time.monotonic() - start_time - pause_seconds++def _mark_copilot_total_timeout_if_elapsed(ctx: Any, start_time: float) -> None:- if time.monotonic() - start_time >= TOTAL_TIMEOUT_SECONDS:+ if _elapsed_run_seconds(ctx, start_time) >= TOTAL_TIMEOUT_SECONDS:_mark_copilot_total_timeout(ctx)@@ -1516,6 +1533,14 @@ def _check_enforcement()raise_if_turn_halt(ctx, verified=verified)+ # A pending credential pause pre-empts every hygiene nudge below, not just+ # the failed-test one: a credential-blocked update_and_run_blocks call+ # satisfies post_update (test not run) and, when the diagnosis contract+ # is the source, the generic failed-test nudge too. None of those nudges+ # can be acted on without the credential the pause is about to ask for.+ if credential_pause_would_fire(ctx, config):+ return None+# A permanent navigation error (DNS / cert / SSL / invalid URL) cannot be# resolved by observing a prior navigate or by testing an updated# workflow against the same bad URL, so let it speak first.@@ -2095,7 +2120,7 @@ async def _run_streamed_with_deadline(``MIN_DEADLINE_REMAINING_SECONDS`` floors ``remaining`` so``wait_for(timeout=0)`` never panics on an already-spent budget."""- elapsed = time.monotonic() - start_time+ elapsed = _elapsed_run_seconds(ctx, start_time)remaining = max(MIN_DEADLINE_REMAINING_SECONDS, TOTAL_TIMEOUT_SECONDS - elapsed)result = Runner.run_streamed(agent, input=current_input, context=ctx, session=session, **runner_kwargs)try:@@ -2919,7 +2944,7 @@ async def run_with_enforcement(# SSE stream silently drops events once the browser is gone, but# the agent keeps running so the reply can be persisted to the# chat history on the server side (see SKY-8986).- elapsed = time.monotonic() - start_time+ elapsed = _elapsed_run_seconds(ctx, start_time)if elapsed > TOTAL_TIMEOUT_SECONDS:_mark_copilot_total_timeout(ctx)raise CopilotTotalTimeoutError()@@ -3073,14 +3098,41 @@ async def run_with_enforcement(synthesized_msg = _maybe_synthesized_block_offer_msg(ctx)spine_checkpoint_nudge = False- if nudge is None and synthesized_msg is None:- spine_nudge = _scouted_spine_turn_end_nudge(ctx)- if spine_nudge is None:- _consume_pending_screenshots(ctx)- _maybe_raise_non_retriable_nav(ctx)- return result- nudge = spine_nudge- spine_checkpoint_nudge = True+ if nudge is None:+ # Checked whenever there's no regular nudge, even if a synthesized+ # offer is also pending: a credential-blocked run's diagnosis can+ # coincide with a reopened synthesized-block offer, and the pause+ # must win so the loop doesn't send the offer instead of the card.+ pause_used_before_this_call = getattr(ctx, "credential_pause_used", False)+ resume_msgs = await maybe_credential_pause(ctx, result, stream, copilot_config)+ if resume_msgs is not None:+ current_input = (+ resume_msgs if session is not None else _prune_input_list(result.to_input_list()) + resume_msgs+ )+ iteration += 1+ continue+ if (+ not pause_used_before_this_call+ and getattr(ctx, "credential_pause_used", False)+ and getattr(ctx, "credential_pause_outcome", None) == "declined"+ ):+ # The latch just flipped on THIS call with no frame ever sent+ # (disconnect, cache gone, or the reason vanished under the+ # async-only checks credential_pause_would_fire's docstring notes+ # it excludes) -- fall back to whatever nudge this iteration would+ # have gotten without the pre-empt, instead of silently finalizing+ # an uncorrected reply. Gated on the latch's own transition (not+ # just the outcome value) so a later iteration's unrelated+ # nudge=None doesn't re-trigger this off a stale "declined".+ nudge = _check_enforcement(ctx, result, copilot_config)+ if nudge is None and synthesized_msg is None:+ spine_nudge = _scouted_spine_turn_end_nudge(ctx)+ if spine_nudge is None:+ _consume_pending_screenshots(ctx)+ _maybe_raise_non_retriable_nav(ctx)+ return result+ nudge = spine_nudge+ spine_checkpoint_nudge = Trueif nudge is not None and not spine_checkpoint_nudge and nudge == _nudge(copilot_config, "post_update"):if ctx.post_update_nudge_count >= MAX_POST_UPDATE_NUDGES:diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.pyindex 36a21c538..33a743cd3 100644--- a/skyvern/forge/sdk/copilot/tools/__init__.py+++ b/skyvern/forge/sdk/copilot/tools/__init__.py@@ -615,6 +615,12 @@ async def update_and_run_blocks_tool("parameters": parameters or {},}skip_run_after_update = _request_policy_allows_update_and_skip_run(copilot_ctx, "update_and_run_blocks")+ # Cleared unconditionally up front and only set True at the actual skip+ # branch below — reflects "we skipped a run", not "the policy would have+ # allowed a skip if we got that far". A stale True from an earlier call, or+ # a premature True from a policy check ahead of an unrelated update_workflow+ # failure, would misreport an authoring error as a credential ask.+ copilot_ctx.last_run_skipped_unbound_credentials = Falseauthority_error = _authority_tool_error(copilot_ctx,"update_and_run_blocks",@@ -721,6 +727,7 @@ async def update_and_run_blocks_tool(return json.dumps(sanitized)if skip_run_after_update:+ copilot_ctx.last_run_skipped_unbound_credentials = Trueskip_message = "Skipped test run: required credentials are not configured."skip_result = {"ok": True,diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.pyindex 378c04a9f..9094efaad 100644--- a/skyvern/forge/sdk/copilot/tools/_shared.py+++ b/skyvern/forge/sdk/copilot/tools/_shared.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport asyncioimport json-import timefrom collections.abc import Mappingfrom typing import Anyfrom urllib.parse import urlparse@@ -26,7 +25,7 @@ from skyvern.forge.sdk.copilot.composition_browser_expressions import ()from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema, parse_composition_structuredfrom skyvern.forge.sdk.copilot.context import CopilotContext-from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS+from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS, _elapsed_run_secondsfrom skyvern.forge.sdk.copilot.runtime import AgentContextfrom skyvern.forge.sdk.copilot.task_output_envelope import (_TASK_ENVELOPE_BLOCK_TYPES,@@ -270,7 +269,7 @@ def _copilot_seconds_remaining(ctx: AgentContext) -> float | None:started_at = getattr(ctx, "copilot_run_start_monotonic", None)if not isinstance(started_at, int | float):return None- return TOTAL_TIMEOUT_SECONDS - (time.monotonic() - float(started_at))+ return TOTAL_TIMEOUT_SECONDS - _elapsed_run_seconds(ctx, float(started_at))def _same_page_ignoring_fragment(left: str | None, right: str | None) -> bool:diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex 4d37e170a..9caadb4a1 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -42,6 +42,11 @@ from skyvern.forge.sdk.copilot.context import (render_loaded_result_context_for_prompt,sanitize_global_llm_context_for_prompt,)+from skyvern.forge.sdk.copilot.credential_pause import (+ CredentialPauseRejection,+ check_credential_pause_resumable,+ resolve_credential_pause,+)from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failurefrom skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handlerfrom skyvern.forge.sdk.copilot.output_utils import truncate_output@@ -77,6 +82,7 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotChatSender,WorkflowCopilotChatSummary,WorkflowCopilotClearProposedWorkflowRequest,+ WorkflowCopilotCredentialResponseRequest,WorkflowCopilotProcessingUpdate,WorkflowCopilotStreamErrorUpdate,WorkflowCopilotStreamMessageType,@@ -2352,6 +2358,72 @@ async def workflow_copilot_cancel()+@base_router.post(+ "/workflow/copilot/credential-response", include_in_schema=False, status_code=status.HTTP_204_NO_CONTENT+)+async def workflow_copilot_credential_response(+ response_request: WorkflowCopilotCredentialResponseRequest,+ organization: Organization = Depends(org_auth_service.get_current_org),+) -> None:+ """Resume a turn paused on ``credential_required`` with the user's card response.++ The resume path is not authorized by org auth + ``turn_id`` alone: the caller+ must echo back the one-time ``resume_token`` and ``workflow_copilot_chat_id``+ from the frame, which ``resolve_credential_pause`` checks against a still-pending+ active-pause record before it writes the flag the paused loop polls. Returns+ 503 when ``app.CACHE`` is absent, 422 when ``action="connected"`` omits a+ ``credential_id``, 404 when that ID doesn't resolve in this organization or no+ active pause matches, 403 on a bad token, and 409 once the pause is consumed.+ """+ cache = getattr(app, "CACHE", None)+ if cache is None:+ raise HTTPException(+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,+ detail="Credential response not supported in this environment",+ )++ # Validate the resume token BEFORE the credential-id lookup below: org auth+ # alone doesn't authorize this turn, and checking token validity first avoids+ # using an unauthorized-for-this-turn caller's credential_id as a small+ # authenticated existence oracle (a bogus token still gets a real 404/not-found).+ try:+ await check_credential_pause_resumable(+ cache,+ organization_id=organization.organization_id,+ workflow_copilot_chat_id=response_request.workflow_copilot_chat_id,+ turn_id=response_request.turn_id,+ resume_token=response_request.resume_token,+ )+ except CredentialPauseRejection as rejection:+ raise HTTPException(status_code=rejection.status_code, detail=rejection.detail)++ credential_id = response_request.credential_id+ if response_request.action == "connected":+ if not credential_id:+ raise HTTPException(+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,+ detail="credential_id is required when action is 'connected'",+ )+ existing = await app.DATABASE.credentials.get_credentials_by_ids(+ [credential_id], organization_id=organization.organization_id+ )+ if not existing:+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Credential {credential_id} not found")++ try:+ await resolve_credential_pause(+ cache,+ organization_id=organization.organization_id,+ workflow_copilot_chat_id=response_request.workflow_copilot_chat_id,+ turn_id=response_request.turn_id,+ resume_token=response_request.resume_token,+ action=response_request.action,+ credential_id=credential_id,+ )+ except CredentialPauseRejection as rejection:+ raise HTTPException(status_code=rejection.status_code, detail=rejection.detail)++@base_router.post("/workflow/copilot/clear-proposed-workflow", include_in_schema=False, status_code=status.HTTP_204_NO_CONTENT)diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex 3436c0b0b..89cc8ac9b 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -124,6 +124,14 @@ class WorkflowCopilotChatRequest(BaseModel):"otherwise it is a no-op."),)+ supports_credential_pause: bool = Field(+ False,+ description=(+ "True when the client can render the credential_required frame. Older clients default to False "+ "and silently drop unknown frame types, so the backend must not pause for them even if the "+ "server-side flag is on."+ ),+ )keep_pending_proposal: bool = Field(False,description=(@@ -137,6 +145,14 @@ class WorkflowCopilotCancelRequest(BaseModel):cancel_token: str = Field(..., description="The cancel_token sent on the original /chat-post request")+class WorkflowCopilotCredentialResponseRequest(BaseModel):+ turn_id: str = Field(..., description="turn_id from the matching credential_required frame")+ workflow_copilot_chat_id: str = Field(..., description="chat ID from the matching credential_required frame")+ resume_token: str = Field(..., description="One-time resume token from the matching credential_required frame")+ action: Literal["connected", "skip"] = Field(..., description="The user's response to the credential card")+ credential_id: str | None = Field(None, description="Saved credential ID; required when action is 'connected'")++class WorkflowCopilotClearProposedWorkflowRequest(BaseModel):workflow_copilot_chat_id: str = Field(..., description="The chat ID to update")auto_accept: bool = Field(..., description="Whether to auto-accept future workflow updates")@@ -199,6 +215,7 @@ class WorkflowCopilotStreamMessageType(StrEnum):DESIGN_START = "design_start"DESIGN_END = "design_end"WORKFLOW_DRAFT = "workflow_draft"+ CREDENTIAL_REQUIRED = "credential_required"CODEGEN_PROGRESS = "codegen_progress"@@ -406,6 +423,26 @@ class WorkflowCopilotWorkflowDraftUpdate(BaseModel):)+class WorkflowCopilotCredentialRequiredUpdate(BaseModel):+ type: WorkflowCopilotStreamMessageType = Field(+ WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED, description="Message type"+ )+ turn_id: str = Field(..., description="UUID for the paused turn; correlates with credential-response POSTs")+ workflow_copilot_chat_id: str = Field(..., description="The chat ID")+ resume_token: str = Field(..., description="One-time token the credential-response POST must echo back to resume")+ reason: Literal[+ "workflow_credential_inputs_unbound",+ "missing_credential_run_failure",+ "credential_deferred_draft",+ ] = Field(..., description="Typed signal that triggered the pause")+ message: str = Field(..., description="The agent's explanatory text at the moment of pausing")+ login_page_urls: list[str] = Field(default_factory=list, description="Candidate login page URLs, if known")+ credential_refs: list[str] = Field(default_factory=list, description="Credential IDs or names referenced")+ timeout_seconds: int = Field(..., description="How long the backend will wait before degrading to terminal")+ expires_at: datetime = Field(..., description="Server time after which the pause degrades to terminal")+ timestamp: datetime = Field(..., description="Server timestamp")++class WorkflowCopilotCodegenProgressUpdate(BaseModel):"""Live-only drafting progress while the LLM streams authoring-tool arguments. Not persisted; theworkflow_draft frame supersedes it."""diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.pynew file mode 100644index 000000000..1a8719f71--- /dev/null+++ b/tests/unit/test_copilot_credential_pause.py@@ -0,0 +1,1835 @@+"""Tests for the copilot mid-build credential pause-and-wait (SKY-12138).++Covers:++- ``credential_pause_reason`` fires only on the three typed mid-build signals+ and never on ``credential_prompt_reason``'s text-marker tier.+- ``maybe_credential_pause``'s waiter: connect mutates ``RequestPolicy`` and+ resolves, skip leaves the policy untouched, timeout/disconnect/no-cache+ degrade to None without sending a frame, an invalid/foreign credential id+ degrades rather than crashing, and CancelledError always propagates.+- ``run_with_enforcement`` loop integration: a typed signal at finalize sends+ exactly one ``credential_required`` frame and re-enters the loop with the+ resume message, instead of returning on the first finalize.+- Pause time is credited back against ``TOTAL_TIMEOUT_SECONDS`` so a slow+ pause doesn't trip the total-timeout on the resumed iteration.+- The one-pause-per-turn latch and kill-switch-off parity.+- The ``/workflow/copilot/credential-response`` route's validation and TTL.+"""++from __future__ import annotations++import asyncio+import json+import time+from datetime import datetime, timedelta, timezone+from types import SimpleNamespace+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest+from agents import RunConfig+from fastapi import HTTPException, status++from skyvern.config import settings+from skyvern.forge import app+from skyvern.forge.sdk.cache.base import NoopLock+from skyvern.forge.sdk.copilot import credential_pause as credential_pause_module+from skyvern.forge.sdk.copilot import tools as tools_module+from skyvern.forge.sdk.copilot.config import CopilotConfig+from skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.credential_pause import (+ CredentialPauseRejection,+ CredentialPauseResolution,+ _encode_active_pause,+ credential_pause_active_key,+ credential_response_cache_key,+ encode_credential_response,+ maybe_credential_pause,+ resolve_credential_pause,+)+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (+ DiagnosisFailureType,+ DiagnosisInput,+ DiagnosisRepairContract,+ DiagnosisResult,+ RepairDecision,+ RepairNextAction,+ RepairRootCauseIdentity,+ VerificationResult,+)+from skyvern.forge.sdk.copilot.enforcement import (+ NUDGE_SENTINEL,+ _check_enforcement,+ _elapsed_run_seconds,+ run_with_enforcement,+)+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, credential_prompt_reason+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode+from skyvern.forge.sdk.routes.workflow_copilot import (+ WorkflowCopilotCredentialResponseRequest,+ workflow_copilot_credential_response,+)+from skyvern.forge.sdk.schemas.credentials import Credential, CredentialType, CredentialVaultType+from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotStreamMessageType+from tests.unit.conftest import make_copilot_context+++def _repair_contract(+ next_action: Any,+ failure_type: Any = DiagnosisFailureType.UNKNOWN,+ categories: tuple[str, ...] = ("CREDENTIAL_ERROR",),+) -> DiagnosisRepairContract:+ return DiagnosisRepairContract(+ diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),+ diagnosis_result=DiagnosisResult(+ suspected_failure_type=failure_type,+ root_cause_identity=RepairRootCauseIdentity(failure_categories=categories),+ ),+ repair_decision=RepairDecision(next_action=next_action),+ verification_result=VerificationResult(),+ )+++def _make_credential(credential_id: str = "cred_1", name: str = "Example Login") -> Credential:+ now = datetime(2026, 1, 1, tzinfo=timezone.utc)+ return Credential(+ credential_id=credential_id,+ organization_id="org-1",+ name=name,+ vault_type=CredentialVaultType.SKYVERN,+ item_id="item_1",+ credential_type=CredentialType.PASSWORD,+ username="[email protected]",+ card_last4=None,+ card_brand=None,+ created_at=now,+ modified_at=now,+ )+++class _FakeCache:+ """Minimal in-memory double of the ``get`` / ``set`` / ``get_lock`` surface of app.CACHE.++ ``is_shared = True`` by default (stands in for Redis); flip an instance's+ to False to simulate a LocalCache-shaped non-shared cache.+ """++ is_shared = True++ def __init__(self) -> None:+ self.store: dict[str, Any] = {}+ self.set_calls: list[tuple[str, Any, Any]] = []++ async def get(self, key: str) -> Any:+ return self.store.get(key)++ async def set(self, key: str, value: Any, ex: Any = None) -> None:+ self.store[key] = value+ self.set_calls.append((key, value, ex))++ def get_lock(self, lock_name: str, blocking_timeout: int = 5, timeout: int = 10) -> NoopLock:+ return NoopLock(lock_name, blocking_timeout, timeout)+++def _seed_active_pause(+ cache: _FakeCache,+ org: str,+ chat: str,+ turn: str,+ token: str,+ *,+ expires_at: datetime | None = None,+) -> None:+ """Write a pending active-pause record the way maybe_credential_pause would."""+ cache.store[credential_pause_active_key(org, chat, turn)] = _encode_active_pause(+ token, expires_at or (datetime.now(timezone.utc) + timedelta(minutes=5))+ )+++def _make_stream(*, disconnected: bool = False) -> MagicMock:+ stream = MagicMock()+ stream.send = AsyncMock(return_value=True)+ stream.is_disconnected = AsyncMock(return_value=disconnected)+ return stream+++def _fake_result() -> MagicMock:+ result = MagicMock()+ result.final_output = "Done."+ result.new_items = []+ result.to_input_list.return_value = []+ return result+++# ---------------------------------------------------------------------------+# 1 - credential_pause_reason detector+# ---------------------------------------------------------------------------+++def test_reason_fires_on_skipped_unbound_credentials() -> None:+ ctx = SimpleNamespace(last_run_skipped_unbound_credentials=True)+ assert credential_pause_module.credential_pause_reason(ctx) == "workflow_credential_inputs_unbound"+++def test_reason_ignores_skipped_unbound_credentials_when_user_explicitly_skipped_testing() -> None:+ """_request_policy_allows_update_and_skip_run (guardrails.py) skips the run for a+ skip_test-deferred credential too -- pausing here would contradict the same explicit+ request the credential_deferred_draft branch already excludes skip_test for."""+ policy = RequestPolicy(testing_intent="skip_test")+ ctx = SimpleNamespace(last_run_skipped_unbound_credentials=True, request_policy=policy)+ assert credential_pause_module.credential_pause_reason(ctx) is None+++def test_reason_fires_on_missing_credential_run_failure() -> None:+ ctx = SimpleNamespace(+ last_run_skipped_unbound_credentials=False,+ latest_diagnosis_repair_contract=_repair_contract(+ RepairNextAction.ASK, DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT+ ),+ )+ assert credential_pause_module.credential_pause_reason(ctx) == "missing_credential_run_failure"+++def test_reason_does_not_fire_when_ask_is_for_a_different_failure_type() -> None:+ ctx = SimpleNamespace(+ last_run_skipped_unbound_credentials=False,+ latest_diagnosis_repair_contract=_repair_contract(RepairNextAction.ASK, DiagnosisFailureType.UNKNOWN),+ )+ assert credential_pause_module.credential_pause_reason(ctx) is None+++def test_reason_ignores_missing_credential_or_init_when_not_categorized_as_credential_error() -> None:+ """MISSING_CREDENTIAL_OR_INIT also covers param-binding/lookup failures; only the+ CREDENTIAL_ERROR category should preempt with a card that can't unblock those."""+ ctx = SimpleNamespace(+ last_run_skipped_unbound_credentials=False,+ latest_diagnosis_repair_contract=_repair_contract(+ RepairNextAction.ASK,+ DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT,+ categories=("PARAMETER_BINDING_ERROR",),+ ),+ )+ assert credential_pause_module.credential_pause_reason(ctx) is None+++def test_reason_fires_on_credential_deferred_draft() -> None:+ policy = RequestPolicy(credential_draft_deferred_explicitly=True)+ ctx = SimpleNamespace(+ last_run_skipped_unbound_credentials=False,+ latest_diagnosis_repair_contract=None,+ request_policy=policy,+ update_workflow_called=True,+ )+ assert credential_pause_module.credential_pause_reason(ctx) == "credential_deferred_draft"+++def test_reason_requires_update_workflow_called_for_deferred_draft() -> None:+ policy = RequestPolicy(credential_draft_deferred_explicitly=True)+ ctx = SimpleNamespace(+ last_run_skipped_unbound_credentials=False,+ latest_diagnosis_repair_contract=None,+ request_policy=policy,+ update_workflow_called=False,+ )+ assert credential_pause_module.credential_pause_reason(ctx) is None+++def test_reason_ignores_deferred_draft_when_user_explicitly_skipped_testing() -> None:+ """skip_test means the user already said not to run/verify this -- pausing to+ ask for a credential would contradict that explicit request."""+ policy = RequestPolicy(credential_draft_deferred_explicitly=True, testing_intent="skip_test")+ ctx = SimpleNamespace(+ last_run_skipped_unbound_credentials=False,+ latest_diagnosis_repair_contract=None,+ request_policy=policy,+ update_workflow_called=True,+ )+ assert credential_pause_module.credential_pause_reason(ctx) is None+++def test_reason_ignores_text_marker_tier_that_credential_prompt_reason_catches() -> None:+ """Pins the SKY-11988 false-positive lesson: no text-marker fallback here."""+ policy = RequestPolicy()+ final_text = "I couldn't test this. Please add the credential via the Credentials UI."++ # The sibling function DOES classify this via its text-marker tier.+ assert credential_prompt_reason(policy, final_text) == "assistant_directed"++ ctx = SimpleNamespace(+ last_run_skipped_unbound_credentials=False,+ latest_diagnosis_repair_contract=None,+ request_policy=policy,+ update_workflow_called=True,+ )+ assert credential_pause_module.credential_pause_reason(ctx) is None+++def test_reason_none_when_no_signals_present() -> None:+ ctx = make_copilot_context()+ assert credential_pause_module.credential_pause_reason(ctx) is None+++# ---------------------------------------------------------------------------+# 2 - maybe_credential_pause waiter+# ---------------------------------------------------------------------------+++async def test_connected_action_mutates_policy_and_resolves(monkeypatch: pytest.MonkeyPatch) -> None:+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-1"+ ctx.client_supports_credential_pause = True+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")] = encode_credential_response(+ "connected", "cred_1"+ )+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ credential = _make_credential()+ monkeypatch.setattr(+ credential_pause_module.app,+ "DATABASE",+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[credential]))),+ )+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)++ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert resume_msgs is not None+ assert ctx.credential_pause_outcome == "connected"+ assert ctx.request_policy.resolved_credentials == [credential]+ assert ctx.request_policy.allow_run_blocks is True+ assert ctx.request_policy.clarification_reason == "none"+ assert ctx.request_policy.allow_missing_credentials_in_draft is False+ assert ctx.request_policy.requires_user_clarification is False+ resume_text = resume_msgs[-1]["content"]+ assert resume_text.startswith(NUDGE_SENTINEL)+ assert "cred_1" in resume_text+ sent_types = [call.args[0].type for call in stream.send.await_args_list]+ assert sent_types == [WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED]+++async def test_connected_action_unlatches_test_after_update_done(monkeypatch: pytest.MonkeyPatch) -> None:+ """The skipped run before the pause already stamped test_after_update_done=True;+ a connected resume must reset it so a reply that never re-runs the blocks still+ gets forced back through update_and_run_blocks by the post_update nudge."""+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-1"+ ctx.client_supports_credential_pause = True+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()+ ctx.update_workflow_called = True+ ctx.test_after_update_done = True++ cache = _FakeCache()+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")] = encode_credential_response(+ "connected", "cred_1"+ )+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(+ credential_pause_module.app,+ "DATABASE",+ SimpleNamespace(+ credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[_make_credential()]))+ ),+ )+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)++ resume_msgs = await maybe_credential_pause(+ ctx,+ _fake_result(),+ _make_stream(),+ CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5),+ )++ assert resume_msgs is not None+ assert ctx.test_after_update_done is False+++async def test_skip_action_leaves_policy_untouched(monkeypatch: pytest.MonkeyPatch) -> None:+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-2"+ ctx.client_supports_credential_pause = True+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-2")] = encode_credential_response("skip", None)+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)++ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert resume_msgs is not None+ assert ctx.credential_pause_outcome == "skipped"+ assert ctx.request_policy.resolved_credentials == []+ assert ctx.request_policy == RequestPolicy()+ assert "chose not to connect a credential now" in resume_msgs[-1]["content"]+++async def test_skip_clears_stale_last_test_ok_from_the_diagnosed_failure(monkeypatch: pytest.MonkeyPatch) -> None:+ """A missing_credential_run_failure pause means the diagnosed run left+ last_test_ok=False; skip must clear it or the resumed reply is intercepted+ by the generic failed-test nudge instead of honoring the skip decision."""+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-skip-clears"+ ctx.client_supports_credential_pause = True+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.last_test_ok = False+ ctx.test_after_update_done = True+ ctx.latest_diagnosis_repair_contract = _repair_contract(+ RepairNextAction.ASK, DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT+ )+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-skip-clears")] = encode_credential_response(+ "skip", None+ )+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)++ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert resume_msgs is not None+ assert ctx.last_test_ok is None+ assert _check_enforcement(ctx, result=None, config=config) is None+++async def test_timeout_returns_none_and_marks_outcome(monkeypatch: pytest.MonkeyPatch) -> None:+ ctx = make_copilot_context()+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache() # never populated+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)++ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=0)++ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert resume_msgs is None+ assert ctx.credential_pause_outcome == "timeout"+ assert ctx.copilot_credential_pause_seconds >= 0.0+++async def test_response_racing_in_at_the_timeout_instant_is_rescued(monkeypatch: pytest.MonkeyPatch) -> None:+ """Fails on old code: resolve_credential_pause writes under a per-turn lock,+ but the timeout invalidate wrote the same active-pause key unlocked -- a+ response that lands in the same instant the waiter gives up would validate+ against the still-pending record (204 to the client) yet never get read,+ since the waiter already returned None. The invalidate must re-check under+ the same lock and use the response if one raced in, not silently drop it."""+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-race"+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)++ async def fake_wait_races_in_a_response(response_key: str, ctx: Any, stream: Any, timeout_seconds: int) -> None:+ # Simulates resolve_credential_pause's lock-protected write landing in+ # the same instant the waiter's own poll loop concludes None.+ cache.store[response_key] = encode_credential_response("skip", None)+ return None++ monkeypatch.setattr(credential_pause_module, "_wait_for_credential_response", fake_wait_races_in_a_response)++ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert resume_msgs is not None+ assert ctx.credential_pause_outcome == "skipped"+++async def test_client_disconnect_mid_wait_degrades_early(monkeypatch: pytest.MonkeyPatch) -> None:+ ctx = make_copilot_context()+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache() # never populated+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)++ stream = MagicMock()+ stream.send = AsyncMock(return_value=True)+ # Connected for the initial guard and the post-send re-check, gone by the first poll.+ stream.is_disconnected = AsyncMock(side_effect=[False, False, True])+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)++ start = time.monotonic()+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)+ elapsed = time.monotonic() - start++ assert resume_msgs is None+ assert ctx.credential_pause_outcome == "timeout"+ assert elapsed < 1.0 # degraded on the first poll, not the full 30s timeout+++async def test_disconnect_racing_the_send_itself_declines_without_waiting(monkeypatch: pytest.MonkeyPatch) -> None:+ """Fails on old code: send() returns True even when the client is already+ gone (its own contract), so a disconnect that lands between the pre-send+ guard and delivery would otherwise wait out the full timeout for a card+ nobody ever saw, and land on 'timeout' instead of 'declined'."""+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-race-send"+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)++ stream = MagicMock()+ stream.send = AsyncMock(return_value=True)+ # Connected for the pre-send guard, gone by the time send() returns.+ stream.is_disconnected = AsyncMock(side_effect=[False, True])+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)++ start = time.monotonic()+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)+ elapsed = time.monotonic() - start++ assert resume_msgs is None+ assert ctx.credential_pause_outcome == "declined"+ assert elapsed < 1.0 # never entered the wait loop+++async def test_already_disconnected_before_send_declines_without_sending_a_frame(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The predictor (credential_pause_would_fire) excludes the async-only+ disconnect check by design -- a client gone before the frame is even sent+ must latch and tag 'declined' (not 'timeout') so the caller can fall back+ to a normal nudge instead of treating this like a delivered-and-waited pause."""+ ctx = make_copilot_context()+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)++ stream = _make_stream(disconnected=True)+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)++ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert resume_msgs is None+ assert ctx.credential_pause_outcome == "declined"+ assert ctx.credential_pause_used is True+ stream.send.assert_not_called()+++async def test_disconnect_invalidates_the_active_pause_record(monkeypatch: pytest.MonkeyPatch) -> None:+ """A disconnect degrades well before expires_at; a late POST after the tab+ reconnects must be rejected instead of silently writing an unread response."""+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-disconnect"+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)++ stream = MagicMock()+ stream.send = AsyncMock(return_value=True)+ stream.is_disconnected = AsyncMock(side_effect=[False, True])+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)++ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)+ assert resume_msgs is None++ frame = stream.send.await_args_list[0].args[0]+ with pytest.raises(CredentialPauseRejection) as excinfo:+ await resolve_credential_pause(+ cache,+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ turn_id="turn-disconnect",+ resume_token=frame.resume_token,+ action="skip",+ credential_id=None,+ )+ assert excinfo.value.status_code == status.HTTP_409_CONFLICT+++async def test_no_cache_returns_none_without_sending_a_frame(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(app._inst, "CACHE", None, raising=False)+ ctx = make_copilot_context()+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True)++ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert resume_msgs is None+ stream.send.assert_not_awaited()+++async def test_cancelled_error_propagates_and_still_accumulates_pause_seconds(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-cancel"+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)++ async def _raise_cancelled(*args: object, **kwargs: object) -> None:+ raise asyncio.CancelledError()++ monkeypatch.setattr(credential_pause_module.asyncio, "sleep", _raise_cancelled)++ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)++ with pytest.raises(asyncio.CancelledError):+ await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert ctx.copilot_credential_pause_seconds > 0.0+ assert ctx.credential_pause_outcome is None # neither timeout nor resolved path ran++ frame = stream.send.await_args_list[0].args[0]+ with pytest.raises(CredentialPauseRejection) as excinfo:+ await resolve_credential_pause(+ cache,+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ turn_id="turn-cancel",+ resume_token=frame.resume_token,+ action="skip",+ credential_id=None,+ )+ assert excinfo.value.status_code == status.HTTP_409_CONFLICT+++async def test_unexpected_error_in_waiter_also_invalidates_the_active_pause_record(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Not just CancelledError: any unexpected failure in the wait loop leaves the+ frame's resume token live client-side, so the record must be invalidated too."""+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-crash"+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)++ async def _raise_runtime_error(*args: object, **kwargs: object) -> None:+ raise RuntimeError("cache connection reset")++ monkeypatch.setattr(credential_pause_module.asyncio, "sleep", _raise_runtime_error)++ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)++ with pytest.raises(RuntimeError):+ await maybe_credential_pause(ctx, _fake_result(), stream, config)++ frame = stream.send.await_args_list[0].args[0]+ with pytest.raises(CredentialPauseRejection) as excinfo:+ await resolve_credential_pause(+ cache,+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ turn_id="turn-crash",+ resume_token=frame.resume_token,+ action="skip",+ credential_id=None,+ )+ assert excinfo.value.status_code == status.HTTP_409_CONFLICT+++async def test_invalid_credential_id_degrades_instead_of_crashing(monkeypatch: pytest.MonkeyPatch) -> None:+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-3"+ ctx.client_supports_credential_pause = True+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-3")] = encode_credential_response(+ "connected", "cred_foreign"+ )+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(+ credential_pause_module.app,+ "DATABASE",+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[]))),+ )+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)++ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert resume_msgs is None+ assert ctx.credential_pause_outcome == "timeout"+ assert ctx.request_policy.resolved_credentials == []+++async def test_full_round_trip_binds_frame_token_to_active_pause(monkeypatch: pytest.MonkeyPatch) -> None:+ """End-to-end: establishing the pause writes an active record + one-time token,+ the response endpoint resolves only with that token, and the resolved pause is+ consumed so a replay conflicts.+ """+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-rt"+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)++ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)+ active_key = credential_pause_active_key("org-1", "chat-1", "turn-rt")++ async def _respond_using_frame_token() -> None:+ # Read the one-time token exactly as the FE would from the frame, then resolve.+ for _ in range(2000):+ raw = await cache.get(active_key)+ record = json.loads(raw) if raw else None+ if record and record.get("status") == "pending":+ await resolve_credential_pause(+ cache,+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ turn_id="turn-rt",+ resume_token=record["resume_token"],+ action="skip",+ credential_id=None,+ )+ return+ await asyncio.sleep(0.001)++ responder = asyncio.ensure_future(_respond_using_frame_token())+ try:+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)+ finally:+ await responder++ assert resume_msgs is not None+ assert ctx.credential_pause_outcome == "skipped"+ frame = stream.send.await_args_list[0].args[0]+ assert frame.resume_token++ with pytest.raises(CredentialPauseRejection) as excinfo:+ await resolve_credential_pause(+ cache,+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ turn_id="turn-rt",+ resume_token=frame.resume_token,+ action="skip",+ credential_id=None,+ )+ assert excinfo.value.status_code == status.HTTP_409_CONFLICT+++# ---------------------------------------------------------------------------+# 5 - one-pause-per-turn latch / 6 - kill-switch-off parity+# ---------------------------------------------------------------------------+++async def test_latch_prevents_second_pause_this_turn() -> None:+ ctx = make_copilot_context()+ ctx.credential_pause_used = True+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()+ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True)++ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert resume_msgs is None+ stream.send.assert_not_awaited()+++async def test_kill_switch_off_returns_none_without_touching_cache_or_stream() -> None:+ ctx = make_copilot_context()+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()+ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=False)++ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert resume_msgs is None+ stream.send.assert_not_awaited()+ stream.is_disconnected.assert_not_awaited()+++# ---------------------------------------------------------------------------+# 3 - run_with_enforcement loop integration (regression pin)+# ---------------------------------------------------------------------------+++async def test_loop_pauses_at_finalize_and_resumes_same_turn(monkeypatch: pytest.MonkeyPatch) -> None:+ """Fails on old code: without the pause hook this returns on the first finalize."""+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-loop"+ ctx.client_supports_credential_pause = True+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-loop")] = encode_credential_response(+ "skip", None+ )+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)++ stream = _make_stream()+ fake_result = _fake_result()+ calls: list[dict[str, Any]] = []++ def fake_run_streamed(*args: Any, **kwargs: Any) -> Any:+ calls.append(kwargs)+ return fake_result++ async def fake_stream_to_sse(result: Any, s: Any, c: Any) -> None:+ return None++ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", fake_run_streamed)+ monkeypatch.setattr("skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse", fake_stream_to_sse)++ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ returned = await run_with_enforcement(+ agent=MagicMock(),+ initial_input="hello",+ ctx=ctx,+ stream=stream,+ run_config=RunConfig(),+ copilot_config=config,+ )++ assert returned is fake_result+ assert len(calls) == 2, "second Runner invocation must happen after the pause resumes"+ second_call_input = calls[1]["input"]+ assert any(NUDGE_SENTINEL in item.get("content", "") for item in second_call_input)+ assert ctx.credential_pause_used is True+ frame_types = [call.args[0].type for call in stream.send.await_args_list]+ assert frame_types.count(WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED) == 1+++async def test_credential_pause_preempts_a_concurrent_synthesized_offer(monkeypatch: pytest.MonkeyPatch) -> None:+ """Fails on old code: the finalize branch only tried the pause when nudge AND+ synthesized_msg were both None, so a reopened synthesized-block offer coinciding+ with a credential diagnosis sent the offer instead of the credential_required frame.+ """+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-offer-race"+ ctx.client_supports_credential_pause = True+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-offer-race")] = encode_credential_response(+ "skip", None+ )+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)++ stream = _make_stream()+ fake_result = _fake_result()+ calls: list[dict[str, Any]] = []++ def fake_synthesized_offer(_ctx: Any) -> dict[str, Any] | None:+ # _check_enforcement runs after the Runner call, so `calls` already has+ # 1 entry on the pre-pause finalize check; gone by the post-resume check+ # (2 entries), so the test isolates the preemption itself.+ return {"role": "user", "content": "offer to add the reopened block"} if len(calls) <= 1 else None++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.enforcement._maybe_synthesized_block_offer_msg",+ fake_synthesized_offer,+ )++ def fake_run_streamed(*args: Any, **kwargs: Any) -> Any:+ calls.append(kwargs)+ return fake_result++ async def fake_stream_to_sse(result: Any, s: Any, c: Any) -> None:+ return None++ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", fake_run_streamed)+ monkeypatch.setattr("skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse", fake_stream_to_sse)++ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ await run_with_enforcement(+ agent=MagicMock(),+ initial_input="hello",+ ctx=ctx,+ stream=stream,+ run_config=RunConfig(),+ copilot_config=config,+ )++ frame_types = [call.args[0].type for call in stream.send.await_args_list]+ assert frame_types.count(WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED) == 1+ # On old code the offer nudge wins the first check (pause only fires once the+ # offer stops recurring), costing an extra Runner call before the skip resume+ # text appears -- this pins that the pause wins on the very first check.+ assert len(calls) == 2+ second_call_input = calls[1]["input"]+ assert any("chose not to connect a credential now" in item.get("content", "") for item in second_call_input)+++async def test_declined_pause_falls_back_to_the_normal_nudge_instead_of_finalizing(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Fails on old code: an already-disconnected stream made maybe_credential_pause+ return None with nothing sent, and run_with_enforcement finalized the turn as-is+ instead of the post_update nudge this iteration would otherwise have gotten."""+ ctx = make_copilot_context()+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.update_workflow_called = True+ ctx.test_after_update_done = False+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)++ stream = _make_stream(disconnected=True)+ fake_result = _fake_result()+ calls: list[dict[str, Any]] = []++ def fake_run_streamed(*args: Any, **kwargs: Any) -> Any:+ calls.append(kwargs)+ return fake_result++ async def fake_stream_to_sse(result: Any, s: Any, c: Any) -> None:+ return None++ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", fake_run_streamed)+ monkeypatch.setattr("skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse", fake_stream_to_sse)++ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)++ await run_with_enforcement(+ agent=MagicMock(),+ initial_input="hello",+ ctx=ctx,+ stream=stream,+ run_config=RunConfig(),+ copilot_config=config,+ )++ assert ctx.credential_pause_outcome == "declined"+ stream.send.assert_not_called()+ # On old code this returns after the first call (nudge=None, pause declined,+ # synthesized_msg=None -> immediate finalize). The fallback must fire the+ # post_update nudge instead; MAX_POST_UPDATE_NUDGES allows retries before the+ # fake (which never "tests" anything) exhausts them, so >1 call is the claim.+ assert len(calls) > 1, "the post_update nudge must fire instead of an immediate finalize"+ second_call_input = calls[1]["input"]+ assert any("did not test it" in item.get("content", "") for item in second_call_input)+++# ---------------------------------------------------------------------------+# 4 - timeout credit+# ---------------------------------------------------------------------------+++def test_elapsed_run_seconds_subtracts_pause_time() -> None:+ ctx = SimpleNamespace(copilot_credential_pause_seconds=50.0)+ start_time = time.monotonic() - 60.0++ elapsed = _elapsed_run_seconds(ctx, start_time)++ assert 9.0 < elapsed < 11.0+++async def test_paused_loop_does_not_trip_total_timeout_on_resume(monkeypatch: pytest.MonkeyPatch) -> None:+ """Fails on old code: a slow pause would consume the (tiny) real timeout budget."""+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 0.15)++ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-credit"+ ctx.client_supports_credential_pause = True+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.2)++ async def _populate_after_first_poll() -> None:+ await asyncio.sleep(0.25)+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-credit")] = encode_credential_response(+ "skip", None+ )++ stream = _make_stream()+ fake_result = _fake_result()+ calls: list[dict[str, Any]] = []++ def fake_run_streamed(*args: Any, **kwargs: Any) -> Any:+ calls.append(kwargs)+ return fake_result++ async def fake_stream_to_sse(result: Any, s: Any, c: Any) -> None:+ return None++ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", fake_run_streamed)+ monkeypatch.setattr("skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse", fake_stream_to_sse)++ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ populate_task = asyncio.ensure_future(_populate_after_first_poll())+ try:+ returned = await run_with_enforcement(+ agent=MagicMock(),+ initial_input="hello",+ ctx=ctx,+ stream=stream,+ run_config=RunConfig(),+ copilot_config=config,+ )+ finally:+ await populate_task++ assert returned is fake_result+ assert len(calls) == 2, "pause time must be credited so the resumed iteration isn't timed out"+ assert ctx.copilot_total_timeout_exceeded is False+++# ---------------------------------------------------------------------------+# 7 - /workflow/copilot/credential-response route+# ---------------------------------------------------------------------------+++def _response_request(+ *,+ turn_id: str = "turn-1",+ chat_id: str = "chat-1",+ resume_token: str = "tok-1",+ action: str = "skip",+ credential_id: str | None = None,+) -> WorkflowCopilotCredentialResponseRequest:+ return WorkflowCopilotCredentialResponseRequest(+ turn_id=turn_id,+ workflow_copilot_chat_id=chat_id,+ resume_token=resume_token,+ action=action, # type: ignore[arg-type]+ credential_id=credential_id,+ )+++async def test_route_503_when_cache_missing(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(app._inst, "CACHE", None, raising=False)+ organization = SimpleNamespace(organization_id="org-1")++ with pytest.raises(HTTPException) as excinfo:+ await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)+ assert excinfo.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE+++async def test_route_204_writes_flag_for_skip_without_credential_id(monkeypatch: pytest.MonkeyPatch) -> None:+ cache = _FakeCache()+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ organization = SimpleNamespace(organization_id="org-1")++ result = await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)++ assert result is None+ expected_key = credential_response_cache_key("org-1", "chat-1", "turn-1")+ assert cache.store[expected_key] == encode_credential_response("skip", None)+ response_set = next(call for call in cache.set_calls if call[0] == expected_key)+ assert response_set[1] == encode_credential_response("skip", None)+ assert response_set[2] == timedelta(seconds=settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS + 300)+++async def test_route_422_when_connected_without_credential_id(monkeypatch: pytest.MonkeyPatch) -> None:+ cache = _FakeCache()+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ organization = SimpleNamespace(organization_id="org-1")++ with pytest.raises(HTTPException) as excinfo:+ await workflow_copilot_credential_response(_response_request(action="connected"), organization=organization)+ assert excinfo.value.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY+ assert cache.set_calls == []+++async def test_route_404_when_credential_unknown_or_foreign_org(monkeypatch: pytest.MonkeyPatch) -> None:+ cache = _FakeCache()+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(+ app,+ "DATABASE",+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[]))),+ )+ organization = SimpleNamespace(organization_id="org-1")++ with pytest.raises(HTTPException) as excinfo:+ await workflow_copilot_credential_response(+ _response_request(action="connected", credential_id="cred_other_org"),+ organization=organization,+ )+ assert excinfo.value.status_code == status.HTTP_404_NOT_FOUND+ assert cache.set_calls == []+++async def test_route_204_writes_flag_for_connected_with_valid_credential(monkeypatch: pytest.MonkeyPatch) -> None:+ cache = _FakeCache()+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(+ app,+ "DATABASE",+ SimpleNamespace(+ credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[_make_credential()]))+ ),+ )+ organization = SimpleNamespace(organization_id="org-1")++ result = await workflow_copilot_credential_response(+ _response_request(action="connected", credential_id="cred_1"),+ organization=organization,+ )++ assert result is None+ expected_key = credential_response_cache_key("org-1", "chat-1", "turn-1")+ assert cache.store[expected_key] == encode_credential_response("connected", "cred_1")+++async def test_route_404_when_no_active_pause_record(monkeypatch: pytest.MonkeyPatch) -> None:+ """A valid-looking token can't resolve a turn that was never paused."""+ cache = _FakeCache() # no active-pause record seeded+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ organization = SimpleNamespace(organization_id="org-1")++ with pytest.raises(HTTPException) as excinfo:+ await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)+ assert excinfo.value.status_code == status.HTTP_404_NOT_FOUND+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store+++async def test_route_403_when_resume_token_mismatches(monkeypatch: pytest.MonkeyPatch) -> None:+ """A leaked/guessed turn+chat id without the frame's token is rejected."""+ cache = _FakeCache()+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-real")+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ organization = SimpleNamespace(organization_id="org-1")++ with pytest.raises(HTTPException) as excinfo:+ await workflow_copilot_credential_response(+ _response_request(action="skip", resume_token="tok-wrong"),+ organization=organization,+ )+ assert excinfo.value.status_code == status.HTTP_403_FORBIDDEN+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store+++async def test_route_rejects_bad_token_before_looking_up_credential_id(monkeypatch: pytest.MonkeyPatch) -> None:+ """A bad token must not reach the credential-id DB lookup -- org auth alone+ can't authorize this turn, and doing the lookup first turns a wrong-token+ response into a small authenticated credential-id existence oracle."""+ cache = _FakeCache()+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-real")+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ lookup = AsyncMock(return_value=[_make_credential(credential_id="cred_real")])+ monkeypatch.setattr(app, "DATABASE", SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=lookup)))+ organization = SimpleNamespace(organization_id="org-1")++ with pytest.raises(HTTPException) as excinfo:+ await workflow_copilot_credential_response(+ _response_request(action="connected", credential_id="cred_real", resume_token="tok-wrong"),+ organization=organization,+ )+ assert excinfo.value.status_code == status.HTTP_403_FORBIDDEN+ lookup.assert_not_called()+++async def test_route_404_when_chat_id_does_not_match_the_pause(monkeypatch: pytest.MonkeyPatch) -> None:+ """The pause is keyed by chat+turn: a response with the right turn but a foreign chat misses it."""+ cache = _FakeCache()+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ organization = SimpleNamespace(organization_id="org-1")++ with pytest.raises(HTTPException) as excinfo:+ await workflow_copilot_credential_response(+ _response_request(action="skip", chat_id="chat-foreign"),+ organization=organization,+ )+ assert excinfo.value.status_code == status.HTTP_404_NOT_FOUND+++async def test_route_409_on_replay_after_first_accepted_response(monkeypatch: pytest.MonkeyPatch) -> None:+ """First accepted response consumes the pause; a replay of the same POST is rejected."""+ cache = _FakeCache()+ _seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ organization = SimpleNamespace(organization_id="org-1")++ first = await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)+ assert first is None++ with pytest.raises(HTTPException) as excinfo:+ await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)+ assert excinfo.value.status_code == status.HTTP_409_CONFLICT+++async def test_route_410_when_response_arrives_after_the_frame_deadline(monkeypatch: pytest.MonkeyPatch) -> None:+ """The record's Redis TTL outlives the wait window for grace on a slow poll,+ but a response after the deadline the frame told the client about must still+ be rejected -- the waiter already gave up and the turn finalized without it."""+ cache = _FakeCache()+ _seed_active_pause(+ cache, "org-1", "chat-1", "turn-1", "tok-1", expires_at=datetime.now(timezone.utc) - timedelta(seconds=1)+ )+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ organization = SimpleNamespace(organization_id="org-1")++ with pytest.raises(HTTPException) as excinfo:+ await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)+ assert excinfo.value.status_code == status.HTTP_410_GONE+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store+++async def test_resolve_credential_pause_rejections_are_typed() -> None:+ """resolve_credential_pause raises CredentialPauseRejection (not HTTPException) so it stays route-agnostic."""+ cache = _FakeCache()++ with pytest.raises(CredentialPauseRejection) as excinfo:+ await resolve_credential_pause(+ cache,+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ turn_id="turn-1",+ resume_token="tok-1",+ action="skip",+ credential_id=None,+ )+ assert excinfo.value.status_code == status.HTTP_404_NOT_FOUND+++# ---------------------------------------------------------------------------+# A diagnosed missing-credential run failure must pre-empt the generic+# failed-test nudge, or the pause hook (gated on nudge is None) never runs on+# the turn's first credential ask.+# ---------------------------------------------------------------------------+++def _failed_test_ctx() -> CopilotContext:+ """A real CopilotContext (all enforcement fields defaulted) with just the+ failed-test + missing-credential-diagnosis fields set, so _check_enforcement+ doesn't AttributeError on some unrelated field this hand-picked set omits.+ """+ ctx = make_copilot_context()+ ctx.client_supports_credential_pause = True+ ctx.last_test_ok = False+ ctx.test_after_update_done = True+ ctx.latest_diagnosis_repair_contract = _repair_contract(+ RepairNextAction.ASK, DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT+ )+ ctx.request_policy = RequestPolicy()+ return ctx+++def test_missing_credential_failure_suppresses_generic_failed_test_nudge_when_pause_enabled() -> None:+ """Fails on old code: the generic 'post_failed_test' nudge fires here instead."""+ ctx = _failed_test_ctx()+ config = CopilotConfig(credential_pause_enabled=True)++ nudge = _check_enforcement(ctx, result=None, config=config)++ assert nudge is None+ assert ctx.failed_test_nudge_count == 0+++def test_missing_credential_failure_still_nudges_when_kill_switch_off() -> None:+ """Flag-off parity: unrelated to the fix, the generic nudge still fires as before."""+ ctx = _failed_test_ctx()+ config = CopilotConfig(credential_pause_enabled=False)++ nudge = _check_enforcement(ctx, result=None, config=config)++ assert nudge is not None+ assert ctx.failed_test_nudge_count == 1+++def test_missing_credential_failure_nudges_normally_once_pause_already_used() -> None:+ ctx = _failed_test_ctx()+ ctx.credential_pause_used = True+ config = CopilotConfig(credential_pause_enabled=True)++ nudge = _check_enforcement(ctx, result=None, config=config)++ assert nudge is not None+ assert ctx.failed_test_nudge_count == 1+++def _skipped_run_ctx() -> CopilotContext:+ """Defensive fixture: even if test_after_update_done were False after a+ skipped run (in the real runtime it isn't -- streaming_adapter's+ _update_enforcement_from_tool sets it True unconditionally for+ update_and_run_blocks/run_blocks_and_collect_debug regardless of skip;+ see test_workflow_credential_inputs_unbound_skip_does_not_nudge_post_update+ for the real-state pin), the pause must still pre-empt the post_update+ nudge, a second nudge that can race the pause hook.+ """+ ctx = make_copilot_context()+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.update_workflow_called = True+ ctx.test_after_update_done = False+ ctx.request_policy = RequestPolicy()+ return ctx+++def test_skipped_run_suppresses_post_update_nudge_when_pause_enabled() -> None:+ """Fails on old code: the earlier 'post_update' nudge fires instead."""+ ctx = _skipped_run_ctx()+ config = CopilotConfig(credential_pause_enabled=True)++ nudge = _check_enforcement(ctx, result=None, config=config)++ assert nudge is None+++def test_skipped_run_still_nudges_post_update_when_kill_switch_off() -> None:+ ctx = _skipped_run_ctx()+ config = CopilotConfig(credential_pause_enabled=False)++ nudge = _check_enforcement(ctx, result=None, config=config)++ assert nudge is not None+ assert "updated the workflow but did not test it" in nudge+++def test_workflow_credential_inputs_unbound_skip_does_not_nudge_post_update() -> None:+ """Pins the real post-skip state (not the defensive _skipped_run_ctx fixture):+ _update_enforcement_from_tool sets test_after_update_done=True unconditionally+ for update_and_run_blocks, even on the credential-unbound skip branch, so the+ post_update nudge condition (not test_after_update_done) is already False by+ the time enforcement runs -- verified via a direct call to that function with+ the real skip_result shape from tools/__init__.py's skip branch."""+ ctx = make_copilot_context()+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.update_workflow_called = True+ ctx.test_after_update_done = True+ ctx.request_policy = RequestPolicy()+ config = CopilotConfig(credential_pause_enabled=True)++ assert _check_enforcement(ctx, result=None, config=config) is None++ ctx.credential_pause_used = True+ assert _check_enforcement(ctx, result=None, config=config) is None+++async def test_missing_credential_run_failure_pauses_the_loop(monkeypatch: pytest.MonkeyPatch) -> None:+ """Loop-level pin for the same fix, through the real finalize seam."""+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-nudge-race"+ ctx.client_supports_credential_pause = True+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.last_test_ok = False+ ctx.test_after_update_done = True+ ctx.latest_diagnosis_repair_contract = _repair_contract(+ RepairNextAction.ASK, DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT+ )+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-nudge-race")] = encode_credential_response(+ "skip", None+ )+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)++ stream = _make_stream()+ fake_result = _fake_result()+ calls: list[dict[str, Any]] = []++ def fake_run_streamed(*args: Any, **kwargs: Any) -> Any:+ calls.append(kwargs)+ return fake_result++ async def fake_stream_to_sse(result: Any, s: Any, c: Any) -> None:+ return None++ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", fake_run_streamed)+ monkeypatch.setattr("skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse", fake_stream_to_sse)++ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ returned = await run_with_enforcement(+ agent=MagicMock(),+ initial_input="hello",+ ctx=ctx,+ stream=stream,+ run_config=RunConfig(),+ copilot_config=config,+ )++ assert returned is fake_result+ # If maybe_credential_pause didn't clear the stale last_test_ok=False on+ # skip, the resumed reply would be intercepted by the failed-test nudge+ # for a 3rd Runner call instead of finalizing here.+ assert len(calls) == 2+ frame_types = [call.args[0].type for call in stream.send.await_args_list]+ assert frame_types.count(WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED) == 1+++# ---------------------------------------------------------------------------+# last_run_skipped_unbound_credentials must reflect the MOST RECENT+# update_and_run_blocks call, not "ever skipped this turn" — otherwise a+# later successful call leaves a stale True and pauses an already-passing turn.+# ---------------------------------------------------------------------------+++async def test_last_run_skipped_flag_clears_on_a_later_successful_call(monkeypatch: pytest.MonkeyPatch) -> None:+ """Fails on old code: the flag stays True after the second, successful call."""+ ctx = make_copilot_context()+ ctx.turn_intent = TurnIntent(+ mode=TurnIntentMode.BUILD,+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),+ )+ ctx.request_policy = RequestPolicy()++ workflow_yaml = "workflow_definition:\n parameters: []\n blocks:\n - block_type: code\n label: step_one\n"++ async def fake_prior_definition(update_ctx: Any) -> object:+ return None++ async def fake_update_workflow(payload: dict, update_ctx: Any, **kwargs: object) -> dict:+ workflow = SimpleNamespace(workflow_definition={"blocks": [{"label": "step_one"}]})+ update_ctx.last_workflow = workflow+ update_ctx.last_update_block_count = 1+ return {"ok": True, "_workflow": workflow, "data": {"block_count": 1}}++ async def fake_run_blocks(params: dict, run_ctx: Any, **kwargs: object) -> dict:+ return {"ok": True, "data": {"workflow_run_id": "wr-1", "overall_status": "completed", "blocks": []}}++ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)+ monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)+ monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *a, **k: None)+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", fake_prior_definition)+ monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)+ monkeypatch.setattr(tools_module, "_plan_frontier", lambda *args: (["step_one"], {}, "step_one"))+ monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *args: None)+ monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", fake_run_blocks)+ monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)+ monkeypatch.setattr(tools_module, "enqueue_screenshot_from_result", lambda *args, **kwargs: None)++ call_args = json.dumps({"workflow_yaml": workflow_yaml, "block_labels": ["step_one"], "parameters": {}})++ # Call 1: policy forces a skip (unbound credential).+ monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: True)+ result_1 = await tools_module.update_and_run_blocks_tool.on_invoke_tool(+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"), call_args+ )+ assert json.loads(result_1)["data"]["skip_reason"] == "workflow_credential_inputs_unbound"+ assert ctx.last_run_skipped_unbound_credentials is True++ # Call 2: credential now bound, policy allows the real run.+ monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: False)+ result_2 = await tools_module.update_and_run_blocks_tool.on_invoke_tool(+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"), call_args+ )+ assert json.loads(result_2)["ok"] is True+ assert ctx.last_run_skipped_unbound_credentials is False+ assert credential_pause_module.credential_pause_reason(ctx) is None+++async def test_last_run_skipped_flag_stays_false_when_update_workflow_fails_before_skip_branch(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Fails on old code: the flag was set True from the policy check alone,+ before _update_workflow ever ran, so an unrelated authoring failure got+ misreported as a credential ask.+ """+ ctx = make_copilot_context()+ ctx.turn_intent = TurnIntent(+ mode=TurnIntentMode.BUILD,+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),+ )+ ctx.request_policy = RequestPolicy()+ workflow_yaml = "workflow_definition:\n parameters: []\n blocks:\n - block_type: code\n label: step_one\n"++ async def fake_prior_definition(update_ctx: Any) -> object:+ return None++ async def failing_update_workflow(payload: dict, update_ctx: Any, **kwargs: object) -> dict:+ return {"ok": False, "error": "workflow_yaml is not valid: bad block reference"}++ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)+ monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)+ monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *a, **k: None)+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", fake_prior_definition)+ monkeypatch.setattr(tools_module, "_update_workflow", failing_update_workflow)+ monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)+ # The policy would allow a skip if we got that far — but we never do.+ monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: True)++ result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),+ json.dumps({"workflow_yaml": workflow_yaml, "block_labels": ["step_one"], "parameters": {}}),+ )++ assert json.loads(result)["ok"] is False+ assert ctx.last_run_skipped_unbound_credentials is False+ assert credential_pause_module.credential_pause_reason(ctx) is None+++# ---------------------------------------------------------------------------+# _copilot_seconds_remaining must credit pause time, or a long pause late in+# the turn FORBIDS the very test the connected credential was for+# (blockers.py's _late_block_running_call_signal).+# ---------------------------------------------------------------------------+++def test_copilot_seconds_remaining_credits_pause_time() -> None:+ from skyvern.forge.sdk.copilot.tools._shared import _copilot_seconds_remaining++ ctx = SimpleNamespace(copilot_run_start_monotonic=time.monotonic() - 1000.0, copilot_credential_pause_seconds=300.0)++ remaining = _copilot_seconds_remaining(ctx)++ # Uncredited: 900 - 1000 = -100s (would forbid). Credited: 900 - 700 = 200s.+ assert remaining is not None+ assert remaining > 150.0+++def test_late_block_running_call_signal_allows_call_after_credited_pause() -> None:+ """Fails on old code: a 300s pause 700s into the turn wrongly forbids the+ resumed test run because the wall-clock check never saw the pause credit."""+ from skyvern.forge.sdk.copilot.tools.blockers import _late_block_running_call_signal++ ctx = SimpleNamespace(+ copilot_run_start_monotonic=time.monotonic() - 1000.0,+ copilot_credential_pause_seconds=300.0,+ last_failed_workflow_yaml=None,+ last_good_workflow_yaml=None,+ )++ assert _late_block_running_call_signal(ctx, "update_and_run_blocks") is None+++def test_late_block_running_call_signal_forbids_same_elapsed_without_pause_credit() -> None:+ """Sanity pair for the test above: the identical wall-clock elapsed DOES+ forbid the call when there was no pause to credit, proving the crediting+ -- not something else -- is what keeps the resumed call allowed."""+ from skyvern.forge.sdk.copilot.tools.blockers import _late_block_running_call_signal++ ctx = SimpleNamespace(+ copilot_run_start_monotonic=time.monotonic() - 1000.0,+ copilot_credential_pause_seconds=0.0,+ last_failed_workflow_yaml=None,+ last_good_workflow_yaml=None,+ )++ assert _late_block_running_call_signal(ctx, "update_and_run_blocks") is not None+++# ---------------------------------------------------------------------------+# A client that can't render credential_required must never be paused for,+# or it stares at "Working..." until the timeout.+# ---------------------------------------------------------------------------+++def test_would_fire_false_when_client_does_not_support_pause() -> None:+ ctx = make_copilot_context()+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()+ config = CopilotConfig(credential_pause_enabled=True)++ assert credential_pause_module.credential_pause_would_fire(ctx, config) is False+++def test_would_fire_true_once_client_support_is_set() -> None:+ ctx = make_copilot_context()+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()+ config = CopilotConfig(credential_pause_enabled=True)++ assert credential_pause_module.credential_pause_would_fire(ctx, config) is True+++async def test_unsupported_client_never_pauses_even_with_everything_else_true(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Fails on old code: no client-support guard existed, so a pre-12139 FE+ build (or the env flag on with an unwired client) got paused and the FE+ silently dropped the unknown frame -- a guaranteed ~timeout-length hang."""+ ctx = make_copilot_context()+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()+ # ctx.client_supports_credential_pause left at its False default.++ cache = _FakeCache()+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert resume_msgs is None+ stream.send.assert_not_awaited()+++# ---------------------------------------------------------------------------+# A successful connect must clear credential_draft_deferred_explicitly, or+# credential_prompt_reason() keeps stamping credentialPrompt right next to+# credentialPause: connected.+# ---------------------------------------------------------------------------+++async def test_connect_clears_credential_draft_deferred_explicitly(monkeypatch: pytest.MonkeyPatch) -> None:+ """Fails on old code: the flag survives a successful connect, so the FE+ would see a contradictory connect-CTA right after the user connected."""+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-connect-clear"+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy(credential_draft_deferred_explicitly=True)++ cache = _FakeCache()+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-connect-clear")] = encode_credential_response(+ "connected", "cred_1"+ )+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ credential = _make_credential()+ monkeypatch.setattr(+ credential_pause_module.app,+ "DATABASE",+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[credential]))),+ )+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)++ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert ctx.request_policy.credential_draft_deferred_explicitly is False+ assert credential_prompt_reason(ctx.request_policy, "any final text") is None+++async def test_skip_leaves_credential_draft_deferred_explicitly_untouched(monkeypatch: pytest.MonkeyPatch) -> None:+ """Skip/timeout correctly keep the flag -- only a successful connect clears it."""+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-skip-keep"+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy(credential_draft_deferred_explicitly=True)++ cache = _FakeCache()+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-skip-keep")] = encode_credential_response(+ "skip", None+ )+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)++ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert ctx.request_policy.credential_draft_deferred_explicitly is True+++# ---------------------------------------------------------------------------+# The poller must resolve before checking disconnect, or an already-arrived+# response (connect-then-refresh) is discarded as a timeout instead of being+# read.+# ---------------------------------------------------------------------------+++async def test_poller_resolves_before_checking_disconnect(monkeypatch: pytest.MonkeyPatch) -> None:+ """Fails on old code: is_disconnected() was checked first in the poll loop,+ so a response that arrives during the sleep window is discarded as a+ timeout once the client has since disconnected."""+ ctx = make_copilot_context()+ ctx.organization_id = "org-1"+ ctx.turn_id = "turn-race"+ ctx.workflow_copilot_chat_id = "chat-1"+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.1)++ async def _populate_during_sleep() -> None:+ await asyncio.sleep(0.02)+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-race")] = encode_credential_response(+ "skip", None+ )++ stream = MagicMock()+ stream.send = AsyncMock(return_value=True)+ # Connected for the top-level guard and the post-send re-check; "gone" for+ # every poll-loop check after.+ stream.is_disconnected = AsyncMock(side_effect=[False, False] + [True] * 10)+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ populate_task = asyncio.ensure_future(_populate_during_sleep())+ try:+ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)+ finally:+ await populate_task++ assert resume_msgs is not None+ assert ctx.credential_pause_outcome == "skipped"+++# ---------------------------------------------------------------------------+# The pause must gate on a SHARED cache, not merely a non-None one -- a+# same-process-only cache (LocalCache) guarantees a hang in a multi-worker+# deployment since the poller and the POST can land on different workers.+# ---------------------------------------------------------------------------+++def test_local_cache_is_not_shared() -> None:+ from skyvern.forge.sdk.cache.local import LocalCache++ assert LocalCache().is_shared is False+++async def test_non_shared_cache_never_pauses(monkeypatch: pytest.MonkeyPatch) -> None:+ """Fails on old code: `cache is None` was the only cache guard, so a+ same-process-only cache (LocalCache in a multi-worker OSS deployment)+ passed it and produced a guaranteed hang."""+ ctx = make_copilot_context()+ ctx.client_supports_credential_pause = True+ ctx.last_run_skipped_unbound_credentials = True+ ctx.request_policy = RequestPolicy()++ cache = _FakeCache()+ cache.is_shared = False+ monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)+ stream = _make_stream()+ config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)++ resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)++ assert resume_msgs is None+ stream.send.assert_not_awaited()+++# ---------------------------------------------------------------------------+# CredentialPauseResolution sanity+# ---------------------------------------------------------------------------+++def test_credential_pause_resolution_defaults_credential_to_none() -> None:+ resolution = CredentialPauseResolution(action="skip")+ assert resolution.credential is None