Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
"""Tests for the copilot mid-build credential pause-and-wait (SKY-12138).
Covers:
- ``credential_pause_reason`` fires only on the three typed mid-build signals.
- ``maybe_credential_pause``'s waiter: connect mutates ``RequestPolicy`` and
resolves, skip leaves the policy untouched, timeout/disconnect/no-cache
degrade to None without sending a frame, an invalid/foreign credential id
degrades rather than crashing, and CancelledError always propagates.
- ``run_with_enforcement`` loop integration: a typed signal at finalize sends
exactly one ``credential_required`` frame and re-enters the loop with the
resume message, instead of returning on the first finalize.
- Pause time is credited back against ``TOTAL_TIMEOUT_SECONDS`` so a slow
pause doesn't trip the total-timeout on the resumed iteration.
- The one-pause-per-turn latch and kill-switch-off parity.
- The ``/workflow/copilot/credential-response`` route's validation and TTL.
"""
from __future__ import annotations
import asyncio
import io
import json
import logging
import time
from collections.abc import Awaitable, Callable, Iterator
from copy import deepcopy
from dataclasses import replace
from datetime import datetime, timedelta, timezone
from itertools import count
from types import SimpleNamespace
from typing import Any, Literal
from unittest.mock import AsyncMock, MagicMock
import pytest
import structlog
from agents import RunConfig
from agents.tool_context import ToolContext
from fastapi import BackgroundTasks, HTTPException, status
from structlog.testing import capture_logs
from skyvern.config import settings
from skyvern.exceptions import HttpException as VaultHttpException
from skyvern.forge import app
from skyvern.forge.sdk.cache.base import NoopLock
from skyvern.forge.sdk.copilot import credential_pause as credential_pause_module
from skyvern.forge.sdk.copilot import credential_resolution
from skyvern.forge.sdk.copilot import tools as tools_module
from skyvern.forge.sdk.copilot.agent import (
_finalize_result_with_blocker_override,
_make_agent_result,
_turn_credential_prompt_reason,
)
from skyvern.forge.sdk.copilot.blocker_signal import (
CREDENTIAL_ORIGIN_RECOVERY_DECLINED_REASON_CODE,
CREDENTIAL_ORIGIN_RECOVERY_PENDING_REASON_CODE,
CopilotToolBlockerSignal,
)
from skyvern.forge.sdk.copilot.code_block_synthesis import CREDENTIAL_FILL_TOOL_NAME
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
from skyvern.forge.sdk.copilot.context import (
AgentResult,
ApprovedCredential,
CopilotContext,
ProposedCredential,
StructuredContext,
record_approved_credentials_in_global_llm_context,
)
from skyvern.forge.sdk.copilot.credential_generation import SAFE_SYMBOLS, generate_registration_password
from skyvern.forge.sdk.copilot.credential_pause import (
CredentialPauseRejection,
CredentialPauseResolution,
_decode_active_pause,
_encode_active_pause,
await_pending_credential_pause,
claim_credential_generation,
credential_pause_active_key,
credential_response_cache_key,
encode_credential_response,
maybe_credential_pause,
resolve_credential_pause,
)
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisFailureType,
DiagnosisInput,
DiagnosisRepairContract,
DiagnosisResult,
RepairDecision,
RepairNextAction,
RepairRootCauseIdentity,
VerificationResult,
)
from skyvern.forge.sdk.copilot.enforcement import (
NUDGE_SENTINEL,
_elapsed_run_seconds,
enforcement_decision,
run_with_enforcement,
)
from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks
from skyvern.forge.sdk.copilot.human_input_wait import HumanInputWait, pause_human_input
from skyvern.forge.sdk.copilot.narration import NarratorState, build_tool_call_activity
from skyvern.forge.sdk.copilot.request_policy import (
RequestPolicy,
_seed_prior_approved_credentials,
_seed_proposed_credential,
)
from skyvern.forge.sdk.copilot.runtime import (
CredentialOriginRecovery,
CredentialOriginRecoveryState,
)
from skyvern.forge.sdk.copilot.tools import credential_fill as credential_fill_module
from skyvern.forge.sdk.copilot.tools._shared import TOTAL_TIMEOUT_SECONDS, _copilot_seconds_remaining
from skyvern.forge.sdk.copilot.tools.credential_fill import _request_credential
from skyvern.forge.sdk.copilot.tools.guardrails import _authority_tool_error
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.event_source_stream import EventSourceStream
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.forge_log import setup_logger
from skyvern.forge.sdk.routes import workflow_copilot as copilot_routes
from skyvern.forge.sdk.routes.workflow_copilot import (
WorkflowCopilotCredentialResponseRequest,
workflow_copilot_credential_generate,
workflow_copilot_credential_response,
)
from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
from skyvern.forge.sdk.schemas.credentials import (
CreateCredentialRequest,
Credential,
CredentialItem,
CredentialType,
CredentialVaultType,
PasswordCredential,
TotpType,
)
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotCredentialGenerateRequest,
WorkflowCopilotCredentialGenerateResult,
WorkflowCopilotCredentialPauseResolvedUpdate,
WorkflowCopilotCredentialRegistration,
WorkflowCopilotCredentialRequiredUpdate,
WorkflowCopilotStreamMessageType,
)
from tests.unit.conftest import make_copilot_context
from tests.unit.copilot_test_helpers import wire_credential_vault
from tests.unit.test_copilot_make_agent_result_backfill import _payload as _backfill_payload
@pytest.fixture(autouse=True)
def _sequential_resume_tokens(monkeypatch: pytest.MonkeyPatch) -> None:
"""Each card gets tok-1, tok-2, ... so a test can pre-seed the answer for the card it expects."""
tokens = (f"tok-{n}" for n in count(1))
monkeypatch.setattr(credential_pause_module, "_new_resume_token", lambda: next(tokens))
def _repair_contract(
next_action: Any,
failure_type: Any = DiagnosisFailureType.UNKNOWN,
categories: tuple[str, ...] = ("CREDENTIAL_ERROR",),
) -> DiagnosisRepairContract:
return DiagnosisRepairContract(
diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),
diagnosis_result=DiagnosisResult(
suspected_failure_type=failure_type,
root_cause_identity=RepairRootCauseIdentity(failure_categories=categories),
),
repair_decision=RepairDecision(next_action=next_action),
verification_result=VerificationResult(),
)
def _make_credential(credential_id: str = "cred_1", name: str = "Example Login") -> Credential:
now = datetime(2026, 1, 1, tzinfo=timezone.utc)
return Credential(
credential_id=credential_id,
organization_id="org-1",
name=name,
vault_type=CredentialVaultType.SKYVERN,
item_id="item_1",
credential_type=CredentialType.PASSWORD,
username="[email protected]",
card_last4=None,
card_brand=None,
created_at=now,
modified_at=now,
)
class _FakeCache:
"""Minimal in-memory double of the ``get`` / ``set`` / ``get_lock`` surface of app.CACHE.
``is_shared = True`` by default (stands in for Redis); flip an instance's
to False to simulate a LocalCache-shaped non-shared cache.
"""
is_shared = True
def __init__(self) -> None:
self.store: dict[str, Any] = {}
self.set_calls: list[tuple[str, Any, Any]] = []
async def get(self, key: str) -> Any:
return self.store.get(key)
async def set(self, key: str, value: Any, ex: Any = None) -> None:
self.store[key] = value
self.set_calls.append((key, value, ex))
def get_lock(self, lock_name: str, blocking_timeout: int = 5, timeout: int = 10) -> NoopLock:
return NoopLock(lock_name, blocking_timeout, timeout)
def _seed_active_pause(
cache: _FakeCache,
org: str,
chat: str,
turn: str,
token: str,
*,
expires_at: datetime | None = None,
) -> None:
"""Write a pending active-pause record the way maybe_credential_pause would."""
cache.store[credential_pause_active_key(org, chat, turn)] = _encode_active_pause(
token, expires_at or (datetime.now(timezone.utc) + timedelta(minutes=5))
)
def _make_stream(*, disconnected: bool = False) -> MagicMock:
stream = MagicMock()
stream.send = AsyncMock(return_value=True)
stream.is_disconnected = AsyncMock(return_value=disconnected)
return stream
def _fake_result() -> MagicMock:
result = MagicMock()
result.final_output = "Done."
result.new_items = []
result.to_input_list.return_value = []
return result
# ---------------------------------------------------------------------------
# 1 - credential_pause_reason detector
# ---------------------------------------------------------------------------
def test_reason_fires_on_skipped_unbound_credentials() -> None:
ctx = SimpleNamespace(last_run_skipped_unbound_credentials=True)
assert credential_pause_module.credential_pause_reason(ctx) == "workflow_credential_inputs_unbound"
def test_reason_does_not_let_testing_intent_override_concrete_skipped_run() -> None:
"""A classifier-produced testing label cannot erase a concrete credential boundary fact."""
policy = RequestPolicy(testing_intent="skip_test")
ctx = SimpleNamespace(last_run_skipped_unbound_credentials=True, request_policy=policy)
assert credential_pause_module.credential_pause_reason(ctx) == "workflow_credential_inputs_unbound"
def test_reason_fires_on_missing_credential_run_failure() -> None:
ctx = SimpleNamespace(
last_run_skipped_unbound_credentials=False,
latest_diagnosis_repair_contract=_repair_contract(
RepairNextAction.ASK, DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
),
)
assert credential_pause_module.credential_pause_reason(ctx) == "missing_credential_run_failure"
def test_reason_does_not_fire_when_ask_is_for_a_different_failure_type() -> None:
ctx = SimpleNamespace(
last_run_skipped_unbound_credentials=False,
latest_diagnosis_repair_contract=_repair_contract(RepairNextAction.ASK, DiagnosisFailureType.UNKNOWN),
)
assert credential_pause_module.credential_pause_reason(ctx) is None
def test_reason_ignores_missing_credential_or_init_when_not_categorized_as_credential_error() -> None:
"""MISSING_CREDENTIAL_OR_INIT also covers param-binding/lookup failures; only the
CREDENTIAL_ERROR category should preempt with a card that can't unblock those."""
ctx = SimpleNamespace(
last_run_skipped_unbound_credentials=False,
latest_diagnosis_repair_contract=_repair_contract(
RepairNextAction.ASK,
DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT,
categories=("PARAMETER_BINDING_ERROR",),
),
)
assert credential_pause_module.credential_pause_reason(ctx) is None
def test_reason_does_not_fire_from_generic_deferred_draft_policy() -> None:
policy = RequestPolicy(credential_draft_deferred_explicitly=True)
ctx = SimpleNamespace(
last_run_skipped_unbound_credentials=False,
latest_diagnosis_repair_contract=None,
request_policy=policy,
update_workflow_called=True,
)
assert credential_pause_module.credential_pause_reason(ctx) is None
def test_reason_requires_update_workflow_called_for_deferred_draft() -> None:
policy = RequestPolicy(credential_draft_deferred_explicitly=True)
ctx = SimpleNamespace(
last_run_skipped_unbound_credentials=False,
latest_diagnosis_repair_contract=None,
request_policy=policy,
update_workflow_called=False,
)
assert credential_pause_module.credential_pause_reason(ctx) is None
def test_reason_ignores_deferred_draft_when_user_explicitly_skipped_testing() -> None:
"""skip_test means the user already said not to run/verify this -- pausing to
ask for a credential would contradict that explicit request."""
policy = RequestPolicy(credential_draft_deferred_explicitly=True, testing_intent="skip_test")
ctx = SimpleNamespace(
last_run_skipped_unbound_credentials=False,
latest_diagnosis_repair_contract=None,
request_policy=policy,
update_workflow_called=True,
)
assert credential_pause_module.credential_pause_reason(ctx) is None
def test_reason_none_when_no_signals_present() -> None:
ctx = make_copilot_context()
assert credential_pause_module.credential_pause_reason(ctx) is None
# ---------------------------------------------------------------------------
# 2 - maybe_credential_pause waiter
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_connected_action_mutates_policy_and_resolves(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-1"
ctx.client_supports_credential_pause = True
ctx.workflow_copilot_chat_id = "chat-1"
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
ctx.narrator_state = NarratorState()
ctx.narrator_state.record_activity(
build_tool_call_activity("run_blocks_and_collect_debug", 0, "call-run", timestamp=datetime.now(timezone.utc))
)
cache = _FakeCache()
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")] = encode_credential_response(
"connected", "cred_1", "tok-1"
)
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
credential = _make_credential()
record_resume = AsyncMock()
monkeypatch.setattr(
credential_pause_module.app,
"DATABASE",
SimpleNamespace(
credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[credential])),
workflow_params=SimpleNamespace(record_pending_copilot_turn_credential_resume=record_resume),
),
)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is not None
# Reconcile measures abandonment from this stamp, so a resumed turn is not recovered as dead.
record_resume.assert_awaited_with(organization_id="org-1", workflow_copilot_chat_id="chat-1", turn_id="turn-1")
assert ctx.credential_pause_outcome == "connected"
assert ctx.request_policy.resolved_credentials == [credential]
assert ctx.request_policy.allow_run_blocks is True
assert ctx.request_policy.clarification_reason == "none"
assert ctx.request_policy.allow_missing_credentials_in_draft is False
assert ctx.request_policy.requires_user_clarification is False
resume_text = resume_msgs[-1]["content"]
assert resume_text.startswith(NUDGE_SENTINEL)
assert "cred_1" in resume_text
card, resolved = (call.args[0] for call in stream.send.await_args_list)
assert card.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED
# The card renders after the row that was newest when it was raised, live and after reload.
assert card.anchor_tool_call_id == ctx.credential_pause_anchor_tool_call_id == "call-run"
assert resolved.type == WorkflowCopilotStreamMessageType.CREDENTIAL_PAUSE_RESOLVED
assert (resolved.resume_token, resolved.outcome, resolved.credential_id, resolved.name) == (
card.resume_token,
"connected",
"cred_1",
"Example Login",
)
@pytest.mark.asyncio
async def test_resolved_frame_is_sent_promptly_after_the_response_write(monkeypatch: pytest.MonkeyPatch) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache, poll_seconds=None)
ctx.last_run_skipped_unbound_credentials = True
card_sent = asyncio.Event()
sent: list[
tuple[float, WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate]
] = []
async def _record_send(
frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
) -> bool:
sent.append((time.monotonic(), frame))
card_sent.set()
return True
ctx.stream.send = AsyncMock(side_effect=_record_send)
pause = asyncio.ensure_future(maybe_credential_pause(ctx, _fake_result(), ctx.stream, ctx.copilot_config))
await card_sent.wait()
# Let the waiter pass its immediate first check and enter the poll sleep.
await asyncio.sleep(0.05)
card = sent[0][1]
await resolve_credential_pause(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id="turn-1",
resume_token=card.resume_token,
action="skip",
credential_id=None,
)
written_at = time.monotonic()
await pause
resolved_at, resolved = sent[-1]
assert resolved.type == WorkflowCopilotStreamMessageType.CREDENTIAL_PAUSE_RESOLVED
assert resolved_at - written_at < 1.0
@pytest.mark.asyncio
async def test_connected_action_unlatches_test_after_update_done(monkeypatch: pytest.MonkeyPatch) -> None:
"""The skipped run before the pause already stamped test_after_update_done=True;
a connected resume must reset it so a reply that never re-runs the blocks still
gets forced back through update_and_run_blocks by the post_update nudge."""
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-1"
ctx.client_supports_credential_pause = True
ctx.workflow_copilot_chat_id = "chat-1"
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
ctx.update_workflow_called = True
ctx.test_after_update_done = True
cache = _FakeCache()
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")] = encode_credential_response(
"connected", "cred_1", "tok-1"
)
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(
credential_pause_module.app,
"DATABASE",
SimpleNamespace(
credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[_make_credential()]))
),
)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
resume_msgs = await maybe_credential_pause(
ctx,
_fake_result(),
_make_stream(),
CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5),
)
assert resume_msgs is not None
assert ctx.test_after_update_done is False
@pytest.mark.asyncio
async def test_skip_action_leaves_policy_untouched(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-2"
ctx.client_supports_credential_pause = True
ctx.workflow_copilot_chat_id = "chat-1"
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-2")] = encode_credential_response(
"skip", None, "tok-1"
)
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is not None
assert ctx.credential_pause_outcome == "skipped"
assert ctx.request_policy.resolved_credentials == []
assert ctx.request_policy == RequestPolicy()
assert "chose not to connect a credential" in resume_msgs[-1]["content"]
@pytest.mark.asyncio
async def test_skip_clears_stale_last_test_ok_from_the_diagnosed_failure(monkeypatch: pytest.MonkeyPatch) -> None:
"""A missing_credential_run_failure pause means the diagnosed run left
last_test_ok=False; skip must clear it or the resumed reply is intercepted
by the generic failed-test nudge instead of honoring the skip decision."""
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-skip-clears"
ctx.client_supports_credential_pause = True
ctx.workflow_copilot_chat_id = "chat-1"
ctx.last_test_ok = False
ctx.test_after_update_done = True
ctx.latest_diagnosis_repair_contract = _repair_contract(
RepairNextAction.ASK, DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
)
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-skip-clears")] = encode_credential_response(
"skip", None, "tok-1"
)
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is not None
assert ctx.last_test_ok is None
assert enforcement_decision(ctx, result=None, config=config) is None
@pytest.mark.asyncio
async def test_timeout_returns_none_and_marks_outcome(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_context()
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache() # never populated
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=0)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is None
assert ctx.credential_pause_outcome == "timeout"
assert ctx.copilot_credential_pause_seconds >= 0.0
@pytest.mark.asyncio
async def test_response_racing_in_at_the_timeout_instant_is_rescued(monkeypatch: pytest.MonkeyPatch) -> None:
"""Fails on old code: resolve_credential_pause writes under a per-turn lock,
but the timeout invalidate wrote the same active-pause key unlocked -- a
response that lands in the same instant the waiter gives up would validate
against the still-pending record (204 to the client) yet never get read,
since the waiter already returned None. The invalidate must re-check under
the same lock and use the response if one raced in, not silently drop it."""
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-race"
ctx.workflow_copilot_chat_id = "chat-1"
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
async def fake_wait_races_in_a_response(
response_key: str, ctx: CopilotContext, stream: EventSourceStream, timeout_seconds: int, resume_token: str
) -> None:
# Simulates resolve_credential_pause's lock-protected write landing in
# the same instant the waiter's own poll loop concludes None.
cache.store[response_key] = encode_credential_response("skip", None, "tok-1")
return None
monkeypatch.setattr(credential_pause_module, "_wait_for_credential_response", fake_wait_races_in_a_response)
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is not None
assert ctx.credential_pause_outcome == "skipped"
@pytest.mark.asyncio
async def test_client_disconnect_mid_wait_degrades_early(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_context()
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache() # never populated
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
stream = MagicMock()
stream.send = AsyncMock(return_value=True)
# Connected for the initial guard and the post-send re-check, gone by the first poll.
stream.is_disconnected = AsyncMock(side_effect=[False, False, True])
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
start = time.monotonic()
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
elapsed = time.monotonic() - start
assert resume_msgs is None
assert ctx.credential_pause_outcome == "timeout"
assert elapsed < 1.0 # degraded on the first poll, not the full 30s timeout
@pytest.mark.asyncio
async def test_disconnect_racing_the_send_itself_declines_without_waiting(monkeypatch: pytest.MonkeyPatch) -> None:
"""Fails on old code: send() returns True even when the client is already
gone (its own contract), so a disconnect that lands between the pre-send
guard and delivery would otherwise wait out the full timeout for a card
nobody ever saw, and land on 'timeout' instead of 'declined'."""
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-race-send"
ctx.workflow_copilot_chat_id = "chat-1"
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
stream = MagicMock()
stream.send = AsyncMock(return_value=True)
# Connected for the pre-send guard, gone by the time send() returns.
stream.is_disconnected = AsyncMock(side_effect=[False, True])
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
start = time.monotonic()
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
elapsed = time.monotonic() - start
assert resume_msgs is None
assert ctx.credential_pause_outcome == "declined"
assert elapsed < 1.0 # never entered the wait loop
@pytest.mark.asyncio
async def test_already_disconnected_before_send_declines_without_sending_a_frame(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The predictor (credential_pause_transport_ready) excludes the async-only
disconnect check by design -- a client gone before the frame is even sent
must latch and tag 'declined' (not 'timeout') so the caller can fall back
to a normal nudge instead of treating this like a delivered-and-waited pause."""
ctx = make_copilot_context()
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
stream = _make_stream(disconnected=True)
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is None
assert ctx.credential_pause_outcome == "declined"
assert ctx.credential_pause_used is True
stream.send.assert_not_called()
@pytest.mark.asyncio
async def test_disconnect_invalidates_the_active_pause_record(monkeypatch: pytest.MonkeyPatch) -> None:
"""A disconnect degrades well before expires_at; a late POST after the tab
reconnects must be rejected instead of silently writing an unread response."""
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-disconnect"
ctx.workflow_copilot_chat_id = "chat-1"
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
stream = MagicMock()
stream.send = AsyncMock(return_value=True)
stream.is_disconnected = AsyncMock(side_effect=[False, True])
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is None
frame = stream.send.await_args_list[0].args[0]
with pytest.raises(CredentialPauseRejection) as excinfo:
await resolve_credential_pause(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id="turn-disconnect",
resume_token=frame.resume_token,
action="skip",
credential_id=None,
)
assert excinfo.value.status_code == status.HTTP_409_CONFLICT
@pytest.mark.asyncio
async def test_no_cache_returns_none_without_sending_a_frame(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(app._inst, "CACHE", None, raising=False)
ctx = make_copilot_context()
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is None
stream.send.assert_not_awaited()
@pytest.mark.asyncio
async def test_cancelled_error_propagates_and_still_accumulates_pause_seconds(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-cancel"
ctx.workflow_copilot_chat_id = "chat-1"
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
async def _raise_cancelled(*args: object, **kwargs: object) -> None:
raise asyncio.CancelledError()
monkeypatch.setattr(credential_pause_module.asyncio, "sleep", _raise_cancelled)
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
with pytest.raises(asyncio.CancelledError):
await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert ctx.copilot_credential_pause_seconds > 0.0
assert ctx.credential_pause_outcome is None # neither timeout nor resolved path ran
frame = stream.send.await_args_list[0].args[0]
with pytest.raises(CredentialPauseRejection) as excinfo:
await resolve_credential_pause(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id="turn-cancel",
resume_token=frame.resume_token,
action="skip",
credential_id=None,
)
assert excinfo.value.status_code == status.HTTP_409_CONFLICT
@pytest.mark.asyncio
async def test_unexpected_error_in_waiter_also_invalidates_the_active_pause_record(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Not just CancelledError: any unexpected failure in the wait loop leaves the
frame's resume token live client-side, so the record must be invalidated too."""
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-crash"
ctx.workflow_copilot_chat_id = "chat-1"
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
async def _raise_runtime_error(*args: object, **kwargs: object) -> None:
raise RuntimeError("cache connection reset")
monkeypatch.setattr(credential_pause_module.asyncio, "sleep", _raise_runtime_error)
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
with pytest.raises(RuntimeError):
await maybe_credential_pause(ctx, _fake_result(), stream, config)
frame = stream.send.await_args_list[0].args[0]
with pytest.raises(CredentialPauseRejection) as excinfo:
await resolve_credential_pause(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id="turn-crash",
resume_token=frame.resume_token,
action="skip",
credential_id=None,
)
assert excinfo.value.status_code == status.HTTP_409_CONFLICT
@pytest.mark.asyncio
async def test_invalid_credential_id_degrades_instead_of_crashing(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-3"
ctx.client_supports_credential_pause = True
ctx.workflow_copilot_chat_id = "chat-1"
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-3")] = encode_credential_response(
"connected", "cred_foreign", "tok-1"
)
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(
credential_pause_module.app,
"DATABASE",
SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[]))),
)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=1)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is None
assert ctx.credential_pause_outcome == "timeout"
assert ctx.request_policy.resolved_credentials == []
@pytest.mark.asyncio
async def test_full_round_trip_binds_frame_token_to_active_pause(monkeypatch: pytest.MonkeyPatch) -> None:
"""End-to-end: establishing the pause writes an active record + one-time token,
the response endpoint resolves only with that token, and the resolved pause is
consumed so a replay conflicts.
"""
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-rt"
ctx.workflow_copilot_chat_id = "chat-1"
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
active_key = credential_pause_active_key("org-1", "chat-1", "turn-rt")
async def _respond_using_frame_token() -> None:
# Read the one-time token exactly as the FE would from the frame, then resolve.
for _ in range(2000):
raw = await cache.get(active_key)
record = json.loads(raw) if raw else None
if record and record.get("status") == "pending":
await resolve_credential_pause(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id="turn-rt",
resume_token=record["resume_token"],
action="skip",
credential_id=None,
)
return
await asyncio.sleep(0.001)
responder = asyncio.ensure_future(_respond_using_frame_token())
try:
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
finally:
await responder
assert resume_msgs is not None
assert ctx.credential_pause_outcome == "skipped"
frame = stream.send.await_args_list[0].args[0]
assert frame.resume_token
with pytest.raises(CredentialPauseRejection) as excinfo:
await resolve_credential_pause(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id="turn-rt",
resume_token=frame.resume_token,
action="skip",
credential_id=None,
)
assert excinfo.value.status_code == status.HTTP_409_CONFLICT
# ---------------------------------------------------------------------------
# 5 - one-pause-per-turn latch / 6 - kill-switch-off parity
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_latch_prevents_second_pause_this_turn() -> None:
ctx = make_copilot_context()
ctx.credential_pause_used = True
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is None
stream.send.assert_not_awaited()
@pytest.mark.asyncio
async def test_kill_switch_off_returns_none_without_touching_cache_or_stream() -> None:
ctx = make_copilot_context()
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=False)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is None
stream.send.assert_not_awaited()
stream.is_disconnected.assert_not_awaited()
# ---------------------------------------------------------------------------
# 3 - run_with_enforcement loop integration (regression pin)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_loop_pauses_at_finalize_and_resumes_same_turn(monkeypatch: pytest.MonkeyPatch) -> None:
"""Fails on old code: without the pause hook this returns on the first finalize."""
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-loop"
ctx.client_supports_credential_pause = True
ctx.workflow_copilot_chat_id = "chat-1"
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-loop")] = encode_credential_response(
"skip", None, "tok-1"
)
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
stream = _make_stream()
fake_result = _fake_result()
calls: list[dict[str, Any]] = []
def fake_run_streamed(*args: Any, **kwargs: Any) -> Any:
calls.append(kwargs)
return fake_result
async def fake_stream_to_sse(result: Any, s: Any, c: Any) -> None:
return None
monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", fake_run_streamed)
monkeypatch.setattr("skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse", fake_stream_to_sse)
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
returned = await run_with_enforcement(
agent=MagicMock(),
initial_input="hello",
ctx=ctx,
stream=stream,
run_config=RunConfig(),
copilot_config=config,
)
assert returned is fake_result
assert len(calls) == 2, "second Runner invocation must happen after the pause resumes"
second_call_input = calls[1]["input"]
assert any(NUDGE_SENTINEL in item.get("content", "") for item in second_call_input)
assert ctx.credential_pause_used is True
frame_types = [call.args[0].type for call in stream.send.await_args_list]
assert frame_types.count(WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED) == 1
def test_elapsed_run_seconds_subtracts_pause_time() -> None:
ctx = SimpleNamespace(copilot_credential_pause_seconds=50.0)
start_time = time.monotonic() - 60.0
elapsed = _elapsed_run_seconds(ctx, start_time)
assert 9.0 < elapsed < 11.0
@pytest.mark.asyncio
async def test_paused_loop_does_not_trip_total_timeout_on_resume(monkeypatch: pytest.MonkeyPatch) -> None:
"""Fails on old code: a slow pause would consume the real timeout budget.
Four values must stay ordered: non-pause work < TOTAL_TIMEOUT_SECONDS < pause <
credential_pause_timeout_seconds. Non-pause work was measured at ~2s on a cold CI shard
(whichever test imports the agent SDK first pays for it), so a sub-second budget only
ever passed on a warm runner. The pause must still exceed the budget or the assertion
proves nothing, and must stay under the pause timeout or it resolves as a timeout
instead of a response.
"""
monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 4.0)
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-credit"
ctx.client_supports_credential_pause = True
ctx.workflow_copilot_chat_id = "chat-1"
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.5)
async def _populate_after_first_poll() -> None:
await asyncio.sleep(4.5)
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-credit")] = encode_credential_response(
"skip", None, "tok-1"
)
stream = _make_stream()
fake_result = _fake_result()
calls: list[dict[str, Any]] = []
def fake_run_streamed(*args: Any, **kwargs: Any) -> Any:
calls.append(kwargs)
return fake_result
async def fake_stream_to_sse(result: Any, s: Any, c: Any) -> None:
return None
monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", fake_run_streamed)
monkeypatch.setattr("skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse", fake_stream_to_sse)
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
populate_task = asyncio.ensure_future(_populate_after_first_poll())
try:
returned = await run_with_enforcement(
agent=MagicMock(),
initial_input="hello",
ctx=ctx,
stream=stream,
run_config=RunConfig(),
copilot_config=config,
)
finally:
await populate_task
assert returned is fake_result
assert len(calls) == 2, "pause time must be credited so the resumed iteration isn't timed out"
assert ctx.copilot_total_timeout_exceeded is False
# ---------------------------------------------------------------------------
# 7 - /workflow/copilot/credential-response route
# ---------------------------------------------------------------------------
def _response_request(
*,
turn_id: str = "turn-1",
chat_id: str = "chat-1",
resume_token: str = "tok-1",
action: str = "skip",
credential_id: str | None = None,
) -> WorkflowCopilotCredentialResponseRequest:
return WorkflowCopilotCredentialResponseRequest(
turn_id=turn_id,
workflow_copilot_chat_id=chat_id,
resume_token=resume_token,
action=action, # type: ignore[arg-type]
credential_id=credential_id,
)
@pytest.mark.asyncio
async def test_route_503_when_cache_missing(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(app._inst, "CACHE", None, raising=False)
organization = SimpleNamespace(organization_id="org-1")
with pytest.raises(HTTPException) as excinfo:
await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)
assert excinfo.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE
@pytest.mark.asyncio
async def test_route_writes_flag_for_skip_without_credential_id(monkeypatch: pytest.MonkeyPatch) -> None:
cache = _FakeCache()
_seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
organization = SimpleNamespace(organization_id="org-1")
result = await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)
assert result.result == "accepted"
expected_key = credential_response_cache_key("org-1", "chat-1", "turn-1")
assert cache.store[expected_key] == encode_credential_response("skip", None, "tok-1")
response_set = next(call for call in cache.set_calls if call[0] == expected_key)
assert response_set[1] == encode_credential_response("skip", None, "tok-1")
assert response_set[2] == timedelta(seconds=settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS + 300)
@pytest.mark.asyncio
async def test_route_422_when_connected_without_credential_id(monkeypatch: pytest.MonkeyPatch) -> None:
cache = _FakeCache()
_seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
organization = SimpleNamespace(organization_id="org-1")
with pytest.raises(HTTPException) as excinfo:
await workflow_copilot_credential_response(_response_request(action="connected"), organization=organization)
assert excinfo.value.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY
assert cache.set_calls == []
@pytest.mark.asyncio
async def test_route_404_when_credential_unknown_or_foreign_org(monkeypatch: pytest.MonkeyPatch) -> None:
cache = _FakeCache()
_seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(
app,
"DATABASE",
SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[]))),
)
organization = SimpleNamespace(organization_id="org-1")
with pytest.raises(HTTPException) as excinfo:
await workflow_copilot_credential_response(
_response_request(action="connected", credential_id="cred_other_org"),
organization=organization,
)
assert excinfo.value.status_code == status.HTTP_404_NOT_FOUND
assert cache.set_calls == []
@pytest.mark.asyncio
async def test_route_404_when_connected_credential_is_in_a_hidden_folder(monkeypatch: pytest.MonkeyPatch) -> None:
cache = _FakeCache()
_seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")
monkeypatch.setattr(app, "CACHE", cache, raising=False)
credential_lookup = AsyncMock(return_value=[])
monkeypatch.setattr(
app, "DATABASE", SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=credential_lookup))
)
monkeypatch.setattr(
credential_resolution, "credential_folder_ids_for_current_request", AsyncMock(return_value={"folder_visible"})
)
organization = SimpleNamespace(organization_id="org-1")
with pytest.raises(HTTPException) as excinfo:
await workflow_copilot_credential_response(
_response_request(action="connected", credential_id="cred_hidden"),
organization=organization,
)
assert excinfo.value.status_code == status.HTTP_404_NOT_FOUND
credential_lookup.assert_awaited_once_with(
["cred_hidden"],
organization_id="org-1",
allowed_folder_ids={"folder_visible"},
include_deleted=False,
)
assert cache.set_calls == []
@pytest.mark.asyncio
async def test_route_writes_flag_for_connected_with_valid_credential(monkeypatch: pytest.MonkeyPatch) -> None:
cache = _FakeCache()
_seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(
app,
"DATABASE",
SimpleNamespace(
credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[_make_credential()]))
),
)
organization = SimpleNamespace(organization_id="org-1")
result = await workflow_copilot_credential_response(
_response_request(action="connected", credential_id="cred_1"),
organization=organization,
)
assert result.result == "accepted"
expected_key = credential_response_cache_key("org-1", "chat-1", "turn-1")
assert cache.store[expected_key] == encode_credential_response("connected", "cred_1", "tok-1")
@pytest.mark.asyncio
async def test_route_404_when_no_active_pause_record(monkeypatch: pytest.MonkeyPatch) -> None:
"""A valid-looking token can't resolve a turn that was never paused."""
cache = _FakeCache() # no active-pause record seeded
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
organization = SimpleNamespace(organization_id="org-1")
with pytest.raises(HTTPException) as excinfo:
await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)
assert excinfo.value.status_code == status.HTTP_404_NOT_FOUND
assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store
@pytest.mark.asyncio
async def test_route_403_when_resume_token_mismatches(monkeypatch: pytest.MonkeyPatch) -> None:
"""A leaked/guessed turn+chat id without the frame's token is rejected."""
cache = _FakeCache()
_seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-real")
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
organization = SimpleNamespace(organization_id="org-1")
with pytest.raises(HTTPException) as excinfo:
await workflow_copilot_credential_response(
_response_request(action="skip", resume_token="tok-wrong"),
organization=organization,
)
assert excinfo.value.status_code == status.HTTP_403_FORBIDDEN
assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store
@pytest.mark.asyncio
async def test_route_rejects_bad_token_before_looking_up_credential_id(monkeypatch: pytest.MonkeyPatch) -> None:
"""A bad token must not reach the credential-id DB lookup -- org auth alone
can't authorize this turn, and doing the lookup first turns a wrong-token
response into a small authenticated credential-id existence oracle."""
cache = _FakeCache()
_seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-real")
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
lookup = AsyncMock(return_value=[_make_credential(credential_id="cred_real")])
monkeypatch.setattr(app, "DATABASE", SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=lookup)))
organization = SimpleNamespace(organization_id="org-1")
with pytest.raises(HTTPException) as excinfo:
await workflow_copilot_credential_response(
_response_request(action="connected", credential_id="cred_real", resume_token="tok-wrong"),
organization=organization,
)
assert excinfo.value.status_code == status.HTTP_403_FORBIDDEN
lookup.assert_not_called()
@pytest.mark.asyncio
async def test_route_404_when_chat_id_does_not_match_the_pause(monkeypatch: pytest.MonkeyPatch) -> None:
"""The pause is keyed by chat+turn: a response with the right turn but a foreign chat misses it."""
cache = _FakeCache()
_seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
organization = SimpleNamespace(organization_id="org-1")
with pytest.raises(HTTPException) as excinfo:
await workflow_copilot_credential_response(
_response_request(action="skip", chat_id="chat-foreign"),
organization=organization,
)
assert excinfo.value.status_code == status.HTTP_404_NOT_FOUND
@pytest.mark.asyncio
async def test_route_409_on_replay_after_first_accepted_response(monkeypatch: pytest.MonkeyPatch) -> None:
"""First accepted response consumes the pause; a replay of the same POST is rejected."""
cache = _FakeCache()
_seed_active_pause(cache, "org-1", "chat-1", "turn-1", "tok-1")
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
organization = SimpleNamespace(organization_id="org-1")
first = await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)
assert first.result == "accepted"
with pytest.raises(HTTPException) as excinfo:
await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)
assert excinfo.value.status_code == status.HTTP_409_CONFLICT
@pytest.mark.asyncio
async def test_route_410_when_response_arrives_after_the_frame_deadline(monkeypatch: pytest.MonkeyPatch) -> None:
"""The record's Redis TTL outlives the wait window for grace on a slow poll,
but a response after the deadline the frame told the client about must still
be rejected -- the waiter already gave up and the turn finalized without it."""
cache = _FakeCache()
_seed_active_pause(
cache, "org-1", "chat-1", "turn-1", "tok-1", expires_at=datetime.now(timezone.utc) - timedelta(seconds=1)
)
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
organization = SimpleNamespace(organization_id="org-1")
with pytest.raises(HTTPException) as excinfo:
await workflow_copilot_credential_response(_response_request(action="skip"), organization=organization)
assert excinfo.value.status_code == status.HTTP_410_GONE
assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store
@pytest.mark.asyncio
async def test_resolve_credential_pause_rejections_are_typed() -> None:
"""resolve_credential_pause raises CredentialPauseRejection (not HTTPException) so it stays route-agnostic."""
cache = _FakeCache()
with pytest.raises(CredentialPauseRejection) as excinfo:
await resolve_credential_pause(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id="turn-1",
resume_token="tok-1",
action="skip",
credential_id=None,
)
assert excinfo.value.status_code == status.HTTP_404_NOT_FOUND
def test_workflow_credential_inputs_unbound_skip_does_not_nudge_post_update() -> None:
"""Pins the real post-skip state (not the defensive _skipped_run_ctx fixture):
_update_enforcement_from_tool sets test_after_update_done=True unconditionally
for update_and_run_blocks, even on the credential-unbound skip branch, so the
post_update nudge condition (not test_after_update_done) is already False by
the time enforcement runs -- verified via a direct call to that function with
the real skip_result shape from tools/__init__.py's skip branch."""
ctx = make_copilot_context()
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.update_workflow_called = True
ctx.test_after_update_done = True
ctx.request_policy = RequestPolicy()
config = CopilotConfig(credential_pause_enabled=True)
assert enforcement_decision(ctx, result=None, config=config) is None
ctx.credential_pause_used = True
assert enforcement_decision(ctx, result=None, config=config) is None
@pytest.mark.asyncio
async def test_missing_credential_run_failure_pauses_the_loop(monkeypatch: pytest.MonkeyPatch) -> None:
"""Loop-level pin for the same fix, through the real finalize seam."""
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-nudge-race"
ctx.client_supports_credential_pause = True
ctx.workflow_copilot_chat_id = "chat-1"
ctx.last_test_ok = False
ctx.test_after_update_done = True
ctx.latest_diagnosis_repair_contract = _repair_contract(
RepairNextAction.ASK, DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT
)
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-nudge-race")] = encode_credential_response(
"skip", None, "tok-1"
)
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
stream = _make_stream()
fake_result = _fake_result()
calls: list[dict[str, Any]] = []
def fake_run_streamed(*args: Any, **kwargs: Any) -> Any:
calls.append(kwargs)
return fake_result
async def fake_stream_to_sse(result: Any, s: Any, c: Any) -> None:
return None
monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.Runner.run_streamed", fake_run_streamed)
monkeypatch.setattr("skyvern.forge.sdk.copilot.streaming_adapter.stream_to_sse", fake_stream_to_sse)
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
returned = await run_with_enforcement(
agent=MagicMock(),
initial_input="hello",
ctx=ctx,
stream=stream,
run_config=RunConfig(),
copilot_config=config,
)
assert returned is fake_result
# If maybe_credential_pause didn't clear the stale last_test_ok=False on
# skip, the resumed reply would be intercepted by the failed-test nudge
# for a 3rd Runner call instead of finalizing here.
assert len(calls) == 2
frame_types = [call.args[0].type for call in stream.send.await_args_list]
assert frame_types.count(WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED) == 1
# ---------------------------------------------------------------------------
# last_run_skipped_unbound_credentials must reflect the MOST RECENT
# update_and_run_blocks call, not "ever skipped this turn" — otherwise a
# later successful call leaves a stale True and pauses an already-passing turn.
# ---------------------------------------------------------------------------
def _skip_flag_workflow_yaml() -> str:
return "workflow_definition:\n parameters: []\n blocks:\n - block_type: code\n label: step_one\n"
def _skip_flag_ctx() -> CopilotContext:
ctx = make_copilot_context()
ctx.request_policy = RequestPolicy()
return ctx
async def _no_prior_definition(update_ctx: CopilotContext) -> object:
return None
# ---------------------------------------------------------------------------
# _copilot_seconds_remaining must credit pause time, or a long pause late in
# the turn FORBIDS the very test the connected credential was for
# (blockers.py's _late_block_running_call_signal).
# ---------------------------------------------------------------------------
def test_copilot_seconds_remaining_credits_pause_time() -> None:
ctx = SimpleNamespace(copilot_run_start_monotonic=time.monotonic() - 1000.0, copilot_credential_pause_seconds=300.0)
remaining = _copilot_seconds_remaining(ctx)
# Uncredited, the budget would be spent against 1000s rather than the 700s of counted work.
assert remaining is not None
assert remaining == pytest.approx(TOTAL_TIMEOUT_SECONDS - 700.0, abs=1.0)
# ---------------------------------------------------------------------------
# A client that can't render credential_required must never be paused for,
# or it stares at "Working..." until the timeout.
# ---------------------------------------------------------------------------
def test_transport_not_ready_when_client_does_not_support_pause() -> None:
ctx = make_copilot_context()
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
config = CopilotConfig(credential_pause_enabled=True)
assert credential_pause_module.credential_pause_transport_ready(ctx, config) is False
def test_transport_ready_once_client_support_is_set() -> None:
ctx = make_copilot_context()
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
config = CopilotConfig(credential_pause_enabled=True)
assert credential_pause_module.credential_pause_transport_ready(ctx, config) is True
@pytest.mark.asyncio
async def test_unsupported_client_never_pauses_even_with_everything_else_true(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Fails on old code: no client-support guard existed, so a pre-12139 FE
build (or the env flag on with an unwired client) got paused and the FE
silently dropped the unknown frame -- a guaranteed ~timeout-length hang."""
ctx = make_copilot_context()
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
# ctx.client_supports_credential_pause left at its False default.
cache = _FakeCache()
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is None
stream.send.assert_not_awaited()
# ---------------------------------------------------------------------------
# A successful connect must clear credential_draft_deferred_explicitly, or
# credential_prompt_reason() keeps stamping credentialPrompt right next to
# credentialPause: connected.
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_connect_clears_credential_draft_deferred_explicitly(monkeypatch: pytest.MonkeyPatch) -> None:
"""Fails on old code: the flag survives a successful connect, so the FE
would see a contradictory connect-CTA right after the user connected."""
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-connect-clear"
ctx.workflow_copilot_chat_id = "chat-1"
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy(credential_draft_deferred_explicitly=True)
cache = _FakeCache()
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-connect-clear")] = encode_credential_response(
"connected", "cred_1", "tok-1"
)
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
credential = _make_credential()
monkeypatch.setattr(
credential_pause_module.app,
"DATABASE",
SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[credential]))),
)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert ctx.request_policy.credential_draft_deferred_explicitly is False
assert _turn_credential_prompt_reason(ctx) is None
@pytest.mark.asyncio
async def test_skip_leaves_credential_draft_deferred_explicitly_untouched(monkeypatch: pytest.MonkeyPatch) -> None:
"""Skip/timeout correctly keep the flag -- only a successful connect clears it."""
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-skip-keep"
ctx.workflow_copilot_chat_id = "chat-1"
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy(credential_draft_deferred_explicitly=True)
cache = _FakeCache()
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-skip-keep")] = encode_credential_response(
"skip", None, "tok-1"
)
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert ctx.request_policy.credential_draft_deferred_explicitly is True
# ---------------------------------------------------------------------------
# The poller must resolve before checking disconnect, or an already-arrived
# response (connect-then-refresh) is discarded as a timeout instead of being
# read.
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_poller_resolves_before_checking_disconnect(monkeypatch: pytest.MonkeyPatch) -> None:
"""Fails on old code: is_disconnected() was checked first in the poll loop,
so a response that arrives during the sleep window is discarded as a
timeout once the client has since disconnected."""
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-race"
ctx.workflow_copilot_chat_id = "chat-1"
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.1)
async def _populate_during_sleep() -> None:
await asyncio.sleep(0.02)
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-race")] = encode_credential_response(
"skip", None, "tok-1"
)
stream = MagicMock()
stream.send = AsyncMock(return_value=True)
# Connected for the top-level guard and the post-send re-check; "gone" for
# every poll-loop check after.
stream.is_disconnected = AsyncMock(side_effect=[False, False] + [True] * 10)
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
populate_task = asyncio.ensure_future(_populate_during_sleep())
try:
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
finally:
await populate_task
assert resume_msgs is not None
assert ctx.credential_pause_outcome == "skipped"
# ---------------------------------------------------------------------------
# The pause must gate on a SHARED cache, not merely a non-None one -- a
# same-process-only cache (LocalCache) guarantees a hang in a multi-worker
# deployment since the poller and the POST can land on different workers.
# ---------------------------------------------------------------------------
def test_local_cache_is_not_shared() -> None:
from skyvern.forge.sdk.cache.local import LocalCache
assert LocalCache().is_shared is False
@pytest.mark.asyncio
async def test_non_shared_cache_never_pauses(monkeypatch: pytest.MonkeyPatch) -> None:
"""Fails on old code: `cache is None` was the only cache guard, so a
same-process-only cache (LocalCache in a multi-worker OSS deployment)
passed it and produced a guaranteed hang."""
ctx = make_copilot_context()
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy()
cache = _FakeCache()
cache.is_shared = False
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
stream = _make_stream()
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is None
stream.send.assert_not_awaited()
# ---------------------------------------------------------------------------
# CredentialPauseResolution sanity
# ---------------------------------------------------------------------------
def test_credential_pause_resolution_defaults_credential_to_none() -> None:
resolution = CredentialPauseResolution(action="skip")
assert resolution.credential is None
def _ask_origin_policy(*ask_login_page_urls: str) -> RequestPolicy:
return RequestPolicy(credential_ask_login_page_urls=list(ask_login_page_urls))
def _named_site_policy(site_url: str = "https://portal.example.com/login") -> RequestPolicy:
return RequestPolicy(user_provided_site_urls=[site_url])
def _tool_ctx(
monkeypatch: pytest.MonkeyPatch, cache: _FakeCache | None = None, poll_seconds: float | None = 0.01
) -> CopilotContext:
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-1"
ctx.workflow_copilot_chat_id = "chat-1"
ctx.client_supports_credential_pause = True
ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
ctx.request_policy = _named_site_policy()
ctx.stream = _make_stream()
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache or _FakeCache(), raising=False)
if poll_seconds is not None:
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", poll_seconds)
return ctx
async def _ask(ctx: CopilotContext, login_page_url: str = "https://portal.example.com/login") -> dict[str, Any]:
return await _request_credential(login_page_url, "This site needs a sign-in.", ctx)
def test_connected_resume_stamps_the_single_ask_origin() -> None:
policy = _ask_origin_policy("https://portal.example.com/login")
credential_pause_module.apply_connected_credential_to_policy(make_copilot_context(), policy, _make_credential())
assert policy.live_page_admitted_urls == {"cred_1": "https://portal.example.com/login"}
def test_connected_resume_stamps_nothing_when_the_ask_spans_several_origins() -> None:
policy = _ask_origin_policy("https://portal.example.com/login", "https://other.example.com/login")
credential_pause_module.apply_connected_credential_to_policy(make_copilot_context(), policy, _make_credential())
assert policy.live_page_admitted_urls == {}
@pytest.mark.asyncio
async def test_generated_credential_connected_from_a_multi_origin_ask_gets_no_fill_grant(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _tool_ctx(monkeypatch)
ctx.request_policy = _ask_origin_policy("https://portal.example.com/login", "https://other.example.com/login")
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
ctx.org_credentials_for_turn = []
credential_pause_module.apply_connected_credential_to_policy(ctx, ctx.request_policy, _make_credential("cred_new"))
grant, error = await tools_module.credential_fill._credential_fill_origin_grant(ctx, "cred_new")
assert ctx.request_policy.live_page_admitted_urls == {}
assert grant is None and error
def test_connected_resume_names_the_picked_credential_superseding_an_earlier_mention() -> None:
"""The fill seam's which-credential check is set equality, so a card answer must replace an
earlier prose mention, not join it."""
policy = _ask_origin_policy("https://portal.example.com/login")
policy.current_turn_named_credential_ids = {"cred_earlier"}
credential_pause_module.apply_connected_credential_to_policy(make_copilot_context(), policy, _make_credential())
assert policy.current_turn_named_credential_ids == {"cred_1"}
def test_connected_resume_does_not_duplicate_an_already_resolved_credential() -> None:
policy = _ask_origin_policy("https://portal.example.com/login")
credential = _make_credential()
policy.resolved_credentials = [credential]
credential_pause_module.apply_connected_credential_to_policy(make_copilot_context(), policy, credential)
assert [resolved.credential_id for resolved in policy.resolved_credentials] == ["cred_1"]
@pytest.mark.parametrize(
"login_urls",
[[], ["https://portal.example.com/login"], ["https://portal.example.com/login", "https://other.example.com/login"]],
)
def test_a_card_connected_credential_keeps_its_durable_cross_turn_approval(login_urls: list[str]) -> None:
ctx = make_copilot_context()
policy = _ask_origin_policy(*login_urls)
ctx.request_policy = policy
credential_pause_module.apply_connected_credential_to_policy(ctx, policy, _make_credential())
carried = record_approved_credentials_in_global_llm_context(ctx, None)
assert [record.credential_id for record in StructuredContext.from_json_str(carried).approved_credentials] == [
"cred_1"
]
@pytest.mark.asyncio
@pytest.mark.parametrize("previous_origin", [None, "", "https://old.example.com/login"])
@pytest.mark.parametrize(
"login_url",
[
"https://portal.example.com/login",
"https://portal.example.com/login?code=synthetic-code&state=synthetic-state#synthetic-token",
],
)
async def test_card_selection_keeps_its_origin_on_the_next_turn(
monkeypatch: pytest.MonkeyPatch, previous_origin: str | None, login_url: str
) -> None:
ctx = make_copilot_context()
ctx.request_policy = _ask_origin_policy(login_url)
ctx.request_policy.persisted_workflow_credential_ids = {"cred_1"}
previous = StructuredContext()
if previous_origin is not None:
previous.approved_credentials = [ApprovedCredential(credential_id="cred_1", admitted_url=previous_origin)]
credential_pause_module.apply_connected_credential_to_policy(ctx, ctx.request_policy, _make_credential())
carried = record_approved_credentials_in_global_llm_context(ctx, previous.to_json_str())
assert carried is not None
records = StructuredContext.from_json_str(carried).approved_credentials
assert records == [ApprovedCredential(credential_id="cred_1", admitted_url="https://portal.example.com/login")]
next_policy = RequestPolicy()
_stub_credential_lookup(monkeypatch, _make_credential())
await _seed_prior_approved_credentials(next_policy, organization_id="org-1", global_llm_context=carried)
next_ctx = make_copilot_context()
next_ctx.request_policy = next_policy
next_ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
grant, error = await tools_module.credential_fill._credential_fill_origin_grant(next_ctx, "cred_1")
assert error is None
assert grant is not None
assert tools_module.credential_fill._within_grant("https://portal.example.com/password", grant)
assert not tools_module.credential_fill._within_grant("https://elsewhere.example.com/login", grant)
assert not tools_module.credential_fill._within_grant("https://old.example.com/login", grant)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"login_url",
[
r"https://trusted.example\@evil.example/login",
"https://synthetic-user:[email protected]/login",
"https://portal.example.com:invalid/login",
],
)
async def test_invalid_card_origin_cannot_create_cross_turn_approval(
monkeypatch: pytest.MonkeyPatch, login_url: str
) -> None:
ctx = _tool_ctx(monkeypatch, _answered_cache("connected", "cred_1"))
_stub_credential_lookup(monkeypatch, _make_credential())
result = await _ask(ctx, login_url)
assert result["ok"] is False
assert ctx.credential_pause_connected_credential_id is None
# Retained state must also fail closed if it predates validation at the card boundary.
ctx.request_policy = _ask_origin_policy(login_url)
credential_pause_module.apply_connected_credential_to_policy(ctx, ctx.request_policy, _make_credential())
carried = record_approved_credentials_in_global_llm_context(ctx, None)
assert StructuredContext.from_json_str(carried).approved_credentials == []
@pytest.mark.asyncio
@pytest.mark.parametrize("user_named", [False, True])
async def test_saved_workflow_resolution_requires_user_selection_for_chat_approval(
monkeypatch: pytest.MonkeyPatch, user_named: bool
) -> None:
ctx = make_copilot_context()
ctx.request_policy = RequestPolicy(
resolved_credentials=[_make_credential()],
persisted_workflow_credential_ids={"cred_1"},
current_turn_named_credential_ids={"cred_1"} if user_named else set(),
)
carried = record_approved_credentials_in_global_llm_context(ctx, None)
next_policy = RequestPolicy()
_stub_credential_lookup(monkeypatch, _make_credential())
await _seed_prior_approved_credentials(next_policy, organization_id="org-1", global_llm_context=carried)
assert [credential.credential_id for credential in next_policy.resolved_credentials] == (
["cred_1"] if user_named else []
)
@pytest.mark.asyncio
async def test_card_approval_preserves_explicit_port_zero(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_context()
ctx.request_policy = _ask_origin_policy("https://portal.example.com:0/login?code=synthetic-code")
credential_pause_module.apply_connected_credential_to_policy(ctx, ctx.request_policy, _make_credential())
carried = record_approved_credentials_in_global_llm_context(ctx, None)
next_policy = RequestPolicy()
_stub_credential_lookup(monkeypatch, _make_credential())
await _seed_prior_approved_credentials(next_policy, organization_id="org-1", global_llm_context=carried)
next_ctx = make_copilot_context()
next_ctx.request_policy = next_policy
next_ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
grant, error = await tools_module.credential_fill._credential_fill_origin_grant(next_ctx, "cred_1")
assert error is None
assert grant is not None
assert tools_module.credential_fill._within_grant("https://portal.example.com:0/password", grant)
assert not tools_module.credential_fill._within_grant("https://portal.example.com/password", grant)
@pytest.mark.parametrize("already_approved", [False, True])
def test_page_only_admission_does_not_become_durable_approval(already_approved: bool) -> None:
ctx = make_copilot_context()
ctx.request_policy = RequestPolicy(
resolved_credentials=[_make_credential()],
live_page_admitted_urls={"cred_1": "https://portal.example.com/login"},
)
previous = StructuredContext()
if already_approved:
previous.approved_credentials = [
ApprovedCredential(credential_id="cred_1", admitted_url="https://old.example.com/login")
]
carried = record_approved_credentials_in_global_llm_context(ctx, previous.to_json_str())
assert StructuredContext.from_json_str(carried).approved_credentials == previous.approved_credentials
def _answered_cache(action: str, credential_id: str | None = None) -> _FakeCache:
cache = _FakeCache()
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")] = encode_credential_response(
action, credential_id, "tok-1"
)
return cache
def _stub_credential_lookup(
monkeypatch: pytest.MonkeyPatch, credential: Credential, vault_row: AsyncMock | None = None
) -> None:
monkeypatch.setattr(
credential_pause_module.app,
"DATABASE",
SimpleNamespace(
credentials=SimpleNamespace(
get_credentials_by_ids=AsyncMock(return_value=[credential]),
get_credential=vault_row or AsyncMock(return_value=None),
)
),
)
@pytest.mark.asyncio
async def test_a_card_pick_fills_on_a_turn_whose_named_login_was_not_saved(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = _tool_ctx(monkeypatch, _answered_cache("connected", "cred_1"))
_stub_credential_lookup(monkeypatch, _make_credential())
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
ctx.request_policy.record_named_login_not_found()
result = await _ask(ctx)
grant, error = await tools_module.credential_fill._credential_fill_origin_grant(ctx, "cred_1")
assert result["status"] == "connected"
assert error is None and grant is not None
@pytest.mark.asyncio
async def test_the_tool_call_raises_the_card_and_returns_the_connected_credential_to_the_model(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Fails without the tool: nothing pre-run raises the card, so the turn finalizes with a prose
ask and no frame. Fails again if the answer is not returned as this call's own typed result."""
ctx = _tool_ctx(monkeypatch, _answered_cache("connected", "cred_1"))
_stub_credential_lookup(monkeypatch, _make_credential())
result = await _ask(ctx)
assert result["status"] == "connected"
assert result["credential_id"] == "cred_1"
frame = ctx.stream.send.await_args_list[0].args[0]
assert frame.type is WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED
assert frame.reason == "login_credentials_unresolved"
assert frame.login_page_urls == ["https://portal.example.com/login"]
assert frame.message == "This site needs a sign-in."
assert ctx.request_policy.live_page_admitted_urls == {"cred_1": "https://portal.example.com/login"}
assert ctx.request_policy.current_turn_named_credential_ids == {"cred_1"}
assert ctx.request_policy.allow_run_blocks is True
@pytest.mark.asyncio
@pytest.mark.parametrize(
("first_answer", "first_status"),
[(("signed_in", None), "signed_in"), (("connected", "cred_1"), "connected"), (("skip", None), "skipped")],
ids=["signed_in", "connected", "skipped"],
)
async def test_a_settled_card_leaves_the_models_next_ask_free_to_open_a_fresh_card(
monkeypatch: pytest.MonkeyPatch, first_answer: _CardAnswer, first_status: str
) -> None:
"""A Done can save a browser state whose next test is still signed out, so the model's next ask must
reach the user as a new card rather than an already_asked from the settled one."""
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
_stub_credential_lookup(monkeypatch, _make_credential())
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [first_answer, ("connected", "cred_1")]))
first = await _call_ask_tool(ctx)
ctx.last_test_ok = False
second = await _call_ask_tool(ctx)
assert first["status"] == first_status
assert (second["status"], second.get("credential_id")) == ("connected", "cred_1")
assert [card.resume_token for card in _sent_cards(ctx)] == ["tok-1", "tok-2"]
assert [(resolved.resume_token, resolved.outcome) for resolved in _sent_resolutions(ctx)] == [
("tok-1", first_status),
("tok-2", "connected"),
]
assert ctx.credential_pause_outcome == "connected"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("first_card", "credited"), [("unanswered", False), ("start_sign_in", True)], ids=["nobody_there", "signing_in"]
)
async def test_an_unanswered_card_returns_past_the_deadline_and_counts_against_the_turn_unless_someone_signed_in(
monkeypatch: pytest.MonkeyPatch, first_card: Literal["unanswered", "start_sign_in"], credited: bool
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=1)
ctx.browser_session_id = "pbs_debug"
monkeypatch.setattr(settings, "WORKFLOW_COPILOT_MANUAL_SIGN_IN_TIMEOUT_SECONDS", 1)
_stub_credential_lookup(monkeypatch, _make_credential())
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [(first_card, None), ("connected", "cred_1")]))
# The turn's backstop is shorter than the card's wait.
async with asyncio.timeout(0.3) as deadline:
ctx.model_stream_deadline = deadline
first = await _call_ask_tool(ctx)
credited_after_first = ctx.copilot_credential_pause_seconds
backstop_after_first = deadline.when() or 0.0
backstop_still_ahead = backstop_after_first > asyncio.get_running_loop().time()
second = await _call_ask_tool(ctx)
assert (first["status"], first["outcome"]) == ("unanswered", "timeout")
assert backstop_still_ahead
# The backstop's grace is one-shot: a later card in a spent turn no longer pushes it out.
assert deadline.when() == backstop_after_first
assert (credited_after_first > 0.0) is credited
assert (second["status"], second.get("credential_id")) == ("connected", "cred_1")
assert [card.resume_token for card in _sent_cards(ctx)] == ["tok-1", "tok-2"]
def test_overlapping_waits_credit_only_the_time_someone_was_answering() -> None:
def waiting_ctx() -> SimpleNamespace:
return SimpleNamespace(
human_input_wait=HumanInputWait(),
model_stream_deadline=None,
copilot_question_pause_seconds=0.0,
copilot_credential_pause_seconds=0.0,
)
question_outlasts_card = waiting_ctx()
with pause_human_input(question_outlasts_card, "question"): # type: ignore[arg-type]
with pause_human_input(question_outlasts_card, "credential") as card: # type: ignore[arg-type]
card.unattended = True
time.sleep(0.01)
card_outlasts_question = waiting_ctx()
with pause_human_input(card_outlasts_question, "credential") as card: # type: ignore[arg-type]
with pause_human_input(card_outlasts_question, "question"): # type: ignore[arg-type]
pass
time.sleep(0.2)
card.unattended = True
assert question_outlasts_card.copilot_question_pause_seconds > 0.0
assert card_outlasts_question.copilot_credential_pause_seconds < 0.1
@pytest.mark.asyncio
async def test_an_ask_issued_while_a_card_is_open_opens_no_second_card(monkeypatch: pytest.MonkeyPatch) -> None:
response_key = credential_response_cache_key("org-1", "chat-1", "turn-1")
ctx = _tool_ctx(monkeypatch, _SlowAnswerCache(response_key, 0.2, "connected"))
_stub_credential_lookup(monkeypatch, _make_credential())
results = await asyncio.gather(_call_ask_tool(ctx), _call_ask_tool(ctx))
refused = next(result for result in results if result["status"] == "already_asked")
assert sorted(result["status"] for result in results) == ["already_asked", "connected"]
assert "outcome" not in refused
assert [card.resume_token for card in _sent_cards(ctx)] == ["tok-1"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("answers", "settled_outcome"),
[
([("signed_in", None)], "signed_in"),
([("skip", None)], "skipped"),
([("connected", "cred_1"), ("skip", None)], "skipped"),
],
ids=["signed_in", "skipped", "connected_then_skipped"],
)
async def test_a_run_derived_ask_does_not_reopen_a_card_the_model_raised(
monkeypatch: pytest.MonkeyPatch, answers: list[_CardAnswer], settled_outcome: str
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
_stub_credential_lookup(monkeypatch, _make_credential())
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, answers))
for _ in answers:
await _call_ask_tool(ctx)
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy.login_page_urls = ["https://portal.example.com/login"]
stream = _make_stream()
assert credential_pause_module.credential_pause_reason(ctx) == "workflow_credential_inputs_unbound"
assert await maybe_credential_pause(ctx, _fake_result(), stream, ctx.copilot_config) is None
stream.send.assert_not_awaited()
assert ctx.credential_pause_outcome == settled_outcome
@pytest.mark.asyncio
async def test_two_cards_answered_with_different_logins_both_become_durable_approvals(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx, statuses = await _answer_two_cards_with_different_logins(monkeypatch)
carried = StructuredContext.from_json_str(record_approved_credentials_in_global_llm_context(ctx, None))
assert statuses == ["connected", "connected"]
assert {approval.credential_id: approval.admitted_url for approval in carried.approved_credentials} == {
"cred_a": "https://portal.example.com/login",
"cred_b": "https://portal.example.com/login",
}
@pytest.mark.asyncio
@pytest.mark.parametrize("server_bound_before_the_card", [False, True])
async def test_an_earlier_card_pick_survives_a_later_card_while_fill_targets_the_later_one(
monkeypatch: pytest.MonkeyPatch, server_bound_before_the_card: bool
) -> None:
ctx, _ = await _answer_two_cards_with_different_logins(monkeypatch, server_bound_a=server_bound_before_the_card)
ctx.request_policy.canonical_user_message = "use cred_missing"
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
earlier_pick = await tools_module.credentials._resolve_exact_credential("cred_a", ctx)
missing = await tools_module.credentials._resolve_exact_credential("cred_missing", ctx)
grant, error = await tools_module.credential_fill._credential_fill_origin_grant(ctx, "cred_b")
assert earlier_pick["ok"] is True
assert missing["data"]["withdrawn_credential_ids"] == []
assert "cred_a" in {credential.credential_id for credential in ctx.request_policy.resolved_credentials}
assert ctx.request_policy.current_turn_named_credential_ids == {"cred_b"}
assert error is None and grant is not None
async def _answer_two_cards_with_different_logins(
monkeypatch: pytest.MonkeyPatch, *, server_bound_a: bool = False
) -> tuple[CopilotContext, list[str]]:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
logins = [_make_credential("cred_a", "Login A"), _make_credential("cred_b", "Login B")]
if server_bound_a:
ctx.request_policy.resolved_credentials = [logins[0]]
ctx.request_policy.auto_bound_credentials = [logins[0]]
monkeypatch.setattr(tools_module.credentials, "load_credentials", AsyncMock(return_value=logins))
monkeypatch.setattr(
credential_pause_module.app,
"DATABASE",
SimpleNamespace(
credentials=SimpleNamespace(
get_credentials_by_ids=AsyncMock(
side_effect=lambda ids, **_: [login for login in logins if login.credential_id in ids]
),
get_credential=AsyncMock(return_value=None),
),
organizations=SimpleNamespace(get_valid_org_auth_token=AsyncMock(return_value=None)),
),
)
ctx.stream.send = AsyncMock(
side_effect=_answer_each_card(cache, [("connected", "cred_a"), ("connected", "cred_b")])
)
return ctx, [(await _call_ask_tool(ctx))["status"] for _ in logins]
@pytest.mark.asyncio
async def test_each_settled_card_keeps_its_own_turn_end_receipt(monkeypatch: pytest.MonkeyPatch) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
_stub_credential_lookup(monkeypatch, _make_credential())
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [("connected", "cred_1"), ("skip", None)]))
await _call_ask_tool(ctx)
await _call_ask_tool(ctx)
result = _make_agent_result(
ctx,
user_response="ok",
updated_workflow=None,
turn_outcome=TurnOutcome(response_kind=ResponseKind.BUILD),
narrative_payload=_backfill_payload(),
)
assert result.narrative_payload is not None
assert result.narrative_payload["credentialPause"] == {"outcome": "skipped"}
assert result.narrative_payload["credentialPauses"] == [
{"outcome": "connected", "credentialId": "cred_1"},
{"outcome": "skipped"},
]
@pytest.mark.asyncio
async def test_a_second_card_cancelled_mid_wait_keeps_the_first_cards_turn_end_receipt(
monkeypatch: pytest.MonkeyPatch,
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
_stub_credential_lookup(monkeypatch, _make_credential())
answer_first = _answer_each_card(cache, [("connected", "cred_1")])
second_card_open = asyncio.Event()
async def send(
frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
) -> bool:
if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED and frame.resume_token == "tok-2":
second_card_open.set()
return True
return await answer_first(frame)
ctx.stream.send = AsyncMock(side_effect=send)
assert (await _call_ask_tool(ctx))["status"] == "connected"
second = asyncio.create_task(_call_ask_tool(ctx))
await asyncio.wait_for(second_card_open.wait(), 1)
second.cancel()
with pytest.raises(asyncio.CancelledError):
await second
result = _make_agent_result(
ctx,
user_response="ok",
updated_workflow=None,
turn_outcome=TurnOutcome(response_kind=ResponseKind.BUILD),
narrative_payload=_backfill_payload(),
)
assert result.narrative_payload is not None
assert result.narrative_payload["credentialPause"] == {"outcome": "connected", "credentialId": "cred_1"}
@pytest.mark.asyncio
async def test_an_unanswered_card_returns_a_typed_outcome_rather_than_an_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The user can walk away from an open card; the model must get a typed arm it can act on
instead of a failed call it would retry or narrate as a breakage."""
ctx = _tool_ctx(monkeypatch)
ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=0)
result = await _ask(ctx)
assert result["ok"] is True
assert result["status"] == "unanswered"
assert result["outcome"] == "timeout"
assert result["next"]
ctx.stream.send.assert_awaited()
@pytest.mark.asyncio
async def test_the_tool_offers_a_discovered_site_and_binds_only_the_users_card_answer(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _tool_ctx(monkeypatch, _answered_cache("connected", "cred_1"))
_stub_credential_lookup(monkeypatch, _make_credential())
result = await _ask(ctx, "https://elsewhere.example.net/login")
assert result["status"] == "connected"
frame = ctx.stream.send.await_args_list[0].args[0]
assert frame.login_page_urls == ["https://elsewhere.example.net/login"]
assert ctx.request_policy.live_page_admitted_urls == {"cred_1": "https://elsewhere.example.net/login"}
assert ctx.request_policy.current_turn_named_credential_ids == {"cred_1"}
@pytest.mark.asyncio
@pytest.mark.parametrize("login_url", ["", "javascript:alert(1)", "not-a-url"])
async def test_the_card_requires_a_real_sign_in_url(monkeypatch: pytest.MonkeyPatch, login_url: str) -> None:
ctx = _tool_ctx(monkeypatch)
result = await _ask(ctx, login_url)
assert result["ok"] is False
ctx.stream.send.assert_not_awaited()
@pytest.mark.asyncio
async def test_the_tool_reports_unavailable_with_a_prose_fallback_when_the_card_cannot_render(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _tool_ctx(monkeypatch)
ctx.client_supports_credential_pause = False
result = await _ask(ctx)
assert result["status"] == "unavailable"
assert "Credentials page" in result["fallback"]
assert ctx.request_policy.credential_ask_login_page_urls == []
ctx.stream.send.assert_not_awaited()
@pytest.mark.asyncio
async def test_the_tool_reports_unavailable_rather_than_raising_when_no_cache_is_configured(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A deployment with no cache at all must degrade, not fault the turn on an attribute of None."""
ctx = _tool_ctx(monkeypatch)
monkeypatch.setattr(app._inst, "CACHE", None, raising=False)
result = await _ask(ctx)
assert result["status"] == "unavailable"
ctx.stream.send.assert_not_awaited()
class _SlowAnswerCache(_FakeCache):
"""Answers the card only after ``delay_seconds``, standing in for a user who takes a moment."""
def __init__(self, response_key: str, delay_seconds: float, action: str = "skip") -> None:
super().__init__()
self._response_key = response_key
self._action = action
self._ready_at = time.monotonic() + delay_seconds
async def get(self, key: str) -> Any:
if key == self._response_key and time.monotonic() >= self._ready_at:
return encode_credential_response(self._action, "cred_1" if self._action == "connected" else None, "tok-1")
return await super().get(key)
@pytest.mark.asyncio
async def test_cancelling_the_turn_while_the_card_is_open_stays_a_cancellation(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The suspended deadline is handed back on the way out; a failure to reschedule it must not
replace the cancellation with a RuntimeError the enforcement loop does not handle."""
ctx = _tool_ctx(
monkeypatch,
_SlowAnswerCache(credential_response_cache_key("org-1", "chat-1", "turn-1"), 30),
)
async def ask_under_deadline() -> None:
async with asyncio.timeout(30) as deadline:
ctx.model_stream_deadline = deadline
await _ask(ctx)
task = asyncio.create_task(ask_under_deadline())
await asyncio.sleep(0.05)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert ctx.copilot_credential_pause_seconds > 0
_RUN_TOOLS_GATED_ON_AN_OPEN_ASK = [
("run_blocks_and_collect_debug", {"block_labels": ["login"], "parameters": {}}),
(
"edit_block_and_run",
{
"label": "login",
"expected_code": "old",
"replacement_code": "new",
"block_labels": ["login"],
"parameters": {},
},
),
("update_and_run_blocks", {"workflow": {"title": "draft"}, "block_labels": ["login"], "parameters": {}}),
]
@pytest.mark.asyncio
@pytest.mark.parametrize(("tool_name", "arguments"), _RUN_TOOLS_GATED_ON_AN_OPEN_ASK)
async def test_a_run_tool_called_alongside_the_ask_waits_for_the_user_to_answer(
monkeypatch: pytest.MonkeyPatch,
tool_name: str,
arguments: dict[str, Any],
) -> None:
"""The provider issues sibling tool calls in one model response, so without this wait the run
starts against the pre-card policy and the user answers a login the run already went without."""
ctx = _tool_ctx(
monkeypatch,
_SlowAnswerCache(credential_response_cache_key("org-1", "chat-1", "turn-1"), 0.25, "connected"),
)
_stub_credential_lookup(monkeypatch, _make_credential())
ctx.turn_origin = TurnOrigin.code_block_ai_fallback
tool = {
"run_blocks_and_collect_debug": tools_module.run_blocks_tool,
"edit_block_and_run": tools_module.edit_block_and_run_tool,
"update_and_run_blocks": tools_module.update_and_run_blocks_tool,
}[tool_name]
finished: list[str] = []
async def ask() -> None:
await _ask(ctx)
finished.append("ask")
async def run() -> None:
await tool.on_invoke_tool(SimpleNamespace(context=ctx, tool_name=tool_name), json.dumps(arguments))
finished.append("run")
await asyncio.gather(ask(), run())
assert finished == ["ask", "run"]
assert ctx.credential_pause_outcome == "connected"
@pytest.mark.asyncio
async def test_an_ask_issued_alongside_a_run_renders_after_its_own_row(monkeypatch: pytest.MonkeyPatch) -> None:
"""Sibling calls in one model response log their rows before any of them runs, so the newest row
when the ask starts is a sibling's rather than the ask's own."""
ctx = _tool_ctx(
monkeypatch,
_SlowAnswerCache(credential_response_cache_key("org-1", "chat-1", "turn-1"), 0.01, "connected"),
)
_stub_credential_lookup(monkeypatch, _make_credential())
ctx.narrator_state = NarratorState()
for tool_name, call_id in (("request_credential", "call-ask"), ("run_blocks_and_collect_debug", "call-run")):
ctx.narrator_state.record_activity(
build_tool_call_activity(tool_name, 0, call_id, timestamp=datetime.now(timezone.utc))
)
arguments = json.dumps({"login_page_url": "https://portal.example.com/login", "reason": "Needs a sign-in."})
await tools_module.request_credential_tool.on_invoke_tool(
ToolContext(context=ctx, tool_name="request_credential", tool_call_id="call-ask", tool_arguments=arguments),
arguments,
)
(card,) = _sent_cards(ctx)
assert card.anchor_tool_call_id == ctx.credential_pause_anchor_tool_call_id == "call-ask"
async def _announce_ask(ctx: CopilotContext) -> None:
response = SimpleNamespace(output=[SimpleNamespace(name="request_credential")])
await CopilotRunHooks(ctx).on_llm_end(SimpleNamespace(context=ctx), MagicMock(), response) # type: ignore[arg-type]
async def _invoke_ask_tool(ctx: CopilotContext, login_page_url: str) -> None:
await tools_module.request_credential_tool.on_invoke_tool(
SimpleNamespace(context=ctx, tool_name="request_credential"), # type: ignore[arg-type]
json.dumps({"login_page_url": login_page_url, "reason": "This site needs a sign-in."}),
)
def test_the_ask_tool_is_registered_in_the_production_tool_set() -> None:
"""Every other test here invokes the tool object directly, so unregistering it would take the
whole capability away without reddening one of them."""
assert "request_credential" in {tool.name for tool in tools_module.NATIVE_TOOLS}
@pytest.mark.asyncio
@pytest.mark.parametrize(("tool_name", "arguments"), _RUN_TOOLS_GATED_ON_AN_OPEN_ASK)
async def test_a_run_tool_scheduled_ahead_of_the_ask_in_one_response_still_waits(
monkeypatch: pytest.MonkeyPatch,
tool_name: str,
arguments: dict[str, Any],
) -> None:
"""Tool calls in one model response run as concurrent tasks in an order the model picks, so a
gate armed only inside the ask's own coroutine is passed by whichever sibling is scheduled first."""
ctx = _tool_ctx(
monkeypatch,
_SlowAnswerCache(credential_response_cache_key("org-1", "chat-1", "turn-1"), 0.25, "connected"),
)
_stub_credential_lookup(monkeypatch, _make_credential())
await _announce_ask(ctx)
finished: list[str] = []
tool = {
"run_blocks_and_collect_debug": tools_module.run_blocks_tool,
"edit_block_and_run": tools_module.edit_block_and_run_tool,
"update_and_run_blocks": tools_module.update_and_run_blocks_tool,
}[tool_name]
async def run() -> None:
await tool.on_invoke_tool(SimpleNamespace(context=ctx, tool_name=tool_name), json.dumps(arguments)) # type: ignore[arg-type]
finished.append("run")
async def ask() -> None:
await _invoke_ask_tool(ctx, "https://portal.example.com/login")
finished.append("ask")
await asyncio.gather(run(), ask())
# Without this the ask refuses instantly and the ordering is proven around a card that never
# opened, which is the one shape the wrapper's release runs in.
assert ctx.credential_pause_outcome == "connected"
assert finished == ["ask", "run"]
@pytest.mark.asyncio
async def test_an_announced_ask_that_never_reaches_the_card_still_releases_the_waiting_run_tool(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Arming on the response means the gate closes before the ask can refuse, so every exit of the
tool has to open it again or the sibling waits out the whole turn."""
ctx = _tool_ctx(monkeypatch)
await _announce_ask(ctx)
async def ask() -> None:
await _invoke_ask_tool(ctx, "not-a-url")
await asyncio.wait_for(asyncio.gather(await_pending_credential_pause(ctx), ask()), timeout=5)
assert ctx.request_policy.credential_ask_login_page_urls == []
def test_the_card_message_is_capped_and_stripped_before_the_user_sees_it() -> None:
"""`reason` is model-authored and leads the card, above the hostname. An unbounded one can push
the site the credential is actually for off the viewport."""
shouted = 'Connect your bank login\n"now"' + "!" * 500
safe = credential_pause_module.defang_card_text(shouted)
assert len(safe) <= 200
assert "\n" not in safe
assert '"' not in safe
@pytest.mark.asyncio
async def test_a_repeat_ask_in_a_later_response_releases_the_gate_it_armed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A later response arms a fresh gate, so an ask that settles without releasing it parks every
sibling on an Event no one owns until the turn times out."""
ctx = _tool_ctx(monkeypatch, _answered_cache("skip"))
await _announce_ask(ctx)
await _invoke_ask_tool(ctx, "https://portal.example.com/login")
await asyncio.wait_for(await_pending_credential_pause(ctx), timeout=1)
def _redacted_secret_policy(*site_urls: str) -> RequestPolicy:
policy = RequestPolicy(user_provided_site_urls=list(site_urls))
policy.apply_raw_secret_redacted_draft()
return policy
@pytest.mark.asyncio
@pytest.mark.parametrize(
("site_urls", "handling", "model_url"),
[
((), "redacted_draft", "https://portal.example.com/login"),
(("https://portal.example.com/login",), "redacted_draft", "https://elsewhere.example.net/login"),
(("https://portal.example.com/login",), "block", "https://portal.example.com/login"),
],
ids=["no_user_url", "model_only_url", "blocked_turn"],
)
async def test_a_raw_secret_turn_opens_no_card_without_a_site_the_user_gave(
monkeypatch: pytest.MonkeyPatch, site_urls: tuple[str, ...], handling: str, model_url: str
) -> None:
ctx = _tool_ctx(monkeypatch, _answered_cache("connected", "cred_1"))
ctx.request_policy = _redacted_secret_policy(*site_urls)
ctx.request_policy.raw_secret_handling = handling
result = await _ask(ctx, model_url)
assert result["ok"] is False
ctx.stream.send.assert_not_awaited()
assert ctx.credential_pause_used is False
assert ctx.request_policy.credential_ask_login_page_urls == []
@pytest.mark.asyncio
async def test_a_raw_secret_turn_opens_no_authenticator_card(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = _tool_ctx(monkeypatch, _answered_cache("connected", "cred_1"))
credential = wire_credential_vault(monkeypatch, PasswordCredential(username="u", password="p", totp=None))
ctx.request_policy = _redacted_secret_policy("https://portal.example.com/login")
ctx.request_policy.resolved_credentials = [credential]
result = await _request_credential("https://portal.example.com/login", "Needs 2FA.", ctx, "cred_1")
assert result["ok"] is False
ctx.stream.send.assert_not_awaited()
assert ctx.credential_totp_update_asked is False
_CardAnswer = tuple[Literal["connected", "skip", "signed_in", "unanswered", "start_sign_in"], str | None]
_SIGNED_IN_PORTAL = credential_pause_module.SignedInProfile(
browser_profile_id="bp_signed_in",
profile_name="Sign-in for portal.example.com",
site="portal.example.com",
cookie_count=2,
)
def _answer_each_card(
cache: _FakeCache, answers: list[_CardAnswer]
) -> Callable[
[WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate], Awaitable[bool]
]:
"""Answer each card through the resume route's own writer as it is sent."""
pending = list(answers)
async def send(
card: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
) -> bool:
if card.type != WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:
return True
action, credential_id = pending.pop(0)
if action == "unanswered":
return True
if action == "start_sign_in":
await credential_pause_module.start_manual_sign_in(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id=card.turn_id,
resume_token=card.resume_token,
)
return True
if action == "signed_in":
# The Done route saves the profile and writes this response; saving a profile is not under test here.
cache.store[credential_response_cache_key("org-1", "chat-1", card.turn_id)] = encode_credential_response(
action, None, card.resume_token, signed_in=_SIGNED_IN_PORTAL
)
return True
await resolve_credential_pause(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id=card.turn_id,
resume_token=card.resume_token,
action=action,
credential_id=credential_id,
)
return True
return send
async def _call_ask_tool(ctx: CopilotContext, **arguments: object) -> dict[str, Any]:
raw = await tools_module.request_credential_tool.on_invoke_tool(
SimpleNamespace(context=ctx, tool_name="request_credential"), # type: ignore[arg-type]
json.dumps({"login_page_url": "https://portal.example.com/login", "reason": "Needs 2FA.", **arguments}),
)
return json.loads(raw)
def _sent_cards(ctx: CopilotContext) -> list[WorkflowCopilotCredentialRequiredUpdate]:
sent = [call.args[0] for call in ctx.stream.send.await_args_list]
return [frame for frame in sent if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED]
def _sent_resolutions(ctx: CopilotContext) -> list[WorkflowCopilotCredentialPauseResolvedUpdate]:
sent = [call.args[0] for call in ctx.stream.send.await_args_list]
return [frame for frame in sent if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_PAUSE_RESOLVED]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("totp_after_save", "expected_status"),
[(None, "saved_without_authenticator"), ("fake-seed", "authenticator_added")],
)
async def test_a_credential_with_no_authenticator_raises_an_update_card_naming_it(
monkeypatch: pytest.MonkeyPatch, totp_after_save: str | None, expected_status: str
) -> None:
"""The update answer grants nothing, and success is read from the saved record, not from the answer."""
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
credential = wire_credential_vault(monkeypatch, PasswordCredential(username="u", password="p", totp=None))
app.CREDENTIAL_VAULT_SERVICES[CredentialVaultType.SKYVERN].get_credential_item.side_effect = [
SimpleNamespace(name="authtest simple", credential=PasswordCredential(username="u", password="p", totp=None)),
SimpleNamespace(
name="authtest simple", credential=PasswordCredential(username="u", password="p", totp=totp_after_save)
),
]
ctx.request_policy.resolved_credentials = [credential]
ctx.request_policy.live_page_admitted_urls = {"cred_1": "https://app.example.org/signin"}
ctx.request_policy.credential_ask_login_page_urls = ["https://app.example.org/signin"]
ctx.request_policy.allow_run_blocks = False
policy_before = deepcopy(ctx.request_policy)
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [("connected", "cred_1")]))
result = await _call_ask_tool(ctx, credential_id="cred_1")
[card] = _sent_cards(ctx)
assert (card.reason, card.credential_refs, card.login_page_urls) == (
"credential_missing_totp",
["cred_1"],
["https://portal.example.com/login"],
)
[resolved] = _sent_resolutions(ctx)
assert (resolved.resume_token, resolved.outcome, resolved.credential_id) == (
card.resume_token,
"connected",
"cred_1",
)
assert (result["status"], result["credential_id"]) == (expected_status, "cred_1")
assert ctx.request_policy == policy_before
assert ctx.credential_pause_outcome is None
@pytest.mark.asyncio
async def test_an_update_card_answered_with_another_credential_is_not_honored(
monkeypatch: pytest.MonkeyPatch,
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
credential = wire_credential_vault(monkeypatch, PasswordCredential(username="u", password="p", totp=None))
ctx.request_policy.resolved_credentials = [credential]
policy_before = deepcopy(ctx.request_policy)
app.DATABASE.credentials.get_credentials_by_ids = AsyncMock(return_value=[_make_credential("cred_2")])
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [("connected", "cred_2")]))
result = await _call_ask_tool(ctx, credential_id="cred_1")
assert result["status"] == "unanswered"
assert "credential_id" not in result
assert ctx.request_policy == policy_before
assert _sent_resolutions(ctx) == []
@pytest.mark.asyncio
async def test_an_update_ask_waits_out_a_card_already_on_screen(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = _tool_ctx(monkeypatch)
credential = wire_credential_vault(monkeypatch, PasswordCredential(username="u", password="p", totp=None))
ctx.request_policy.resolved_credentials = [credential]
ctx.credential_ask_in_flight = True
result = await _request_credential("https://portal.example.com/login", "Needs 2FA.", ctx, "cred_1")
assert result["status"] == "already_asked"
ctx.stream.send.assert_not_awaited()
@pytest.mark.asyncio
async def test_the_update_card_after_a_pick_waits_for_its_own_answer(monkeypatch: pytest.MonkeyPatch) -> None:
"""Both cards share the turn's response key, so the update card must not read the pick's answer."""
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
wire_credential_vault(monkeypatch, PasswordCredential(username="u", password="p", totp=None))
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [("connected", "cred_1"), ("skip", None)]))
picked = await _call_ask_tool(ctx)
ctx.last_test_ok = False
update = await _call_ask_tool(ctx, credential_id="cred_1")
repeat = await _call_ask_tool(ctx, credential_id="cred_1")
assert [picked["status"], update["status"], repeat["status"]] == ["connected", "skipped", "already_asked"]
assert "outcome" not in repeat
assert [card.reason for card in _sent_cards(ctx)] == ["login_credentials_unresolved", "credential_missing_totp"]
assert (ctx.credential_pause_outcome, ctx.credential_pause_connected_credential_id) == ("connected", "cred_1")
assert ctx.last_test_ok is False
@pytest.mark.asyncio
async def test_a_pick_card_carries_the_one_login_the_user_named_and_an_update_card_does_not(
monkeypatch: pytest.MonkeyPatch,
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
wire_credential_vault(monkeypatch, PasswordCredential(username="u", password="p", totp=None))
ctx.request_policy.current_turn_named_credential_ids = {"cred_1"}
ctx.vault_login_uris_by_credential_id["cred_1"] = []
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [("connected", "cred_1"), ("skip", None)]))
await _call_ask_tool(ctx)
await _call_ask_tool(ctx, credential_id="cred_1")
assert [card.named_credential_id for card in _sent_cards(ctx)] == ["cred_1", None]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("saved_url", "confirmed_url", "vault_sites", "offered"),
[
("https://other.example.com/login", None, [], None),
(None, "https://other.example.com/login", [], None),
(None, None, ["https://other.example.org"], None),
(None, None, None, None),
("https://portal.example.com/account", "https://portal.example.com/login", ["example.com"], "cred_1"),
],
ids=[
"saved_for_another_site",
"confirmed_for_another_site",
"vault_names_another_site",
"vault_unreadable",
"same_site",
],
)
async def test_a_named_login_known_for_another_site_is_left_to_the_picker(
monkeypatch: pytest.MonkeyPatch,
saved_url: str | None,
confirmed_url: str | None,
vault_sites: list[str] | None,
offered: str | None,
) -> None:
ctx = _tool_ctx(monkeypatch, _answered_cache("skip"))
if vault_sites is not None:
ctx.vault_login_uris_by_credential_id["cred_1"] = vault_sites
ctx.request_policy.current_turn_named_credential_ids = {"cred_1"}
ctx.request_policy.resolved_credentials = [_make_credential().model_copy(update={"tested_url": saved_url})]
if confirmed_url:
ctx.request_policy.live_page_admitted_urls["cred_1"] = confirmed_url
await _ask(ctx, "https://portal.example.com/login")
assert [card.named_credential_id for card in _sent_cards(ctx)] == [offered]
@pytest.mark.asyncio
async def test_denied_vault_site_read_leaves_a_named_login_to_the_picker(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = _tool_ctx(monkeypatch, _answered_cache("skip"))
ctx.request_policy.current_turn_named_credential_ids = {"cred_1"}
monkeypatch.setattr(
credential_fill_module,
"_read_vault_named_sites",
AsyncMock(side_effect=PermissionError("credential not accessible")),
)
await _ask(ctx, "https://portal.example.com/login")
assert [card.named_credential_id for card in _sent_cards(ctx)] == [None]
@pytest.mark.asyncio
@pytest.mark.parametrize("carried_as", ["approval", "proposal"])
async def test_a_login_carried_in_for_another_site_is_left_to_the_picker_when_named_again(
monkeypatch: pytest.MonkeyPatch, carried_as: str
) -> None:
ctx = _tool_ctx(monkeypatch, _answered_cache("skip"))
ctx.vault_login_uris_by_credential_id["cred_1"] = []
policy = ctx.request_policy
policy.resolved_credentials = [_make_credential()]
_stub_credential_lookup(monkeypatch, _make_credential())
earlier = StructuredContext()
other_site = "https://other.example.com/login"
if carried_as == "approval":
earlier.approved_credentials = [ApprovedCredential(credential_id="cred_1", admitted_url=other_site)]
seed = _seed_prior_approved_credentials
else:
earlier.proposed_credential = ProposedCredential(credential_id="cred_1", admitted_url=other_site)
seed = _seed_proposed_credential
await seed(policy, organization_id="org-1", global_llm_context=earlier.to_json_str())
# What resolving a re-named carried credential does before any card is built.
policy.live_page_admitted_urls.pop("cred_1")
policy.current_turn_named_credential_ids = {"cred_1"}
await _ask(ctx, "https://portal.example.com/login")
assert [card.named_credential_id for card in _sent_cards(ctx)] == [None]
@pytest.mark.asyncio
async def test_an_update_ask_leaves_the_pick_budget_for_a_later_card(monkeypatch: pytest.MonkeyPatch) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
credential = wire_credential_vault(monkeypatch, PasswordCredential(username="u", password="p", totp=None))
ctx.request_policy.resolved_credentials = [credential]
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [("skip", None), ("connected", "cred_1")]))
update = await _call_ask_tool(ctx, credential_id="cred_1")
run_card_ready = credential_pause_module.credential_pause_transport_ready(ctx, ctx.copilot_config)
picked = await _call_ask_tool(ctx)
assert [update["status"], picked["status"]] == ["skipped", "connected"]
assert run_card_ready is True
assert [card.reason for card in _sent_cards(ctx)] == ["credential_missing_totp", "login_credentials_unresolved"]
@pytest.mark.asyncio
async def test_an_answer_without_a_resume_token_resolves_no_card(monkeypatch: pytest.MonkeyPatch) -> None:
cache = _FakeCache()
key = credential_response_cache_key("org-1", "chat-1", "turn-1")
cache.store[key] = json.dumps({"action": "connected", "credential_id": "cred_1"})
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
assert await credential_pause_module._try_resolve_credential_response(key, "org-1", "tok-1") == "pending"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("answer", "status", "resolved_outcomes"),
[("skip", "skipped", ["skipped"]), ("timeout", "unanswered", []), ("unsupported_client", "unavailable", [])],
)
async def test_a_raw_secret_card_keeps_the_typed_non_connect_outcomes(
monkeypatch: pytest.MonkeyPatch, answer: str, status: str, resolved_outcomes: list[str]
) -> None:
ctx = _tool_ctx(monkeypatch, _answered_cache("skip") if answer == "skip" else None)
ctx.request_policy = _redacted_secret_policy("https://portal.example.com/login")
if answer == "timeout":
ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=0)
if answer == "unsupported_client":
ctx.client_supports_credential_pause = False
result = await _ask(ctx)
assert result["status"] == status
assert ctx.request_policy.allow_run_blocks is False
assert ctx.request_policy.current_turn_named_credential_ids == set()
assert [frame.outcome for frame in _sent_resolutions(ctx)] == resolved_outcomes
@pytest.mark.asyncio
@pytest.mark.parametrize(
("secrets", "resolved", "expected"),
[
pytest.param(
PasswordCredential(username="u", password="p", totp="fake-seed"),
True,
{"ok": True, "status": "has_code_method", "method": "authenticator"},
id="authenticator",
),
pytest.param(
PasswordCredential(
username="u", password="p", totp=None, totp_type=TotpType.EMAIL, totp_identifier="[email protected]"
),
True,
{"ok": True, "status": "has_code_method", "method": "email_or_text"},
id="email-otp",
),
pytest.param(
PasswordCredential(username="u", password="p", totp=None),
False,
{"ok": False},
id="not-resolved-for-this-request",
),
],
)
async def test_no_update_card_for_a_credential_with_a_code_method_or_outside_the_request(
monkeypatch: pytest.MonkeyPatch, secrets: PasswordCredential, resolved: bool, expected: dict[str, Any]
) -> None:
ctx = _tool_ctx(monkeypatch)
credential = wire_credential_vault(monkeypatch, secrets)
ctx.request_policy.resolved_credentials = [credential] if resolved else []
result = await _call_ask_tool(ctx, credential_id="cred_1")
assert {key: result[key] for key in expected} == expected
ctx.stream.send.assert_not_awaited()
@pytest.mark.asyncio
async def test_an_authenticator_code_method_names_the_fill_tool_only_when_the_turn_has_it(
monkeypatch: pytest.MonkeyPatch,
) -> None:
results = []
for native_tool_names in ((CREDENTIAL_FILL_TOOL_NAME,), ()):
ctx = _tool_ctx(monkeypatch)
ctx.eval_native_tool_names = native_tool_names
secrets = PasswordCredential(username="u", password="p", totp="fake-seed")
ctx.request_policy.resolved_credentials = [wire_credential_vault(monkeypatch, secrets)]
results.append(await _call_ask_tool(ctx, credential_id="cred_1"))
with_tool, without_tool = results
assert (without_tool["ok"], without_tool["status"], without_tool["method"]) == (
True,
"has_code_method",
"authenticator",
)
assert without_tool["detail"]
assert {key: value for key, value in with_tool.items() if key != "next"} == without_tool
assert with_tool["next"]
_IDP_ORIGIN = "https://idp.example.test"
_PENDING = CredentialOriginRecovery(_IDP_ORIGIN, "pending", refused_credential_id="cred_service")
def _idp_credential(credential_id: str = "cred_idp") -> Credential:
return _make_credential(credential_id, "Provider Login").model_copy(update={"tested_url": f"{_IDP_ORIGIN}/login"})
def _recovery_ctx(monkeypatch: pytest.MonkeyPatch, cache: _FakeCache) -> CopilotContext:
ctx = _tool_ctx(monkeypatch, cache)
ctx.credential_pause_used = True
ctx.credential_pause_outcome = "connected"
ctx.credential_origin_recovery = _PENDING
ctx.blocker_signal = CopilotToolBlockerSignal(
blocker_kind="authority_denied",
agent_steering_text="Ask for a login on the provider.",
user_facing_reason="The sign-in continues on another site.",
recovery_hint="ask_user_clarifying",
internal_reason_code=CREDENTIAL_ORIGIN_RECOVERY_PENDING_REASON_CODE,
renders_final_reply=False,
)
return ctx
def _assert_declined(ctx: CopilotContext, result: dict[str, Any], status: str) -> None:
assert result["status"] == status
assert ctx.credential_origin_recovery == replace(_PENDING, state="declined")
declined = ctx.blocker_signal
assert declined.internal_reason_code == CREDENTIAL_ORIGIN_RECOVERY_DECLINED_REASON_CODE
assert CREDENTIAL_ORIGIN_RECOVERY_PENDING_REASON_CODE not in {
signal.internal_reason_code for signal in ctx.tool_blocker_signals
}
assert declined.preserves_workflow_draft is True
for tool_name in ("update_workflow", "run_blocks_and_collect_debug", "edit_block_and_run", "update_and_run_blocks"):
assert _authority_tool_error(ctx, tool_name) is None
models_own_report = AgentResult(user_response="Ran it; it failed.", updated_workflow=None, global_llm_context=None)
assert _finalize_result_with_blocker_override(ctx, models_own_report) is models_own_report
assert _IDP_ORIGIN in result["next"] # nosemgrep: incomplete-url-substring-sanitization
@pytest.mark.asyncio
@pytest.mark.parametrize("connect_first", [True, False], ids=["after_a_connect", "while_pending"])
@pytest.mark.parametrize(
"later_ask_url",
["https://idp.provider.com/login", "https://login.provider.com/login"],
ids=["same_origin", "same_site"],
)
async def test_a_pick_ask_on_a_recovery_site_keeps_its_admission_check_before_and_after_a_connect(
monkeypatch: pytest.MonkeyPatch, later_ask_url: str, connect_first: bool
) -> None:
# A registrable domain, unlike .test, so a sibling host shares the recovery's site.
provider = "https://idp.provider.com"
cache = _FakeCache()
ctx = _recovery_ctx(monkeypatch, cache)
ctx.credential_origin_recovery = CredentialOriginRecovery(provider, "pending", refused_credential_id="cred_service")
refused = _make_credential("cred_service").model_copy(update={"tested_url": "https://portal.example.com/login"})
provider_login = _make_credential("cred_idp", "Provider Login").model_copy(
update={"tested_url": f"{provider}/login"}
)
logins = [provider_login, refused]
monkeypatch.setattr(
credential_pause_module.app,
"DATABASE",
SimpleNamespace(
credentials=SimpleNamespace(
get_credentials_by_ids=AsyncMock(
side_effect=lambda ids, **_: [login for login in logins if login.credential_id in ids]
),
get_credential=AsyncMock(return_value=None),
)
),
)
answers: list[_CardAnswer] = [("connected", "cred_idp")] if connect_first else []
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [*answers, ("connected", "cred_service")]))
if connect_first:
result = await _ask(ctx, f"{provider}/login?state=opaque-state#fragment")
assert result["status"] == "connected"
assert result["credential_id"] == "cred_idp"
[card] = _sent_cards(ctx)
assert card.login_page_urls == [f"{provider}/login"]
assert ctx.request_policy.live_page_admitted_urls["cred_idp"] == f"{provider}/login"
assert ctx.credential_origin_recovery is None
assert ctx.blocker_signal is None
again = await _ask(ctx, later_ask_url)
assert again["status"] == "connected_other_site"
assert [resolved.outcome for resolved in _sent_resolutions(ctx)][-1] == "not_admitted"
assert "cred_service" not in ctx.request_policy.live_page_admitted_urls
@pytest.mark.asyncio
@pytest.mark.parametrize(
("steps", "later_origin", "picked"),
[
(
[
("refuse", "https://a.provider.com", "cred_a"),
("card", "https://a.provider.com", "cred_b"),
("refuse", "https://b.provider.com", "cred_d"),
("card", "https://b.provider.com", "cred_e"),
],
"https://b.provider.com",
"cred_d",
),
(
[
("refuse", "https://a.provider.com", "cred_a"),
("card", "https://a.provider.com", "cred_b"),
("refuse", "https://a.provider.com", "cred_d"),
("card", "https://a.provider.com", "cred_b"),
],
"https://a.provider.com",
"cred_a",
),
(
[
("refuse", "https://a.provider.com", "cred_a"),
("card", "https://a.provider.com", "cred_b"),
("refuse", "https://b.provider.com", "cred_d"),
],
"https://a.provider.com",
"cred_a",
),
(
[
("refuse", "https://idp.other.com", "cred_x"),
("refuse", "https://b.provider.com", "cred_d"),
("card", "https://idp.other.com", "cred_o"),
],
"https://b.provider.com",
"cred_d",
),
],
ids=["sibling_origins", "same_origin_twice", "sibling_still_pending", "refused_while_another_site_pending"],
)
async def test_a_later_card_on_a_recovery_site_never_admits_a_login_refused_anywhere_on_it(
monkeypatch: pytest.MonkeyPatch,
steps: list[tuple[Literal["refuse", "card"], str, str]],
later_origin: str,
picked: str,
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
# The refused logins are saved on the provider's own site, so only the refusal can keep them off it.
logins = {
"cred_a": _make_credential("cred_a").model_copy(update={"tested_url": "https://login.provider.com/a"}),
"cred_d": _make_credential("cred_d").model_copy(update={"tested_url": "https://login.provider.com/d"}),
"cred_b": _make_credential("cred_b").model_copy(update={"tested_url": "https://a.provider.com/login"}),
"cred_e": _make_credential("cred_e").model_copy(update={"tested_url": "https://b.provider.com/login"}),
"cred_o": _make_credential("cred_o").model_copy(update={"tested_url": "https://idp.other.com/login"}),
}
monkeypatch.setattr(
credential_pause_module.app,
"DATABASE",
SimpleNamespace(
credentials=SimpleNamespace(
get_credentials_by_ids=AsyncMock(side_effect=lambda ids, **_: [logins[i] for i in ids if i in logins]),
get_credential=AsyncMock(return_value=None),
)
),
)
answers: list[_CardAnswer] = [("connected", login) for step, _, login in steps if step == "card"]
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [*answers, ("connected", picked)]))
for step, origin, login in steps:
if step == "refuse":
credential_fill_module._start_credential_origin_recovery(ctx, origin, login)
else:
assert (await _ask(ctx, f"{origin}/login"))["credential_id"] == login
later = await _ask(ctx, f"{later_origin}/login")
assert later["status"] == "connected_other_site"
assert picked not in ctx.request_policy.live_page_admitted_urls
@pytest.mark.asyncio
async def test_a_card_for_another_site_after_a_declined_recovery_still_offers_signing_in(
monkeypatch: pytest.MonkeyPatch,
) -> None:
cache = _FakeCache()
ctx = _recovery_ctx(monkeypatch, cache)
ctx.browser_session_id = "pbs_debug"
_stub_credential_lookup(monkeypatch, _make_credential())
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [("skip", None), ("skip", None)]))
_assert_declined(ctx, await _ask(ctx, f"{_IDP_ORIGIN}/login"), "skipped")
await _ask(ctx, "https://portal.example.com/login")
recovery_card, other_site_card = _sent_cards(ctx)
assert recovery_card.sign_in_browser_session_id is None
assert other_site_card.sign_in_browser_session_id == "pbs_debug"
@pytest.mark.asyncio
async def test_a_retry_after_a_declined_origin_recovery_opens_a_card_that_keeps_its_admission_check(
monkeypatch: pytest.MonkeyPatch,
) -> None:
cache = _FakeCache()
ctx = _recovery_ctx(monkeypatch, cache)
_stub_credential_lookup(monkeypatch, _idp_credential())
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [("skip", None), ("connected", "cred_idp")]))
_assert_declined(ctx, await _ask(ctx, f"{_IDP_ORIGIN}/login"), "skipped")
again = await _ask(ctx, f"{_IDP_ORIGIN}/login")
assert again["status"] == "connected"
assert [card.credential_refs for card in _sent_cards(ctx)] == [[], []]
assert ctx.request_policy.live_page_admitted_urls == {"cred_idp": f"{_IDP_ORIGIN}/login"}
assert ctx.credential_origin_recovery is None
assert ctx.blocker_signal is None
@pytest.mark.asyncio
async def test_a_connected_credential_whose_own_site_differs_is_not_rebound_to_the_recovery_origin(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _recovery_ctx(monkeypatch, _answered_cache("connected", "cred_1"))
ctx.request_policy.current_turn_named_credential_ids = {"cred_1"}
service = _make_credential().model_copy(update={"tested_url": "https://portal.example.com/login"})
_stub_credential_lookup(monkeypatch, service)
result = await _ask(ctx, f"{_IDP_ORIGIN}/login")
[card] = _sent_cards(ctx)
[resolved] = _sent_resolutions(ctx)
assert (resolved.resume_token, resolved.outcome, resolved.credential_id) == (
card.resume_token,
"not_admitted",
None,
)
assert "cred_1" not in ctx.request_policy.live_page_admitted_urls
assert ctx.credential_pause_connected_credential_id is None
assert ctx.credential_pause_outcome == "not_admitted"
_assert_declined(ctx, result, "connected_other_site")
@pytest.mark.asyncio
@pytest.mark.parametrize(
("connected", "vault_row"),
[
(_idp_credential("cred_service"), None),
(_make_credential("cred_blank"), None),
(_idp_credential(), AsyncMock(side_effect=RuntimeError("vault unavailable"))),
],
ids=["refused_credential", "no_site_evidence", "vault_read_error"],
)
async def test_origin_recovery_admits_no_credential_it_cannot_place_on_the_provider(
monkeypatch: pytest.MonkeyPatch, connected: Credential, vault_row: AsyncMock | None
) -> None:
ctx = _recovery_ctx(monkeypatch, _answered_cache("connected", connected.credential_id))
ctx.request_policy.credential_refs = ["cred_service"]
_stub_credential_lookup(monkeypatch, connected, vault_row)
result = await _ask(ctx, f"{_IDP_ORIGIN}/login")
assert [card.credential_refs for card in _sent_cards(ctx)] == [[]]
assert connected.credential_id not in ctx.request_policy.live_page_admitted_urls
assert ctx.credential_pause_outcome == "not_admitted"
_assert_declined(ctx, result, "connected_other_site")
@pytest.mark.asyncio
async def test_denied_vault_site_read_does_not_abort_origin_recovery(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = _recovery_ctx(monkeypatch, _answered_cache("connected", "cred_idp"))
monkeypatch.setattr(
credential_fill_module,
"_read_vault_named_sites",
AsyncMock(side_effect=PermissionError("credential not accessible")),
)
_stub_credential_lookup(monkeypatch, _idp_credential())
monkeypatch.setattr(
credential_pause_module.app.DATABASE,
"workflow_params",
SimpleNamespace(record_pending_copilot_turn_credential_resume=AsyncMock()),
raising=False,
)
result = await _ask(ctx, f"{_IDP_ORIGIN}/login")
assert ctx.credential_pause_outcome == "not_admitted"
_assert_declined(ctx, result, "connected_other_site")
@pytest.mark.asyncio
async def test_a_recovery_connect_keeps_the_named_service_login_approved_beside_the_provider_login(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _recovery_ctx(monkeypatch, _answered_cache("connected", "cred_idp"))
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
service = _make_credential("cred_service").model_copy(update={"tested_url": "https://portal.example.com/login"})
policy = ctx.request_policy
policy.resolved_credentials = [service]
policy.current_turn_named_credential_ids = {"cred_service"}
policy.persisted_workflow_credential_ids = ["cred_service"]
_stub_credential_lookup(monkeypatch, _idp_credential())
result = await _ask(ctx, f"{_IDP_ORIGIN}/login")
assert result["status"] == "connected"
assert credential_fill_module._request_settled_credential(policy, "cred_idp")
grants = {
credential_id: (await credential_fill_module._credential_fill_origin_grant(ctx, credential_id))[0]
for credential_id in ("cred_service", "cred_idp")
}
assert grants["cred_service"].intended_url == "https://portal.example.com/login"
assert grants["cred_idp"].intended_url == f"{_IDP_ORIGIN}/login"
approved = StructuredContext.from_json_str(record_approved_credentials_in_global_llm_context(ctx, None))
assert {record.credential_id for record in approved.approved_credentials} == {"cred_service", "cred_idp"}
@pytest.mark.asyncio
async def test_a_card_that_raises_declines_origin_recovery(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = _recovery_ctx(monkeypatch, _FakeCache())
monkeypatch.setattr(
credential_fill_module, "request_credential_pause", AsyncMock(side_effect=RuntimeError("stream closed"))
)
with pytest.raises(RuntimeError, match="stream closed"):
await _ask(ctx, f"{_IDP_ORIGIN}/login")
assert ctx.credential_origin_recovery == replace(_PENDING, state="declined")
assert ctx.blocker_signal.internal_reason_code == CREDENTIAL_ORIGIN_RECOVERY_DECLINED_REASON_CODE
@pytest.mark.asyncio
async def test_a_connect_that_resolves_no_credential_declines_origin_recovery(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = _recovery_ctx(monkeypatch, _FakeCache())
monkeypatch.setattr(
credential_fill_module,
"request_credential_pause",
AsyncMock(return_value=CredentialPauseResolution(action="connected")),
)
result = await _ask(ctx, f"{_IDP_ORIGIN}/login")
_assert_declined(ctx, result, "connected_unresolved")
@pytest.mark.asyncio
async def test_origin_recovery_declines_when_the_card_cannot_be_shown(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = _recovery_ctx(monkeypatch, _FakeCache())
ctx.client_supports_credential_pause = False
with capture_logs() as logs:
result = await _ask(ctx, f"{_IDP_ORIGIN}/login")
assert "copilot_credential_card_unavailable" in {entry["event"] for entry in logs}
ctx.stream.send.assert_not_awaited()
assert ctx.credential_pause_used is True
_assert_declined(ctx, result, "unavailable")
@pytest.mark.asyncio
async def test_a_card_for_another_origin_leaves_origin_recovery_pending(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = _recovery_ctx(monkeypatch, _answered_cache("connected", "cred_1"))
ctx.credential_pause_used = False
_stub_credential_lookup(monkeypatch, _make_credential())
result = await _ask(ctx, "https://portal.example.com/login")
assert result["status"] == "connected"
assert ctx.credential_origin_recovery == _PENDING
assert ctx.blocker_signal.internal_reason_code == CREDENTIAL_ORIGIN_RECOVERY_PENDING_REASON_CODE
@pytest.mark.asyncio
async def test_origin_recovery_raises_no_second_card_while_one_is_in_flight(monkeypatch: pytest.MonkeyPatch) -> None:
cache = _FakeCache()
ctx = _recovery_ctx(monkeypatch, cache)
_stub_credential_lookup(monkeypatch, _idp_credential())
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [("connected", "cred_idp")]))
ctx.credential_ask_in_flight = True
result = await _ask(ctx, f"{_IDP_ORIGIN}/login")
assert result["status"] == "already_asked"
ctx.stream.send.assert_not_awaited()
assert ctx.credential_origin_recovery == _PENDING
assert ctx.credential_origin_refusals == set()
@pytest.mark.asyncio
async def test_an_authenticator_update_ask_neither_spends_nor_resolves_origin_recovery(
monkeypatch: pytest.MonkeyPatch,
) -> None:
cache = _FakeCache()
ctx = _recovery_ctx(monkeypatch, cache)
service = wire_credential_vault(
monkeypatch, PasswordCredential(username="u", password="p", totp=None), credential_id="cred_service"
)
ctx.request_policy.resolved_credentials = [service]
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [("skip", None)]))
await _request_credential(f"{_IDP_ORIGIN}/login", "Add 2FA.", ctx, credential_id="cred_service")
assert len(_sent_cards(ctx)) == 1
assert ctx.credential_origin_recovery == _PENDING
assert ctx.credential_origin_refusals == set()
@pytest.mark.asyncio
@pytest.mark.parametrize(("answer", "status"), [("skip", "skipped"), (None, "unanswered")])
async def test_a_declined_origin_recovery_ends_the_turn_naming_the_missing_login(
monkeypatch: pytest.MonkeyPatch, answer: str | None, status: str
) -> None:
ctx = _recovery_ctx(monkeypatch, _answered_cache(answer) if answer else _FakeCache())
ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=1)
result = await _ask(ctx, f"{_IDP_ORIGIN}/login")
assert len(_sent_cards(ctx)) == 1
_assert_declined(ctx, result, status)
@pytest.mark.asyncio
@pytest.mark.parametrize("state", ["pending", "declined"])
async def test_a_run_derived_pause_does_not_reopen_the_card_during_origin_recovery(
monkeypatch: pytest.MonkeyPatch, state: CredentialOriginRecoveryState
) -> None:
ctx = _recovery_ctx(monkeypatch, _answered_cache("connected", "cred_1"))
ctx.credential_origin_recovery = replace(_PENDING, state=state)
ctx.credential_pause_used = False
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy.login_page_urls = [f"{_IDP_ORIGIN}/login"]
_stub_credential_lookup(monkeypatch, _make_credential())
resume = await maybe_credential_pause(ctx, _fake_result(), ctx.stream, ctx.copilot_config)
assert resume is None
ctx.stream.send.assert_not_awaited()
assert "cred_1" not in ctx.request_policy.live_page_admitted_urls
@pytest.mark.asyncio
@pytest.mark.parametrize(
("answer", "expected_status"),
[(("connected", "cred_1"), "updated"), (("skip", None), "skipped")],
)
async def test_a_site_rejected_credential_gets_an_update_card_after_a_pick_even_with_an_authenticator(
monkeypatch: pytest.MonkeyPatch, answer: _CardAnswer, expected_status: str
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
wire_credential_vault(monkeypatch, PasswordCredential(username="u", password="p", totp="wrong-seed"))
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [("connected", "cred_1"), answer]))
picked = await _call_ask_tool(ctx)
policy_before = deepcopy(ctx.request_policy)
update = await _call_ask_tool(ctx, credential_id="cred_1", rejected_by_site=True)
assert (picked["status"], update["status"]) == ("connected", expected_status)
assert [(card.reason, card.credential_refs) for card in _sent_cards(ctx)][1] == (
"credential_rejected_by_site",
["cred_1"],
)
assert ctx.request_policy == policy_before
assert ctx.credential_pause_outcome == "connected"
assert (await _call_ask_tool(ctx, credential_id="cred_1", rejected_by_site=True))["status"] == "already_asked"
@pytest.mark.asyncio
async def test_a_saved_workflow_binding_the_chat_never_named_can_open_the_update_card(
monkeypatch: pytest.MonkeyPatch,
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
wire_credential_vault(monkeypatch, PasswordCredential(username="u", password="p", totp="wrong-seed"))
ctx.request_policy.resolved_credentials = []
ctx.request_policy.persisted_workflow_credential_ids = {"cred_1"}
ctx.stream.send = AsyncMock(side_effect=_answer_each_card(cache, [("connected", "cred_1")]))
update = await _call_ask_tool(ctx, credential_id="cred_1", rejected_by_site=True)
assert update["status"] == "updated"
assert [card.reason for card in _sent_cards(ctx)] == ["credential_rejected_by_site"]
_TERMINAL_SENTINEL = "Sentinel-Pw-7731"
@pytest.mark.asyncio
async def test_the_finalize_seam_opens_no_card_on_a_redacted_secret_draft(monkeypatch: pytest.MonkeyPatch) -> None:
"""The model owns the card on a raw-secret turn through request_credential; the finalizer never sequences it."""
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-1"
ctx.workflow_copilot_chat_id = "chat-1"
ctx.client_supports_credential_pause = True
ctx.request_policy = _redacted_secret_policy("https://portal.example.com/login")
ctx.last_run_skipped_unbound_credentials = True
monkeypatch.setattr(
credential_pause_module.app._inst, "CACHE", _answered_cache("connected", "cred_1"), raising=False
)
stream = _make_stream()
resume = await maybe_credential_pause(
ctx, _fake_result(), stream, CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
)
assert resume is None
stream.send.assert_not_awaited()
assert ctx.request_policy.allow_run_blocks is False
@pytest.mark.parametrize(
("user_url", "origin"),
[
(
f"https://portal.example.com:8443/{_TERMINAL_SENTINEL}?p={_TERMINAL_SENTINEL}",
"https://portal.example.com:8443",
),
("http://[::1]:8900/login", "http://[::1]:8900"),
("https://[2001:db8::1]/sign-in", "https://[2001:db8::1]"),
("www.portal.example.com/login", "https://www.portal.example.com"),
(f"https://ops:{_TERMINAL_SENTINEL}@portal.example.com/login", ""),
],
ids=["path_and_query", "ipv6_loopback_port", "ipv6", "schemeless", "userinfo"],
)
def test_the_raw_secret_card_sees_only_the_origin(user_url: str, origin: str) -> None:
assert credential_pause_module.raw_secret_card_origin(user_url) == origin
_SIGNED_IN_SCOPE_PHRASES = ("this turn's tests only", "start signed out")
@pytest.mark.asyncio
async def test_a_run_derived_card_answered_by_signing_in_resumes_with_facts_and_does_not_fire_again(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = make_copilot_context()
ctx.organization_id = "org-1"
ctx.turn_id = "turn-1"
ctx.workflow_copilot_chat_id = "chat-1"
ctx.client_supports_credential_pause = True
ctx.last_run_skipped_unbound_credentials = True
ctx.request_policy = RequestPolicy(
login_page_urls=["https://portal.example.com/login"], credential_draft_deferred_explicitly=True
)
signed_in = credential_pause_module.SignedInProfile(
browser_profile_id="bp_signed_in",
profile_name="Sign-in for portal.example.com",
site="portal.example.com",
cookie_count=2,
)
cache = _FakeCache()
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")] = encode_credential_response(
"signed_in", None, "tok-1", signed_in=signed_in
)
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), _make_stream(), config)
assert resume_msgs is not None
resume_text = json.dumps(resume_msgs)
assert credential_pause_module.signed_in_facts(signed_in) in resume_text
assert all(phrase in resume_text for phrase in _SIGNED_IN_SCOPE_PHRASES)
assert ctx.credential_pause_outcome == "signed_in"
assert _turn_credential_prompt_reason(ctx) is None
assert await maybe_credential_pause(ctx, _fake_result(), _make_stream(), config) is None
@pytest.mark.asyncio
async def test_a_card_answered_by_signing_in_tells_the_tool_what_happened(monkeypatch: pytest.MonkeyPatch) -> None:
signed_in = credential_pause_module.SignedInProfile(
browser_profile_id="bp_signed_in",
profile_name="Sign-in for portal.example.com",
site="portal.example.com",
cookie_count=2,
)
cache = _FakeCache()
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")] = encode_credential_response(
"signed_in", None, "tok-1", signed_in=signed_in
)
ctx = _tool_ctx(monkeypatch, cache)
result = await _call_ask_tool(ctx)
assert (result["status"], result["browser_profile_id"]) == ("signed_in", "bp_signed_in")
assert all(phrase in result["detail"] for phrase in _SIGNED_IN_SCOPE_PHRASES)
_REGISTRATION = {
"username": "[email protected]",
"credential_name": "Portal test account",
"password_length": 24,
"charset": "alphanumeric_symbols",
}
@pytest.mark.parametrize(
("length", "charset", "classes"),
[
(24, "alphanumeric_symbols", 4),
(128, "alphanumeric_symbols", 4),
(24, "alphanumeric", 3),
(128, "alphanumeric", 3),
],
)
def test_registration_password_has_the_length_and_every_class_of_its_charset(
length: int, charset: Literal["alphanumeric", "alphanumeric_symbols"], classes: int
) -> None:
for _ in range(50):
password = generate_registration_password(length, charset)
present = [
any(c.islower() for c in password),
any(c.isupper() for c in password),
any(c.isdigit() for c in password),
any(c in SAFE_SYMBOLS for c in password),
]
assert len(password) == length
assert sum(present) == classes
assert all(c.isalnum() or c in SAFE_SYMBOLS for c in password)
@pytest.mark.parametrize("length", [23, 129])
def test_registration_password_refuses_a_length_outside_its_bounds(length: int) -> None:
with pytest.raises(ValueError):
generate_registration_password(length, "alphanumeric")
class _RegistrationVault:
"""Stands in for the shared create path: records what it was asked to store and can fail or stall."""
def __init__(
self,
credential: Credential,
*,
reject: bool = False,
fail: bool = False,
delay: float = 0.0,
stored_password: str | None = None,
read_delay: float = 0.0,
read_fails: bool = False,
vault_status: int | None = None,
prepare_delay: float = 0.0,
):
self.credential = credential
self.prepare_delay = prepare_delay
self.vault_status = vault_status
self.reject = reject
self.fail = fail
self.delay = delay
self.stored_password = stored_password
self.read_delay = read_delay
self.read_fails = read_fails
self.prepared = 0
self.passwords: list[str] = []
self.stored = 0
self.hooks_run: list[str] = []
async def prepare(self, organization_id: str, data: CreateCredentialRequest) -> _RegistrationVault:
self.prepared += 1
await asyncio.sleep(self.prepare_delay)
if self.reject:
raise HTTPException(status_code=400, detail="No credential vault is configured")
return self
async def store(
self,
organization_id: str,
data: CreateCredentialRequest,
created_by: str | None,
service: _RegistrationVault,
tasks: BackgroundTasks,
) -> Credential:
self.passwords.append(data.credential.password)
await asyncio.sleep(self.delay)
if self.fail:
try:
raise RuntimeError(f"vault rejected password '{data.credential.password}'")
except RuntimeError:
logging.getLogger("skyvern.test.vault").exception("vault create failed")
raise HTTPException(status_code=502, detail=f"Custom credential service returned HTTP 500 for {data.name}")
self.stored += 1
tasks.add_task(self._run_hook, self.credential.credential_id)
return self.credential
# Starlette sends a sync task to a worker thread, so a sync append lands whenever that thread runs.
async def _run_hook(self, credential_id: str) -> None:
self.hooks_run.append(credential_id)
async def create_credential(
self, organization_id: str, data: CreateCredentialRequest, created_by: str | None
) -> Credential:
self.passwords.append(data.credential.password)
assert self.vault_status is not None
raise VaultHttpException(self.vault_status, "https://vault.example.test/credentials")
async def get_credential_item(self, db_credential: Credential) -> CredentialItem:
await asyncio.sleep(self.read_delay)
password = self.stored_password or self.passwords[-1]
if self.read_fails:
raise RuntimeError(f"vault read returned password '{password}' in an unexpected shape")
return CredentialItem(
item_id=db_credential.item_id,
name=db_credential.name,
credential_type=CredentialType.PASSWORD,
credential=PasswordCredential(username="[email protected]", password=password),
)
def install(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(copilot_routes, "prepare_credential_create", self.prepare)
if self.vault_status is None:
monkeypatch.setattr(copilot_routes, "store_new_credential", self.store)
def _registration_database(
monkeypatch: pytest.MonkeyPatch, credential: Credential, readback: list[Credential] | None = None
) -> SimpleNamespace:
database = SimpleNamespace(
credentials=SimpleNamespace(
get_credentials_by_ids=AsyncMock(return_value=[credential]),
get_credentials=AsyncMock(return_value=readback or []),
),
workflow_params=SimpleNamespace(record_pending_copilot_turn_credential_resume=AsyncMock()),
)
monkeypatch.setattr(credential_pause_module.app, "DATABASE", database)
return database
def _seed_registration_pause(
cache: _FakeCache, *, expires_at: datetime | None = None, registration: bool = True
) -> None:
cache.store[credential_pause_active_key("org-1", "chat-1", "turn-1")] = _encode_active_pause(
"tok-1",
expires_at or datetime.now(timezone.utc) + timedelta(minutes=5),
registration=WorkflowCopilotCredentialRegistration(**_REGISTRATION) if registration else None,
)
async def _generate(
*, resume_token: str = "tok-1", organization_id: str = "org-1"
) -> WorkflowCopilotCredentialGenerateResult:
with skyvern_context.scoped(SkyvernContext()):
return await workflow_copilot_credential_generate(
WorkflowCopilotCredentialGenerateRequest(
turn_id="turn-1", workflow_copilot_chat_id="chat-1", resume_token=resume_token
),
organization=SimpleNamespace(organization_id=organization_id), # type: ignore[arg-type]
current_user_id="user-1",
)
def _registration_record(cache: _FakeCache) -> WorkflowCopilotCredentialRegistration | None:
record = _decode_active_pause(cache.store[credential_pause_active_key("org-1", "chat-1", "turn-1")])
assert record is not None and record.status == "pending"
return record.registration
@pytest.mark.asyncio
async def test_registration_card_generates_one_credential_and_resumes_with_only_its_id(
monkeypatch: pytest.MonkeyPatch,
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
ctx.client_supports_credential_generation = True
credential = _make_credential("cred_new", "Portal test account")
database = _registration_database(monkeypatch, credential)
vault = _RegistrationVault(credential)
vault.install(monkeypatch)
responses: list[WorkflowCopilotCredentialGenerateResult] = []
async def answer_with_generate(
frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
) -> bool:
if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:
responses.append(await _generate(resume_token=frame.resume_token))
return True
ctx.stream.send.side_effect = answer_with_generate
with capture_logs() as logs:
result = await _call_ask_tool(ctx, registration=_REGISTRATION)
[card] = _sent_cards(ctx)
assert card.reason == "credential_registration"
assert card.login_page_urls == ["https://portal.example.com/login"]
assert card.registration is not None
assert (card.registration.username, card.registration.credential_name) == (
"[email protected]",
"Portal test account",
)
assert [response.result for response in responses] == ["connected"]
assert result["status"] == "connected"
assert result["credential_id"] == "cred_new"
assert (result["generated"], result["site_effect"]) == (True, "none")
assert "next" not in result and "data" not in result
assert ctx.request_policy.live_page_admitted_urls == {"cred_new": "https://portal.example.com/login"}
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
grant, error = await tools_module.credential_fill._credential_fill_origin_grant(ctx, "cred_new")
assert error is None and grant is not None
assert tools_module.credential_fill._within_grant("https://portal.example.com/signup", grant)
assert not tools_module.credential_fill._within_grant("https://elsewhere.example.com/signup", grant)
assert vault.stored == 1
await asyncio.sleep(0)
assert vault.hooks_run == ["cred_new"]
database.workflow_params.record_pending_copilot_turn_credential_resume.assert_awaited()
[password] = vault.passwords
surfaces = [
json.dumps(result),
*(call.args[0].model_dump_json() for call in ctx.stream.send.await_args_list),
*(str(value) for value in cache.store.values()),
*(response.model_dump_json() for response in responses),
json.dumps(logs, default=str),
]
assert all(password not in surface for surface in surfaces)
@pytest.mark.asyncio
@pytest.mark.parametrize("answer", ["generate", "pick"])
async def test_registration_card_during_origin_recovery_admits_its_generated_credential_without_site_evidence(
monkeypatch: pytest.MonkeyPatch, answer: str
) -> None:
cache = _FakeCache()
ctx = _recovery_ctx(monkeypatch, cache)
ctx.client_supports_credential_generation = True
generated = _make_credential("cred_new", "Portal test account")
picked = _make_credential("cred_blank")
database = _registration_database(monkeypatch, generated if answer == "generate" else picked)
database.credentials.get_credential = AsyncMock(return_value=None)
vault = _RegistrationVault(generated)
vault.install(monkeypatch)
async def answer_card(
frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
) -> bool:
if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:
if answer == "generate":
await _generate(resume_token=frame.resume_token)
else:
cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")] = encode_credential_response(
"connected", "cred_blank", frame.resume_token
)
return True
ctx.stream.send.side_effect = answer_card
result = await _call_ask_tool(ctx, login_page_url=f"{_IDP_ORIGIN}/login", registration=_REGISTRATION)
[card] = _sent_cards(ctx)
assert card.registration is not None
if answer == "generate":
assert (result["status"], result["credential_id"]) == ("connected", "cred_new")
assert ctx.credential_pause_outcome == "connected"
assert ctx.request_policy.live_page_admitted_urls["cred_new"] == f"{_IDP_ORIGIN}/login"
assert ctx.credential_origin_recovery is None
else:
assert vault.passwords == []
assert "cred_blank" not in ctx.request_policy.live_page_admitted_urls
assert ctx.credential_pause_outcome == "not_admitted"
_assert_declined(ctx, result, "connected_other_site")
@pytest.mark.asyncio
async def test_registration_ask_on_an_origin_recovery_handback_after_a_generate_raises_no_card(
monkeypatch: pytest.MonkeyPatch,
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
ctx.client_supports_credential_generation = True
generated = _make_credential("cred_new", "Portal test account")
_registration_database(monkeypatch, generated)
vault = _RegistrationVault(generated)
vault.install(monkeypatch)
async def answer_with_generate(
frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
) -> bool:
if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:
await _generate(resume_token=frame.resume_token)
return True
ctx.stream.send.side_effect = answer_with_generate
first = await _call_ask_tool(ctx, registration=_REGISTRATION)
assert first["status"] == "connected"
ctx.credential_origin_recovery = _PENDING
handback = await _call_ask_tool(ctx, login_page_url=f"{_IDP_ORIGIN}/login", registration=_REGISTRATION)
assert handback["status"] == "already_generated"
assert len(_sent_cards(ctx)) == 1
assert (vault.stored, len(vault.passwords)) == (1, 1)
assert ctx.credential_origin_recovery == _PENDING
@pytest.mark.asyncio
@pytest.mark.parametrize("reask", ["origin_recovery", "plain_card"])
@pytest.mark.parametrize("outcome", ["unknown", "rejected"])
async def test_registration_reask_after_an_unconfirmed_generate_never_creates_a_second_credential(
monkeypatch: pytest.MonkeyPatch, outcome: str, reask: str
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
ctx.client_supports_credential_generation = True
_registration_database(monkeypatch, _make_credential("cred_new", "Portal test account"))
vault = _RegistrationVault(
_make_credential("cred_new", "Portal test account"), reject=outcome == "rejected", fail=outcome == "unknown"
)
vault.install(monkeypatch)
generated: list[str] = []
async def generate_then_skip(
frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
) -> bool:
if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:
generated.append((await _generate(resume_token=frame.resume_token)).result)
await resolve_credential_pause(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id=frame.turn_id,
resume_token=frame.resume_token,
action="skip",
credential_id=None,
)
return True
ctx.stream.send.side_effect = generate_then_skip
first = await _call_ask_tool(ctx, registration=_REGISTRATION)
assert (first["status"], generated) == ("skipped", [outcome])
login_page_url = "https://portal.example.com/login"
if reask == "origin_recovery":
ctx.credential_origin_recovery = _PENDING
login_page_url = f"{_IDP_ORIGIN}/login"
second = await _call_ask_tool(ctx, login_page_url=login_page_url, registration=_REGISTRATION)
if outcome == "unknown":
assert second["status"] == "already_generated"
assert len(_sent_cards(ctx)) == 1
assert (vault.prepared, len(vault.passwords), generated) == (1, 1, ["unknown"])
else:
assert len(_sent_cards(ctx)) == 2
assert (vault.prepared, vault.passwords, generated) == (2, [], ["rejected", "rejected"])
@pytest.mark.asyncio
async def test_registration_card_skipped_creates_no_credential_and_cannot_generate_afterwards(
monkeypatch: pytest.MonkeyPatch,
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
ctx.client_supports_credential_generation = True
_registration_database(monkeypatch, _make_credential())
vault = _RegistrationVault(_make_credential())
vault.install(monkeypatch)
ctx.stream.send.side_effect = _answer_each_card(cache, [("skip", None)])
result = await _call_ask_tool(ctx, registration=_REGISTRATION)
assert result["status"] == "skipped"
with pytest.raises(HTTPException) as late:
await _generate()
assert late.value.status_code == status.HTTP_409_CONFLICT
assert (vault.prepared, vault.passwords) == (0, [])
@pytest.mark.asyncio
async def test_registration_card_answered_with_a_saved_login_does_not_report_it_generated(
monkeypatch: pytest.MonkeyPatch,
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
ctx.client_supports_credential_generation = True
_registration_database(monkeypatch, _make_credential("cred_saved", "Saved login"))
vault = _RegistrationVault(_make_credential())
vault.install(monkeypatch)
ctx.stream.send.side_effect = _answer_each_card(cache, [("connected", "cred_saved")])
result = await _call_ask_tool(ctx, registration=_REGISTRATION)
assert (result["status"], result["credential_id"]) == ("connected", "cred_saved")
assert "generated" not in result and "site_effect" not in result
assert (vault.prepared, vault.passwords) == (0, [])
@pytest.mark.asyncio
@pytest.mark.parametrize(
("registration", "expected"),
[
({**_REGISTRATION, "password_length": 23}, "unsupported_constraints"),
({**_REGISTRATION, "password_length": 129}, "unsupported_constraints"),
({**_REGISTRATION, "username": " "}, None),
],
)
async def test_registration_ask_outside_the_typed_bounds_raises_no_card(
monkeypatch: pytest.MonkeyPatch, registration: dict[str, Any], expected: str | None
) -> None:
ctx = _tool_ctx(monkeypatch)
ctx.client_supports_credential_generation = True
result = await _call_ask_tool(ctx, registration=registration)
assert result.get("status") == expected
assert "fallback" not in result and "next" not in result
assert _sent_cards(ctx) == []
assert ctx.credential_pause_used is False
@pytest.mark.asyncio
async def test_registration_ask_is_unavailable_when_the_client_cannot_generate(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _tool_ctx(monkeypatch)
ctx.client_supports_credential_generation = False
result = await _call_ask_tool(ctx, registration=_REGISTRATION)
assert result["status"] == "unavailable"
assert "fallback" not in result and "next" not in result
assert _sent_cards(ctx) == []
@pytest.mark.asyncio
@pytest.mark.parametrize("conflict", ["credential_id", "raw_secret"])
async def test_registration_ask_is_refused_beside_a_credential_id_or_on_a_secret_turn(
monkeypatch: pytest.MonkeyPatch, conflict: str
) -> None:
ctx = _tool_ctx(monkeypatch)
ctx.client_supports_credential_generation = True
extra: dict[str, Any] = {}
if conflict == "credential_id":
extra["credential_id"] = "cred_1"
else:
ctx.request_policy = _redacted_secret_policy("https://portal.example.com/login")
result = await _call_ask_tool(ctx, registration=_REGISTRATION, **extra)
assert result["ok"] is False
assert _sent_cards(ctx) == []
def _readback_credential(credential_id: str, **update: Any) -> Credential:
return _make_credential(credential_id, "Portal test account").model_copy(
update={
"username": "[email protected]",
"created_by": "user-1",
"created_at": datetime.now(timezone.utc) + timedelta(seconds=5),
**update,
}
)
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["rejected", "slow_prepare", "vault_error", "timeout"])
async def test_registration_generate_failure_with_no_readback_match_reopens_the_card_once(
monkeypatch: pytest.MonkeyPatch, failure: str
) -> None:
cache = _FakeCache()
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
_seed_registration_pause(cache)
late = _readback_credential("cred_late")
stale = _readback_credential("cred_stale", created_at=datetime.now(timezone.utc) - timedelta(hours=1))
other_user = _readback_credential("cred_other", created_by="user-2")
database = _registration_database(monkeypatch, late, readback=[other_user, stale])
vault = _RegistrationVault(
late,
reject=failure == "rejected",
fail=failure == "vault_error",
delay=0.2 if failure == "timeout" else 0,
prepare_delay=5 if failure == "slow_prepare" else 0,
)
vault.install(monkeypatch)
monkeypatch.setattr(credential_pause_module, "MANUAL_SIGN_IN_CLAIM_SECONDS", 1)
monkeypatch.setattr(copilot_routes, "_CLAIM_FINISH_MARGIN_SECONDS", 0.5)
monkeypatch.setattr(copilot_routes, "_READBACK_RESERVE_SECONDS", 0.5)
with capture_logs() as logs:
started = time.monotonic()
result = await _generate()
assert time.monotonic() - started < 2
expected = "rejected" if failure in ("rejected", "slow_prepare") else "unknown"
assert result.result == expected
assert result.credential_id is None
if expected == "rejected":
database.credentials.get_credentials.assert_not_awaited()
registration = _registration_record(cache)
assert registration is not None
assert (registration.attempted, registration.outcome) == (True, expected)
assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store
database.workflow_params.record_pending_copilot_turn_credential_resume.assert_not_awaited()
with pytest.raises(HTTPException) as again:
await _generate()
assert again.value.status_code == status.HTTP_409_CONFLICT
assert len(vault.passwords) == (0 if expected == "rejected" else 1)
await asyncio.sleep(0.3)
assert vault.stored == (1 if failure == "timeout" else 0)
assert vault.hooks_run == (["cred_late"] if failure == "timeout" else [])
late_creates = [log["credential_id"] for log in logs if log["event"] == "copilot_credential_generation_late_create"]
assert late_creates == (["cred_late"] if failure == "timeout" else [])
for password in vault.passwords:
assert password not in json.dumps(logs, default=str)
assert password not in result.model_dump_json()
assert password not in str(cache.store)
@pytest.mark.asyncio
@pytest.mark.parametrize(("create_seconds", "expected_id"), [(0.2, "cred_new"), (1.5, None)])
async def test_registration_slow_create_connects_only_when_it_lands_before_the_wait_cutoff(
monkeypatch: pytest.MonkeyPatch, create_seconds: float, expected_id: str | None
) -> None:
cache = _FakeCache()
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(credential_pause_module, "MANUAL_SIGN_IN_CLAIM_SECONDS", 2)
monkeypatch.setattr(copilot_routes, "_CLAIM_FINISH_MARGIN_SECONDS", 0.5)
monkeypatch.setattr(copilot_routes, "_READBACK_RESERVE_SECONDS", 0.5)
_seed_registration_pause(cache)
credential = _make_credential("cred_new", "Portal test account")
database = _registration_database(monkeypatch, credential)
vault = _RegistrationVault(credential, delay=create_seconds)
vault.install(monkeypatch)
result = await _generate()
with pytest.raises(HTTPException) as again:
await _generate()
assert again.value.status_code == status.HTTP_409_CONFLICT
assert len(vault.passwords) == 1
if expected_id is not None:
assert (result.result, result.credential_id) == ("connected", expected_id)
database.credentials.get_credentials.assert_not_awaited()
else:
assert (result.result, result.credential_id) == ("unknown", None)
database.credentials.get_credentials.assert_awaited_once()
registration = _registration_record(cache)
assert registration is not None and registration.outcome == "unknown"
await asyncio.sleep(create_seconds)
assert vault.stored == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(("vault_status", "expected"), [(422, "rejected"), (500, "unknown")])
async def test_registration_vault_refusal_is_rejected_only_for_a_4xx(
monkeypatch: pytest.MonkeyPatch, vault_status: int, expected: str
) -> None:
cache = _FakeCache()
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
_seed_registration_pause(cache)
credential = _readback_credential("cred_unrelated", created_by="user-2")
database = _registration_database(monkeypatch, credential, readback=[credential])
vault = _RegistrationVault(credential, vault_status=vault_status)
vault.install(monkeypatch)
with capture_logs() as logs:
result = await _generate()
with pytest.raises(HTTPException) as again:
await _generate()
assert (result.result, result.credential_id) == (expected, None)
assert again.value.status_code == status.HTTP_409_CONFLICT
assert len(vault.passwords) == 1
if expected == "rejected":
database.credentials.get_credentials.assert_not_awaited()
else:
database.credentials.get_credentials.assert_awaited()
registration = _registration_record(cache)
assert registration is not None and registration.outcome == expected
password = vault.passwords[0]
assert password not in json.dumps(logs, default=str)
assert password not in result.model_dump_json()
assert password not in str(cache.store)
@pytest.mark.asyncio
@pytest.mark.parametrize("readback", ["one", "two", "full_page", "other_password", "slow_read"])
async def test_registration_unknown_save_connects_only_a_single_readback_match(
monkeypatch: pytest.MonkeyPatch, readback: str
) -> None:
cache = _FakeCache()
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
monkeypatch.setattr(credential_pause_module, "MANUAL_SIGN_IN_CLAIM_SECONDS", 1)
monkeypatch.setattr(copilot_routes, "_CLAIM_FINISH_MARGIN_SECONDS", 0.5)
_seed_registration_pause(cache)
match = _readback_credential("cred_match")
rows = {
"two": [match, _readback_credential("cred_twin")],
"full_page": [match, *(_readback_credential(f"cred_{i}", username=f"u{i}@example.com") for i in range(99))],
}.get(readback, [match])
database = _registration_database(monkeypatch, match, readback=rows)
vault = _RegistrationVault(
match,
fail=True,
stored_password="Another-create's-password-1" if readback == "other_password" else None,
read_delay=30 if readback == "slow_read" else 0,
)
vault.install(monkeypatch)
started = time.monotonic()
result = await _generate()
assert time.monotonic() - started < 5
assert len(vault.passwords) == 1
with pytest.raises(HTTPException) as again:
await _generate()
assert again.value.status_code == status.HTTP_409_CONFLICT
assert len(vault.passwords) == 1
if readback == "one":
assert (result.result, result.credential_id) == ("connected", "cred_match")
response = json.loads(cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")])
assert (response["action"], response["credential_id"]) == ("connected", "cred_match")
database.workflow_params.record_pending_copilot_turn_credential_resume.assert_awaited()
else:
assert (result.result, result.credential_id) == ("unknown", None)
assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store
registration = _registration_record(cache)
assert registration is not None and registration.outcome == "unknown"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("case", "expected_status"),
[
("bad_token", status.HTTP_403_FORBIDDEN),
("expired", status.HTTP_410_GONE),
("consumed", status.HTTP_409_CONFLICT),
("other_org", status.HTTP_404_NOT_FOUND),
("not_a_registration_card", status.HTTP_409_CONFLICT),
],
)
async def test_registration_generate_refuses_before_any_vault_call(
monkeypatch: pytest.MonkeyPatch, case: str, expected_status: int
) -> None:
cache = _FakeCache()
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
_seed_registration_pause(
cache,
expires_at=datetime.now(timezone.utc) - timedelta(seconds=1) if case == "expired" else None,
registration=case != "not_a_registration_card",
)
if case == "consumed":
await resolve_credential_pause(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id="turn-1",
resume_token="tok-1",
action="skip",
credential_id=None,
)
_registration_database(monkeypatch, _make_credential())
vault = _RegistrationVault(_make_credential())
vault.install(monkeypatch)
with pytest.raises(HTTPException) as refused:
await _generate(
resume_token="tok-wrong" if case == "bad_token" else "tok-1",
organization_id="org-2" if case == "other_org" else "org-1",
)
assert refused.value.status_code == expected_status
assert (vault.prepared, vault.passwords) == (0, [])
@pytest.mark.asyncio
async def test_registration_created_after_the_pause_ended_is_reported_but_not_connected(
monkeypatch: pytest.MonkeyPatch,
) -> None:
cache = _FakeCache()
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
_seed_registration_pause(cache)
created = _make_credential("cred_late", "Portal test account")
_registration_database(monkeypatch, created)
vault = _RegistrationVault(created)
vault.install(monkeypatch)
store = vault.store
async def store_while_the_waiter_gives_up(
organization_id: str,
data: CreateCredentialRequest,
created_by: str | None,
service: str,
tasks: BackgroundTasks,
) -> Credential:
cache.store[credential_pause_active_key("org-1", "chat-1", "turn-1")] = _encode_active_pause(
"tok-1", datetime.now(timezone.utc), consumed=True
)
return await store(organization_id, data, created_by, service, tasks)
monkeypatch.setattr(copilot_routes, "store_new_credential", store_while_the_waiter_gives_up)
result = await _generate()
assert result.result == "created_not_connected"
assert result.credential_id == "cred_late"
assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store
@pytest.mark.asyncio
async def test_registration_created_but_unrecorded_on_the_pause_is_reported_with_its_id(
monkeypatch: pytest.MonkeyPatch,
) -> None:
cache = _FakeCache()
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
_seed_registration_pause(cache)
created = _make_credential("cred_saved", "Portal test account")
_registration_database(monkeypatch, created)
vault = _RegistrationVault(created)
vault.install(monkeypatch)
response_key = credential_response_cache_key("org-1", "chat-1", "turn-1")
cache_set = cache.set
async def set_failing_on_the_response(key: str, value: Any, ex: Any = None) -> None:
if key == response_key:
raise ConnectionError("cache unavailable")
await cache_set(key, value, ex)
monkeypatch.setattr(cache, "set", set_failing_on_the_response)
result = await _generate()
assert (result.result, result.credential_id) == ("created_not_connected", "cred_saved")
assert len(vault.passwords) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["rejected", "vault_error"])
async def test_registration_card_reopened_after_its_deadline_passed_mid_save_is_still_answerable(
monkeypatch: pytest.MonkeyPatch, failure: str
) -> None:
cache = _FakeCache()
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
_seed_registration_pause(cache, expires_at=datetime.now(timezone.utc) + timedelta(milliseconds=50))
_registration_database(monkeypatch, _make_credential())
vault = _RegistrationVault(
_make_credential(), reject=failure == "rejected", fail=failure == "vault_error", delay=0.1
)
vault.install(monkeypatch)
result = await _generate()
record = _decode_active_pause(cache.store[credential_pause_active_key("org-1", "chat-1", "turn-1")])
assert record is not None and record.status == "pending"
assert result.expires_at == record.expires_at
assert record.expires_at > datetime.now(timezone.utc) + timedelta(
seconds=settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS - 5
)
await resolve_credential_pause(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id="turn-1",
resume_token="tok-1",
action="skip",
credential_id=None,
)
assert credential_response_cache_key("org-1", "chat-1", "turn-1") in cache.store
@pytest.mark.asyncio
async def test_registration_ask_with_a_charset_outside_the_enum_returns_an_error_the_model_sees(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _tool_ctx(monkeypatch)
ctx.client_supports_credential_generation = True
result = await _call_ask_tool(ctx, registration={**_REGISTRATION, "charset": "letters_and_spaces"})
assert result["ok"] is False
assert "registration.charset" in result["error"]
assert _sent_cards(ctx) == []
assert ctx.credential_pause_used is False
@pytest.fixture(params=[True, False], ids=["json", "console"])
def rendered_logs(monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest) -> Iterator[io.StringIO]:
monkeypatch.setattr(settings, "JSON_LOGGING", request.param)
root = logging.getLogger()
saved_config = structlog.get_config()
saved_handlers, saved_level = root.handlers[:], root.level
stream = io.StringIO()
try:
setup_logger()
handler = root.handlers[0]
assert isinstance(handler, logging.StreamHandler)
handler.setStream(stream)
yield stream
finally:
structlog.configure(**saved_config)
root.handlers[:] = saved_handlers
root.setLevel(saved_level)
@pytest.mark.asyncio
async def test_registration_password_quoted_by_a_vault_error_is_scrubbed_from_rendered_logs_and_tracebacks(
monkeypatch: pytest.MonkeyPatch, rendered_logs: io.StringIO
) -> None:
cache = _FakeCache()
monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
_seed_registration_pause(cache)
match = _readback_credential("cred_match")
_registration_database(monkeypatch, match, readback=[match])
vault = _RegistrationVault(match, fail=True, read_fails=True)
vault.install(monkeypatch)
result = await _generate()
[password] = vault.passwords
rendered = rendered_logs.getvalue()
assert result.result == "unknown"
assert "vault create failed" in rendered
assert "copilot_credential_generation_readback_failed" in rendered
assert "in an unexpected shape" in rendered
assert "Traceback" in rendered
assert password not in rendered
@pytest.mark.asyncio
@pytest.mark.parametrize("answer", ["skip", "timeout"])
async def test_registration_card_reopened_after_an_unconfirmed_save_reports_it_when_skipped_or_unanswered(
monkeypatch: pytest.MonkeyPatch, answer: str
) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
ctx.client_supports_credential_generation = True
ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=1)
monkeypatch.setattr(settings, "WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS", 1)
_registration_database(monkeypatch, _make_credential())
vault = _RegistrationVault(_make_credential(), fail=True)
vault.install(monkeypatch)
async def generate_then_answer(
frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
) -> bool:
if frame.type != WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:
return True
assert (await _generate(resume_token=frame.resume_token)).result == "unknown"
if answer == "skip":
await resolve_credential_pause(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id="turn-1",
resume_token=frame.resume_token,
action="skip",
credential_id=None,
)
return True
ctx.stream.send.side_effect = generate_then_answer
result = await _call_ask_tool(ctx, registration=_REGISTRATION)
assert result["status"] == ("skipped" if answer == "skip" else "unanswered")
assert result["registration_outcome"] == "unknown"
assert "next" not in result
assert len(vault.passwords) == 1
assert vault.passwords[0] not in json.dumps(result)
@pytest.mark.asyncio
async def test_registration_card_that_ran_out_mid_create_reports_unknown(monkeypatch: pytest.MonkeyPatch) -> None:
cache = _FakeCache()
ctx = _tool_ctx(monkeypatch, cache)
ctx.client_supports_credential_generation = True
ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=1)
monkeypatch.setattr(settings, "WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS", 1)
monkeypatch.setattr(credential_pause_module, "MANUAL_SIGN_IN_CLAIM_SECONDS", 1)
async def claim_and_abandon(
frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
) -> bool:
if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:
await claim_credential_generation(
cache,
organization_id="org-1",
workflow_copilot_chat_id="chat-1",
turn_id="turn-1",
resume_token=frame.resume_token,
)
return True
ctx.stream.send.side_effect = claim_and_abandon
result = await _call_ask_tool(ctx, registration=_REGISTRATION)
assert result["status"] == "unanswered"
assert result["registration_outcome"] == "unknown"
@pytest.mark.asyncio
async def test_registration_outcome_from_an_earlier_ask_is_not_reported_on_the_next(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = _tool_ctx(monkeypatch, _FakeCache())
ctx.client_supports_credential_generation = True
ctx.credential_registration_outcome = "rejected"
ctx.client_supports_credential_pause_recovery = False
ctx.stream.is_disconnected.return_value = True
result = await _call_ask_tool(ctx, registration=_REGISTRATION)
assert result["status"] == "unanswered"
assert "registration_outcome" not in result