Skip to content

Automate browser based workflows with AI

72.1 KB
Raw Blame
"""Shared builders for copilot unit tests."""

from __future__ import annotations

import asyncio
import base64
import io
import json
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence
from contextlib import asynccontextmanager, suppress
from datetime import UTC, datetime, timedelta
from functools import cache
from itertools import count
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Protocol, TypeVar
from unittest.mock import AsyncMock, MagicMock
from urllib.parse import urlparse

import pytest
from agents import Agent, FunctionTool, OutputGuardrail, RunConfig, SQLiteSession
from agents.items import ModelResponse, TResponseInputItem
from agents.models.interface import Model, ModelProvider
from agents.result import RunResultStreaming
from openai.types.responses import (
    ResponseCompletedEvent,
    ResponseOutputItemDoneEvent,
    ResponseOutputMessage,
    ResponseOutputText,
)
from openai.types.responses.response import Response
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
from PIL import Image
from playwright.async_api import Page, Route, async_playwright
from playwright.sync_api import sync_playwright

from skyvern.forge import app as forge_app
from skyvern.forge.sdk.api.llm import api_handler_factory
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot import agent as copilot_agent
from skyvern.forge.sdk.copilot import runtime as copilot_runtime
from skyvern.forge.sdk.copilot.active_run_session import ActiveRunSessionAssociation
from skyvern.forge.sdk.copilot.agent import _run_agent_loop_with_surface, run_copilot_agent
from skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalMode
from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome
from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
    DiagnosisInput,
    DiagnosisRepairContract,
    DiagnosisResult,
    RepairDecision,
    RepairNextAction,
    VerificationResult,
)
from skyvern.forge.sdk.copilot.enforcement import (
    CopilotTotalTimeoutError,
    _mark_copilot_total_timeout,
    run_with_enforcement,
)
from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks
from skyvern.forge.sdk.copilot.repair_origin_run import RepairOriginBinding
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion
from skyvern.forge.sdk.copilot.review_gate import build_review_projection, workflow_block_fingerprints
from skyvern.forge.sdk.copilot.runtime import (
    AgentContext,
    OriginRunRedactionRegistry,
)
from skyvern.forge.sdk.copilot.screenshot_utils import (
    ScreenshotActionRelation,
    ScreenshotEntry,
    ScreenshotProvenance,
    resize_screenshot_b64,
)
from skyvern.forge.sdk.copilot.session_factory import (
    copilot_call_model_input_filter,
    copilot_session_input_callback,
)
from skyvern.forge.sdk.copilot.tools import reply_ends_turn, reply_tool
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
from skyvern.forge.sdk.copilot.tools import scouting as scouting_module
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
from skyvern.forge.sdk.copilot.workflow_yaml import _process_workflow_yaml as process_workflow_yaml
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType
from skyvern.forge.sdk.routes.workflow_copilot import _assistant_execution_receipts
from skyvern.forge.sdk.schemas.credentials import Credential, CredentialType, CredentialVaultType, PasswordCredential
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
from skyvern.forge.sdk.schemas.workflow_copilot import (
    WorkflowCopilotChatMessage,
    WorkflowCopilotChatRequest,
    WorkflowCopilotChatSender,
    WorkflowCopilotStreamMessageType,
    WorkflowCopilotTitleUpdate,
)
from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock
from skyvern.forge.sdk.workflow.models.block import CodeBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, WorkflowParameter, WorkflowParameterType
from skyvern.forge.sdk.workflow.models.workflow import (
    Workflow,
    WorkflowDefinition,
    WorkflowRun,
    WorkflowRunOutputParameter,
    WorkflowRunParameter,
    WorkflowRunStatus,
)
from skyvern.schemas.proxy_location import ProxyLocationInput
from skyvern.schemas.runs import ProxyLocation
from skyvern.schemas.self_heal import HealEpisode
from skyvern.schemas.workflows import BlockType
from skyvern.services import workflow_service as workflow_service_module
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import ActionStatus

DISPATCHED_LOGIN_GATE_HTML = (
    "<html><head><title>Sign in</title></head><body><main>"
    "<h1>Sign in to continue</h1>"
    '<form id="signin" action="/session" method="post">'
    '<label for="account-email">Email</label>'
    '<input id="account-email" name="email" type="email" required />'
    '<label for="account-password">Password</label>'
    '<input id="account-password" name="password" type="password" required />'
    '<button type="submit">Sign in</button>'
    "</form></main></body></html>"
)
DISPATCHED_RESULTS_HTML = (
    "<html><head><title>Available providers</title></head><body><main>"
    "<h1>Available providers</h1>"
    '<table id="provider-results"><tbody>'
    "<tr><td>Example Fiber</td><td>up to 500 Mbps</td></tr>"
    "<tr><td>Example Cable</td><td>up to 300 Mbps</td></tr>"
    "</tbody></table></main></body></html>"
)
DISPATCHED_NAV_ONLY_HTML = (
    "<html><head><title>Site map</title></head><body><main>"
    '<a href="https://example.test/plans">Plans</a>'
    '<a href="https://example.test/support">Support</a>'
    "</main></body></html>"
)


def wire_credential_vault(
    monkeypatch: pytest.MonkeyPatch,
    secrets: PasswordCredential,
    *,
    credential_id: str = "cred_1",
    name: str = "authtest simple",
) -> Credential:
    """Serve one saved credential from the org database and ``secrets`` from its vault."""
    now = datetime(2026, 1, 1, tzinfo=UTC)
    credential = Credential(
        credential_id=credential_id,
        organization_id="org-1",
        name=name,
        vault_type=CredentialVaultType.SKYVERN,
        item_id="item_1",
        credential_type=CredentialType.PASSWORD,
        username=secrets.username,
        card_last4=None,
        card_brand=None,
        created_at=now,
        modified_at=now,
    )
    monkeypatch.setattr(
        forge_app.DATABASE,
        "credentials",
        SimpleNamespace(
            get_credential=AsyncMock(return_value=credential),
            get_credentials_by_ids=AsyncMock(return_value=[credential]),
        ),
        raising=False,
    )
    vault = SimpleNamespace(get_credential_item=AsyncMock(return_value=SimpleNamespace(name=name, credential=secrets)))
    # `app` is an AppHolder proxy without __delattr__; patch the underlying instance so teardown can delete it.
    monkeypatch.setattr(
        object.__getattribute__(forge_app, "_inst"),
        "CREDENTIAL_VAULT_SERVICES",
        {CredentialVaultType.SKYVERN: vault},
        raising=False,
    )
    return credential


def make_stub_artifact(
    artifact_id: str,
    file_name: str,
    file_size: int | None,
    artifact_type: ArtifactType = ArtifactType.DOWNLOAD,
) -> SimpleNamespace:
    return SimpleNamespace(
        artifact_id=artifact_id,
        uri=f"s3://bucket/{file_name}",
        file_size=file_size,
        artifact_type=artifact_type,
    )


def make_stub_html_artifact(
    artifact_id: str,
    artifact_type: ArtifactType,
    file_size: int | None = 400,
    created_at: datetime | None = None,
) -> SimpleNamespace:
    artifact = make_stub_artifact(artifact_id, f"{artifact_id}.html", file_size, artifact_type=artifact_type)
    artifact.created_at = created_at or datetime(2026, 7, 9, tzinfo=UTC)
    return artifact


def stub_artifact_app(
    monkeypatch: pytest.MonkeyPatch,
    artifacts: list[SimpleNamespace],
    retrieved: dict[str, bytes],
    *,
    by_ids: list[SimpleNamespace] | None = None,
) -> list[str]:
    retrieved_ids: list[str] = []

    async def fake_get_artifacts_for_run(
        run_id: str, *, organization_id: str, artifact_types: object
    ) -> list[SimpleNamespace]:
        return artifacts

    async def fake_get_artifacts_by_ids(artifact_ids: list[str], *, organization_id: str) -> list[SimpleNamespace]:
        pool = {artifact.artifact_id: artifact for artifact in (by_ids if by_ids is not None else artifacts)}
        return [pool[artifact_id] for artifact_id in artifact_ids if artifact_id in pool]

    async def fake_retrieve_artifact(artifact: SimpleNamespace) -> bytes:
        retrieved_ids.append(artifact.artifact_id)
        return retrieved.get(artifact.artifact_id, b"")

    fake_app = SimpleNamespace(
        DATABASE=SimpleNamespace(
            artifacts=SimpleNamespace(
                get_artifacts_for_run=fake_get_artifacts_for_run,
                get_artifacts_by_ids=fake_get_artifacts_by_ids,
            )
        ),
        ARTIFACT_MANAGER=SimpleNamespace(retrieve_artifact=fake_retrieve_artifact),
    )
    monkeypatch.setattr(run_execution_module, "app", fake_app)
    return retrieved_ids


_RUN_SESSION_ID = "pbs_run"
_CHAT_SESSION_ID = "pbs_chat"
# The chat session answers with a different proxy than the run session, so a lookup that reads the
# wrong session shows up as the wrong label instead of passing.
_CHAT_SESSION_PROXY_LOCATION = ProxyLocation.RESIDENTIAL_ZA


def _fake_workflow_run(status: str, failure_category: list[dict[str, Any]] | None = None) -> WorkflowRun:
    return WorkflowRun(
        workflow_run_id="wr_paused",
        workflow_id="w_source",
        workflow_permanent_id="wfp-1",
        organization_id="org-1",
        status=WorkflowRunStatus(status),
        created_at=datetime(2026, 4, 21, 11, 0, 0),
        modified_at=datetime(2026, 4, 21, 12, 0, 0, tzinfo=UTC),
        trigger_type=None,
        browser_session_id=None,
        failure_reason=None,
        failure_category=failure_category,
    )


HARNESS_RUN_CREATED_AT = datetime(2026, 4, 21, 12, 0, 0)


def harness_run(
    workflow_run_id: str,
    *,
    created_at: datetime = HARNESS_RUN_CREATED_AT,
    status: str = "completed",
    trigger_type: WorkflowRunTriggerType | None = None,
    copilot_session_id: str | None = None,
    workflow_permanent_id: str = "wpid-1",
    organization_id: str = "org-1",
    workflow_definition_sha256: str | None = None,
) -> SimpleNamespace:
    """Naive-UTC created_at, as the database returns it."""
    return SimpleNamespace(
        workflow_run_id=workflow_run_id,
        organization_id=organization_id,
        workflow_permanent_id=workflow_permanent_id,
        workflow_id="wf-1",
        status=status,
        created_at=created_at,
        trigger_type=trigger_type,
        copilot_session_id=copilot_session_id,
        failure_reason=None,
        browser_session_id="pbs-1",
        workflow_definition_sha256=workflow_definition_sha256,
    )


def install_get_run_results_harness(
    monkeypatch: pytest.MonkeyPatch,
    *,
    blocks: list[MagicMock],
    run_status: str = "failed",
    dispatch_to_worker: bool = True,
    workflow_parameters: list[dict[str, str]] | None = None,
    attach_action_traces: Callable[..., Awaitable[None]] | None = None,
    recent_actions: list[MagicMock] | None = None,
    attach_failed_block_screenshots: Callable[..., Awaitable[None]] | None = None,
    other_runs: list[SimpleNamespace] | None = None,
    carried_successful_run_id: str | None = None,
    carried_run_id: str | None = None,
    heal_episodes: list[HealEpisode] | None = None,
    run_definition_sha256: str | None = None,
) -> SimpleNamespace:
    """Stub the collaborators ``_get_run_results`` reaches and return the ctx to call it with; the run pool is
    ``wr-1`` plus ``other_runs``, and run lookup and history listing honor their arguments."""
    pool = [
        harness_run("wr-1", status=run_status, workflow_definition_sha256=run_definition_sha256),
        *(other_runs or []),
    ]
    workflow = SimpleNamespace(
        workflow_id="wf-1",
        workflow_permanent_id="wpid-1",
        version=1,
        created_by=None,
        modified_at=HARNESS_RUN_CREATED_AT,
        workflow_definition=SimpleNamespace(
            parameters=[SimpleNamespace(**parameter) for parameter in workflow_parameters or []], blocks=[]
        ),
    )

    async def get_workflow_run(workflow_run_id: str, organization_id: str | None = None) -> SimpleNamespace | None:
        return next(
            (r for r in pool if r.workflow_run_id == workflow_run_id and r.organization_id == organization_id),
            None,
        )

    async def list_runs(
        *,
        workflow_permanent_id: str,
        organization_id: str,
        page: int = 1,
        page_size: int = 10,
        status: list[WorkflowRunStatus] | None = None,
        created_at_start: datetime | None = None,
    ) -> list[SimpleNamespace]:
        matching = [
            r
            for r in pool
            if r.workflow_permanent_id == workflow_permanent_id
            and r.organization_id == organization_id
            and r.copilot_session_id is None
            and (not status or r.status in status)
            and (created_at_start is None or r.created_at >= created_at_start)
        ]
        matching.sort(key=lambda r: r.created_at, reverse=True)
        return matching[(page - 1) * page_size : page * page_size]

    class _AppStub:
        class DATABASE:
            workflow_runs = SimpleNamespace(
                get_workflow_run=get_workflow_run,
                get_workflow_runs_for_workflow_permanent_id=list_runs,
            )

            class workflows:
                get_workflow = AsyncMock(return_value=None)
                get_workflow_for_workflow_run = AsyncMock(return_value=workflow)

            class observer:
                get_workflow_run_blocks = AsyncMock(return_value=blocks)

            class tasks:
                get_recent_actions_for_tasks = AsyncMock(return_value=list(recent_actions or []))

            class self_heal:
                get_heal_episodes_for_run = AsyncMock(return_value=list(heal_episodes or []))

        class AGENT_FUNCTION:
            should_dispatch_copilot_block_run_to_worker = AsyncMock(return_value=dispatch_to_worker)

        WORKFLOW_SERVICE = SimpleNamespace(get_workflow_runs_for_workflow_permanent_id=list_runs)

    monkeypatch.setattr(run_execution_module, "app", _AppStub())
    if attach_action_traces is not None:
        monkeypatch.setattr(run_execution_module, "_attach_action_traces", attach_action_traces)
    # Stubbed by default so most callers need no artifact store; pass the real function to assert
    # that at-failure evidence actually reaches the packet rather than that the call was made.
    monkeypatch.setattr(
        run_execution_module,
        "_attach_failed_block_screenshots",
        attach_failed_block_screenshots or AsyncMock(),
    )
    monkeypatch.setattr(run_execution_module, "_attach_registered_output_parameter_values", AsyncMock(return_value={}))
    monkeypatch.setattr(run_execution_module, "_fetch_dispatched_terminal_page_evidence", AsyncMock(return_value=None))
    return SimpleNamespace(
        organization_id="org-1",
        workflow_permanent_id="wpid-1",
        copilot_total_timeout_exceeded=False,
        last_successful_run_blocks_workflow_run_id=carried_successful_run_id,
        last_run_blocks_workflow_run_id=carried_run_id,
        proposal_workflow_run_id=None,
        dispatched_run_ids_this_turn=set(),
        seeded_only_labels_by_run_id={},
        workflow_yaml=None,
        staged_workflow_yaml=None,
    )


def historical_code_version(
    blocks: Mapping[str, str],
    *,
    parameter_keys: Sequence[str] = ("applicant_name",),
    default_value: str = "Fixture default",
    workflow_id: str = "wf-1",
    modified_at: datetime = HARNESS_RUN_CREATED_AT,
) -> SimpleNamespace:
    """A saved version of ``wpid-1`` holding one CodeBlock per ``blocks`` label, as a finished harness run read it."""
    return SimpleNamespace(
        workflow_id=workflow_id,
        workflow_permanent_id="wpid-1",
        version=3,
        created_by=None,
        modified_at=modified_at,
        workflow_definition=WorkflowDefinition(
            parameters=[
                WorkflowParameter(
                    workflow_parameter_id=f"wp_{key}",
                    workflow_parameter_type=WorkflowParameterType.STRING,
                    key=key,
                    default_value=default_value,
                    workflow_id=workflow_id,
                    created_at=HARNESS_RUN_CREATED_AT,
                    modified_at=HARNESS_RUN_CREATED_AT,
                )
                for key in parameter_keys
            ],
            blocks=[
                CodeBlock(
                    label=label,
                    code=code,
                    output_parameter=OutputParameter(
                        output_parameter_id=f"op_{label}",
                        key=f"{label}_output",
                        workflow_id=workflow_id,
                        created_at=HARNESS_RUN_CREATED_AT,
                        modified_at=HARNESS_RUN_CREATED_AT,
                    ),
                )
                for label, code in blocks.items()
            ],
        ),
    )


def install_historical_run(
    monkeypatch: pytest.MonkeyPatch,
    version: SimpleNamespace | None,
    *,
    failed_label: str,
    failing_line: int = 1,
    run_definition_sha256: str | None = None,
) -> None:
    """Harness run ``wr-1`` failed at ``failing_line`` of ``failed_label`` while executing ``version``."""

    async def stamp_failing_line(
        _rows: object, results: list[dict[str, Any]], _org: str, include_completed: bool = False
    ) -> None:
        results[0]["action_trace"] = [{"action": "NULL_ACTION", "status": "failed", "code_line": failing_line}]

    block = run_result_block_row(
        failed_label, "failed", failure_reason="NameError: a name is not defined", error_codes=["user_code_error"]
    )
    block.created_at = HARNESS_RUN_CREATED_AT
    block.parent_workflow_run_block_id = None
    block.current_index = None
    block.current_value = None
    install_get_run_results_harness(
        monkeypatch,
        blocks=[block],
        attach_action_traces=stamp_failing_line,
        run_definition_sha256=run_definition_sha256,
    )
    run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run.return_value = version


def run_result_action_row(
    task_id: str,
    action_type: ActionType,
    status: ActionStatus,
    *,
    code_line: int | None = None,
) -> MagicMock:
    action = MagicMock()
    action.task_id = task_id
    action.step_id = f"stp_{task_id}"
    action.action_type = action_type
    action.status = status
    action.reasoning = None
    action.element_id = None
    action.response = None
    action.output = {"code_line": code_line} if code_line is not None else None
    return action


def run_result_block_row(
    label: str,
    status: str,
    final_url: str | None = None,
    *,
    failure_reason: str | None = None,
    error_codes: list[str] | None = None,
    task_id: str | None = None,
) -> MagicMock:
    row = MagicMock()
    row.label = label
    row.block_type = SimpleNamespace(name="code")
    row.status = status
    row.failure_reason = failure_reason
    row.error_codes = error_codes or []
    row.output = None
    row.task_id = task_id
    row.final_url = final_url
    row.workflow_run_block_id = f"wrb_{label}"
    return row


async def install_run_blocks_harness(
    monkeypatch: pytest.MonkeyPatch,
    *,
    workflow_yaml: str,
    polled_status: str,
    dispatch_to_worker: bool = False,
    terminal_blocks: list[WorkflowRunBlock] | None = None,
    recent_actions: list[MagicMock] | None = None,
    run_proxy_location: ProxyLocationInput = None,
    run_session_proxy_location: ProxyLocationInput = None,
    polled_failure_category: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
    """Stub the collaborators an inline ``_run_blocks_and_collect_debug`` call reaches, with the
    polled run parked on ``polled_status`` so the watchdog decides the exit."""
    monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
    workflow = await process_workflow_yaml(
        settings_fallback_yaml="enable_self_healing: false",
        workflow_id="w_source",
        workflow_permanent_id="wfp-1",
        organization_id="org-1",
        workflow_yaml=workflow_yaml,
    )
    now = datetime.now(UTC)
    organization = Organization(
        organization_id="org-1",
        organization_name="Test Org",
        created_at=now,
        modified_at=now,
    )
    captured: dict[str, Any] = {"workflow": workflow, "executor_cancelled": False}

    database = MagicMock()
    database.workflows.get_workflow_by_permanent_id = AsyncMock(return_value=workflow)
    database.workflows.get_workflow = AsyncMock(return_value=workflow)
    database.organizations.get_organization = AsyncMock(return_value=organization)
    persisted_output_params = [p for p in workflow.workflow_definition.parameters if isinstance(p, OutputParameter)]
    persisted_workflow_params = [p for p in workflow.workflow_definition.parameters if isinstance(p, WorkflowParameter)]
    database.workflow_params.get_workflow_output_parameters = AsyncMock(return_value=persisted_output_params)
    database.observer.get_workflow_run_blocks = AsyncMock(return_value=terminal_blocks or [])
    database.tasks.get_recent_actions_for_tasks = AsyncMock(return_value=list(recent_actions or []))
    database.self_heal.get_heal_episodes_for_run = AsyncMock(return_value=[])
    database.workflow_runs.get_workflow_run = AsyncMock(return_value=_fake_workflow_run(status=polled_status))
    database.workflow_runs.get_workflow_run_output_parameters = AsyncMock(return_value=[])
    database.workflow_runs.get_workflow_runs_for_workflow_permanent_id = AsyncMock(return_value=[])
    monkeypatch.setattr(forge_app, "DATABASE", database)

    async def _execute_workflow(**_kwargs: Any) -> None:
        try:
            await asyncio.Event().wait()
        except asyncio.CancelledError:
            captured["executor_cancelled"] = True
            raise

    workflow_service = MagicMock()
    workflow_service.get_workflow_parameters = AsyncMock(return_value=persisted_workflow_params)
    workflow_service.execute_workflow = AsyncMock(side_effect=_execute_workflow)
    workflow_service.create_copilot_dispatch_draft_version = AsyncMock(return_value=workflow)
    monkeypatch.setattr(forge_app, "WORKFLOW_SERVICE", workflow_service)
    monkeypatch.setattr(
        forge_app.AGENT_FUNCTION,
        "should_dispatch_copilot_block_run_to_worker",
        AsyncMock(return_value=dispatch_to_worker),
    )
    monkeypatch.setattr(
        forge_app.AGENT_FUNCTION,
        "allow_copilot_inline_code_execution",
        MagicMock(return_value=False),
    )

    workflow_run = WorkflowRun(
        workflow_run_id="wr_paused",
        workflow_id="w_source",
        workflow_permanent_id="wfp-1",
        organization_id="org-1",
        status=WorkflowRunStatus.running,
        created_at=now,
        modified_at=now,
        sequential_credential_id=None,
        proxy_location=run_proxy_location,
        runnable_id=None,
    )
    monkeypatch.setattr(workflow_service_module, "prepare_workflow", AsyncMock(return_value=workflow_run))

    async def _get_session(session_id: str, _organization_id: str | None = None) -> PersistentBrowserSession:
        proxy_location = run_session_proxy_location if session_id == _RUN_SESSION_ID else _CHAT_SESSION_PROXY_LOCATION
        return PersistentBrowserSession(
            persistent_browser_session_id=session_id,
            organization_id="org-1",
            created_at=now,
            modified_at=now,
            proxy_location=proxy_location,
            browser_profile_id=None,
            browser_profile_loaded=True,
        )

    monkeypatch.setattr(forge_app.PERSISTENT_SESSIONS_MANAGER, "get_session", _get_session)

    polled_run = _fake_workflow_run(status=polled_status, failure_category=polled_failure_category)

    async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, Any]:
        return polled_run, now

    monkeypatch.setattr(run_execution_module, "_read_progress_sources", _read_progress)
    monkeypatch.setattr(run_execution_module, "RUN_BLOCKS_POLL_INTERVAL_SECONDS", 0)
    monkeypatch.setattr(run_execution_module, "_fallback_page_info", AsyncMock(return_value=("", "")))

    association = ActiveRunSessionAssociation(
        organization_id="org-1",
        workflow_permanent_id="wfp-1",
        debug_browser_session_id=_CHAT_SESSION_ID,
        run_browser_session_id=_RUN_SESSION_ID,
        workflow_run_id="wr_paused",
        turn_id="turn-1",
        generation="gen-1",
        expires_at=now + timedelta(minutes=5),
    )
    captured["publish"] = AsyncMock(return_value=association)
    captured["clear"] = AsyncMock(return_value=True)
    captured["cancel_run_task"] = AsyncMock(return_value=None)
    captured["cooperative_cancel"] = AsyncMock(return_value=None)
    monkeypatch.setattr(run_execution_module, "publish_active_run_session", captured["publish"])
    monkeypatch.setattr(run_execution_module, "clear_active_run_session", captured["clear"])
    monkeypatch.setattr(run_execution_module, "_cancel_run_task_if_not_final", captured["cancel_run_task"])
    monkeypatch.setattr(run_execution_module, "_cooperative_cancel_dispatched_run", captured["cooperative_cancel"])
    if dispatch_to_worker:
        captured["worker_execute"] = AsyncMock(return_value=None)
        monkeypatch.setattr(
            run_execution_module.AsyncExecutorFactory,
            "get_executor",
            MagicMock(return_value=SimpleNamespace(execute_workflow=captured["worker_execute"])),
        )
        monkeypatch.setattr(run_execution_module, "_delete_dispatch_draft_if_run_final", AsyncMock(return_value=None))
        monkeypatch.setattr(
            run_execution_module, "_capture_dispatched_terminal_page_evidence", AsyncMock(return_value=None)
        )
        monkeypatch.setattr(
            run_execution_module, "_attach_registered_output_parameter_values", AsyncMock(return_value={})
        )
    return captured


_REAL_CANCEL_RUN_TASK_IF_NOT_FINAL = run_execution_module._cancel_run_task_if_not_final
_REAL_COOPERATIVE_CANCEL_DISPATCHED_RUN = run_execution_module._cooperative_cancel_dispatched_run
_REAL_DELETE_DISPATCH_DRAFT_IF_RUN_FINAL = run_execution_module._delete_dispatch_draft_if_run_final

OWNED_RUN_ID = "wr_build_test"


class OwnedRunRow:
    """One run row shared by the build tool's poll, the real conditional cancel and draft-retire paths and its
    worker, installed over ``install_run_blocks_harness`` so a write lands on the row a later read reports."""

    def __init__(
        self, monkeypatch: pytest.MonkeyPatch, *, block_first_read: bool = False, block_first_cancel: bool = False
    ) -> None:
        self.row = _fake_workflow_run(status="running").model_copy(update={"workflow_run_id": OWNED_RUN_ID})
        self.cancel_writes: list[str] = []
        self.draft_deletes = AsyncMock()
        self.polled = asyncio.Event()
        self.cancel_started = asyncio.Event()
        self._block_first_read = block_first_read
        self._block_first_cancel = block_first_cancel
        monkeypatch.setattr(workflow_service_module, "prepare_workflow", AsyncMock(return_value=self.row.model_copy()))
        monkeypatch.setattr(run_execution_module, "_cancel_run_task_if_not_final", _REAL_CANCEL_RUN_TASK_IF_NOT_FINAL)
        monkeypatch.setattr(
            run_execution_module, "_cooperative_cancel_dispatched_run", _REAL_COOPERATIVE_CANCEL_DISPATCHED_RUN
        )
        monkeypatch.setattr(
            run_execution_module, "_delete_dispatch_draft_if_run_final", _REAL_DELETE_DISPATCH_DRAFT_IF_RUN_FINAL
        )
        monkeypatch.setattr(
            forge_app.WORKFLOW_SERVICE, "mark_workflow_run_as_canceled_if_not_final", self._cancel_if_not_final
        )
        monkeypatch.setattr(forge_app.DATABASE.workflow_runs, "get_workflow_run", self._get_workflow_run)
        monkeypatch.setattr(forge_app.DATABASE.workflows, "soft_delete_workflow_by_id", self.draft_deletes)
        monkeypatch.setattr(run_execution_module, "_read_progress_sources", self._read_progress_sources)

    def finish(self, status: str) -> None:
        if not self.row.status.is_final():
            self.row.status = WorkflowRunStatus(status)

    async def _cancel_if_not_final(self, workflow_run_id: str) -> None:
        self.cancel_started.set()
        if self._block_first_cancel:
            self._block_first_cancel = False
            await asyncio.Event().wait()
        if not self.row.status.is_final():
            self.cancel_writes.append(workflow_run_id)
            self.finish("canceled")

    async def _get_workflow_run(self, **_kwargs: object) -> WorkflowRun:
        return self.row

    async def _read_progress_sources(self, _ctx: CopilotContext, _run_id: str) -> tuple[WorkflowRun, datetime]:
        self.polled.set()
        if self._block_first_read:
            await asyncio.Event().wait()
        return self.row, datetime.now(UTC)


def make_build_test_ctx(
    harness: dict[str, Any], monkeypatch: pytest.MonkeyPatch, *, on_chat_browser: bool = False, **overrides: object
) -> CopilotContext:
    ctx = make_copilot_ctx(browser_session_id="pbs_chat", **overrides)
    ctx.staged_workflow = harness["workflow"]
    ctx.frontier_resume_session_id = "pbs_chat" if on_chat_browser else "pbs_run"
    if on_chat_browser:
        monkeypatch.setattr(run_execution_module, "acquire_build_test_browser_session", AsyncMock(return_value=None))
    return ctx


def install_secret_carrying_run_registry(monkeypatch: pytest.MonkeyPatch, *, all_registered: bool = False) -> None:
    async def _secret_carrying_registry(
        _ctx: CopilotContext, *, workflow_run_id: str, **_kwargs: object
    ) -> OriginRunRedactionRegistry:
        return OriginRunRedactionRegistry(
            workflow_run_id=workflow_run_id,
            parameters={},
            contains_sensitive_values=True,
            contains_all_sensitive_values=all_registered,
        )

    monkeypatch.setattr(run_execution_module, "_bind_origin_run_redaction_registry", _secret_carrying_registry)


def adopted_detached_tasks(before: set[asyncio.Task]) -> list[asyncio.Task]:
    return [task for task in run_execution_module._DETACHED_CLEANUP_TASKS if task not in before]


async def settle_cancelled_tools() -> None:
    for _ in range(10):
        await asyncio.sleep(0.01)


HANDBACK_WORKFLOW_YAML = """
title: extraction example
workflow_definition:
  parameters: []
  blocks:
    - block_type: extraction
      label: extract_heading
      url: https://example.com
      data_extraction_goal: Extract the page heading.
"""


def terminal_extraction_block(
    status: str,
    *,
    final_url: str | None = None,
    label: str = "extract_heading",
    failure_reason: str | None = None,
    task_id: str | None = None,
) -> WorkflowRunBlock:
    return WorkflowRunBlock(
        label=label,
        block_type=BlockType.EXTRACTION,
        status=status,
        final_url=final_url,
        task_id=task_id,
        failure_reason=(
            (failure_reason or 'Timeout exceeded: waiting for locator("#heading") to be visible')
            if status == "failed"
            else None
        ),
        workflow_run_block_id=f"wrb_{label}",
        workflow_run_id="wr_paused",
        organization_id="org-1",
        created_at=datetime(2026, 4, 21, 12, 5, tzinfo=UTC),
        modified_at=datetime(2026, 4, 21, 12, 5, tzinfo=UTC),
    )


SEARCH_THEN_SELECT_WORKFLOW_YAML = """
title: search then select
workflow_definition:
  parameters: []
  blocks:
    - block_type: extraction
      label: run_search
      url: https://fixture.test
      data_extraction_goal: Extract the search results.
    - block_type: extraction
      label: select_first_result
      data_extraction_goal: Extract the selected result.
"""


def page_only_failed_block() -> WorkflowRunBlock:
    """A failed block with no failure_reason, so the post-run page is its only structural signal."""
    return WorkflowRunBlock(
        label="extract_heading",
        block_type=BlockType.EXTRACTION,
        status="failed",
        failure_reason=None,
        workflow_run_block_id="wrb_extract_heading",
        workflow_run_id="wr_paused",
        organization_id="org-1",
        created_at=datetime(2026, 4, 21, 12, 5, tzinfo=UTC),
        modified_at=datetime(2026, 4, 21, 12, 5, tzinfo=UTC),
    )


def same_run_page_evidence() -> dict[str, object]:
    return {
        "observed_after_workflow_run": True,
        "workflow_run_id": "wr_paused",
        "source_browser_session_id": "pbs_run",
        "current_url": "https://example.com/done",
        "page_title": "Done",
        "inspected_url": "https://example.com/done",
    }


def count_record_and_send(monkeypatch: pytest.MonkeyPatch) -> dict[str, int]:
    counts = {"record": 0, "send": 0}
    real_record = run_execution_module.record_build_test_outcome
    real_send = run_execution_module._send_run_outcome_update

    def _record(ctx: object, outcome: object) -> None:
        counts["record"] += 1
        real_record(ctx, outcome)

    async def _send(*args: object, **kwargs: object) -> None:
        counts["send"] += 1
        await real_send(*args, **kwargs)

    monkeypatch.setattr(run_execution_module, "record_build_test_outcome", _record)
    monkeypatch.setattr(run_execution_module, "_send_run_outcome_update", _send)
    return counts


async def handback_ctx(
    monkeypatch: pytest.MonkeyPatch,
    *,
    polled_status: str,
    block_status: str,
    terminal_blocks: list[WorkflowRunBlock] | None = None,
) -> CopilotContext:
    harness = await install_run_blocks_harness(
        monkeypatch,
        workflow_yaml=HANDBACK_WORKFLOW_YAML,
        polled_status=polled_status,
        terminal_blocks=terminal_blocks or [terminal_extraction_block(block_status)],
    )
    ctx = make_copilot_ctx(browser_session_id="pbs_chat")
    ctx.staged_workflow = harness["workflow"]
    ctx.frontier_resume_session_id = "pbs_run"
    return ctx


class FakeTab:
    def __init__(self, url: str) -> None:
        self.url = url
        self.closed = False

    def is_closed(self) -> bool:
        return self.closed


class BrowserTab(Protocol):
    @property
    def url(self) -> str: ...

    @property
    def frames(self) -> Sequence[Any]: ...

    async def title(self) -> str: ...

    def is_closed(self) -> bool: ...


FakeBrowserTab = FakeTab | BrowserTab


class FakeTabbedBrowserState:
    """A browser whose tab list and selected tab the test moves between calls: navigate by assigning a
    tab's ``url``, close one with ``close``, select one by assigning ``active``. A URL becomes a ``FakeTab``;
    a page (a real one, or a stand-in) is used as it is."""

    def __init__(self, *tabs: str | BrowserTab, active: int = 0) -> None:
        self.tabs: list[FakeBrowserTab] = [FakeTab(tab) if isinstance(tab, str) else tab for tab in tabs]
        self.active: FakeBrowserTab | None = self.tabs[active] if self.tabs else None
        self.browser_context = SimpleNamespace(pages=self.tabs)

    def close(self, tab: FakeBrowserTab) -> None:
        assert isinstance(tab, FakeTab), "only a tab opened from a URL can be closed here"
        tab.closed = True
        if self.active is tab:
            self.active = next((open_tab for open_tab in self.tabs if not open_tab.is_closed()), None)

    async def get_working_page(self, *, prune_excess_pages: bool = True) -> FakeBrowserTab | None:
        return self.active

    async def get_or_create_page(self) -> FakeBrowserTab | None:
        return self.active

    async def list_valid_pages(self, max_pages: int = 0) -> list[FakeBrowserTab]:
        return [tab for tab in self.tabs if not tab.is_closed()]


def patch_browser_tabs(
    monkeypatch: pytest.MonkeyPatch, states: FakeTabbedBrowserState | dict[str, FakeTabbedBrowserState] | None
) -> list[str | None]:
    """Resolve browser sessions to the given tabbed state (one for every session, or one per session id);
    None resolves to no browser at all, which the taint guard treats as unreadable. Returns the session ids
    resolved, in order."""
    resolved: list[str | None] = []

    async def resolve(_ctx: AgentContext, *, session_id: str | None = None) -> FakeTabbedBrowserState | None:
        resolved.append(session_id)
        if isinstance(states, dict):
            return states.get(session_id or "")
        return states

    monkeypatch.setattr(copilot_runtime, "resolve_browser_state_for_context", resolve)
    monkeypatch.setattr(scouting_module, "resolve_browser_state_for_context", resolve)
    return resolved


T = TypeVar("T")

CHALLENGE_URL = (
    "https://challenges.cloudflare.com/cdn-cgi/challenge-platform/turnstile/if/ov2/av0/fake?sitekey=0xTESTKEY"
)
AUTO_RENDER_TURNSTILE_HTML = f"""<!DOCTYPE html>
<html><body>
  <form><input id="email" type="email" /><button type="submit">Submit</button></form>
  <div class="cf-turnstile" data-sitekey="0xTESTKEY">
    <iframe src="{CHALLENGE_URL}" style="width:300px;height:65px"></iframe>
  </div>
</body></html>
"""
CATALOG_PAGE_HTML = "<!DOCTYPE html><html><head><title>Catalog</title></head><body><p>ZX-4471</p></body></html>"
HIDDEN_TURNSTILE_HELPER_HTML = f"""<!DOCTYPE html>
<html><head><title>Catalog</title></head><body>
  <p>ZX-4471</p>
  <iframe src="{CHALLENGE_URL}" style="width:300px;height:65px;visibility:hidden"></iframe>
</body></html>
"""
_RECAPTCHA_URL = "https://www.google.com/recaptcha/api2"
INVISIBLE_RECAPTCHA_BADGE_HTML = f"""<!DOCTYPE html>
<html><head><title>Catalog</title></head><body>
  <p>ZX-4471</p>
  <div class="grecaptcha-badge" style="width:256px;height:60px;position:fixed;right:0;bottom:14px">
    <iframe title="reCAPTCHA" src="{_RECAPTCHA_URL}/anchor?ar=1&k=6LeTEST&size=invisible" width="256" height="60"></iframe>
  </div>
</body></html>
"""
OPENED_INVISIBLE_RECAPTCHA_HTML = INVISIBLE_RECAPTCHA_BADGE_HTML.replace(
    "</body>",
    f'<iframe title="recaptcha challenge" src="{_RECAPTCHA_URL}/bframe?k=6LeTEST" width="400" height="580"></iframe></body>',
)
FIXTURE_SITE_ORIGIN = "https://captcha-fixture.test"


def page_with_unprobeable_vendor_embedder() -> SimpleNamespace:
    """A rendered-size vendor frame inside an embedding frame whose own element cannot be read."""
    top = SimpleNamespace(parent_frame=None, url="https://orders.example.test/")
    embedder = SimpleNamespace(
        parent_frame=top,
        url="https://embedder.example.test/",
        frame_element=AsyncMock(side_effect=RuntimeError("frame detached")),
    )
    vendor = SimpleNamespace(
        parent_frame=embedder,
        url=CHALLENGE_URL,
        frame_element=AsyncMock(return_value=SimpleNamespace(evaluate=AsyncMock(return_value=19500.0))),
    )
    return SimpleNamespace(
        url="https://orders.example.test/",
        title=AsyncMock(return_value="Orders"),
        frames=[top, embedder, vendor],
        is_closed=lambda: False,
    )


@cache
def _has_playwright_browser() -> bool:
    try:
        with sync_playwright() as playwright:
            return Path(playwright.chromium.executable_path).exists()
    except Exception:
        return False


def skip_no_browser(test: T) -> T:
    return pytest.mark.skipif(
        not _has_playwright_browser(),
        reason="Requires Playwright browsers installed (run: playwright install chromium)",
    )(test)


async def _fulfill_challenge(route: Route) -> None:
    await route.fulfill(status=200, content_type="text/html", body="<html><body>Verify you are human</body></html>")


def _serve_fixture_site(pages: Mapping[str, str]) -> Callable[[Route], Awaitable[None]]:
    async def serve(route: Route) -> None:
        path = urlparse(route.request.url).path
        body = pages.get(path)
        if body is None:
            await route.fulfill(status=404, body="")
            return
        content_type = "application/javascript" if path.endswith(".js") else "text/html"
        await route.fulfill(status=200, content_type=content_type, body=body)

    return serve


@asynccontextmanager
async def challenge_browser_page(
    html: str | None = None,
    *,
    site: Mapping[str, str] | None = None,
    path: str = "/",
    expect_challenge_frame: bool = True,
) -> AsyncIterator[Page]:
    """Load ``html`` directly, or ``path`` on a routed fixture origin that serves ``site`` by path, so a fixture
    spanning several documents (form, widget frames, results page) lives in the test rather than on disk."""
    async with async_playwright() as playwright:
        # Production runs headful chromium, where a cross-origin widget frame is an out-of-process iframe;
        # without site isolation a headless probe would never exercise that path.
        browser = await playwright.chromium.launch(headless=True, args=["--site-per-process"])
        try:
            context = await browser.new_context()
            await context.route("https://challenges.cloudflare.com/**", _fulfill_challenge)
            await context.route(f"{_RECAPTCHA_URL}/**", _fulfill_challenge)
            if site is not None:
                await context.route(f"{FIXTURE_SITE_ORIGIN}/**", _serve_fixture_site(site))
            page = await context.new_page()
            if site is not None:
                await page.goto(f"{FIXTURE_SITE_ORIGIN}{path}", wait_until="load")
            else:
                assert html is not None
                await page.set_content(html, wait_until="load")
            if expect_challenge_frame:
                assert any(urlparse(frame.url).hostname == "challenges.cloudflare.com" for frame in page.frames), (
                    "challenge frame did not commit"
                )
            yield page
        finally:
            await browser.close()


def redact_parameter_values(monkeypatch: pytest.MonkeyPatch, ctx: AgentContext, parameters: dict[str, str]) -> None:
    """Give ``ctx`` code-block parameters and a redactor that replaces each of their values with ``[redacted]``."""
    ctx.codeblock_redaction_parameters = parameters

    def redact(value: Any, active: dict[str, Any], **_budget: object) -> Any:
        if isinstance(value, str):
            for secret in active.values():
                value = value.replace(secret, "[redacted]")
            return value
        if isinstance(value, dict):
            return {key: redact(item, active) for key, item in value.items()}
        if isinstance(value, list):
            return [redact(item, active) for item in value]
        return value

    monkeypatch.setattr(forge_app.AGENT_FUNCTION, "redact_codeblock_parameter_values", redact)


def origin_run_input(
    key: str,
    value: bool | float | str | dict | list,
    ptype: WorkflowParameterType = WorkflowParameterType.FILE_URL,
    *,
    default_value: str | None = None,
) -> tuple[WorkflowParameter, WorkflowRunParameter]:
    now = datetime.now(UTC)
    parameter = WorkflowParameter(
        workflow_parameter_id=f"wp_origin_{key}",
        workflow_parameter_type=ptype,
        key=key,
        description=None,
        workflow_id="wf_origin",
        default_value=default_value,
        created_at=now,
        modified_at=now,
    )
    return parameter, WorkflowRunParameter(
        workflow_run_id="wr_origin",
        workflow_parameter_id=parameter.workflow_parameter_id,
        value=value,
        created_at=now,
    )


INERT_APPROVAL_WORKFLOW_YAML = """
title: inert approval
workflow_definition:
  parameters:
    - parameter_type: workflow
      workflow_parameter_type: string
      key: request_id
  blocks:
    - block_type: extraction
      label: approval
      url: https://example.test/approval
      data_extraction_goal: "Extract whether request {{ request_id }} is authorized."
      parameter_keys:
        - request_id
    - block_type: extraction
      label: source_status
      data_extraction_goal: "Report the source status for authorization {{ approval.output.authorized }}."
"""

REPAIRED_APPROVAL_WORKFLOW_YAML = INERT_APPROVAL_WORKFLOW_YAML.replace(
    "Report the source status for authorization {{ approval.output.authorized }}.",
    "Report the repaired source status for authorization {{ approval.output.authorized }} "
    "and {{ approval_output.extracted_information.authorized }}.",
)

ORIGIN_RUN_ID = "wr_origin"


async def inert_approval_workflow(workflow_yaml: str, *, workflow_id: str) -> Workflow:
    """A converted version mints its own parameter ids, as each persisted version does."""
    return await process_workflow_yaml(
        settings_fallback_yaml="enable_self_healing: false",
        workflow_id=workflow_id,
        workflow_permanent_id="wfp-1",
        organization_id="org-1",
        workflow_yaml=workflow_yaml,
    )


ORIGIN_OUTPUT_SENTINEL = "origin-output-sentinel-7f3a"


def origin_block_rows(
    workflow: Workflow,
    label: str,
    *,
    status: str = "completed",
    registered: bool = True,
    value: dict | list | str | None = None,
    row_output: dict | list | str | None = None,
    minute: int = 0,
    parent: str | None = None,
) -> tuple[list[WorkflowRunBlock], list[WorkflowRunOutputParameter]]:
    now = datetime(2026, 9, 1, 12, minute, tzinfo=UTC)
    run_block = WorkflowRunBlock(
        workflow_run_block_id=f"wrb_origin_{label}_{minute}",
        workflow_run_id=ORIGIN_RUN_ID,
        organization_id="org-1",
        parent_workflow_run_block_id=parent,
        block_type="extraction",
        label=label,
        status=status,
        output=row_output,
        created_at=now,
        modified_at=now,
    )
    output_parameter = workflow.get_output_parameter(label)
    if not registered or output_parameter is None:
        return [run_block], []
    return [run_block], [
        WorkflowRunOutputParameter(
            workflow_run_id=ORIGIN_RUN_ID,
            output_parameter_id=output_parameter.output_parameter_id,
            value=value,
            created_at=now,
        )
    ]


def merge_origin_rows(
    *rows: tuple[list[WorkflowRunBlock], list[WorkflowRunOutputParameter]],
) -> tuple[list[WorkflowRunBlock], list[WorkflowRunOutputParameter]]:
    return [block for blocks, _ in rows for block in blocks], [row for _, registered in rows for row in registered]


ORIGIN_RUN_CREATED_AT = datetime.now(UTC) + timedelta(days=1)


def origin_run_row(**overrides: object) -> WorkflowRun:
    fields: dict[str, object] = {
        "workflow_run_id": ORIGIN_RUN_ID,
        "workflow_id": "w_origin",
        "organization_id": "org-1",
        "workflow_permanent_id": "wfp-1",
        "browser_session_id": "pbs_origin",
        "status": WorkflowRunStatus.completed,
        # Later than any workflow version a test builds, as a real run starts after its version was saved.
        "created_at": ORIGIN_RUN_CREATED_AT,
        "modified_at": ORIGIN_RUN_CREATED_AT,
    }
    return WorkflowRun.model_validate({**fields, **overrides})


def install_origin_run(
    monkeypatch: pytest.MonkeyPatch,
    *,
    origin_workflow: Workflow,
    rows: tuple[list[WorkflowRunBlock], list[WorkflowRunOutputParameter]],
    **run_overrides: object,
) -> None:
    run = origin_run_row(**{"workflow_id": origin_workflow.workflow_id, **run_overrides})
    monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_run", AsyncMock(return_value=run))
    monkeypatch.setattr(forge_app.DATABASE.workflow_runs, "get_workflow_run_parameters", AsyncMock(return_value=[]))
    monkeypatch.setattr(forge_app.DATABASE.workflows, "get_workflow", AsyncMock(return_value=origin_workflow))
    monkeypatch.setattr(forge_app.DATABASE.observer, "get_workflow_run_blocks", AsyncMock(return_value=rows[0]))
    monkeypatch.setattr(
        forge_app.DATABASE.workflow_runs, "get_workflow_run_output_parameters", AsyncMock(return_value=rows[1])
    )


def make_copilot_ctx(**overrides: object) -> CopilotContext:
    defaults: dict[str, object] = dict(
        organization_id="org-1",
        workflow_id="wf-1",
        workflow_permanent_id="wfp-1",
        workflow_yaml="",
        browser_session_id=None,
        stream=MagicMock(),
    )
    defaults.update(overrides)
    return CopilotContext(**defaults)


def make_verified_goal_contract(
    *, next_action: RepairNextAction = RepairNextAction.NO_CHANGE
) -> DiagnosisRepairContract:
    return DiagnosisRepairContract(
        diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),
        diagnosis_result=DiagnosisResult(),
        repair_decision=RepairDecision(next_action=next_action),
        verification_result=VerificationResult(
            user_goal_satisfied=True,
            completion_contract_satisfied=True,
        ),
    )


def make_completion_criterion(
    cid: str,
    outcome: str,
    *,
    level: str = "run",
    method_mandated: bool = False,
    output_path: str | None = None,
    contingent_on: str | None = None,
    contingent_antecedent_output_path: str | None = None,
    antecedent_family: str | None = None,
    kind: str = "outcome",
    terminal_action_family: str | None = None,
    deliverable_kind: str | None = None,
    deliverable_confirmation_criterion_id: str | None = None,
    expected_output_value: str | None = None,
    expected_output_shape: str | None = None,
    requested_output_evidence_source: str = "runtime_output",
    requested_output_path_mint_source: str | None = None,
    classification_output_key: str | None = None,
    expected_classification: str | bool | None = None,
    requested_output_corroborator: bool = False,
    mint_degrade: str | None = None,
    requested_output_floor_rekeyed: bool = False,
    floor_rekeyed_from_path: str | None = None,
) -> CompletionCriterion:
    return CompletionCriterion(
        id=cid,
        outcome=outcome,
        level=level,  # type: ignore[arg-type]
        method_mandated=method_mandated,
        output_path=output_path,
        contingent_on=contingent_on,
        contingent_antecedent_output_path=contingent_antecedent_output_path,
        antecedent_family=antecedent_family,  # type: ignore[arg-type]
        kind=kind,  # type: ignore[arg-type]
        terminal_action_family=terminal_action_family,  # type: ignore[arg-type]
        deliverable_kind=deliverable_kind,  # type: ignore[arg-type]
        deliverable_confirmation_criterion_id=deliverable_confirmation_criterion_id,
        expected_output_value=expected_output_value,
        expected_output_shape=expected_output_shape,  # type: ignore[arg-type]
        requested_output_evidence_source=requested_output_evidence_source,  # type: ignore[arg-type]
        requested_output_path_mint_source=requested_output_path_mint_source,  # type: ignore[arg-type]
        classification_output_key=classification_output_key,
        expected_classification=expected_classification,
        requested_output_corroborator=requested_output_corroborator,
        mint_degrade=mint_degrade,  # type: ignore[arg-type]
        requested_output_floor_rekeyed=requested_output_floor_rekeyed,
        floor_rekeyed_from_path=floor_rekeyed_from_path,
    )


def two_page_login_yaml(*, submit_selector: str = "Login") -> str:
    """The shape copilot emits in code-block mode: branch bodies are code inside one always-executed
    block, so a passing run can traverse it without reaching the guarded call."""
    return f"""
    title: Sign in and read the metric
    workflow_definition:
      blocks:
      - block_type: code
        label: sign_in_and_read
        code: |
          await page.fill("#user", "demo")
          await page.click("#submit")
          if await page.locator("#token").count():
              await page.get_by_role("button", name="{submit_selector}", exact=True).click()
          return {{"visitors": "9.42K"}}
    """


def failed_second_factor_run(run_id: str) -> RecordedBuildTestOutcome:
    return RecordedBuildTestOutcome(
        phase="persisted_block_run",
        attempted_tool="update_and_run_blocks",
        attempted_block_label="sign_in_and_read",
        verdict="repairable_failure",
        reason_code="runtime_block_failure",
        workflow_run_id=run_id,
        block_labels=["sign_in_and_read"],
        structural_failure_identity="locator-timeout-identity",
    )


def passing_run(run_id: str, block_labels: list[str]) -> RecordedBuildTestOutcome:
    return RecordedBuildTestOutcome(
        phase="persisted_block_run",
        attempted_tool="update_and_run_blocks",
        verdict="progress_observed",
        reason_code="run_completed_unevaluated",
        workflow_run_id=run_id,
        block_labels=block_labels,
        structural_failure_identity="",
        evidence_refs=["rows:1"],
    )


InteractionFieldValue = str | int | bool | None | list[Any] | dict[str, Any]


def carried_interaction(**fields: InteractionFieldValue) -> dict[str, Any]:
    """One entry of the cross-turn carried trajectory.

    The record is plain interaction dicts, so this only spares tests the brace noise.
    """
    return dict(fields)


def captured_frame_entry(width: int = 1920, height: int = 947) -> ScreenshotEntry:
    """A frame staged from a real PNG; each call is a new capture event."""
    buf = io.BytesIO()
    Image.new("RGB", (width, height), (40, 90, 160)).save(buf, format="PNG")
    return resize_screenshot_b64(
        base64.b64encode(buf.getvalue()).decode("ascii"),
        provenance=ScreenshotProvenance(
            source_tool="get_browser_screenshot",
            captured_url="https://example.com/page",
            observation_step=None,
            browser_session_id="pbs_123",
            workflow_run_id=None,
            action_relation=ScreenshotActionRelation.TOOL_RESULT,
            viewport_css_px=(width, height),
        ),
    )


def sent_image_px(message: Any) -> list[tuple[int, int]]:
    content = message.get("content") if isinstance(message, dict) else None
    return [
        Image.open(io.BytesIO(base64.b64decode(part["image_url"].partition(";base64,")[2]))).size
        for part in (content if isinstance(content, list) else [])
        if part.get("type") == "input_image"
    ]


def make_model_input_data(items: list[Any], *, instructions: str | None = None, context: Any = None) -> Any:
    """Build a fake CallModelData payload with a model_data.input list.

    ``CallModelData.context`` is the run context itself (``TContext | None``), not a wrapper around
    one; a fake that nests it hides an attribute error behind a passing test.
    """
    return SimpleNamespace(
        model_data=SimpleNamespace(input=list(items), instructions=instructions),
        context=context,
    )


class FakeMCPServerManager:
    def __init__(self, servers: object) -> None:
        self.active_servers = servers

    async def __aenter__(self) -> FakeMCPServerManager:
        return self

    async def __aexit__(self, *args: object) -> None:
        return None


def stub_copilot_agent_loop(
    monkeypatch: pytest.MonkeyPatch, run_with_enforcement: Callable[..., Awaitable[object]]
) -> None:
    def fake_resolve_model_config(
        _handler: object, *, copilot_config: object = None, llm_key_override: str | None = None
    ) -> tuple[str, object, str, bool]:
        return f"model-{llm_key_override or 'PRIMARY'}", object(), llm_key_override or "PRIMARY", True

    monkeypatch.setattr(
        "skyvern.forge.sdk.copilot.agent._resolve_live_browser_session_id", AsyncMock(return_value=None)
    )
    monkeypatch.setattr("agents.mcp.MCPServerManager", FakeMCPServerManager)
    monkeypatch.setattr("skyvern.forge.sdk.copilot.model_resolver.resolve_model_config", fake_resolve_model_config)
    monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.run_with_enforcement", run_with_enforcement)
    monkeypatch.setattr(copilot_agent, "schedule_agent_naming", lambda *_args: None)


TURN_EXIT_PATHS = ("normal", "model_error", "deadline", "cancel")


async def run_turn_to_exit(
    monkeypatch: pytest.MonkeyPatch,
    *,
    manager: MagicMock,
    exit_path: str,
    session_id: str | None = "pbs_turn",
    browser_state: MagicMock | None = None,
    resolve_session_id: str | None = None,
    eval_mode: CopilotEvalMode | None = None,
    turn_origin: TurnOrigin = TurnOrigin.interactive,
    attach_through: str = "resolve",
    on_attached: Callable[[CopilotContext], Awaitable[None]] | None = None,
    get_browser_state: AsyncMock | None = None,
) -> tuple[AgentResult | None, BaseException | None]:
    """Drive one real ``run_copilot_agent`` turn to ``exit_path`` against ``manager``, attaching
    through the real resolve funnel so the finalizer releases only what that funnel recorded."""
    manager.get_browser_state = get_browser_state or AsyncMock(return_value=browser_state)
    if turn_origin == TurnOrigin.code_block_ai_fallback and browser_state is not None:
        browser_state.get_working_page = AsyncMock(return_value=MagicMock())

    async def _exit(ctx: CopilotContext) -> None:
        if on_attached is not None:
            await on_attached(ctx)
        if exit_path == "model_error":
            raise RuntimeError("model stream failed")
        if exit_path == "deadline":
            _mark_copilot_total_timeout(ctx, elapsed_seconds=901.0, iteration=1)
            raise CopilotTotalTimeoutError()
        if exit_path == "cancel":
            raise asyncio.CancelledError()

    async def fake_turn(**kwargs: Any) -> SimpleNamespace:
        ctx = kwargs["ctx"]
        ctx.browser_session_id = session_id
        ctx.turn_origin = turn_origin
        ctx.injected_browser_state = browser_state
        ctx.heal_workflow_run_id = "wr_heal"
        if attach_through == "liveness_probe":
            assert session_id is not None
            await copilot_agent._registered_browser_state_liveness(session_id, ctx.organization_id, ctx)
        elif attach_through == "resolve":
            await copilot_runtime.resolve_browser_state_for_context(ctx, session_id=resolve_session_id)
        await _exit(ctx)
        return SimpleNamespace(final_output=json.dumps({"type": "REPLY", "user_response": "ok"}), new_items=[])

    stub_copilot_agent_loop(monkeypatch, fake_turn)
    workflow_run_id: str | None = None
    if attach_through == "prior_run_hydration":
        assert session_id is not None
        workflow_run_id = "wr_origin"
        origin = RepairOriginBinding(
            workflow_run_id=workflow_run_id,
            browser_session_id=session_id,
            refusal=None,
            status=WorkflowRunStatus.failed,
        )
        monkeypatch.setattr(copilot_agent, "seed_repair_origin_run", AsyncMock(return_value=origin))

        async def hydrate_and_exit(ctx: CopilotContext, *, workflow_run_id: str | None) -> None:
            await copilot_runtime.resolve_browser_state_for_context(ctx, session_id=session_id)
            await _exit(ctx)

        monkeypatch.setattr(copilot_agent, "hydrate_prior_run_packet", hydrate_and_exit)
    mock_app = MagicMock()
    mock_app.PERSISTENT_SESSIONS_MANAGER = manager
    monkeypatch.setattr(copilot_runtime, "app", mock_app)
    monkeypatch.setattr(copilot_agent, "_manager_can_probe_registered_browser_state", lambda: True)

    try:
        result = await run_copilot_agent(
            stream=MagicMock(),
            organization_id="org-1",
            chat_request=WorkflowCopilotChatRequest(
                workflow_permanent_id="wfp-1",
                workflow_id="wf-1",
                workflow_copilot_chat_id="chat-1",
                message="read the page title and tell me what it is",
                workflow_yaml="",
                workflow_run_id=workflow_run_id,
            ),
            chat_history=[],
            global_llm_context=None,
            llm_api_handler=SimpleNamespace(llm_key="PRIMARY"),
            raw_secret_safety_handler=AsyncMock(
                return_value={"version": "1", "state": "clean", "handling": "none", "citations": []}
            ),
            api_key="sk-test",
            eval_mode=eval_mode,
        )
    except BaseException as exc:
        return None, exc
    return result, None


async def run_concurrent_turns_on_one_session(
    monkeypatch: pytest.MonkeyPatch,
    *,
    manager: MagicMock,
    browser_states: tuple[MagicMock, MagicMock],
    exit_order: tuple[int, int] = (0, 1),
    after_first_exit: Callable[[], None] = lambda: None,
) -> None:
    """Two turns attached to one session, each on its own generation, released in ``exit_order``;
    ``after_first_exit`` is the point where only the earlier turn's release has run."""
    attached = (asyncio.Event(), asyncio.Event())
    may_exit = (asyncio.Event(), asyncio.Event())
    arrivals = count()

    async def _gate() -> None:
        index = next(arrivals)
        attached[index].set()
        await may_exit[index].wait()

    def _turn(index: int) -> asyncio.Task[tuple[AgentResult | None, BaseException | None]]:
        return asyncio.ensure_future(
            run_turn_to_exit(
                monkeypatch,
                manager=manager,
                exit_path="normal",
                browser_state=browser_states[index],
                on_attached=lambda _ctx: _gate(),
            )
        )

    first = _turn(0)
    await asyncio.wait_for(attached[0].wait(), 5)
    second = _turn(1)
    await asyncio.wait_for(attached[1].wait(), 5)
    turns = (first, second)

    may_exit[exit_order[0]].set()
    await turns[exit_order[0]]
    after_first_exit()
    may_exit[exit_order[1]].set()
    await turns[exit_order[1]]


async def run_turn_attaching_during_release(
    monkeypatch: pytest.MonkeyPatch,
    *,
    manager: MagicMock,
    browser_states: tuple[MagicMock, MagicMock],
    lookup_already_in_flight: bool = False,
) -> None:
    """A second turn resolves the session while the first turn's exit evict is still in flight.
    The manager hands out the first generation until that evict completes and the second after.
    With ``lookup_already_in_flight`` the second turn's lookup was issued before the release began
    and only answers once the evict is under way."""
    first_attached, evicting, evicted, contested = (asyncio.Event() for _ in range(4))

    async def _evict(*_args: object, **_kwargs: object) -> bool:
        if evicted.is_set():
            return True
        evicting.set()
        with suppress(TimeoutError):
            await asyncio.wait_for(contested.wait(), 1.0)
        evicted.set()
        return True

    async def _current_generation(**_kwargs: object) -> MagicMock:
        first_attached.set()
        if evicting.is_set() and not evicted.is_set():
            contested.set()
        return browser_states[1] if evicted.is_set() else browser_states[0]

    async def _generation_answered_mid_release(**kwargs: object) -> MagicMock:
        await asyncio.wait_for(evicting.wait(), 5)
        return await _current_generation(**kwargs)

    manager.evict_cached_browser_state = AsyncMock(side_effect=_evict)
    first = asyncio.ensure_future(
        run_turn_to_exit(
            monkeypatch,
            manager=manager,
            exit_path="normal",
            browser_state=browser_states[0],
            get_browser_state=AsyncMock(side_effect=_current_generation),
        )
    )
    second_lookup = _current_generation
    if lookup_already_in_flight:
        await asyncio.wait_for(first_attached.wait(), 5)
        second_lookup = _generation_answered_mid_release
    else:
        await asyncio.wait_for(evicting.wait(), 5)
    await run_turn_to_exit(
        monkeypatch,
        manager=manager,
        exit_path="normal",
        browser_state=browser_states[1],
        get_browser_state=AsyncMock(side_effect=second_lookup),
    )
    await first


async def run_turn_cancelled_during_cleanup(
    monkeypatch: pytest.MonkeyPatch,
    *,
    manager: MagicMock,
    browser_state: MagicMock,
    evicting: asyncio.Event,
    on_attached: Callable[[CopilotContext], Awaitable[None]] | None = None,
) -> tuple[AgentResult | None, BaseException | None]:
    """Cancel the turn once its exit release has reached the manager, which is the only window in
    which a cancel can land on the finalizer rather than on the turn's own work."""
    turn = asyncio.ensure_future(
        run_turn_to_exit(
            monkeypatch,
            manager=manager,
            exit_path="normal",
            browser_state=browser_state,
            on_attached=on_attached,
        )
    )
    await asyncio.wait_for(evicting.wait(), 5)
    turn.cancel()
    return await turn


def install_org_secondary_llm_override(monkeypatch: pytest.MonkeyPatch, handler: object) -> None:
    """Make ``get_org_aware_secondary_llm_api_handler`` resolve an org's routed secondary model."""
    monkeypatch.setattr(
        api_handler_factory.skyvern_context,
        "current",
        lambda: SkyvernContext(organization_id="o_test", org_default_secondary_llm_key="CUSTOM_LLM_oat_fast"),
    )
    monkeypatch.setattr(api_handler_factory, "is_custom_llm_owned_by_organization", lambda _id, _org: True)
    monkeypatch.setattr(api_handler_factory.LLMConfigRegistry, "is_registered", lambda _key: True)
    monkeypatch.setattr(api_handler_factory.LLMAPIHandlerFactory, "get_llm_api_handler", lambda _key: handler)


class FakeCopilotStream:
    """EventSourceStream stand-in recording what was sent and whether the send was accepted."""

    def __init__(self, send_ok: bool = True) -> None:
        self.send_ok = send_ok
        self.sent: list[Any] = []

    async def send(self, payload: Any) -> bool:
        self.sent.append(payload)
        return self.send_ok

    async def is_disconnected(self) -> bool:
        return False

    def titles(self) -> list[str]:
        return [event.title for event in self.sent if isinstance(event, WorkflowCopilotTitleUpdate)]


def _scripted_response() -> Response:
    return Response(
        id="resp_1",
        created_at=0.0,
        model="gpt-4o",
        object="response",
        output=[],
        parallel_tool_calls=True,
        tool_choice="auto",
        tools=[],
    )


ScriptedOutput = list[ResponseFunctionToolCall | ResponseOutputMessage]


def scripted_call(name: str, arguments: dict[str, Any], call_id: str = "call_held") -> ResponseFunctionToolCall:
    return ResponseFunctionToolCall(
        id=f"fc_{call_id}", call_id=call_id, name=name, arguments=json.dumps(arguments), type="function_call"
    )


def scripted_text(text: str) -> ResponseOutputMessage:
    return ResponseOutputMessage(
        id="msg_done",
        role="assistant",
        status="completed",
        type="message",
        content=[ResponseOutputText(type="output_text", text=text, annotations=[])],
    )


class ScriptedModel(Model):
    """Deterministic SDK input source: one scripted output per model call, None for a stream that never completes."""

    def __init__(self, script: list[ScriptedOutput | None]) -> None:
        self.script = script
        self.inputs: list[list[Any]] = []
        self.served_instructions: str | None = None
        self.served_tools: list[Any] = []
        self.served_tool_names_by_call: list[set[str]] = []

    async def get_response(self, *args: Any, **kwargs: Any) -> ModelResponse:
        raise AssertionError("streamed execution required")

    async def stream_response(
        self,
        system_instructions: str | None,
        input: Any,
        model_settings: Any,
        tools: list[Any],
        *args: Any,
        **kwargs: Any,
    ):
        self.served_instructions = system_instructions
        self.served_tools = tools
        self.served_tool_names_by_call.append({tool.name for tool in tools})
        self.inputs.append(list(input))
        output = self.script[len(self.inputs) - 1]
        if output is None:
            return
        for index, item in enumerate(output):
            yield ResponseOutputItemDoneEvent(
                item=item, output_index=index, sequence_number=index + 1, type="response.output_item.done"
            )
        response = _scripted_response().model_copy(update={"output": output, "status": "completed"})
        yield ResponseCompletedEvent(response=response, sequence_number=len(output) + 1, type="response.completed")


REPLY_ARGUMENTS = {"user_response": "Use a `for_loop` block.", "global_llm_context": {"user_goal": "loop over URLs"}}


async def run_scripted_turn(
    model: ScriptedModel,
    *,
    tools: list[FunctionTool] | None = None,
    max_turns: int = 10,
    output_guardrails: list[OutputGuardrail[CopilotContext]] | None = None,
    saved_items: list[TResponseInputItem] | None = None,
    ctx: CopilotContext | None = None,
) -> tuple[RunResultStreaming, CopilotContext, FakeCopilotStream]:
    ctx = ctx or make_copilot_ctx(
        organization_id="org_test", workflow_permanent_id="wpid_test", workflow_id=None, workflow_yaml=None, stream=None
    )
    stream = FakeCopilotStream()
    ctx.stream = stream
    session = SQLiteSession("scripted-turn")
    try:
        result = await run_with_enforcement(
            agent=Agent(
                name="controlled",
                model=model,
                tools=[*(tools or []), reply_tool],
                tool_use_behavior=reply_ends_turn,
                output_guardrails=output_guardrails or [],
            ),
            initial_input="How do I loop over a list of URLs?",
            ctx=ctx,
            stream=stream,
            session=session,
            max_turns=max_turns,
            hooks=CopilotRunHooks(ctx),
            run_config=RunConfig(
                tracing_disabled=True,
                session_input_callback=copilot_session_input_callback,
                call_model_input_filter=copilot_call_model_input_filter,
            ),
        )
    finally:
        if saved_items is not None:
            saved_items.extend(await session.get_items())
        session.close()
    return result, ctx, stream


def tool_outputs_for(model_input: list[Any], call_id: str) -> list[str]:
    return [
        item["output"]
        for item in model_input
        if isinstance(item, dict) and item.get("type") == "function_call_output" and item.get("call_id") == call_id
    ]


def tool_frame_types(stream: FakeCopilotStream) -> list[WorkflowCopilotStreamMessageType]:
    return [
        frame.type
        for frame in stream.sent
        if frame.type in (WorkflowCopilotStreamMessageType.TOOL_CALL, WorkflowCopilotStreamMessageType.TOOL_RESULT)
    ]


class ScriptedProvider(ModelProvider):
    def __init__(self, model: Model) -> None:
        self.model = model

    def get_model(self, model_name: str | None) -> Model:
        return self.model


async def run_production_loop(
    model: ScriptedModel,
    *,
    final_reply: bool = False,
    output_guardrails: list[OutputGuardrail[CopilotContext]] | None = None,
    native_tools: list[FunctionTool] | None = None,
) -> tuple[RunResultStreaming, CopilotContext, FakeCopilotStream]:
    ctx = make_copilot_ctx(
        organization_id="org_test", workflow_permanent_id="wpid_test", workflow_id=None, workflow_yaml=None, stream=None
    )
    ctx.api_key = "test-in-process-key"
    stream = FakeCopilotStream()
    ctx.stream = stream
    result = await _run_agent_loop_with_surface(
        ctx=ctx,
        stream=stream,
        chat_id="chat-1",
        initial_input="How do I loop over a list of URLs?",
        system_prompt="system prompt",
        model_name="scripted",
        run_config=RunConfig(tracing_disabled=True, model_provider=ScriptedProvider(model)),
        llm_key="PRIMARY",
        copilot_config=CopilotConfig(),
        native_tools=native_tools or [reply_tool],
        alias_map={},
        overlays={},
        output_guardrails=output_guardrails or [],
        final_reply=final_reply,
    )
    return result, ctx, stream


RECEIPT_WORKFLOW_YAML = """
title: Metric
workflow_definition:
  parameters: []
  blocks:
    - block_type: task
      label: sign_in
      prompt: Sign in
    - block_type: task
      label: read_metric
      prompt: Read the metric
"""


def receipt_message(narrative_payload: dict[str, Any]) -> MagicMock:
    return MagicMock(sender=WorkflowCopilotChatSender.AI, narrative_payload=narrative_payload)


def all_blocks_tested_message() -> MagicMock:
    current = workflow_block_fingerprints(RECEIPT_WORKFLOW_YAML)
    return receipt_message({"testedBlockFingerprints": {label: sorted(fps) for label, fps in current.items()}})


def receipt_coverage_after(
    messages: Sequence[WorkflowCopilotChatMessage], proposal: dict[str, Any] | None = None
) -> dict[str, str]:
    seed = _assistant_execution_receipts(list(messages), proposal)
    projection = build_review_projection(RECEIPT_WORKFLOW_YAML, RECEIPT_WORKFLOW_YAML, seed)
    assert projection is not None
    return {block["label"]: block["coverage"] for block in projection["blocks"]}