Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-15316: Commit the Copilot build-test run outcome before post-run browser enrichment (#8394)
Andrew Neilsonb6813d6c9d9b
diff --git a/skyvern/forge/sdk/copilot/build_test_connect_failure.py b/skyvern/forge/sdk/copilot/build_test_connect_failure.pyindex 2f88282b9..b54471ce8 100644--- a/skyvern/forge/sdk/copilot/build_test_connect_failure.py+++ b/skyvern/forge/sdk/copilot/build_test_connect_failure.py@@ -18,3 +18,11 @@ class BuildTestConnectFailure(BaseModel):task_id: str | None = Nonebrowser_session_id: str | None = Noneretry_action: Literal["test_end_to_end"] = "test_end_to_end"+++def build_test_connect_failure_sentence(failure: BuildTestConnectFailure) -> str:+ """A run id means the session was lost after the run started, so the operator must not be+ pointed at provisioning."""+ if failure.workflow_run_id:+ return f"Build-test browser session was unavailable before any block ran: {failure.state}."+ return f"Build-test browser acquisition stopped: {failure.state}."diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pyindex 9f477e99f..383b74594 100644--- a/skyvern/forge/sdk/copilot/build_test_outcome.py+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -13,7 +13,10 @@ import structlogimport yamlfrom pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator-from skyvern.forge.sdk.copilot.build_test_connect_failure import BuildTestConnectFailure+from skyvern.forge.sdk.copilot.build_test_connect_failure import (+ BuildTestConnectFailure,+ build_test_connect_failure_sentence,+)from skyvern.forge.sdk.copilot.challenge_evidence import (carrier_backed_anti_bot_categories,interactive_challenge_controls,@@ -456,6 +459,30 @@ class _RecordedBuildTestOutcomeContext(Protocol):recorded_persisted_block_run_workflow_run_id: str | None+def _history_entry(ctx: _RecordedBuildTestOutcomeContext, outcome: RecordedBuildTestOutcome) -> dict[str, object]:+ return {+ "phase": outcome.phase,+ "reason_code": outcome.reason_code,+ "verdict": outcome.verdict,+ "structural_key": outcome.structural_key,+ "is_authoritative": outcome.is_authoritative,+ "workflow_run_id": outcome.workflow_run_id,+ "authored_structure_signature": outcome.authored_structure_signature,+ "block_labels": list(outcome.block_labels),+ "attempted_block_label": outcome.attempted_block_label,+ "attempted_block_signature": _attempted_block_signature(ctx, outcome),+ "attempted_call_ref": outcome.attempted_call_ref,+ "code_safety_rejection_facts": [fact.model_dump(mode="json") for fact in outcome.code_safety_rejection_facts],+ "failed_operation": (+ outcome.failed_operation.model_dump(mode="json") if outcome.failed_operation is not None else None+ ),+ "connect_failure": (+ outcome.connect_failure.model_dump(mode="json") if outcome.connect_failure is not None else None+ ),+ "failed_operation_call_signature": outcome.failed_operation_call_signature,+ }++def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: RecordedBuildTestOutcome | None) -> None:if outcome is None:latest = getattr(ctx, "latest_recorded_build_test_outcome", None)@@ -504,31 +531,7 @@ def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: Rectx.latest_recorded_build_test_outcome = outcomeraw_history = getattr(ctx, "recorded_build_test_outcome_history", None)history: list[dict[str, object]] = raw_history if isinstance(raw_history, list) else []- history.append(- {- "phase": outcome.phase,- "reason_code": outcome.reason_code,- "verdict": outcome.verdict,- "structural_key": outcome.structural_key,- "is_authoritative": outcome.is_authoritative,- "workflow_run_id": outcome.workflow_run_id,- "authored_structure_signature": outcome.authored_structure_signature,- "block_labels": list(outcome.block_labels),- "attempted_block_label": outcome.attempted_block_label,- "attempted_block_signature": _attempted_block_signature(ctx, outcome),- "attempted_call_ref": outcome.attempted_call_ref,- "code_safety_rejection_facts": [- fact.model_dump(mode="json") for fact in outcome.code_safety_rejection_facts- ],- "failed_operation": (- outcome.failed_operation.model_dump(mode="json") if outcome.failed_operation is not None else None- ),- "connect_failure": (- outcome.connect_failure.model_dump(mode="json") if outcome.connect_failure is not None else None- ),- "failed_operation_call_signature": outcome.failed_operation_call_signature,- }- )+ history.append(_history_entry(ctx, outcome))del history[:-_HISTORY_LIMIT]ctx.recorded_build_test_outcome_history = historyif outcome.phase == "persisted_block_run" and outcome.is_authoritative and outcome.workflow_run_id:@@ -800,6 +803,58 @@ def bind_post_run_page_path_failure(return True+def bind_post_run_page_evidence(+ ctx: _RecordedBuildTestOutcomeContext,+ data: Mapping[str, object],+ page_evidence: Mapping[str, object] | None,+ *,+ regraded: RecordedBuildTestOutcome | None = None,+) -> bool:+ """Settle the outcome for this run now that post-run page evidence exists, replacing the+ entry recorded before it rather than appending a second one. Page evidence is a grading+ input rather than a decoration, so ``regraded`` supersedes a verdict reached without it."""+ run_id = _safe_str(data.get("workflow_run_id"))+ if not run_id:+ return False+ if regraded is not None and (regraded.phase != "persisted_block_run" or regraded.workflow_run_id != run_id):+ regraded = None+ latest = ctx.latest_recorded_build_test_outcome+ if latest is None or latest.phase != "persisted_block_run" or latest.workflow_run_id != run_id:+ # Grading before enrichment had no page to key on, so this run may have recorded nothing at all.+ if regraded is None:+ return False+ record_build_test_outcome(ctx, regraded)+ return True+ graded = page_evidence if _post_run_page_evidence_matches_result(data, page_evidence) else None+ page_fields: dict[str, object] = {+ "page_evidence_refs": _page_evidence_refs(graded),+ "page_capture": post_run_page_capture_from_result(data, graded),+ "page_path_failure": _post_run_page_path_failure(graded, run_id),+ "observed_page_value_excerpt": _observed_page_value_excerpt(graded),+ }+ if regraded is not None:+ # record_build_test_outcome resolved these against the executed code; the regrade cannot see it.+ carried = (+ {+ "failed_operation": latest.failed_operation,+ "failed_operation_call_signature": latest.failed_operation_call_signature,+ "failed_operation_code_signature": latest.failed_operation_code_signature,+ }+ if latest.failed_operation is not None+ else {}+ )+ updated = regraded.model_copy(update={**page_fields, **carried})+ else:+ updated = latest.model_copy(update=page_fields)+ ctx.latest_recorded_build_test_outcome = updated+ history = ctx.recorded_build_test_outcome_history+ if history and history[-1].get("workflow_run_id") == run_id:+ history[-1] = _history_entry(ctx, updated)+ if updated.is_authoritative:+ ctx.recorded_persisted_block_run_workflow_run_id = run_id+ return True++def authored_structure_signature_from_workflow(workflow_yaml: str | None,code_artifact_metadata: object = None,@@ -1091,7 +1146,7 @@ def recorded_outcome_from_run_blocks_result(requested_block_labels=requested_block_labels,structural_failure_identity=f"build_test_connect:{connect_failure.state}",connect_failure=connect_failure,- observed_evidence_summary=f"Build-test browser acquisition stopped: {connect_failure.state}.",+ observed_evidence_summary=build_test_connect_failure_sentence(connect_failure),key_provenance={"structural_failure_identity": "typed build-test browser acquisition fact"},)executed_block_labels = _clean_list(diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex 3363e3613..2c11945b5 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -43,6 +43,8 @@ if TYPE_CHECKING:LOG = structlog.get_logger()_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY = "_copilot_internal_run_cancelled_by_watchdog"+_INTERNAL_RUN_OUTCOME_RECORDED_KEY = "_copilot_internal_run_outcome_recorded"+_INTERNAL_GOAL_PATH_OMISSIONS_KEY = "_copilot_internal_goal_path_omissions"_BASE64_IMAGE_OMITTED_MESSAGE = "[base64 image omitted — screenshot was taken successfully]"BUILD_TEST_PACKET_KEY = "build_test_packet"@@ -1216,6 +1218,8 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict"timing_ms","_workflow",_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,+ _INTERNAL_RUN_OUTCOME_RECORDED_KEY,+ _INTERNAL_GOAL_PATH_OMISSIONS_KEY,),drop_data_keys=("sdk_equivalent", "authored_locator_observations"),replacement_fields={"screenshot_base64": _BASE64_IMAGE_OMITTED_MESSAGE},diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 650bc28a7..217003a47 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -30,7 +30,10 @@ from skyvern.cli.core.session_manager import ()from skyvern.config import settingsfrom skyvern.forge import app-from skyvern.forge.sdk.copilot.build_test_connect_failure import BuildTestConnectFailure+from skyvern.forge.sdk.copilot.build_test_connect_failure import (+ BuildTestConnectFailure,+ build_test_connect_failure_sentence,+)from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicyfrom skyvern.forge.sdk.copilot.screenshot_utils import PendingFrameLease, ScreenshotEntryfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_span@@ -1225,12 +1228,13 @@ async def _drop_browser_session_id_at_its_fixed_deadline(ctx: AgentContext) -> Ndef _build_test_connect_failure_result(failure: BuildTestConnectFailure) -> dict[str, Any]:+ sentence = build_test_connect_failure_sentence(failure)return {"ok": False,- "error": f"Build-test browser acquisition stopped: {failure.state}.",+ "error": sentence,"data": {"overall_status": "setup_failed",- "failure_reason": f"Build-test browser acquisition stopped: {failure.state}.",+ "failure_reason": sentence,"browser_session_id": failure.browser_session_id,"build_test_connect_failure": failure.model_dump(mode="json", exclude_none=True),"blocks": [],diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 4cd574273..c74aa1b9a 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -28,6 +28,11 @@ from skyvern.forge.sdk.copilot.authoring_parameter_binding import _literal_locatfrom skyvern.forge.sdk.copilot.blocker_signal import (stash_blocker_signal,)+from skyvern.forge.sdk.copilot.build_test_connect_failure import (+ BuildTestConnectFailure,+ BuildTestConnectFailureState,+ build_test_connect_failure_sentence,+)from skyvern.forge.sdk.copilot.build_test_outcome import (INFRASTRUCTURE_RUNNER_ERROR_CODES,BuildTestEvidencePacket,@@ -46,6 +51,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (RecordedBuildTestOutcome,authored_block_parameter_keys_from_workflow,authored_structure_signature_from_workflow,+ bind_post_run_page_evidence,connect_failure_from_run_blocks_result,failed_operation_from_run_blocks_result,post_run_page_capture_from_result,@@ -96,7 +102,9 @@ from skyvern.forge.sdk.copilot.narration import handler_available as narration_hfrom skyvern.forge.sdk.copilot.narration import narrator_poll_tickfrom skyvern.forge.sdk.copilot.outcome_verification_trace import record_gate_decisionfrom skyvern.forge.sdk.copilot.output_utils import (+ _INTERNAL_GOAL_PATH_OMISSIONS_KEY,_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,+ _INTERNAL_RUN_OUTCOME_RECORDED_KEY,BUILD_TEST_PACKET_KEY,build_run_blocks_response,iter_failure_reasons,@@ -2424,6 +2432,141 @@ async def run_workflow_end_to_end(ctx: CopilotContext, workflow_yaml: str) -> di)+async def _attach_post_run_browser_enrichment(+ ctx: CopilotContext,+ result_data: dict[str, Any],+ *,+ workflow: Workflow | None,+ workflow_run_id: str,+ run_block_rows: list[WorkflowRunBlock],+ results: list[dict[str, Any]],+ block_outputs_by_label: Mapping[str, Any],+ failed_block_code: str | None,+ run_session_id: str | None,+ dispatch_to_worker: bool,+ sensitive_origin_run: bool,+ run_ok: bool,+ used_fresh_run_session: bool,+ run_detached_from_chat: bool,+) -> tuple[str, dict[str, Any] | None]:+ """Probe the browser for post-run facts and stamp them onto an already-recorded run result.+ The outcome is what is protected, not these awaits: it is committed before this runs, so a probe+ the deadline cancels costs enrichment and never the record. Only the screenshot capture is+ guarded here; every other await propagates and fails the tool call."""+ # Dispatched runs: the worker owns the run session; do not touch it over CDP from the API.+ # The frontier's anchors and the page the run ended on both come from the rows the worker+ # persisted instead. A dispatched run has no page title to report.+ current_url, page_title, dispatched_end_url = await _resolve_post_run_page_info(+ ctx,+ run_block_rows=run_block_rows,+ dispatch_to_worker=dispatch_to_worker,+ sensitive_origin_run=sensitive_origin_run,+ run_session_id=run_session_id,+ )++ screenshot_b64: str | None = None+ # Dispatched runs: the worker owns the persistent browser session, so the API side must not+ # grab the live page over CDP. Their at-failure frames come from worker-persisted artifacts.+ if not dispatch_to_worker and not run_ok and run_session_id and not sensitive_origin_run:+ try:+ browser_state = await resolve_persistent_browser_state(+ session_id=run_session_id,+ organization_id=ctx.organization_id,+ )+ if browser_state:+ page = await browser_state.get_or_create_page()+ if SettingsManager.get_settings().BROWSER_CURSOR_VISUALIZATION:+ try:+ await SkyvernFrame.hide_cursor_overlay(page)+ except Exception:+ pass+ try:+ screenshot_bytes = await page.screenshot(type="png")+ finally:+ if SettingsManager.get_settings().BROWSER_CURSOR_VISUALIZATION:+ try:+ await SkyvernFrame.show_cursor_overlay(page)+ except Exception:+ pass+ screenshot_b64 = base64.b64encode(screenshot_bytes).decode("utf-8")+ except Exception:+ LOG.debug("Failed to capture post-run screenshot", exc_info=True)++ locator_observations: list[AuthoredLocatorObservationRow] | None = None+ post_run_page_capture: BuildTestPacketPageCapture | None = None+ if (+ not dispatch_to_worker+ and run_session_id+ and _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER+ and not ctx.copilot_total_timeout_exceeded+ ):+ # Structured evidence is admitted through the origin registry scrubber. Pixel capture+ # and locator probes remain withheld for sensitive runs because they have no equivalent+ # exact-value disclosure boundary.+ _pin_pre_run_page_reference(ctx, workflow_run_id)+ post_run_page_capture = await _capture_and_store_post_run_page(+ ctx,+ run_session_id=run_session_id,+ run_id=workflow_run_id,+ current_url=current_url,+ )++ if not sensitive_origin_run:+ locator_observations = await _observe_authored_locators(+ ctx,+ run_session_id=run_session_id,+ failed_block_code=failed_block_code,+ worker_owned=dispatch_to_worker,+ observation_deadline_exceeded=ctx.copilot_total_timeout_exceeded,+ )++ if not dispatch_to_worker and not ctx.copilot_total_timeout_exceeded:+ await _capture_registered_artifact_evidence(+ ctx,+ run_id=workflow_run_id,+ organization_id=ctx.organization_id,+ downloaded_artifact_ids=_collect_downloaded_artifact_ids(block_outputs_by_label),+ )++ # Dispatched runs are worker-owned, so the API cannot CDP-capture the terminal page; read the+ # worker-persisted terminal HTML artifact instead and route it through the same post-run sink.+ if dispatch_to_worker and run_session_id and not ctx.copilot_total_timeout_exceeded:+ await _capture_dispatched_terminal_page_evidence(+ ctx,+ workflow=workflow,+ run_id=workflow_run_id,+ run_session_id=run_session_id,+ organization_id=ctx.organization_id,+ current_url=current_url,+ origin_redaction_registry=ctx.origin_run_redaction_registry,+ )++ result_data["current_url"] = current_url+ result_data["page_title"] = page_title+ if locator_observations is not None:+ result_data["authored_locator_observations"] = locator_observations+ if not dispatch_to_worker and current_url:+ result_data["current_url_live_observed"] = True+ if dispatch_to_worker and dispatched_end_url is None:+ result_data["current_url_evidence"] = NO_PERSISTED_END_URL+ post_run_page_evidence = _same_run_page_evidence_for_result(ctx, workflow_run_id)+ if post_run_page_evidence is not None:+ result_data["post_run_page_evidence"] = model_visible_composition_evidence(post_run_page_evidence)+ if post_run_page_capture is not None:+ result_data["post_run_page_capture"] = post_run_page_capture.model_dump(mode="json")+ _attach_run_session_facts(+ result_data,+ used_fresh_run_session=used_fresh_run_session,+ run_detached_from_chat=run_detached_from_chat,+ run_ok=run_ok,+ page_evidence=post_run_page_evidence,+ )+ resolved_screenshot_b64 = _resolve_run_screenshot_b64(live_capture=screenshot_b64, results=results, run_ok=run_ok)+ if resolved_screenshot_b64 is not None:+ result_data["screenshot_base64"] = resolved_screenshot_b64+ return current_url, post_run_page_evidence++async def _run_blocks_and_collect_debug(params: dict[str, Any],ctx: CopilotContext,@@ -3272,94 +3415,7 @@ async def _run_blocks_and_collect_debug(for entry in results:entry.pop("action_trace", None)- # Dispatched runs: the worker owns the run session; do not touch it over CDP from the API.- # The frontier's anchors and the page the run ended on both come from the rows the worker- # persisted instead. A dispatched run has no page title to report.block_end_urls = {} if sensitive_origin_run else _block_end_urls_by_label(run_block_rows)- current_url, page_title, dispatched_end_url = await _resolve_post_run_page_info(- ctx,- run_block_rows=run_block_rows,- dispatch_to_worker=dispatch_to_worker,- sensitive_origin_run=sensitive_origin_run,- run_session_id=run_session_id,- )-- screenshot_b64: str | None = None- # Dispatched runs: the worker owns the persistent browser session, so the API side must not- # grab the live page over CDP. Their at-failure frames come from worker-persisted artifacts.- if not dispatch_to_worker and not run_ok and run_session_id and not sensitive_origin_run:- try:- browser_state = await resolve_persistent_browser_state(- session_id=run_session_id,- organization_id=ctx.organization_id,- )- if browser_state:- page = await browser_state.get_or_create_page()- if SettingsManager.get_settings().BROWSER_CURSOR_VISUALIZATION:- try:- await SkyvernFrame.hide_cursor_overlay(page)- except Exception:- pass- try:- screenshot_bytes = await page.screenshot(type="png")- finally:- if SettingsManager.get_settings().BROWSER_CURSOR_VISUALIZATION:- try:- await SkyvernFrame.show_cursor_overlay(page)- except Exception:- pass- screenshot_b64 = base64.b64encode(screenshot_bytes).decode("utf-8")- except Exception:- LOG.debug("Failed to capture post-run screenshot", exc_info=True)-- locator_observations: list[AuthoredLocatorObservationRow] | None = None- post_run_page_capture: BuildTestPacketPageCapture | None = None- if (- not dispatch_to_worker- and run_session_id- and _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER- and not ctx.copilot_total_timeout_exceeded- ):- # Structured evidence is admitted through the origin registry scrubber. Pixel capture- # and locator probes remain withheld for sensitive runs because they have no equivalent- # exact-value disclosure boundary.- _pin_pre_run_page_reference(ctx, workflow_run.workflow_run_id)- post_run_page_capture = await _capture_and_store_post_run_page(- ctx,- run_session_id=run_session_id,- run_id=workflow_run.workflow_run_id,- current_url=current_url,- )-- if not sensitive_origin_run:- locator_observations = await _observe_authored_locators(- ctx,- run_session_id=run_session_id,- failed_block_code=failed_block_code,- worker_owned=dispatch_to_worker,- observation_deadline_exceeded=ctx.copilot_total_timeout_exceeded,- )-- if not dispatch_to_worker and not ctx.copilot_total_timeout_exceeded:- await _capture_registered_artifact_evidence(- ctx,- run_id=workflow_run.workflow_run_id,- organization_id=ctx.organization_id,- downloaded_artifact_ids=_collect_downloaded_artifact_ids(block_outputs_by_label),- )-- # Dispatched runs are worker-owned, so the API cannot CDP-capture the terminal page; read the- # worker-persisted terminal HTML artifact instead and route it through the same post-run sink.- if dispatch_to_worker and run_session_id and not ctx.copilot_total_timeout_exceeded:- await _capture_dispatched_terminal_page_evidence(- ctx,- workflow=workflow,- run_id=workflow_run.workflow_run_id,- run_session_id=run_session_id,- organization_id=ctx.organization_id,- current_url=current_url,- origin_redaction_registry=ctx.origin_run_redaction_registry,- )result_data: dict[str, Any] = {"workflow_run_id": workflow_run.workflow_run_id,@@ -3369,39 +3425,18 @@ async def _run_blocks_and_collect_debug("executed_block_labels": list(labels_to_execute),"frontier_start_label": frontier_start_label,"blocks": results,- "current_url": current_url,- "page_title": page_title,"action_trace_summary": action_trace_summary,"failing_code_line": failing_code_line,"action_observations": action_observations,}- if locator_observations is not None:- result_data["authored_locator_observations"] = locator_observations- if not dispatch_to_worker and current_url:- result_data["current_url_live_observed"] = True- if dispatch_to_worker and dispatched_end_url is None:- result_data["current_url_evidence"] = NO_PERSISTED_END_URL- post_run_page_evidence = _same_run_page_evidence_for_result(ctx, workflow_run.workflow_run_id)- if post_run_page_evidence is not None:- result_data["post_run_page_evidence"] = model_visible_composition_evidence(post_run_page_evidence)- if post_run_page_capture is not None:- result_data["post_run_page_capture"] = post_run_page_capture.model_dump(mode="json")- _attach_run_session_facts(- result_data,- used_fresh_run_session=used_fresh_run_session,- run_detached_from_chat=run_detached_from_chat,- run_ok=run_ok,- page_evidence=post_run_page_evidence,- )if runtime_frontier_anchor_url is not None:result_data["runtime_frontier_anchor_url"] = runtime_frontier_anchor_urlif runtime_frontier_starter_url_seeded:result_data["runtime_frontier_starter_url_seeded"] = True- screenshot_b64 = _resolve_run_screenshot_b64(live_capture=screenshot_b64, results=results, run_ok=run_ok)- if screenshot_b64 is not None:- result_data["screenshot_base64"] = screenshot_b64if not run_ok and run and getattr(run, "failure_reason", None):result_data["failure_reason"] = run.failure_reason+ if not run_ok and run and getattr(run, "failure_category", None):+ result_data["failure_category"] = run.failure_categoryoutput_identity_workflow = _registered_output_identity_workflow(dispatch_to_worker=dispatch_to_worker,@@ -3423,18 +3458,23 @@ async def _run_blocks_and_collect_debug(data=result_data,persisted_output_parameters=all_output_params,)+ # The run's terminal facts are known here, so the outcome is committed before any+ # browser-dependent probe: a probe the turn deadline cancels must not erase what the run did.+ # The post-run capture that drops another run's page now happens after the record, and the+ # anti-bot read has no run-id check of its own, so stale evidence would grade this run.+ if not post_run_inspection_cleanly_matches(ctx.composition_page_evidence, workflow_run.workflow_run_id):+ ctx.composition_page_evidence = None+for label, output in registered_outputs_by_label.items():if isinstance(output, dict) and output:block_outputs_by_label[label] = output- # Update verified prefix state ONLY on a fully-successful run. A failed- # suffix run leaves the browser in post-failure state, so we must not- # trust blocks that individually succeeded inside it.- if run_ok and all(r.get("status") == "completed" for r in results):- for label, output in block_outputs_by_label.items():- ctx.verified_block_outputs[label] = output+ # The record below derives "fully tested" from these credits and prefers its own+ # extracted_data for terminal replies, so both have to land before it, not after.+ run_fully_completed = run_ok and all(r.get("status") == "completed" for r in results)+ if run_fully_completed:ctx.verified_terminal_block_outputs = dict(block_outputs_by_label)- existing_prefix = list(getattr(ctx, "verified_prefix_labels", []) or [])+ existing_prefix = list(ctx.verified_prefix_labels or [])existing_set = set(existing_prefix)for label in labels_to_execute:if label not in existing_set:@@ -3442,6 +3482,48 @@ async def _run_blocks_and_collect_debug(existing_set.add(label)ctx.verified_prefix_labels = existing_prefix_credit_composition_verified_labels(ctx, labels_to_execute, start_provenance)++ response = build_run_blocks_response(run_ok, result_data)+ committed_run_outcome = _commit_run_blocks_record(ctx, response)++ current_url, post_run_page_evidence = await _attach_post_run_browser_enrichment(+ ctx,+ result_data,+ workflow=workflow,+ workflow_run_id=workflow_run.workflow_run_id,+ run_block_rows=run_block_rows,+ results=results,+ block_outputs_by_label=block_outputs_by_label,+ failed_block_code=failed_block_code,+ run_session_id=run_session_id,+ dispatch_to_worker=dispatch_to_worker,+ sensitive_origin_run=sensitive_origin_run,+ run_ok=run_ok,+ used_fresh_run_session=used_fresh_run_session,+ run_detached_from_chat=run_detached_from_chat,+ )+ settle_terminal_challenge_after_enrichment(ctx, response)+ settled_page_evidence = post_run_page_evidence or ctx.composition_page_evidence+ bind_post_run_page_evidence(+ ctx,+ result_data,+ # Singular selectors are stripped at this boundary; page_evidence_refs reaches a prompt.+ model_visible_composition_evidence(settled_page_evidence) if settled_page_evidence else None,+ regraded=_build_recorded_build_test_outcome(+ ctx,+ response,+ ctx.last_run_outcome or committed_run_outcome,+ response.get(_INTERNAL_GOAL_PATH_OMISSIONS_KEY),+ ),+ )+ _update_verification_evidence_from_run_result(ctx, response)++ # Update verified prefix state ONLY on a fully-successful run. A failed+ # suffix run leaves the browser in post-failure state, so we must not+ # trust blocks that individually succeeded inside it.+ if run_fully_completed:+ for label, output in block_outputs_by_label.items():+ ctx.verified_block_outputs[label] = outputif sensitive_origin_run:# Preserve verified labels and outputs, but never carry browser-position URLs from# a credential-bearing run into a later frontier or model-visible result.@@ -3457,7 +3539,7 @@ async def _run_blocks_and_collect_debug(if verified_current_url is not None:ctx.verified_prefix_current_url = verified_current_url- return build_run_blocks_response(run_ok, result_data)+ return responsefinally:# A paused run keeps its association so the pane still follows the run the person was# asked to act on; the next run's generation replaces it.@@ -3809,6 +3891,82 @@ def _block_labels_from_result_data(data: Mapping[str, object]) -> tuple[str, ...return tuple(dict.fromkeys(labels))+def _apply_terminal_challenge_latches(+ copilot_ctx: CopilotContext, result: dict[str, Any], terminal_challenge: TerminalChallengeEvidence+) -> None:+ result["ok"] = False+ result.setdefault("error", terminal_challenge.reason)+ data = result.get("data")+ if isinstance(data, dict):+ data.setdefault("failure_reason", terminal_challenge.reason)+ _ensure_terminal_challenge_category(+ data, challenge_evidence_source=terminal_challenge.challenge_evidence_source+ )+ copilot_ctx.last_failure_category_top = "ANTI_BOT_DETECTION"+ LOG.info(+ "copilot anti-bot evidence stamp",+ anti_bot_evidence_source=terminal_challenge.challenge_evidence_source,+ stamp_seam="terminal_challenge",+ )+ copilot_ctx.last_test_ok = False+ copilot_ctx.last_test_suspicious_success = False+ copilot_ctx.last_test_failure_reason = terminal_challenge.reason+ copilot_ctx.last_test_anti_bot = terminal_challenge.reason+ copilot_ctx.last_full_workflow_test_ok = False+ copilot_ctx.last_failed_workflow_yaml = copilot_ctx.workflow_yaml+ # A challenge used to fire before the success branch could run, so these were unreachable for a+ # challenged run; the grade now precedes the settle and can set them.+ copilot_ctx.verified_terminal_proposal_ready = False+++def settle_terminal_challenge_after_enrichment(copilot_ctx: CopilotContext, result: dict[str, Any]) -> bool:+ """Re-decide the terminal challenge once post-run page evidence exists, since the grade committed+ at handback could not see it. It may only turn a reported pass into a failure, never the reverse."""+ if result.get("ok") is not True:+ return False+ structured_blocker = _run_blocks_structured_blocker_message(result, copilot_ctx)+ anti_bot_match, _, failure_categories, _ = _analyze_run_blocks(result, copilot_ctx)+ anti_bot_source = first_carrier_backed_anti_bot_source(failure_categories) if anti_bot_match else None+ anti_bot_evidence_source = anti_bot_source.value if anti_bot_source else None+ if not anti_bot_match:+ anti_bot_match = _composition_anti_bot_reason(copilot_ctx)+ if anti_bot_match:+ composition_source = composition_challenge_carrier(copilot_ctx.composition_page_evidence)+ anti_bot_evidence_source = composition_source.value if composition_source else None+ if anti_bot_match and not anti_bot_evidence_source:+ # Keyword-only, same as the pre-enrichment path: drop the match but keep going, because a+ # structured blocker plus a challenge artifact still decides this without it.+ LOG.info("copilot anti-bot latch keyword-only-suppressed")+ anti_bot_match = None+ terminal_challenge = _terminal_challenge_evidence(+ result,+ failure_categories=failure_categories,+ structured_blocker=structured_blocker,+ anti_bot_match=anti_bot_match,+ anti_bot_evidence_source=anti_bot_evidence_source,+ artifact_flag_key=_artifact_challenge_flag_from_result(result, copilot_ctx),+ challenge_kind=_terminal_challenge_kind(copilot_ctx, result),+ )+ if terminal_challenge is None:+ return False+ _apply_terminal_challenge_latches(copilot_ctx, result, terminal_challenge)+ # The committed outcome still reads as the pass this run looked like; the emitted envelope would+ # otherwise contradict the failure the caller now returns.+ data = result.get("data")+ run_id = data.get("workflow_run_id") if isinstance(data, dict) else None+ _stash_recorded_run_outcome(+ copilot_ctx,+ RecordedRunOutcome(+ verdict="not_demonstrated",+ reason_code="blocker_reported",+ display_reason=run_outcome_display_reason(terminal_challenge.reason),+ workflow_run_id=run_id if isinstance(run_id, str) else None,+ run_completed=False,+ ),+ )+ return True++def _terminal_challenge_evidence(result: dict[str, Any],*,@@ -4007,7 +4165,10 @@ def _retained_terminal_output_has_value(value: Any) -> bool:def _record_run_blocks_result(- copilot_ctx: Any, result: dict[str, Any], completion_verification: CompletionVerificationResult | None = None+ copilot_ctx: Any,+ result: dict[str, Any],+ completion_verification: CompletionVerificationResult | None = None,+ connect_failure_reason: str | None = None,) -> RecordedRunOutcome | None:"""Record the run result on ctx without letting a second judge rewrite it."""_record_executed_block_labels(copilot_ctx, result)@@ -4154,27 +4315,8 @@ def _record_run_blocks_result(}if terminal_challenge is not None:- result["ok"] = False+ _apply_terminal_challenge_latches(copilot_ctx, result, terminal_challenge)run_ok = False- result.setdefault("error", terminal_challenge.reason)- data = result.get("data")- if isinstance(data, dict):- data.setdefault("failure_reason", terminal_challenge.reason)- _ensure_terminal_challenge_category(- data, challenge_evidence_source=terminal_challenge.challenge_evidence_source- )- copilot_ctx.last_failure_category_top = "ANTI_BOT_DETECTION"- LOG.info(- "copilot anti-bot evidence stamp",- anti_bot_evidence_source=terminal_challenge.challenge_evidence_source,- stamp_seam="terminal_challenge",- )- copilot_ctx.last_test_ok = False- copilot_ctx.last_test_suspicious_success = False- copilot_ctx.last_test_failure_reason = terminal_challenge.reason- copilot_ctx.last_test_anti_bot = terminal_challenge.reason- copilot_ctx.last_full_workflow_test_ok = False- copilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)if run_ok:registered_output_identity_mismatch = bool(@@ -4255,12 +4397,18 @@ def _record_run_blocks_result(copilot_ctx.last_test_failure_reason = next(iter_failure_reasons(result), None)if result.get("error") and copilot_ctx.last_test_failure_reason is None:copilot_ctx.last_test_failure_reason = str(result["error"])+ if connect_failure_reason:+ # User-facing surfaces read this; the result dict keeps the run's own reason for the+ # repair signature, the failure summary and loop detection.+ copilot_ctx.last_test_failure_reason = connect_failure_reason_update_verification_evidence_from_run_result(copilot_ctx, result)recorded_outcome = RecordedRunOutcome(verdict="not_demonstrated",reason_code="blocker_reported",display_reason=run_outcome_display_reason(- copilot_ctx.last_test_failure_reason or str(result.get("error") or "The run failed.")+ connect_failure_reason+ or copilot_ctx.last_test_failure_reason+ or str(result.get("error") or "The run failed.")),workflow_run_id=run_id if isinstance(run_id, str) else None,run_completed=False,@@ -4272,6 +4420,19 @@ def _record_run_blocks_result(_EXECUTED_BLOCK_STATUSES = frozenset(status.value for status in BlockStatus if status != BlockStatus.skipped)+def _commit_run_blocks_record(copilot_ctx: CopilotContext, result: dict[str, Any]) -> RecordedRunOutcome | None:+ """Commit the run's structured outcome once, marking the result so the shared seam does not redo it.++ The browser-loss stamp runs here rather than at the shared seam: the commit happens upstream of+ that seam, so a stamp applied there would never reach the committed record."""+ connect_failure_reason = _stamp_run_side_connect_failure(copilot_ctx, result)+ recorded = _record_run_blocks_result(+ copilot_ctx, result, completion_verification=None, connect_failure_reason=connect_failure_reason+ )+ result[_INTERNAL_RUN_OUTCOME_RECORDED_KEY] = True+ return recorded++def _record_executed_block_labels(copilot_ctx: CopilotContext, result: dict[str, Any]) -> None:data = result.get("data")blocks = data.get("blocks") if isinstance(data, dict) else None@@ -4358,6 +4519,8 @@ def _record_build_test_outcome(recorded_run_outcome: RecordedRunOutcome | None,declared_goal_path_omissions: Sequence[Mapping[str, str]] | None = None,) -> None:+ # Kept on the result so a post-enrichment regrade reads the same omission facts this grade did.+ result[_INTERNAL_GOAL_PATH_OMISSIONS_KEY] = list(declared_goal_path_omissions or [])record_build_test_outcome(copilot_ctx,_build_recorded_build_test_outcome(@@ -4546,24 +4709,75 @@ def _safe_reason_code(value: object) -> str:return value if isinstance(value, str) else ""+_RUN_SIDE_CONNECT_STATES: dict[str, BuildTestConnectFailureState] = {+ "browser_session_closed": "already_closed",+ "browser_session_startup_timeout": "provisioning_unavailable",+}+++def _run_side_connect_state(failure_category: object) -> BuildTestConnectFailureState | None:+ if not isinstance(failure_category, list):+ return None+ for entry in failure_category:+ if isinstance(entry, dict) and entry.get("category") == "BROWSER_ERROR":+ state = _RUN_SIDE_CONNECT_STATES.get(str(entry.get("reason_code") or ""))+ if state is not None:+ return state+ return None+++def _stamp_run_side_connect_failure(copilot_ctx: CopilotContext, result: dict[str, Any]) -> str | None:+ """Attach the typed browser-loss fact the run itself persisted at its lease seam, beside the+ run's own failure text, and return the sentence the user should see. Nothing here is inferred:+ a run that failed for any other reason carries no BROWSER_ERROR reason code and stays untyped,+ however its session row happens to look afterwards."""+ if result.get("ok"):+ return None+ data = result.get("data")+ if not isinstance(data, dict) or data.get("build_test_connect_failure") is not None:+ return None+ if data.get("blocks") or data.get("control_signal") is not None:+ return None+ workflow_run_id = data.get("workflow_run_id")+ if not isinstance(workflow_run_id, str) or not workflow_run_id:+ return None+ state = _run_side_connect_state(data.get("failure_category"))+ if state is None:+ return None+ session_id = data.get("browser_session_id")+ failure = BuildTestConnectFailure(+ state=state,+ browser_session_id=session_id if isinstance(session_id, str) and session_id else None,+ workflow_run_id=workflow_run_id,+ )+ data["build_test_connect_failure"] = failure.model_dump(mode="json", exclude_none=True)+ return build_test_connect_failure_sentence(failure)++async def _verify_and_record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any], _handler_start: float) -> RecordedBuildTestOutcome | None:"""Record and emit the run fact once; no authoring judge may rewrite it."""- await _resolve_captcha_solver_availability(copilot_ctx)- recorded = _record_run_blocks_result(copilot_ctx, result, completion_verification=None)+ if result.get(_INTERNAL_RUN_OUTCOME_RECORDED_KEY) is True:+ recorded = copilot_ctx.last_run_outcome+ else:+ recorded = _commit_run_blocks_record(copilot_ctx, result)if not result.get("ok"):_mark_stored_post_run_failure_page(copilot_ctx)+ if recorded is not None:+ await _send_run_outcome_update(+ copilot_ctx,+ result,+ verdict=recorded.verdict,+ role=recorded.role,+ reason_code=recorded.reason_code,+ display_reason=recorded.display_reason,+ )+ # The solver probe reaches the browser and can be cancelled by the turn deadline, so it runs+ # only after the outcome is committed and emitted, and still before the caller builds repair context.+ await _resolve_captcha_solver_availability(copilot_ctx)if recorded is None:return None- await _send_run_outcome_update(- copilot_ctx,- result,- verdict=recorded.verdict,- role=recorded.role,- reason_code=recorded.reason_code,- display_reason=recorded.display_reason,- )build_test_outcome = getattr(copilot_ctx, "latest_recorded_build_test_outcome", None)result_data = result.get("data")result_run_id = result_data.get("workflow_run_id") if isinstance(result_data, dict) else Nonediff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 277ec6996..fa64cefec 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -54,6 +54,7 @@ from skyvern.exceptions import (BrowserSessionClosed,BrowserSessionNotFound,BrowserSessionNotRenewable,+ BrowserSessionStartupTimeout,DisabledBlockExecutionError,DownloadSaveIncompleteError,InProcessScriptExecutionDenied,@@ -1387,6 +1388,28 @@ def _build_managed_browser_profile_name(workflow_title: str | None, rendered_keyreturn f"{title}{suffix}"+def _browser_lease_failure_category(exc: Exception) -> list[dict] | None:+ """The lease seam still holds the typed exception; persist its identity so a reader does not+ have to rediscover it from prose or from the session row, which closes the same way after+ every run."""+ if isinstance(exc, BrowserSessionClosed):+ reason_code = "browser_session_closed"+ reasoning = "The browser session had already closed before the run could lease it"+ elif isinstance(exc, BrowserSessionStartupTimeout):+ reason_code = "browser_session_startup_timeout"+ reasoning = "The browser session did not start within its startup timeout"+ else:+ return None+ return [+ {+ "category": "BROWSER_ERROR",+ "confidence_float": 1.0,+ "reason_code": reason_code,+ "reasoning": reasoning,+ }+ ]++class WorkflowService:# Prevent GC of fire-and-forget asyncio tasks (e.g. task_run sync)._background_tasks: set[asyncio.Task] = set() # noqa: RUF012@@ -4999,6 +5022,7 @@ class WorkflowService:workflow_run = await self.mark_workflow_run_as_failed(workflow_run_id=workflow_run_id,failure_reason=failure_reason,+ failure_category=_browser_lease_failure_category(e),)await self.clean_up_workflow(workflow=workflow,diff --git a/tests/unit/copilot_test_helpers.py b/tests/unit/copilot_test_helpers.pyindex 3d3b0c485..b84244747 100644--- a/tests/unit/copilot_test_helpers.py+++ b/tests/unit/copilot_test_helpers.py@@ -2,15 +2,18 @@from __future__ import annotations+import asynciofrom collections.abc import Awaitable, Callable-from datetime import datetime, timezone+from datetime import UTC, datetime, timedelta, timezonefrom types import SimpleNamespacefrom typing import Anyfrom unittest.mock import AsyncMock, MagicMockimport pytest+from skyvern.forge import app as forge_appfrom skyvern.forge.sdk.artifact.models import ArtifactType+from skyvern.forge.sdk.copilot.active_run_session import ActiveRunSessionAssociationfrom skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcomefrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (@@ -23,6 +26,12 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import ()from skyvern.forge.sdk.copilot.request_policy import CompletionCriterionfrom skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module+from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml as process_workflow_yaml+from skyvern.forge.sdk.schemas.organizations import Organization+from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, WorkflowParameter+from skyvern.schemas.workflows import BlockType+from skyvern.services import workflow_service as workflow_service_moduleDISPATCHED_LOGIN_GATE_HTML = ("<html><head><title>Sign in</title></head><body><main>"@@ -111,6 +120,215 @@ def stub_artifact_app(return retrieved_ids+def _fake_workflow_run(status: str) -> SimpleNamespace:+ return SimpleNamespace(+ status=status,+ modified_at=datetime(2026, 4, 21, 12, 0, 0, tzinfo=timezone.utc),+ browser_session_id=None,+ )+++async def install_run_blocks_harness(+ monkeypatch: pytest.MonkeyPatch,+ *,+ workflow_yaml: str,+ polled_status: str,+ dispatch_to_worker: bool = False,+ terminal_blocks: list[WorkflowRunBlock] | None = None,+) -> dict[str, Any]:+ """Stub the collaborators an inline ``_run_blocks_and_collect_debug`` call reaches, with the+ polled run parked on ``polled_status`` so the watchdog decides the exit."""+ workflow = await process_workflow_yaml(+ settings_fallback_yaml="enable_self_healing: false",+ workflow_id="w_source",+ workflow_permanent_id="wfp-1",+ organization_id="org-1",+ workflow_yaml=workflow_yaml,+ )+ now = datetime.now(timezone.utc)+ organization = Organization(+ organization_id="org-1",+ organization_name="Test Org",+ created_at=now,+ modified_at=now,+ )+ captured: dict[str, Any] = {"workflow": workflow, "executor_cancelled": False}++ database = MagicMock()+ database.workflows.get_workflow_by_permanent_id = AsyncMock(return_value=workflow)+ database.organizations.get_organization = AsyncMock(return_value=organization)+ persisted_output_params = [p for p in workflow.workflow_definition.parameters if isinstance(p, OutputParameter)]+ persisted_workflow_params = [p for p in workflow.workflow_definition.parameters if isinstance(p, WorkflowParameter)]+ database.workflow_params.get_workflow_output_parameters = AsyncMock(return_value=persisted_output_params)+ database.observer.get_workflow_run_blocks = AsyncMock(return_value=terminal_blocks or [])+ database.workflow_runs.get_workflow_run = AsyncMock(return_value=_fake_workflow_run(status=polled_status))+ monkeypatch.setattr(forge_app, "DATABASE", database)++ async def _execute_workflow(**_kwargs: Any) -> None:+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ captured["executor_cancelled"] = True+ raise++ workflow_service = MagicMock()+ workflow_service.get_workflow_parameters = AsyncMock(return_value=persisted_workflow_params)+ workflow_service.execute_workflow = AsyncMock(side_effect=_execute_workflow)+ workflow_service.create_copilot_dispatch_draft_version = AsyncMock(return_value=workflow)+ monkeypatch.setattr(forge_app, "WORKFLOW_SERVICE", workflow_service)+ monkeypatch.setattr(+ forge_app.AGENT_FUNCTION,+ "should_dispatch_copilot_block_run_to_worker",+ AsyncMock(return_value=dispatch_to_worker),+ )+ monkeypatch.setattr(+ forge_app.AGENT_FUNCTION,+ "allow_copilot_inline_code_execution",+ MagicMock(return_value=False),+ )++ workflow_run = SimpleNamespace(+ workflow_run_id="wr_paused",+ workflow_id="w_source",+ sequential_credential_id=None,+ )+ monkeypatch.setattr(workflow_service_module, "prepare_workflow", AsyncMock(return_value=workflow_run))++ polled_run = _fake_workflow_run(status=polled_status)++ async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, Any, Any]:+ return polled_run, now, now++ monkeypatch.setattr(run_execution_module, "_read_progress_sources", _read_progress)+ monkeypatch.setattr(run_execution_module, "RUN_BLOCKS_POLL_INTERVAL_SECONDS", 0)+ monkeypatch.setattr(run_execution_module, "_fallback_page_info", AsyncMock(return_value=("", "")))++ association = ActiveRunSessionAssociation(+ organization_id="org-1",+ workflow_permanent_id="wfp-1",+ debug_browser_session_id="pbs_chat",+ run_browser_session_id="pbs_run",+ workflow_run_id="wr_paused",+ turn_id="turn-1",+ generation="gen-1",+ expires_at=now + timedelta(minutes=5),+ )+ captured["publish"] = AsyncMock(return_value=association)+ captured["clear"] = AsyncMock(return_value=True)+ captured["cancel_run_task"] = AsyncMock(return_value=None)+ captured["cooperative_cancel"] = AsyncMock(return_value=None)+ monkeypatch.setattr(run_execution_module, "publish_active_run_session", captured["publish"])+ monkeypatch.setattr(run_execution_module, "clear_active_run_session", captured["clear"])+ monkeypatch.setattr(run_execution_module, "_cancel_run_task_if_not_final", captured["cancel_run_task"])+ monkeypatch.setattr(run_execution_module, "_cooperative_cancel_dispatched_run", captured["cooperative_cancel"])+ if dispatch_to_worker:+ captured["worker_execute"] = AsyncMock(return_value=None)+ monkeypatch.setattr(+ run_execution_module.AsyncExecutorFactory,+ "get_executor",+ MagicMock(return_value=SimpleNamespace(execute_workflow=captured["worker_execute"])),+ )+ monkeypatch.setattr(run_execution_module, "_delete_dispatch_draft_if_run_final", AsyncMock(return_value=None))+ monkeypatch.setattr(+ run_execution_module, "_capture_dispatched_terminal_page_evidence", AsyncMock(return_value=None)+ )+ monkeypatch.setattr(+ run_execution_module, "_attach_registered_output_parameter_values", AsyncMock(return_value={})+ )+ return captured+++HANDBACK_WORKFLOW_YAML = """+title: extraction example+workflow_definition:+ parameters: []+ blocks:+ - block_type: extraction+ label: extract_heading+ url: https://example.com+ data_extraction_goal: Extract the page heading.+"""+++def terminal_extraction_block(status: str) -> WorkflowRunBlock:+ return WorkflowRunBlock(+ label="extract_heading",+ block_type=BlockType.EXTRACTION,+ status=status,+ failure_reason=(+ 'Timeout exceeded: waiting for locator("#heading") to be visible' if status == "failed" else None+ ),+ workflow_run_block_id="wrb_extract_heading",+ workflow_run_id="wr_paused",+ organization_id="org-1",+ created_at=datetime(2026, 4, 21, 12, 5, tzinfo=UTC),+ modified_at=datetime(2026, 4, 21, 12, 5, tzinfo=UTC),+ )+++def page_only_failed_block() -> WorkflowRunBlock:+ """A failed block with no failure_reason, so the post-run page is its only structural signal."""+ return WorkflowRunBlock(+ label="extract_heading",+ block_type=BlockType.EXTRACTION,+ status="failed",+ failure_reason=None,+ workflow_run_block_id="wrb_extract_heading",+ workflow_run_id="wr_paused",+ organization_id="org-1",+ created_at=datetime(2026, 4, 21, 12, 5, tzinfo=UTC),+ modified_at=datetime(2026, 4, 21, 12, 5, tzinfo=UTC),+ )+++def same_run_page_evidence() -> dict[str, object]:+ return {+ "observed_after_workflow_run": True,+ "workflow_run_id": "wr_paused",+ "source_browser_session_id": "pbs_run",+ "current_url": "https://example.com/done",+ "page_title": "Done",+ "inspected_url": "https://example.com/done",+ }+++def count_record_and_send(monkeypatch: pytest.MonkeyPatch) -> dict[str, int]:+ counts = {"record": 0, "send": 0}+ real_record = run_execution_module.record_build_test_outcome+ real_send = run_execution_module._send_run_outcome_update++ def _record(ctx: object, outcome: object) -> None:+ counts["record"] += 1+ real_record(ctx, outcome)++ async def _send(*args: object, **kwargs: object) -> None:+ counts["send"] += 1+ await real_send(*args, **kwargs)++ monkeypatch.setattr(run_execution_module, "record_build_test_outcome", _record)+ monkeypatch.setattr(run_execution_module, "_send_run_outcome_update", _send)+ return counts+++async def handback_ctx(+ monkeypatch: pytest.MonkeyPatch,+ *,+ polled_status: str,+ block_status: str,+ terminal_blocks: list[WorkflowRunBlock] | None = None,+) -> CopilotContext:+ harness = await install_run_blocks_harness(+ monkeypatch,+ workflow_yaml=HANDBACK_WORKFLOW_YAML,+ polled_status=polled_status,+ terminal_blocks=terminal_blocks or [terminal_extraction_block(block_status)],+ )+ ctx = make_copilot_ctx(browser_session_id="pbs_chat")+ ctx.staged_workflow = harness["workflow"]+ ctx.frontier_resume_session_id = "pbs_run"+ return ctx++def make_copilot_ctx(**overrides: object) -> CopilotContext:defaults: dict[str, object] = dict(organization_id="org-1",diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.pyindex 62b287778..9b86ddb1b 100644--- a/tests/unit/test_copilot_build_test_outcome.py+++ b/tests/unit/test_copilot_build_test_outcome.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport copyimport jsonfrom datetime import UTC, datetime@@ -17,6 +18,7 @@ from skyvern.forge.sdk.copilot.agent import (_prior_run_debug_text,_recorded_build_test_outcome_prompt,)+from skyvern.forge.sdk.copilot.build_test_connect_failure import build_test_connect_failure_sentencefrom skyvern.forge.sdk.copilot.build_test_outcome import (BuildTestConnectFailure,BuildTestEvidencePacket,@@ -42,7 +44,10 @@ from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificafrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfigfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContextfrom skyvern.forge.sdk.copilot.failure_tracking import selector_identity_from_failure-from skyvern.forge.sdk.copilot.output_utils import project_build_test_packet_for_llm+from skyvern.forge.sdk.copilot.output_utils import (+ _INTERNAL_RUN_OUTCOME_RECORDED_KEY,+ project_build_test_packet_for_llm,+)from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.runtime_authoring_repair import inject_runtime_authoring_repair_contextfrom skyvern.forge.sdk.copilot.secret_scrub import clear_session_scrub_values, register_secret_scrub_value@@ -55,6 +60,8 @@ from skyvern.forge.sdk.copilot.tools.run_execution import (_first_failed_result,_record_run_blocks_result,_recorded_run_block_result,+ _run_blocks_and_collect_debug,+ _verify_and_record_run_blocks_result,build_test_evidence_packet,)from skyvern.forge.sdk.copilot.workflow_yaml import runner_code_block_associations@@ -63,10 +70,14 @@ from skyvern.forge.sdk.workflow.models.block import CodeBlockfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterTypefrom skyvern.webeye.browser_artifacts import BrowserArtifactsfrom tests.unit.copilot_test_helpers import (+ count_record_and_send,failed_second_factor_run,+ handback_ctx,make_copilot_ctx,make_stub_html_artifact,+ page_only_failed_block,passing_run,+ same_run_page_evidence,straight_line_login_yaml,two_page_login_yaml,)@@ -173,6 +184,7 @@ def test_connect_failure_projects_through_recorded_outcome_and_packet() -> None:outcome = recorded_outcome_from_run_blocks_result(result)assert outcome is not Noneassert outcome.connect_failure == failure+ assert outcome.observed_evidence_summary == build_test_connect_failure_sentence(failure)packet = build_test_evidence_packet(ctx, result, recorded_outcome=outcome)assert packet.failure is not Noneassert packet.failure.connect_failure == failure@@ -4629,3 +4641,191 @@ async def test_the_repaired_block_executes_and_the_run_owns_the_outputs_it_was_aif output["output_parameter_key"] == "collect_payment_options_output"]assert run_owned == [after_output], "the corrected run does not hand back the output its code produced"+++async def test_completed_run_is_recorded_before_the_browser_enrichment_await(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")+ order: list[str] = []+ real_record = run_execution_module.record_build_test_outcome++ def _record(record_ctx: object, outcome: object) -> None:+ order.append("record")+ real_record(record_ctx, outcome)++ async def _enrichment(*_args: object, **_kwargs: object) -> tuple[str, dict[str, object] | None]:+ order.append("enrichment")+ return "", None++ monkeypatch.setattr(run_execution_module, "record_build_test_outcome", _record)+ monkeypatch.setattr(run_execution_module, "_attach_post_run_browser_enrichment", _enrichment)++ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)++ assert result["ok"] is True, result+ assert order == ["record", "enrichment"]+ assert ctx.latest_recorded_build_test_outcome is not None+ assert ctx.latest_recorded_build_test_outcome.workflow_run_id == "wr_paused"+++async def test_failed_run_still_records_the_failure_and_marks_the_post_run_page(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = await handback_ctx(monkeypatch, polled_status="failed", block_status="failed")+ marks: list[str] = []++ async def _enrichment(*_args: object, **_kwargs: object) -> tuple[str, dict[str, object] | None]:+ return "", None++ monkeypatch.setattr(run_execution_module, "_attach_post_run_browser_enrichment", _enrichment)+ monkeypatch.setattr(+ run_execution_module,+ "_mark_stored_post_run_failure_page",+ lambda _ctx: marks.append("marked"),+ )+ counts = count_record_and_send(monkeypatch)++ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)+ await _verify_and_record_run_blocks_result(ctx, result, 0.0)++ assert result["ok"] is False, result+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.workflow_run_id == "wr_paused"+ assert outcome.verdict == "repairable_failure"+ assert outcome.reason_code != ""+ assert marks == ["marked"]+ assert counts == {"record": 1, "send": 1}+ assert len(ctx.recorded_build_test_outcome_history) == 1+++async def test_enrichment_facts_still_land_on_the_recorded_outcome_and_agree_with_history(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")+ evidence = same_run_page_evidence()++ async def _enrichment(+ enrich_ctx: CopilotContext, result_data: dict[str, object], **_kwargs: object+ ) -> tuple[str, dict[str, object]]:+ enrich_ctx.composition_page_evidence = evidence+ result_data["current_url"] = "https://example.com/done"+ result_data["post_run_page_evidence"] = evidence+ result_data["post_run_page_capture"] = {"status": "captured"}+ return "https://example.com/done", evidence++ monkeypatch.setattr(run_execution_module, "_attach_post_run_browser_enrichment", _enrichment)+ monkeypatch.setattr(+ run_execution_module.app.AGENT_FUNCTION,+ "captcha_solving_available",+ AsyncMock(return_value=True),+ )+ counts = count_record_and_send(monkeypatch)++ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)+ await _verify_and_record_run_blocks_result(ctx, result, 0.0)++ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert ctx.captcha_solver_available is True+ assert ctx.captcha_solver_available_for_url == "https://example.com/done"+ assert outcome.page_evidence_refs, outcome+ assert outcome.page_capture is not None and outcome.page_capture.status == "captured"+ assert counts == {"record": 1, "send": 1}+ assert len(ctx.recorded_build_test_outcome_history) == 1+ entry = ctx.recorded_build_test_outcome_history[-1]+ assert entry["workflow_run_id"] == outcome.workflow_run_id+ assert entry["verdict"] == outcome.verdict+ assert entry["reason_code"] == outcome.reason_code+ assert entry["structural_key"] == outcome.structural_key+ assert outcome.is_authoritative is True+ assert entry["is_authoritative"] is True+ assert ctx.recorded_persisted_block_run_workflow_run_id == outcome.workflow_run_id+++async def test_failed_run_known_only_by_its_page_is_graded_after_enrichment(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A failed block carrying no failure_reason has no structural identity until the post-run page+ arrives, so grading it before enrichment records nothing at all."""+ ctx = await handback_ctx(+ monkeypatch,+ polled_status="failed",+ block_status="failed",+ terminal_blocks=[page_only_failed_block()],+ )+ evidence = same_run_page_evidence()++ async def _enrichment(+ enrich_ctx: CopilotContext, result_data: dict[str, object], **_kwargs: object+ ) -> tuple[str, dict[str, object]]:+ enrich_ctx.composition_page_evidence = evidence+ result_data["post_run_page_evidence"] = evidence+ return "https://example.com/done", evidence++ monkeypatch.setattr(run_execution_module, "_attach_post_run_browser_enrichment", _enrichment)+ counts = count_record_and_send(monkeypatch)++ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)+ await _verify_and_record_run_blocks_result(ctx, result, 0.0)++ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None, "the run's failure was lost because the page was unknown at record time"+ assert outcome.verdict == "repairable_failure", outcome+ assert outcome.page_evidence_refs, outcome+ assert outcome.key_provenance.get("structural_failure_identity") != (+ "no typed verification/page/output identity available"+ )+ assert counts["record"] == 1+ assert len(ctx.recorded_build_test_outcome_history) == 1+ entry = ctx.recorded_build_test_outcome_history[-1]+ assert entry["verdict"] == outcome.verdict+ assert entry["reason_code"] == outcome.reason_code+ assert entry["structural_key"] == outcome.structural_key+ assert entry["is_authoritative"] == outcome.is_authoritative+++async def test_watchdog_paused_result_is_recorded_before_the_captcha_probe(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = await handback_ctx(monkeypatch, polled_status="paused", block_status="completed")+ record_build_test_outcome(ctx, failed_second_factor_run("wr_prior_failure"))+ counts = count_record_and_send(monkeypatch)+ observed_at_probe: list[RecordedBuildTestOutcome | None] = []++ async def _cancelled_probe(*_args: object, **_kwargs: object) -> bool:+ observed_at_probe.append(ctx.latest_recorded_build_test_outcome)+ raise asyncio.CancelledError++ monkeypatch.setattr(+ run_execution_module.app.AGENT_FUNCTION,+ "captcha_solving_available",+ _cancelled_probe,+ )++ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)++ assert result["ok"] is False, result+ assert result["data"]["control_signal"]["kind"] == "watchdog_paused"+ assert _INTERNAL_RUN_OUTCOME_RECORDED_KEY not in result++ with pytest.raises(asyncio.CancelledError):+ await _verify_and_record_run_blocks_result(ctx, result, 0.0)++ assert len(observed_at_probe) == 1+ at_probe = observed_at_probe[0]+ assert at_probe is not None and at_probe.workflow_run_id == "wr_paused"+ assert at_probe.verdict == "not_authoritative"+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None and outcome.workflow_run_id == "wr_paused"+ assert counts == {"record": 1, "send": 1}+ assert [entry["workflow_run_id"] for entry in ctx.recorded_build_test_outcome_history] == [+ "wr_prior_failure",+ "wr_paused",+ ]diff --git a/tests/unit/test_copilot_runner_deadline.py b/tests/unit/test_copilot_runner_deadline.pyindex d32230ac7..d822db493 100644--- a/tests/unit/test_copilot_runner_deadline.py+++ b/tests/unit/test_copilot_runner_deadline.py@@ -5,12 +5,16 @@ from __future__ import annotationsimport asyncioimport contextlibimport time+from collections.abc import Awaitable, Callablefrom typing import Any, castfrom unittest.mock import AsyncMock, MagicMockimport pytestfrom structlog.testing import capture_logs+from skyvern.forge.sdk.copilot.build_test_outcome import record_build_test_outcome+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy+from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.enforcement import (TOTAL_TIMEOUT_SECONDS,CopilotTotalTimeoutError,@@ -22,6 +26,13 @@ from skyvern.forge.sdk.copilot.pending_operation import (pending_operation,pending_operation_fields,)+from skyvern.forge.sdk.copilot.tools import run_execution+from skyvern.forge.sdk.copilot.tools.run_execution import _run_blocks_and_collect_debug+from tests.unit.copilot_test_helpers import (+ count_record_and_send,+ failed_second_factor_run,+ handback_ctx,+)def _fake_result() -> MagicMock:@@ -470,3 +481,179 @@ async def test_the_deadline_path_is_unchanged_when_the_fingerprint_contributes_nwith capture_logs() as already_marked:_mark_copilot_total_timeout(ctx, elapsed_seconds=99.0, iteration=1)assert _deadline_events(already_marked) == []+++async def _run_with_failing_enrichment(+ monkeypatch: pytest.MonkeyPatch,+ enrichment: Callable[..., Awaitable[tuple[str, dict[str, object] | None]]],+) -> tuple[CopilotContext, dict[str, int]]:+ ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")+ record_build_test_outcome(ctx, failed_second_factor_run("wr_prior_failure"))+ counts = count_record_and_send(monkeypatch)+ monkeypatch.setattr(run_execution, "_attach_post_run_browser_enrichment", enrichment)+ return ctx, counts+++def _assert_later_run_survived(ctx: CopilotContext, counts: dict[str, int]) -> None:+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.workflow_run_id == "wr_paused"+ assert outcome.verdict != "repairable_failure"+ assert counts["record"] == 1+ assert [entry["workflow_run_id"] for entry in ctx.recorded_build_test_outcome_history] == [+ "wr_prior_failure",+ "wr_paused",+ ]+++async def test_completed_run_survives_an_enrichment_probe_that_raises(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def _raising(*_args: object, **_kwargs: object) -> tuple[str, dict[str, object] | None]:+ raise RuntimeError("post-run browser probe failed")++ ctx, counts = await _run_with_failing_enrichment(monkeypatch, _raising)++ with pytest.raises(RuntimeError, match="post-run browser probe failed"):+ await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)++ _assert_later_run_survived(ctx, counts)+++async def test_completed_run_survives_an_enrichment_probe_cancelled_by_the_deadline(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ cancelled: list[str] = []++ async def _hanging(*_args: object, **_kwargs: object) -> tuple[str, dict[str, object] | None]:+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ cancelled.append("cancelled")+ raise+ return "", None++ ctx, counts = await _run_with_failing_enrichment(monkeypatch, _hanging)++ with pytest.raises(asyncio.TimeoutError):+ await asyncio.wait_for(+ _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx),+ timeout=0.05,+ )++ assert cancelled == ["cancelled"]+ _assert_later_run_survived(ctx, counts)+++async def test_bot_wall_seen_only_by_the_post_run_page_still_fails_the_run(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Every block reports completed and the run output alone looks clean; the challenge is visible+ only in the page evidence that arrives after the outcome is committed."""+ ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")+ challenge_evidence = {+ "observed_after_workflow_run": True,+ "workflow_run_id": "wr_paused",+ "source_browser_session_id": "pbs_run",+ "current_url": "https://example.com/verify",+ "inspected_url": "https://example.com/verify",+ "anti_bot_indicators": ["captcha"],+ "challenge_state": {+ "detected": True,+ "kind": "captcha",+ "requires_human_verification": True,+ },+ "message": "Please complete the CAPTCHA to continue",+ }++ async def _enrichment(+ enrich_ctx: CopilotContext, result_data: dict[str, object], **_kwargs: object+ ) -> tuple[str, dict[str, object]]:+ enrich_ctx.composition_page_evidence = challenge_evidence+ result_data["post_run_page_evidence"] = challenge_evidence+ return "https://example.com/verify", challenge_evidence++ monkeypatch.setattr(run_execution, "_attach_post_run_browser_enrichment", _enrichment)++ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)++ assert result["ok"] is False, "a run that ended on a bot wall was reported as passing"+ assert ctx.last_test_anti_bot, ctx.last_test_anti_bot+ assert ctx.last_full_workflow_test_ok is False+ # The emitted envelope must agree with the failure the caller returns.+ assert ctx.last_run_outcome is not None+ assert ctx.last_run_outcome.verdict == "not_demonstrated", ctx.last_run_outcome+ assert ctx.last_run_outcome.run_completed is False+ # The success-branch latches must not survive a run the settle turned into a failure.+ assert ctx.verified_terminal_proposal_ready is False+++async def test_prior_run_page_evidence_does_not_fail_this_run(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A challenge page left on ctx by an earlier run must not grade this one: the capture that drops+ another run's page runs after the record, and the anti-bot read has no run-id check."""+ ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")+ ctx.composition_page_evidence = {+ "observed_after_workflow_run": True,+ "workflow_run_id": "wr_some_older_run",+ "source_browser_session_id": "pbs_run",+ "current_url": "https://example.com/verify",+ "anti_bot_indicators": ["captcha"],+ "challenge_state": {"detected": True, "kind": "captcha", "requires_human_verification": True},+ "message": "Please complete the CAPTCHA to continue",+ }++ async def _enrichment(+ _enrich_ctx: CopilotContext, _result_data: dict[str, object], **_kwargs: object+ ) -> tuple[str, dict[str, object] | None]:+ return "https://example.com/done", None++ monkeypatch.setattr(run_execution, "_attach_post_run_browser_enrichment", _enrichment)++ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)++ assert result["ok"] is True, "another run's page evidence failed this run"+ assert ctx.last_test_anti_bot is None, ctx.last_test_anti_bot+++async def test_enrichment_supplied_page_reaches_the_grade(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Drives the real _attach_post_run_browser_enrichment. Every other test stubs the helper and+ plants the page on ctx first, which cannot tell a page enrichment supplied from one already+ there -- the distinction this ordering is entirely about."""+ ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ assert ctx.composition_page_evidence is None, "the page must arrive from enrichment, not the fixture"++ challenge_page = {+ "source_tool": "inspect_page_for_composition",+ "observed_after_workflow_run": True,+ "workflow_run_id": "wr_paused",+ "current_url": "https://example.com/verify",+ "inspected_url": "https://example.com/verify",+ "anti_bot_indicators": ["captcha"],+ "challenge_state": {"detected": True, "kind": "captcha", "requires_human_verification": True},+ "message": "Please complete the CAPTCHA to continue",+ }++ async def _page_info(*_args: object, **_kwargs: object) -> tuple[str, str | None, str | None]:+ return "https://example.com/verify", "Verify you are human", None++ async def _read_evidence(*_args: object, **_kwargs: object) -> tuple[dict[str, object], str, None, None]:+ return challenge_page, "pbs_run", None, None++ monkeypatch.setattr(run_execution, "_resolve_post_run_page_info", _page_info)+ monkeypatch.setattr(run_execution, "_read_run_session_page_evidence", _read_evidence)++ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)++ assert ctx.composition_page_evidence is not None, "the real enrichment never stored the page"+ assert result["ok"] is False, "a page supplied by enrichment did not reach the grade"+ assert ctx.last_test_anti_bot, ctx.last_test_anti_botdiff --git a/tests/unit/test_copilot_terminal_envelope.py b/tests/unit/test_copilot_terminal_envelope.pyindex b92f88eab..8da2aa191 100644--- a/tests/unit/test_copilot_terminal_envelope.py+++ b/tests/unit/test_copilot_terminal_envelope.py@@ -1,16 +1,20 @@from __future__ import annotationsimport time+from datetime import UTC, datetimefrom types import SimpleNamespacefrom typing import Any+from unittest.mock import AsyncMock, MagicMockimport pytestimport structlog.testing+from skyvern.exceptions import get_user_facing_exception_messagefrom skyvern.forge.sdk.copilot import agent as agent_modulefrom skyvern.forge.sdk.copilot.blocker_signal import (CopilotToolBlockerSignal,)+from skyvern.forge.sdk.copilot.build_test_connect_failure import build_test_connect_failure_sentencefrom skyvern.forge.sdk.copilot.build_test_outcome import BuildTestConnectFailure, BuildTestFailedOperationfrom skyvern.forge.sdk.copilot.config import CopilotConfigfrom skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome@@ -23,10 +27,130 @@ from skyvern.forge.sdk.copilot.terminal_envelope import (reason_in_reply_shadow,render_terminal_message,)-from skyvern.forge.sdk.copilot.tools.run_execution import _stash_recorded_run_outcome+from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module+from skyvern.forge.sdk.copilot.tools.run_execution import (+ _record_run_blocks_result,+ _stamp_run_side_connect_failure,+ _stash_recorded_run_outcome,+)from skyvern.forge.sdk.copilot.tools.workflow_update import _record_workflow_update_result+from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSessionfrom tests.unit.copilot_test_helpers import make_copilot_ctx+RUN_SIDE_SHARED_REASON = get_user_facing_exception_message(+ Exception("connect_over_cdp failed: WebSocket error: connection closed")+)+RUN_SIDE_TYPED_REASON = build_test_connect_failure_sentence(+ BuildTestConnectFailure(state="already_closed", workflow_run_id="wr_run_side")+)+++def _run_side_failed_result(+ *, failure_reason: str = RUN_SIDE_SHARED_REASON, reason_code: str | None = None+) -> dict[str, Any]:+ data: dict[str, Any] = {+ "workflow_run_id": "wr_run_side",+ "browser_session_id": "pbs_run_side",+ "overall_status": "failed",+ "blocks": [],+ "failure_reason": failure_reason,+ }+ if reason_code is not None:+ data["failure_category"] = [{"category": "BROWSER_ERROR", "confidence_float": 1.0, "reason_code": reason_code}]+ return {"ok": False, "error": failure_reason, "data": data}+++def _install_session_record(monkeypatch: pytest.MonkeyPatch, session: PersistentBrowserSession | None) -> None:+ mock_app = MagicMock()+ mock_app.DATABASE.browser_sessions.get_persistent_browser_session = AsyncMock(return_value=session)+ monkeypatch.setattr(run_execution_module, "app", mock_app)+++def _session_record(*, closed: bool) -> PersistentBrowserSession:+ now = datetime.now(UTC)+ return PersistentBrowserSession(+ persistent_browser_session_id="pbs_run_side",+ organization_id="org-1",+ status="completed" if closed else "running",+ completed_at=now if closed else None,+ created_at=now,+ modified_at=now,+ )+++def test_a_persisted_session_closed_reason_types_the_stop_without_touching_the_runs_own_text() -> None:+ result = _run_side_failed_result(reason_code="browser_session_closed")++ sentence = _stamp_run_side_connect_failure(make_copilot_ctx(), result)++ assert sentence == RUN_SIDE_TYPED_REASON+ assert result["data"]["build_test_connect_failure"]["state"] == "already_closed"+ assert result["error"] == RUN_SIDE_SHARED_REASON+ assert result["data"]["failure_reason"] == RUN_SIDE_SHARED_REASON+++def test_a_persisted_startup_timeout_types_provisioning_unavailable() -> None:+ result = _run_side_failed_result(reason_code="browser_session_startup_timeout")++ sentence = _stamp_run_side_connect_failure(make_copilot_ctx(), result)++ assert sentence is not None+ assert result["data"]["build_test_connect_failure"]["state"] == "provisioning_unavailable"+++def test_a_run_that_failed_for_its_own_reason_stays_untyped_however_its_session_row_closed(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A mistyped block label fails before any block runs and its session row closes normally+ afterwards; the run persisted no browser reason code, so the author must see the typo, not+ a browser-loss retry."""+ _install_session_record(monkeypatch, _session_record(closed=True))+ typo = "Unable to find block with label extract_invoice_total"+ result = _run_side_failed_result(failure_reason=typo)++ sentence = _stamp_run_side_connect_failure(make_copilot_ctx(), result)++ assert sentence is None+ assert result["data"].get("build_test_connect_failure") is None+ assert result["error"] == typo+ assert result["data"]["failure_reason"] == typo+++async def test_a_typed_run_side_stop_keeps_the_shared_prose_out_of_the_chat_reply(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _install_session_record(monkeypatch, _session_record(closed=True))+ ctx = make_copilot_ctx()+ ctx.last_update_block_count = 2+ result = _run_side_failed_result(reason_code="browser_session_closed")++ sentence = _stamp_run_side_connect_failure(ctx, result)+ recorded = _record_run_blocks_result(ctx, result, connect_failure_reason=sentence)+ reply = agent_module._rewrite_failed_test_response("The test failed.", ctx)++ assert recorded is not None and recorded.display_reason == RUN_SIDE_TYPED_REASON+ assert RUN_SIDE_TYPED_REASON in reply+ assert RUN_SIDE_SHARED_REASON not in reply+ assert "high demand" not in reply.lower()+++async def test_a_typed_run_side_stop_keeps_the_shared_prose_out_of_the_recorded_summary(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _install_session_record(monkeypatch, _session_record(closed=True))+ ctx = make_copilot_ctx()+ result = _run_side_failed_result(reason_code="browser_session_closed")++ sentence = _stamp_run_side_connect_failure(ctx, result)+ _record_run_blocks_result(ctx, result, connect_failure_reason=sentence)+ reason, _ = agent_module._recorded_failure_summary(ctx)++ assert reason and reason in RUN_SIDE_TYPED_REASON+ assert RUN_SIDE_SHARED_REASON not in reason+ assert "high demand" not in reason.lower()+def _run_outcome(verdict: str, display_reason: str | None = None) -> RecordedRunOutcome:return RecordedRunOutcome(verdict=verdict, display_reason=display_reason)diff --git a/tests/unit/test_copilot_watchdog.py b/tests/unit/test_copilot_watchdog.pyindex a82ec4296..265c34ab9 100644--- a/tests/unit/test_copilot_watchdog.py+++ b/tests/unit/test_copilot_watchdog.py@@ -36,7 +36,6 @@ from unittest.mock import AsyncMock, MagicMockimport pytestfrom skyvern.forge import app as forge_app-from skyvern.forge.sdk.copilot.active_run_session import ActiveRunSessionAssociationfrom skyvern.forge.sdk.copilot.blocker_signal import (assert_clean_user_facing_text,contains_internal_machinery_leak,@@ -58,12 +57,9 @@ from skyvern.forge.sdk.copilot.tools.run_execution import (_watchdog_user_facing_summary,)from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin-from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml-from skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock-from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, WorkflowParameterfrom skyvern.schemas.workflows import BlockType-from skyvern.services import workflow_service as workflow_service_module+from tests.unit.copilot_test_helpers import install_run_blocks_harness as _install_run_harnessfrom tests.unit.copilot_test_helpers import make_copilot_ctx@@ -591,112 +587,6 @@ workflow_definition:"""-async def _install_run_harness(- monkeypatch: pytest.MonkeyPatch,- *,- workflow_yaml: str,- polled_status: str,- dispatch_to_worker: bool = False,- terminal_blocks: list[WorkflowRunBlock] | None = None,-) -> dict[str, Any]:- """Stub the collaborators an inline ``_run_blocks_and_collect_debug`` call reaches, with the- polled run parked on ``polled_status`` so the watchdog decides the exit."""- workflow = await _process_workflow_yaml(- settings_fallback_yaml="enable_self_healing: false",- workflow_id="w_source",- workflow_permanent_id="wfp-1",- organization_id="org-1",- workflow_yaml=workflow_yaml,- )- now = datetime.now(timezone.utc)- organization = Organization(- organization_id="org-1",- organization_name="Test Org",- created_at=now,- modified_at=now,- )- captured: dict[str, Any] = {"workflow": workflow, "executor_cancelled": False}-- database = MagicMock()- database.workflows.get_workflow_by_permanent_id = AsyncMock(return_value=workflow)- database.organizations.get_organization = AsyncMock(return_value=organization)- persisted_output_params = [p for p in workflow.workflow_definition.parameters if isinstance(p, OutputParameter)]- persisted_workflow_params = [p for p in workflow.workflow_definition.parameters if isinstance(p, WorkflowParameter)]- database.workflow_params.get_workflow_output_parameters = AsyncMock(return_value=persisted_output_params)- database.observer.get_workflow_run_blocks = AsyncMock(return_value=terminal_blocks or [])- database.workflow_runs.get_workflow_run = AsyncMock(return_value=_fake_run(status=polled_status))- monkeypatch.setattr(forge_app, "DATABASE", database)-- async def _execute_workflow(**_kwargs: Any) -> None:- try:- await asyncio.Event().wait()- except asyncio.CancelledError:- captured["executor_cancelled"] = True- raise-- workflow_service = MagicMock()- workflow_service.get_workflow_parameters = AsyncMock(return_value=persisted_workflow_params)- workflow_service.execute_workflow = AsyncMock(side_effect=_execute_workflow)- workflow_service.create_copilot_dispatch_draft_version = AsyncMock(return_value=workflow)- monkeypatch.setattr(forge_app, "WORKFLOW_SERVICE", workflow_service)- monkeypatch.setattr(- forge_app.AGENT_FUNCTION,- "should_dispatch_copilot_block_run_to_worker",- AsyncMock(return_value=dispatch_to_worker),- )- monkeypatch.setattr(- forge_app.AGENT_FUNCTION,- "allow_copilot_inline_code_execution",- MagicMock(return_value=False),- )-- workflow_run = SimpleNamespace(- workflow_run_id="wr_paused",- workflow_id="w_source",- sequential_credential_id=None,- )- monkeypatch.setattr(workflow_service_module, "prepare_workflow", AsyncMock(return_value=workflow_run))-- polled_run = _fake_run(status=polled_status)-- async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, Any, Any]:- return polled_run, now, now-- monkeypatch.setattr(run_execution, "_read_progress_sources", _read_progress)- monkeypatch.setattr(run_execution, "RUN_BLOCKS_POLL_INTERVAL_SECONDS", 0)- monkeypatch.setattr(run_execution, "_fallback_page_info", AsyncMock(return_value=("", "")))-- association = ActiveRunSessionAssociation(- organization_id="org-1",- workflow_permanent_id="wfp-1",- debug_browser_session_id="pbs_chat",- run_browser_session_id="pbs_run",- workflow_run_id="wr_paused",- turn_id="turn-1",- generation="gen-1",- expires_at=now + timedelta(minutes=5),- )- captured["publish"] = AsyncMock(return_value=association)- captured["clear"] = AsyncMock(return_value=True)- captured["cancel_run_task"] = AsyncMock(return_value=None)- captured["cooperative_cancel"] = AsyncMock(return_value=None)- monkeypatch.setattr(run_execution, "publish_active_run_session", captured["publish"])- monkeypatch.setattr(run_execution, "clear_active_run_session", captured["clear"])- monkeypatch.setattr(run_execution, "_cancel_run_task_if_not_final", captured["cancel_run_task"])- monkeypatch.setattr(run_execution, "_cooperative_cancel_dispatched_run", captured["cooperative_cancel"])- if dispatch_to_worker:- captured["worker_execute"] = AsyncMock(return_value=None)- monkeypatch.setattr(- run_execution.AsyncExecutorFactory,- "get_executor",- MagicMock(return_value=SimpleNamespace(execute_workflow=captured["worker_execute"])),- )- monkeypatch.setattr(run_execution, "_delete_dispatch_draft_if_run_final", AsyncMock(return_value=None))- monkeypatch.setattr(run_execution, "_capture_dispatched_terminal_page_evidence", AsyncMock(return_value=None))- monkeypatch.setattr(run_execution, "_attach_registered_output_parameter_values", AsyncMock(return_value={}))- return captured--def _adopted_detached_tasks(before: set[Any]) -> list[Any]:return [task for task in run_execution._DETACHED_CLEANUP_TASKS if task not in before]diff --git a/tests/unit/test_workflow_service_cancel.py b/tests/unit/test_workflow_service_cancel.pyindex 70d058ca6..3bfb20de5 100644--- a/tests/unit/test_workflow_service_cancel.py+++ b/tests/unit/test_workflow_service_cancel.py@@ -545,3 +545,16 @@ async def test_shielded_finalize_skipped_when_pre_finally_status_unset() -> Noneawait taskassert not finalize_called, "finalize must not run when pre_finally_status is unset"+++def test_lease_failure_category_persists_only_typed_browser_loss() -> None:+ from skyvern.exceptions import BrowserSessionClosed, BrowserSessionStartupTimeout+ from skyvern.forge.sdk.workflow.service import _browser_lease_failure_category++ closed = _browser_lease_failure_category(BrowserSessionClosed("pbs_x"))+ timeout = _browser_lease_failure_category(BrowserSessionStartupTimeout("pbs_x"))++ assert closed is not None and closed[0]["category"] == "BROWSER_ERROR"+ assert closed[0]["reason_code"] == "browser_session_closed"+ assert timeout is not None and timeout[0]["reason_code"] == "browser_session_startup_timeout"+ assert _browser_lease_failure_category(RuntimeError("a required workflow parameter was not provided")) is None