Skip to content

Automate browser based workflows with AI

SKY-15316: Commit the Copilot build-test run outcome before post-run browser enrichment (#8394)

Andrew Neilsonb6813d6c9d9b
diff --git a/skyvern/forge/sdk/copilot/build_test_connect_failure.py b/skyvern/forge/sdk/copilot/build_test_connect_failure.py
index 2f88282b9..b54471ce8 100644
--- a/skyvern/forge/sdk/copilot/build_test_connect_failure.py
+++ b/skyvern/forge/sdk/copilot/build_test_connect_failure.py
@@ -18,3 +18,11 @@ class BuildTestConnectFailure(BaseModel):
task_id: str | None = None
browser_session_id: str | None = None
retry_action: Literal["test_end_to_end"] = "test_end_to_end"
+
+
+def build_test_connect_failure_sentence(failure: BuildTestConnectFailure) -> str:
+ """A run id means the session was lost after the run started, so the operator must not be
+ pointed at provisioning."""
+ if failure.workflow_run_id:
+ return f"Build-test browser session was unavailable before any block ran: {failure.state}."
+ return f"Build-test browser acquisition stopped: {failure.state}."
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 9f477e99f..383b74594 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -13,7 +13,10 @@ import structlog
import yaml
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
-from skyvern.forge.sdk.copilot.build_test_connect_failure import BuildTestConnectFailure
+from skyvern.forge.sdk.copilot.build_test_connect_failure import (
+ BuildTestConnectFailure,
+ build_test_connect_failure_sentence,
+)
from skyvern.forge.sdk.copilot.challenge_evidence import (
carrier_backed_anti_bot_categories,
interactive_challenge_controls,
@@ -456,6 +459,30 @@ class _RecordedBuildTestOutcomeContext(Protocol):
recorded_persisted_block_run_workflow_run_id: str | None
+def _history_entry(ctx: _RecordedBuildTestOutcomeContext, outcome: RecordedBuildTestOutcome) -> dict[str, object]:
+ return {
+ "phase": outcome.phase,
+ "reason_code": outcome.reason_code,
+ "verdict": outcome.verdict,
+ "structural_key": outcome.structural_key,
+ "is_authoritative": outcome.is_authoritative,
+ "workflow_run_id": outcome.workflow_run_id,
+ "authored_structure_signature": outcome.authored_structure_signature,
+ "block_labels": list(outcome.block_labels),
+ "attempted_block_label": outcome.attempted_block_label,
+ "attempted_block_signature": _attempted_block_signature(ctx, outcome),
+ "attempted_call_ref": outcome.attempted_call_ref,
+ "code_safety_rejection_facts": [fact.model_dump(mode="json") for fact in outcome.code_safety_rejection_facts],
+ "failed_operation": (
+ outcome.failed_operation.model_dump(mode="json") if outcome.failed_operation is not None else None
+ ),
+ "connect_failure": (
+ outcome.connect_failure.model_dump(mode="json") if outcome.connect_failure is not None else None
+ ),
+ "failed_operation_call_signature": outcome.failed_operation_call_signature,
+ }
+
+
def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: RecordedBuildTestOutcome | None) -> None:
if outcome is None:
latest = getattr(ctx, "latest_recorded_build_test_outcome", None)
@@ -504,31 +531,7 @@ def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: Re
ctx.latest_recorded_build_test_outcome = outcome
raw_history = getattr(ctx, "recorded_build_test_outcome_history", None)
history: list[dict[str, object]] = raw_history if isinstance(raw_history, list) else []
- history.append(
- {
- "phase": outcome.phase,
- "reason_code": outcome.reason_code,
- "verdict": outcome.verdict,
- "structural_key": outcome.structural_key,
- "is_authoritative": outcome.is_authoritative,
- "workflow_run_id": outcome.workflow_run_id,
- "authored_structure_signature": outcome.authored_structure_signature,
- "block_labels": list(outcome.block_labels),
- "attempted_block_label": outcome.attempted_block_label,
- "attempted_block_signature": _attempted_block_signature(ctx, outcome),
- "attempted_call_ref": outcome.attempted_call_ref,
- "code_safety_rejection_facts": [
- fact.model_dump(mode="json") for fact in outcome.code_safety_rejection_facts
- ],
- "failed_operation": (
- outcome.failed_operation.model_dump(mode="json") if outcome.failed_operation is not None else None
- ),
- "connect_failure": (
- outcome.connect_failure.model_dump(mode="json") if outcome.connect_failure is not None else None
- ),
- "failed_operation_call_signature": outcome.failed_operation_call_signature,
- }
- )
+ history.append(_history_entry(ctx, outcome))
del history[:-_HISTORY_LIMIT]
ctx.recorded_build_test_outcome_history = history
if outcome.phase == "persisted_block_run" and outcome.is_authoritative and outcome.workflow_run_id:
@@ -800,6 +803,58 @@ def bind_post_run_page_path_failure(
return True
+def bind_post_run_page_evidence(
+ ctx: _RecordedBuildTestOutcomeContext,
+ data: Mapping[str, object],
+ page_evidence: Mapping[str, object] | None,
+ *,
+ regraded: RecordedBuildTestOutcome | None = None,
+) -> bool:
+ """Settle the outcome for this run now that post-run page evidence exists, replacing the
+ entry recorded before it rather than appending a second one. Page evidence is a grading
+ input rather than a decoration, so ``regraded`` supersedes a verdict reached without it."""
+ run_id = _safe_str(data.get("workflow_run_id"))
+ if not run_id:
+ return False
+ if regraded is not None and (regraded.phase != "persisted_block_run" or regraded.workflow_run_id != run_id):
+ regraded = None
+ latest = ctx.latest_recorded_build_test_outcome
+ if latest is None or latest.phase != "persisted_block_run" or latest.workflow_run_id != run_id:
+ # Grading before enrichment had no page to key on, so this run may have recorded nothing at all.
+ if regraded is None:
+ return False
+ record_build_test_outcome(ctx, regraded)
+ return True
+ graded = page_evidence if _post_run_page_evidence_matches_result(data, page_evidence) else None
+ page_fields: dict[str, object] = {
+ "page_evidence_refs": _page_evidence_refs(graded),
+ "page_capture": post_run_page_capture_from_result(data, graded),
+ "page_path_failure": _post_run_page_path_failure(graded, run_id),
+ "observed_page_value_excerpt": _observed_page_value_excerpt(graded),
+ }
+ if regraded is not None:
+ # record_build_test_outcome resolved these against the executed code; the regrade cannot see it.
+ carried = (
+ {
+ "failed_operation": latest.failed_operation,
+ "failed_operation_call_signature": latest.failed_operation_call_signature,
+ "failed_operation_code_signature": latest.failed_operation_code_signature,
+ }
+ if latest.failed_operation is not None
+ else {}
+ )
+ updated = regraded.model_copy(update={**page_fields, **carried})
+ else:
+ updated = latest.model_copy(update=page_fields)
+ ctx.latest_recorded_build_test_outcome = updated
+ history = ctx.recorded_build_test_outcome_history
+ if history and history[-1].get("workflow_run_id") == run_id:
+ history[-1] = _history_entry(ctx, updated)
+ if updated.is_authoritative:
+ ctx.recorded_persisted_block_run_workflow_run_id = run_id
+ return True
+
+
def authored_structure_signature_from_workflow(
workflow_yaml: str | None,
code_artifact_metadata: object = None,
@@ -1091,7 +1146,7 @@ def recorded_outcome_from_run_blocks_result(
requested_block_labels=requested_block_labels,
structural_failure_identity=f"build_test_connect:{connect_failure.state}",
connect_failure=connect_failure,
- observed_evidence_summary=f"Build-test browser acquisition stopped: {connect_failure.state}.",
+ observed_evidence_summary=build_test_connect_failure_sentence(connect_failure),
key_provenance={"structural_failure_identity": "typed build-test browser acquisition fact"},
)
executed_block_labels = _clean_list(
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 3363e3613..2c11945b5 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -43,6 +43,8 @@ if TYPE_CHECKING:
LOG = structlog.get_logger()
_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY = "_copilot_internal_run_cancelled_by_watchdog"
+_INTERNAL_RUN_OUTCOME_RECORDED_KEY = "_copilot_internal_run_outcome_recorded"
+_INTERNAL_GOAL_PATH_OMISSIONS_KEY = "_copilot_internal_goal_path_omissions"
_BASE64_IMAGE_OMITTED_MESSAGE = "[base64 image omitted — screenshot was taken successfully]"
BUILD_TEST_PACKET_KEY = "build_test_packet"
@@ -1216,6 +1218,8 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict
"timing_ms",
"_workflow",
_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
+ _INTERNAL_RUN_OUTCOME_RECORDED_KEY,
+ _INTERNAL_GOAL_PATH_OMISSIONS_KEY,
),
drop_data_keys=("sdk_equivalent", "authored_locator_observations"),
replacement_fields={"screenshot_base64": _BASE64_IMAGE_OMITTED_MESSAGE},
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 650bc28a7..217003a47 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -30,7 +30,10 @@ from skyvern.cli.core.session_manager import (
)
from skyvern.config import settings
from skyvern.forge import app
-from skyvern.forge.sdk.copilot.build_test_connect_failure import BuildTestConnectFailure
+from skyvern.forge.sdk.copilot.build_test_connect_failure import (
+ BuildTestConnectFailure,
+ build_test_connect_failure_sentence,
+)
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.screenshot_utils import PendingFrameLease, ScreenshotEntry
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
@@ -1225,12 +1228,13 @@ async def _drop_browser_session_id_at_its_fixed_deadline(ctx: AgentContext) -> N
def _build_test_connect_failure_result(failure: BuildTestConnectFailure) -> dict[str, Any]:
+ sentence = build_test_connect_failure_sentence(failure)
return {
"ok": False,
- "error": f"Build-test browser acquisition stopped: {failure.state}.",
+ "error": sentence,
"data": {
"overall_status": "setup_failed",
- "failure_reason": f"Build-test browser acquisition stopped: {failure.state}.",
+ "failure_reason": sentence,
"browser_session_id": failure.browser_session_id,
"build_test_connect_failure": failure.model_dump(mode="json", exclude_none=True),
"blocks": [],
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 4cd574273..c74aa1b9a 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -28,6 +28,11 @@ from skyvern.forge.sdk.copilot.authoring_parameter_binding import _literal_locat
from skyvern.forge.sdk.copilot.blocker_signal import (
stash_blocker_signal,
)
+from skyvern.forge.sdk.copilot.build_test_connect_failure import (
+ BuildTestConnectFailure,
+ BuildTestConnectFailureState,
+ build_test_connect_failure_sentence,
+)
from skyvern.forge.sdk.copilot.build_test_outcome import (
INFRASTRUCTURE_RUNNER_ERROR_CODES,
BuildTestEvidencePacket,
@@ -46,6 +51,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
authored_block_parameter_keys_from_workflow,
authored_structure_signature_from_workflow,
+ bind_post_run_page_evidence,
connect_failure_from_run_blocks_result,
failed_operation_from_run_blocks_result,
post_run_page_capture_from_result,
@@ -96,7 +102,9 @@ from skyvern.forge.sdk.copilot.narration import handler_available as narration_h
from skyvern.forge.sdk.copilot.narration import narrator_poll_tick
from skyvern.forge.sdk.copilot.outcome_verification_trace import record_gate_decision
from skyvern.forge.sdk.copilot.output_utils import (
+ _INTERNAL_GOAL_PATH_OMISSIONS_KEY,
_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
+ _INTERNAL_RUN_OUTCOME_RECORDED_KEY,
BUILD_TEST_PACKET_KEY,
build_run_blocks_response,
iter_failure_reasons,
@@ -2424,6 +2432,141 @@ async def run_workflow_end_to_end(ctx: CopilotContext, workflow_yaml: str) -> di
)
+async def _attach_post_run_browser_enrichment(
+ ctx: CopilotContext,
+ result_data: dict[str, Any],
+ *,
+ workflow: Workflow | None,
+ workflow_run_id: str,
+ run_block_rows: list[WorkflowRunBlock],
+ results: list[dict[str, Any]],
+ block_outputs_by_label: Mapping[str, Any],
+ failed_block_code: str | None,
+ run_session_id: str | None,
+ dispatch_to_worker: bool,
+ sensitive_origin_run: bool,
+ run_ok: bool,
+ used_fresh_run_session: bool,
+ run_detached_from_chat: bool,
+) -> tuple[str, dict[str, Any] | None]:
+ """Probe the browser for post-run facts and stamp them onto an already-recorded run result.
+ The outcome is what is protected, not these awaits: it is committed before this runs, so a probe
+ the deadline cancels costs enrichment and never the record. Only the screenshot capture is
+ guarded here; every other await propagates and fails the tool call."""
+ # Dispatched runs: the worker owns the run session; do not touch it over CDP from the API.
+ # The frontier's anchors and the page the run ended on both come from the rows the worker
+ # persisted instead. A dispatched run has no page title to report.
+ current_url, page_title, dispatched_end_url = await _resolve_post_run_page_info(
+ ctx,
+ run_block_rows=run_block_rows,
+ dispatch_to_worker=dispatch_to_worker,
+ sensitive_origin_run=sensitive_origin_run,
+ run_session_id=run_session_id,
+ )
+
+ screenshot_b64: str | None = None
+ # Dispatched runs: the worker owns the persistent browser session, so the API side must not
+ # grab the live page over CDP. Their at-failure frames come from worker-persisted artifacts.
+ if not dispatch_to_worker and not run_ok and run_session_id and not sensitive_origin_run:
+ try:
+ browser_state = await resolve_persistent_browser_state(
+ session_id=run_session_id,
+ organization_id=ctx.organization_id,
+ )
+ if browser_state:
+ page = await browser_state.get_or_create_page()
+ if SettingsManager.get_settings().BROWSER_CURSOR_VISUALIZATION:
+ try:
+ await SkyvernFrame.hide_cursor_overlay(page)
+ except Exception:
+ pass
+ try:
+ screenshot_bytes = await page.screenshot(type="png")
+ finally:
+ if SettingsManager.get_settings().BROWSER_CURSOR_VISUALIZATION:
+ try:
+ await SkyvernFrame.show_cursor_overlay(page)
+ except Exception:
+ pass
+ screenshot_b64 = base64.b64encode(screenshot_bytes).decode("utf-8")
+ except Exception:
+ LOG.debug("Failed to capture post-run screenshot", exc_info=True)
+
+ locator_observations: list[AuthoredLocatorObservationRow] | None = None
+ post_run_page_capture: BuildTestPacketPageCapture | None = None
+ if (
+ not dispatch_to_worker
+ and run_session_id
+ and _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ and not ctx.copilot_total_timeout_exceeded
+ ):
+ # Structured evidence is admitted through the origin registry scrubber. Pixel capture
+ # and locator probes remain withheld for sensitive runs because they have no equivalent
+ # exact-value disclosure boundary.
+ _pin_pre_run_page_reference(ctx, workflow_run_id)
+ post_run_page_capture = await _capture_and_store_post_run_page(
+ ctx,
+ run_session_id=run_session_id,
+ run_id=workflow_run_id,
+ current_url=current_url,
+ )
+
+ if not sensitive_origin_run:
+ locator_observations = await _observe_authored_locators(
+ ctx,
+ run_session_id=run_session_id,
+ failed_block_code=failed_block_code,
+ worker_owned=dispatch_to_worker,
+ observation_deadline_exceeded=ctx.copilot_total_timeout_exceeded,
+ )
+
+ if not dispatch_to_worker and not ctx.copilot_total_timeout_exceeded:
+ await _capture_registered_artifact_evidence(
+ ctx,
+ run_id=workflow_run_id,
+ organization_id=ctx.organization_id,
+ downloaded_artifact_ids=_collect_downloaded_artifact_ids(block_outputs_by_label),
+ )
+
+ # Dispatched runs are worker-owned, so the API cannot CDP-capture the terminal page; read the
+ # worker-persisted terminal HTML artifact instead and route it through the same post-run sink.
+ if dispatch_to_worker and run_session_id and not ctx.copilot_total_timeout_exceeded:
+ await _capture_dispatched_terminal_page_evidence(
+ ctx,
+ workflow=workflow,
+ run_id=workflow_run_id,
+ run_session_id=run_session_id,
+ organization_id=ctx.organization_id,
+ current_url=current_url,
+ origin_redaction_registry=ctx.origin_run_redaction_registry,
+ )
+
+ result_data["current_url"] = current_url
+ result_data["page_title"] = page_title
+ if locator_observations is not None:
+ result_data["authored_locator_observations"] = locator_observations
+ if not dispatch_to_worker and current_url:
+ result_data["current_url_live_observed"] = True
+ if dispatch_to_worker and dispatched_end_url is None:
+ result_data["current_url_evidence"] = NO_PERSISTED_END_URL
+ post_run_page_evidence = _same_run_page_evidence_for_result(ctx, workflow_run_id)
+ if post_run_page_evidence is not None:
+ result_data["post_run_page_evidence"] = model_visible_composition_evidence(post_run_page_evidence)
+ if post_run_page_capture is not None:
+ result_data["post_run_page_capture"] = post_run_page_capture.model_dump(mode="json")
+ _attach_run_session_facts(
+ result_data,
+ used_fresh_run_session=used_fresh_run_session,
+ run_detached_from_chat=run_detached_from_chat,
+ run_ok=run_ok,
+ page_evidence=post_run_page_evidence,
+ )
+ resolved_screenshot_b64 = _resolve_run_screenshot_b64(live_capture=screenshot_b64, results=results, run_ok=run_ok)
+ if resolved_screenshot_b64 is not None:
+ result_data["screenshot_base64"] = resolved_screenshot_b64
+ return current_url, post_run_page_evidence
+
+
async def _run_blocks_and_collect_debug(
params: dict[str, Any],
ctx: CopilotContext,
@@ -3272,94 +3415,7 @@ async def _run_blocks_and_collect_debug(
for entry in results:
entry.pop("action_trace", None)
- # Dispatched runs: the worker owns the run session; do not touch it over CDP from the API.
- # The frontier's anchors and the page the run ended on both come from the rows the worker
- # persisted instead. A dispatched run has no page title to report.
block_end_urls = {} if sensitive_origin_run else _block_end_urls_by_label(run_block_rows)
- current_url, page_title, dispatched_end_url = await _resolve_post_run_page_info(
- ctx,
- run_block_rows=run_block_rows,
- dispatch_to_worker=dispatch_to_worker,
- sensitive_origin_run=sensitive_origin_run,
- run_session_id=run_session_id,
- )
-
- screenshot_b64: str | None = None
- # Dispatched runs: the worker owns the persistent browser session, so the API side must not
- # grab the live page over CDP. Their at-failure frames come from worker-persisted artifacts.
- if not dispatch_to_worker and not run_ok and run_session_id and not sensitive_origin_run:
- try:
- browser_state = await resolve_persistent_browser_state(
- session_id=run_session_id,
- organization_id=ctx.organization_id,
- )
- if browser_state:
- page = await browser_state.get_or_create_page()
- if SettingsManager.get_settings().BROWSER_CURSOR_VISUALIZATION:
- try:
- await SkyvernFrame.hide_cursor_overlay(page)
- except Exception:
- pass
- try:
- screenshot_bytes = await page.screenshot(type="png")
- finally:
- if SettingsManager.get_settings().BROWSER_CURSOR_VISUALIZATION:
- try:
- await SkyvernFrame.show_cursor_overlay(page)
- except Exception:
- pass
- screenshot_b64 = base64.b64encode(screenshot_bytes).decode("utf-8")
- except Exception:
- LOG.debug("Failed to capture post-run screenshot", exc_info=True)
-
- locator_observations: list[AuthoredLocatorObservationRow] | None = None
- post_run_page_capture: BuildTestPacketPageCapture | None = None
- if (
- not dispatch_to_worker
- and run_session_id
- and _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER
- and not ctx.copilot_total_timeout_exceeded
- ):
- # Structured evidence is admitted through the origin registry scrubber. Pixel capture
- # and locator probes remain withheld for sensitive runs because they have no equivalent
- # exact-value disclosure boundary.
- _pin_pre_run_page_reference(ctx, workflow_run.workflow_run_id)
- post_run_page_capture = await _capture_and_store_post_run_page(
- ctx,
- run_session_id=run_session_id,
- run_id=workflow_run.workflow_run_id,
- current_url=current_url,
- )
-
- if not sensitive_origin_run:
- locator_observations = await _observe_authored_locators(
- ctx,
- run_session_id=run_session_id,
- failed_block_code=failed_block_code,
- worker_owned=dispatch_to_worker,
- observation_deadline_exceeded=ctx.copilot_total_timeout_exceeded,
- )
-
- if not dispatch_to_worker and not ctx.copilot_total_timeout_exceeded:
- await _capture_registered_artifact_evidence(
- ctx,
- run_id=workflow_run.workflow_run_id,
- organization_id=ctx.organization_id,
- downloaded_artifact_ids=_collect_downloaded_artifact_ids(block_outputs_by_label),
- )
-
- # Dispatched runs are worker-owned, so the API cannot CDP-capture the terminal page; read the
- # worker-persisted terminal HTML artifact instead and route it through the same post-run sink.
- if dispatch_to_worker and run_session_id and not ctx.copilot_total_timeout_exceeded:
- await _capture_dispatched_terminal_page_evidence(
- ctx,
- workflow=workflow,
- run_id=workflow_run.workflow_run_id,
- run_session_id=run_session_id,
- organization_id=ctx.organization_id,
- current_url=current_url,
- origin_redaction_registry=ctx.origin_run_redaction_registry,
- )
result_data: dict[str, Any] = {
"workflow_run_id": workflow_run.workflow_run_id,
@@ -3369,39 +3425,18 @@ async def _run_blocks_and_collect_debug(
"executed_block_labels": list(labels_to_execute),
"frontier_start_label": frontier_start_label,
"blocks": results,
- "current_url": current_url,
- "page_title": page_title,
"action_trace_summary": action_trace_summary,
"failing_code_line": failing_code_line,
"action_observations": action_observations,
}
- if locator_observations is not None:
- result_data["authored_locator_observations"] = locator_observations
- if not dispatch_to_worker and current_url:
- result_data["current_url_live_observed"] = True
- if dispatch_to_worker and dispatched_end_url is None:
- result_data["current_url_evidence"] = NO_PERSISTED_END_URL
- post_run_page_evidence = _same_run_page_evidence_for_result(ctx, workflow_run.workflow_run_id)
- if post_run_page_evidence is not None:
- result_data["post_run_page_evidence"] = model_visible_composition_evidence(post_run_page_evidence)
- if post_run_page_capture is not None:
- result_data["post_run_page_capture"] = post_run_page_capture.model_dump(mode="json")
- _attach_run_session_facts(
- result_data,
- used_fresh_run_session=used_fresh_run_session,
- run_detached_from_chat=run_detached_from_chat,
- run_ok=run_ok,
- page_evidence=post_run_page_evidence,
- )
if runtime_frontier_anchor_url is not None:
result_data["runtime_frontier_anchor_url"] = runtime_frontier_anchor_url
if runtime_frontier_starter_url_seeded:
result_data["runtime_frontier_starter_url_seeded"] = True
- screenshot_b64 = _resolve_run_screenshot_b64(live_capture=screenshot_b64, results=results, run_ok=run_ok)
- if screenshot_b64 is not None:
- result_data["screenshot_base64"] = screenshot_b64
if not run_ok and run and getattr(run, "failure_reason", None):
result_data["failure_reason"] = run.failure_reason
+ if not run_ok and run and getattr(run, "failure_category", None):
+ result_data["failure_category"] = run.failure_category
output_identity_workflow = _registered_output_identity_workflow(
dispatch_to_worker=dispatch_to_worker,
@@ -3423,18 +3458,23 @@ async def _run_blocks_and_collect_debug(
data=result_data,
persisted_output_parameters=all_output_params,
)
+ # The run's terminal facts are known here, so the outcome is committed before any
+ # browser-dependent probe: a probe the turn deadline cancels must not erase what the run did.
+ # The post-run capture that drops another run's page now happens after the record, and the
+ # anti-bot read has no run-id check of its own, so stale evidence would grade this run.
+ if not post_run_inspection_cleanly_matches(ctx.composition_page_evidence, workflow_run.workflow_run_id):
+ ctx.composition_page_evidence = None
+
for label, output in registered_outputs_by_label.items():
if isinstance(output, dict) and output:
block_outputs_by_label[label] = output
- # Update verified prefix state ONLY on a fully-successful run. A failed
- # suffix run leaves the browser in post-failure state, so we must not
- # trust blocks that individually succeeded inside it.
- if run_ok and all(r.get("status") == "completed" for r in results):
- for label, output in block_outputs_by_label.items():
- ctx.verified_block_outputs[label] = output
+ # The record below derives "fully tested" from these credits and prefers its own
+ # extracted_data for terminal replies, so both have to land before it, not after.
+ run_fully_completed = run_ok and all(r.get("status") == "completed" for r in results)
+ if run_fully_completed:
ctx.verified_terminal_block_outputs = dict(block_outputs_by_label)
- existing_prefix = list(getattr(ctx, "verified_prefix_labels", []) or [])
+ existing_prefix = list(ctx.verified_prefix_labels or [])
existing_set = set(existing_prefix)
for label in labels_to_execute:
if label not in existing_set:
@@ -3442,6 +3482,48 @@ async def _run_blocks_and_collect_debug(
existing_set.add(label)
ctx.verified_prefix_labels = existing_prefix
_credit_composition_verified_labels(ctx, labels_to_execute, start_provenance)
+
+ response = build_run_blocks_response(run_ok, result_data)
+ committed_run_outcome = _commit_run_blocks_record(ctx, response)
+
+ current_url, post_run_page_evidence = await _attach_post_run_browser_enrichment(
+ ctx,
+ result_data,
+ workflow=workflow,
+ workflow_run_id=workflow_run.workflow_run_id,
+ run_block_rows=run_block_rows,
+ results=results,
+ block_outputs_by_label=block_outputs_by_label,
+ failed_block_code=failed_block_code,
+ run_session_id=run_session_id,
+ dispatch_to_worker=dispatch_to_worker,
+ sensitive_origin_run=sensitive_origin_run,
+ run_ok=run_ok,
+ used_fresh_run_session=used_fresh_run_session,
+ run_detached_from_chat=run_detached_from_chat,
+ )
+ settle_terminal_challenge_after_enrichment(ctx, response)
+ settled_page_evidence = post_run_page_evidence or ctx.composition_page_evidence
+ bind_post_run_page_evidence(
+ ctx,
+ result_data,
+ # Singular selectors are stripped at this boundary; page_evidence_refs reaches a prompt.
+ model_visible_composition_evidence(settled_page_evidence) if settled_page_evidence else None,
+ regraded=_build_recorded_build_test_outcome(
+ ctx,
+ response,
+ ctx.last_run_outcome or committed_run_outcome,
+ response.get(_INTERNAL_GOAL_PATH_OMISSIONS_KEY),
+ ),
+ )
+ _update_verification_evidence_from_run_result(ctx, response)
+
+ # Update verified prefix state ONLY on a fully-successful run. A failed
+ # suffix run leaves the browser in post-failure state, so we must not
+ # trust blocks that individually succeeded inside it.
+ if run_fully_completed:
+ for label, output in block_outputs_by_label.items():
+ ctx.verified_block_outputs[label] = output
if sensitive_origin_run:
# Preserve verified labels and outputs, but never carry browser-position URLs from
# a credential-bearing run into a later frontier or model-visible result.
@@ -3457,7 +3539,7 @@ async def _run_blocks_and_collect_debug(
if verified_current_url is not None:
ctx.verified_prefix_current_url = verified_current_url
- return build_run_blocks_response(run_ok, result_data)
+ return response
finally:
# A paused run keeps its association so the pane still follows the run the person was
# asked to act on; the next run's generation replaces it.
@@ -3809,6 +3891,82 @@ def _block_labels_from_result_data(data: Mapping[str, object]) -> tuple[str, ...
return tuple(dict.fromkeys(labels))
+def _apply_terminal_challenge_latches(
+ copilot_ctx: CopilotContext, result: dict[str, Any], terminal_challenge: TerminalChallengeEvidence
+) -> None:
+ result["ok"] = False
+ result.setdefault("error", terminal_challenge.reason)
+ data = result.get("data")
+ if isinstance(data, dict):
+ data.setdefault("failure_reason", terminal_challenge.reason)
+ _ensure_terminal_challenge_category(
+ data, challenge_evidence_source=terminal_challenge.challenge_evidence_source
+ )
+ copilot_ctx.last_failure_category_top = "ANTI_BOT_DETECTION"
+ LOG.info(
+ "copilot anti-bot evidence stamp",
+ anti_bot_evidence_source=terminal_challenge.challenge_evidence_source,
+ stamp_seam="terminal_challenge",
+ )
+ copilot_ctx.last_test_ok = False
+ copilot_ctx.last_test_suspicious_success = False
+ copilot_ctx.last_test_failure_reason = terminal_challenge.reason
+ copilot_ctx.last_test_anti_bot = terminal_challenge.reason
+ copilot_ctx.last_full_workflow_test_ok = False
+ copilot_ctx.last_failed_workflow_yaml = copilot_ctx.workflow_yaml
+ # A challenge used to fire before the success branch could run, so these were unreachable for a
+ # challenged run; the grade now precedes the settle and can set them.
+ copilot_ctx.verified_terminal_proposal_ready = False
+
+
+def settle_terminal_challenge_after_enrichment(copilot_ctx: CopilotContext, result: dict[str, Any]) -> bool:
+ """Re-decide the terminal challenge once post-run page evidence exists, since the grade committed
+ at handback could not see it. It may only turn a reported pass into a failure, never the reverse."""
+ if result.get("ok") is not True:
+ return False
+ structured_blocker = _run_blocks_structured_blocker_message(result, copilot_ctx)
+ anti_bot_match, _, failure_categories, _ = _analyze_run_blocks(result, copilot_ctx)
+ anti_bot_source = first_carrier_backed_anti_bot_source(failure_categories) if anti_bot_match else None
+ anti_bot_evidence_source = anti_bot_source.value if anti_bot_source else None
+ if not anti_bot_match:
+ anti_bot_match = _composition_anti_bot_reason(copilot_ctx)
+ if anti_bot_match:
+ composition_source = composition_challenge_carrier(copilot_ctx.composition_page_evidence)
+ anti_bot_evidence_source = composition_source.value if composition_source else None
+ if anti_bot_match and not anti_bot_evidence_source:
+ # Keyword-only, same as the pre-enrichment path: drop the match but keep going, because a
+ # structured blocker plus a challenge artifact still decides this without it.
+ LOG.info("copilot anti-bot latch keyword-only-suppressed")
+ anti_bot_match = None
+ terminal_challenge = _terminal_challenge_evidence(
+ result,
+ failure_categories=failure_categories,
+ structured_blocker=structured_blocker,
+ anti_bot_match=anti_bot_match,
+ anti_bot_evidence_source=anti_bot_evidence_source,
+ artifact_flag_key=_artifact_challenge_flag_from_result(result, copilot_ctx),
+ challenge_kind=_terminal_challenge_kind(copilot_ctx, result),
+ )
+ if terminal_challenge is None:
+ return False
+ _apply_terminal_challenge_latches(copilot_ctx, result, terminal_challenge)
+ # The committed outcome still reads as the pass this run looked like; the emitted envelope would
+ # otherwise contradict the failure the caller now returns.
+ data = result.get("data")
+ run_id = data.get("workflow_run_id") if isinstance(data, dict) else None
+ _stash_recorded_run_outcome(
+ copilot_ctx,
+ RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="blocker_reported",
+ display_reason=run_outcome_display_reason(terminal_challenge.reason),
+ workflow_run_id=run_id if isinstance(run_id, str) else None,
+ run_completed=False,
+ ),
+ )
+ return True
+
+
def _terminal_challenge_evidence(
result: dict[str, Any],
*,
@@ -4007,7 +4165,10 @@ def _retained_terminal_output_has_value(value: Any) -> bool:
def _record_run_blocks_result(
- copilot_ctx: Any, result: dict[str, Any], completion_verification: CompletionVerificationResult | None = None
+ copilot_ctx: Any,
+ result: dict[str, Any],
+ completion_verification: CompletionVerificationResult | None = None,
+ connect_failure_reason: str | None = None,
) -> RecordedRunOutcome | None:
"""Record the run result on ctx without letting a second judge rewrite it."""
_record_executed_block_labels(copilot_ctx, result)
@@ -4154,27 +4315,8 @@ def _record_run_blocks_result(
}
if terminal_challenge is not None:
- result["ok"] = False
+ _apply_terminal_challenge_latches(copilot_ctx, result, terminal_challenge)
run_ok = False
- result.setdefault("error", terminal_challenge.reason)
- data = result.get("data")
- if isinstance(data, dict):
- data.setdefault("failure_reason", terminal_challenge.reason)
- _ensure_terminal_challenge_category(
- data, challenge_evidence_source=terminal_challenge.challenge_evidence_source
- )
- copilot_ctx.last_failure_category_top = "ANTI_BOT_DETECTION"
- LOG.info(
- "copilot anti-bot evidence stamp",
- anti_bot_evidence_source=terminal_challenge.challenge_evidence_source,
- stamp_seam="terminal_challenge",
- )
- copilot_ctx.last_test_ok = False
- copilot_ctx.last_test_suspicious_success = False
- copilot_ctx.last_test_failure_reason = terminal_challenge.reason
- copilot_ctx.last_test_anti_bot = terminal_challenge.reason
- copilot_ctx.last_full_workflow_test_ok = False
- copilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)
if run_ok:
registered_output_identity_mismatch = bool(
@@ -4255,12 +4397,18 @@ def _record_run_blocks_result(
copilot_ctx.last_test_failure_reason = next(iter_failure_reasons(result), None)
if result.get("error") and copilot_ctx.last_test_failure_reason is None:
copilot_ctx.last_test_failure_reason = str(result["error"])
+ if connect_failure_reason:
+ # User-facing surfaces read this; the result dict keeps the run's own reason for the
+ # repair signature, the failure summary and loop detection.
+ copilot_ctx.last_test_failure_reason = connect_failure_reason
_update_verification_evidence_from_run_result(copilot_ctx, result)
recorded_outcome = RecordedRunOutcome(
verdict="not_demonstrated",
reason_code="blocker_reported",
display_reason=run_outcome_display_reason(
- copilot_ctx.last_test_failure_reason or str(result.get("error") or "The run failed.")
+ connect_failure_reason
+ or copilot_ctx.last_test_failure_reason
+ or str(result.get("error") or "The run failed.")
),
workflow_run_id=run_id if isinstance(run_id, str) else None,
run_completed=False,
@@ -4272,6 +4420,19 @@ def _record_run_blocks_result(
_EXECUTED_BLOCK_STATUSES = frozenset(status.value for status in BlockStatus if status != BlockStatus.skipped)
+def _commit_run_blocks_record(copilot_ctx: CopilotContext, result: dict[str, Any]) -> RecordedRunOutcome | None:
+ """Commit the run's structured outcome once, marking the result so the shared seam does not redo it.
+
+ The browser-loss stamp runs here rather than at the shared seam: the commit happens upstream of
+ that seam, so a stamp applied there would never reach the committed record."""
+ connect_failure_reason = _stamp_run_side_connect_failure(copilot_ctx, result)
+ recorded = _record_run_blocks_result(
+ copilot_ctx, result, completion_verification=None, connect_failure_reason=connect_failure_reason
+ )
+ result[_INTERNAL_RUN_OUTCOME_RECORDED_KEY] = True
+ return recorded
+
+
def _record_executed_block_labels(copilot_ctx: CopilotContext, result: dict[str, Any]) -> None:
data = result.get("data")
blocks = data.get("blocks") if isinstance(data, dict) else None
@@ -4358,6 +4519,8 @@ def _record_build_test_outcome(
recorded_run_outcome: RecordedRunOutcome | None,
declared_goal_path_omissions: Sequence[Mapping[str, str]] | None = None,
) -> None:
+ # Kept on the result so a post-enrichment regrade reads the same omission facts this grade did.
+ result[_INTERNAL_GOAL_PATH_OMISSIONS_KEY] = list(declared_goal_path_omissions or [])
record_build_test_outcome(
copilot_ctx,
_build_recorded_build_test_outcome(
@@ -4546,24 +4709,75 @@ def _safe_reason_code(value: object) -> str:
return value if isinstance(value, str) else ""
+_RUN_SIDE_CONNECT_STATES: dict[str, BuildTestConnectFailureState] = {
+ "browser_session_closed": "already_closed",
+ "browser_session_startup_timeout": "provisioning_unavailable",
+}
+
+
+def _run_side_connect_state(failure_category: object) -> BuildTestConnectFailureState | None:
+ if not isinstance(failure_category, list):
+ return None
+ for entry in failure_category:
+ if isinstance(entry, dict) and entry.get("category") == "BROWSER_ERROR":
+ state = _RUN_SIDE_CONNECT_STATES.get(str(entry.get("reason_code") or ""))
+ if state is not None:
+ return state
+ return None
+
+
+def _stamp_run_side_connect_failure(copilot_ctx: CopilotContext, result: dict[str, Any]) -> str | None:
+ """Attach the typed browser-loss fact the run itself persisted at its lease seam, beside the
+ run's own failure text, and return the sentence the user should see. Nothing here is inferred:
+ a run that failed for any other reason carries no BROWSER_ERROR reason code and stays untyped,
+ however its session row happens to look afterwards."""
+ if result.get("ok"):
+ return None
+ data = result.get("data")
+ if not isinstance(data, dict) or data.get("build_test_connect_failure") is not None:
+ return None
+ if data.get("blocks") or data.get("control_signal") is not None:
+ return None
+ workflow_run_id = data.get("workflow_run_id")
+ if not isinstance(workflow_run_id, str) or not workflow_run_id:
+ return None
+ state = _run_side_connect_state(data.get("failure_category"))
+ if state is None:
+ return None
+ session_id = data.get("browser_session_id")
+ failure = BuildTestConnectFailure(
+ state=state,
+ browser_session_id=session_id if isinstance(session_id, str) and session_id else None,
+ workflow_run_id=workflow_run_id,
+ )
+ data["build_test_connect_failure"] = failure.model_dump(mode="json", exclude_none=True)
+ return build_test_connect_failure_sentence(failure)
+
+
async def _verify_and_record_run_blocks_result(
copilot_ctx: Any, result: dict[str, Any], _handler_start: float
) -> RecordedBuildTestOutcome | None:
"""Record and emit the run fact once; no authoring judge may rewrite it."""
- await _resolve_captcha_solver_availability(copilot_ctx)
- recorded = _record_run_blocks_result(copilot_ctx, result, completion_verification=None)
+ if result.get(_INTERNAL_RUN_OUTCOME_RECORDED_KEY) is True:
+ recorded = copilot_ctx.last_run_outcome
+ else:
+ recorded = _commit_run_blocks_record(copilot_ctx, result)
if not result.get("ok"):
_mark_stored_post_run_failure_page(copilot_ctx)
+ if recorded is not None:
+ await _send_run_outcome_update(
+ copilot_ctx,
+ result,
+ verdict=recorded.verdict,
+ role=recorded.role,
+ reason_code=recorded.reason_code,
+ display_reason=recorded.display_reason,
+ )
+ # The solver probe reaches the browser and can be cancelled by the turn deadline, so it runs
+ # only after the outcome is committed and emitted, and still before the caller builds repair context.
+ await _resolve_captcha_solver_availability(copilot_ctx)
if recorded is None:
return None
- await _send_run_outcome_update(
- copilot_ctx,
- result,
- verdict=recorded.verdict,
- role=recorded.role,
- reason_code=recorded.reason_code,
- display_reason=recorded.display_reason,
- )
build_test_outcome = getattr(copilot_ctx, "latest_recorded_build_test_outcome", None)
result_data = result.get("data")
result_run_id = result_data.get("workflow_run_id") if isinstance(result_data, dict) else None
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 277ec6996..fa64cefec 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -54,6 +54,7 @@ from skyvern.exceptions import (
BrowserSessionClosed,
BrowserSessionNotFound,
BrowserSessionNotRenewable,
+ BrowserSessionStartupTimeout,
DisabledBlockExecutionError,
DownloadSaveIncompleteError,
InProcessScriptExecutionDenied,
@@ -1387,6 +1388,28 @@ def _build_managed_browser_profile_name(workflow_title: str | None, rendered_key
return f"{title}{suffix}"
+def _browser_lease_failure_category(exc: Exception) -> list[dict] | None:
+ """The lease seam still holds the typed exception; persist its identity so a reader does not
+ have to rediscover it from prose or from the session row, which closes the same way after
+ every run."""
+ if isinstance(exc, BrowserSessionClosed):
+ reason_code = "browser_session_closed"
+ reasoning = "The browser session had already closed before the run could lease it"
+ elif isinstance(exc, BrowserSessionStartupTimeout):
+ reason_code = "browser_session_startup_timeout"
+ reasoning = "The browser session did not start within its startup timeout"
+ else:
+ return None
+ return [
+ {
+ "category": "BROWSER_ERROR",
+ "confidence_float": 1.0,
+ "reason_code": reason_code,
+ "reasoning": reasoning,
+ }
+ ]
+
+
class WorkflowService:
# Prevent GC of fire-and-forget asyncio tasks (e.g. task_run sync).
_background_tasks: set[asyncio.Task] = set() # noqa: RUF012
@@ -4999,6 +5022,7 @@ class WorkflowService:
workflow_run = await self.mark_workflow_run_as_failed(
workflow_run_id=workflow_run_id,
failure_reason=failure_reason,
+ failure_category=_browser_lease_failure_category(e),
)
await self.clean_up_workflow(
workflow=workflow,
diff --git a/tests/unit/copilot_test_helpers.py b/tests/unit/copilot_test_helpers.py
index 3d3b0c485..b84244747 100644
--- a/tests/unit/copilot_test_helpers.py
+++ b/tests/unit/copilot_test_helpers.py
@@ -2,15 +2,18 @@
from __future__ import annotations
+import asyncio
from collections.abc import Awaitable, Callable
-from datetime import datetime, timezone
+from datetime import UTC, datetime, timedelta, timezone
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
+from skyvern.forge import app as forge_app
from skyvern.forge.sdk.artifact.models import ArtifactType
+from skyvern.forge.sdk.copilot.active_run_session import ActiveRunSessionAssociation
from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
@@ -23,6 +26,12 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
)
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
+from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml as process_workflow_yaml
+from skyvern.forge.sdk.schemas.organizations import Organization
+from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock
+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, WorkflowParameter
+from skyvern.schemas.workflows import BlockType
+from skyvern.services import workflow_service as workflow_service_module
DISPATCHED_LOGIN_GATE_HTML = (
"<html><head><title>Sign in</title></head><body><main>"
@@ -111,6 +120,215 @@ def stub_artifact_app(
return retrieved_ids
+def _fake_workflow_run(status: str) -> SimpleNamespace:
+ return SimpleNamespace(
+ status=status,
+ modified_at=datetime(2026, 4, 21, 12, 0, 0, tzinfo=timezone.utc),
+ browser_session_id=None,
+ )
+
+
+async def install_run_blocks_harness(
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ workflow_yaml: str,
+ polled_status: str,
+ dispatch_to_worker: bool = False,
+ terminal_blocks: list[WorkflowRunBlock] | None = None,
+) -> dict[str, Any]:
+ """Stub the collaborators an inline ``_run_blocks_and_collect_debug`` call reaches, with the
+ polled run parked on ``polled_status`` so the watchdog decides the exit."""
+ workflow = await process_workflow_yaml(
+ settings_fallback_yaml="enable_self_healing: false",
+ workflow_id="w_source",
+ workflow_permanent_id="wfp-1",
+ organization_id="org-1",
+ workflow_yaml=workflow_yaml,
+ )
+ now = datetime.now(timezone.utc)
+ organization = Organization(
+ organization_id="org-1",
+ organization_name="Test Org",
+ created_at=now,
+ modified_at=now,
+ )
+ captured: dict[str, Any] = {"workflow": workflow, "executor_cancelled": False}
+
+ database = MagicMock()
+ database.workflows.get_workflow_by_permanent_id = AsyncMock(return_value=workflow)
+ database.organizations.get_organization = AsyncMock(return_value=organization)
+ persisted_output_params = [p for p in workflow.workflow_definition.parameters if isinstance(p, OutputParameter)]
+ persisted_workflow_params = [p for p in workflow.workflow_definition.parameters if isinstance(p, WorkflowParameter)]
+ database.workflow_params.get_workflow_output_parameters = AsyncMock(return_value=persisted_output_params)
+ database.observer.get_workflow_run_blocks = AsyncMock(return_value=terminal_blocks or [])
+ database.workflow_runs.get_workflow_run = AsyncMock(return_value=_fake_workflow_run(status=polled_status))
+ monkeypatch.setattr(forge_app, "DATABASE", database)
+
+ async def _execute_workflow(**_kwargs: Any) -> None:
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ captured["executor_cancelled"] = True
+ raise
+
+ workflow_service = MagicMock()
+ workflow_service.get_workflow_parameters = AsyncMock(return_value=persisted_workflow_params)
+ workflow_service.execute_workflow = AsyncMock(side_effect=_execute_workflow)
+ workflow_service.create_copilot_dispatch_draft_version = AsyncMock(return_value=workflow)
+ monkeypatch.setattr(forge_app, "WORKFLOW_SERVICE", workflow_service)
+ monkeypatch.setattr(
+ forge_app.AGENT_FUNCTION,
+ "should_dispatch_copilot_block_run_to_worker",
+ AsyncMock(return_value=dispatch_to_worker),
+ )
+ monkeypatch.setattr(
+ forge_app.AGENT_FUNCTION,
+ "allow_copilot_inline_code_execution",
+ MagicMock(return_value=False),
+ )
+
+ workflow_run = SimpleNamespace(
+ workflow_run_id="wr_paused",
+ workflow_id="w_source",
+ sequential_credential_id=None,
+ )
+ monkeypatch.setattr(workflow_service_module, "prepare_workflow", AsyncMock(return_value=workflow_run))
+
+ polled_run = _fake_workflow_run(status=polled_status)
+
+ async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, Any, Any]:
+ return polled_run, now, now
+
+ monkeypatch.setattr(run_execution_module, "_read_progress_sources", _read_progress)
+ monkeypatch.setattr(run_execution_module, "RUN_BLOCKS_POLL_INTERVAL_SECONDS", 0)
+ monkeypatch.setattr(run_execution_module, "_fallback_page_info", AsyncMock(return_value=("", "")))
+
+ association = ActiveRunSessionAssociation(
+ organization_id="org-1",
+ workflow_permanent_id="wfp-1",
+ debug_browser_session_id="pbs_chat",
+ run_browser_session_id="pbs_run",
+ workflow_run_id="wr_paused",
+ turn_id="turn-1",
+ generation="gen-1",
+ expires_at=now + timedelta(minutes=5),
+ )
+ captured["publish"] = AsyncMock(return_value=association)
+ captured["clear"] = AsyncMock(return_value=True)
+ captured["cancel_run_task"] = AsyncMock(return_value=None)
+ captured["cooperative_cancel"] = AsyncMock(return_value=None)
+ monkeypatch.setattr(run_execution_module, "publish_active_run_session", captured["publish"])
+ monkeypatch.setattr(run_execution_module, "clear_active_run_session", captured["clear"])
+ monkeypatch.setattr(run_execution_module, "_cancel_run_task_if_not_final", captured["cancel_run_task"])
+ monkeypatch.setattr(run_execution_module, "_cooperative_cancel_dispatched_run", captured["cooperative_cancel"])
+ if dispatch_to_worker:
+ captured["worker_execute"] = AsyncMock(return_value=None)
+ monkeypatch.setattr(
+ run_execution_module.AsyncExecutorFactory,
+ "get_executor",
+ MagicMock(return_value=SimpleNamespace(execute_workflow=captured["worker_execute"])),
+ )
+ monkeypatch.setattr(run_execution_module, "_delete_dispatch_draft_if_run_final", AsyncMock(return_value=None))
+ monkeypatch.setattr(
+ run_execution_module, "_capture_dispatched_terminal_page_evidence", AsyncMock(return_value=None)
+ )
+ monkeypatch.setattr(
+ run_execution_module, "_attach_registered_output_parameter_values", AsyncMock(return_value={})
+ )
+ return captured
+
+
+HANDBACK_WORKFLOW_YAML = """
+title: extraction example
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: extraction
+ label: extract_heading
+ url: https://example.com
+ data_extraction_goal: Extract the page heading.
+"""
+
+
+def terminal_extraction_block(status: str) -> WorkflowRunBlock:
+ return WorkflowRunBlock(
+ label="extract_heading",
+ block_type=BlockType.EXTRACTION,
+ status=status,
+ failure_reason=(
+ 'Timeout exceeded: waiting for locator("#heading") to be visible' if status == "failed" else None
+ ),
+ workflow_run_block_id="wrb_extract_heading",
+ workflow_run_id="wr_paused",
+ organization_id="org-1",
+ created_at=datetime(2026, 4, 21, 12, 5, tzinfo=UTC),
+ modified_at=datetime(2026, 4, 21, 12, 5, tzinfo=UTC),
+ )
+
+
+def page_only_failed_block() -> WorkflowRunBlock:
+ """A failed block with no failure_reason, so the post-run page is its only structural signal."""
+ return WorkflowRunBlock(
+ label="extract_heading",
+ block_type=BlockType.EXTRACTION,
+ status="failed",
+ failure_reason=None,
+ workflow_run_block_id="wrb_extract_heading",
+ workflow_run_id="wr_paused",
+ organization_id="org-1",
+ created_at=datetime(2026, 4, 21, 12, 5, tzinfo=UTC),
+ modified_at=datetime(2026, 4, 21, 12, 5, tzinfo=UTC),
+ )
+
+
+def same_run_page_evidence() -> dict[str, object]:
+ return {
+ "observed_after_workflow_run": True,
+ "workflow_run_id": "wr_paused",
+ "source_browser_session_id": "pbs_run",
+ "current_url": "https://example.com/done",
+ "page_title": "Done",
+ "inspected_url": "https://example.com/done",
+ }
+
+
+def count_record_and_send(monkeypatch: pytest.MonkeyPatch) -> dict[str, int]:
+ counts = {"record": 0, "send": 0}
+ real_record = run_execution_module.record_build_test_outcome
+ real_send = run_execution_module._send_run_outcome_update
+
+ def _record(ctx: object, outcome: object) -> None:
+ counts["record"] += 1
+ real_record(ctx, outcome)
+
+ async def _send(*args: object, **kwargs: object) -> None:
+ counts["send"] += 1
+ await real_send(*args, **kwargs)
+
+ monkeypatch.setattr(run_execution_module, "record_build_test_outcome", _record)
+ monkeypatch.setattr(run_execution_module, "_send_run_outcome_update", _send)
+ return counts
+
+
+async def handback_ctx(
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ polled_status: str,
+ block_status: str,
+ terminal_blocks: list[WorkflowRunBlock] | None = None,
+) -> CopilotContext:
+ harness = await install_run_blocks_harness(
+ monkeypatch,
+ workflow_yaml=HANDBACK_WORKFLOW_YAML,
+ polled_status=polled_status,
+ terminal_blocks=terminal_blocks or [terminal_extraction_block(block_status)],
+ )
+ ctx = make_copilot_ctx(browser_session_id="pbs_chat")
+ ctx.staged_workflow = harness["workflow"]
+ ctx.frontier_resume_session_id = "pbs_run"
+ return ctx
+
+
def make_copilot_ctx(**overrides: object) -> CopilotContext:
defaults: dict[str, object] = dict(
organization_id="org-1",
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index 62b287778..9b86ddb1b 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import copy
import json
from datetime import UTC, datetime
@@ -17,6 +18,7 @@ from skyvern.forge.sdk.copilot.agent import (
_prior_run_debug_text,
_recorded_build_test_outcome_prompt,
)
+from skyvern.forge.sdk.copilot.build_test_connect_failure import build_test_connect_failure_sentence
from skyvern.forge.sdk.copilot.build_test_outcome import (
BuildTestConnectFailure,
BuildTestEvidencePacket,
@@ -42,7 +44,10 @@ from skyvern.forge.sdk.copilot.completion_verification import CompletionVerifica
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext
from skyvern.forge.sdk.copilot.failure_tracking import selector_identity_from_failure
-from skyvern.forge.sdk.copilot.output_utils import project_build_test_packet_for_llm
+from skyvern.forge.sdk.copilot.output_utils import (
+ _INTERNAL_RUN_OUTCOME_RECORDED_KEY,
+ project_build_test_packet_for_llm,
+)
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.runtime_authoring_repair import inject_runtime_authoring_repair_context
from skyvern.forge.sdk.copilot.secret_scrub import clear_session_scrub_values, register_secret_scrub_value
@@ -55,6 +60,8 @@ from skyvern.forge.sdk.copilot.tools.run_execution import (
_first_failed_result,
_record_run_blocks_result,
_recorded_run_block_result,
+ _run_blocks_and_collect_debug,
+ _verify_and_record_run_blocks_result,
build_test_evidence_packet,
)
from skyvern.forge.sdk.copilot.workflow_yaml import runner_code_block_associations
@@ -63,10 +70,14 @@ from skyvern.forge.sdk.workflow.models.block import CodeBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
from skyvern.webeye.browser_artifacts import BrowserArtifacts
from tests.unit.copilot_test_helpers import (
+ count_record_and_send,
failed_second_factor_run,
+ handback_ctx,
make_copilot_ctx,
make_stub_html_artifact,
+ page_only_failed_block,
passing_run,
+ same_run_page_evidence,
straight_line_login_yaml,
two_page_login_yaml,
)
@@ -173,6 +184,7 @@ def test_connect_failure_projects_through_recorded_outcome_and_packet() -> None:
outcome = recorded_outcome_from_run_blocks_result(result)
assert outcome is not None
assert outcome.connect_failure == failure
+ assert outcome.observed_evidence_summary == build_test_connect_failure_sentence(failure)
packet = build_test_evidence_packet(ctx, result, recorded_outcome=outcome)
assert packet.failure is not None
assert packet.failure.connect_failure == failure
@@ -4629,3 +4641,191 @@ async def test_the_repaired_block_executes_and_the_run_owns_the_outputs_it_was_a
if output["output_parameter_key"] == "collect_payment_options_output"
]
assert run_owned == [after_output], "the corrected run does not hand back the output its code produced"
+
+
+async def test_completed_run_is_recorded_before_the_browser_enrichment_await(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")
+ order: list[str] = []
+ real_record = run_execution_module.record_build_test_outcome
+
+ def _record(record_ctx: object, outcome: object) -> None:
+ order.append("record")
+ real_record(record_ctx, outcome)
+
+ async def _enrichment(*_args: object, **_kwargs: object) -> tuple[str, dict[str, object] | None]:
+ order.append("enrichment")
+ return "", None
+
+ monkeypatch.setattr(run_execution_module, "record_build_test_outcome", _record)
+ monkeypatch.setattr(run_execution_module, "_attach_post_run_browser_enrichment", _enrichment)
+
+ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+
+ assert result["ok"] is True, result
+ assert order == ["record", "enrichment"]
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.workflow_run_id == "wr_paused"
+
+
+async def test_failed_run_still_records_the_failure_and_marks_the_post_run_page(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = await handback_ctx(monkeypatch, polled_status="failed", block_status="failed")
+ marks: list[str] = []
+
+ async def _enrichment(*_args: object, **_kwargs: object) -> tuple[str, dict[str, object] | None]:
+ return "", None
+
+ monkeypatch.setattr(run_execution_module, "_attach_post_run_browser_enrichment", _enrichment)
+ monkeypatch.setattr(
+ run_execution_module,
+ "_mark_stored_post_run_failure_page",
+ lambda _ctx: marks.append("marked"),
+ )
+ counts = count_record_and_send(monkeypatch)
+
+ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+ await _verify_and_record_run_blocks_result(ctx, result, 0.0)
+
+ assert result["ok"] is False, result
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.workflow_run_id == "wr_paused"
+ assert outcome.verdict == "repairable_failure"
+ assert outcome.reason_code != ""
+ assert marks == ["marked"]
+ assert counts == {"record": 1, "send": 1}
+ assert len(ctx.recorded_build_test_outcome_history) == 1
+
+
+async def test_enrichment_facts_still_land_on_the_recorded_outcome_and_agree_with_history(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")
+ evidence = same_run_page_evidence()
+
+ async def _enrichment(
+ enrich_ctx: CopilotContext, result_data: dict[str, object], **_kwargs: object
+ ) -> tuple[str, dict[str, object]]:
+ enrich_ctx.composition_page_evidence = evidence
+ result_data["current_url"] = "https://example.com/done"
+ result_data["post_run_page_evidence"] = evidence
+ result_data["post_run_page_capture"] = {"status": "captured"}
+ return "https://example.com/done", evidence
+
+ monkeypatch.setattr(run_execution_module, "_attach_post_run_browser_enrichment", _enrichment)
+ monkeypatch.setattr(
+ run_execution_module.app.AGENT_FUNCTION,
+ "captcha_solving_available",
+ AsyncMock(return_value=True),
+ )
+ counts = count_record_and_send(monkeypatch)
+
+ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+ await _verify_and_record_run_blocks_result(ctx, result, 0.0)
+
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert ctx.captcha_solver_available is True
+ assert ctx.captcha_solver_available_for_url == "https://example.com/done"
+ assert outcome.page_evidence_refs, outcome
+ assert outcome.page_capture is not None and outcome.page_capture.status == "captured"
+ assert counts == {"record": 1, "send": 1}
+ assert len(ctx.recorded_build_test_outcome_history) == 1
+ entry = ctx.recorded_build_test_outcome_history[-1]
+ assert entry["workflow_run_id"] == outcome.workflow_run_id
+ assert entry["verdict"] == outcome.verdict
+ assert entry["reason_code"] == outcome.reason_code
+ assert entry["structural_key"] == outcome.structural_key
+ assert outcome.is_authoritative is True
+ assert entry["is_authoritative"] is True
+ assert ctx.recorded_persisted_block_run_workflow_run_id == outcome.workflow_run_id
+
+
+async def test_failed_run_known_only_by_its_page_is_graded_after_enrichment(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A failed block carrying no failure_reason has no structural identity until the post-run page
+ arrives, so grading it before enrichment records nothing at all."""
+ ctx = await handback_ctx(
+ monkeypatch,
+ polled_status="failed",
+ block_status="failed",
+ terminal_blocks=[page_only_failed_block()],
+ )
+ evidence = same_run_page_evidence()
+
+ async def _enrichment(
+ enrich_ctx: CopilotContext, result_data: dict[str, object], **_kwargs: object
+ ) -> tuple[str, dict[str, object]]:
+ enrich_ctx.composition_page_evidence = evidence
+ result_data["post_run_page_evidence"] = evidence
+ return "https://example.com/done", evidence
+
+ monkeypatch.setattr(run_execution_module, "_attach_post_run_browser_enrichment", _enrichment)
+ counts = count_record_and_send(monkeypatch)
+
+ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+ await _verify_and_record_run_blocks_result(ctx, result, 0.0)
+
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None, "the run's failure was lost because the page was unknown at record time"
+ assert outcome.verdict == "repairable_failure", outcome
+ assert outcome.page_evidence_refs, outcome
+ assert outcome.key_provenance.get("structural_failure_identity") != (
+ "no typed verification/page/output identity available"
+ )
+ assert counts["record"] == 1
+ assert len(ctx.recorded_build_test_outcome_history) == 1
+ entry = ctx.recorded_build_test_outcome_history[-1]
+ assert entry["verdict"] == outcome.verdict
+ assert entry["reason_code"] == outcome.reason_code
+ assert entry["structural_key"] == outcome.structural_key
+ assert entry["is_authoritative"] == outcome.is_authoritative
+
+
+async def test_watchdog_paused_result_is_recorded_before_the_captcha_probe(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = await handback_ctx(monkeypatch, polled_status="paused", block_status="completed")
+ record_build_test_outcome(ctx, failed_second_factor_run("wr_prior_failure"))
+ counts = count_record_and_send(monkeypatch)
+ observed_at_probe: list[RecordedBuildTestOutcome | None] = []
+
+ async def _cancelled_probe(*_args: object, **_kwargs: object) -> bool:
+ observed_at_probe.append(ctx.latest_recorded_build_test_outcome)
+ raise asyncio.CancelledError
+
+ monkeypatch.setattr(
+ run_execution_module.app.AGENT_FUNCTION,
+ "captcha_solving_available",
+ _cancelled_probe,
+ )
+
+ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+
+ assert result["ok"] is False, result
+ assert result["data"]["control_signal"]["kind"] == "watchdog_paused"
+ assert _INTERNAL_RUN_OUTCOME_RECORDED_KEY not in result
+
+ with pytest.raises(asyncio.CancelledError):
+ await _verify_and_record_run_blocks_result(ctx, result, 0.0)
+
+ assert len(observed_at_probe) == 1
+ at_probe = observed_at_probe[0]
+ assert at_probe is not None and at_probe.workflow_run_id == "wr_paused"
+ assert at_probe.verdict == "not_authoritative"
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None and outcome.workflow_run_id == "wr_paused"
+ assert counts == {"record": 1, "send": 1}
+ assert [entry["workflow_run_id"] for entry in ctx.recorded_build_test_outcome_history] == [
+ "wr_prior_failure",
+ "wr_paused",
+ ]
diff --git a/tests/unit/test_copilot_runner_deadline.py b/tests/unit/test_copilot_runner_deadline.py
index d32230ac7..d822db493 100644
--- a/tests/unit/test_copilot_runner_deadline.py
+++ b/tests/unit/test_copilot_runner_deadline.py
@@ -5,12 +5,16 @@ from __future__ import annotations
import asyncio
import contextlib
import time
+from collections.abc import Awaitable, Callable
from typing import Any, cast
from unittest.mock import AsyncMock, MagicMock
import pytest
from structlog.testing import capture_logs
+from skyvern.forge.sdk.copilot.build_test_outcome import record_build_test_outcome
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
+from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import (
TOTAL_TIMEOUT_SECONDS,
CopilotTotalTimeoutError,
@@ -22,6 +26,13 @@ from skyvern.forge.sdk.copilot.pending_operation import (
pending_operation,
pending_operation_fields,
)
+from skyvern.forge.sdk.copilot.tools import run_execution
+from skyvern.forge.sdk.copilot.tools.run_execution import _run_blocks_and_collect_debug
+from tests.unit.copilot_test_helpers import (
+ count_record_and_send,
+ failed_second_factor_run,
+ handback_ctx,
+)
def _fake_result() -> MagicMock:
@@ -470,3 +481,179 @@ async def test_the_deadline_path_is_unchanged_when_the_fingerprint_contributes_n
with capture_logs() as already_marked:
_mark_copilot_total_timeout(ctx, elapsed_seconds=99.0, iteration=1)
assert _deadline_events(already_marked) == []
+
+
+async def _run_with_failing_enrichment(
+ monkeypatch: pytest.MonkeyPatch,
+ enrichment: Callable[..., Awaitable[tuple[str, dict[str, object] | None]]],
+) -> tuple[CopilotContext, dict[str, int]]:
+ ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")
+ record_build_test_outcome(ctx, failed_second_factor_run("wr_prior_failure"))
+ counts = count_record_and_send(monkeypatch)
+ monkeypatch.setattr(run_execution, "_attach_post_run_browser_enrichment", enrichment)
+ return ctx, counts
+
+
+def _assert_later_run_survived(ctx: CopilotContext, counts: dict[str, int]) -> None:
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.workflow_run_id == "wr_paused"
+ assert outcome.verdict != "repairable_failure"
+ assert counts["record"] == 1
+ assert [entry["workflow_run_id"] for entry in ctx.recorded_build_test_outcome_history] == [
+ "wr_prior_failure",
+ "wr_paused",
+ ]
+
+
+async def test_completed_run_survives_an_enrichment_probe_that_raises(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def _raising(*_args: object, **_kwargs: object) -> tuple[str, dict[str, object] | None]:
+ raise RuntimeError("post-run browser probe failed")
+
+ ctx, counts = await _run_with_failing_enrichment(monkeypatch, _raising)
+
+ with pytest.raises(RuntimeError, match="post-run browser probe failed"):
+ await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+
+ _assert_later_run_survived(ctx, counts)
+
+
+async def test_completed_run_survives_an_enrichment_probe_cancelled_by_the_deadline(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ cancelled: list[str] = []
+
+ async def _hanging(*_args: object, **_kwargs: object) -> tuple[str, dict[str, object] | None]:
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ cancelled.append("cancelled")
+ raise
+ return "", None
+
+ ctx, counts = await _run_with_failing_enrichment(monkeypatch, _hanging)
+
+ with pytest.raises(asyncio.TimeoutError):
+ await asyncio.wait_for(
+ _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx),
+ timeout=0.05,
+ )
+
+ assert cancelled == ["cancelled"]
+ _assert_later_run_survived(ctx, counts)
+
+
+async def test_bot_wall_seen_only_by_the_post_run_page_still_fails_the_run(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Every block reports completed and the run output alone looks clean; the challenge is visible
+ only in the page evidence that arrives after the outcome is committed."""
+ ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")
+ challenge_evidence = {
+ "observed_after_workflow_run": True,
+ "workflow_run_id": "wr_paused",
+ "source_browser_session_id": "pbs_run",
+ "current_url": "https://example.com/verify",
+ "inspected_url": "https://example.com/verify",
+ "anti_bot_indicators": ["captcha"],
+ "challenge_state": {
+ "detected": True,
+ "kind": "captcha",
+ "requires_human_verification": True,
+ },
+ "message": "Please complete the CAPTCHA to continue",
+ }
+
+ async def _enrichment(
+ enrich_ctx: CopilotContext, result_data: dict[str, object], **_kwargs: object
+ ) -> tuple[str, dict[str, object]]:
+ enrich_ctx.composition_page_evidence = challenge_evidence
+ result_data["post_run_page_evidence"] = challenge_evidence
+ return "https://example.com/verify", challenge_evidence
+
+ monkeypatch.setattr(run_execution, "_attach_post_run_browser_enrichment", _enrichment)
+
+ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+
+ assert result["ok"] is False, "a run that ended on a bot wall was reported as passing"
+ assert ctx.last_test_anti_bot, ctx.last_test_anti_bot
+ assert ctx.last_full_workflow_test_ok is False
+ # The emitted envelope must agree with the failure the caller returns.
+ assert ctx.last_run_outcome is not None
+ assert ctx.last_run_outcome.verdict == "not_demonstrated", ctx.last_run_outcome
+ assert ctx.last_run_outcome.run_completed is False
+ # The success-branch latches must not survive a run the settle turned into a failure.
+ assert ctx.verified_terminal_proposal_ready is False
+
+
+async def test_prior_run_page_evidence_does_not_fail_this_run(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A challenge page left on ctx by an earlier run must not grade this one: the capture that drops
+ another run's page runs after the record, and the anti-bot read has no run-id check."""
+ ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")
+ ctx.composition_page_evidence = {
+ "observed_after_workflow_run": True,
+ "workflow_run_id": "wr_some_older_run",
+ "source_browser_session_id": "pbs_run",
+ "current_url": "https://example.com/verify",
+ "anti_bot_indicators": ["captcha"],
+ "challenge_state": {"detected": True, "kind": "captcha", "requires_human_verification": True},
+ "message": "Please complete the CAPTCHA to continue",
+ }
+
+ async def _enrichment(
+ _enrich_ctx: CopilotContext, _result_data: dict[str, object], **_kwargs: object
+ ) -> tuple[str, dict[str, object] | None]:
+ return "https://example.com/done", None
+
+ monkeypatch.setattr(run_execution, "_attach_post_run_browser_enrichment", _enrichment)
+
+ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+
+ assert result["ok"] is True, "another run's page evidence failed this run"
+ assert ctx.last_test_anti_bot is None, ctx.last_test_anti_bot
+
+
+async def test_enrichment_supplied_page_reaches_the_grade(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Drives the real _attach_post_run_browser_enrichment. Every other test stubs the helper and
+ plants the page on ctx first, which cannot tell a page enrichment supplied from one already
+ there -- the distinction this ordering is entirely about."""
+ ctx = await handback_ctx(monkeypatch, polled_status="completed", block_status="completed")
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ assert ctx.composition_page_evidence is None, "the page must arrive from enrichment, not the fixture"
+
+ challenge_page = {
+ "source_tool": "inspect_page_for_composition",
+ "observed_after_workflow_run": True,
+ "workflow_run_id": "wr_paused",
+ "current_url": "https://example.com/verify",
+ "inspected_url": "https://example.com/verify",
+ "anti_bot_indicators": ["captcha"],
+ "challenge_state": {"detected": True, "kind": "captcha", "requires_human_verification": True},
+ "message": "Please complete the CAPTCHA to continue",
+ }
+
+ async def _page_info(*_args: object, **_kwargs: object) -> tuple[str, str | None, str | None]:
+ return "https://example.com/verify", "Verify you are human", None
+
+ async def _read_evidence(*_args: object, **_kwargs: object) -> tuple[dict[str, object], str, None, None]:
+ return challenge_page, "pbs_run", None, None
+
+ monkeypatch.setattr(run_execution, "_resolve_post_run_page_info", _page_info)
+ monkeypatch.setattr(run_execution, "_read_run_session_page_evidence", _read_evidence)
+
+ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+
+ assert ctx.composition_page_evidence is not None, "the real enrichment never stored the page"
+ assert result["ok"] is False, "a page supplied by enrichment did not reach the grade"
+ assert ctx.last_test_anti_bot, ctx.last_test_anti_bot
diff --git a/tests/unit/test_copilot_terminal_envelope.py b/tests/unit/test_copilot_terminal_envelope.py
index b92f88eab..8da2aa191 100644
--- a/tests/unit/test_copilot_terminal_envelope.py
+++ b/tests/unit/test_copilot_terminal_envelope.py
@@ -1,16 +1,20 @@
from __future__ import annotations
import time
+from datetime import UTC, datetime
from types import SimpleNamespace
from typing import Any
+from unittest.mock import AsyncMock, MagicMock
import pytest
import structlog.testing
+from skyvern.exceptions import get_user_facing_exception_message
from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot.blocker_signal import (
CopilotToolBlockerSignal,
)
+from skyvern.forge.sdk.copilot.build_test_connect_failure import build_test_connect_failure_sentence
from skyvern.forge.sdk.copilot.build_test_outcome import BuildTestConnectFailure, BuildTestFailedOperation
from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
@@ -23,10 +27,130 @@ from skyvern.forge.sdk.copilot.terminal_envelope import (
reason_in_reply_shadow,
render_terminal_message,
)
-from skyvern.forge.sdk.copilot.tools.run_execution import _stash_recorded_run_outcome
+from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
+from skyvern.forge.sdk.copilot.tools.run_execution import (
+ _record_run_blocks_result,
+ _stamp_run_side_connect_failure,
+ _stash_recorded_run_outcome,
+)
from skyvern.forge.sdk.copilot.tools.workflow_update import _record_workflow_update_result
+from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
from tests.unit.copilot_test_helpers import make_copilot_ctx
+RUN_SIDE_SHARED_REASON = get_user_facing_exception_message(
+ Exception("connect_over_cdp failed: WebSocket error: connection closed")
+)
+RUN_SIDE_TYPED_REASON = build_test_connect_failure_sentence(
+ BuildTestConnectFailure(state="already_closed", workflow_run_id="wr_run_side")
+)
+
+
+def _run_side_failed_result(
+ *, failure_reason: str = RUN_SIDE_SHARED_REASON, reason_code: str | None = None
+) -> dict[str, Any]:
+ data: dict[str, Any] = {
+ "workflow_run_id": "wr_run_side",
+ "browser_session_id": "pbs_run_side",
+ "overall_status": "failed",
+ "blocks": [],
+ "failure_reason": failure_reason,
+ }
+ if reason_code is not None:
+ data["failure_category"] = [{"category": "BROWSER_ERROR", "confidence_float": 1.0, "reason_code": reason_code}]
+ return {"ok": False, "error": failure_reason, "data": data}
+
+
+def _install_session_record(monkeypatch: pytest.MonkeyPatch, session: PersistentBrowserSession | None) -> None:
+ mock_app = MagicMock()
+ mock_app.DATABASE.browser_sessions.get_persistent_browser_session = AsyncMock(return_value=session)
+ monkeypatch.setattr(run_execution_module, "app", mock_app)
+
+
+def _session_record(*, closed: bool) -> PersistentBrowserSession:
+ now = datetime.now(UTC)
+ return PersistentBrowserSession(
+ persistent_browser_session_id="pbs_run_side",
+ organization_id="org-1",
+ status="completed" if closed else "running",
+ completed_at=now if closed else None,
+ created_at=now,
+ modified_at=now,
+ )
+
+
+def test_a_persisted_session_closed_reason_types_the_stop_without_touching_the_runs_own_text() -> None:
+ result = _run_side_failed_result(reason_code="browser_session_closed")
+
+ sentence = _stamp_run_side_connect_failure(make_copilot_ctx(), result)
+
+ assert sentence == RUN_SIDE_TYPED_REASON
+ assert result["data"]["build_test_connect_failure"]["state"] == "already_closed"
+ assert result["error"] == RUN_SIDE_SHARED_REASON
+ assert result["data"]["failure_reason"] == RUN_SIDE_SHARED_REASON
+
+
+def test_a_persisted_startup_timeout_types_provisioning_unavailable() -> None:
+ result = _run_side_failed_result(reason_code="browser_session_startup_timeout")
+
+ sentence = _stamp_run_side_connect_failure(make_copilot_ctx(), result)
+
+ assert sentence is not None
+ assert result["data"]["build_test_connect_failure"]["state"] == "provisioning_unavailable"
+
+
+def test_a_run_that_failed_for_its_own_reason_stays_untyped_however_its_session_row_closed(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A mistyped block label fails before any block runs and its session row closes normally
+ afterwards; the run persisted no browser reason code, so the author must see the typo, not
+ a browser-loss retry."""
+ _install_session_record(monkeypatch, _session_record(closed=True))
+ typo = "Unable to find block with label extract_invoice_total"
+ result = _run_side_failed_result(failure_reason=typo)
+
+ sentence = _stamp_run_side_connect_failure(make_copilot_ctx(), result)
+
+ assert sentence is None
+ assert result["data"].get("build_test_connect_failure") is None
+ assert result["error"] == typo
+ assert result["data"]["failure_reason"] == typo
+
+
+async def test_a_typed_run_side_stop_keeps_the_shared_prose_out_of_the_chat_reply(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _install_session_record(monkeypatch, _session_record(closed=True))
+ ctx = make_copilot_ctx()
+ ctx.last_update_block_count = 2
+ result = _run_side_failed_result(reason_code="browser_session_closed")
+
+ sentence = _stamp_run_side_connect_failure(ctx, result)
+ recorded = _record_run_blocks_result(ctx, result, connect_failure_reason=sentence)
+ reply = agent_module._rewrite_failed_test_response("The test failed.", ctx)
+
+ assert recorded is not None and recorded.display_reason == RUN_SIDE_TYPED_REASON
+ assert RUN_SIDE_TYPED_REASON in reply
+ assert RUN_SIDE_SHARED_REASON not in reply
+ assert "high demand" not in reply.lower()
+
+
+async def test_a_typed_run_side_stop_keeps_the_shared_prose_out_of_the_recorded_summary(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _install_session_record(monkeypatch, _session_record(closed=True))
+ ctx = make_copilot_ctx()
+ result = _run_side_failed_result(reason_code="browser_session_closed")
+
+ sentence = _stamp_run_side_connect_failure(ctx, result)
+ _record_run_blocks_result(ctx, result, connect_failure_reason=sentence)
+ reason, _ = agent_module._recorded_failure_summary(ctx)
+
+ assert reason and reason in RUN_SIDE_TYPED_REASON
+ assert RUN_SIDE_SHARED_REASON not in reason
+ assert "high demand" not in reason.lower()
+
def _run_outcome(verdict: str, display_reason: str | None = None) -> RecordedRunOutcome:
return RecordedRunOutcome(verdict=verdict, display_reason=display_reason)
diff --git a/tests/unit/test_copilot_watchdog.py b/tests/unit/test_copilot_watchdog.py
index a82ec4296..265c34ab9 100644
--- a/tests/unit/test_copilot_watchdog.py
+++ b/tests/unit/test_copilot_watchdog.py
@@ -36,7 +36,6 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from skyvern.forge import app as forge_app
-from skyvern.forge.sdk.copilot.active_run_session import ActiveRunSessionAssociation
from skyvern.forge.sdk.copilot.blocker_signal import (
assert_clean_user_facing_text,
contains_internal_machinery_leak,
@@ -58,12 +57,9 @@ from skyvern.forge.sdk.copilot.tools.run_execution import (
_watchdog_user_facing_summary,
)
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
-from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
-from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock
-from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, WorkflowParameter
from skyvern.schemas.workflows import BlockType
-from skyvern.services import workflow_service as workflow_service_module
+from tests.unit.copilot_test_helpers import install_run_blocks_harness as _install_run_harness
from tests.unit.copilot_test_helpers import make_copilot_ctx
@@ -591,112 +587,6 @@ workflow_definition:
"""
-async def _install_run_harness(
- monkeypatch: pytest.MonkeyPatch,
- *,
- workflow_yaml: str,
- polled_status: str,
- dispatch_to_worker: bool = False,
- terminal_blocks: list[WorkflowRunBlock] | None = None,
-) -> dict[str, Any]:
- """Stub the collaborators an inline ``_run_blocks_and_collect_debug`` call reaches, with the
- polled run parked on ``polled_status`` so the watchdog decides the exit."""
- workflow = await _process_workflow_yaml(
- settings_fallback_yaml="enable_self_healing: false",
- workflow_id="w_source",
- workflow_permanent_id="wfp-1",
- organization_id="org-1",
- workflow_yaml=workflow_yaml,
- )
- now = datetime.now(timezone.utc)
- organization = Organization(
- organization_id="org-1",
- organization_name="Test Org",
- created_at=now,
- modified_at=now,
- )
- captured: dict[str, Any] = {"workflow": workflow, "executor_cancelled": False}
-
- database = MagicMock()
- database.workflows.get_workflow_by_permanent_id = AsyncMock(return_value=workflow)
- database.organizations.get_organization = AsyncMock(return_value=organization)
- persisted_output_params = [p for p in workflow.workflow_definition.parameters if isinstance(p, OutputParameter)]
- persisted_workflow_params = [p for p in workflow.workflow_definition.parameters if isinstance(p, WorkflowParameter)]
- database.workflow_params.get_workflow_output_parameters = AsyncMock(return_value=persisted_output_params)
- database.observer.get_workflow_run_blocks = AsyncMock(return_value=terminal_blocks or [])
- database.workflow_runs.get_workflow_run = AsyncMock(return_value=_fake_run(status=polled_status))
- monkeypatch.setattr(forge_app, "DATABASE", database)
-
- async def _execute_workflow(**_kwargs: Any) -> None:
- try:
- await asyncio.Event().wait()
- except asyncio.CancelledError:
- captured["executor_cancelled"] = True
- raise
-
- workflow_service = MagicMock()
- workflow_service.get_workflow_parameters = AsyncMock(return_value=persisted_workflow_params)
- workflow_service.execute_workflow = AsyncMock(side_effect=_execute_workflow)
- workflow_service.create_copilot_dispatch_draft_version = AsyncMock(return_value=workflow)
- monkeypatch.setattr(forge_app, "WORKFLOW_SERVICE", workflow_service)
- monkeypatch.setattr(
- forge_app.AGENT_FUNCTION,
- "should_dispatch_copilot_block_run_to_worker",
- AsyncMock(return_value=dispatch_to_worker),
- )
- monkeypatch.setattr(
- forge_app.AGENT_FUNCTION,
- "allow_copilot_inline_code_execution",
- MagicMock(return_value=False),
- )
-
- workflow_run = SimpleNamespace(
- workflow_run_id="wr_paused",
- workflow_id="w_source",
- sequential_credential_id=None,
- )
- monkeypatch.setattr(workflow_service_module, "prepare_workflow", AsyncMock(return_value=workflow_run))
-
- polled_run = _fake_run(status=polled_status)
-
- async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, Any, Any]:
- return polled_run, now, now
-
- monkeypatch.setattr(run_execution, "_read_progress_sources", _read_progress)
- monkeypatch.setattr(run_execution, "RUN_BLOCKS_POLL_INTERVAL_SECONDS", 0)
- monkeypatch.setattr(run_execution, "_fallback_page_info", AsyncMock(return_value=("", "")))
-
- association = ActiveRunSessionAssociation(
- organization_id="org-1",
- workflow_permanent_id="wfp-1",
- debug_browser_session_id="pbs_chat",
- run_browser_session_id="pbs_run",
- workflow_run_id="wr_paused",
- turn_id="turn-1",
- generation="gen-1",
- expires_at=now + timedelta(minutes=5),
- )
- captured["publish"] = AsyncMock(return_value=association)
- captured["clear"] = AsyncMock(return_value=True)
- captured["cancel_run_task"] = AsyncMock(return_value=None)
- captured["cooperative_cancel"] = AsyncMock(return_value=None)
- monkeypatch.setattr(run_execution, "publish_active_run_session", captured["publish"])
- monkeypatch.setattr(run_execution, "clear_active_run_session", captured["clear"])
- monkeypatch.setattr(run_execution, "_cancel_run_task_if_not_final", captured["cancel_run_task"])
- monkeypatch.setattr(run_execution, "_cooperative_cancel_dispatched_run", captured["cooperative_cancel"])
- if dispatch_to_worker:
- captured["worker_execute"] = AsyncMock(return_value=None)
- monkeypatch.setattr(
- run_execution.AsyncExecutorFactory,
- "get_executor",
- MagicMock(return_value=SimpleNamespace(execute_workflow=captured["worker_execute"])),
- )
- monkeypatch.setattr(run_execution, "_delete_dispatch_draft_if_run_final", AsyncMock(return_value=None))
- monkeypatch.setattr(run_execution, "_capture_dispatched_terminal_page_evidence", AsyncMock(return_value=None))
- monkeypatch.setattr(run_execution, "_attach_registered_output_parameter_values", AsyncMock(return_value={}))
- return captured
-
-
def _adopted_detached_tasks(before: set[Any]) -> list[Any]:
return [task for task in run_execution._DETACHED_CLEANUP_TASKS if task not in before]
diff --git a/tests/unit/test_workflow_service_cancel.py b/tests/unit/test_workflow_service_cancel.py
index 70d058ca6..3bfb20de5 100644
--- a/tests/unit/test_workflow_service_cancel.py
+++ b/tests/unit/test_workflow_service_cancel.py
@@ -545,3 +545,16 @@ async def test_shielded_finalize_skipped_when_pre_finally_status_unset() -> None
await task
assert not finalize_called, "finalize must not run when pre_finally_status is unset"
+
+
+def test_lease_failure_category_persists_only_typed_browser_loss() -> None:
+ from skyvern.exceptions import BrowserSessionClosed, BrowserSessionStartupTimeout
+ from skyvern.forge.sdk.workflow.service import _browser_lease_failure_category
+
+ closed = _browser_lease_failure_category(BrowserSessionClosed("pbs_x"))
+ timeout = _browser_lease_failure_category(BrowserSessionStartupTimeout("pbs_x"))
+
+ assert closed is not None and closed[0]["category"] == "BROWSER_ERROR"
+ assert closed[0]["reason_code"] == "browser_session_closed"
+ assert timeout is not None and timeout[0]["reason_code"] == "browser_session_startup_timeout"
+ assert _browser_lease_failure_category(RuntimeError("a required workflow parameter was not provided")) is None