Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11545: Add recorded outcome spine (#6939)
Andrew Neilson1de08f3eebfd
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 81c84a0c9..52cef77c1 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -44,6 +44,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import ()from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_datafrom skyvern.forge.sdk.copilot.build_phase import initial_build_phase+from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcomefrom skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODEfrom skyvern.forge.sdk.copilot.code_block_synthesis import (is_optional_dismissal_only_trajectory,@@ -715,6 +716,56 @@ def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:return "\n\n" + "\n".join(line for line in lines if line)+def _recorded_build_test_outcome_prompt(ctx: CopilotContext | None) -> str:+ if ctx is None:+ return ""+ if normalize_block_authoring_policy(ctx.block_authoring_policy) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:+ return ""+ outcome = ctx.latest_recorded_build_test_outcome+ if not isinstance(outcome, RecordedBuildTestOutcome) or not outcome.is_authoritative:+ return ""+ LOG.info(+ "copilot recorded build-test outcome rendered",+ phase=outcome.phase,+ reason_code=outcome.reason_code,+ structural_key=outcome.structural_key,+ workflow_run_id=outcome.workflow_run_id,+ )+ lines = [+ "RECORDED BUILD-TEST OUTCOME:",+ f"phase: {_clean_authoring_repair_prompt_atom(outcome.phase)}",+ f"attempted_tool: {_clean_authoring_repair_prompt_atom(outcome.attempted_tool)}",+ f"attempted_target: {_clean_authoring_repair_prompt_atom(outcome.attempted_target)}",+ f"attempted_block_label: {_clean_authoring_repair_prompt_atom(outcome.attempted_block_label)}",+ f"verdict: {_clean_authoring_repair_prompt_atom(outcome.verdict)}",+ f"reason_code: {_clean_authoring_repair_prompt_atom(outcome.reason_code)}",+ f"structural_key: {_clean_authoring_repair_prompt_atom(outcome.structural_key or '')}",+ f"block_labels: {_render_authoring_repair_prompt_list(outcome.block_labels)}",+ f"page_evidence_refs: {_render_authoring_repair_prompt_list(outcome.page_evidence_refs)}",+ ]+ if outcome.missing_requested_output_facts:+ lines.append("missing_requested_output_facts:")+ for fact in outcome.missing_requested_output_facts[:8]:+ if not isinstance(fact, dict):+ continue+ fields = []+ for key in ("output_root", "output_path", "value_status", "reason_code"):+ value = fact.get(key)+ if isinstance(value, str) and value.strip():+ fields.append(f"{key}={_clean_authoring_repair_prompt_atom(value)}")+ if fields:+ lines.append(f"- {'; '.join(fields)}")+ if outcome.workflow_run_id:+ lines.append(f"workflow_run_id: {_clean_authoring_repair_prompt_atom(outcome.workflow_run_id)}")+ if outcome.observed_evidence_summary:+ lines.append(f"observed_evidence: {_clean_authoring_repair_prompt_atom(outcome.observed_evidence_summary)}")+ lines.append(+ "Before saving or rerunning, change the next authored step, selector, extraction, or binding based on this "+ "recorded structure. Do not re-emit the same plan against the same structural key."+ )+ return "\n\n" + "\n".join(line for line in lines if line)++def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:"""Pre-authoring offer of the synthesized code block.@@ -804,6 +855,7 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -return (prompt+ _runtime_verification_evidence_prompt(ctx)+ + _recorded_build_test_outcome_prompt(ctx)+ _code_authoring_repair_context_prompt(ctx)+ _synthesized_block_offer_prompt(ctx)+ _docs_answer_turn_directive(ctx.turn_intent)diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pynew file mode 100644index 000000000..7beb642bc--- /dev/null+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -0,0 +1,816 @@+from __future__ import annotations++import hashlib+import json+import re+import textwrap+from collections.abc import Iterable, Mapping, Sequence+from typing import Literal, Protocol+from urllib.parse import urlsplit++import structlog+import yaml+from pydantic import BaseModel, ConfigDict, Field++from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult+from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt+from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome++LOG = structlog.get_logger()++BuildTestOutcomePhase = Literal["scout_evaluate", "persisted_block_run", "author_time_reject"]+BuildTestOutcomeVerdict = Literal[+ "progress_observed",+ "repairable_failure",+ "authoring_rejected",+ "not_authoritative",+]+BuildTestOutcomeReasonCode = Literal[+ "loaded_result_targets_observed",+ "runtime_block_failure",+ "sandbox_unresolved_name",+ "synthesized_parameter_binding_ambiguous",+ "code_safety_reject",+ "credential_scout_reject",+ "schema_incompatibility",+ "verified_success",+ "outcome_not_demonstrated",+ "no_meaningful_output",+ "terminal_challenge_blocker",+ "blocker_reported",+ "failed_run",+ "suspicious_success",+ "missing_structural_evidence",+ "unchanged_after_recorded_outcome",+ "metadata_reject",+]++_STRUCTURAL_KEY_VERSION = "recorded_build_test_outcome:v1"+_AUTHORED_STRUCTURE_VERSION = "recorded_build_test_outcome_authored_structure:v1"+_TEXT_MAX = 180+_REF_TEXT_MAX = 96+_HISTORY_LIMIT = 8+_PLAYWRIGHT_LOCATOR_WAIT_RE = re.compile(+ r"waiting for locator\((?P<quote>['\"])(?P<selector>.*?)(?P=quote)\)"+ r"(?P<locator_chain>(?:\.[A-Za-z_][A-Za-z0-9_]*(?:\([^)]*\))?)*)\s+to be (?P<state>[a-z_]+)",+ re.IGNORECASE,+)+_PLAYWRIGHT_HIDDEN_TAG_RE = re.compile(r"locator resolved to hidden <(?P<tag>[a-z0-9:-]+)", re.IGNORECASE)+++class RecordedBuildTestOutcome(BaseModel):+ model_config = ConfigDict(extra="forbid")++ phase: BuildTestOutcomePhase+ attempted_tool: str = ""+ attempted_target: str = ""+ attempted_block_label: str = ""+ verdict: BuildTestOutcomeVerdict+ reason_code: BuildTestOutcomeReasonCode+ observed_evidence_summary: str = ""+ workflow_run_id: str | None = None+ block_labels: list[str] = Field(default_factory=list)+ structural_failure_identity: str = ""+ verified_progress_marker: str = ""+ page_evidence_refs: list[str] = Field(default_factory=list)+ evidence_refs: list[str] = Field(default_factory=list)+ missing_requested_output_facts: list[dict[str, object]] = Field(default_factory=list)+ authored_structure_signature: str | None = None+ display_text: str = ""+ key_provenance: dict[str, str] = Field(default_factory=dict)++ @property+ def structural_key_payload(self) -> dict[str, object] | None:+ if not (self.structural_failure_identity or self.verified_progress_marker or self.page_evidence_refs):+ return None+ return {+ "version": _STRUCTURAL_KEY_VERSION,+ "phase": self.phase,+ "attempted_tool": self.attempted_tool,+ "attempted_target": self.attempted_target,+ "reason_code": self.reason_code,+ "verdict": self.verdict,+ "structural_failure_identity": self.structural_failure_identity,+ "verified_progress_marker": self.verified_progress_marker,+ "page_evidence_refs": sorted(self.page_evidence_refs),+ "evidence_refs": sorted(self.evidence_refs),+ "missing_requested_output_facts": self.missing_requested_output_facts,+ }++ @property+ def structural_key(self) -> str | None:+ payload = self.structural_key_payload+ if payload is None:+ return None+ return _stable_hash(payload)++ @property+ def is_authoritative(self) -> bool:+ return self.structural_key is not None+++class _RecordedBuildTestOutcomeContext(Protocol):+ latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None+ recorded_build_test_outcome_history: list[dict[str, object]]+++def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: RecordedBuildTestOutcome | None) -> None:+ if outcome is None:+ ctx.latest_recorded_build_test_outcome = None+ return+ ctx.latest_recorded_build_test_outcome = outcome+ history = getattr(ctx, "recorded_build_test_outcome_history", None)+ if not isinstance(history, list):+ history = []+ history.append(+ {+ "phase": outcome.phase,+ "reason_code": outcome.reason_code,+ "verdict": outcome.verdict,+ "structural_key": outcome.structural_key,+ "is_authoritative": outcome.is_authoritative,+ "workflow_run_id": outcome.workflow_run_id,+ "authored_structure_signature": outcome.authored_structure_signature,+ }+ )+ del history[:-_HISTORY_LIMIT]+ ctx.recorded_build_test_outcome_history = history+ LOG.info(+ "copilot recorded build-test outcome stored",+ phase=outcome.phase,+ reason_code=outcome.reason_code,+ verdict=outcome.verdict,+ structural_key=outcome.structural_key,+ is_authoritative=outcome.is_authoritative,+ workflow_run_id=outcome.workflow_run_id,+ authored_structure_signature=outcome.authored_structure_signature,+ )+++def authored_structure_signature_from_workflow(+ workflow_yaml: str | None,+ code_artifact_metadata: object = None,+) -> str | None:+ payload = _authored_structure_payload_from_workflow(workflow_yaml, code_artifact_metadata)+ if payload is None:+ return None+ return _stable_hash(payload)+++def latest_recorded_build_test_outcome_repeated(ctx: object) -> bool | None:+ history = getattr(ctx, "recorded_build_test_outcome_history", None)+ if not isinstance(history, list) or not history:+ return None+ latest = history[-1]+ if not isinstance(latest, dict) or not isinstance(latest.get("structural_key"), str):+ return None+ for previous in reversed(history[:-1]):+ if not isinstance(previous, dict):+ continue+ previous_key = previous.get("structural_key")+ if isinstance(previous_key, str):+ return previous_key == latest["structural_key"]+ return None+++def recorded_outcome_from_loaded_result_evidence(+ evidence: LoadedResultCompositionEvidence,+) -> RecordedBuildTestOutcome:+ page_refs = [+ f"result_containers:{evidence.result_container_count}",+ f"table_result_containers:{evidence.table_result_container_count}",+ ]+ for target in evidence.targets:+ page_refs.append(+ "target:"+ + _stable_hash(+ {+ "is_table": target.is_table,+ "row_count": target.row_count,+ "structure_signature": target.structure_signature,+ }+ )+ )+ return RecordedBuildTestOutcome(+ phase="scout_evaluate",+ attempted_tool="evaluate",+ attempted_target="loaded_result_targets",+ verdict="progress_observed",+ reason_code="loaded_result_targets_observed",+ structural_failure_identity=f"loaded_result:{evidence.structure_signature}",+ page_evidence_refs=page_refs,+ observed_evidence_summary=(+ f"{evidence.result_container_count} result container(s), "+ f"{evidence.table_result_container_count} table-like container(s)."+ ),+ key_provenance={+ "structural_failure_identity": "LoadedResultCompositionEvidence.structure_signature",+ "page_evidence_refs": "loaded-result target structural signatures",+ },+ )+++def recorded_outcome_from_run_blocks_result(+ result: Mapping[str, object],+ *,+ page_evidence: Mapping[str, object] | None = None,+ recorded_run_outcome: RecordedRunOutcome | None = None,+ completion_verification: CompletionVerificationResult | None = None,+ authored_structure_signature: str | None = None,+) -> RecordedBuildTestOutcome | None:+ data = _dict(result.get("data"))+ workflow_run_id = _safe_str(data.get("workflow_run_id"))+ blocks = _block_dicts(data.get("blocks"))+ failed_block = _first_failed_block(blocks)+ block_labels = [_safe_str(block.get("label")) for block in blocks if _safe_str(block.get("label"))]+ page_refs = _page_evidence_refs(page_evidence)+ output_refs = _output_evidence_refs(blocks)+ verification_identity = _completion_verification_identity(completion_verification)+ missing_output_facts = _missing_requested_output_facts(completion_verification, blocks)+ if recorded_run_outcome is not None:+ reason_code = _run_outcome_reason_code(recorded_run_outcome)+ if reason_code == "terminal_challenge_blocker":+ return RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="not_authoritative",+ reason_code=reason_code,+ workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,+ block_labels=block_labels,+ authored_structure_signature=authored_structure_signature,+ observed_evidence_summary=recorded_run_outcome.display_reason or "",+ key_provenance={"structural_failure_identity": "terminal blocker precedence suppresses repair prompt"},+ )+ if recorded_run_outcome.verdict == "demonstrated":+ return RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="progress_observed",+ reason_code="verified_success",+ workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,+ block_labels=block_labels,+ verified_progress_marker=verification_identity or "run_completed_verified",+ evidence_refs=output_refs,+ authored_structure_signature=authored_structure_signature,+ observed_evidence_summary=recorded_run_outcome.display_reason or "Completion verification passed.",+ key_provenance={+ "verified_progress_marker": "CompletionVerificationResult satisfied criteria",+ "evidence_refs": "run output structure",+ },+ )+ structural_identity = verification_identity+ evidence_refs = output_refs+ if not structural_identity and not page_refs and not evidence_refs:+ return RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="not_authoritative",+ reason_code=reason_code,+ workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,+ block_labels=block_labels,+ authored_structure_signature=authored_structure_signature,+ observed_evidence_summary=recorded_run_outcome.display_reason or "",+ key_provenance={"structural_failure_identity": "no typed verification/page/output identity available"},+ )+ return RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code=reason_code,+ workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,+ block_labels=block_labels,+ structural_failure_identity=structural_identity,+ page_evidence_refs=page_refs,+ evidence_refs=evidence_refs,+ missing_requested_output_facts=missing_output_facts,+ authored_structure_signature=authored_structure_signature,+ observed_evidence_summary=recorded_run_outcome.display_reason or "",+ key_provenance={+ "structural_failure_identity": "CompletionVerificationResult verdict structure",+ "page_evidence_refs": "bounded post-run page evidence",+ "evidence_refs": "run output structure",+ "missing_requested_output_facts": "CompletionVerificationResult unsatisfied output paths and run output shape",+ },+ )+ run_status = _safe_str(data.get("overall_status"))+ failure_type = _safe_str(data.get("failure_type"))+ failure_categories = _failure_category_refs(data.get("failure_categories"))+ status = _safe_str(failed_block.get("status")) if failed_block is not None else run_status+ runtime_failure_identity = _runtime_failure_identity(failed_block)+ if not (failure_categories or failure_type or runtime_failure_identity or page_refs or output_refs):+ return None+ structural_identity = (+ _stable_hash(+ {+ "failure_type": failure_type,+ "failure_categories": failure_categories,+ "runtime_failure_identity": runtime_failure_identity,+ "status": status,+ }+ )+ if failure_categories or failure_type or runtime_failure_identity+ else ""+ )+ verdict: BuildTestOutcomeVerdict = "repairable_failure" if bool(result.get("ok")) is False else "progress_observed"+ if not structural_identity and not page_refs and not output_refs:+ verdict = "not_authoritative"+ return RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ attempted_block_label=_safe_str(failed_block.get("label")) if failed_block is not None else "",+ verdict=verdict,+ reason_code="runtime_block_failure" if failed_block is not None or not bool(result.get("ok")) else "failed_run",+ workflow_run_id=workflow_run_id or None,+ block_labels=block_labels,+ structural_failure_identity=structural_identity,+ page_evidence_refs=page_refs,+ evidence_refs=output_refs,+ authored_structure_signature=authored_structure_signature,+ observed_evidence_summary=_bounded_text(run_status),+ key_provenance={+ "structural_failure_identity": (+ "typed runtime failure structure"+ if runtime_failure_identity+ else "typed failure categories or failure_type"+ ),+ "page_evidence_refs": "bounded post-run page evidence",+ "evidence_refs": "run output structure",+ },+ )+++def _stable_hash(payload: object) -> str:+ canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()+++def _authored_structure_payload_from_workflow(+ workflow_yaml: str | None,+ code_artifact_metadata: object,+) -> dict[str, object] | None:+ if not isinstance(workflow_yaml, str) or not workflow_yaml.strip():+ return None+ parsed = _parse_workflow_yaml(workflow_yaml)+ if not isinstance(parsed, Mapping):+ return None+ definition = _dict(parsed.get("workflow_definition"))+ code_blocks = _code_block_signature_payloads(definition.get("blocks"))+ if not code_blocks:+ return None+ metadata_by_label = _artifact_metadata_by_label(code_artifact_metadata)+ return {+ "version": _AUTHORED_STRUCTURE_VERSION,+ "workflow_parameter_keys": _workflow_parameter_keys(definition),+ "code_blocks": [+ {+ **block,+ "output_metadata": _artifact_output_metadata_signature(metadata_by_label.get(str(block["label"]))),+ }+ for block in code_blocks+ ],+ }+++def _parse_workflow_yaml(workflow_yaml: str) -> object:+ try:+ return yaml.safe_load(workflow_yaml)+ except yaml.YAMLError:+ return None+++def _workflow_parameter_keys(definition: Mapping[str, object]) -> list[str]:+ keys: list[str] = []+ for parameter in _mapping_list(definition.get("parameters")):+ key = _safe_str(parameter.get("key"))+ if key:+ keys.append(key)+ return sorted(dict.fromkeys(keys))+++def _code_block_signature_payloads(value: object) -> list[dict[str, object]]:+ payloads: list[dict[str, object]] = []+ for block in _mapping_list(value):+ block_type = _safe_str(block.get("block_type")).lower()+ if block_type == "code":+ label = _safe_str(block.get("label"))+ code = block.get("code")+ if label and isinstance(code, str):+ payloads.append(+ {+ "label": label,+ "code_hash": _stable_hash(_normalized_code_text(code)),+ "parameter_keys": _string_list(block.get("parameter_keys")),+ }+ )+ for child_key in ("blocks",):+ payloads.extend(_code_block_signature_payloads(block.get(child_key)))+ for branch in _mapping_list(block.get("branch_conditions")):+ payloads.extend(_code_block_signature_payloads(branch.get("blocks")))+ return sorted(payloads, key=lambda item: str(item.get("label")))+++def _normalized_code_text(code: str) -> str:+ return "\n".join(line.rstrip() for line in textwrap.dedent(code).strip().splitlines())+++def _artifact_metadata_by_label(code_artifact_metadata: object) -> dict[str, Mapping[str, object]]:+ rows: Iterable[tuple[object, object]]+ if isinstance(code_artifact_metadata, Mapping):+ rows = code_artifact_metadata.items()+ elif isinstance(code_artifact_metadata, list):+ rows = [(None, row) for row in code_artifact_metadata]+ else:+ return {}+ by_label: dict[str, Mapping[str, object]] = {}+ for fallback_label, row in rows:+ if not isinstance(row, Mapping):+ continue+ label = _safe_str(row.get("block_label")) or _safe_str(fallback_label)+ if label:+ by_label[label] = row+ return by_label+++def _artifact_output_metadata_signature(artifact: Mapping[str, object] | None) -> dict[str, object]:+ if artifact is None:+ return {}+ return {+ "claimed_outcomes": _output_rows_signature(artifact.get("claimed_outcomes")),+ "terminal_verifier_expectations": _output_rows_signature(artifact.get("terminal_verifier_expectations")),+ }+++def _output_rows_signature(value: object) -> list[dict[str, object]]:+ rows: list[dict[str, object]] = []+ for row in _mapping_list(value):+ rows.append(+ {+ "goal_value_paths": _string_list(row.get("goal_value_paths")),+ "extraction_schema_paths": _extraction_schema_paths(row.get("extraction_schema")),+ }+ )+ return rows+++def _extraction_schema_paths(value: object) -> list[str]:+ schema: object = value+ if isinstance(value, str):+ try:+ schema = json.loads(value)+ except json.JSONDecodeError:+ return []+ if not isinstance(schema, Mapping):+ return []+ paths: list[str] = []+ _collect_schema_paths(schema, prefix="", paths=paths)+ return sorted(dict.fromkeys(paths))+++def _collect_schema_paths(schema: Mapping[str, object], *, prefix: str, paths: list[str]) -> None:+ schema_type = _safe_str(schema.get("type"))+ if prefix:+ paths.append(f"{prefix}:{schema_type or 'unknown'}")+ properties = schema.get("properties")+ if isinstance(properties, Mapping):+ for key, child in sorted(properties.items(), key=lambda item: str(item[0])):+ if isinstance(child, Mapping):+ child_prefix = f"{prefix}.{key}" if prefix else str(key)+ _collect_schema_paths(child, prefix=child_prefix, paths=paths)+ items = schema.get("items")+ if isinstance(items, Mapping):+ _collect_schema_paths(items, prefix=f"{prefix}[]" if prefix else "[]", paths=paths)+++def _string_list(value: object) -> list[str]:+ if not isinstance(value, list):+ return []+ return sorted(dict.fromkeys(item for item in value if isinstance(item, str)))+++def _bounded_text(value: object, max_chars: int = _TEXT_MAX) -> str:+ if not isinstance(value, str):+ return ""+ text = redact_raw_secrets_for_prompt(" ".join(value.split()))+ return text[:max_chars]+++def _bounded_ref(value: object, max_chars: int = _REF_TEXT_MAX) -> str:+ if not isinstance(value, str):+ return ""+ return " ".join(value.split())[:max_chars]+++def _safe_str(value: object) -> str:+ return value if isinstance(value, str) else ""+++def _dict(value: object) -> Mapping[str, object]:+ return value if isinstance(value, Mapping) else {}+++def _block_dicts(value: object) -> list[Mapping[str, object]]:+ return [item for item in value if isinstance(item, Mapping)] if isinstance(value, list) else []+++def _first_failed_block(blocks: Sequence[Mapping[str, object]]) -> Mapping[str, object] | None:+ for block in blocks:+ if _safe_str(block.get("status")).lower() in {"failed", "terminated", "canceled", "timed_out"}:+ return block+ return None+++def _runtime_failure_identity(failed_block: Mapping[str, object] | None) -> str:+ if failed_block is None:+ return ""+ failure_reason = _safe_str(failed_block.get("failure_reason"))+ if not failure_reason:+ return ""+ locator_wait_match = _PLAYWRIGHT_LOCATOR_WAIT_RE.search(failure_reason)+ if locator_wait_match is None:+ return ""+ hidden_tag_match = _PLAYWRIGHT_HIDDEN_TAG_RE.search(failure_reason)+ selector = _bounded_ref(locator_wait_match.group("selector"))+ locator_chain = _bounded_ref(locator_wait_match.group("locator_chain"))+ state = _bounded_ref(locator_wait_match.group("state").casefold())+ hidden_tag = _bounded_ref(hidden_tag_match.group("tag").casefold()) if hidden_tag_match is not None else ""+ return _stable_hash(+ {+ "source": "playwright_locator_wait",+ "selector": selector,+ "locator_chain": locator_chain,+ "state": state,+ "hidden_tag": hidden_tag,+ "block_label": _safe_str(failed_block.get("label")),+ "block_status": _safe_str(failed_block.get("status")),+ }+ )+++def _clean_list(values: Sequence[str]) -> list[str]:+ return [cleaned for value in values for cleaned in [_bounded_ref(value)] if cleaned]+++def _origin_ref(value: object) -> str:+ if not isinstance(value, str) or not value.strip():+ return ""+ parsed = urlsplit(value)+ if not parsed.scheme or not parsed.netloc:+ return ""+ return f"origin:{parsed.scheme}://{parsed.netloc}"+++def _page_evidence_refs(page_evidence: Mapping[str, object] | None) -> list[str]:+ if page_evidence is None:+ return []+ refs: list[str] = []+ origin = _origin_ref(page_evidence.get("current_url")) or _origin_ref(page_evidence.get("inspected_url"))+ if origin:+ refs.append(origin)+ refs.extend(_form_refs(page_evidence.get("forms")))+ refs.extend(_result_refs(page_evidence.get("result_containers")))+ refs.extend(_action_refs(page_evidence.get("navigation_targets")))+ return refs[:12]+++def _form_refs(value: object) -> list[str]:+ if not isinstance(value, list):+ return []+ refs: list[str] = []+ for form in value[:3]:+ if not isinstance(form, Mapping):+ continue+ for field in _mapping_list(form.get("fields"))[:3]:+ label = _bounded_ref(field.get("label"))+ selector = _bounded_ref(field.get("selector"))+ if label or selector:+ refs.append(f"form:{' '.join(item for item in (label, selector) if item)}")+ for control in _mapping_list(form.get("submit_controls"))[:2]:+ text = _bounded_ref(control.get("text"))+ selector = _bounded_ref(control.get("selector"))+ if text or selector:+ refs.append(f"submit:{' '.join(item for item in (text, selector) if item)}")+ return refs+++def _result_refs(value: object) -> list[str]:+ refs: list[str] = []+ for container in _mapping_list(value)[:4]:+ selector = _bounded_ref(container.get("selector")) or "unknown"+ row_count = container.get("row_count")+ row_text = str(row_count) if isinstance(row_count, int) else "unknown"+ refs.append(f"result:{selector} rows={row_text}")+ return refs+++def _action_refs(value: object) -> list[str]:+ refs: list[str] = []+ for action in _mapping_list(value)[:4]:+ text = _bounded_ref(action.get("text"))+ selector = _bounded_ref(action.get("selector"))+ if text or selector:+ refs.append(f"action:{' '.join(item for item in (text, selector) if item)}")+ return refs+++def _mapping_list(value: object) -> list[Mapping[str, object]]:+ return [item for item in value if isinstance(item, Mapping)] if isinstance(value, list) else []+++def _run_outcome_reason_code(recorded_run_outcome: RecordedRunOutcome) -> BuildTestOutcomeReasonCode:+ reason_code = recorded_run_outcome.reason_code+ if reason_code in {+ "outcome_not_demonstrated",+ "no_meaningful_output",+ "terminal_challenge_blocker",+ "blocker_reported",+ }:+ return reason_code+ if recorded_run_outcome.verdict == "demonstrated":+ return "verified_success"+ return "failed_run"+++def _completion_verification_identity(completion_verification: CompletionVerificationResult | None) -> str:+ if completion_verification is None or completion_verification.status != "evaluated":+ return ""+ verdict_payload = [+ {+ "criterion_id": verdict.criterion_id,+ "state": verdict.state,+ "reason_code": verdict.reason_code,+ "output_path": verdict.output_path,+ "grounding_mode": verdict.grounding_mode,+ "expected_output_shape": verdict.expected_output_shape,+ "has_exact_value": verdict.has_exact_value,+ }+ for verdict in completion_verification.verdicts+ ]+ payload = {+ "criterion_ids": sorted(completion_verification.criterion_ids),+ "verdicts": verdict_payload,+ "no_gradeable_run_plane": completion_verification.no_gradeable_run_plane,+ "structural_unfired_criterion_ids": sorted(completion_verification.structural_unfired_criterion_ids),+ }+ return "completion:" + _stable_hash(payload)+++def _missing_requested_output_facts(+ completion_verification: CompletionVerificationResult | None,+ blocks: Sequence[Mapping[str, object]],+) -> list[dict[str, object]]:+ if completion_verification is None or completion_verification.status != "evaluated":+ return []+ empty_output_block_labels = _empty_output_block_labels(blocks)+ facts: list[dict[str, object]] = []+ for verdict in completion_verification.verdicts:+ if verdict.satisfied or not verdict.output_path:+ continue+ output_path = _bounded_ref(verdict.output_path)+ output_root = _output_path_root(output_path)+ if not output_root:+ continue+ fact: dict[str, object] = {+ "criterion_id": _bounded_ref(verdict.criterion_id),+ "output_path": output_path,+ "output_root": output_root,+ "reason_code": _bounded_ref(verdict.reason_code),+ "value_status": _output_path_value_status(blocks, output_path, verdict),+ }+ if verdict.grounding_mode:+ fact["grounding_mode"] = verdict.grounding_mode+ if verdict.expected_output_shape:+ fact["expected_output_shape"] = _bounded_ref(verdict.expected_output_shape)+ if empty_output_block_labels:+ fact["empty_output_block_labels"] = empty_output_block_labels+ partial_labels = _partial_output_block_labels(blocks, output_root)+ if partial_labels:+ fact["partial_output_block_labels"] = partial_labels+ facts.append(fact)+ return sorted(facts, key=lambda item: str(item.get("output_path") or ""))+++def _output_path_root(output_path: str) -> str:+ return _bounded_ref(output_path.split(".", 1)[0].split("[", 1)[0])+++def _output_path_value_status(+ blocks: Sequence[Mapping[str, object]],+ output_path: str,+ verdict: object,+) -> str:+ values: list[object] = []+ for block in blocks:+ extracted = block.get("extracted_data")+ if extracted is None:+ continue+ value, present = _value_at_output_path(extracted, output_path)+ if present:+ values.append(value)+ if not values:+ return "no_typed_value"+ if all(_is_empty_output_value(value) for value in values):+ return "empty_typed_value"+ grounding_mode = getattr(verdict, "grounding_mode", None)+ has_exact_value = getattr(verdict, "has_exact_value", False)+ if grounding_mode == "shape" and not has_exact_value:+ return "presence_only_evidence"+ return "typed_value_unverified"+++def _value_at_output_path(value: object, output_path: str) -> tuple[object | None, bool]:+ current = value+ for segment in [part for part in re.split(r"\.|\[\]", output_path) if part]:+ if isinstance(current, Mapping):+ if segment not in current:+ return None, False+ current = current[segment]+ continue+ if isinstance(current, list):+ found_values = [item.get(segment) for item in current if isinstance(item, Mapping) and segment in item]+ if not found_values:+ return None, False+ current = found_values+ continue+ return None, False+ return current, True+++def _is_empty_output_value(value: object) -> bool:+ if value is None:+ return True+ if isinstance(value, str):+ return not value.strip()+ if isinstance(value, (list, tuple, set, dict)):+ return len(value) == 0+ return False+++def _empty_output_block_labels(blocks: Sequence[Mapping[str, object]]) -> list[str]:+ labels: list[str] = []+ for block in blocks:+ extracted = block.get("extracted_data")+ if extracted is None:+ continue+ if _is_empty_output_value(extracted):+ label = _bounded_ref(block.get("label"))+ if label:+ labels.append(label)+ return labels+++def _partial_output_block_labels(blocks: Sequence[Mapping[str, object]], output_root: str) -> list[str]:+ labels: list[str] = []+ for block in blocks:+ extracted = block.get("extracted_data")+ if not isinstance(extracted, Mapping) or output_root in extracted or not extracted:+ continue+ label = _bounded_ref(block.get("label"))+ if label:+ labels.append(label)+ return labels+++def _failure_category_refs(value: object) -> list[str]:+ refs: list[str] = []+ for entry in _mapping_list(value):+ category = _bounded_ref(entry.get("category"))+ reason = _bounded_ref(entry.get("reason_code"))+ if category or reason:+ refs.append(":".join(part for part in (category, reason) if part))+ return refs+++def _output_evidence_refs(blocks: Sequence[Mapping[str, object]]) -> list[str]:+ refs: list[str] = []+ for block in blocks[:8]:+ extracted = block.get("extracted_data")+ if extracted is None:+ continue+ refs.append("output:" + _stable_hash(_value_shape(extracted)))+ return refs+++def _value_shape(value: object, *, depth: int = 0) -> object:+ if depth > 6:+ return "max_depth"+ if isinstance(value, Mapping):+ return {+ str(key): _value_shape(item, depth=depth + 1)+ for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))+ }+ if isinstance(value, list):+ return {+ "type": "list",+ "length": len(value),+ "items": [_value_shape(item, depth=depth + 1) for item in value[:3]],+ }+ if isinstance(value, bool):+ return {"type": "bool", "value": value}+ if isinstance(value, (int, float)) and not isinstance(value, bool):+ return {"type": type(value).__name__, "zero": value == 0}+ if isinstance(value, str):+ return {"type": "str", "empty": value == ""}+ if value is None:+ return "none"+ return type(value).__name__diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 9f164121a..246ac4189 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -97,6 +97,7 @@ class TurnNarrativePayload(TypedDict):if TYPE_CHECKING:from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal+ from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcomefrom skyvern.forge.sdk.copilot.completion_criteria_store import CompletionCriteriaTurnStatefrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import DiagnosisRepairContractfrom skyvern.forge.sdk.copilot.narration import NarratorState@@ -619,6 +620,8 @@ class CopilotContext(AgentContext):last_code_authoring_reject_was_credential_priority: bool = Falsepending_code_authoring_runtime_repair_context: CodeAuthoringRepairContext | None = Nonelast_code_authoring_repair_context: CodeAuthoringRepairContext | None = None+ latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None = None+ recorded_build_test_outcome_history: list[dict[str, object]] = field(default_factory=list)# Turn-scoped monotonic marks of verified forward progress: the union of# completion criteria the judge confirmed satisfied so far this turn, and the# high-water length of the verified block prefix. A repair that grows eitherdiff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex f227ef346..7dc3068c7 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -25,6 +25,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (stash_blocker_signal,)from skyvern.forge.sdk.copilot.build_phase import DISCOVERY_PERMITTED_PHASES+from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcomefrom skyvern.forge.sdk.copilot.code_block_synthesis import (is_durable_fallback_entry_target,is_generic_entry_opener_click,@@ -167,6 +168,9 @@ SYNTHESIZED_BLOCK_PERSISTENCE_TOOL = "update_and_run_blocks"_SYNTHESIZED_BLOCK_PERSISTENCE_ALLOWED_TOOLS = frozenset({SYNTHESIZED_BLOCK_PERSISTENCE_TOOL, "fill_credential_field", "update_workflow"})+_SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS = frozenset(+ {"click", "press_key", "type_text", "select_option", "navigate_browser"}+)_SYNTHESIZED_BLOCK_COMMIT_TOOLS = frozenset({"click", "press_key"})# OpenAI detail=high cost per resized image. If we support other providers,# pull from model config — this value will silently over/undercount otherwise.@@ -2127,10 +2131,63 @@ def _should_force_synthesized_block_persistence(ctx: Any) -> bool:return synthesized_trajectory_is_goal_complete(ctx)+def _should_block_mutating_tool_after_synthesized_offer(ctx: Any, tool_name: str) -> bool:+ if tool_name not in _SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS:+ return False+ if getattr(ctx, "update_workflow_called", False) and not synthesized_persistence_reopened_after_failed_run(ctx):+ return False+ if not _turn_intent_can_update_and_run_without_user_input(getattr(ctx, "turn_intent", None)):+ return False+ if normalize_block_authoring_policy(getattr(ctx, "block_authoring_policy", None)) != (+ BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ):+ return False+ if not getattr(ctx, "synthesized_block_offered", False):+ return False+ trajectory = getattr(ctx, "scout_trajectory", None) or []+ return (getattr(ctx, "synthesized_block_offered_trajectory_len", 0) or 0) == len(trajectory)+++def _should_block_mutating_tool_after_unresolved_recorded_outcome(ctx: Any, tool_name: str) -> bool:+ if tool_name not in _SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS:+ return False+ if not _turn_intent_can_update_and_run_without_user_input(getattr(ctx, "turn_intent", None)):+ return False+ if normalize_block_authoring_policy(getattr(ctx, "block_authoring_policy", None)) != (+ BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ):+ return False+ latest = getattr(ctx, "latest_recorded_build_test_outcome", None)+ if not isinstance(latest, RecordedBuildTestOutcome) or not latest.is_authoritative:+ return False+ if latest.phase != "persisted_block_run" or latest.reason_code != "outcome_not_demonstrated":+ return False+ return _completion_verification_unsatisfied(ctx)++def synthesized_block_persistence_signal(ctx: Any, tool_name: str) -> CopilotToolBlockerSignal | None:if tool_name in _SYNTHESIZED_BLOCK_PERSISTENCE_ALLOWED_TOOLS:return None- if not _should_force_synthesized_block_persistence(ctx):+ if _should_block_mutating_tool_after_unresolved_recorded_outcome(ctx, tool_name):+ return CopilotToolBlockerSignal(+ blocker_kind="tool_error",+ agent_steering_text=(+ "The last recorded test outcome is authoritative and still has unsatisfied completion criteria. "+ f"Call {SYNTHESIZED_BLOCK_PERSISTENCE_TOOL} with a materially changed authored workflow now; "+ "do not spend the repair attempt on browser interaction."+ ),+ user_facing_reason="I need to revise and test the workflow code instead of interacting with the page.",+ recovery_hint="retry_with_different_tool",+ cleared_by_tools=frozenset({SYNTHESIZED_BLOCK_PERSISTENCE_TOOL}),+ preserves_workflow_draft=True,+ renders_final_reply=False,+ internal_reason_code=SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,+ blocked_tool=tool_name,+ extra={"recorded_outcome_reason_code": "outcome_not_demonstrated"},+ )+ if not _should_force_synthesized_block_persistence(ctx) and not _should_block_mutating_tool_after_synthesized_offer(+ ctx, tool_name+ ):return Nonereturn CopilotToolBlockerSignal(blocker_kind="tool_error",diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex a0ab45709..9e7f777d6 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -35,6 +35,7 @@ from skyvern.library.skyvern_browser import SkyvernBrowserif TYPE_CHECKING:from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal+ from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcomefrom skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResultfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContextfrom skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget@@ -241,6 +242,8 @@ class AgentContext:last_run_blocks_block_labels: list[str] = field(default_factory=list)last_run_outcome: RecordedRunOutcome | None = Nonelast_run_outcome_block_labels: list[str] = field(default_factory=list)+ latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None = None+ recorded_build_test_outcome_history: list[dict[str, object]] = field(default_factory=list)completion_verification_result: CompletionVerificationResult | None = Noneverified_terminal_proposal_ready: bool = Falseoutcome_verification_trace_snapshot: dict[str, Any] = field(default_factory=dict)diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 5d8b758aa..41ff011e9 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -23,6 +23,12 @@ from skyvern.forge.sdk.copilot.blocker_signal import (clear_blocker_signal_for_reason_codes,stash_blocker_signal,)+from skyvern.forge.sdk.copilot.build_test_outcome import (+ RecordedBuildTestOutcome,+ authored_structure_signature_from_workflow,+ record_build_test_outcome,+ recorded_outcome_from_run_blocks_result,+)from skyvern.forge.sdk.copilot.code_block_security import (COPILOT_CODE_SECURITY_FAILURE_CATEGORY,CodeBlockSecurityInput,@@ -2184,15 +2190,14 @@ def _record_run_blocks_result(stash_turn_halt_from_blocker_signal(copilot_ctx, signal, source="run_execution")update_repeated_failure_state(copilot_ctx, result)_update_verification_evidence_from_run_result(copilot_ctx, result)- return _stash_recorded_run_outcome(- copilot_ctx,- RecordedRunOutcome(- verdict="not_demonstrated",- reason_code=TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,- display_reason=run_outcome_display_reason(terminal_challenge.reason),- workflow_run_id=terminal_challenge.workflow_run_id,- ),+ recorded_outcome = RecordedRunOutcome(+ verdict="not_demonstrated",+ reason_code=TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,+ display_reason=run_outcome_display_reason(terminal_challenge.reason),+ workflow_run_id=terminal_challenge.workflow_run_id,)+ _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, recorded_outcome)+ return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)if run_ok:_mark_page_inspected(copilot_ctx)@@ -2220,14 +2225,13 @@ def _record_run_blocks_result(data.setdefault("failure_reason", failure_reason)update_repeated_failure_state(copilot_ctx, result)_update_verification_evidence_from_run_result(copilot_ctx, result)- return _stash_recorded_run_outcome(- copilot_ctx,- RecordedRunOutcome(- verdict="not_demonstrated",- reason_code="blocker_reported",- display_reason=run_outcome_display_reason(structured_blocker),- ),+ recorded_outcome = RecordedRunOutcome(+ verdict="not_demonstrated",+ reason_code="blocker_reported",+ display_reason=run_outcome_display_reason(structured_blocker),)+ _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, recorded_outcome)+ return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)if completion_fully_satisfied:# ``verified_terminal_proposal_ready`` is telemetry only (the barrier keys# on ``outcome_fully_verified(ctx)``); clearing the stale suspicious-success@@ -2249,14 +2253,13 @@ def _record_run_blocks_result(copilot_ctx.probable_site_block_streak_count = 0update_repeated_failure_state(copilot_ctx, result)_update_verification_evidence_from_run_result(copilot_ctx, result)- return _stash_recorded_run_outcome(- copilot_ctx,- RecordedRunOutcome(- verdict="not_demonstrated",- reason_code="no_meaningful_output",- display_reason=run_outcome_display_reason(copilot_ctx.last_test_failure_reason),- ),+ recorded_outcome = RecordedRunOutcome(+ verdict="not_demonstrated",+ reason_code="no_meaningful_output",+ display_reason=run_outcome_display_reason(copilot_ctx.last_test_failure_reason),)+ _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, recorded_outcome)+ return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)if prior_committed_outcome is not None and artifact_reason is None:# artifact_reason is current-run health; prior_committed_outcome already passed prior ctx artifact-health.copilot_ctx.last_full_workflow_test_ok = True@@ -2312,7 +2315,9 @@ def _record_run_blocks_result()update_repeated_failure_state(copilot_ctx, result)_update_verification_evidence_from_run_result(copilot_ctx, result)- return _stash_recorded_run_outcome(copilot_ctx, _adjudicated_run_outcome(copilot_ctx, completion_verification))+ recorded_outcome = _adjudicated_run_outcome(copilot_ctx, completion_verification)+ _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, recorded_outcome)+ return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)if outcome_fully_verified(copilot_ctx):copilot_ctx.last_test_suspicious_success = False@@ -2327,7 +2332,9 @@ def _record_run_blocks_result(copilot_ctx.last_good_workflow_yaml = copilot_ctx.last_workflow_yamlupdate_repeated_failure_state(copilot_ctx, result)_update_verification_evidence_from_run_result(copilot_ctx, result)- return _stash_recorded_run_outcome(copilot_ctx, _adjudicated_run_outcome(copilot_ctx, completion_verification))+ recorded_outcome = _adjudicated_run_outcome(copilot_ctx, completion_verification)+ _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, recorded_outcome)+ return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)copilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)copilot_ctx.last_test_non_retriable_nav_error = _detect_non_retriable_nav_error(result)@@ -2350,9 +2357,31 @@ def _record_run_blocks_result(copilot_ctx.last_test_failure_reason = str(result["error"])update_repeated_failure_state(copilot_ctx, result)_update_verification_evidence_from_run_result(copilot_ctx, result)+ _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, None)return None+def _record_adjudicated_build_test_outcome(+ copilot_ctx: Any,+ result: dict[str, Any],+ completion_verification: CompletionVerificationResult | None,+ recorded_run_outcome: RecordedRunOutcome | None,+) -> None:+ record_build_test_outcome(+ copilot_ctx,+ recorded_outcome_from_run_blocks_result(+ result,+ page_evidence=getattr(copilot_ctx, "composition_page_evidence", None),+ recorded_run_outcome=recorded_run_outcome,+ completion_verification=completion_verification,+ authored_structure_signature=authored_structure_signature_from_workflow(+ getattr(copilot_ctx, "workflow_yaml", None),+ getattr(copilot_ctx, "code_artifact_metadata", None),+ ),+ ),+ )++def _stash_recorded_run_outcome(copilot_ctx: Any, outcome: RecordedRunOutcome) -> RecordedRunOutcome:if outcome.workflow_run_id is None:outcome = replace(outcome, workflow_run_id=getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None))@@ -2490,9 +2519,12 @@ async def _verify_and_record_run_blocks_result(return completion_verification-def _repair_non_convergence_signature(contract: DiagnosisRepairContract) -> str | None:+def _repair_non_convergence_signature(copilot_ctx: Any, contract: DiagnosisRepairContract) -> str | None:if contract.repair_decision.next_action is not RepairNextAction.REPAIR:return None+ recorded = getattr(copilot_ctx, "latest_recorded_build_test_outcome", None)+ if isinstance(recorded, RecordedBuildTestOutcome) and recorded.structural_key is not None:+ return f"recorded_build_test_outcome:{recorded.structural_key}"identity = contract.diagnosis_result.root_cause_identityif identity.primary_category == _AUTHORING_REPAIR_CATEGORY and identity.root_cause_signature:return identity.root_cause_signature@@ -2533,7 +2565,7 @@ def _update_repair_loop_state(copilot_ctx: Any, contract: DiagnosisRepairContracif progressed:reset_no_progress_interaction_count(copilot_ctx)- signature = _repair_non_convergence_signature(contract)+ signature = _repair_non_convergence_signature(copilot_ctx, contract)if signature is None or progressed:copilot_ctx.consecutive_non_converging_repair_count = 0copilot_ctx.last_repair_non_convergence_signature = Nonediff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.pyindex e30f8fa45..11729299d 100644--- a/skyvern/forge/sdk/copilot/tools/scouting.py+++ b/skyvern/forge/sdk/copilot/tools/scouting.py@@ -12,6 +12,10 @@ import structlogfrom skyvern.config import settingsfrom skyvern.forge import app+from skyvern.forge.sdk.copilot.build_test_outcome import (+ record_build_test_outcome,+ recorded_outcome_from_loaded_result_evidence,+)from skyvern.forge.sdk.copilot.composition_browser_expressions import (scout_accessible_role_name_expression as _scout_accessible_role_name_expression,)@@ -945,6 +949,7 @@ async def _maybe_steer_evaluate_to_action(if loaded_results is not None:_reset_evaluate_tracker(ctx)ctx.latest_evaluate_result_composition_steer = loaded_results+ record_build_test_outcome(ctx, recorded_outcome_from_loaded_result_evidence(loaded_results))data.pop("actionable_targets", None)data["composition_targets"] = loaded_result_composition_target_summary(loaded_results)data["next_action"] = "compose_extraction"diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 67916620f..932ff66a9 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -22,6 +22,7 @@ from skyvern.forge.sdk.copilot.agent import ()from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignalfrom skyvern.forge.sdk.copilot.build_phase import BuildPhase+from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcomefrom skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODEfrom skyvern.forge.sdk.copilot.completion_criteria_store import (StoredCriteriaSet,@@ -836,6 +837,30 @@ workflow_definition:assert "adapt the next code block to the observed page state" in promptassert "do not re-emit the same failing selector or name path" in prompt+ def test_recorded_build_test_outcome_prompt_renders_structural_grounding(self) -> None:+ ctx = _ctx(+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,+ latest_recorded_build_test_outcome=RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ attempted_block_label="search_records",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_failed",+ structural_failure_identity="runtime:timeout_waiting_for_selector:failed",+ page_evidence_refs=["form:Search #search", "result:#results rows=unknown"],+ observed_evidence_summary="Timeout waiting for results.",+ ),+ )++ prompt = agent_module._recorded_build_test_outcome_prompt(ctx)++ assert "RECORDED BUILD-TEST OUTCOME" in prompt+ assert "phase: persisted_block_run" in prompt+ assert "reason_code: runtime_block_failure" in prompt+ assert "page_evidence_refs: form:Search #search, result:#results rows=unknown" in prompt+ assert "Do not re-emit the same plan against the same structural key" in prompt+class TestVerifiedWorkflowOrNone:"""SKY-9143 strict invariant: a proposal surfaces only after a passing test this turn."""diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.pynew file mode 100644index 000000000..19056103e--- /dev/null+++ b/tests/unit/test_copilot_build_test_outcome.py@@ -0,0 +1,429 @@+from __future__ import annotations++from types import SimpleNamespace++from skyvern.forge.sdk.copilot.build_test_outcome import (+ RecordedBuildTestOutcome,+ authored_structure_signature_from_workflow,+ record_build_test_outcome,+ recorded_outcome_from_loaded_result_evidence,+ recorded_outcome_from_run_blocks_result,+)+from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict+from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence, LoadedResultCompositionTarget+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome+++def test_structural_key_changes_when_page_or_result_structure_changes() -> None:+ first = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_first",+ block_labels=["search_records"],+ structural_failure_identity="runtime:timeout_waiting_for_selector:failed",+ page_evidence_refs=["origin_present", "results:empty"],+ observed_evidence_summary="No matching records.",+ )+ second = first.model_copy(+ update={+ "workflow_run_id": "wr_second",+ "page_evidence_refs": ["origin_present", "results:table_rows"],+ "observed_evidence_summary": "A table with one result row appeared.",+ }+ )++ assert first.structural_key is not None+ assert second.structural_key is not None+ assert first.structural_key != second.structural_key+++def test_structural_key_ignores_display_prose_and_workflow_run_id() -> None:+ first = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_first",+ block_labels=["search_records"],+ structural_failure_identity="runtime:timeout_waiting_for_selector:failed",+ page_evidence_refs=["origin_present", "form:search"],+ observed_evidence_summary="Timeout waiting for #results.",+ display_text="The page did not show results.",+ )+ second = first.model_copy(+ update={+ "workflow_run_id": "wr_second",+ "observed_evidence_summary": "Different explanation with the same structural observation.",+ "display_text": "Another user-facing sentence.",+ }+ )++ assert first.structural_key == second.structural_key+++def test_structural_key_ignores_authored_signature_but_retains_it() -> None:+ first = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:typed-outcome",+ page_evidence_refs=["origin:https://example.com", "result:#results rows=0"],+ authored_structure_signature="authored:first",+ )+ second = first.model_copy(update={"authored_structure_signature": "authored:second"})++ assert first.structural_key is not None+ assert first.structural_key == second.structural_key+ assert first.authored_structure_signature == "authored:first"+ assert second.authored_structure_signature == "authored:second"+++def test_structural_key_does_not_require_fixture_slug_or_raw_transcript_text() -> None:+ outcome = recorded_outcome_from_loaded_result_evidence(+ LoadedResultCompositionEvidence(+ result_container_count=1,+ table_result_container_count=1,+ targets=(+ LoadedResultCompositionTarget(+ selector="#results",+ is_table=True,+ row_count=3,+ sample_rows=("synthetic row text that must not become identity",),+ text_excerpt="synthetic transcript excerpt that must not become identity",+ structure_signature="target-structure",+ ),+ ),+ structure_signature="result-structure",+ )+ )++ assert outcome.structural_key is not None+ key_payload = outcome.structural_key_payload+ assert "fixture" not in str(key_payload).lower()+ assert "synthetic row text" not in str(key_payload)+ assert "synthetic transcript excerpt" not in str(key_payload)+++def test_prose_or_label_only_typed_outcome_is_not_authoritative() -> None:+ outcome = RecordedBuildTestOutcome(+ phase="author_time_reject",+ attempted_tool="update_workflow",+ attempted_block_label="search_records",+ verdict="authoring_rejected",+ reason_code="code_safety_reject",+ block_labels=["search_records"],+ observed_evidence_summary="This sounds like the same failure.",+ display_text="Use the prior failure reason.",+ )++ assert outcome.structural_key is None+ assert outcome.is_authoritative is False+++def test_record_none_clears_stale_latest_outcome() -> None:+ ctx = SimpleNamespace(latest_recorded_build_test_outcome=None, recorded_build_test_outcome_history=[])+ record_build_test_outcome(+ ctx,+ RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ structural_failure_identity="typed-runtime-identity",+ ),+ )+ assert ctx.latest_recorded_build_test_outcome is not None++ record_build_test_outcome(ctx, None)++ assert ctx.latest_recorded_build_test_outcome is None+++def test_authored_structure_signature_is_stable_and_excludes_raw_code_or_prose() -> None:+ workflow_yaml = """+ title: Registry lookup+ workflow_definition:+ parameters:+ - parameter_type: workflow+ workflow_parameter_type: string+ key: provider_query+ blocks:+ - block_type: code+ label: search_registry+ parameter_keys:+ - provider_query+ code: |+ await page.goto("https://example.com/search")+ return {"records": [{"npi": "123"}]}+ """+ metadata = [+ {+ "block_label": "search_registry",+ "declared_goal": "Find the exact provider row from the page prose.",+ "claimed_outcomes": [+ {+ "id": "claim:provider",+ "text": "The provider was found in the directory.",+ "goal_value_paths": ["records[].npi"],+ "extraction_schema": {+ "type": "object",+ "properties": {"records": {"type": "array", "items": {"type": "object"}}},+ },+ }+ ],+ }+ ]+ prose_changed_metadata = [+ {+ **metadata[0],+ "declared_goal": "Different page prose for the same structure.",+ "claimed_outcomes": [{**metadata[0]["claimed_outcomes"][0], "text": "Different prose."}],+ }+ ]++ signature = authored_structure_signature_from_workflow(workflow_yaml, metadata)+ same_structure = authored_structure_signature_from_workflow(workflow_yaml, prose_changed_metadata)+ dumped = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:typed",+ authored_structure_signature=signature,+ ).model_dump(mode="json")++ assert signature is not None+ assert signature == same_structure+ assert "page.goto" not in str(dumped)+ assert "Find the exact provider" not in str(dumped)+++def test_authored_structure_signature_changes_on_code_parameter_or_output_structure() -> None:+ base = """+ title: Registry lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: search_registry+ parameter_keys:+ - provider_query+ code: |+ return {"records": [{"npi": "123"}]}+ """+ changed_code = base.replace('"123"', '"456"')+ changed_parameter = base.replace("provider_query", "provider_name")+ metadata = [{"block_label": "search_registry", "claimed_outcomes": [{"goal_value_paths": ["records[].npi"]}]}]+ changed_metadata = [+ {"block_label": "search_registry", "claimed_outcomes": [{"goal_value_paths": ["records[].license"]}]}+ ]++ signature = authored_structure_signature_from_workflow(base, metadata)++ assert signature is not None+ assert authored_structure_signature_from_workflow(changed_code, metadata) != signature+ assert authored_structure_signature_from_workflow(changed_parameter, metadata) != signature+ assert authored_structure_signature_from_workflow(base, changed_metadata) != signature+++def test_outcome_not_demonstrated_keeps_authoritative_unsatisfied_criteria_identity() -> None:+ result = {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_partial",+ "overall_status": "completed",+ "blocks": [+ {+ "label": "lookup_provider_and_extract_credentials",+ "status": "completed",+ "extracted_data": {"npi": "", "evidence_text": "address and statuses appear in page text"},+ }+ ],+ },+ }+ verification = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["npi", "locations", "statuses", "overall"],+ verdicts=[+ CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence", output_path="npi"),+ CriterionVerdict(+ criterion_id="locations",+ state="unsatisfied",+ reason_code="no_evidence",+ output_path="locations",+ ),+ CriterionVerdict(+ criterion_id="statuses",+ state="unsatisfied",+ reason_code="no_evidence",+ output_path="credentialing_statuses",+ ),+ CriterionVerdict(+ criterion_id="overall",+ state="unsatisfied",+ reason_code="no_evidence",+ output_path="overall_credentialing_status",+ ),+ ],+ )++ outcome = recorded_outcome_from_run_blocks_result(+ result,+ recorded_run_outcome=RecordedRunOutcome(+ verdict="not_demonstrated",+ reason_code="outcome_not_demonstrated",+ workflow_run_id="wr_partial",+ ),+ completion_verification=verification,+ authored_structure_signature="authored:partial-location-only",+ )++ assert outcome is not None+ assert outcome.phase == "persisted_block_run"+ assert outcome.reason_code == "outcome_not_demonstrated"+ assert outcome.workflow_run_id == "wr_partial"+ assert outcome.is_authoritative is True+ assert outcome.structural_key is not None+ assert outcome.authored_structure_signature == "authored:partial-location-only"+ assert outcome.key_provenance["structural_failure_identity"] == "CompletionVerificationResult verdict structure"+ assert outcome.missing_requested_output_facts == [+ {+ "criterion_id": "statuses",+ "output_path": "credentialing_statuses",+ "output_root": "credentialing_statuses",+ "reason_code": "no_evidence",+ "value_status": "no_typed_value",+ "partial_output_block_labels": ["lookup_provider_and_extract_credentials"],+ },+ {+ "criterion_id": "locations",+ "output_path": "locations",+ "output_root": "locations",+ "reason_code": "no_evidence",+ "value_status": "no_typed_value",+ "partial_output_block_labels": ["lookup_provider_and_extract_credentials"],+ },+ {+ "criterion_id": "npi",+ "output_path": "npi",+ "output_root": "npi",+ "reason_code": "no_evidence",+ "value_status": "empty_typed_value",+ },+ {+ "criterion_id": "overall",+ "output_path": "overall_credentialing_status",+ "output_root": "overall_credentialing_status",+ "reason_code": "no_evidence",+ "value_status": "no_typed_value",+ "partial_output_block_labels": ["lookup_provider_and_extract_credentials"],+ },+ ]+ payload_text = str(outcome.structural_key_payload)+ assert "evidence_text" not in payload_text+ assert "address and statuses" not in payload_text+++def test_runtime_block_failure_outcome_includes_bounded_page_state_and_run_id() -> None:+ result = {+ "ok": False,+ "error": "Timeout waiting for results.",+ "data": {+ "workflow_run_id": "wr_failed",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "search_records",+ "status": "failed",+ "failure_reason": "Timeout waiting for results.",+ }+ ],+ },+ }+ page_evidence = {+ "workflow_run_id": "wr_failed",+ "observed_after_workflow_run": True,+ "source_tool": "inspect_page_for_composition",+ "current_url": "https://example.test/search?secret=redacted",+ "page_title": "Search",+ "forms": [{"fields": [{"label": "Search", "selector": "#search"}]}],+ "result_containers": [{"selector": "#results", "text_excerpt": "No matching records"}],+ }++ outcome = recorded_outcome_from_run_blocks_result(result, page_evidence=page_evidence)++ assert outcome.workflow_run_id == "wr_failed"+ assert outcome.phase == "persisted_block_run"+ assert outcome.reason_code == "runtime_block_failure"+ assert outcome.structural_key is not None+ assert "form:Search #search" in outcome.page_evidence_refs+ assert "result:#results rows=unknown" in outcome.page_evidence_refs+++def test_runtime_block_failure_outcome_keys_playwright_hidden_locator_structure() -> None:+ table_result = {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_failed",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "extract_records",+ "status": "failed",+ "failure_reason": (+ "Failed to execute code block. Reason: TimeoutError: Locator.wait_for: "+ 'Timeout 15000ms exceeded.\nCall log:\nwaiting for locator("table").first '+ "to be visible\n - locator resolved to hidden <table>...</table>\n"+ ),+ }+ ],+ },+ }+ row_result = {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_failed_again",+ "overall_status": "failed",+ "blocks": [+ {+ "label": "extract_records",+ "status": "failed",+ "failure_reason": (+ "Failed to execute code block. Reason: TimeoutError: Locator.wait_for: "+ 'Timeout 15000ms exceeded.\nCall log:\nwaiting for locator("table tbody tr").first '+ "to be visible\n - locator resolved to hidden <tr>...</tr>\n"+ ),+ }+ ],+ },+ }++ table_outcome = recorded_outcome_from_run_blocks_result(table_result)+ row_outcome = recorded_outcome_from_run_blocks_result(row_result)++ assert table_outcome is not None+ assert row_outcome is not None+ assert table_outcome.is_authoritative is True+ assert row_outcome.is_authoritative is True+ assert table_outcome.structural_key != row_outcome.structural_key+ assert table_outcome.key_provenance["structural_failure_identity"] == "typed runtime failure structure"+++def test_persisted_run_prose_only_failure_is_not_authoritative() -> None:+ result = {+ "ok": False,+ "error": "The registry form failed after waiting for the same selector.",+ "data": {+ "workflow_run_id": "wr_failed",+ "overall_status": "failed",+ "failure_reason": "Timeout waiting for selector #results on the registry form.",+ "blocks": [{"label": "search_records", "status": "failed"}],+ },+ }++ outcome = recorded_outcome_from_run_blocks_result(result)++ assert outcome is None or outcome.is_authoritative is False+ assert outcome is None or outcome.structural_key is Nonediff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.pyindex a3d71a8bf..5e06b5c05 100644--- a/tests/unit/test_copilot_enforcement_pruning.py+++ b/tests/unit/test_copilot_enforcement_pruning.py@@ -17,6 +17,7 @@ import pytestfrom agents import RunConfigfrom skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, stash_blocker_signal+from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcomefrom skyvern.forge.sdk.copilot.code_block_synthesis import SynthesizedCodeBlockfrom skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdictfrom skyvern.forge.sdk.copilot.config import SYNTHESIZED_OFFER_REFRESH_STEP_THRESHOLD, BlockAuthoringPolicy@@ -274,6 +275,101 @@ class TestSynthesizedOfferPersistenceGate:assert signal.renders_final_reply is Falseassert synthesized_block_persistence_signal(ctx, "update_and_run_blocks") is None+ def test_authoring_offer_blocks_page_mutating_tool_before_loop_detection(self) -> None:+ ctx = _Ctx()+ ctx.turn_intent = TurnIntent(+ mode=TurnIntentMode.BUILD,+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),+ )+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.synthesized_block_offered = True+ ctx.synthesized_block_offered_trajectory_len = 1+ ctx.synthesized_block_offered_goal_complete = False+ ctx.scout_trajectory = [{"tool_name": "type_text", "selector": "input[name='q']", "accessible_name": "Search"}]++ signal = synthesized_block_persistence_signal(ctx, "click")++ assert isinstance(signal, CopilotToolBlockerSignal)+ assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE+ assert signal.blocked_tool == "click"+ assert signal.cleared_by_tools == frozenset({"update_and_run_blocks"})+ assert signal.renders_final_reply is False++ @pytest.mark.parametrize(+ "tool_name",+ ["update_and_run_blocks", "update_workflow", "fill_credential_field"],+ )+ def test_authoring_offer_keeps_allowed_tools_unblocked(self, tool_name: str) -> None:+ ctx = _Ctx()+ ctx.turn_intent = TurnIntent(+ mode=TurnIntentMode.BUILD,+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),+ )+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.synthesized_block_offered = True+ ctx.synthesized_block_offered_trajectory_len = 1+ ctx.synthesized_block_offered_goal_complete = False+ ctx.scout_trajectory = [{"tool_name": "type_text", "selector": "input[name='q']", "accessible_name": "Search"}]++ assert synthesized_block_persistence_signal(ctx, tool_name) is None++ def test_authoring_offer_does_not_block_mutating_tool_when_trajectory_changed(self) -> None:+ ctx = _Ctx()+ ctx.turn_intent = TurnIntent(+ mode=TurnIntentMode.BUILD,+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),+ )+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.synthesized_block_offered = True+ ctx.synthesized_block_offered_trajectory_len = 1+ ctx.synthesized_block_offered_goal_complete = False+ ctx.scout_trajectory = [+ {"tool_name": "type_text", "selector": "input[name='q']", "accessible_name": "Search"},+ {"tool_name": "click", "selector": "button[data-action='search']", "accessible_name": "Search"},+ ]++ assert synthesized_block_persistence_signal(ctx, "click") is None++ def test_authoring_offer_does_not_block_mutating_tool_without_offer(self) -> None:+ ctx = _Ctx()+ ctx.turn_intent = TurnIntent(+ mode=TurnIntentMode.BUILD,+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),+ )+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.synthesized_block_offered = False+ ctx.synthesized_block_offered_trajectory_len = 1+ ctx.scout_trajectory = [{"tool_name": "type_text", "selector": "input[name='q']", "accessible_name": "Search"}]++ assert synthesized_block_persistence_signal(ctx, "click") is None++ def test_unresolved_recorded_outcome_blocks_page_mutating_tool_until_update_and_run_blocks(self) -> None:+ ctx = _Ctx()+ ctx.turn_intent = TurnIntent(+ mode=TurnIntentMode.BUILD,+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),+ )+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.completion_verification_result = self._unsatisfied_verification()+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:partial",+ )++ signal = synthesized_block_persistence_signal(ctx, "click")++ assert isinstance(signal, CopilotToolBlockerSignal)+ assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE+ assert signal.blocked_tool == "click"+ assert signal.cleared_by_tools == frozenset({"update_and_run_blocks"})+ assert "last recorded test outcome" in signal.agent_steering_text+ assert synthesized_block_persistence_signal(ctx, "update_and_run_blocks") is None+ assert synthesized_block_persistence_signal(ctx, "evaluate") is None+@pytest.mark.parametrize("ctx_attrs",[diff --git a/tests/unit/test_copilot_run_output_content_gate.py b/tests/unit/test_copilot_run_output_content_gate.pyindex 7554a799e..57a0630d3 100644--- a/tests/unit/test_copilot_run_output_content_gate.py+++ b/tests/unit/test_copilot_run_output_content_gate.py@@ -772,6 +772,56 @@ def test_satisfied_completion_prevents_empty_output_suspicious_success() -> Noneassert ctx.last_test_suspicious_success is Falseassert ctx.last_test_failure_reason is Noneassert verified_goal_satisfied_context(ctx) is True+ assert ctx.latest_recorded_build_test_outcome is not None+ assert ctx.latest_recorded_build_test_outcome.verdict == "progress_observed"+ assert ctx.latest_recorded_build_test_outcome.reason_code == "verified_success"+ assert ctx.latest_recorded_build_test_outcome.structural_key is not None+++def test_unverified_success_records_goal_relative_build_test_outcome() -> None:+ result = _run_result([_code_block("search_registry_person", {"records": [], "result_count": 0})])+ ctx = _ctx(result["data"]["blocks"])++ recorded = _record_run_blocks_result(ctx, result, completion_verification=_no_evidence("c0"))++ assert recorded is not None+ assert recorded.verdict == "not_demonstrated"+ assert recorded.reason_code == "outcome_not_demonstrated"+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.phase == "persisted_block_run"+ assert outcome.verdict == "repairable_failure"+ assert outcome.reason_code == "outcome_not_demonstrated"+ assert outcome.structural_key is not None+++def test_changed_output_structure_changes_unverified_success_structural_key() -> None:+ first_result = _run_result([_code_block("search_registry_person", {"records": [], "result_count": 0})])+ second_result = _run_result(+ [_code_block("search_registry_person", {"records": [{"status_present": True}], "result_count": 1})]+ )+ ctx = _ctx(first_result["data"]["blocks"])++ _record_run_blocks_result(ctx, first_result, completion_verification=_no_evidence("c0"))+ first_key = ctx.latest_recorded_build_test_outcome.structural_key+ _record_run_blocks_result(ctx, second_result, completion_verification=_no_evidence("c0"))++ assert first_key is not None+ assert ctx.latest_recorded_build_test_outcome.structural_key is not None+ assert ctx.latest_recorded_build_test_outcome.structural_key != first_key+++def test_terminal_blocker_does_not_leave_authoritative_prompt_outcome() -> None:+ result = _blocked_flag_run_result()+ ctx = _ctx(result["data"]["blocks"])++ _record_run_blocks_result(ctx, result, completion_verification=None)++ assert ctx.blocker_signal is not None+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "terminal_challenge_blocker"+ assert outcome.is_authoritative is Falsedef test_all_null_metadata_goal_fields_are_flagged_as_no_goal_content() -> None:diff --git a/tests/unit/test_evaluate_actionable_target.py b/tests/unit/test_evaluate_actionable_target.pyindex 7d7275234..328262ac2 100644--- a/tests/unit/test_evaluate_actionable_target.py+++ b/tests/unit/test_evaluate_actionable_target.py@@ -346,6 +346,10 @@ async def test_first_loaded_result_table_sheds_large_payload_but_keeps_compositiserialized = json.dumps(result, default=str)assert result["data"].get("next_action") == "compose_extraction"assert result["data"].get("composition_targets")+ assert ctx.latest_recorded_build_test_outcome is not None+ assert ctx.latest_recorded_build_test_outcome.phase == "scout_evaluate"+ assert ctx.latest_recorded_build_test_outcome.reason_code == "loaded_result_targets_observed"+ assert ctx.latest_recorded_build_test_outcome.structural_key is not Noneassert len(serialized) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAPassert '"next_action"' in serialized[: scouting._RECENT_TOOL_OUTPUT_CHAR_CAP]assert "compose_extraction" in serialized