Skip to content

Automate browser based workflows with AI

SKY-11545: Add recorded outcome spine (#6939)

Andrew Neilson1de08f3eebfd
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 81c84a0c9..52cef77c1 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -44,6 +44,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
)
from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_data
from skyvern.forge.sdk.copilot.build_phase import initial_build_phase
+from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome
from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.code_block_synthesis import (
is_optional_dismissal_only_trajectory,
@@ -715,6 +716,56 @@ def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:
return "\n\n" + "\n".join(line for line in lines if line)
+def _recorded_build_test_outcome_prompt(ctx: CopilotContext | None) -> str:
+ if ctx is None:
+ return ""
+ if normalize_block_authoring_policy(ctx.block_authoring_policy) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return ""
+ outcome = ctx.latest_recorded_build_test_outcome
+ if not isinstance(outcome, RecordedBuildTestOutcome) or not outcome.is_authoritative:
+ return ""
+ LOG.info(
+ "copilot recorded build-test outcome rendered",
+ phase=outcome.phase,
+ reason_code=outcome.reason_code,
+ structural_key=outcome.structural_key,
+ workflow_run_id=outcome.workflow_run_id,
+ )
+ lines = [
+ "RECORDED BUILD-TEST OUTCOME:",
+ f"phase: {_clean_authoring_repair_prompt_atom(outcome.phase)}",
+ f"attempted_tool: {_clean_authoring_repair_prompt_atom(outcome.attempted_tool)}",
+ f"attempted_target: {_clean_authoring_repair_prompt_atom(outcome.attempted_target)}",
+ f"attempted_block_label: {_clean_authoring_repair_prompt_atom(outcome.attempted_block_label)}",
+ f"verdict: {_clean_authoring_repair_prompt_atom(outcome.verdict)}",
+ f"reason_code: {_clean_authoring_repair_prompt_atom(outcome.reason_code)}",
+ f"structural_key: {_clean_authoring_repair_prompt_atom(outcome.structural_key or '')}",
+ f"block_labels: {_render_authoring_repair_prompt_list(outcome.block_labels)}",
+ f"page_evidence_refs: {_render_authoring_repair_prompt_list(outcome.page_evidence_refs)}",
+ ]
+ if outcome.missing_requested_output_facts:
+ lines.append("missing_requested_output_facts:")
+ for fact in outcome.missing_requested_output_facts[:8]:
+ if not isinstance(fact, dict):
+ continue
+ fields = []
+ for key in ("output_root", "output_path", "value_status", "reason_code"):
+ value = fact.get(key)
+ if isinstance(value, str) and value.strip():
+ fields.append(f"{key}={_clean_authoring_repair_prompt_atom(value)}")
+ if fields:
+ lines.append(f"- {'; '.join(fields)}")
+ if outcome.workflow_run_id:
+ lines.append(f"workflow_run_id: {_clean_authoring_repair_prompt_atom(outcome.workflow_run_id)}")
+ if outcome.observed_evidence_summary:
+ lines.append(f"observed_evidence: {_clean_authoring_repair_prompt_atom(outcome.observed_evidence_summary)}")
+ lines.append(
+ "Before saving or rerunning, change the next authored step, selector, extraction, or binding based on this "
+ "recorded structure. Do not re-emit the same plan against the same structural key."
+ )
+ return "\n\n" + "\n".join(line for line in lines if line)
+
+
def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:
"""Pre-authoring offer of the synthesized code block.
@@ -804,6 +855,7 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -
return (
prompt
+ _runtime_verification_evidence_prompt(ctx)
+ + _recorded_build_test_outcome_prompt(ctx)
+ _code_authoring_repair_context_prompt(ctx)
+ _synthesized_block_offer_prompt(ctx)
+ _docs_answer_turn_directive(ctx.turn_intent)
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
new file mode 100644
index 000000000..7beb642bc
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -0,0 +1,816 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import textwrap
+from collections.abc import Iterable, Mapping, Sequence
+from typing import Literal, Protocol
+from urllib.parse import urlsplit
+
+import structlog
+import yaml
+from pydantic import BaseModel, ConfigDict, Field
+
+from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
+from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
+from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+
+LOG = structlog.get_logger()
+
+BuildTestOutcomePhase = Literal["scout_evaluate", "persisted_block_run", "author_time_reject"]
+BuildTestOutcomeVerdict = Literal[
+ "progress_observed",
+ "repairable_failure",
+ "authoring_rejected",
+ "not_authoritative",
+]
+BuildTestOutcomeReasonCode = Literal[
+ "loaded_result_targets_observed",
+ "runtime_block_failure",
+ "sandbox_unresolved_name",
+ "synthesized_parameter_binding_ambiguous",
+ "code_safety_reject",
+ "credential_scout_reject",
+ "schema_incompatibility",
+ "verified_success",
+ "outcome_not_demonstrated",
+ "no_meaningful_output",
+ "terminal_challenge_blocker",
+ "blocker_reported",
+ "failed_run",
+ "suspicious_success",
+ "missing_structural_evidence",
+ "unchanged_after_recorded_outcome",
+ "metadata_reject",
+]
+
+_STRUCTURAL_KEY_VERSION = "recorded_build_test_outcome:v1"
+_AUTHORED_STRUCTURE_VERSION = "recorded_build_test_outcome_authored_structure:v1"
+_TEXT_MAX = 180
+_REF_TEXT_MAX = 96
+_HISTORY_LIMIT = 8
+_PLAYWRIGHT_LOCATOR_WAIT_RE = re.compile(
+ r"waiting for locator\((?P<quote>['\"])(?P<selector>.*?)(?P=quote)\)"
+ r"(?P<locator_chain>(?:\.[A-Za-z_][A-Za-z0-9_]*(?:\([^)]*\))?)*)\s+to be (?P<state>[a-z_]+)",
+ re.IGNORECASE,
+)
+_PLAYWRIGHT_HIDDEN_TAG_RE = re.compile(r"locator resolved to hidden <(?P<tag>[a-z0-9:-]+)", re.IGNORECASE)
+
+
+class RecordedBuildTestOutcome(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ phase: BuildTestOutcomePhase
+ attempted_tool: str = ""
+ attempted_target: str = ""
+ attempted_block_label: str = ""
+ verdict: BuildTestOutcomeVerdict
+ reason_code: BuildTestOutcomeReasonCode
+ observed_evidence_summary: str = ""
+ workflow_run_id: str | None = None
+ block_labels: list[str] = Field(default_factory=list)
+ structural_failure_identity: str = ""
+ verified_progress_marker: str = ""
+ page_evidence_refs: list[str] = Field(default_factory=list)
+ evidence_refs: list[str] = Field(default_factory=list)
+ missing_requested_output_facts: list[dict[str, object]] = Field(default_factory=list)
+ authored_structure_signature: str | None = None
+ display_text: str = ""
+ key_provenance: dict[str, str] = Field(default_factory=dict)
+
+ @property
+ def structural_key_payload(self) -> dict[str, object] | None:
+ if not (self.structural_failure_identity or self.verified_progress_marker or self.page_evidence_refs):
+ return None
+ return {
+ "version": _STRUCTURAL_KEY_VERSION,
+ "phase": self.phase,
+ "attempted_tool": self.attempted_tool,
+ "attempted_target": self.attempted_target,
+ "reason_code": self.reason_code,
+ "verdict": self.verdict,
+ "structural_failure_identity": self.structural_failure_identity,
+ "verified_progress_marker": self.verified_progress_marker,
+ "page_evidence_refs": sorted(self.page_evidence_refs),
+ "evidence_refs": sorted(self.evidence_refs),
+ "missing_requested_output_facts": self.missing_requested_output_facts,
+ }
+
+ @property
+ def structural_key(self) -> str | None:
+ payload = self.structural_key_payload
+ if payload is None:
+ return None
+ return _stable_hash(payload)
+
+ @property
+ def is_authoritative(self) -> bool:
+ return self.structural_key is not None
+
+
+class _RecordedBuildTestOutcomeContext(Protocol):
+ latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None
+ recorded_build_test_outcome_history: list[dict[str, object]]
+
+
+def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: RecordedBuildTestOutcome | None) -> None:
+ if outcome is None:
+ ctx.latest_recorded_build_test_outcome = None
+ return
+ ctx.latest_recorded_build_test_outcome = outcome
+ history = getattr(ctx, "recorded_build_test_outcome_history", None)
+ if not isinstance(history, list):
+ history = []
+ history.append(
+ {
+ "phase": outcome.phase,
+ "reason_code": outcome.reason_code,
+ "verdict": outcome.verdict,
+ "structural_key": outcome.structural_key,
+ "is_authoritative": outcome.is_authoritative,
+ "workflow_run_id": outcome.workflow_run_id,
+ "authored_structure_signature": outcome.authored_structure_signature,
+ }
+ )
+ del history[:-_HISTORY_LIMIT]
+ ctx.recorded_build_test_outcome_history = history
+ LOG.info(
+ "copilot recorded build-test outcome stored",
+ phase=outcome.phase,
+ reason_code=outcome.reason_code,
+ verdict=outcome.verdict,
+ structural_key=outcome.structural_key,
+ is_authoritative=outcome.is_authoritative,
+ workflow_run_id=outcome.workflow_run_id,
+ authored_structure_signature=outcome.authored_structure_signature,
+ )
+
+
+def authored_structure_signature_from_workflow(
+ workflow_yaml: str | None,
+ code_artifact_metadata: object = None,
+) -> str | None:
+ payload = _authored_structure_payload_from_workflow(workflow_yaml, code_artifact_metadata)
+ if payload is None:
+ return None
+ return _stable_hash(payload)
+
+
+def latest_recorded_build_test_outcome_repeated(ctx: object) -> bool | None:
+ history = getattr(ctx, "recorded_build_test_outcome_history", None)
+ if not isinstance(history, list) or not history:
+ return None
+ latest = history[-1]
+ if not isinstance(latest, dict) or not isinstance(latest.get("structural_key"), str):
+ return None
+ for previous in reversed(history[:-1]):
+ if not isinstance(previous, dict):
+ continue
+ previous_key = previous.get("structural_key")
+ if isinstance(previous_key, str):
+ return previous_key == latest["structural_key"]
+ return None
+
+
+def recorded_outcome_from_loaded_result_evidence(
+ evidence: LoadedResultCompositionEvidence,
+) -> RecordedBuildTestOutcome:
+ page_refs = [
+ f"result_containers:{evidence.result_container_count}",
+ f"table_result_containers:{evidence.table_result_container_count}",
+ ]
+ for target in evidence.targets:
+ page_refs.append(
+ "target:"
+ + _stable_hash(
+ {
+ "is_table": target.is_table,
+ "row_count": target.row_count,
+ "structure_signature": target.structure_signature,
+ }
+ )
+ )
+ return RecordedBuildTestOutcome(
+ phase="scout_evaluate",
+ attempted_tool="evaluate",
+ attempted_target="loaded_result_targets",
+ verdict="progress_observed",
+ reason_code="loaded_result_targets_observed",
+ structural_failure_identity=f"loaded_result:{evidence.structure_signature}",
+ page_evidence_refs=page_refs,
+ observed_evidence_summary=(
+ f"{evidence.result_container_count} result container(s), "
+ f"{evidence.table_result_container_count} table-like container(s)."
+ ),
+ key_provenance={
+ "structural_failure_identity": "LoadedResultCompositionEvidence.structure_signature",
+ "page_evidence_refs": "loaded-result target structural signatures",
+ },
+ )
+
+
+def recorded_outcome_from_run_blocks_result(
+ result: Mapping[str, object],
+ *,
+ page_evidence: Mapping[str, object] | None = None,
+ recorded_run_outcome: RecordedRunOutcome | None = None,
+ completion_verification: CompletionVerificationResult | None = None,
+ authored_structure_signature: str | None = None,
+) -> RecordedBuildTestOutcome | None:
+ data = _dict(result.get("data"))
+ workflow_run_id = _safe_str(data.get("workflow_run_id"))
+ blocks = _block_dicts(data.get("blocks"))
+ failed_block = _first_failed_block(blocks)
+ block_labels = [_safe_str(block.get("label")) for block in blocks if _safe_str(block.get("label"))]
+ page_refs = _page_evidence_refs(page_evidence)
+ output_refs = _output_evidence_refs(blocks)
+ verification_identity = _completion_verification_identity(completion_verification)
+ missing_output_facts = _missing_requested_output_facts(completion_verification, blocks)
+ if recorded_run_outcome is not None:
+ reason_code = _run_outcome_reason_code(recorded_run_outcome)
+ if reason_code == "terminal_challenge_blocker":
+ return RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="not_authoritative",
+ reason_code=reason_code,
+ workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,
+ block_labels=block_labels,
+ authored_structure_signature=authored_structure_signature,
+ observed_evidence_summary=recorded_run_outcome.display_reason or "",
+ key_provenance={"structural_failure_identity": "terminal blocker precedence suppresses repair prompt"},
+ )
+ if recorded_run_outcome.verdict == "demonstrated":
+ return RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="progress_observed",
+ reason_code="verified_success",
+ workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,
+ block_labels=block_labels,
+ verified_progress_marker=verification_identity or "run_completed_verified",
+ evidence_refs=output_refs,
+ authored_structure_signature=authored_structure_signature,
+ observed_evidence_summary=recorded_run_outcome.display_reason or "Completion verification passed.",
+ key_provenance={
+ "verified_progress_marker": "CompletionVerificationResult satisfied criteria",
+ "evidence_refs": "run output structure",
+ },
+ )
+ structural_identity = verification_identity
+ evidence_refs = output_refs
+ if not structural_identity and not page_refs and not evidence_refs:
+ return RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="not_authoritative",
+ reason_code=reason_code,
+ workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,
+ block_labels=block_labels,
+ authored_structure_signature=authored_structure_signature,
+ observed_evidence_summary=recorded_run_outcome.display_reason or "",
+ key_provenance={"structural_failure_identity": "no typed verification/page/output identity available"},
+ )
+ return RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code=reason_code,
+ workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,
+ block_labels=block_labels,
+ structural_failure_identity=structural_identity,
+ page_evidence_refs=page_refs,
+ evidence_refs=evidence_refs,
+ missing_requested_output_facts=missing_output_facts,
+ authored_structure_signature=authored_structure_signature,
+ observed_evidence_summary=recorded_run_outcome.display_reason or "",
+ key_provenance={
+ "structural_failure_identity": "CompletionVerificationResult verdict structure",
+ "page_evidence_refs": "bounded post-run page evidence",
+ "evidence_refs": "run output structure",
+ "missing_requested_output_facts": "CompletionVerificationResult unsatisfied output paths and run output shape",
+ },
+ )
+ run_status = _safe_str(data.get("overall_status"))
+ failure_type = _safe_str(data.get("failure_type"))
+ failure_categories = _failure_category_refs(data.get("failure_categories"))
+ status = _safe_str(failed_block.get("status")) if failed_block is not None else run_status
+ runtime_failure_identity = _runtime_failure_identity(failed_block)
+ if not (failure_categories or failure_type or runtime_failure_identity or page_refs or output_refs):
+ return None
+ structural_identity = (
+ _stable_hash(
+ {
+ "failure_type": failure_type,
+ "failure_categories": failure_categories,
+ "runtime_failure_identity": runtime_failure_identity,
+ "status": status,
+ }
+ )
+ if failure_categories or failure_type or runtime_failure_identity
+ else ""
+ )
+ verdict: BuildTestOutcomeVerdict = "repairable_failure" if bool(result.get("ok")) is False else "progress_observed"
+ if not structural_identity and not page_refs and not output_refs:
+ verdict = "not_authoritative"
+ return RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ attempted_block_label=_safe_str(failed_block.get("label")) if failed_block is not None else "",
+ verdict=verdict,
+ reason_code="runtime_block_failure" if failed_block is not None or not bool(result.get("ok")) else "failed_run",
+ workflow_run_id=workflow_run_id or None,
+ block_labels=block_labels,
+ structural_failure_identity=structural_identity,
+ page_evidence_refs=page_refs,
+ evidence_refs=output_refs,
+ authored_structure_signature=authored_structure_signature,
+ observed_evidence_summary=_bounded_text(run_status),
+ key_provenance={
+ "structural_failure_identity": (
+ "typed runtime failure structure"
+ if runtime_failure_identity
+ else "typed failure categories or failure_type"
+ ),
+ "page_evidence_refs": "bounded post-run page evidence",
+ "evidence_refs": "run output structure",
+ },
+ )
+
+
+def _stable_hash(payload: object) -> str:
+ canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _authored_structure_payload_from_workflow(
+ workflow_yaml: str | None,
+ code_artifact_metadata: object,
+) -> dict[str, object] | None:
+ if not isinstance(workflow_yaml, str) or not workflow_yaml.strip():
+ return None
+ parsed = _parse_workflow_yaml(workflow_yaml)
+ if not isinstance(parsed, Mapping):
+ return None
+ definition = _dict(parsed.get("workflow_definition"))
+ code_blocks = _code_block_signature_payloads(definition.get("blocks"))
+ if not code_blocks:
+ return None
+ metadata_by_label = _artifact_metadata_by_label(code_artifact_metadata)
+ return {
+ "version": _AUTHORED_STRUCTURE_VERSION,
+ "workflow_parameter_keys": _workflow_parameter_keys(definition),
+ "code_blocks": [
+ {
+ **block,
+ "output_metadata": _artifact_output_metadata_signature(metadata_by_label.get(str(block["label"]))),
+ }
+ for block in code_blocks
+ ],
+ }
+
+
+def _parse_workflow_yaml(workflow_yaml: str) -> object:
+ try:
+ return yaml.safe_load(workflow_yaml)
+ except yaml.YAMLError:
+ return None
+
+
+def _workflow_parameter_keys(definition: Mapping[str, object]) -> list[str]:
+ keys: list[str] = []
+ for parameter in _mapping_list(definition.get("parameters")):
+ key = _safe_str(parameter.get("key"))
+ if key:
+ keys.append(key)
+ return sorted(dict.fromkeys(keys))
+
+
+def _code_block_signature_payloads(value: object) -> list[dict[str, object]]:
+ payloads: list[dict[str, object]] = []
+ for block in _mapping_list(value):
+ block_type = _safe_str(block.get("block_type")).lower()
+ if block_type == "code":
+ label = _safe_str(block.get("label"))
+ code = block.get("code")
+ if label and isinstance(code, str):
+ payloads.append(
+ {
+ "label": label,
+ "code_hash": _stable_hash(_normalized_code_text(code)),
+ "parameter_keys": _string_list(block.get("parameter_keys")),
+ }
+ )
+ for child_key in ("blocks",):
+ payloads.extend(_code_block_signature_payloads(block.get(child_key)))
+ for branch in _mapping_list(block.get("branch_conditions")):
+ payloads.extend(_code_block_signature_payloads(branch.get("blocks")))
+ return sorted(payloads, key=lambda item: str(item.get("label")))
+
+
+def _normalized_code_text(code: str) -> str:
+ return "\n".join(line.rstrip() for line in textwrap.dedent(code).strip().splitlines())
+
+
+def _artifact_metadata_by_label(code_artifact_metadata: object) -> dict[str, Mapping[str, object]]:
+ rows: Iterable[tuple[object, object]]
+ if isinstance(code_artifact_metadata, Mapping):
+ rows = code_artifact_metadata.items()
+ elif isinstance(code_artifact_metadata, list):
+ rows = [(None, row) for row in code_artifact_metadata]
+ else:
+ return {}
+ by_label: dict[str, Mapping[str, object]] = {}
+ for fallback_label, row in rows:
+ if not isinstance(row, Mapping):
+ continue
+ label = _safe_str(row.get("block_label")) or _safe_str(fallback_label)
+ if label:
+ by_label[label] = row
+ return by_label
+
+
+def _artifact_output_metadata_signature(artifact: Mapping[str, object] | None) -> dict[str, object]:
+ if artifact is None:
+ return {}
+ return {
+ "claimed_outcomes": _output_rows_signature(artifact.get("claimed_outcomes")),
+ "terminal_verifier_expectations": _output_rows_signature(artifact.get("terminal_verifier_expectations")),
+ }
+
+
+def _output_rows_signature(value: object) -> list[dict[str, object]]:
+ rows: list[dict[str, object]] = []
+ for row in _mapping_list(value):
+ rows.append(
+ {
+ "goal_value_paths": _string_list(row.get("goal_value_paths")),
+ "extraction_schema_paths": _extraction_schema_paths(row.get("extraction_schema")),
+ }
+ )
+ return rows
+
+
+def _extraction_schema_paths(value: object) -> list[str]:
+ schema: object = value
+ if isinstance(value, str):
+ try:
+ schema = json.loads(value)
+ except json.JSONDecodeError:
+ return []
+ if not isinstance(schema, Mapping):
+ return []
+ paths: list[str] = []
+ _collect_schema_paths(schema, prefix="", paths=paths)
+ return sorted(dict.fromkeys(paths))
+
+
+def _collect_schema_paths(schema: Mapping[str, object], *, prefix: str, paths: list[str]) -> None:
+ schema_type = _safe_str(schema.get("type"))
+ if prefix:
+ paths.append(f"{prefix}:{schema_type or 'unknown'}")
+ properties = schema.get("properties")
+ if isinstance(properties, Mapping):
+ for key, child in sorted(properties.items(), key=lambda item: str(item[0])):
+ if isinstance(child, Mapping):
+ child_prefix = f"{prefix}.{key}" if prefix else str(key)
+ _collect_schema_paths(child, prefix=child_prefix, paths=paths)
+ items = schema.get("items")
+ if isinstance(items, Mapping):
+ _collect_schema_paths(items, prefix=f"{prefix}[]" if prefix else "[]", paths=paths)
+
+
+def _string_list(value: object) -> list[str]:
+ if not isinstance(value, list):
+ return []
+ return sorted(dict.fromkeys(item for item in value if isinstance(item, str)))
+
+
+def _bounded_text(value: object, max_chars: int = _TEXT_MAX) -> str:
+ if not isinstance(value, str):
+ return ""
+ text = redact_raw_secrets_for_prompt(" ".join(value.split()))
+ return text[:max_chars]
+
+
+def _bounded_ref(value: object, max_chars: int = _REF_TEXT_MAX) -> str:
+ if not isinstance(value, str):
+ return ""
+ return " ".join(value.split())[:max_chars]
+
+
+def _safe_str(value: object) -> str:
+ return value if isinstance(value, str) else ""
+
+
+def _dict(value: object) -> Mapping[str, object]:
+ return value if isinstance(value, Mapping) else {}
+
+
+def _block_dicts(value: object) -> list[Mapping[str, object]]:
+ return [item for item in value if isinstance(item, Mapping)] if isinstance(value, list) else []
+
+
+def _first_failed_block(blocks: Sequence[Mapping[str, object]]) -> Mapping[str, object] | None:
+ for block in blocks:
+ if _safe_str(block.get("status")).lower() in {"failed", "terminated", "canceled", "timed_out"}:
+ return block
+ return None
+
+
+def _runtime_failure_identity(failed_block: Mapping[str, object] | None) -> str:
+ if failed_block is None:
+ return ""
+ failure_reason = _safe_str(failed_block.get("failure_reason"))
+ if not failure_reason:
+ return ""
+ locator_wait_match = _PLAYWRIGHT_LOCATOR_WAIT_RE.search(failure_reason)
+ if locator_wait_match is None:
+ return ""
+ hidden_tag_match = _PLAYWRIGHT_HIDDEN_TAG_RE.search(failure_reason)
+ selector = _bounded_ref(locator_wait_match.group("selector"))
+ locator_chain = _bounded_ref(locator_wait_match.group("locator_chain"))
+ state = _bounded_ref(locator_wait_match.group("state").casefold())
+ hidden_tag = _bounded_ref(hidden_tag_match.group("tag").casefold()) if hidden_tag_match is not None else ""
+ return _stable_hash(
+ {
+ "source": "playwright_locator_wait",
+ "selector": selector,
+ "locator_chain": locator_chain,
+ "state": state,
+ "hidden_tag": hidden_tag,
+ "block_label": _safe_str(failed_block.get("label")),
+ "block_status": _safe_str(failed_block.get("status")),
+ }
+ )
+
+
+def _clean_list(values: Sequence[str]) -> list[str]:
+ return [cleaned for value in values for cleaned in [_bounded_ref(value)] if cleaned]
+
+
+def _origin_ref(value: object) -> str:
+ if not isinstance(value, str) or not value.strip():
+ return ""
+ parsed = urlsplit(value)
+ if not parsed.scheme or not parsed.netloc:
+ return ""
+ return f"origin:{parsed.scheme}://{parsed.netloc}"
+
+
+def _page_evidence_refs(page_evidence: Mapping[str, object] | None) -> list[str]:
+ if page_evidence is None:
+ return []
+ refs: list[str] = []
+ origin = _origin_ref(page_evidence.get("current_url")) or _origin_ref(page_evidence.get("inspected_url"))
+ if origin:
+ refs.append(origin)
+ refs.extend(_form_refs(page_evidence.get("forms")))
+ refs.extend(_result_refs(page_evidence.get("result_containers")))
+ refs.extend(_action_refs(page_evidence.get("navigation_targets")))
+ return refs[:12]
+
+
+def _form_refs(value: object) -> list[str]:
+ if not isinstance(value, list):
+ return []
+ refs: list[str] = []
+ for form in value[:3]:
+ if not isinstance(form, Mapping):
+ continue
+ for field in _mapping_list(form.get("fields"))[:3]:
+ label = _bounded_ref(field.get("label"))
+ selector = _bounded_ref(field.get("selector"))
+ if label or selector:
+ refs.append(f"form:{' '.join(item for item in (label, selector) if item)}")
+ for control in _mapping_list(form.get("submit_controls"))[:2]:
+ text = _bounded_ref(control.get("text"))
+ selector = _bounded_ref(control.get("selector"))
+ if text or selector:
+ refs.append(f"submit:{' '.join(item for item in (text, selector) if item)}")
+ return refs
+
+
+def _result_refs(value: object) -> list[str]:
+ refs: list[str] = []
+ for container in _mapping_list(value)[:4]:
+ selector = _bounded_ref(container.get("selector")) or "unknown"
+ row_count = container.get("row_count")
+ row_text = str(row_count) if isinstance(row_count, int) else "unknown"
+ refs.append(f"result:{selector} rows={row_text}")
+ return refs
+
+
+def _action_refs(value: object) -> list[str]:
+ refs: list[str] = []
+ for action in _mapping_list(value)[:4]:
+ text = _bounded_ref(action.get("text"))
+ selector = _bounded_ref(action.get("selector"))
+ if text or selector:
+ refs.append(f"action:{' '.join(item for item in (text, selector) if item)}")
+ return refs
+
+
+def _mapping_list(value: object) -> list[Mapping[str, object]]:
+ return [item for item in value if isinstance(item, Mapping)] if isinstance(value, list) else []
+
+
+def _run_outcome_reason_code(recorded_run_outcome: RecordedRunOutcome) -> BuildTestOutcomeReasonCode:
+ reason_code = recorded_run_outcome.reason_code
+ if reason_code in {
+ "outcome_not_demonstrated",
+ "no_meaningful_output",
+ "terminal_challenge_blocker",
+ "blocker_reported",
+ }:
+ return reason_code
+ if recorded_run_outcome.verdict == "demonstrated":
+ return "verified_success"
+ return "failed_run"
+
+
+def _completion_verification_identity(completion_verification: CompletionVerificationResult | None) -> str:
+ if completion_verification is None or completion_verification.status != "evaluated":
+ return ""
+ verdict_payload = [
+ {
+ "criterion_id": verdict.criterion_id,
+ "state": verdict.state,
+ "reason_code": verdict.reason_code,
+ "output_path": verdict.output_path,
+ "grounding_mode": verdict.grounding_mode,
+ "expected_output_shape": verdict.expected_output_shape,
+ "has_exact_value": verdict.has_exact_value,
+ }
+ for verdict in completion_verification.verdicts
+ ]
+ payload = {
+ "criterion_ids": sorted(completion_verification.criterion_ids),
+ "verdicts": verdict_payload,
+ "no_gradeable_run_plane": completion_verification.no_gradeable_run_plane,
+ "structural_unfired_criterion_ids": sorted(completion_verification.structural_unfired_criterion_ids),
+ }
+ return "completion:" + _stable_hash(payload)
+
+
+def _missing_requested_output_facts(
+ completion_verification: CompletionVerificationResult | None,
+ blocks: Sequence[Mapping[str, object]],
+) -> list[dict[str, object]]:
+ if completion_verification is None or completion_verification.status != "evaluated":
+ return []
+ empty_output_block_labels = _empty_output_block_labels(blocks)
+ facts: list[dict[str, object]] = []
+ for verdict in completion_verification.verdicts:
+ if verdict.satisfied or not verdict.output_path:
+ continue
+ output_path = _bounded_ref(verdict.output_path)
+ output_root = _output_path_root(output_path)
+ if not output_root:
+ continue
+ fact: dict[str, object] = {
+ "criterion_id": _bounded_ref(verdict.criterion_id),
+ "output_path": output_path,
+ "output_root": output_root,
+ "reason_code": _bounded_ref(verdict.reason_code),
+ "value_status": _output_path_value_status(blocks, output_path, verdict),
+ }
+ if verdict.grounding_mode:
+ fact["grounding_mode"] = verdict.grounding_mode
+ if verdict.expected_output_shape:
+ fact["expected_output_shape"] = _bounded_ref(verdict.expected_output_shape)
+ if empty_output_block_labels:
+ fact["empty_output_block_labels"] = empty_output_block_labels
+ partial_labels = _partial_output_block_labels(blocks, output_root)
+ if partial_labels:
+ fact["partial_output_block_labels"] = partial_labels
+ facts.append(fact)
+ return sorted(facts, key=lambda item: str(item.get("output_path") or ""))
+
+
+def _output_path_root(output_path: str) -> str:
+ return _bounded_ref(output_path.split(".", 1)[0].split("[", 1)[0])
+
+
+def _output_path_value_status(
+ blocks: Sequence[Mapping[str, object]],
+ output_path: str,
+ verdict: object,
+) -> str:
+ values: list[object] = []
+ for block in blocks:
+ extracted = block.get("extracted_data")
+ if extracted is None:
+ continue
+ value, present = _value_at_output_path(extracted, output_path)
+ if present:
+ values.append(value)
+ if not values:
+ return "no_typed_value"
+ if all(_is_empty_output_value(value) for value in values):
+ return "empty_typed_value"
+ grounding_mode = getattr(verdict, "grounding_mode", None)
+ has_exact_value = getattr(verdict, "has_exact_value", False)
+ if grounding_mode == "shape" and not has_exact_value:
+ return "presence_only_evidence"
+ return "typed_value_unverified"
+
+
+def _value_at_output_path(value: object, output_path: str) -> tuple[object | None, bool]:
+ current = value
+ for segment in [part for part in re.split(r"\.|\[\]", output_path) if part]:
+ if isinstance(current, Mapping):
+ if segment not in current:
+ return None, False
+ current = current[segment]
+ continue
+ if isinstance(current, list):
+ found_values = [item.get(segment) for item in current if isinstance(item, Mapping) and segment in item]
+ if not found_values:
+ return None, False
+ current = found_values
+ continue
+ return None, False
+ return current, True
+
+
+def _is_empty_output_value(value: object) -> bool:
+ if value is None:
+ return True
+ if isinstance(value, str):
+ return not value.strip()
+ if isinstance(value, (list, tuple, set, dict)):
+ return len(value) == 0
+ return False
+
+
+def _empty_output_block_labels(blocks: Sequence[Mapping[str, object]]) -> list[str]:
+ labels: list[str] = []
+ for block in blocks:
+ extracted = block.get("extracted_data")
+ if extracted is None:
+ continue
+ if _is_empty_output_value(extracted):
+ label = _bounded_ref(block.get("label"))
+ if label:
+ labels.append(label)
+ return labels
+
+
+def _partial_output_block_labels(blocks: Sequence[Mapping[str, object]], output_root: str) -> list[str]:
+ labels: list[str] = []
+ for block in blocks:
+ extracted = block.get("extracted_data")
+ if not isinstance(extracted, Mapping) or output_root in extracted or not extracted:
+ continue
+ label = _bounded_ref(block.get("label"))
+ if label:
+ labels.append(label)
+ return labels
+
+
+def _failure_category_refs(value: object) -> list[str]:
+ refs: list[str] = []
+ for entry in _mapping_list(value):
+ category = _bounded_ref(entry.get("category"))
+ reason = _bounded_ref(entry.get("reason_code"))
+ if category or reason:
+ refs.append(":".join(part for part in (category, reason) if part))
+ return refs
+
+
+def _output_evidence_refs(blocks: Sequence[Mapping[str, object]]) -> list[str]:
+ refs: list[str] = []
+ for block in blocks[:8]:
+ extracted = block.get("extracted_data")
+ if extracted is None:
+ continue
+ refs.append("output:" + _stable_hash(_value_shape(extracted)))
+ return refs
+
+
+def _value_shape(value: object, *, depth: int = 0) -> object:
+ if depth > 6:
+ return "max_depth"
+ if isinstance(value, Mapping):
+ return {
+ str(key): _value_shape(item, depth=depth + 1)
+ for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))
+ }
+ if isinstance(value, list):
+ return {
+ "type": "list",
+ "length": len(value),
+ "items": [_value_shape(item, depth=depth + 1) for item in value[:3]],
+ }
+ if isinstance(value, bool):
+ return {"type": "bool", "value": value}
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
+ return {"type": type(value).__name__, "zero": value == 0}
+ if isinstance(value, str):
+ return {"type": "str", "empty": value == ""}
+ if value is None:
+ return "none"
+ return type(value).__name__
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 9f164121a..246ac4189 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -97,6 +97,7 @@ class TurnNarrativePayload(TypedDict):
if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+ from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome
from skyvern.forge.sdk.copilot.completion_criteria_store import CompletionCriteriaTurnState
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import DiagnosisRepairContract
from skyvern.forge.sdk.copilot.narration import NarratorState
@@ -619,6 +620,8 @@ class CopilotContext(AgentContext):
last_code_authoring_reject_was_credential_priority: bool = False
pending_code_authoring_runtime_repair_context: CodeAuthoringRepairContext | None = None
last_code_authoring_repair_context: CodeAuthoringRepairContext | None = None
+ latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None = None
+ recorded_build_test_outcome_history: list[dict[str, object]] = field(default_factory=list)
# Turn-scoped monotonic marks of verified forward progress: the union of
# completion criteria the judge confirmed satisfied so far this turn, and the
# high-water length of the verified block prefix. A repair that grows either
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index f227ef346..7dc3068c7 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -25,6 +25,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
stash_blocker_signal,
)
from skyvern.forge.sdk.copilot.build_phase import DISCOVERY_PERMITTED_PHASES
+from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome
from skyvern.forge.sdk.copilot.code_block_synthesis import (
is_durable_fallback_entry_target,
is_generic_entry_opener_click,
@@ -167,6 +168,9 @@ SYNTHESIZED_BLOCK_PERSISTENCE_TOOL = "update_and_run_blocks"
_SYNTHESIZED_BLOCK_PERSISTENCE_ALLOWED_TOOLS = frozenset(
{SYNTHESIZED_BLOCK_PERSISTENCE_TOOL, "fill_credential_field", "update_workflow"}
)
+_SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS = frozenset(
+ {"click", "press_key", "type_text", "select_option", "navigate_browser"}
+)
_SYNTHESIZED_BLOCK_COMMIT_TOOLS = frozenset({"click", "press_key"})
# OpenAI detail=high cost per resized image. If we support other providers,
# pull from model config — this value will silently over/undercount otherwise.
@@ -2127,10 +2131,63 @@ def _should_force_synthesized_block_persistence(ctx: Any) -> bool:
return synthesized_trajectory_is_goal_complete(ctx)
+def _should_block_mutating_tool_after_synthesized_offer(ctx: Any, tool_name: str) -> bool:
+ if tool_name not in _SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS:
+ return False
+ if getattr(ctx, "update_workflow_called", False) and not synthesized_persistence_reopened_after_failed_run(ctx):
+ return False
+ if not _turn_intent_can_update_and_run_without_user_input(getattr(ctx, "turn_intent", None)):
+ return False
+ if normalize_block_authoring_policy(getattr(ctx, "block_authoring_policy", None)) != (
+ BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ):
+ return False
+ if not getattr(ctx, "synthesized_block_offered", False):
+ return False
+ trajectory = getattr(ctx, "scout_trajectory", None) or []
+ return (getattr(ctx, "synthesized_block_offered_trajectory_len", 0) or 0) == len(trajectory)
+
+
+def _should_block_mutating_tool_after_unresolved_recorded_outcome(ctx: Any, tool_name: str) -> bool:
+ if tool_name not in _SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS:
+ return False
+ if not _turn_intent_can_update_and_run_without_user_input(getattr(ctx, "turn_intent", None)):
+ return False
+ if normalize_block_authoring_policy(getattr(ctx, "block_authoring_policy", None)) != (
+ BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ):
+ return False
+ latest = getattr(ctx, "latest_recorded_build_test_outcome", None)
+ if not isinstance(latest, RecordedBuildTestOutcome) or not latest.is_authoritative:
+ return False
+ if latest.phase != "persisted_block_run" or latest.reason_code != "outcome_not_demonstrated":
+ return False
+ return _completion_verification_unsatisfied(ctx)
+
+
def synthesized_block_persistence_signal(ctx: Any, tool_name: str) -> CopilotToolBlockerSignal | None:
if tool_name in _SYNTHESIZED_BLOCK_PERSISTENCE_ALLOWED_TOOLS:
return None
- if not _should_force_synthesized_block_persistence(ctx):
+ if _should_block_mutating_tool_after_unresolved_recorded_outcome(ctx, tool_name):
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=(
+ "The last recorded test outcome is authoritative and still has unsatisfied completion criteria. "
+ f"Call {SYNTHESIZED_BLOCK_PERSISTENCE_TOOL} with a materially changed authored workflow now; "
+ "do not spend the repair attempt on browser interaction."
+ ),
+ user_facing_reason="I need to revise and test the workflow code instead of interacting with the page.",
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=frozenset({SYNTHESIZED_BLOCK_PERSISTENCE_TOOL}),
+ preserves_workflow_draft=True,
+ renders_final_reply=False,
+ internal_reason_code=SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,
+ blocked_tool=tool_name,
+ extra={"recorded_outcome_reason_code": "outcome_not_demonstrated"},
+ )
+ if not _should_force_synthesized_block_persistence(ctx) and not _should_block_mutating_tool_after_synthesized_offer(
+ ctx, tool_name
+ ):
return None
return CopilotToolBlockerSignal(
blocker_kind="tool_error",
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index a0ab45709..9e7f777d6 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -35,6 +35,7 @@ from skyvern.library.skyvern_browser import SkyvernBrowser
if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+ from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
@@ -241,6 +242,8 @@ class AgentContext:
last_run_blocks_block_labels: list[str] = field(default_factory=list)
last_run_outcome: RecordedRunOutcome | None = None
last_run_outcome_block_labels: list[str] = field(default_factory=list)
+ latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None = None
+ recorded_build_test_outcome_history: list[dict[str, object]] = field(default_factory=list)
completion_verification_result: CompletionVerificationResult | None = None
verified_terminal_proposal_ready: bool = False
outcome_verification_trace_snapshot: dict[str, Any] = field(default_factory=dict)
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 5d8b758aa..41ff011e9 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -23,6 +23,12 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
clear_blocker_signal_for_reason_codes,
stash_blocker_signal,
)
+from skyvern.forge.sdk.copilot.build_test_outcome import (
+ RecordedBuildTestOutcome,
+ authored_structure_signature_from_workflow,
+ record_build_test_outcome,
+ recorded_outcome_from_run_blocks_result,
+)
from skyvern.forge.sdk.copilot.code_block_security import (
COPILOT_CODE_SECURITY_FAILURE_CATEGORY,
CodeBlockSecurityInput,
@@ -2184,15 +2190,14 @@ def _record_run_blocks_result(
stash_turn_halt_from_blocker_signal(copilot_ctx, signal, source="run_execution")
update_repeated_failure_state(copilot_ctx, result)
_update_verification_evidence_from_run_result(copilot_ctx, result)
- return _stash_recorded_run_outcome(
- copilot_ctx,
- RecordedRunOutcome(
- verdict="not_demonstrated",
- reason_code=TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,
- display_reason=run_outcome_display_reason(terminal_challenge.reason),
- workflow_run_id=terminal_challenge.workflow_run_id,
- ),
+ recorded_outcome = RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code=TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,
+ display_reason=run_outcome_display_reason(terminal_challenge.reason),
+ workflow_run_id=terminal_challenge.workflow_run_id,
)
+ _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, recorded_outcome)
+ return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)
if run_ok:
_mark_page_inspected(copilot_ctx)
@@ -2220,14 +2225,13 @@ def _record_run_blocks_result(
data.setdefault("failure_reason", failure_reason)
update_repeated_failure_state(copilot_ctx, result)
_update_verification_evidence_from_run_result(copilot_ctx, result)
- return _stash_recorded_run_outcome(
- copilot_ctx,
- RecordedRunOutcome(
- verdict="not_demonstrated",
- reason_code="blocker_reported",
- display_reason=run_outcome_display_reason(structured_blocker),
- ),
+ recorded_outcome = RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="blocker_reported",
+ display_reason=run_outcome_display_reason(structured_blocker),
)
+ _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, recorded_outcome)
+ return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)
if completion_fully_satisfied:
# ``verified_terminal_proposal_ready`` is telemetry only (the barrier keys
# on ``outcome_fully_verified(ctx)``); clearing the stale suspicious-success
@@ -2249,14 +2253,13 @@ def _record_run_blocks_result(
copilot_ctx.probable_site_block_streak_count = 0
update_repeated_failure_state(copilot_ctx, result)
_update_verification_evidence_from_run_result(copilot_ctx, result)
- return _stash_recorded_run_outcome(
- copilot_ctx,
- RecordedRunOutcome(
- verdict="not_demonstrated",
- reason_code="no_meaningful_output",
- display_reason=run_outcome_display_reason(copilot_ctx.last_test_failure_reason),
- ),
+ recorded_outcome = RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="no_meaningful_output",
+ display_reason=run_outcome_display_reason(copilot_ctx.last_test_failure_reason),
)
+ _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, recorded_outcome)
+ return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)
if prior_committed_outcome is not None and artifact_reason is None:
# artifact_reason is current-run health; prior_committed_outcome already passed prior ctx artifact-health.
copilot_ctx.last_full_workflow_test_ok = True
@@ -2312,7 +2315,9 @@ def _record_run_blocks_result(
)
update_repeated_failure_state(copilot_ctx, result)
_update_verification_evidence_from_run_result(copilot_ctx, result)
- return _stash_recorded_run_outcome(copilot_ctx, _adjudicated_run_outcome(copilot_ctx, completion_verification))
+ recorded_outcome = _adjudicated_run_outcome(copilot_ctx, completion_verification)
+ _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, recorded_outcome)
+ return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)
if outcome_fully_verified(copilot_ctx):
copilot_ctx.last_test_suspicious_success = False
@@ -2327,7 +2332,9 @@ def _record_run_blocks_result(
copilot_ctx.last_good_workflow_yaml = copilot_ctx.last_workflow_yaml
update_repeated_failure_state(copilot_ctx, result)
_update_verification_evidence_from_run_result(copilot_ctx, result)
- return _stash_recorded_run_outcome(copilot_ctx, _adjudicated_run_outcome(copilot_ctx, completion_verification))
+ recorded_outcome = _adjudicated_run_outcome(copilot_ctx, completion_verification)
+ _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, recorded_outcome)
+ return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)
copilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)
copilot_ctx.last_test_non_retriable_nav_error = _detect_non_retriable_nav_error(result)
@@ -2350,9 +2357,31 @@ def _record_run_blocks_result(
copilot_ctx.last_test_failure_reason = str(result["error"])
update_repeated_failure_state(copilot_ctx, result)
_update_verification_evidence_from_run_result(copilot_ctx, result)
+ _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, None)
return None
+def _record_adjudicated_build_test_outcome(
+ copilot_ctx: Any,
+ result: dict[str, Any],
+ completion_verification: CompletionVerificationResult | None,
+ recorded_run_outcome: RecordedRunOutcome | None,
+) -> None:
+ record_build_test_outcome(
+ copilot_ctx,
+ recorded_outcome_from_run_blocks_result(
+ result,
+ page_evidence=getattr(copilot_ctx, "composition_page_evidence", None),
+ recorded_run_outcome=recorded_run_outcome,
+ completion_verification=completion_verification,
+ authored_structure_signature=authored_structure_signature_from_workflow(
+ getattr(copilot_ctx, "workflow_yaml", None),
+ getattr(copilot_ctx, "code_artifact_metadata", None),
+ ),
+ ),
+ )
+
+
def _stash_recorded_run_outcome(copilot_ctx: Any, outcome: RecordedRunOutcome) -> RecordedRunOutcome:
if outcome.workflow_run_id is None:
outcome = replace(outcome, workflow_run_id=getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None))
@@ -2490,9 +2519,12 @@ async def _verify_and_record_run_blocks_result(
return completion_verification
-def _repair_non_convergence_signature(contract: DiagnosisRepairContract) -> str | None:
+def _repair_non_convergence_signature(copilot_ctx: Any, contract: DiagnosisRepairContract) -> str | None:
if contract.repair_decision.next_action is not RepairNextAction.REPAIR:
return None
+ recorded = getattr(copilot_ctx, "latest_recorded_build_test_outcome", None)
+ if isinstance(recorded, RecordedBuildTestOutcome) and recorded.structural_key is not None:
+ return f"recorded_build_test_outcome:{recorded.structural_key}"
identity = contract.diagnosis_result.root_cause_identity
if identity.primary_category == _AUTHORING_REPAIR_CATEGORY and identity.root_cause_signature:
return identity.root_cause_signature
@@ -2533,7 +2565,7 @@ def _update_repair_loop_state(copilot_ctx: Any, contract: DiagnosisRepairContrac
if progressed:
reset_no_progress_interaction_count(copilot_ctx)
- signature = _repair_non_convergence_signature(contract)
+ signature = _repair_non_convergence_signature(copilot_ctx, contract)
if signature is None or progressed:
copilot_ctx.consecutive_non_converging_repair_count = 0
copilot_ctx.last_repair_non_convergence_signature = None
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index e30f8fa45..11729299d 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -12,6 +12,10 @@ import structlog
from skyvern.config import settings
from skyvern.forge import app
+from skyvern.forge.sdk.copilot.build_test_outcome import (
+ record_build_test_outcome,
+ recorded_outcome_from_loaded_result_evidence,
+)
from skyvern.forge.sdk.copilot.composition_browser_expressions import (
scout_accessible_role_name_expression as _scout_accessible_role_name_expression,
)
@@ -945,6 +949,7 @@ async def _maybe_steer_evaluate_to_action(
if loaded_results is not None:
_reset_evaluate_tracker(ctx)
ctx.latest_evaluate_result_composition_steer = loaded_results
+ record_build_test_outcome(ctx, recorded_outcome_from_loaded_result_evidence(loaded_results))
data.pop("actionable_targets", None)
data["composition_targets"] = loaded_result_composition_target_summary(loaded_results)
data["next_action"] = "compose_extraction"
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 67916620f..932ff66a9 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -22,6 +22,7 @@ from skyvern.forge.sdk.copilot.agent import (
)
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.build_phase import BuildPhase
+from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome
from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.completion_criteria_store import (
StoredCriteriaSet,
@@ -836,6 +837,30 @@ workflow_definition:
assert "adapt the next code block to the observed page state" in prompt
assert "do not re-emit the same failing selector or name path" in prompt
+ def test_recorded_build_test_outcome_prompt_renders_structural_grounding(self) -> None:
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ latest_recorded_build_test_outcome=RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ attempted_block_label="search_records",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_failed",
+ structural_failure_identity="runtime:timeout_waiting_for_selector:failed",
+ page_evidence_refs=["form:Search #search", "result:#results rows=unknown"],
+ observed_evidence_summary="Timeout waiting for results.",
+ ),
+ )
+
+ prompt = agent_module._recorded_build_test_outcome_prompt(ctx)
+
+ assert "RECORDED BUILD-TEST OUTCOME" in prompt
+ assert "phase: persisted_block_run" in prompt
+ assert "reason_code: runtime_block_failure" in prompt
+ assert "page_evidence_refs: form:Search #search, result:#results rows=unknown" in prompt
+ assert "Do not re-emit the same plan against the same structural key" in prompt
+
class TestVerifiedWorkflowOrNone:
"""SKY-9143 strict invariant: a proposal surfaces only after a passing test this turn."""
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
new file mode 100644
index 000000000..19056103e
--- /dev/null
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -0,0 +1,429 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from skyvern.forge.sdk.copilot.build_test_outcome import (
+ RecordedBuildTestOutcome,
+ authored_structure_signature_from_workflow,
+ record_build_test_outcome,
+ recorded_outcome_from_loaded_result_evidence,
+ recorded_outcome_from_run_blocks_result,
+)
+from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
+from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence, LoadedResultCompositionTarget
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+
+
+def test_structural_key_changes_when_page_or_result_structure_changes() -> None:
+ first = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_first",
+ block_labels=["search_records"],
+ structural_failure_identity="runtime:timeout_waiting_for_selector:failed",
+ page_evidence_refs=["origin_present", "results:empty"],
+ observed_evidence_summary="No matching records.",
+ )
+ second = first.model_copy(
+ update={
+ "workflow_run_id": "wr_second",
+ "page_evidence_refs": ["origin_present", "results:table_rows"],
+ "observed_evidence_summary": "A table with one result row appeared.",
+ }
+ )
+
+ assert first.structural_key is not None
+ assert second.structural_key is not None
+ assert first.structural_key != second.structural_key
+
+
+def test_structural_key_ignores_display_prose_and_workflow_run_id() -> None:
+ first = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_first",
+ block_labels=["search_records"],
+ structural_failure_identity="runtime:timeout_waiting_for_selector:failed",
+ page_evidence_refs=["origin_present", "form:search"],
+ observed_evidence_summary="Timeout waiting for #results.",
+ display_text="The page did not show results.",
+ )
+ second = first.model_copy(
+ update={
+ "workflow_run_id": "wr_second",
+ "observed_evidence_summary": "Different explanation with the same structural observation.",
+ "display_text": "Another user-facing sentence.",
+ }
+ )
+
+ assert first.structural_key == second.structural_key
+
+
+def test_structural_key_ignores_authored_signature_but_retains_it() -> None:
+ first = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:typed-outcome",
+ page_evidence_refs=["origin:https://example.com", "result:#results rows=0"],
+ authored_structure_signature="authored:first",
+ )
+ second = first.model_copy(update={"authored_structure_signature": "authored:second"})
+
+ assert first.structural_key is not None
+ assert first.structural_key == second.structural_key
+ assert first.authored_structure_signature == "authored:first"
+ assert second.authored_structure_signature == "authored:second"
+
+
+def test_structural_key_does_not_require_fixture_slug_or_raw_transcript_text() -> None:
+ outcome = recorded_outcome_from_loaded_result_evidence(
+ LoadedResultCompositionEvidence(
+ result_container_count=1,
+ table_result_container_count=1,
+ targets=(
+ LoadedResultCompositionTarget(
+ selector="#results",
+ is_table=True,
+ row_count=3,
+ sample_rows=("synthetic row text that must not become identity",),
+ text_excerpt="synthetic transcript excerpt that must not become identity",
+ structure_signature="target-structure",
+ ),
+ ),
+ structure_signature="result-structure",
+ )
+ )
+
+ assert outcome.structural_key is not None
+ key_payload = outcome.structural_key_payload
+ assert "fixture" not in str(key_payload).lower()
+ assert "synthetic row text" not in str(key_payload)
+ assert "synthetic transcript excerpt" not in str(key_payload)
+
+
+def test_prose_or_label_only_typed_outcome_is_not_authoritative() -> None:
+ outcome = RecordedBuildTestOutcome(
+ phase="author_time_reject",
+ attempted_tool="update_workflow",
+ attempted_block_label="search_records",
+ verdict="authoring_rejected",
+ reason_code="code_safety_reject",
+ block_labels=["search_records"],
+ observed_evidence_summary="This sounds like the same failure.",
+ display_text="Use the prior failure reason.",
+ )
+
+ assert outcome.structural_key is None
+ assert outcome.is_authoritative is False
+
+
+def test_record_none_clears_stale_latest_outcome() -> None:
+ ctx = SimpleNamespace(latest_recorded_build_test_outcome=None, recorded_build_test_outcome_history=[])
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ structural_failure_identity="typed-runtime-identity",
+ ),
+ )
+ assert ctx.latest_recorded_build_test_outcome is not None
+
+ record_build_test_outcome(ctx, None)
+
+ assert ctx.latest_recorded_build_test_outcome is None
+
+
+def test_authored_structure_signature_is_stable_and_excludes_raw_code_or_prose() -> None:
+ workflow_yaml = """
+ title: Registry lookup
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: provider_query
+ blocks:
+ - block_type: code
+ label: search_registry
+ parameter_keys:
+ - provider_query
+ code: |
+ await page.goto("https://example.com/search")
+ return {"records": [{"npi": "123"}]}
+ """
+ metadata = [
+ {
+ "block_label": "search_registry",
+ "declared_goal": "Find the exact provider row from the page prose.",
+ "claimed_outcomes": [
+ {
+ "id": "claim:provider",
+ "text": "The provider was found in the directory.",
+ "goal_value_paths": ["records[].npi"],
+ "extraction_schema": {
+ "type": "object",
+ "properties": {"records": {"type": "array", "items": {"type": "object"}}},
+ },
+ }
+ ],
+ }
+ ]
+ prose_changed_metadata = [
+ {
+ **metadata[0],
+ "declared_goal": "Different page prose for the same structure.",
+ "claimed_outcomes": [{**metadata[0]["claimed_outcomes"][0], "text": "Different prose."}],
+ }
+ ]
+
+ signature = authored_structure_signature_from_workflow(workflow_yaml, metadata)
+ same_structure = authored_structure_signature_from_workflow(workflow_yaml, prose_changed_metadata)
+ dumped = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:typed",
+ authored_structure_signature=signature,
+ ).model_dump(mode="json")
+
+ assert signature is not None
+ assert signature == same_structure
+ assert "page.goto" not in str(dumped)
+ assert "Find the exact provider" not in str(dumped)
+
+
+def test_authored_structure_signature_changes_on_code_parameter_or_output_structure() -> None:
+ base = """
+ title: Registry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_registry
+ parameter_keys:
+ - provider_query
+ code: |
+ return {"records": [{"npi": "123"}]}
+ """
+ changed_code = base.replace('"123"', '"456"')
+ changed_parameter = base.replace("provider_query", "provider_name")
+ metadata = [{"block_label": "search_registry", "claimed_outcomes": [{"goal_value_paths": ["records[].npi"]}]}]
+ changed_metadata = [
+ {"block_label": "search_registry", "claimed_outcomes": [{"goal_value_paths": ["records[].license"]}]}
+ ]
+
+ signature = authored_structure_signature_from_workflow(base, metadata)
+
+ assert signature is not None
+ assert authored_structure_signature_from_workflow(changed_code, metadata) != signature
+ assert authored_structure_signature_from_workflow(changed_parameter, metadata) != signature
+ assert authored_structure_signature_from_workflow(base, changed_metadata) != signature
+
+
+def test_outcome_not_demonstrated_keeps_authoritative_unsatisfied_criteria_identity() -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_partial",
+ "overall_status": "completed",
+ "blocks": [
+ {
+ "label": "lookup_provider_and_extract_credentials",
+ "status": "completed",
+ "extracted_data": {"npi": "", "evidence_text": "address and statuses appear in page text"},
+ }
+ ],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["npi", "locations", "statuses", "overall"],
+ verdicts=[
+ CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence", output_path="npi"),
+ CriterionVerdict(
+ criterion_id="locations",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="locations",
+ ),
+ CriterionVerdict(
+ criterion_id="statuses",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="credentialing_statuses",
+ ),
+ CriterionVerdict(
+ criterion_id="overall",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="overall_credentialing_status",
+ ),
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_partial",
+ ),
+ completion_verification=verification,
+ authored_structure_signature="authored:partial-location-only",
+ )
+
+ assert outcome is not None
+ assert outcome.phase == "persisted_block_run"
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert outcome.workflow_run_id == "wr_partial"
+ assert outcome.is_authoritative is True
+ assert outcome.structural_key is not None
+ assert outcome.authored_structure_signature == "authored:partial-location-only"
+ assert outcome.key_provenance["structural_failure_identity"] == "CompletionVerificationResult verdict structure"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "criterion_id": "statuses",
+ "output_path": "credentialing_statuses",
+ "output_root": "credentialing_statuses",
+ "reason_code": "no_evidence",
+ "value_status": "no_typed_value",
+ "partial_output_block_labels": ["lookup_provider_and_extract_credentials"],
+ },
+ {
+ "criterion_id": "locations",
+ "output_path": "locations",
+ "output_root": "locations",
+ "reason_code": "no_evidence",
+ "value_status": "no_typed_value",
+ "partial_output_block_labels": ["lookup_provider_and_extract_credentials"],
+ },
+ {
+ "criterion_id": "npi",
+ "output_path": "npi",
+ "output_root": "npi",
+ "reason_code": "no_evidence",
+ "value_status": "empty_typed_value",
+ },
+ {
+ "criterion_id": "overall",
+ "output_path": "overall_credentialing_status",
+ "output_root": "overall_credentialing_status",
+ "reason_code": "no_evidence",
+ "value_status": "no_typed_value",
+ "partial_output_block_labels": ["lookup_provider_and_extract_credentials"],
+ },
+ ]
+ payload_text = str(outcome.structural_key_payload)
+ assert "evidence_text" not in payload_text
+ assert "address and statuses" not in payload_text
+
+
+def test_runtime_block_failure_outcome_includes_bounded_page_state_and_run_id() -> None:
+ result = {
+ "ok": False,
+ "error": "Timeout waiting for results.",
+ "data": {
+ "workflow_run_id": "wr_failed",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "search_records",
+ "status": "failed",
+ "failure_reason": "Timeout waiting for results.",
+ }
+ ],
+ },
+ }
+ page_evidence = {
+ "workflow_run_id": "wr_failed",
+ "observed_after_workflow_run": True,
+ "source_tool": "inspect_page_for_composition",
+ "current_url": "https://example.test/search?secret=redacted",
+ "page_title": "Search",
+ "forms": [{"fields": [{"label": "Search", "selector": "#search"}]}],
+ "result_containers": [{"selector": "#results", "text_excerpt": "No matching records"}],
+ }
+
+ outcome = recorded_outcome_from_run_blocks_result(result, page_evidence=page_evidence)
+
+ assert outcome.workflow_run_id == "wr_failed"
+ assert outcome.phase == "persisted_block_run"
+ assert outcome.reason_code == "runtime_block_failure"
+ assert outcome.structural_key is not None
+ assert "form:Search #search" in outcome.page_evidence_refs
+ assert "result:#results rows=unknown" in outcome.page_evidence_refs
+
+
+def test_runtime_block_failure_outcome_keys_playwright_hidden_locator_structure() -> None:
+ table_result = {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_failed",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "extract_records",
+ "status": "failed",
+ "failure_reason": (
+ "Failed to execute code block. Reason: TimeoutError: Locator.wait_for: "
+ 'Timeout 15000ms exceeded.\nCall log:\nwaiting for locator("table").first '
+ "to be visible\n - locator resolved to hidden <table>...</table>\n"
+ ),
+ }
+ ],
+ },
+ }
+ row_result = {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_failed_again",
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "extract_records",
+ "status": "failed",
+ "failure_reason": (
+ "Failed to execute code block. Reason: TimeoutError: Locator.wait_for: "
+ 'Timeout 15000ms exceeded.\nCall log:\nwaiting for locator("table tbody tr").first '
+ "to be visible\n - locator resolved to hidden <tr>...</tr>\n"
+ ),
+ }
+ ],
+ },
+ }
+
+ table_outcome = recorded_outcome_from_run_blocks_result(table_result)
+ row_outcome = recorded_outcome_from_run_blocks_result(row_result)
+
+ assert table_outcome is not None
+ assert row_outcome is not None
+ assert table_outcome.is_authoritative is True
+ assert row_outcome.is_authoritative is True
+ assert table_outcome.structural_key != row_outcome.structural_key
+ assert table_outcome.key_provenance["structural_failure_identity"] == "typed runtime failure structure"
+
+
+def test_persisted_run_prose_only_failure_is_not_authoritative() -> None:
+ result = {
+ "ok": False,
+ "error": "The registry form failed after waiting for the same selector.",
+ "data": {
+ "workflow_run_id": "wr_failed",
+ "overall_status": "failed",
+ "failure_reason": "Timeout waiting for selector #results on the registry form.",
+ "blocks": [{"label": "search_records", "status": "failed"}],
+ },
+ }
+
+ outcome = recorded_outcome_from_run_blocks_result(result)
+
+ assert outcome is None or outcome.is_authoritative is False
+ assert outcome is None or outcome.structural_key is None
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index a3d71a8bf..5e06b5c05 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -17,6 +17,7 @@ import pytest
from agents import RunConfig
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, stash_blocker_signal
+from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome
from skyvern.forge.sdk.copilot.code_block_synthesis import SynthesizedCodeBlock
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.config import SYNTHESIZED_OFFER_REFRESH_STEP_THRESHOLD, BlockAuthoringPolicy
@@ -274,6 +275,101 @@ class TestSynthesizedOfferPersistenceGate:
assert signal.renders_final_reply is False
assert synthesized_block_persistence_signal(ctx, "update_and_run_blocks") is None
+ def test_authoring_offer_blocks_page_mutating_tool_before_loop_detection(self) -> None:
+ ctx = _Ctx()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ )
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.synthesized_block_offered = True
+ ctx.synthesized_block_offered_trajectory_len = 1
+ ctx.synthesized_block_offered_goal_complete = False
+ ctx.scout_trajectory = [{"tool_name": "type_text", "selector": "input[name='q']", "accessible_name": "Search"}]
+
+ signal = synthesized_block_persistence_signal(ctx, "click")
+
+ assert isinstance(signal, CopilotToolBlockerSignal)
+ assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
+ assert signal.blocked_tool == "click"
+ assert signal.cleared_by_tools == frozenset({"update_and_run_blocks"})
+ assert signal.renders_final_reply is False
+
+ @pytest.mark.parametrize(
+ "tool_name",
+ ["update_and_run_blocks", "update_workflow", "fill_credential_field"],
+ )
+ def test_authoring_offer_keeps_allowed_tools_unblocked(self, tool_name: str) -> None:
+ ctx = _Ctx()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ )
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.synthesized_block_offered = True
+ ctx.synthesized_block_offered_trajectory_len = 1
+ ctx.synthesized_block_offered_goal_complete = False
+ ctx.scout_trajectory = [{"tool_name": "type_text", "selector": "input[name='q']", "accessible_name": "Search"}]
+
+ assert synthesized_block_persistence_signal(ctx, tool_name) is None
+
+ def test_authoring_offer_does_not_block_mutating_tool_when_trajectory_changed(self) -> None:
+ ctx = _Ctx()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ )
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.synthesized_block_offered = True
+ ctx.synthesized_block_offered_trajectory_len = 1
+ ctx.synthesized_block_offered_goal_complete = False
+ ctx.scout_trajectory = [
+ {"tool_name": "type_text", "selector": "input[name='q']", "accessible_name": "Search"},
+ {"tool_name": "click", "selector": "button[data-action='search']", "accessible_name": "Search"},
+ ]
+
+ assert synthesized_block_persistence_signal(ctx, "click") is None
+
+ def test_authoring_offer_does_not_block_mutating_tool_without_offer(self) -> None:
+ ctx = _Ctx()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ )
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.synthesized_block_offered = False
+ ctx.synthesized_block_offered_trajectory_len = 1
+ ctx.scout_trajectory = [{"tool_name": "type_text", "selector": "input[name='q']", "accessible_name": "Search"}]
+
+ assert synthesized_block_persistence_signal(ctx, "click") is None
+
+ def test_unresolved_recorded_outcome_blocks_page_mutating_tool_until_update_and_run_blocks(self) -> None:
+ ctx = _Ctx()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ )
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.completion_verification_result = self._unsatisfied_verification()
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:partial",
+ )
+
+ signal = synthesized_block_persistence_signal(ctx, "click")
+
+ assert isinstance(signal, CopilotToolBlockerSignal)
+ assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
+ assert signal.blocked_tool == "click"
+ assert signal.cleared_by_tools == frozenset({"update_and_run_blocks"})
+ assert "last recorded test outcome" in signal.agent_steering_text
+ assert synthesized_block_persistence_signal(ctx, "update_and_run_blocks") is None
+ assert synthesized_block_persistence_signal(ctx, "evaluate") is None
+
@pytest.mark.parametrize(
"ctx_attrs",
[
diff --git a/tests/unit/test_copilot_run_output_content_gate.py b/tests/unit/test_copilot_run_output_content_gate.py
index 7554a799e..57a0630d3 100644
--- a/tests/unit/test_copilot_run_output_content_gate.py
+++ b/tests/unit/test_copilot_run_output_content_gate.py
@@ -772,6 +772,56 @@ def test_satisfied_completion_prevents_empty_output_suspicious_success() -> None
assert ctx.last_test_suspicious_success is False
assert ctx.last_test_failure_reason is None
assert verified_goal_satisfied_context(ctx) is True
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.verdict == "progress_observed"
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "verified_success"
+ assert ctx.latest_recorded_build_test_outcome.structural_key is not None
+
+
+def test_unverified_success_records_goal_relative_build_test_outcome() -> None:
+ result = _run_result([_code_block("search_registry_person", {"records": [], "result_count": 0})])
+ ctx = _ctx(result["data"]["blocks"])
+
+ recorded = _record_run_blocks_result(ctx, result, completion_verification=_no_evidence("c0"))
+
+ assert recorded is not None
+ assert recorded.verdict == "not_demonstrated"
+ assert recorded.reason_code == "outcome_not_demonstrated"
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.phase == "persisted_block_run"
+ assert outcome.verdict == "repairable_failure"
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert outcome.structural_key is not None
+
+
+def test_changed_output_structure_changes_unverified_success_structural_key() -> None:
+ first_result = _run_result([_code_block("search_registry_person", {"records": [], "result_count": 0})])
+ second_result = _run_result(
+ [_code_block("search_registry_person", {"records": [{"status_present": True}], "result_count": 1})]
+ )
+ ctx = _ctx(first_result["data"]["blocks"])
+
+ _record_run_blocks_result(ctx, first_result, completion_verification=_no_evidence("c0"))
+ first_key = ctx.latest_recorded_build_test_outcome.structural_key
+ _record_run_blocks_result(ctx, second_result, completion_verification=_no_evidence("c0"))
+
+ assert first_key is not None
+ assert ctx.latest_recorded_build_test_outcome.structural_key is not None
+ assert ctx.latest_recorded_build_test_outcome.structural_key != first_key
+
+
+def test_terminal_blocker_does_not_leave_authoritative_prompt_outcome() -> None:
+ result = _blocked_flag_run_result()
+ ctx = _ctx(result["data"]["blocks"])
+
+ _record_run_blocks_result(ctx, result, completion_verification=None)
+
+ assert ctx.blocker_signal is not None
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "terminal_challenge_blocker"
+ assert outcome.is_authoritative is False
def test_all_null_metadata_goal_fields_are_flagged_as_no_goal_content() -> None:
diff --git a/tests/unit/test_evaluate_actionable_target.py b/tests/unit/test_evaluate_actionable_target.py
index 7d7275234..328262ac2 100644
--- a/tests/unit/test_evaluate_actionable_target.py
+++ b/tests/unit/test_evaluate_actionable_target.py
@@ -346,6 +346,10 @@ async def test_first_loaded_result_table_sheds_large_payload_but_keeps_compositi
serialized = json.dumps(result, default=str)
assert result["data"].get("next_action") == "compose_extraction"
assert result["data"].get("composition_targets")
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.phase == "scout_evaluate"
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "loaded_result_targets_observed"
+ assert ctx.latest_recorded_build_test_outcome.structural_key is not None
assert len(serialized) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAP
assert '"next_action"' in serialized[: scouting._RECENT_TOOL_OUTPUT_CHAR_CAP]
assert "compose_extraction" in serialized