Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-14839 Return complete build-test packets to Copilot (#8280)
Andrew Neilsonbea7dec10eb3
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pyindex f363d8c57..5403423d5 100644--- a/skyvern/forge/sdk/copilot/build_test_outcome.py+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -11,7 +11,7 @@ from urllib.parse import urlsplitimport structlogimport yaml-from pydantic import BaseModel, ConfigDict, Field+from pydantic import BaseModel, ConfigDict, Field, JsonValuefrom skyvern.forge.sdk.copilot.challenge_evidence import (carrier_backed_anti_bot_categories,@@ -67,6 +67,8 @@ BuildTestOutcomeReasonCode = Literal[_TERMINAL_CHALLENGE_REASON_CODES: frozenset[BuildTestOutcomeReasonCode] = frozenset({"terminal_challenge_blocker"})PostRunPagePathKind = Literal["login", "challenge", "incomplete_navigation", "non_page_outcome"]PostRunPagePathTargetKind = Literal["form_submit", "navigation", "clickable", "challenge"]+BuildTestPacketWorkflowSource = Literal["accepted_write_readback", "turn_start_persisted_readback", "unavailable"]+BuildTestPacketUnfinishedKind = Literal["unverified_block", "missing_requested_output"]_STRUCTURAL_KEY_VERSION = "recorded_build_test_outcome:v1"_AUTHORED_STRUCTURE_VERSION = "recorded_build_test_outcome_authored_structure:v1"@@ -105,6 +107,92 @@ class PostRunPagePathFailure(BaseModel):return self.kind != "non_page_outcome" and bool(self.continuation_targets)+class BuildTestPacketRun(BaseModel):+ model_config = ConfigDict(extra="forbid", frozen=True)++ workflow_run_id: str | None = None+ status: str | None = None+++class BuildTestPacketPageState(BaseModel):+ model_config = ConfigDict(extra="forbid", frozen=True)++ current_origin: str | None = None+ current_url: str | None = None+ title: str | None = None+ evidence_source: str | None = None+ observed_after_workflow_run: bool = False+ form_summaries: list[str] = Field(default_factory=list)+ result_summaries: list[str] = Field(default_factory=list)+ action_summaries: list[str] = Field(default_factory=list)+ challenge_summaries: list[str] = Field(default_factory=list)+ obstruction_summaries: list[str] = Field(default_factory=list)+++class BuildTestPacketFailure(BaseModel):+ model_config = ConfigDict(extra="forbid", frozen=True)++ block_label: str | None = None+ block_status: str | None = None+ reason: str | None = None+ action_trace: list[str] = Field(default_factory=list)+ page_state: BuildTestPacketPageState | None = None+++class BuildTestPacketRegisteredOutput(BaseModel):+ model_config = ConfigDict(extra="forbid", frozen=True)++ workflow_run_id: str | None = None+ output_parameter_id: str | None = None+ output_parameter_key: str | None = None+ block_label: str | None = None+ block_type: str | None = None+ value: JsonValue = None+ value_complete: bool = True+++class BuildTestPacketDownload(BaseModel):+ model_config = ConfigDict(extra="forbid", frozen=True)++ artifact_id: str+ file_name: str | None = None+++class BuildTestPacketScreenshot(BaseModel):+ model_config = ConfigDict(extra="forbid", frozen=True)++ present: bool+ provenance: str | None = None+++class BuildTestPacketUnfinishedItem(BaseModel):+ model_config = ConfigDict(extra="forbid", frozen=True)++ kind: BuildTestPacketUnfinishedKind+ label: str | None = None+ output_path: str | None = None+ reason_code: str | None = None+++class BuildTestEvidencePacket(BaseModel):+ model_config = ConfigDict(extra="forbid", frozen=True)++ contract_version: Literal["build_test_evidence_packet_v1"] = "build_test_evidence_packet_v1"+ workflow_permanent_id: str | None = None+ canonical_workflow_yaml: str | None = None+ canonical_workflow_source: BuildTestPacketWorkflowSource+ canonical_workflow_yaml_complete: bool = True+ attempted_block_labels: list[str] = Field(default_factory=list)+ executed_block_labels: list[str] = Field(default_factory=list)+ run: BuildTestPacketRun+ failure: BuildTestPacketFailure | None = None+ registered_outputs: list[BuildTestPacketRegisteredOutput] = Field(default_factory=list)+ downloads: list[BuildTestPacketDownload] = Field(default_factory=list)+ screenshot: BuildTestPacketScreenshot+ unfinished_items: list[BuildTestPacketUnfinishedItem] = Field(default_factory=list)+ omission_notices: list[str] = Field(default_factory=list)++class RecordedBuildTestOutcome(BaseModel):model_config = ConfigDict(extra="forbid")diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex 9aea022da..77166046e 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -10,6 +10,8 @@ import unicodedatafrom collections.abc import Callable, Iterable, Iteratorfrom typing import TYPE_CHECKING, Any+import structlog+from skyvern.forge.sdk.agents.context import sanitize_agent_tool_result_for_llm as sanitize_generic_tool_result_for_llmfrom skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, assert_clean_user_facing_textfrom skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES@@ -21,8 +23,27 @@ from skyvern.schemas.workflows import BlockTypeif TYPE_CHECKING:from agents.result import RunResultStreaming+ from skyvern.forge.sdk.copilot.build_test_outcome import BuildTestEvidencePacket, BuildTestPacketPageState++LOG = structlog.get_logger()+_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY = "_copilot_internal_run_cancelled_by_watchdog"_BASE64_IMAGE_OMITTED_MESSAGE = "[base64 image omitted — screenshot was taken successfully]"+BUILD_TEST_PACKET_KEY = "build_test_packet"++_BUILD_TEST_PACKET_MAX_CHARS = 47_000+_BUILD_TEST_WORKFLOW_MAX_CHARS = 30_000+_BUILD_TEST_IDENTIFIER_MAX_CHARS = 160+_BUILD_TEST_FAILURE_REASON_MAX_CHARS = 1_200+_BUILD_TEST_URL_MAX_CHARS = 1_600+_BUILD_TEST_LABEL_MAX_ITEMS = 24+_BUILD_TEST_OUTPUT_MAX_ITEMS = 12+_BUILD_TEST_OUTPUT_VALUE_MAX_CHARS = 800+_BUILD_TEST_DOWNLOAD_MAX_ITEMS = 12+_BUILD_TEST_UNFINISHED_MAX_ITEMS = 24+_BUILD_TEST_ACTION_TRACE_MAX_ITEMS = 6+_BUILD_TEST_PAGE_SUMMARY_MAX_ITEMS = 8+_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS = 300_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"_JPEG_PREFIX = b"\xff\xd8\xff"@@ -247,6 +268,430 @@ def _summarize_extracted_data(extracted: Any) -> str:return "Extracted data present."+def _append_omission(notices: list[str], notice: str) -> None:+ if notice not in notices:+ notices.append(notice)+++def _bounded_packet_string(+ value: str | None,+ *,+ field_name: str,+ max_chars: int,+ notices: list[str],+) -> str | None:+ if value is None or len(value) <= max_chars:+ return value+ _append_omission(notices, f"{field_name} shortened at {max_chars} characters.")+ return value[: max_chars - 3] + "..."+++def _bounded_packet_strings(+ values: list[str],+ *,+ field_name: str,+ max_items: int,+ max_chars: int | None,+ notices: list[str],+) -> list[str]:+ bounded = values[:max_items]+ if len(values) > max_items:+ _append_omission(notices, f"{field_name} shortened: {len(values) - max_items} item(s) omitted.")+ if max_chars is None:+ return bounded+ rendered: list[str] = []+ shortened = 0+ for value in bounded:+ if len(value) <= max_chars:+ rendered.append(value)+ continue+ rendered.append(value[: max_chars - 3] + "...")+ shortened += 1+ if shortened:+ _append_omission(notices, f"{field_name} shortened: {shortened} text value(s) clipped.")+ return rendered+++def _bounded_packet_page_state(+ page_state: BuildTestPacketPageState | None, notices: list[str]+) -> BuildTestPacketPageState | None:+ if page_state is None:+ return None+ updates = {+ "current_origin": _bounded_packet_string(+ page_state.current_origin,+ field_name="failure.page_state.current_origin",+ max_chars=_BUILD_TEST_URL_MAX_CHARS,+ notices=notices,+ ),+ "current_url": _bounded_packet_string(+ page_state.current_url,+ field_name="failure.page_state.current_url",+ max_chars=_BUILD_TEST_URL_MAX_CHARS,+ notices=notices,+ ),+ "title": _bounded_packet_string(+ page_state.title,+ field_name="failure.page_state.title",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "evidence_source": _bounded_packet_string(+ page_state.evidence_source,+ field_name="failure.page_state.evidence_source",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "form_summaries": _bounded_packet_strings(+ page_state.form_summaries,+ field_name="failure.page_state.form_summaries",+ max_items=_BUILD_TEST_PAGE_SUMMARY_MAX_ITEMS,+ max_chars=_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS,+ notices=notices,+ ),+ "result_summaries": _bounded_packet_strings(+ page_state.result_summaries,+ field_name="failure.page_state.result_summaries",+ max_items=_BUILD_TEST_PAGE_SUMMARY_MAX_ITEMS,+ max_chars=_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS,+ notices=notices,+ ),+ "action_summaries": _bounded_packet_strings(+ page_state.action_summaries,+ field_name="failure.page_state.action_summaries",+ max_items=_BUILD_TEST_PAGE_SUMMARY_MAX_ITEMS,+ max_chars=_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS,+ notices=notices,+ ),+ "challenge_summaries": _bounded_packet_strings(+ page_state.challenge_summaries,+ field_name="failure.page_state.challenge_summaries",+ max_items=_BUILD_TEST_PAGE_SUMMARY_MAX_ITEMS,+ max_chars=_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS,+ notices=notices,+ ),+ "obstruction_summaries": _bounded_packet_strings(+ page_state.obstruction_summaries,+ field_name="failure.page_state.obstruction_summaries",+ max_items=_BUILD_TEST_PAGE_SUMMARY_MAX_ITEMS,+ max_chars=_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS,+ notices=notices,+ ),+ }+ return page_state.model_copy(update=updates)+++def _compact_packet_for_aggregate_limit(+ packet: BuildTestEvidencePacket,+ notices: list[str],+) -> BuildTestEvidencePacket:+ from skyvern.forge.sdk.copilot.build_test_outcome import BuildTestPacketRegisteredOutput++ _append_omission(+ notices,+ f"repeated packet facts shortened further to keep the packet under {_BUILD_TEST_PACKET_MAX_CHARS} characters.",+ )+ outputs: list[BuildTestPacketRegisteredOutput] = []+ for output in packet.registered_outputs[:6]:+ rendered = json.dumps(output.value, ensure_ascii=False, separators=(",", ":"))+ outputs.append(+ output.model_copy(+ update={+ "value": rendered[:197] + "..." if len(rendered) > 200 else output.value,+ "value_complete": output.value_complete and len(rendered) <= 200,+ }+ )+ )+ failure = packet.failure+ if failure is not None:+ page_state = failure.page_state+ if page_state is not None:++ def compact_summaries(values: list[str]) -> list[str]:+ return [value[:117] + "..." if len(value) > 120 else value for value in values[:2]]++ page_state = page_state.model_copy(+ update={+ "form_summaries": compact_summaries(page_state.form_summaries),+ "result_summaries": compact_summaries(page_state.result_summaries),+ "action_summaries": compact_summaries(page_state.action_summaries),+ "challenge_summaries": compact_summaries(page_state.challenge_summaries),+ "obstruction_summaries": compact_summaries(page_state.obstruction_summaries),+ }+ )+ failure = failure.model_copy(+ update={+ "action_trace": [+ value[:117] + "..." if len(value) > 120 else value for value in failure.action_trace[:2]+ ],+ "page_state": page_state,+ }+ )+ return packet.model_copy(+ update={+ "canonical_workflow_yaml": None,+ "canonical_workflow_yaml_complete": False,+ "attempted_block_labels": packet.attempted_block_labels[:12],+ "executed_block_labels": packet.executed_block_labels[:12],+ "failure": failure,+ "registered_outputs": outputs,+ "downloads": packet.downloads[:6],+ "unfinished_items": packet.unfinished_items[:12],+ "omission_notices": notices,+ }+ )+++def project_build_test_packet_for_llm(packet: BuildTestEvidencePacket) -> BuildTestEvidencePacket:+ """Return the one bounded model projection of a factual build-test packet."""+ from skyvern.forge.sdk.copilot.build_test_outcome import BuildTestPacketRegisteredOutput++ notices = list(packet.omission_notices)+ workflow_yaml = packet.canonical_workflow_yaml+ workflow_complete = packet.canonical_workflow_yaml_complete+ if workflow_yaml is not None and len(workflow_yaml) > _BUILD_TEST_WORKFLOW_MAX_CHARS:+ workflow_yaml = workflow_yaml[: _BUILD_TEST_WORKFLOW_MAX_CHARS - 3] + "..."+ workflow_complete = False+ _append_omission(+ notices,+ "canonical_workflow_yaml shortened at 30000 characters; "+ "use the persisted workflow readback for full bytes.",+ )++ attempted = _bounded_packet_strings(+ packet.attempted_block_labels,+ field_name="attempted_block_labels",+ max_items=_BUILD_TEST_LABEL_MAX_ITEMS,+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ )+ executed = _bounded_packet_strings(+ packet.executed_block_labels,+ field_name="executed_block_labels",+ max_items=_BUILD_TEST_LABEL_MAX_ITEMS,+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ )++ registered_outputs: list[BuildTestPacketRegisteredOutput] = []+ for output in packet.registered_outputs[:_BUILD_TEST_OUTPUT_MAX_ITEMS]:+ rendered_value = json.dumps(output.value, ensure_ascii=False, separators=(",", ":"))+ value = output.value+ value_complete = output.value_complete+ if len(rendered_value) > _BUILD_TEST_OUTPUT_VALUE_MAX_CHARS:+ value = rendered_value[: _BUILD_TEST_OUTPUT_VALUE_MAX_CHARS - 3] + "..."+ value_complete = False+ _append_omission(+ notices,+ "registered output "+ f"{output.output_parameter_key or output.output_parameter_id or '(unnamed)'} shortened.",+ )+ registered_outputs.append(+ output.model_copy(+ update={+ "workflow_run_id": _bounded_packet_string(+ output.workflow_run_id,+ field_name="registered_outputs[].workflow_run_id",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "output_parameter_id": _bounded_packet_string(+ output.output_parameter_id,+ field_name="registered_outputs[].output_parameter_id",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "output_parameter_key": _bounded_packet_string(+ output.output_parameter_key,+ field_name="registered_outputs[].output_parameter_key",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "block_label": _bounded_packet_string(+ output.block_label,+ field_name="registered_outputs[].block_label",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "block_type": _bounded_packet_string(+ output.block_type,+ field_name="registered_outputs[].block_type",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "value": value,+ "value_complete": value_complete,+ }+ )+ )+ if len(packet.registered_outputs) > _BUILD_TEST_OUTPUT_MAX_ITEMS:+ _append_omission(+ notices,+ "registered_outputs shortened: "+ f"{len(packet.registered_outputs) - _BUILD_TEST_OUTPUT_MAX_ITEMS} item(s) omitted.",+ )++ failure = packet.failure+ if failure is not None:+ action_trace = _bounded_packet_strings(+ failure.action_trace,+ field_name="failure.action_trace",+ max_items=_BUILD_TEST_ACTION_TRACE_MAX_ITEMS,+ max_chars=_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS,+ notices=notices,+ )+ failure = failure.model_copy(+ update={+ "block_label": _bounded_packet_string(+ failure.block_label,+ field_name="failure.block_label",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "block_status": _bounded_packet_string(+ failure.block_status,+ field_name="failure.block_status",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "reason": _bounded_packet_string(+ failure.reason,+ field_name="failure.reason",+ max_chars=_BUILD_TEST_FAILURE_REASON_MAX_CHARS,+ notices=notices,+ ),+ "action_trace": action_trace,+ "page_state": _bounded_packet_page_state(failure.page_state, notices),+ }+ )++ downloads = [+ download.model_copy(+ update={+ "artifact_id": _bounded_packet_string(+ download.artifact_id,+ field_name="downloads[].artifact_id",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "file_name": _bounded_packet_string(+ download.file_name,+ field_name="downloads[].file_name",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ }+ )+ for download in packet.downloads[:_BUILD_TEST_DOWNLOAD_MAX_ITEMS]+ ]+ if len(packet.downloads) > _BUILD_TEST_DOWNLOAD_MAX_ITEMS:+ _append_omission(+ notices,+ f"downloads shortened: {len(packet.downloads) - _BUILD_TEST_DOWNLOAD_MAX_ITEMS} item(s) omitted.",+ )+ unfinished = [+ item.model_copy(+ update={+ "label": _bounded_packet_string(+ item.label,+ field_name="unfinished_items[].label",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "output_path": _bounded_packet_string(+ item.output_path,+ field_name="unfinished_items[].output_path",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "reason_code": _bounded_packet_string(+ item.reason_code,+ field_name="unfinished_items[].reason_code",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ }+ )+ for item in packet.unfinished_items[:_BUILD_TEST_UNFINISHED_MAX_ITEMS]+ ]+ if len(packet.unfinished_items) > _BUILD_TEST_UNFINISHED_MAX_ITEMS:+ _append_omission(+ notices,+ "unfinished_items shortened: "+ f"{len(packet.unfinished_items) - _BUILD_TEST_UNFINISHED_MAX_ITEMS} item(s) omitted.",+ )++ projected = packet.model_copy(+ update={+ "workflow_permanent_id": _bounded_packet_string(+ packet.workflow_permanent_id,+ field_name="workflow_permanent_id",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "canonical_workflow_yaml": workflow_yaml,+ "canonical_workflow_yaml_complete": workflow_complete,+ "attempted_block_labels": attempted,+ "executed_block_labels": executed,+ "failure": failure,+ "registered_outputs": registered_outputs,+ "downloads": downloads,+ "unfinished_items": unfinished,+ "omission_notices": notices,+ }+ )+ run = projected.run.model_copy(+ update={+ "workflow_run_id": _bounded_packet_string(+ projected.run.workflow_run_id,+ field_name="run.workflow_run_id",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ "status": _bounded_packet_string(+ projected.run.status,+ field_name="run.status",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ ),+ }+ )+ screenshot = projected.screenshot.model_copy(+ update={+ "provenance": _bounded_packet_string(+ projected.screenshot.provenance,+ field_name="screenshot.provenance",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ )+ }+ )+ projected = projected.model_copy(update={"run": run, "screenshot": screenshot, "omission_notices": notices})++ serialized = json.dumps(projected.model_dump(mode="json", exclude_none=True), ensure_ascii=False)+ if len(serialized) > _BUILD_TEST_PACKET_MAX_CHARS and projected.canonical_workflow_yaml is not None:+ excess = len(serialized) - _BUILD_TEST_PACKET_MAX_CHARS+ retained_chars = max(0, len(projected.canonical_workflow_yaml) - excess - 200)+ shortened_workflow = projected.canonical_workflow_yaml[:retained_chars]+ if retained_chars >= 3:+ shortened_workflow = shortened_workflow[:-3] + "..."+ _append_omission(+ notices,+ f"canonical_workflow_yaml shortened further to keep the packet under {_BUILD_TEST_PACKET_MAX_CHARS} characters.",+ )+ projected = projected.model_copy(+ update={+ "canonical_workflow_yaml": shortened_workflow or None,+ "canonical_workflow_yaml_complete": False,+ "omission_notices": notices,+ }+ )+ serialized = json.dumps(projected.model_dump(mode="json", exclude_none=True), ensure_ascii=False)+ if len(serialized) > _BUILD_TEST_PACKET_MAX_CHARS:+ projected = _compact_packet_for_aggregate_limit(projected, notices)+ return projected++def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict[str, Any]:"""Strip large/binary fields from tool results before sending to the LLM."""sanitized = sanitize_generic_tool_result_for_llm(@@ -267,6 +712,7 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dictdata = sanitized.get("data")if isinstance(data, dict):data = dict(data)+ packet_projected = Falseif "schema" in data and isinstance(data["schema"], dict):schema_str = json.dumps(data["schema"])# 2000 chars ~= 500 LLM tokens — enough for the model to see the@@ -303,7 +749,34 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dictelse blockfor block in blocks]- sanitized["data"] = data+ raw_packet = data.get(BUILD_TEST_PACKET_KEY)+ if isinstance(raw_packet, dict):+ from skyvern.forge.sdk.copilot.build_test_outcome import BuildTestEvidencePacket++ try:+ packet = BuildTestEvidencePacket.model_validate(raw_packet)+ except ValueError:+ data.pop(BUILD_TEST_PACKET_KEY, None)+ data["build_test_packet_omitted"] = "The internal packet failed typed validation."+ else:+ try:+ projected_packet = project_build_test_packet_for_llm(packet).model_dump(+ mode="json", exclude_none=True+ )+ except Exception:+ LOG.exception("copilot build test packet projection failed")+ data.pop(BUILD_TEST_PACKET_KEY, None)+ data["build_test_packet_omitted"] = "The internal packet projection failed."+ else:+ data = {+ BUILD_TEST_PACKET_KEY: projected_packet,+ **{key: value for key, value in data.items() if key != BUILD_TEST_PACKET_KEY},+ }+ packet_projected = True+ if packet_projected:+ sanitized = {"data": data, **{key: value for key, value in sanitized.items() if key != "data"}}+ else:+ sanitized["data"] = datareturn sanitizeddiff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.pyindex 3a69761cf..8da797cff 100644--- a/skyvern/forge/sdk/copilot/tools/__init__.py+++ b/skyvern/forge/sdk/copilot/tools/__init__.py@@ -180,9 +180,7 @@ from .run_execution import ()from .run_execution import _progress_marker as _progress_markerfrom .run_execution import _read_progress_sources as _read_progress_sources-from .run_execution import (- _record_diagnosis_repair_contract,-)+from .run_execution import _record_diagnosis_repair_contract as _record_diagnosis_repair_contractfrom .run_execution import _record_run_blocks_result as _record_run_blocks_resultfrom .run_execution import (_run_blocks_and_collect_debug,@@ -190,6 +188,9 @@ from .run_execution import (_verify_and_record_run_blocks_result,)from .run_execution import _watchdog_error_message as _watchdog_error_message+from .run_execution import (+ finalize_build_test_result,+)from .scouting import _MAX_SCOUTED_INTERACTIONS as _MAX_SCOUTED_INTERACTIONSfrom .scouting import _capture_accessible_role_name as _capture_accessible_role_namefrom .scouting import _capture_scout_pre_action as _capture_scout_pre_action@@ -298,12 +299,13 @@ async def update_workflow_tool(_mark_credential_deferred_draft(copilot_ctx, result)_record_workflow_update_result(copilot_ctx, result, prior_definition)record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)- if result.get("ok") is False:- _record_diagnosis_repair_contract(- copilot_ctx,- source_tool="update_workflow",- result=result,- )+ finalize_build_test_result(+ copilot_ctx,+ source_tool="update_workflow",+ result=result,+ workflow_updated=result.get("ok") is True,+ diagnosis_shadow_eligible=result.get("ok") is False,+ )sanitized = sanitize_tool_result_for_llm("update_workflow", result)return json.dumps(sanitized)@@ -334,8 +336,13 @@ async def _persist_block_scoped_edit(result = await _update_workflow(params, copilot_ctx)_record_workflow_update_result(copilot_ctx, result, prior_definition)record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, result)- if result.get("ok") is False:- _record_diagnosis_repair_contract(copilot_ctx, source_tool=tool_name, result=result)+ finalize_build_test_result(+ copilot_ctx,+ source_tool=tool_name,+ result=result,+ workflow_updated=result.get("ok") is True,+ diagnosis_shadow_eligible=result.get("ok") is False,+ )return json.dumps(sanitize_tool_result_for_llm(tool_name, result))@@ -369,7 +376,14 @@ async def edit_block_tool(arguments = {"label": label, "fields": fields, "has_code_edit": expected_code is not None}authority_error = _authority_tool_error(copilot_ctx, "edit_block")if authority_error:- return json.dumps({"ok": False, "error": authority_error})+ result = {"ok": False, "error": authority_error}+ finalize_build_test_result(+ copilot_ctx,+ source_tool="edit_block",+ result=result,+ diagnosis_shadow_eligible=False,+ )+ return json.dumps(sanitize_tool_result_for_llm("edit_block", result))try:workflow_yaml = apply_block_edit(_stored_workflow_yaml(copilot_ctx),@@ -381,7 +395,13 @@ async def edit_block_tool(except BlockEditError as exc:result = {"ok": False, "error": str(exc)}record_tool_step_result_for_ctx(copilot_ctx, "edit_block", arguments, result)- return json.dumps(result)+ finalize_build_test_result(+ copilot_ctx,+ source_tool="edit_block",+ result=result,+ diagnosis_shadow_eligible=False,+ )+ return json.dumps(sanitize_tool_result_for_llm("edit_block", result))return await _persist_block_scoped_edit(copilot_ctx, "edit_block", workflow_yaml, arguments)@@ -434,7 +454,13 @@ async def edit_block_and_run_tool("error": f"block_labels must include the edited block {label!r} so this call tests the persisted repair.",}record_tool_step_result_for_ctx(copilot_ctx, "edit_block_and_run", arguments, result)- return json.dumps(result)+ finalize_build_test_result(+ copilot_ctx,+ source_tool="edit_block_and_run",+ result=result,+ diagnosis_shadow_eligible=False,+ )+ return json.dumps(sanitize_tool_result_for_llm("edit_block_and_run", result))authority_error = _authority_tool_error(copilot_ctx, "edit_block_and_run")if authority_error:return _diagnosis_repair_tool_error(copilot_ctx, "edit_block_and_run", authority_error)@@ -450,8 +476,12 @@ async def edit_block_and_run_tool(except BlockEditError as exc:result = {"ok": False, "error": str(exc)}record_tool_step_result_for_ctx(copilot_ctx, "edit_block_and_run", arguments, result)- _record_diagnosis_repair_contract(copilot_ctx, source_tool="edit_block_and_run", result=result)- return json.dumps(result)+ finalize_build_test_result(+ copilot_ctx,+ source_tool="edit_block_and_run",+ result=result,+ )+ return json.dumps(sanitize_tool_result_for_llm("edit_block_and_run", result))prior_definition = await _get_prior_workflow_definition(copilot_ctx)with copilot_span("edit_block_and_run.update", data={"yaml_length": len(workflow_yaml)}):@@ -463,7 +493,7 @@ async def edit_block_and_run_tool(_record_workflow_update_result(copilot_ctx, update_result, prior_definition)if not update_result.get("ok"):record_tool_step_result_for_ctx(copilot_ctx, "edit_block_and_run", arguments, update_result)- _record_diagnosis_repair_contract(+ finalize_build_test_result(copilot_ctx,source_tool="edit_block_and_run",result=update_result,@@ -523,7 +553,14 @@ async def add_block_tool(arguments = {"after_label": after_label, "parameters": parameters}authority_error = _authority_tool_error(copilot_ctx, "add_block")if authority_error:- return json.dumps({"ok": False, "error": authority_error})+ result = {"ok": False, "error": authority_error}+ finalize_build_test_result(+ copilot_ctx,+ source_tool="add_block",+ result=result,+ diagnosis_shadow_eligible=False,+ )+ return json.dumps(sanitize_tool_result_for_llm("add_block", result))try:workflow_yaml = add_block_to_workflow(_stored_workflow_yaml(copilot_ctx),@@ -534,7 +571,13 @@ async def add_block_tool(except BlockEditError as exc:result = {"ok": False, "error": str(exc)}record_tool_step_result_for_ctx(copilot_ctx, "add_block", arguments, result)- return json.dumps(result)+ finalize_build_test_result(+ copilot_ctx,+ source_tool="add_block",+ result=result,+ diagnosis_shadow_eligible=False,+ )+ return json.dumps(sanitize_tool_result_for_llm("add_block", result))return await _persist_block_scoped_edit(copilot_ctx,"add_block",@@ -556,13 +599,26 @@ async def delete_block_tool(ctx: RunContextWrapper, label: str) -> str:arguments = {"label": label}authority_error = _authority_tool_error(copilot_ctx, "delete_block")if authority_error:- return json.dumps({"ok": False, "error": authority_error})+ result = {"ok": False, "error": authority_error}+ finalize_build_test_result(+ copilot_ctx,+ source_tool="delete_block",+ result=result,+ diagnosis_shadow_eligible=False,+ )+ return json.dumps(sanitize_tool_result_for_llm("delete_block", result))try:workflow_yaml = delete_block_from_workflow(_stored_workflow_yaml(copilot_ctx), label)except BlockEditError as exc:result = {"ok": False, "error": str(exc)}record_tool_step_result_for_ctx(copilot_ctx, "delete_block", arguments, result)- return json.dumps(result)+ finalize_build_test_result(+ copilot_ctx,+ source_tool="delete_block",+ result=result,+ diagnosis_shadow_eligible=False,+ )+ return json.dumps(sanitize_tool_result_for_llm("delete_block", result))return await _persist_block_scoped_edit(copilot_ctx, "delete_block", workflow_yaml, arguments)@@ -806,7 +862,7 @@ async def run_blocks_tool()await _verify_and_record_run_blocks_result(copilot_ctx, result, handler_start)record_tool_step_result_for_ctx(copilot_ctx, "run_blocks_and_collect_debug", arguments, result)- _record_diagnosis_repair_contract(+ finalize_build_test_result(copilot_ctx,source_tool="run_blocks_and_collect_debug",result=result,@@ -961,7 +1017,7 @@ async def update_and_run_blocks_tool(if not update_result.get("ok"):record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, update_result)- _record_diagnosis_repair_contract(+ finalize_build_test_result(copilot_ctx,source_tool="update_and_run_blocks",result=update_result,@@ -1010,7 +1066,7 @@ def _credential_deferred_combined_tool_result(}carry_author_time_findings(update_result, skip_result)record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, skip_result)- _record_diagnosis_repair_contract(+ finalize_build_test_result(copilot_ctx,source_tool=tool_name,result=skip_result,@@ -1021,7 +1077,7 @@ def _credential_deferred_combined_tool_result(tool_name=tool_name,workflow_permanent_id=copilot_ctx.workflow_permanent_id,)- return json.dumps(skip_result)+ return json.dumps(sanitize_tool_result_for_llm(tool_name, skip_result))async def _run_updated_workflow_blocks(@@ -1069,7 +1125,7 @@ async def _run_updated_workflow_blocks(await _verify_and_record_run_blocks_result(copilot_ctx, run_result, handler_start)carry_author_time_findings(update_result, run_result)record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, run_result)- _record_diagnosis_repair_contract(+ finalize_build_test_result(copilot_ctx,source_tool=tool_name,result=run_result,diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 4a877ff35..4ec3cf305 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -27,6 +27,14 @@ from skyvern.forge.sdk.copilot.blocker_signal import (stash_blocker_signal,)from skyvern.forge.sdk.copilot.build_test_outcome import (+ BuildTestEvidencePacket,+ BuildTestPacketDownload,+ BuildTestPacketFailure,+ BuildTestPacketPageState,+ BuildTestPacketRegisteredOutput,+ BuildTestPacketRun,+ BuildTestPacketScreenshot,+ BuildTestPacketUnfinishedItem,authored_block_parameter_keys_from_workflow,authored_structure_signature_from_workflow,record_build_test_outcome,@@ -72,8 +80,10 @@ from skyvern.forge.sdk.copilot.narration import narrator_poll_tickfrom skyvern.forge.sdk.copilot.outcome_verification_trace import record_gate_decisionfrom skyvern.forge.sdk.copilot.output_utils import (_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,+ BUILD_TEST_PACKET_KEY,build_run_blocks_response,iter_failure_reasons,+ sanitize_tool_result_for_llm,truncate_output,)from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprints@@ -3682,10 +3692,299 @@ def _record_diagnosis_repair_contract(return contract+def _packet_string(value: Any) -> str | None:+ return value if isinstance(value, str) and value else None+++def _packet_string_list(value: Any) -> list[str]:+ return [item for item in value if isinstance(item, str) and item] if isinstance(value, list) else []+++def _packet_workflow_readback(copilot_ctx: CopilotContext) -> tuple[str | None, str]:+ accepted = copilot_ctx.last_workflow_yaml+ if isinstance(accepted, str) and accepted.strip():+ return accepted, "accepted_write_readback"+ persisted = copilot_ctx.persisted_workflow_yaml+ if isinstance(persisted, str) and persisted.strip():+ return persisted, "turn_start_persisted_readback"+ return None, "unavailable"+++def _packet_page_state(data: Mapping[str, Any], omission_notices: list[str]) -> BuildTestPacketPageState | None:+ repair_context = data.get("authoring_repair_context")+ run_id = _packet_string(data.get("workflow_run_id"))+ if run_id is None:+ if isinstance(repair_context, Mapping):+ omission_notices.append("failure.page_state omitted: page evidence has no recorded workflow run identity.")+ return None+ repair_run_id = (+ _packet_string(repair_context.get("workflow_run_id")) if isinstance(repair_context, Mapping) else None+ )+ if isinstance(repair_context, Mapping) and repair_run_id != run_id:+ omission_notices.append("failure.page_state omitted repair-context fields belonging to another or unknown run.")+ repair = repair_context if isinstance(repair_context, Mapping) and repair_run_id == run_id else {}+ current_url = _packet_string(repair.get("current_url")) or _packet_string(data.get("current_url"))+ title = _packet_string(repair.get("current_title")) or _packet_string(data.get("page_title"))+ page_state = BuildTestPacketPageState(+ current_origin=_packet_string(repair.get("current_origin")),+ current_url=current_url,+ title=title,+ evidence_source=_packet_string(repair.get("page_evidence_source")),+ observed_after_workflow_run=repair.get("observed_after_workflow_run") is True,+ form_summaries=_packet_string_list(repair.get("page_form_summaries")),+ result_summaries=_packet_string_list(repair.get("page_result_summaries")),+ action_summaries=_packet_string_list(repair.get("page_action_summaries")),+ challenge_summaries=_packet_string_list(repair.get("page_challenge_summaries")),+ obstruction_summaries=_packet_string_list(repair.get("page_obstruction_summaries")),+ )+ return (+ page_state+ if any(+ (+ page_state.current_origin,+ page_state.current_url,+ page_state.title,+ page_state.evidence_source,+ page_state.form_summaries,+ page_state.result_summaries,+ page_state.action_summaries,+ page_state.challenge_summaries,+ page_state.obstruction_summaries,+ )+ )+ else None+ )+++def _packet_registered_outputs(+ data: Mapping[str, Any], run_id: str | None, omission_notices: list[str]+) -> list[BuildTestPacketRegisteredOutput]:+ raw_outputs = data.get("registered_output_parameter_values")+ if not isinstance(raw_outputs, list):+ return []+ outputs: list[BuildTestPacketRegisteredOutput] = []+ malformed = 0+ other_run = 0+ for raw_output in raw_outputs:+ if not isinstance(raw_output, Mapping):+ malformed += 1+ continue+ output_run_id = _packet_string(raw_output.get("workflow_run_id"))+ if run_id is None or output_run_id != run_id:+ other_run += 1+ continue+ try:+ outputs.append(+ BuildTestPacketRegisteredOutput(+ workflow_run_id=_packet_string(raw_output.get("workflow_run_id")),+ output_parameter_id=_packet_string(raw_output.get("output_parameter_id")),+ output_parameter_key=_packet_string(raw_output.get("output_parameter_key")),+ block_label=_packet_string(raw_output.get("block_label")),+ block_type=_packet_string(raw_output.get("block_type")),+ value=raw_output.get("value"),+ )+ )+ except ValueError:+ malformed += 1+ if malformed:+ omission_notices.append(f"registered_outputs omitted {malformed} non-JSON or malformed item(s).")+ if other_run:+ omission_notices.append(f"registered_outputs omitted {other_run} item(s) belonging to another or unknown run.")+ return outputs+++def _packet_downloads(+ copilot_ctx: CopilotContext,+ data: Mapping[str, Any],+ omission_notices: list[str],+) -> list[BuildTestPacketDownload]:+ run_id = _packet_string(data.get("workflow_run_id"))+ if run_id is None:+ return []+ raw_blocks = data.get("blocks")+ blocks = raw_blocks if isinstance(raw_blocks, list) else []+ outputs_by_label = {+ label: extracted+ for block in blocks+ if isinstance(block, Mapping)+ and isinstance((label := block.get("label")), str)+ and isinstance((extracted := block.get("extracted_data")), Mapping)+ }+ artifact_ids = _collect_downloaded_artifact_ids(outputs_by_label)+ evidence = copilot_ctx.registered_artifact_evidence+ names_by_id: dict[str, str | None] = {}+ if isinstance(evidence, RegisteredArtifactEvidence):+ if evidence.workflow_run_id == run_id:+ names_by_id = {entry.artifact_id: entry.file_name for entry in evidence.entries}+ elif artifact_ids:+ omission_notices.append("downloads omitted file names from artifact evidence belonging to another run.")+ return [+ BuildTestPacketDownload(artifact_id=artifact_id, file_name=names_by_id.get(artifact_id))+ for artifact_id in artifact_ids+ ]+++def _packet_unfinished_items(+ copilot_ctx: CopilotContext,+ run_id: str | None,+ omission_notices: list[str],+) -> list[BuildTestPacketUnfinishedItem]:+ outcome = copilot_ctx.latest_recorded_build_test_outcome+ if run_id is None or outcome is None or outcome.workflow_run_id != run_id:+ if copilot_ctx.last_unverified_block_labels or outcome is not None:+ omission_notices.append("unfinished_items omitted: recorded unfinished evidence is not bound to this run.")+ return []+ unfinished = [+ BuildTestPacketUnfinishedItem(kind="unverified_block", label=label)+ for label in dict.fromkeys(copilot_ctx.last_unverified_block_labels)+ if isinstance(label, str) and label+ ]+ for fact in outcome.missing_requested_output_facts:+ output_path = _packet_string(fact.get("output_path"))+ reason_code = _packet_string(fact.get("reason_code"))+ if output_path is not None:+ unfinished.append(+ BuildTestPacketUnfinishedItem(+ kind="missing_requested_output",+ output_path=output_path,+ reason_code=reason_code,+ )+ )+ return unfinished+++def build_test_evidence_packet(copilot_ctx: CopilotContext, result: Mapping[str, Any]) -> BuildTestEvidencePacket:+ raw_data = result.get("data")+ data = raw_data if isinstance(raw_data, Mapping) else {}+ omission_notices: list[str] = []+ workflow_yaml, workflow_source = _packet_workflow_readback(copilot_ctx)+ if workflow_yaml is None:+ omission_notices.append(+ "canonical_workflow_yaml omitted: no accepted or turn-start persistence readback exists."+ )++ attempted_labels = _packet_string_list(data.get("requested_block_labels"))+ executed_labels = _packet_string_list(data.get("executed_block_labels"))+ run_id = _packet_string(data.get("workflow_run_id"))+ run_status = _packet_string(data.get("overall_status"))+ if run_id is None:+ omission_notices.append("run.workflow_run_id omitted: no workflow run was recorded for this result.")+ if run_status is None:+ omission_notices.append("run.status omitted: no recorded run status exists for this result.")+ if not attempted_labels:+ omission_notices.append("attempted_block_labels omitted: no block run attempt was recorded.")+ if not executed_labels:+ omission_notices.append("executed_block_labels omitted: no block execution was recorded.")++ raw_blocks = data.get("blocks")+ blocks = raw_blocks if isinstance(raw_blocks, list) else []+ failed_block = next(+ (block for block in blocks if isinstance(block, Mapping) and block.get("status") in _FAILED_BLOCK_STATUSES),+ None,+ )+ action_trace = _packet_string_list(data.get("action_trace_summary"))+ page_state = _packet_page_state(data, omission_notices)+ failure: BuildTestPacketFailure | None = None+ if failed_block is not None or result.get("ok") is False:+ failure = BuildTestPacketFailure(+ block_label=_packet_string(failed_block.get("label")) if failed_block is not None else None,+ block_status=_packet_string(failed_block.get("status")) if failed_block is not None else run_status,+ reason=(+ _packet_string(failed_block.get("failure_reason"))+ if failed_block is not None+ else _packet_string(data.get("failure_reason")) or _packet_string(result.get("error"))+ ),+ action_trace=action_trace,+ page_state=page_state,+ )+ if not action_trace:+ omission_notices.append("failure.action_trace omitted: no failed action was recorded.")+ if page_state is None:+ omission_notices.append("failure.page_state omitted: no bounded same-run page state was recorded.")+ if failure.reason is None:+ omission_notices.append("failure.reason omitted: no recorded failure reason exists.")+ else:+ omission_notices.append("failure omitted: no failed run or failed block was recorded.")++ registered_outputs = _packet_registered_outputs(data, run_id, omission_notices)+ if not registered_outputs:+ omission_notices.append("registered_outputs empty: no output parameter value was recorded.")+ downloads = _packet_downloads(copilot_ctx, data, omission_notices)+ if not downloads:+ omission_notices.append("downloads empty: no registered download artifact was recorded.")++ screenshot_present = (+ run_id is not None and isinstance(data.get("screenshot_base64"), str) and bool(data.get("screenshot_base64"))+ )+ screenshot_provenance = "data.screenshot_base64" if screenshot_present else None+ if not screenshot_present and run_id is not None:+ failed_screenshot = next(+ (+ block+ for block in blocks+ if isinstance(block, Mapping)+ and isinstance(block.get("screenshot_b64"), str)+ and bool(block.get("screenshot_b64"))+ ),+ None,+ )+ if failed_screenshot is not None:+ screenshot_present = True+ screenshot_provenance = "data.blocks[].screenshot_b64"+ if not screenshot_present:+ omission_notices.append("screenshot omitted: no final or failed-block screenshot was recorded.")++ unfinished_items = _packet_unfinished_items(copilot_ctx, run_id, omission_notices)+ if not unfinished_items:+ omission_notices.append("unfinished_items empty: recorded outcome and workflow evidence identify none.")++ return BuildTestEvidencePacket(+ workflow_permanent_id=copilot_ctx.workflow_permanent_id,+ canonical_workflow_yaml=workflow_yaml,+ canonical_workflow_source=workflow_source,+ canonical_workflow_yaml_complete=workflow_yaml is not None,+ attempted_block_labels=attempted_labels,+ executed_block_labels=executed_labels,+ run=BuildTestPacketRun(workflow_run_id=run_id, status=run_status),+ failure=failure,+ registered_outputs=registered_outputs,+ downloads=downloads,+ screenshot=BuildTestPacketScreenshot(present=screenshot_present, provenance=screenshot_provenance),+ unfinished_items=unfinished_items,+ omission_notices=omission_notices,+ )+++def finalize_build_test_result(+ copilot_ctx: CopilotContext,+ *,+ source_tool: str,+ result: dict[str, Any],+ workflow_updated: bool = False,+ diagnosis_shadow_eligible: bool = True,+) -> dict[str, Any]:+ """Finalize diagnosis state and attach the one shared factual packet."""+ if diagnosis_shadow_eligible:+ _record_diagnosis_repair_contract(+ copilot_ctx,+ source_tool=source_tool,+ result=result,+ workflow_updated=workflow_updated,+ )+ data = result.get("data")+ if not isinstance(data, dict):+ data = {}+ result["data"] = data+ data[BUILD_TEST_PACKET_KEY] = build_test_evidence_packet(copilot_ctx, result).model_dump(+ mode="json", exclude_none=True+ )+ return result++def _diagnosis_repair_tool_error(copilot_ctx: Any, source_tool: str, error: str) -> str:result = {"ok": False, "error": error}- _record_diagnosis_repair_contract(copilot_ctx, source_tool=source_tool, result=result)- return json.dumps(result)+ finalize_build_test_result(copilot_ctx, source_tool=source_tool, result=result)+ return json.dumps(sanitize_tool_result_for_llm(source_tool, result))def _run_blocks_span_data(diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 8b2bb5840..77be5c801 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -7,7 +7,7 @@ import jsonfrom datetime import datetime, timezonefrom pathlib import Pathfrom types import SimpleNamespace-from typing import Any+from typing import Any, ClassVarfrom unittest.mock import AsyncMock, MagicMockimport pytest@@ -1779,10 +1779,16 @@ class TestEditBlockAndRun:),)- assert json.loads(result) == {- "ok": False,- "error": "block_labels must include the edited block 'repair_me' so this call tests the persisted repair.",- }+ parsed = json.loads(result)+ assert parsed["ok"] is False+ assert parsed["error"] == (+ "block_labels must include the edited block 'repair_me' so this call tests the persisted repair."+ )+ assert parsed["data"]["build_test_packet"]["run"] == {}+ assert any(+ "no workflow run was recorded" in notice+ for notice in parsed["data"]["build_test_packet"]["omission_notices"]+ )@pytest.mark.asyncioasync def test_one_call_persists_the_scoped_edit_then_returns_run_debug_evidence(self, monkeypatch) -> None:@@ -1877,6 +1883,7 @@ workflow_definition:async def fake_update_workflow(payload, ctx, *, allow_missing_credentials=False):captured["persisted_yaml"] = payload["workflow_yaml"]captured["allow_missing_credentials"] = allow_missing_credentials+ ctx.last_workflow_yaml = payload["workflow_yaml"]ctx.last_update_block_count = 1return {"ok": True, "data": {"block_count": 1}}@@ -1911,18 +1918,271 @@ workflow_definition:parsed = json.loads(result)assert parsed["ok"] is True- assert parsed["data"] == {+ assert {key: value for key, value in parsed["data"].items() if key != "build_test_packet"} == {"block_count": 1,"workflow_updated": True,"skipped_run": True,"skip_reason": "workflow_credential_inputs_unbound",}+ packet = parsed["data"]["build_test_packet"]+ assert packet["canonical_workflow_yaml"] == workflow_yaml.replace("credential.username", "credential.password")+ assert packet["canonical_workflow_source"] == "accepted_write_readback"+ assert packet["run"] == {}+ assert any("no workflow run was recorded" in notice for notice in packet["omission_notices"])assert captured["allow_missing_credentials"] is Trueassert "credential.password" in str(captured["persisted_yaml"])assert ctx.last_run_skipped_unbound_credentials is Truerun_blocks.assert_not_awaited()+class TestSharedBuildTestPacket:+ _SAVED_WORKFLOW = """title: Test workflow+workflow_definition:+ blocks:+ - block_type: code+ label: read_total+ code: |+ return {"total": await page.inner_text("#final")}+"""+ _PRIOR_WORKFLOW = _SAVED_WORKFLOW.replace('"#final"', '"#old"')+ _FAILED_RUN: ClassVar[dict[str, Any]] = {+ "ok": False,+ "error": "The total was not available.",+ "data": {+ "workflow_run_id": "wr_packet_1",+ "overall_status": "failed",+ "requested_block_labels": ["read_total"],+ "executed_block_labels": ["read_total"],+ "blocks": [+ {+ "label": "read_total",+ "block_type": "CODE",+ "status": "failed",+ "failure_reason": "The total was not available.",+ "extracted_data": {"downloaded_file_artifact_ids": ["artifact_1"]},+ }+ ],+ "current_url": "https://example.test/results",+ "page_title": "Results",+ "action_trace_summary": ["NULL_ACTION failed response=missing total code_line=3"],+ "registered_output_parameter_values": [+ {+ "workflow_run_id": "wr_packet_1",+ "output_parameter_id": "output_1",+ "output_parameter_key": "total",+ "block_label": "read_total",+ "block_type": "CODE",+ "value": None,+ }+ ],+ "screenshot_base64": "frame-bytes",+ "authoring_repair_context": {+ "block_label": "read_total",+ "reason_code": "runtime_block_failure",+ "runtime_failure_reason": "The total was not available.",+ "failed_block_status": "failed",+ "workflow_run_id": "wr_packet_1",+ "current_origin": "https://example.test",+ "current_url": "https://example.test/results",+ "current_title": "Results",+ "page_evidence_source": "inspect_page_for_composition",+ "observed_after_workflow_run": True,+ "page_form_summaries": [],+ "page_result_summaries": ["result container #summary; text=Total unavailable"],+ "page_action_summaries": ["button #refresh; text=Refresh"],+ "page_challenge_summaries": [],+ "page_obstruction_summaries": [],+ },+ },+ }++ @pytest.mark.asyncio+ async def test_same_saved_workflow_and_failed_run_share_one_packet_across_three_paths(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ async def fake_update_workflow(payload, ctx, **_kwargs):+ assert payload["workflow_yaml"] == self._SAVED_WORKFLOW+ ctx.workflow_yaml = self._SAVED_WORKFLOW+ ctx.last_workflow_yaml = self._SAVED_WORKFLOW+ ctx.last_workflow = SimpleNamespace(workflow_definition={"blocks": []})+ return {"ok": True, "data": {"block_count": 1}, "_workflow": ctx.last_workflow}++ async def fake_run_blocks(_params, _ctx, **_kwargs):+ return json.loads(json.dumps(self._FAILED_RUN))++ async def fake_verify(ctx, _result, _handler_start):+ ctx.last_unverified_block_labels = ["read_total"]+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="run_blocks_and_collect_debug",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_packet_1",+ )++ def fake_record_update(ctx, result, _prior_definition=None):+ if result.get("ok"):+ ctx.last_workflow_yaml = ctx.workflow_yaml+ ctx.last_workflow = result.get("_workflow") or ctx.last_workflow+ ctx.last_update_block_count = 1++ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)+ monkeypatch.setattr(tools_module, "_update_and_run_requires_skipped_run", lambda *args: False)+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value={"blocks": []}))+ monkeypatch.setattr(tools_module, "_frontier_runtime_page_url", AsyncMock(return_value=None))+ monkeypatch.setattr(tools_module, "_plan_frontier", lambda *args: (["read_total"], {}, "read_total", "initial"))+ monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)+ monkeypatch.setattr(tools_module, "_record_workflow_update_result", fake_record_update)+ monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", fake_run_blocks)+ monkeypatch.setattr(tools_module, "_verify_and_record_run_blocks_result", fake_verify)+ monkeypatch.setattr(tools_module, "record_tool_step_result_for_ctx", lambda *args, **kwargs: None)+ monkeypatch.setattr(tools_module, "enqueue_screenshot_from_result", lambda *args, **kwargs: None)+ monkeypatch.setattr(tools_module, "_clear_pending_browser_interaction_observation", lambda *args: None)++ def context(workflow_yaml: str) -> CopilotContext:+ return _ctx(+ workflow_yaml=workflow_yaml,+ last_workflow_yaml=workflow_yaml,+ request_policy=RequestPolicy(allow_update_workflow=True, allow_run_blocks=True),+ )++ update_then_run_ctx = context(self._PRIOR_WORKFLOW)+ combined_ctx = context(self._PRIOR_WORKFLOW)+ edit_ctx = context(self._PRIOR_WORKFLOW)+ with capture_logs() as logs:+ await tools_module.update_workflow_tool.on_invoke_tool(+ SimpleNamespace(context=update_then_run_ctx, tool_name="update_workflow"),+ json.dumps({"workflow_yaml": self._SAVED_WORKFLOW}),+ )+ update_then_run = await tools_module.run_blocks_tool.on_invoke_tool(+ SimpleNamespace(context=update_then_run_ctx, tool_name="run_blocks_and_collect_debug"),+ json.dumps({"block_labels": ["read_total"], "parameters": {}}),+ )+ combined = await tools_module.update_and_run_blocks_tool.on_invoke_tool(+ SimpleNamespace(context=combined_ctx, tool_name="update_and_run_blocks"),+ json.dumps({"workflow_yaml": self._SAVED_WORKFLOW, "block_labels": ["read_total"], "parameters": {}}),+ )+ edited = await tools_module.edit_block_and_run_tool.on_invoke_tool(+ SimpleNamespace(context=edit_ctx, tool_name="edit_block_and_run"),+ json.dumps(+ {+ "label": "read_total",+ "expected_code": '"#old"',+ "replacement_code": '"#final"',+ "block_labels": ["read_total"],+ "parameters": {},+ }+ ),+ )++ packets = [json.loads(result)["data"]["build_test_packet"] for result in (update_then_run, combined, edited)]+ assert packets[0] == packets[1] == packets[2]+ packet = packets[0]+ assert packet["workflow_permanent_id"] == "wfp-1"+ assert packet["canonical_workflow_yaml"] == self._SAVED_WORKFLOW+ assert packet["attempted_block_labels"] == ["read_total"]+ assert packet["executed_block_labels"] == ["read_total"]+ assert packet["run"] == {"workflow_run_id": "wr_packet_1", "status": "failed"}+ assert packet["failure"]["block_label"] == "read_total"+ assert packet["failure"]["action_trace"] == ["NULL_ACTION failed response=missing total code_line=3"]+ assert packet["failure"]["page_state"]["result_summaries"] == [+ "result container #summary; text=Total unavailable"+ ]+ assert packet["registered_outputs"][0]["output_parameter_key"] == "total"+ assert packet["downloads"] == [{"artifact_id": "artifact_1"}]+ assert packet["screenshot"] == {"present": True, "provenance": "data.screenshot_base64"}+ assert packet["unfinished_items"] == [{"kind": "unverified_block", "label": "read_total"}]+ assert sum(log.get("event") == "copilot diagnosis repair contract shadow" for log in logs) == 3++ def test_packet_falls_back_to_turn_start_persistence_readback_not_submitted_yaml(self) -> None:+ ctx = _ctx(+ workflow_yaml="title: submitted but rejected",+ persisted_workflow_yaml="title: persisted canonical workflow",+ last_workflow_yaml=None,+ )+ result: dict[str, Any] = {"ok": False, "error": "workflow validation failed"}++ run_execution_module.finalize_build_test_result(+ ctx,+ source_tool="update_workflow",+ result=result,+ )+ packet = result["data"]["build_test_packet"]++ assert packet["canonical_workflow_yaml"] == "title: persisted canonical workflow"+ assert packet["canonical_workflow_source"] == "turn_start_persisted_readback"+ assert "submitted but rejected" not in str(packet)++ def test_packet_omits_evidence_that_is_not_bound_to_the_current_run(self) -> None:+ ctx = _ctx(+ persisted_workflow_yaml="title: persisted canonical workflow",+ last_unverified_block_labels=["stale_block"],+ latest_recorded_build_test_outcome=RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="run_blocks_and_collect_debug",+ verdict="repairable_failure",+ reason_code="runtime_block_failure",+ workflow_run_id="wr_old",+ ),+ )+ result: dict[str, Any] = {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_new",+ "overall_status": "failed",+ "current_url": "https://example.test/current",+ "blocks": [{"label": "read_total", "status": "failed"}],+ "registered_output_parameter_values": [+ {"workflow_run_id": "wr_old", "output_parameter_key": "total", "value": "stale"}+ ],+ "authoring_repair_context": {+ "workflow_run_id": "wr_old",+ "page_result_summaries": ["stale page result"],+ },+ },+ }++ run_execution_module.finalize_build_test_result(+ ctx,+ source_tool="run_blocks_and_collect_debug",+ result=result,+ )+ packet = result["data"]["build_test_packet"]++ assert packet["registered_outputs"] == []+ assert packet["unfinished_items"] == []+ assert packet["failure"]["page_state"]["current_url"] == "https://example.test/current"+ assert packet["failure"]["page_state"]["result_summaries"] == []+ assert any("another or unknown run" in notice for notice in packet["omission_notices"])++ def test_packet_omits_failed_block_screenshot_without_a_recorded_run(self) -> None:+ ctx = _ctx(persisted_workflow_yaml="title: persisted canonical workflow")+ result: dict[str, Any] = {+ "ok": False,+ "data": {+ "overall_status": "failed",+ "blocks": [+ {+ "label": "read_total",+ "status": "failed",+ "screenshot_b64": "unbound-screenshot-bytes",+ }+ ],+ },+ }++ run_execution_module.finalize_build_test_result(+ ctx,+ source_tool="run_blocks_and_collect_debug",+ result=result,+ )+ packet = result["data"]["build_test_packet"]++ assert packet["run"] == {"status": "failed"}+ assert packet["screenshot"] == {"present": False}+ assert any("screenshot omitted" in notice for notice in packet["omission_notices"])++class TestTranslateToAgentResultGating:"""Covers the three SKY-9143 invariants that live in _translate_to_agent_result."""diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.pyindex bc1182432..82cd2af38 100644--- a/tests/unit/test_copilot_output_utils.py+++ b/tests/unit/test_copilot_output_utils.py@@ -2,6 +2,7 @@from __future__ import annotations+import jsonfrom unittest.mock import MagicMockimport pytest@@ -142,6 +143,171 @@ def test_sanitize_run_blocks_debug_strips_block_screenshot_b64() -> None:assert sanitized["data"]["blocks"][0]["final_url"] == "https://portal.example.com/mfa"+def test_sanitize_build_test_packet_bounds_facts_and_preserves_screenshot_provenance() -> None:+ packet = {+ "contract_version": "build_test_evidence_packet_v1",+ "canonical_workflow_yaml": "w" * 30_010,+ "canonical_workflow_source": "accepted_write_readback",+ "canonical_workflow_yaml_complete": True,+ "attempted_block_labels": [f"attempt_{index}" for index in range(30)],+ "executed_block_labels": [f"executed_{index}" for index in range(30)],+ "run": {"workflow_run_id": "wr_1", "status": "failed"},+ "failure": {+ "block_label": "read_total",+ "block_status": "failed",+ "reason": "missing total",+ "action_trace": ["action " + "x" * 500 for _ in range(8)],+ "page_state": {+ "observed_after_workflow_run": True,+ "form_summaries": ["form " + "x" * 500 for _ in range(10)],+ "result_summaries": [],+ "action_summaries": [],+ "challenge_summaries": [],+ "obstruction_summaries": [],+ },+ },+ "registered_outputs": [+ {"output_parameter_key": f"output_{index}", "value": "v" * 1_300} for index in range(13)+ ],+ "downloads": [{"artifact_id": f"artifact_{index}"} for index in range(13)],+ "screenshot": {"present": True, "provenance": "data.screenshot_base64"},+ "unfinished_items": [{"kind": "unverified_block", "label": f"block_{index}"} for index in range(25)],+ "omission_notices": [],+ }++ sanitized = sanitize_tool_result_for_llm(+ "run_blocks_and_collect_debug",+ {"ok": False, "data": {"screenshot_base64": "raw-frame-bytes", "build_test_packet": packet}},+ )+ projected = sanitized["data"]["build_test_packet"]++ assert len(projected["canonical_workflow_yaml"]) <= 30_000+ assert projected["canonical_workflow_yaml_complete"] is False+ assert len(projected["attempted_block_labels"]) == 24+ assert len(projected["failure"]["action_trace"]) == 6+ assert len(projected["failure"]["page_state"]["form_summaries"]) == 8+ assert len(projected["registered_outputs"]) == 12+ assert projected["registered_outputs"][0]["value_complete"] is False+ assert len(projected["downloads"]) == 12+ assert len(projected["unfinished_items"]) == 24+ assert projected["screenshot"] == {"present": True, "provenance": "data.screenshot_base64"}+ assert "raw-frame-bytes" not in str(projected)+ assert any("shortened" in notice for notice in projected["omission_notices"])+ assert len(json.dumps(projected)) <= 47_000+ assert next(iter(sanitized)) == "data"+ assert next(iter(sanitized["data"])) == "build_test_packet"+++def test_sanitize_build_test_packet_exercises_aggregate_compaction() -> None:+ long_identifier = "i" * 200+ long_summary = "s" * 500+ packet = {+ "contract_version": "build_test_evidence_packet_v1",+ "canonical_workflow_yaml": "w" * 30_010,+ "canonical_workflow_source": "accepted_write_readback",+ "canonical_workflow_yaml_complete": True,+ "attempted_block_labels": [long_identifier for _ in range(30)],+ "executed_block_labels": [long_identifier for _ in range(30)],+ "run": {"workflow_run_id": long_identifier, "status": "failed"},+ "failure": {+ "block_label": long_identifier,+ "block_status": long_identifier,+ "reason": "r" * 1_300,+ "action_trace": [long_summary for _ in range(8)],+ "page_state": {+ "current_origin": "https://" + "o" * 2_000,+ "current_url": "https://" + "u" * 2_000,+ "title": long_identifier,+ "evidence_source": long_identifier,+ "observed_after_workflow_run": True,+ "form_summaries": [long_summary for _ in range(10)],+ "result_summaries": [long_summary for _ in range(10)],+ "action_summaries": [long_summary for _ in range(10)],+ "challenge_summaries": [long_summary for _ in range(10)],+ "obstruction_summaries": [long_summary for _ in range(10)],+ },+ },+ "registered_outputs": [+ {+ "workflow_run_id": long_identifier,+ "output_parameter_id": long_identifier,+ "output_parameter_key": long_identifier,+ "block_label": long_identifier,+ "block_type": long_identifier,+ "value": "v" * 1_300,+ }+ for _ in range(13)+ ],+ "downloads": [{"artifact_id": long_identifier, "file_name": long_identifier} for _ in range(13)],+ "screenshot": {"present": False},+ "unfinished_items": [+ {+ "kind": "unverified_block",+ "label": long_identifier,+ "output_path": long_identifier,+ "reason_code": long_identifier,+ }+ for _ in range(25)+ ],+ "omission_notices": [],+ }++ sanitized = sanitize_tool_result_for_llm(+ "run_blocks_and_collect_debug",+ {"ok": False, "data": {"build_test_packet": packet}},+ )+ projected = sanitized["data"]["build_test_packet"]++ assert "canonical_workflow_yaml" not in projected+ assert projected["canonical_workflow_yaml_complete"] is False+ assert len(projected["attempted_block_labels"]) == 12+ assert len(projected["executed_block_labels"]) == 12+ assert len(projected["failure"]["action_trace"]) == 2+ assert len(projected["failure"]["page_state"]["form_summaries"]) == 2+ assert len(projected["registered_outputs"]) == 6+ assert projected["registered_outputs"][0]["value_complete"] is False+ assert len(projected["downloads"]) == 6+ assert len(projected["unfinished_items"]) == 12+ assert any("repeated packet facts shortened further" in notice for notice in projected["omission_notices"])+ assert len(json.dumps(projected)) <= 47_000+++def test_sanitize_build_test_packet_projection_failure_keeps_tool_result_available(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ packet = {+ "contract_version": "build_test_evidence_packet_v1",+ "canonical_workflow_source": "accepted_write_readback",+ "run": {"workflow_run_id": "wr_1", "status": "failed"},+ "screenshot": {"present": False},+ }++ def fail_projection(*_args: object, **_kwargs: object) -> None:+ raise RuntimeError("projection failed")++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.output_utils.project_build_test_packet_for_llm",+ fail_projection,+ )++ sanitized = sanitize_tool_result_for_llm(+ "run_blocks_and_collect_debug",+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_1",+ "overall_status": "failed",+ "build_test_packet": packet,+ },+ },+ )++ assert sanitized["data"]["workflow_run_id"] == "wr_1"+ assert sanitized["data"]["overall_status"] == "failed"+ assert "build_test_packet" not in sanitized["data"]+ assert sanitized["data"]["build_test_packet_omitted"] == "The internal packet projection failed."++def test_sanitize_run_blocks_debug_preserves_post_run_page_evidence() -> None:evidence = {"workflow_run_id": "wr_123",diff --git a/tests/unit/test_copilot_session_factory.py b/tests/unit/test_copilot_session_factory.pyindex cf87c30b8..6a9dffcbf 100644--- a/tests/unit/test_copilot_session_factory.py+++ b/tests/unit/test_copilot_session_factory.py@@ -341,6 +341,52 @@ class TestModelInputCapture:assert len(sorted(tmp_path.glob("call-*.json"))) == 2+def test_build_test_packet_survives_recent_tool_output_head_compaction() -> None:+ from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llm+ from skyvern.forge.sdk.copilot.session_factory import copilot_call_model_input_filter++ packet = {+ "contract_version": "build_test_evidence_packet_v1",+ "workflow_permanent_id": "wfp_compaction",+ "canonical_workflow_yaml": "title: accepted workflow",+ "canonical_workflow_source": "accepted_write_readback",+ "canonical_workflow_yaml_complete": True,+ "attempted_block_labels": ["read_total"],+ "executed_block_labels": ["read_total"],+ "run": {"workflow_run_id": "wr_compaction", "status": "failed"},+ "failure": {"block_label": "read_total", "block_status": "failed", "reason": "missing total"},+ "registered_outputs": [],+ "downloads": [],+ "screenshot": {"present": True, "provenance": "data.screenshot_base64"},+ "unfinished_items": [{"kind": "unverified_block", "label": "read_total"}],+ "omission_notices": [],+ }+ sanitized = sanitize_tool_result_for_llm(+ "run_blocks_and_collect_debug",+ {+ "ok": False,+ "data": {+ "legacy_blob": "x" * 60_000,+ "screenshot_base64": "raw-frame-bytes",+ "build_test_packet": packet,+ },+ },+ )+ output = json.dumps(sanitized)+ filtered = copilot_call_model_input_filter(+ _mk_input_data([{"type": "function_call_output", "call_id": "run-1", "output": output}])+ )+ compacted_output = filtered.input[0]["output"]+ projected_packet = json.dumps(sanitized["data"]["build_test_packet"])++ assert len(output) > 50_000+ assert projected_packet in compacted_output+ assert compacted_output.index("build_test_packet") < compacted_output.index("legacy_blob")+ assert "wfp_compaction" in compacted_output+ assert "wr_compaction" in compacted_output+ assert "raw-frame-bytes" not in compacted_output++def test_model_input_pipeline_has_no_generated_offer_special_case() -> None:from skyvern.forge.sdk.copilot import enforcement