Skip to content

Automate browser based workflows with AI

SKY-14839 Return complete build-test packets to Copilot (#8280)

Andrew Neilsonbea7dec10eb3
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index f363d8c57..5403423d5 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -11,7 +11,7 @@ from urllib.parse import urlsplit
import structlog
import yaml
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, JsonValue
from skyvern.forge.sdk.copilot.challenge_evidence import (
carrier_backed_anti_bot_categories,
@@ -67,6 +67,8 @@ BuildTestOutcomeReasonCode = Literal[
_TERMINAL_CHALLENGE_REASON_CODES: frozenset[BuildTestOutcomeReasonCode] = frozenset({"terminal_challenge_blocker"})
PostRunPagePathKind = Literal["login", "challenge", "incomplete_navigation", "non_page_outcome"]
PostRunPagePathTargetKind = Literal["form_submit", "navigation", "clickable", "challenge"]
+BuildTestPacketWorkflowSource = Literal["accepted_write_readback", "turn_start_persisted_readback", "unavailable"]
+BuildTestPacketUnfinishedKind = Literal["unverified_block", "missing_requested_output"]
_STRUCTURAL_KEY_VERSION = "recorded_build_test_outcome:v1"
_AUTHORED_STRUCTURE_VERSION = "recorded_build_test_outcome_authored_structure:v1"
@@ -105,6 +107,92 @@ class PostRunPagePathFailure(BaseModel):
return self.kind != "non_page_outcome" and bool(self.continuation_targets)
+class BuildTestPacketRun(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ workflow_run_id: str | None = None
+ status: str | None = None
+
+
+class BuildTestPacketPageState(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ current_origin: str | None = None
+ current_url: str | None = None
+ title: str | None = None
+ evidence_source: str | None = None
+ observed_after_workflow_run: bool = False
+ form_summaries: list[str] = Field(default_factory=list)
+ result_summaries: list[str] = Field(default_factory=list)
+ action_summaries: list[str] = Field(default_factory=list)
+ challenge_summaries: list[str] = Field(default_factory=list)
+ obstruction_summaries: list[str] = Field(default_factory=list)
+
+
+class BuildTestPacketFailure(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ block_label: str | None = None
+ block_status: str | None = None
+ reason: str | None = None
+ action_trace: list[str] = Field(default_factory=list)
+ page_state: BuildTestPacketPageState | None = None
+
+
+class BuildTestPacketRegisteredOutput(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ workflow_run_id: str | None = None
+ output_parameter_id: str | None = None
+ output_parameter_key: str | None = None
+ block_label: str | None = None
+ block_type: str | None = None
+ value: JsonValue = None
+ value_complete: bool = True
+
+
+class BuildTestPacketDownload(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ artifact_id: str
+ file_name: str | None = None
+
+
+class BuildTestPacketScreenshot(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ present: bool
+ provenance: str | None = None
+
+
+class BuildTestPacketUnfinishedItem(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ kind: BuildTestPacketUnfinishedKind
+ label: str | None = None
+ output_path: str | None = None
+ reason_code: str | None = None
+
+
+class BuildTestEvidencePacket(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ contract_version: Literal["build_test_evidence_packet_v1"] = "build_test_evidence_packet_v1"
+ workflow_permanent_id: str | None = None
+ canonical_workflow_yaml: str | None = None
+ canonical_workflow_source: BuildTestPacketWorkflowSource
+ canonical_workflow_yaml_complete: bool = True
+ attempted_block_labels: list[str] = Field(default_factory=list)
+ executed_block_labels: list[str] = Field(default_factory=list)
+ run: BuildTestPacketRun
+ failure: BuildTestPacketFailure | None = None
+ registered_outputs: list[BuildTestPacketRegisteredOutput] = Field(default_factory=list)
+ downloads: list[BuildTestPacketDownload] = Field(default_factory=list)
+ screenshot: BuildTestPacketScreenshot
+ unfinished_items: list[BuildTestPacketUnfinishedItem] = Field(default_factory=list)
+ omission_notices: list[str] = Field(default_factory=list)
+
+
class RecordedBuildTestOutcome(BaseModel):
model_config = ConfigDict(extra="forbid")
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 9aea022da..77166046e 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -10,6 +10,8 @@ import unicodedata
from collections.abc import Callable, Iterable, Iterator
from typing import TYPE_CHECKING, Any
+import structlog
+
from skyvern.forge.sdk.agents.context import sanitize_agent_tool_result_for_llm as sanitize_generic_tool_result_for_llm
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, assert_clean_user_facing_text
from skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES
@@ -21,8 +23,27 @@ from skyvern.schemas.workflows import BlockType
if TYPE_CHECKING:
from agents.result import RunResultStreaming
+ from skyvern.forge.sdk.copilot.build_test_outcome import BuildTestEvidencePacket, BuildTestPacketPageState
+
+LOG = structlog.get_logger()
+
_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY = "_copilot_internal_run_cancelled_by_watchdog"
_BASE64_IMAGE_OMITTED_MESSAGE = "[base64 image omitted — screenshot was taken successfully]"
+BUILD_TEST_PACKET_KEY = "build_test_packet"
+
+_BUILD_TEST_PACKET_MAX_CHARS = 47_000
+_BUILD_TEST_WORKFLOW_MAX_CHARS = 30_000
+_BUILD_TEST_IDENTIFIER_MAX_CHARS = 160
+_BUILD_TEST_FAILURE_REASON_MAX_CHARS = 1_200
+_BUILD_TEST_URL_MAX_CHARS = 1_600
+_BUILD_TEST_LABEL_MAX_ITEMS = 24
+_BUILD_TEST_OUTPUT_MAX_ITEMS = 12
+_BUILD_TEST_OUTPUT_VALUE_MAX_CHARS = 800
+_BUILD_TEST_DOWNLOAD_MAX_ITEMS = 12
+_BUILD_TEST_UNFINISHED_MAX_ITEMS = 24
+_BUILD_TEST_ACTION_TRACE_MAX_ITEMS = 6
+_BUILD_TEST_PAGE_SUMMARY_MAX_ITEMS = 8
+_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS = 300
_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
_JPEG_PREFIX = b"\xff\xd8\xff"
@@ -247,6 +268,430 @@ def _summarize_extracted_data(extracted: Any) -> str:
return "Extracted data present."
+def _append_omission(notices: list[str], notice: str) -> None:
+ if notice not in notices:
+ notices.append(notice)
+
+
+def _bounded_packet_string(
+ value: str | None,
+ *,
+ field_name: str,
+ max_chars: int,
+ notices: list[str],
+) -> str | None:
+ if value is None or len(value) <= max_chars:
+ return value
+ _append_omission(notices, f"{field_name} shortened at {max_chars} characters.")
+ return value[: max_chars - 3] + "..."
+
+
+def _bounded_packet_strings(
+ values: list[str],
+ *,
+ field_name: str,
+ max_items: int,
+ max_chars: int | None,
+ notices: list[str],
+) -> list[str]:
+ bounded = values[:max_items]
+ if len(values) > max_items:
+ _append_omission(notices, f"{field_name} shortened: {len(values) - max_items} item(s) omitted.")
+ if max_chars is None:
+ return bounded
+ rendered: list[str] = []
+ shortened = 0
+ for value in bounded:
+ if len(value) <= max_chars:
+ rendered.append(value)
+ continue
+ rendered.append(value[: max_chars - 3] + "...")
+ shortened += 1
+ if shortened:
+ _append_omission(notices, f"{field_name} shortened: {shortened} text value(s) clipped.")
+ return rendered
+
+
+def _bounded_packet_page_state(
+ page_state: BuildTestPacketPageState | None, notices: list[str]
+) -> BuildTestPacketPageState | None:
+ if page_state is None:
+ return None
+ updates = {
+ "current_origin": _bounded_packet_string(
+ page_state.current_origin,
+ field_name="failure.page_state.current_origin",
+ max_chars=_BUILD_TEST_URL_MAX_CHARS,
+ notices=notices,
+ ),
+ "current_url": _bounded_packet_string(
+ page_state.current_url,
+ field_name="failure.page_state.current_url",
+ max_chars=_BUILD_TEST_URL_MAX_CHARS,
+ notices=notices,
+ ),
+ "title": _bounded_packet_string(
+ page_state.title,
+ field_name="failure.page_state.title",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "evidence_source": _bounded_packet_string(
+ page_state.evidence_source,
+ field_name="failure.page_state.evidence_source",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "form_summaries": _bounded_packet_strings(
+ page_state.form_summaries,
+ field_name="failure.page_state.form_summaries",
+ max_items=_BUILD_TEST_PAGE_SUMMARY_MAX_ITEMS,
+ max_chars=_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS,
+ notices=notices,
+ ),
+ "result_summaries": _bounded_packet_strings(
+ page_state.result_summaries,
+ field_name="failure.page_state.result_summaries",
+ max_items=_BUILD_TEST_PAGE_SUMMARY_MAX_ITEMS,
+ max_chars=_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS,
+ notices=notices,
+ ),
+ "action_summaries": _bounded_packet_strings(
+ page_state.action_summaries,
+ field_name="failure.page_state.action_summaries",
+ max_items=_BUILD_TEST_PAGE_SUMMARY_MAX_ITEMS,
+ max_chars=_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS,
+ notices=notices,
+ ),
+ "challenge_summaries": _bounded_packet_strings(
+ page_state.challenge_summaries,
+ field_name="failure.page_state.challenge_summaries",
+ max_items=_BUILD_TEST_PAGE_SUMMARY_MAX_ITEMS,
+ max_chars=_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS,
+ notices=notices,
+ ),
+ "obstruction_summaries": _bounded_packet_strings(
+ page_state.obstruction_summaries,
+ field_name="failure.page_state.obstruction_summaries",
+ max_items=_BUILD_TEST_PAGE_SUMMARY_MAX_ITEMS,
+ max_chars=_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS,
+ notices=notices,
+ ),
+ }
+ return page_state.model_copy(update=updates)
+
+
+def _compact_packet_for_aggregate_limit(
+ packet: BuildTestEvidencePacket,
+ notices: list[str],
+) -> BuildTestEvidencePacket:
+ from skyvern.forge.sdk.copilot.build_test_outcome import BuildTestPacketRegisteredOutput
+
+ _append_omission(
+ notices,
+ f"repeated packet facts shortened further to keep the packet under {_BUILD_TEST_PACKET_MAX_CHARS} characters.",
+ )
+ outputs: list[BuildTestPacketRegisteredOutput] = []
+ for output in packet.registered_outputs[:6]:
+ rendered = json.dumps(output.value, ensure_ascii=False, separators=(",", ":"))
+ outputs.append(
+ output.model_copy(
+ update={
+ "value": rendered[:197] + "..." if len(rendered) > 200 else output.value,
+ "value_complete": output.value_complete and len(rendered) <= 200,
+ }
+ )
+ )
+ failure = packet.failure
+ if failure is not None:
+ page_state = failure.page_state
+ if page_state is not None:
+
+ def compact_summaries(values: list[str]) -> list[str]:
+ return [value[:117] + "..." if len(value) > 120 else value for value in values[:2]]
+
+ page_state = page_state.model_copy(
+ update={
+ "form_summaries": compact_summaries(page_state.form_summaries),
+ "result_summaries": compact_summaries(page_state.result_summaries),
+ "action_summaries": compact_summaries(page_state.action_summaries),
+ "challenge_summaries": compact_summaries(page_state.challenge_summaries),
+ "obstruction_summaries": compact_summaries(page_state.obstruction_summaries),
+ }
+ )
+ failure = failure.model_copy(
+ update={
+ "action_trace": [
+ value[:117] + "..." if len(value) > 120 else value for value in failure.action_trace[:2]
+ ],
+ "page_state": page_state,
+ }
+ )
+ return packet.model_copy(
+ update={
+ "canonical_workflow_yaml": None,
+ "canonical_workflow_yaml_complete": False,
+ "attempted_block_labels": packet.attempted_block_labels[:12],
+ "executed_block_labels": packet.executed_block_labels[:12],
+ "failure": failure,
+ "registered_outputs": outputs,
+ "downloads": packet.downloads[:6],
+ "unfinished_items": packet.unfinished_items[:12],
+ "omission_notices": notices,
+ }
+ )
+
+
+def project_build_test_packet_for_llm(packet: BuildTestEvidencePacket) -> BuildTestEvidencePacket:
+ """Return the one bounded model projection of a factual build-test packet."""
+ from skyvern.forge.sdk.copilot.build_test_outcome import BuildTestPacketRegisteredOutput
+
+ notices = list(packet.omission_notices)
+ workflow_yaml = packet.canonical_workflow_yaml
+ workflow_complete = packet.canonical_workflow_yaml_complete
+ if workflow_yaml is not None and len(workflow_yaml) > _BUILD_TEST_WORKFLOW_MAX_CHARS:
+ workflow_yaml = workflow_yaml[: _BUILD_TEST_WORKFLOW_MAX_CHARS - 3] + "..."
+ workflow_complete = False
+ _append_omission(
+ notices,
+ "canonical_workflow_yaml shortened at 30000 characters; "
+ "use the persisted workflow readback for full bytes.",
+ )
+
+ attempted = _bounded_packet_strings(
+ packet.attempted_block_labels,
+ field_name="attempted_block_labels",
+ max_items=_BUILD_TEST_LABEL_MAX_ITEMS,
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ )
+ executed = _bounded_packet_strings(
+ packet.executed_block_labels,
+ field_name="executed_block_labels",
+ max_items=_BUILD_TEST_LABEL_MAX_ITEMS,
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ )
+
+ registered_outputs: list[BuildTestPacketRegisteredOutput] = []
+ for output in packet.registered_outputs[:_BUILD_TEST_OUTPUT_MAX_ITEMS]:
+ rendered_value = json.dumps(output.value, ensure_ascii=False, separators=(",", ":"))
+ value = output.value
+ value_complete = output.value_complete
+ if len(rendered_value) > _BUILD_TEST_OUTPUT_VALUE_MAX_CHARS:
+ value = rendered_value[: _BUILD_TEST_OUTPUT_VALUE_MAX_CHARS - 3] + "..."
+ value_complete = False
+ _append_omission(
+ notices,
+ "registered output "
+ f"{output.output_parameter_key or output.output_parameter_id or '(unnamed)'} shortened.",
+ )
+ registered_outputs.append(
+ output.model_copy(
+ update={
+ "workflow_run_id": _bounded_packet_string(
+ output.workflow_run_id,
+ field_name="registered_outputs[].workflow_run_id",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "output_parameter_id": _bounded_packet_string(
+ output.output_parameter_id,
+ field_name="registered_outputs[].output_parameter_id",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "output_parameter_key": _bounded_packet_string(
+ output.output_parameter_key,
+ field_name="registered_outputs[].output_parameter_key",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "block_label": _bounded_packet_string(
+ output.block_label,
+ field_name="registered_outputs[].block_label",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "block_type": _bounded_packet_string(
+ output.block_type,
+ field_name="registered_outputs[].block_type",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "value": value,
+ "value_complete": value_complete,
+ }
+ )
+ )
+ if len(packet.registered_outputs) > _BUILD_TEST_OUTPUT_MAX_ITEMS:
+ _append_omission(
+ notices,
+ "registered_outputs shortened: "
+ f"{len(packet.registered_outputs) - _BUILD_TEST_OUTPUT_MAX_ITEMS} item(s) omitted.",
+ )
+
+ failure = packet.failure
+ if failure is not None:
+ action_trace = _bounded_packet_strings(
+ failure.action_trace,
+ field_name="failure.action_trace",
+ max_items=_BUILD_TEST_ACTION_TRACE_MAX_ITEMS,
+ max_chars=_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS,
+ notices=notices,
+ )
+ failure = failure.model_copy(
+ update={
+ "block_label": _bounded_packet_string(
+ failure.block_label,
+ field_name="failure.block_label",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "block_status": _bounded_packet_string(
+ failure.block_status,
+ field_name="failure.block_status",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "reason": _bounded_packet_string(
+ failure.reason,
+ field_name="failure.reason",
+ max_chars=_BUILD_TEST_FAILURE_REASON_MAX_CHARS,
+ notices=notices,
+ ),
+ "action_trace": action_trace,
+ "page_state": _bounded_packet_page_state(failure.page_state, notices),
+ }
+ )
+
+ downloads = [
+ download.model_copy(
+ update={
+ "artifact_id": _bounded_packet_string(
+ download.artifact_id,
+ field_name="downloads[].artifact_id",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "file_name": _bounded_packet_string(
+ download.file_name,
+ field_name="downloads[].file_name",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ }
+ )
+ for download in packet.downloads[:_BUILD_TEST_DOWNLOAD_MAX_ITEMS]
+ ]
+ if len(packet.downloads) > _BUILD_TEST_DOWNLOAD_MAX_ITEMS:
+ _append_omission(
+ notices,
+ f"downloads shortened: {len(packet.downloads) - _BUILD_TEST_DOWNLOAD_MAX_ITEMS} item(s) omitted.",
+ )
+ unfinished = [
+ item.model_copy(
+ update={
+ "label": _bounded_packet_string(
+ item.label,
+ field_name="unfinished_items[].label",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "output_path": _bounded_packet_string(
+ item.output_path,
+ field_name="unfinished_items[].output_path",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "reason_code": _bounded_packet_string(
+ item.reason_code,
+ field_name="unfinished_items[].reason_code",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ }
+ )
+ for item in packet.unfinished_items[:_BUILD_TEST_UNFINISHED_MAX_ITEMS]
+ ]
+ if len(packet.unfinished_items) > _BUILD_TEST_UNFINISHED_MAX_ITEMS:
+ _append_omission(
+ notices,
+ "unfinished_items shortened: "
+ f"{len(packet.unfinished_items) - _BUILD_TEST_UNFINISHED_MAX_ITEMS} item(s) omitted.",
+ )
+
+ projected = packet.model_copy(
+ update={
+ "workflow_permanent_id": _bounded_packet_string(
+ packet.workflow_permanent_id,
+ field_name="workflow_permanent_id",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "canonical_workflow_yaml": workflow_yaml,
+ "canonical_workflow_yaml_complete": workflow_complete,
+ "attempted_block_labels": attempted,
+ "executed_block_labels": executed,
+ "failure": failure,
+ "registered_outputs": registered_outputs,
+ "downloads": downloads,
+ "unfinished_items": unfinished,
+ "omission_notices": notices,
+ }
+ )
+ run = projected.run.model_copy(
+ update={
+ "workflow_run_id": _bounded_packet_string(
+ projected.run.workflow_run_id,
+ field_name="run.workflow_run_id",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ "status": _bounded_packet_string(
+ projected.run.status,
+ field_name="run.status",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
+ }
+ )
+ screenshot = projected.screenshot.model_copy(
+ update={
+ "provenance": _bounded_packet_string(
+ projected.screenshot.provenance,
+ field_name="screenshot.provenance",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ )
+ }
+ )
+ projected = projected.model_copy(update={"run": run, "screenshot": screenshot, "omission_notices": notices})
+
+ serialized = json.dumps(projected.model_dump(mode="json", exclude_none=True), ensure_ascii=False)
+ if len(serialized) > _BUILD_TEST_PACKET_MAX_CHARS and projected.canonical_workflow_yaml is not None:
+ excess = len(serialized) - _BUILD_TEST_PACKET_MAX_CHARS
+ retained_chars = max(0, len(projected.canonical_workflow_yaml) - excess - 200)
+ shortened_workflow = projected.canonical_workflow_yaml[:retained_chars]
+ if retained_chars >= 3:
+ shortened_workflow = shortened_workflow[:-3] + "..."
+ _append_omission(
+ notices,
+ f"canonical_workflow_yaml shortened further to keep the packet under {_BUILD_TEST_PACKET_MAX_CHARS} characters.",
+ )
+ projected = projected.model_copy(
+ update={
+ "canonical_workflow_yaml": shortened_workflow or None,
+ "canonical_workflow_yaml_complete": False,
+ "omission_notices": notices,
+ }
+ )
+ serialized = json.dumps(projected.model_dump(mode="json", exclude_none=True), ensure_ascii=False)
+ if len(serialized) > _BUILD_TEST_PACKET_MAX_CHARS:
+ projected = _compact_packet_for_aggregate_limit(projected, notices)
+ return projected
+
+
def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict[str, Any]:
"""Strip large/binary fields from tool results before sending to the LLM."""
sanitized = sanitize_generic_tool_result_for_llm(
@@ -267,6 +712,7 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict
data = sanitized.get("data")
if isinstance(data, dict):
data = dict(data)
+ packet_projected = False
if "schema" in data and isinstance(data["schema"], dict):
schema_str = json.dumps(data["schema"])
# 2000 chars ~= 500 LLM tokens — enough for the model to see the
@@ -303,7 +749,34 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict
else block
for block in blocks
]
- sanitized["data"] = data
+ raw_packet = data.get(BUILD_TEST_PACKET_KEY)
+ if isinstance(raw_packet, dict):
+ from skyvern.forge.sdk.copilot.build_test_outcome import BuildTestEvidencePacket
+
+ try:
+ packet = BuildTestEvidencePacket.model_validate(raw_packet)
+ except ValueError:
+ data.pop(BUILD_TEST_PACKET_KEY, None)
+ data["build_test_packet_omitted"] = "The internal packet failed typed validation."
+ else:
+ try:
+ projected_packet = project_build_test_packet_for_llm(packet).model_dump(
+ mode="json", exclude_none=True
+ )
+ except Exception:
+ LOG.exception("copilot build test packet projection failed")
+ data.pop(BUILD_TEST_PACKET_KEY, None)
+ data["build_test_packet_omitted"] = "The internal packet projection failed."
+ else:
+ data = {
+ BUILD_TEST_PACKET_KEY: projected_packet,
+ **{key: value for key, value in data.items() if key != BUILD_TEST_PACKET_KEY},
+ }
+ packet_projected = True
+ if packet_projected:
+ sanitized = {"data": data, **{key: value for key, value in sanitized.items() if key != "data"}}
+ else:
+ sanitized["data"] = data
return sanitized
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 3a69761cf..8da797cff 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -180,9 +180,7 @@ from .run_execution import (
)
from .run_execution import _progress_marker as _progress_marker
from .run_execution import _read_progress_sources as _read_progress_sources
-from .run_execution import (
- _record_diagnosis_repair_contract,
-)
+from .run_execution import _record_diagnosis_repair_contract as _record_diagnosis_repair_contract
from .run_execution import _record_run_blocks_result as _record_run_blocks_result
from .run_execution import (
_run_blocks_and_collect_debug,
@@ -190,6 +188,9 @@ from .run_execution import (
_verify_and_record_run_blocks_result,
)
from .run_execution import _watchdog_error_message as _watchdog_error_message
+from .run_execution import (
+ finalize_build_test_result,
+)
from .scouting import _MAX_SCOUTED_INTERACTIONS as _MAX_SCOUTED_INTERACTIONS
from .scouting import _capture_accessible_role_name as _capture_accessible_role_name
from .scouting import _capture_scout_pre_action as _capture_scout_pre_action
@@ -298,12 +299,13 @@ async def update_workflow_tool(
_mark_credential_deferred_draft(copilot_ctx, result)
_record_workflow_update_result(copilot_ctx, result, prior_definition)
record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
- if result.get("ok") is False:
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="update_workflow",
- result=result,
- )
+ finalize_build_test_result(
+ copilot_ctx,
+ source_tool="update_workflow",
+ result=result,
+ workflow_updated=result.get("ok") is True,
+ diagnosis_shadow_eligible=result.get("ok") is False,
+ )
sanitized = sanitize_tool_result_for_llm("update_workflow", result)
return json.dumps(sanitized)
@@ -334,8 +336,13 @@ async def _persist_block_scoped_edit(
result = await _update_workflow(params, copilot_ctx)
_record_workflow_update_result(copilot_ctx, result, prior_definition)
record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, result)
- if result.get("ok") is False:
- _record_diagnosis_repair_contract(copilot_ctx, source_tool=tool_name, result=result)
+ finalize_build_test_result(
+ copilot_ctx,
+ source_tool=tool_name,
+ result=result,
+ workflow_updated=result.get("ok") is True,
+ diagnosis_shadow_eligible=result.get("ok") is False,
+ )
return json.dumps(sanitize_tool_result_for_llm(tool_name, result))
@@ -369,7 +376,14 @@ async def edit_block_tool(
arguments = {"label": label, "fields": fields, "has_code_edit": expected_code is not None}
authority_error = _authority_tool_error(copilot_ctx, "edit_block")
if authority_error:
- return json.dumps({"ok": False, "error": authority_error})
+ result = {"ok": False, "error": authority_error}
+ finalize_build_test_result(
+ copilot_ctx,
+ source_tool="edit_block",
+ result=result,
+ diagnosis_shadow_eligible=False,
+ )
+ return json.dumps(sanitize_tool_result_for_llm("edit_block", result))
try:
workflow_yaml = apply_block_edit(
_stored_workflow_yaml(copilot_ctx),
@@ -381,7 +395,13 @@ async def edit_block_tool(
except BlockEditError as exc:
result = {"ok": False, "error": str(exc)}
record_tool_step_result_for_ctx(copilot_ctx, "edit_block", arguments, result)
- return json.dumps(result)
+ finalize_build_test_result(
+ copilot_ctx,
+ source_tool="edit_block",
+ result=result,
+ diagnosis_shadow_eligible=False,
+ )
+ return json.dumps(sanitize_tool_result_for_llm("edit_block", result))
return await _persist_block_scoped_edit(copilot_ctx, "edit_block", workflow_yaml, arguments)
@@ -434,7 +454,13 @@ async def edit_block_and_run_tool(
"error": f"block_labels must include the edited block {label!r} so this call tests the persisted repair.",
}
record_tool_step_result_for_ctx(copilot_ctx, "edit_block_and_run", arguments, result)
- return json.dumps(result)
+ finalize_build_test_result(
+ copilot_ctx,
+ source_tool="edit_block_and_run",
+ result=result,
+ diagnosis_shadow_eligible=False,
+ )
+ return json.dumps(sanitize_tool_result_for_llm("edit_block_and_run", result))
authority_error = _authority_tool_error(copilot_ctx, "edit_block_and_run")
if authority_error:
return _diagnosis_repair_tool_error(copilot_ctx, "edit_block_and_run", authority_error)
@@ -450,8 +476,12 @@ async def edit_block_and_run_tool(
except BlockEditError as exc:
result = {"ok": False, "error": str(exc)}
record_tool_step_result_for_ctx(copilot_ctx, "edit_block_and_run", arguments, result)
- _record_diagnosis_repair_contract(copilot_ctx, source_tool="edit_block_and_run", result=result)
- return json.dumps(result)
+ finalize_build_test_result(
+ copilot_ctx,
+ source_tool="edit_block_and_run",
+ result=result,
+ )
+ return json.dumps(sanitize_tool_result_for_llm("edit_block_and_run", result))
prior_definition = await _get_prior_workflow_definition(copilot_ctx)
with copilot_span("edit_block_and_run.update", data={"yaml_length": len(workflow_yaml)}):
@@ -463,7 +493,7 @@ async def edit_block_and_run_tool(
_record_workflow_update_result(copilot_ctx, update_result, prior_definition)
if not update_result.get("ok"):
record_tool_step_result_for_ctx(copilot_ctx, "edit_block_and_run", arguments, update_result)
- _record_diagnosis_repair_contract(
+ finalize_build_test_result(
copilot_ctx,
source_tool="edit_block_and_run",
result=update_result,
@@ -523,7 +553,14 @@ async def add_block_tool(
arguments = {"after_label": after_label, "parameters": parameters}
authority_error = _authority_tool_error(copilot_ctx, "add_block")
if authority_error:
- return json.dumps({"ok": False, "error": authority_error})
+ result = {"ok": False, "error": authority_error}
+ finalize_build_test_result(
+ copilot_ctx,
+ source_tool="add_block",
+ result=result,
+ diagnosis_shadow_eligible=False,
+ )
+ return json.dumps(sanitize_tool_result_for_llm("add_block", result))
try:
workflow_yaml = add_block_to_workflow(
_stored_workflow_yaml(copilot_ctx),
@@ -534,7 +571,13 @@ async def add_block_tool(
except BlockEditError as exc:
result = {"ok": False, "error": str(exc)}
record_tool_step_result_for_ctx(copilot_ctx, "add_block", arguments, result)
- return json.dumps(result)
+ finalize_build_test_result(
+ copilot_ctx,
+ source_tool="add_block",
+ result=result,
+ diagnosis_shadow_eligible=False,
+ )
+ return json.dumps(sanitize_tool_result_for_llm("add_block", result))
return await _persist_block_scoped_edit(
copilot_ctx,
"add_block",
@@ -556,13 +599,26 @@ async def delete_block_tool(ctx: RunContextWrapper, label: str) -> str:
arguments = {"label": label}
authority_error = _authority_tool_error(copilot_ctx, "delete_block")
if authority_error:
- return json.dumps({"ok": False, "error": authority_error})
+ result = {"ok": False, "error": authority_error}
+ finalize_build_test_result(
+ copilot_ctx,
+ source_tool="delete_block",
+ result=result,
+ diagnosis_shadow_eligible=False,
+ )
+ return json.dumps(sanitize_tool_result_for_llm("delete_block", result))
try:
workflow_yaml = delete_block_from_workflow(_stored_workflow_yaml(copilot_ctx), label)
except BlockEditError as exc:
result = {"ok": False, "error": str(exc)}
record_tool_step_result_for_ctx(copilot_ctx, "delete_block", arguments, result)
- return json.dumps(result)
+ finalize_build_test_result(
+ copilot_ctx,
+ source_tool="delete_block",
+ result=result,
+ diagnosis_shadow_eligible=False,
+ )
+ return json.dumps(sanitize_tool_result_for_llm("delete_block", result))
return await _persist_block_scoped_edit(copilot_ctx, "delete_block", workflow_yaml, arguments)
@@ -806,7 +862,7 @@ async def run_blocks_tool(
)
await _verify_and_record_run_blocks_result(copilot_ctx, result, handler_start)
record_tool_step_result_for_ctx(copilot_ctx, "run_blocks_and_collect_debug", arguments, result)
- _record_diagnosis_repair_contract(
+ finalize_build_test_result(
copilot_ctx,
source_tool="run_blocks_and_collect_debug",
result=result,
@@ -961,7 +1017,7 @@ async def update_and_run_blocks_tool(
if not update_result.get("ok"):
record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, update_result)
- _record_diagnosis_repair_contract(
+ finalize_build_test_result(
copilot_ctx,
source_tool="update_and_run_blocks",
result=update_result,
@@ -1010,7 +1066,7 @@ def _credential_deferred_combined_tool_result(
}
carry_author_time_findings(update_result, skip_result)
record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, skip_result)
- _record_diagnosis_repair_contract(
+ finalize_build_test_result(
copilot_ctx,
source_tool=tool_name,
result=skip_result,
@@ -1021,7 +1077,7 @@ def _credential_deferred_combined_tool_result(
tool_name=tool_name,
workflow_permanent_id=copilot_ctx.workflow_permanent_id,
)
- return json.dumps(skip_result)
+ return json.dumps(sanitize_tool_result_for_llm(tool_name, skip_result))
async def _run_updated_workflow_blocks(
@@ -1069,7 +1125,7 @@ async def _run_updated_workflow_blocks(
await _verify_and_record_run_blocks_result(copilot_ctx, run_result, handler_start)
carry_author_time_findings(update_result, run_result)
record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, run_result)
- _record_diagnosis_repair_contract(
+ finalize_build_test_result(
copilot_ctx,
source_tool=tool_name,
result=run_result,
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 4a877ff35..4ec3cf305 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -27,6 +27,14 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
stash_blocker_signal,
)
from skyvern.forge.sdk.copilot.build_test_outcome import (
+ BuildTestEvidencePacket,
+ BuildTestPacketDownload,
+ BuildTestPacketFailure,
+ BuildTestPacketPageState,
+ BuildTestPacketRegisteredOutput,
+ BuildTestPacketRun,
+ BuildTestPacketScreenshot,
+ BuildTestPacketUnfinishedItem,
authored_block_parameter_keys_from_workflow,
authored_structure_signature_from_workflow,
record_build_test_outcome,
@@ -72,8 +80,10 @@ from skyvern.forge.sdk.copilot.narration import narrator_poll_tick
from skyvern.forge.sdk.copilot.outcome_verification_trace import record_gate_decision
from skyvern.forge.sdk.copilot.output_utils import (
_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
+ BUILD_TEST_PACKET_KEY,
build_run_blocks_response,
iter_failure_reasons,
+ sanitize_tool_result_for_llm,
truncate_output,
)
from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprints
@@ -3682,10 +3692,299 @@ def _record_diagnosis_repair_contract(
return contract
+def _packet_string(value: Any) -> str | None:
+ return value if isinstance(value, str) and value else None
+
+
+def _packet_string_list(value: Any) -> list[str]:
+ return [item for item in value if isinstance(item, str) and item] if isinstance(value, list) else []
+
+
+def _packet_workflow_readback(copilot_ctx: CopilotContext) -> tuple[str | None, str]:
+ accepted = copilot_ctx.last_workflow_yaml
+ if isinstance(accepted, str) and accepted.strip():
+ return accepted, "accepted_write_readback"
+ persisted = copilot_ctx.persisted_workflow_yaml
+ if isinstance(persisted, str) and persisted.strip():
+ return persisted, "turn_start_persisted_readback"
+ return None, "unavailable"
+
+
+def _packet_page_state(data: Mapping[str, Any], omission_notices: list[str]) -> BuildTestPacketPageState | None:
+ repair_context = data.get("authoring_repair_context")
+ run_id = _packet_string(data.get("workflow_run_id"))
+ if run_id is None:
+ if isinstance(repair_context, Mapping):
+ omission_notices.append("failure.page_state omitted: page evidence has no recorded workflow run identity.")
+ return None
+ repair_run_id = (
+ _packet_string(repair_context.get("workflow_run_id")) if isinstance(repair_context, Mapping) else None
+ )
+ if isinstance(repair_context, Mapping) and repair_run_id != run_id:
+ omission_notices.append("failure.page_state omitted repair-context fields belonging to another or unknown run.")
+ repair = repair_context if isinstance(repair_context, Mapping) and repair_run_id == run_id else {}
+ current_url = _packet_string(repair.get("current_url")) or _packet_string(data.get("current_url"))
+ title = _packet_string(repair.get("current_title")) or _packet_string(data.get("page_title"))
+ page_state = BuildTestPacketPageState(
+ current_origin=_packet_string(repair.get("current_origin")),
+ current_url=current_url,
+ title=title,
+ evidence_source=_packet_string(repair.get("page_evidence_source")),
+ observed_after_workflow_run=repair.get("observed_after_workflow_run") is True,
+ form_summaries=_packet_string_list(repair.get("page_form_summaries")),
+ result_summaries=_packet_string_list(repair.get("page_result_summaries")),
+ action_summaries=_packet_string_list(repair.get("page_action_summaries")),
+ challenge_summaries=_packet_string_list(repair.get("page_challenge_summaries")),
+ obstruction_summaries=_packet_string_list(repair.get("page_obstruction_summaries")),
+ )
+ return (
+ page_state
+ if any(
+ (
+ page_state.current_origin,
+ page_state.current_url,
+ page_state.title,
+ page_state.evidence_source,
+ page_state.form_summaries,
+ page_state.result_summaries,
+ page_state.action_summaries,
+ page_state.challenge_summaries,
+ page_state.obstruction_summaries,
+ )
+ )
+ else None
+ )
+
+
+def _packet_registered_outputs(
+ data: Mapping[str, Any], run_id: str | None, omission_notices: list[str]
+) -> list[BuildTestPacketRegisteredOutput]:
+ raw_outputs = data.get("registered_output_parameter_values")
+ if not isinstance(raw_outputs, list):
+ return []
+ outputs: list[BuildTestPacketRegisteredOutput] = []
+ malformed = 0
+ other_run = 0
+ for raw_output in raw_outputs:
+ if not isinstance(raw_output, Mapping):
+ malformed += 1
+ continue
+ output_run_id = _packet_string(raw_output.get("workflow_run_id"))
+ if run_id is None or output_run_id != run_id:
+ other_run += 1
+ continue
+ try:
+ outputs.append(
+ BuildTestPacketRegisteredOutput(
+ workflow_run_id=_packet_string(raw_output.get("workflow_run_id")),
+ output_parameter_id=_packet_string(raw_output.get("output_parameter_id")),
+ output_parameter_key=_packet_string(raw_output.get("output_parameter_key")),
+ block_label=_packet_string(raw_output.get("block_label")),
+ block_type=_packet_string(raw_output.get("block_type")),
+ value=raw_output.get("value"),
+ )
+ )
+ except ValueError:
+ malformed += 1
+ if malformed:
+ omission_notices.append(f"registered_outputs omitted {malformed} non-JSON or malformed item(s).")
+ if other_run:
+ omission_notices.append(f"registered_outputs omitted {other_run} item(s) belonging to another or unknown run.")
+ return outputs
+
+
+def _packet_downloads(
+ copilot_ctx: CopilotContext,
+ data: Mapping[str, Any],
+ omission_notices: list[str],
+) -> list[BuildTestPacketDownload]:
+ run_id = _packet_string(data.get("workflow_run_id"))
+ if run_id is None:
+ return []
+ raw_blocks = data.get("blocks")
+ blocks = raw_blocks if isinstance(raw_blocks, list) else []
+ outputs_by_label = {
+ label: extracted
+ for block in blocks
+ if isinstance(block, Mapping)
+ and isinstance((label := block.get("label")), str)
+ and isinstance((extracted := block.get("extracted_data")), Mapping)
+ }
+ artifact_ids = _collect_downloaded_artifact_ids(outputs_by_label)
+ evidence = copilot_ctx.registered_artifact_evidence
+ names_by_id: dict[str, str | None] = {}
+ if isinstance(evidence, RegisteredArtifactEvidence):
+ if evidence.workflow_run_id == run_id:
+ names_by_id = {entry.artifact_id: entry.file_name for entry in evidence.entries}
+ elif artifact_ids:
+ omission_notices.append("downloads omitted file names from artifact evidence belonging to another run.")
+ return [
+ BuildTestPacketDownload(artifact_id=artifact_id, file_name=names_by_id.get(artifact_id))
+ for artifact_id in artifact_ids
+ ]
+
+
+def _packet_unfinished_items(
+ copilot_ctx: CopilotContext,
+ run_id: str | None,
+ omission_notices: list[str],
+) -> list[BuildTestPacketUnfinishedItem]:
+ outcome = copilot_ctx.latest_recorded_build_test_outcome
+ if run_id is None or outcome is None or outcome.workflow_run_id != run_id:
+ if copilot_ctx.last_unverified_block_labels or outcome is not None:
+ omission_notices.append("unfinished_items omitted: recorded unfinished evidence is not bound to this run.")
+ return []
+ unfinished = [
+ BuildTestPacketUnfinishedItem(kind="unverified_block", label=label)
+ for label in dict.fromkeys(copilot_ctx.last_unverified_block_labels)
+ if isinstance(label, str) and label
+ ]
+ for fact in outcome.missing_requested_output_facts:
+ output_path = _packet_string(fact.get("output_path"))
+ reason_code = _packet_string(fact.get("reason_code"))
+ if output_path is not None:
+ unfinished.append(
+ BuildTestPacketUnfinishedItem(
+ kind="missing_requested_output",
+ output_path=output_path,
+ reason_code=reason_code,
+ )
+ )
+ return unfinished
+
+
+def build_test_evidence_packet(copilot_ctx: CopilotContext, result: Mapping[str, Any]) -> BuildTestEvidencePacket:
+ raw_data = result.get("data")
+ data = raw_data if isinstance(raw_data, Mapping) else {}
+ omission_notices: list[str] = []
+ workflow_yaml, workflow_source = _packet_workflow_readback(copilot_ctx)
+ if workflow_yaml is None:
+ omission_notices.append(
+ "canonical_workflow_yaml omitted: no accepted or turn-start persistence readback exists."
+ )
+
+ attempted_labels = _packet_string_list(data.get("requested_block_labels"))
+ executed_labels = _packet_string_list(data.get("executed_block_labels"))
+ run_id = _packet_string(data.get("workflow_run_id"))
+ run_status = _packet_string(data.get("overall_status"))
+ if run_id is None:
+ omission_notices.append("run.workflow_run_id omitted: no workflow run was recorded for this result.")
+ if run_status is None:
+ omission_notices.append("run.status omitted: no recorded run status exists for this result.")
+ if not attempted_labels:
+ omission_notices.append("attempted_block_labels omitted: no block run attempt was recorded.")
+ if not executed_labels:
+ omission_notices.append("executed_block_labels omitted: no block execution was recorded.")
+
+ raw_blocks = data.get("blocks")
+ blocks = raw_blocks if isinstance(raw_blocks, list) else []
+ failed_block = next(
+ (block for block in blocks if isinstance(block, Mapping) and block.get("status") in _FAILED_BLOCK_STATUSES),
+ None,
+ )
+ action_trace = _packet_string_list(data.get("action_trace_summary"))
+ page_state = _packet_page_state(data, omission_notices)
+ failure: BuildTestPacketFailure | None = None
+ if failed_block is not None or result.get("ok") is False:
+ failure = BuildTestPacketFailure(
+ block_label=_packet_string(failed_block.get("label")) if failed_block is not None else None,
+ block_status=_packet_string(failed_block.get("status")) if failed_block is not None else run_status,
+ reason=(
+ _packet_string(failed_block.get("failure_reason"))
+ if failed_block is not None
+ else _packet_string(data.get("failure_reason")) or _packet_string(result.get("error"))
+ ),
+ action_trace=action_trace,
+ page_state=page_state,
+ )
+ if not action_trace:
+ omission_notices.append("failure.action_trace omitted: no failed action was recorded.")
+ if page_state is None:
+ omission_notices.append("failure.page_state omitted: no bounded same-run page state was recorded.")
+ if failure.reason is None:
+ omission_notices.append("failure.reason omitted: no recorded failure reason exists.")
+ else:
+ omission_notices.append("failure omitted: no failed run or failed block was recorded.")
+
+ registered_outputs = _packet_registered_outputs(data, run_id, omission_notices)
+ if not registered_outputs:
+ omission_notices.append("registered_outputs empty: no output parameter value was recorded.")
+ downloads = _packet_downloads(copilot_ctx, data, omission_notices)
+ if not downloads:
+ omission_notices.append("downloads empty: no registered download artifact was recorded.")
+
+ screenshot_present = (
+ run_id is not None and isinstance(data.get("screenshot_base64"), str) and bool(data.get("screenshot_base64"))
+ )
+ screenshot_provenance = "data.screenshot_base64" if screenshot_present else None
+ if not screenshot_present and run_id is not None:
+ failed_screenshot = next(
+ (
+ block
+ for block in blocks
+ if isinstance(block, Mapping)
+ and isinstance(block.get("screenshot_b64"), str)
+ and bool(block.get("screenshot_b64"))
+ ),
+ None,
+ )
+ if failed_screenshot is not None:
+ screenshot_present = True
+ screenshot_provenance = "data.blocks[].screenshot_b64"
+ if not screenshot_present:
+ omission_notices.append("screenshot omitted: no final or failed-block screenshot was recorded.")
+
+ unfinished_items = _packet_unfinished_items(copilot_ctx, run_id, omission_notices)
+ if not unfinished_items:
+ omission_notices.append("unfinished_items empty: recorded outcome and workflow evidence identify none.")
+
+ return BuildTestEvidencePacket(
+ workflow_permanent_id=copilot_ctx.workflow_permanent_id,
+ canonical_workflow_yaml=workflow_yaml,
+ canonical_workflow_source=workflow_source,
+ canonical_workflow_yaml_complete=workflow_yaml is not None,
+ attempted_block_labels=attempted_labels,
+ executed_block_labels=executed_labels,
+ run=BuildTestPacketRun(workflow_run_id=run_id, status=run_status),
+ failure=failure,
+ registered_outputs=registered_outputs,
+ downloads=downloads,
+ screenshot=BuildTestPacketScreenshot(present=screenshot_present, provenance=screenshot_provenance),
+ unfinished_items=unfinished_items,
+ omission_notices=omission_notices,
+ )
+
+
+def finalize_build_test_result(
+ copilot_ctx: CopilotContext,
+ *,
+ source_tool: str,
+ result: dict[str, Any],
+ workflow_updated: bool = False,
+ diagnosis_shadow_eligible: bool = True,
+) -> dict[str, Any]:
+ """Finalize diagnosis state and attach the one shared factual packet."""
+ if diagnosis_shadow_eligible:
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool=source_tool,
+ result=result,
+ workflow_updated=workflow_updated,
+ )
+ data = result.get("data")
+ if not isinstance(data, dict):
+ data = {}
+ result["data"] = data
+ data[BUILD_TEST_PACKET_KEY] = build_test_evidence_packet(copilot_ctx, result).model_dump(
+ mode="json", exclude_none=True
+ )
+ return result
+
+
def _diagnosis_repair_tool_error(copilot_ctx: Any, source_tool: str, error: str) -> str:
result = {"ok": False, "error": error}
- _record_diagnosis_repair_contract(copilot_ctx, source_tool=source_tool, result=result)
- return json.dumps(result)
+ finalize_build_test_result(copilot_ctx, source_tool=source_tool, result=result)
+ return json.dumps(sanitize_tool_result_for_llm(source_tool, result))
def _run_blocks_span_data(
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 8b2bb5840..77be5c801 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -7,7 +7,7 @@ import json
from datetime import datetime, timezone
from pathlib import Path
from types import SimpleNamespace
-from typing import Any
+from typing import Any, ClassVar
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -1779,10 +1779,16 @@ class TestEditBlockAndRun:
),
)
- assert json.loads(result) == {
- "ok": False,
- "error": "block_labels must include the edited block 'repair_me' so this call tests the persisted repair.",
- }
+ parsed = json.loads(result)
+ assert parsed["ok"] is False
+ assert parsed["error"] == (
+ "block_labels must include the edited block 'repair_me' so this call tests the persisted repair."
+ )
+ assert parsed["data"]["build_test_packet"]["run"] == {}
+ assert any(
+ "no workflow run was recorded" in notice
+ for notice in parsed["data"]["build_test_packet"]["omission_notices"]
+ )
@pytest.mark.asyncio
async def test_one_call_persists_the_scoped_edit_then_returns_run_debug_evidence(self, monkeypatch) -> None:
@@ -1877,6 +1883,7 @@ workflow_definition:
async def fake_update_workflow(payload, ctx, *, allow_missing_credentials=False):
captured["persisted_yaml"] = payload["workflow_yaml"]
captured["allow_missing_credentials"] = allow_missing_credentials
+ ctx.last_workflow_yaml = payload["workflow_yaml"]
ctx.last_update_block_count = 1
return {"ok": True, "data": {"block_count": 1}}
@@ -1911,18 +1918,271 @@ workflow_definition:
parsed = json.loads(result)
assert parsed["ok"] is True
- assert parsed["data"] == {
+ assert {key: value for key, value in parsed["data"].items() if key != "build_test_packet"} == {
"block_count": 1,
"workflow_updated": True,
"skipped_run": True,
"skip_reason": "workflow_credential_inputs_unbound",
}
+ packet = parsed["data"]["build_test_packet"]
+ assert packet["canonical_workflow_yaml"] == workflow_yaml.replace("credential.username", "credential.password")
+ assert packet["canonical_workflow_source"] == "accepted_write_readback"
+ assert packet["run"] == {}
+ assert any("no workflow run was recorded" in notice for notice in packet["omission_notices"])
assert captured["allow_missing_credentials"] is True
assert "credential.password" in str(captured["persisted_yaml"])
assert ctx.last_run_skipped_unbound_credentials is True
run_blocks.assert_not_awaited()
+class TestSharedBuildTestPacket:
+ _SAVED_WORKFLOW = """title: Test workflow
+workflow_definition:
+ blocks:
+ - block_type: code
+ label: read_total
+ code: |
+ return {"total": await page.inner_text("#final")}
+"""
+ _PRIOR_WORKFLOW = _SAVED_WORKFLOW.replace('"#final"', '"#old"')
+ _FAILED_RUN: ClassVar[dict[str, Any]] = {
+ "ok": False,
+ "error": "The total was not available.",
+ "data": {
+ "workflow_run_id": "wr_packet_1",
+ "overall_status": "failed",
+ "requested_block_labels": ["read_total"],
+ "executed_block_labels": ["read_total"],
+ "blocks": [
+ {
+ "label": "read_total",
+ "block_type": "CODE",
+ "status": "failed",
+ "failure_reason": "The total was not available.",
+ "extracted_data": {"downloaded_file_artifact_ids": ["artifact_1"]},
+ }
+ ],
+ "current_url": "https://example.test/results",
+ "page_title": "Results",
+ "action_trace_summary": ["NULL_ACTION failed response=missing total code_line=3"],
+ "registered_output_parameter_values": [
+ {
+ "workflow_run_id": "wr_packet_1",
+ "output_parameter_id": "output_1",
+ "output_parameter_key": "total",
+ "block_label": "read_total",
+ "block_type": "CODE",
+ "value": None,
+ }
+ ],
+ "screenshot_base64": "frame-bytes",
+ "authoring_repair_context": {
+ "block_label": "read_total",
+ "reason_code": "runtime_block_failure",
+ "runtime_failure_reason": "The total was not available.",
+ "failed_block_status": "failed",
+ "workflow_run_id": "wr_packet_1",
+ "current_origin": "https://example.test",
+ "current_url": "https://example.test/results",
+ "current_title": "Results",
+ "page_evidence_source": "inspect_page_for_composition",
+ "observed_after_workflow_run": True,
+ "page_form_summaries": [],
+ "page_result_summaries": ["result container #summary; text=Total unavailable"],
+ "page_action_summaries": ["button #refresh; text=Refresh"],
+ "page_challenge_summaries": [],
+ "page_obstruction_summaries": [],
+ },
+ },
+ }
+
+ @pytest.mark.asyncio
+ async def test_same_saved_workflow_and_failed_run_share_one_packet_across_three_paths(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ async def fake_update_workflow(payload, ctx, **_kwargs):
+ assert payload["workflow_yaml"] == self._SAVED_WORKFLOW
+ ctx.workflow_yaml = self._SAVED_WORKFLOW
+ ctx.last_workflow_yaml = self._SAVED_WORKFLOW
+ ctx.last_workflow = SimpleNamespace(workflow_definition={"blocks": []})
+ return {"ok": True, "data": {"block_count": 1}, "_workflow": ctx.last_workflow}
+
+ async def fake_run_blocks(_params, _ctx, **_kwargs):
+ return json.loads(json.dumps(self._FAILED_RUN))
+
+ async def fake_verify(ctx, _result, _handler_start):
+ ctx.last_unverified_block_labels = ["read_total"]
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="run_blocks_and_collect_debug",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_packet_1",
+ )
+
+ def fake_record_update(ctx, result, _prior_definition=None):
+ if result.get("ok"):
+ ctx.last_workflow_yaml = ctx.workflow_yaml
+ ctx.last_workflow = result.get("_workflow") or ctx.last_workflow
+ ctx.last_update_block_count = 1
+
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_update_and_run_requires_skipped_run", lambda *args: False)
+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value={"blocks": []}))
+ monkeypatch.setattr(tools_module, "_frontier_runtime_page_url", AsyncMock(return_value=None))
+ monkeypatch.setattr(tools_module, "_plan_frontier", lambda *args: (["read_total"], {}, "read_total", "initial"))
+ monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)
+ monkeypatch.setattr(tools_module, "_record_workflow_update_result", fake_record_update)
+ monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", fake_run_blocks)
+ monkeypatch.setattr(tools_module, "_verify_and_record_run_blocks_result", fake_verify)
+ monkeypatch.setattr(tools_module, "record_tool_step_result_for_ctx", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "enqueue_screenshot_from_result", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_clear_pending_browser_interaction_observation", lambda *args: None)
+
+ def context(workflow_yaml: str) -> CopilotContext:
+ return _ctx(
+ workflow_yaml=workflow_yaml,
+ last_workflow_yaml=workflow_yaml,
+ request_policy=RequestPolicy(allow_update_workflow=True, allow_run_blocks=True),
+ )
+
+ update_then_run_ctx = context(self._PRIOR_WORKFLOW)
+ combined_ctx = context(self._PRIOR_WORKFLOW)
+ edit_ctx = context(self._PRIOR_WORKFLOW)
+ with capture_logs() as logs:
+ await tools_module.update_workflow_tool.on_invoke_tool(
+ SimpleNamespace(context=update_then_run_ctx, tool_name="update_workflow"),
+ json.dumps({"workflow_yaml": self._SAVED_WORKFLOW}),
+ )
+ update_then_run = await tools_module.run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=update_then_run_ctx, tool_name="run_blocks_and_collect_debug"),
+ json.dumps({"block_labels": ["read_total"], "parameters": {}}),
+ )
+ combined = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=combined_ctx, tool_name="update_and_run_blocks"),
+ json.dumps({"workflow_yaml": self._SAVED_WORKFLOW, "block_labels": ["read_total"], "parameters": {}}),
+ )
+ edited = await tools_module.edit_block_and_run_tool.on_invoke_tool(
+ SimpleNamespace(context=edit_ctx, tool_name="edit_block_and_run"),
+ json.dumps(
+ {
+ "label": "read_total",
+ "expected_code": '"#old"',
+ "replacement_code": '"#final"',
+ "block_labels": ["read_total"],
+ "parameters": {},
+ }
+ ),
+ )
+
+ packets = [json.loads(result)["data"]["build_test_packet"] for result in (update_then_run, combined, edited)]
+ assert packets[0] == packets[1] == packets[2]
+ packet = packets[0]
+ assert packet["workflow_permanent_id"] == "wfp-1"
+ assert packet["canonical_workflow_yaml"] == self._SAVED_WORKFLOW
+ assert packet["attempted_block_labels"] == ["read_total"]
+ assert packet["executed_block_labels"] == ["read_total"]
+ assert packet["run"] == {"workflow_run_id": "wr_packet_1", "status": "failed"}
+ assert packet["failure"]["block_label"] == "read_total"
+ assert packet["failure"]["action_trace"] == ["NULL_ACTION failed response=missing total code_line=3"]
+ assert packet["failure"]["page_state"]["result_summaries"] == [
+ "result container #summary; text=Total unavailable"
+ ]
+ assert packet["registered_outputs"][0]["output_parameter_key"] == "total"
+ assert packet["downloads"] == [{"artifact_id": "artifact_1"}]
+ assert packet["screenshot"] == {"present": True, "provenance": "data.screenshot_base64"}
+ assert packet["unfinished_items"] == [{"kind": "unverified_block", "label": "read_total"}]
+ assert sum(log.get("event") == "copilot diagnosis repair contract shadow" for log in logs) == 3
+
+ def test_packet_falls_back_to_turn_start_persistence_readback_not_submitted_yaml(self) -> None:
+ ctx = _ctx(
+ workflow_yaml="title: submitted but rejected",
+ persisted_workflow_yaml="title: persisted canonical workflow",
+ last_workflow_yaml=None,
+ )
+ result: dict[str, Any] = {"ok": False, "error": "workflow validation failed"}
+
+ run_execution_module.finalize_build_test_result(
+ ctx,
+ source_tool="update_workflow",
+ result=result,
+ )
+ packet = result["data"]["build_test_packet"]
+
+ assert packet["canonical_workflow_yaml"] == "title: persisted canonical workflow"
+ assert packet["canonical_workflow_source"] == "turn_start_persisted_readback"
+ assert "submitted but rejected" not in str(packet)
+
+ def test_packet_omits_evidence_that_is_not_bound_to_the_current_run(self) -> None:
+ ctx = _ctx(
+ persisted_workflow_yaml="title: persisted canonical workflow",
+ last_unverified_block_labels=["stale_block"],
+ latest_recorded_build_test_outcome=RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="run_blocks_and_collect_debug",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ workflow_run_id="wr_old",
+ ),
+ )
+ result: dict[str, Any] = {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_new",
+ "overall_status": "failed",
+ "current_url": "https://example.test/current",
+ "blocks": [{"label": "read_total", "status": "failed"}],
+ "registered_output_parameter_values": [
+ {"workflow_run_id": "wr_old", "output_parameter_key": "total", "value": "stale"}
+ ],
+ "authoring_repair_context": {
+ "workflow_run_id": "wr_old",
+ "page_result_summaries": ["stale page result"],
+ },
+ },
+ }
+
+ run_execution_module.finalize_build_test_result(
+ ctx,
+ source_tool="run_blocks_and_collect_debug",
+ result=result,
+ )
+ packet = result["data"]["build_test_packet"]
+
+ assert packet["registered_outputs"] == []
+ assert packet["unfinished_items"] == []
+ assert packet["failure"]["page_state"]["current_url"] == "https://example.test/current"
+ assert packet["failure"]["page_state"]["result_summaries"] == []
+ assert any("another or unknown run" in notice for notice in packet["omission_notices"])
+
+ def test_packet_omits_failed_block_screenshot_without_a_recorded_run(self) -> None:
+ ctx = _ctx(persisted_workflow_yaml="title: persisted canonical workflow")
+ result: dict[str, Any] = {
+ "ok": False,
+ "data": {
+ "overall_status": "failed",
+ "blocks": [
+ {
+ "label": "read_total",
+ "status": "failed",
+ "screenshot_b64": "unbound-screenshot-bytes",
+ }
+ ],
+ },
+ }
+
+ run_execution_module.finalize_build_test_result(
+ ctx,
+ source_tool="run_blocks_and_collect_debug",
+ result=result,
+ )
+ packet = result["data"]["build_test_packet"]
+
+ assert packet["run"] == {"status": "failed"}
+ assert packet["screenshot"] == {"present": False}
+ assert any("screenshot omitted" in notice for notice in packet["omission_notices"])
+
+
class TestTranslateToAgentResultGating:
"""Covers the three SKY-9143 invariants that live in _translate_to_agent_result."""
diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.py
index bc1182432..82cd2af38 100644
--- a/tests/unit/test_copilot_output_utils.py
+++ b/tests/unit/test_copilot_output_utils.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import json
from unittest.mock import MagicMock
import pytest
@@ -142,6 +143,171 @@ def test_sanitize_run_blocks_debug_strips_block_screenshot_b64() -> None:
assert sanitized["data"]["blocks"][0]["final_url"] == "https://portal.example.com/mfa"
+def test_sanitize_build_test_packet_bounds_facts_and_preserves_screenshot_provenance() -> None:
+ packet = {
+ "contract_version": "build_test_evidence_packet_v1",
+ "canonical_workflow_yaml": "w" * 30_010,
+ "canonical_workflow_source": "accepted_write_readback",
+ "canonical_workflow_yaml_complete": True,
+ "attempted_block_labels": [f"attempt_{index}" for index in range(30)],
+ "executed_block_labels": [f"executed_{index}" for index in range(30)],
+ "run": {"workflow_run_id": "wr_1", "status": "failed"},
+ "failure": {
+ "block_label": "read_total",
+ "block_status": "failed",
+ "reason": "missing total",
+ "action_trace": ["action " + "x" * 500 for _ in range(8)],
+ "page_state": {
+ "observed_after_workflow_run": True,
+ "form_summaries": ["form " + "x" * 500 for _ in range(10)],
+ "result_summaries": [],
+ "action_summaries": [],
+ "challenge_summaries": [],
+ "obstruction_summaries": [],
+ },
+ },
+ "registered_outputs": [
+ {"output_parameter_key": f"output_{index}", "value": "v" * 1_300} for index in range(13)
+ ],
+ "downloads": [{"artifact_id": f"artifact_{index}"} for index in range(13)],
+ "screenshot": {"present": True, "provenance": "data.screenshot_base64"},
+ "unfinished_items": [{"kind": "unverified_block", "label": f"block_{index}"} for index in range(25)],
+ "omission_notices": [],
+ }
+
+ sanitized = sanitize_tool_result_for_llm(
+ "run_blocks_and_collect_debug",
+ {"ok": False, "data": {"screenshot_base64": "raw-frame-bytes", "build_test_packet": packet}},
+ )
+ projected = sanitized["data"]["build_test_packet"]
+
+ assert len(projected["canonical_workflow_yaml"]) <= 30_000
+ assert projected["canonical_workflow_yaml_complete"] is False
+ assert len(projected["attempted_block_labels"]) == 24
+ assert len(projected["failure"]["action_trace"]) == 6
+ assert len(projected["failure"]["page_state"]["form_summaries"]) == 8
+ assert len(projected["registered_outputs"]) == 12
+ assert projected["registered_outputs"][0]["value_complete"] is False
+ assert len(projected["downloads"]) == 12
+ assert len(projected["unfinished_items"]) == 24
+ assert projected["screenshot"] == {"present": True, "provenance": "data.screenshot_base64"}
+ assert "raw-frame-bytes" not in str(projected)
+ assert any("shortened" in notice for notice in projected["omission_notices"])
+ assert len(json.dumps(projected)) <= 47_000
+ assert next(iter(sanitized)) == "data"
+ assert next(iter(sanitized["data"])) == "build_test_packet"
+
+
+def test_sanitize_build_test_packet_exercises_aggregate_compaction() -> None:
+ long_identifier = "i" * 200
+ long_summary = "s" * 500
+ packet = {
+ "contract_version": "build_test_evidence_packet_v1",
+ "canonical_workflow_yaml": "w" * 30_010,
+ "canonical_workflow_source": "accepted_write_readback",
+ "canonical_workflow_yaml_complete": True,
+ "attempted_block_labels": [long_identifier for _ in range(30)],
+ "executed_block_labels": [long_identifier for _ in range(30)],
+ "run": {"workflow_run_id": long_identifier, "status": "failed"},
+ "failure": {
+ "block_label": long_identifier,
+ "block_status": long_identifier,
+ "reason": "r" * 1_300,
+ "action_trace": [long_summary for _ in range(8)],
+ "page_state": {
+ "current_origin": "https://" + "o" * 2_000,
+ "current_url": "https://" + "u" * 2_000,
+ "title": long_identifier,
+ "evidence_source": long_identifier,
+ "observed_after_workflow_run": True,
+ "form_summaries": [long_summary for _ in range(10)],
+ "result_summaries": [long_summary for _ in range(10)],
+ "action_summaries": [long_summary for _ in range(10)],
+ "challenge_summaries": [long_summary for _ in range(10)],
+ "obstruction_summaries": [long_summary for _ in range(10)],
+ },
+ },
+ "registered_outputs": [
+ {
+ "workflow_run_id": long_identifier,
+ "output_parameter_id": long_identifier,
+ "output_parameter_key": long_identifier,
+ "block_label": long_identifier,
+ "block_type": long_identifier,
+ "value": "v" * 1_300,
+ }
+ for _ in range(13)
+ ],
+ "downloads": [{"artifact_id": long_identifier, "file_name": long_identifier} for _ in range(13)],
+ "screenshot": {"present": False},
+ "unfinished_items": [
+ {
+ "kind": "unverified_block",
+ "label": long_identifier,
+ "output_path": long_identifier,
+ "reason_code": long_identifier,
+ }
+ for _ in range(25)
+ ],
+ "omission_notices": [],
+ }
+
+ sanitized = sanitize_tool_result_for_llm(
+ "run_blocks_and_collect_debug",
+ {"ok": False, "data": {"build_test_packet": packet}},
+ )
+ projected = sanitized["data"]["build_test_packet"]
+
+ assert "canonical_workflow_yaml" not in projected
+ assert projected["canonical_workflow_yaml_complete"] is False
+ assert len(projected["attempted_block_labels"]) == 12
+ assert len(projected["executed_block_labels"]) == 12
+ assert len(projected["failure"]["action_trace"]) == 2
+ assert len(projected["failure"]["page_state"]["form_summaries"]) == 2
+ assert len(projected["registered_outputs"]) == 6
+ assert projected["registered_outputs"][0]["value_complete"] is False
+ assert len(projected["downloads"]) == 6
+ assert len(projected["unfinished_items"]) == 12
+ assert any("repeated packet facts shortened further" in notice for notice in projected["omission_notices"])
+ assert len(json.dumps(projected)) <= 47_000
+
+
+def test_sanitize_build_test_packet_projection_failure_keeps_tool_result_available(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ packet = {
+ "contract_version": "build_test_evidence_packet_v1",
+ "canonical_workflow_source": "accepted_write_readback",
+ "run": {"workflow_run_id": "wr_1", "status": "failed"},
+ "screenshot": {"present": False},
+ }
+
+ def fail_projection(*_args: object, **_kwargs: object) -> None:
+ raise RuntimeError("projection failed")
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.output_utils.project_build_test_packet_for_llm",
+ fail_projection,
+ )
+
+ sanitized = sanitize_tool_result_for_llm(
+ "run_blocks_and_collect_debug",
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_1",
+ "overall_status": "failed",
+ "build_test_packet": packet,
+ },
+ },
+ )
+
+ assert sanitized["data"]["workflow_run_id"] == "wr_1"
+ assert sanitized["data"]["overall_status"] == "failed"
+ assert "build_test_packet" not in sanitized["data"]
+ assert sanitized["data"]["build_test_packet_omitted"] == "The internal packet projection failed."
+
+
def test_sanitize_run_blocks_debug_preserves_post_run_page_evidence() -> None:
evidence = {
"workflow_run_id": "wr_123",
diff --git a/tests/unit/test_copilot_session_factory.py b/tests/unit/test_copilot_session_factory.py
index cf87c30b8..6a9dffcbf 100644
--- a/tests/unit/test_copilot_session_factory.py
+++ b/tests/unit/test_copilot_session_factory.py
@@ -341,6 +341,52 @@ class TestModelInputCapture:
assert len(sorted(tmp_path.glob("call-*.json"))) == 2
+def test_build_test_packet_survives_recent_tool_output_head_compaction() -> None:
+ from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llm
+ from skyvern.forge.sdk.copilot.session_factory import copilot_call_model_input_filter
+
+ packet = {
+ "contract_version": "build_test_evidence_packet_v1",
+ "workflow_permanent_id": "wfp_compaction",
+ "canonical_workflow_yaml": "title: accepted workflow",
+ "canonical_workflow_source": "accepted_write_readback",
+ "canonical_workflow_yaml_complete": True,
+ "attempted_block_labels": ["read_total"],
+ "executed_block_labels": ["read_total"],
+ "run": {"workflow_run_id": "wr_compaction", "status": "failed"},
+ "failure": {"block_label": "read_total", "block_status": "failed", "reason": "missing total"},
+ "registered_outputs": [],
+ "downloads": [],
+ "screenshot": {"present": True, "provenance": "data.screenshot_base64"},
+ "unfinished_items": [{"kind": "unverified_block", "label": "read_total"}],
+ "omission_notices": [],
+ }
+ sanitized = sanitize_tool_result_for_llm(
+ "run_blocks_and_collect_debug",
+ {
+ "ok": False,
+ "data": {
+ "legacy_blob": "x" * 60_000,
+ "screenshot_base64": "raw-frame-bytes",
+ "build_test_packet": packet,
+ },
+ },
+ )
+ output = json.dumps(sanitized)
+ filtered = copilot_call_model_input_filter(
+ _mk_input_data([{"type": "function_call_output", "call_id": "run-1", "output": output}])
+ )
+ compacted_output = filtered.input[0]["output"]
+ projected_packet = json.dumps(sanitized["data"]["build_test_packet"])
+
+ assert len(output) > 50_000
+ assert projected_packet in compacted_output
+ assert compacted_output.index("build_test_packet") < compacted_output.index("legacy_blob")
+ assert "wfp_compaction" in compacted_output
+ assert "wr_compaction" in compacted_output
+ assert "raw-frame-bytes" not in compacted_output
+
+
def test_model_input_pipeline_has_no_generated_offer_special_case() -> None:
from skyvern.forge.sdk.copilot import enforcement