Skip to content

Automate browser based workflows with AI

SKY-10590 gate click-reached composition evidence (#6338)

Andrew Neilson84fed76b62f5
diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.py
index aaea59743..652681118 100644
--- a/skyvern/forge/sdk/copilot/composition_evidence.py
+++ b/skyvern/forge/sdk/copilot/composition_evidence.py
@@ -2,9 +2,11 @@
from __future__ import annotations
+from collections.abc import Iterable
from typing import Any, Protocol
from urllib.parse import urljoin, urlparse
+import structlog
import yaml
try:
@@ -16,6 +18,8 @@ from skyvern.forge.sdk.copilot.build_phase import BuildPhase
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
from skyvern.utils.yaml_loader import safe_load_no_dates
+LOG = structlog.get_logger()
+
_BUILD_MODE_VALUES: frozenset[str] = frozenset({"build", "draft_only", "edit", "unknown"})
# Block types whose acted page, when no url is on the block, is the current
# frontier (observation of the page suffices). navigation without a url is the
@@ -25,6 +29,9 @@ _FRONTIER_NO_URL_BLOCK_TYPES: frozenset[str] = frozenset({"login", "extraction",
# agent against the reached page, so they need the same observed-evidence floor as a no-url
# navigation, otherwise the agent can author an unobserved click/download/upload.
_INTERACTION_NO_URL_BLOCK_TYPES: frozenset[str] = frozenset({"navigation", "action", "file_download", "file_upload"})
+# Navigation can legitimately split same-page form preparation and submission across
+# blocks, so only no-url action/download/upload blocks force post-interaction refs.
+_POST_INTERACTION_OBS_REQ_BLOCK_TYPES: frozenset[str] = frozenset({"action", "file_download", "file_upload"})
_GOTO_URL_BLOCK_TYPE = "goto_url"
_SCHEMA_EVIDENCE_TOOL = "inspect_page_for_composition"
_STRUCTURED_BROWSER_EVIDENCE_TOOLS: frozenset[str] = frozenset({"evaluate"})
@@ -225,6 +232,20 @@ def merge_visual_composition_evidence(
]
challenge_state["gated_submit_controls"] = (existing_controls + visual_blocked_controls)[:5]
merged["challenge_state"] = challenge_state
+ # Empty-page classification is visual-only: DOM parsing can tell that a page
+ # is schema-empty, but only the screenshot summary distinguishes settled empty
+ # pages from loading shells.
+ if merged.get("schema_empty_page") is True:
+ empty_page_visible = visual_summary.get("empty_page_visible") is True
+ loading_state_visible = visual_summary.get("loading_state_visible") is True
+ if loading_state_visible:
+ merged["empty_page_visual_state"] = "loading_or_progress"
+ elif empty_page_visible:
+ merged["observed_empty_page"] = True
+ merged["empty_page_observation_source"] = VISION_EVIDENCE_SOURCE
+ merged["empty_page_visual_state"] = "settled_empty"
+ else:
+ merged["empty_page_visual_state"] = "unknown"
elif not merged.get("visual_evidence_summary"):
merged["visual_evidence_summary"] = "Screenshot captured because DOM evidence indicated challenge state."
@@ -319,11 +340,15 @@ def _gated_page_acting_blocks(workflow_yaml: str | None, previous_workflow_yaml:
gated: list[dict[str, Any]] = []
nearest_goto: str | None = None
fallback_url = workflow_target_url(workflow_yaml)
+ no_url_interaction_since_url = False
+ click_reached_observation_required = False
for index, block in enumerate(_parse_workflow_blocks(workflow_yaml)):
block_type = str(block.get("block_type") or "").strip().lower()
url = _block_url(block)
if url:
nearest_goto = url
+ no_url_interaction_since_url = False
+ click_reached_observation_required = False
if index == 0 and block_type == _GOTO_URL_BLOCK_TYPE:
# entrypoint scaffold — ungated so the agent can record it and scout from it.
continue
@@ -339,9 +364,19 @@ def _gated_page_acting_blocks(workflow_yaml: str | None, previous_workflow_yaml:
{
"label": label,
"block_type": block_type,
+ "acts_via": acts_via,
"target_url": url or nearest_goto or fallback_url,
+ "requires_observation_ref": acts_via != "url"
+ and (
+ click_reached_observation_required
+ or (block_type in _POST_INTERACTION_OBS_REQ_BLOCK_TYPES and no_url_interaction_since_url)
+ ),
}
)
+ if acts_via == "interaction":
+ no_url_interaction_since_url = True
+ if block_type in _POST_INTERACTION_OBS_REQ_BLOCK_TYPES:
+ click_reached_observation_required = True
return gated
@@ -461,7 +496,11 @@ def has_bounded_page_schema(evidence: dict[str, Any]) -> bool:
if isinstance(value, list) and value:
return True
challenge_state = evidence.get("challenge_state")
- return isinstance(challenge_state, dict) and challenge_state.get("detected") is True
+ if isinstance(challenge_state, dict) and challenge_state.get("detected") is True:
+ return True
+ # A visually confirmed settled-empty page is sufficient composition evidence
+ # even though it has no DOM controls to act on.
+ return evidence.get("observed_empty_page") is True
def _evidence_matches_target(
@@ -523,6 +562,95 @@ def _prior_observed_pages(ctx: Any) -> list[dict[str, Any]]:
return [page for page in (getattr(ctx, "prior_observed_acted_pages", None) or []) if isinstance(page, dict)]
+def _flow_evidence_by_step(ctx: Any) -> dict[int, tuple[dict[str, Any], str]]:
+ by_step: dict[int, tuple[dict[str, Any], str]] = {}
+ for entry in getattr(ctx, "flow_evidence", None) or []:
+ if not isinstance(entry, dict):
+ continue
+ packet = entry.get("evidence")
+ if not isinstance(packet, dict):
+ continue
+ step = entry.get("step")
+ if isinstance(step, bool) or not isinstance(step, int):
+ continue
+ if step in by_step:
+ # Duplicates here mean malformed persisted or deserialized evidence.
+ retained_packet, retained_reached_via = by_step[step]
+ LOG.warning(
+ "copilot_flow_evidence_duplicate_step_ignored",
+ observation_step=step,
+ retained_reached_via=retained_reached_via,
+ ignored_reached_via=str(entry.get("reached_via") or ""),
+ retained_url=retained_packet.get("current_url") or retained_packet.get("inspected_url"),
+ ignored_url=packet.get("current_url") or packet.get("inspected_url"),
+ prior_retained_steps_count=len(by_step),
+ )
+ continue
+ by_step[step] = (packet, str(entry.get("reached_via") or ""))
+ return by_step
+
+
+def _iter_block_observation_ref_items(value: Any, *, warn_malformed: bool) -> Iterable[tuple[Any, Any]] | None:
+ if isinstance(value, dict):
+ return value.items()
+ if isinstance(value, list):
+ items: list[tuple[Any, Any]] = []
+ for item in value:
+ if isinstance(item, dict):
+ items.append((item.get("label"), item.get("observation_step")))
+ elif hasattr(item, "label") and hasattr(item, "observation_step"):
+ # Accept the typed ref shape without coupling the composition gate
+ # to a concrete pydantic model.
+ items.append((item.label, item.observation_step))
+ elif warn_malformed:
+ LOG.warning(
+ "copilot_block_observation_ref_malformed_item_ignored",
+ item_type=type(item).__name__,
+ )
+ return items
+ return None
+
+
+def normalize_block_observation_refs(value: Any) -> dict[str, int]:
+ items = _iter_block_observation_ref_items(value, warn_malformed=True)
+ if items is None:
+ LOG.warning(
+ "copilot_block_observation_refs_unexpected_type_ignored",
+ value_type=type(value).__name__,
+ )
+ return {}
+ refs: dict[str, int] = {}
+ for label, step in items:
+ if not isinstance(label, str) or not label.strip():
+ continue
+ if isinstance(step, bool):
+ continue
+ if isinstance(step, int):
+ refs[label.strip()] = step
+ elif isinstance(step, str):
+ # String steps are ignored so callers can repair malformed refs.
+ LOG.warning(
+ "copilot_block_observation_ref_string_step_ignored",
+ label=label.strip(),
+ step_length=len(step),
+ )
+ return refs
+
+
+def _block_observation_refs(ctx: Any) -> dict[str, int]:
+ return normalize_block_observation_refs(getattr(ctx, "block_observation_refs", None))
+
+
+def _evidence_observed_url(evidence: dict[str, Any]) -> str | None:
+ for key in ("current_url", "inspected_url"):
+ value = evidence.get(key)
+ # "current_page" is the sentinel for inspecting the current browser page
+ # without a known target URL.
+ if isinstance(value, str) and value.strip() and value != "current_page":
+ return value.strip()
+ return None
+
+
def _page_observed(ctx: Any, target_url: str | None, *, allow_post_run: bool) -> bool:
if not target_url:
return False
@@ -540,6 +668,116 @@ def _page_observed(ctx: Any, target_url: str | None, *, allow_post_run: bool) ->
return False
+def _associated_observation_satisfies_block(
+ evidence: dict[str, Any],
+ target_url: str | None,
+ *,
+ acts_via: str,
+ reached_via: str,
+ requires_observation_ref: bool,
+ allow_post_run: bool,
+) -> bool:
+ if acts_via == "url":
+ # URL blocks are grounded by target_url, not observation-ref gates.
+ if requires_observation_ref:
+ return False
+ return _evidence_matches_target(evidence, target_url, allow_post_run_browser_observation=allow_post_run)
+ if requires_observation_ref and reached_via not in {"interaction", "post_run"}:
+ return False
+
+ observed_url = _evidence_observed_url(evidence)
+ if not observed_url:
+ return False
+ return _evidence_matches_target(evidence, observed_url, allow_post_run_browser_observation=allow_post_run)
+
+
+def _block_has_observed_page(
+ ctx: Any,
+ block: dict[str, Any],
+ *,
+ allow_post_run: bool,
+ flow_evidence_by_step: dict[int, tuple[dict[str, Any], str]],
+ block_observation_refs: dict[str, int],
+) -> bool:
+ label = str(block.get("label") or "")
+ if label and label in block_observation_refs:
+ evidence_entry = flow_evidence_by_step.get(block_observation_refs[label])
+ if evidence_entry is None:
+ return False
+ evidence, reached_via = evidence_entry
+ return _associated_observation_satisfies_block(
+ evidence,
+ block.get("target_url"),
+ acts_via=str(block.get("acts_via") or ""),
+ reached_via=reached_via,
+ requires_observation_ref=block.get("requires_observation_ref") is True,
+ allow_post_run=allow_post_run,
+ )
+
+ if block.get("requires_observation_ref") is True:
+ return False
+ return _page_observed(ctx, block.get("target_url"), allow_post_run=allow_post_run)
+
+
+def _missing_observation_ref_step(
+ block: dict[str, Any],
+ *,
+ flow_evidence_by_step: dict[int, tuple[dict[str, Any], str]],
+ block_observation_refs: dict[str, int],
+) -> int | None:
+ label = str(block.get("label") or "")
+ if not label or label not in block_observation_refs:
+ return None
+ step = block_observation_refs[label]
+ return None if step in flow_evidence_by_step else step
+
+
+def _raw_block_observation_ref_step(value: Any, label: str) -> object | None:
+ if not label:
+ return None
+ items = _iter_block_observation_ref_items(value, warn_malformed=False)
+ if items is None:
+ return None
+ for item_label, item_step in items:
+ if isinstance(item_label, str) and item_label.strip() == label:
+ return item_step
+ return None
+
+
+def _string_observation_ref_step(block: dict[str, Any], raw_block_observation_refs: Any) -> str | None:
+ label = str(block.get("label") or "")
+ raw_step = _raw_block_observation_ref_step(raw_block_observation_refs, label)
+ return raw_step if isinstance(raw_step, str) else None
+
+
+def _required_observation_ref_missing(block: dict[str, Any], block_observation_refs: dict[str, int]) -> bool:
+ if block.get("requires_observation_ref") is not True:
+ return False
+ label = str(block.get("label") or "")
+ return bool(label and label not in block_observation_refs)
+
+
+def _wrong_reached_via_observation_ref(
+ block: dict[str, Any],
+ *,
+ flow_evidence_by_step: dict[int, tuple[dict[str, Any], str]],
+ block_observation_refs: dict[str, int],
+) -> tuple[int, str] | None:
+ if block.get("requires_observation_ref") is not True:
+ return None
+ label = str(block.get("label") or "")
+ if not label or label not in block_observation_refs:
+ return None
+ step = block_observation_refs[label]
+ evidence_entry = flow_evidence_by_step.get(step)
+ if evidence_entry is None:
+ return None
+ _, reached_via = evidence_entry
+ if reached_via in {"interaction", "post_run"}:
+ return None
+ return step, reached_via or "<missing>"
+
+
def composition_page_evidence_error(ctx: Any, workflow_yaml: str | None) -> str | None:
"""Return a mutation error when a build adds page-acting blocks before observation.
@@ -563,9 +801,69 @@ def composition_page_evidence_error(ctx: Any, workflow_yaml: str | None) -> str
return None
allow_post_run = bool(previous_workflow_yaml)
+ flow_evidence_by_step = _flow_evidence_by_step(ctx)
+ raw_block_observation_refs = getattr(
+ ctx,
+ "raw_block_observation_refs",
+ getattr(ctx, "block_observation_refs", None),
+ )
+ block_observation_refs = _block_observation_refs(ctx)
for block in gated_blocks:
target_url = block["target_url"]
- if not _page_observed(ctx, target_url, allow_post_run=allow_post_run):
+ if not _block_has_observed_page(
+ ctx,
+ block,
+ allow_post_run=allow_post_run,
+ flow_evidence_by_step=flow_evidence_by_step,
+ block_observation_refs=block_observation_refs,
+ ):
+ missing_step = _missing_observation_ref_step(
+ block,
+ flow_evidence_by_step=flow_evidence_by_step,
+ block_observation_refs=block_observation_refs,
+ )
+ string_step = _string_observation_ref_step(block, raw_block_observation_refs)
+ wrong_reached_via = _wrong_reached_via_observation_ref(
+ block,
+ flow_evidence_by_step=flow_evidence_by_step,
+ block_observation_refs=block_observation_refs,
+ )
+ if string_step is not None:
+ return (
+ "Workflow validation failed: a block_observation_refs entry uses observation_step "
+ f"{string_step!r} as a string. Pass the integer observation_step returned by "
+ "inspect_page_for_composition for click-reached blocks. "
+ f"Offending blocks: {_format_page_block_findings([block])}"
+ )
+ if _required_observation_ref_missing(block, block_observation_refs):
+ return (
+ "Workflow validation failed: a click-reached block requires a block_observation_refs entry. "
+ "Pass an interaction- or post_run-reached observation_step for click-reached blocks before "
+ "composing them. "
+ f"Offending blocks: {_format_page_block_findings([block])}"
+ )
+ if wrong_reached_via is not None:
+ step, reached_via = wrong_reached_via
+ return (
+ "Workflow validation failed: a block references observation_step "
+ f"{step}, but that observed page was reached via {reached_via!r}. "
+ "Pass an interaction- or post_run-reached observation_step for click-reached blocks. "
+ f"Offending blocks: {_format_page_block_findings([block])}"
+ )
+ if missing_step is not None:
+ min_available_step = min(flow_evidence_by_step) if flow_evidence_by_step else None
+ missing_reason = (
+ "that observed page evidence is no longer available in the flow-evidence window"
+ if min_available_step is not None and missing_step < min_available_step
+ else "that observation step was not found in flow evidence"
+ )
+ return (
+ "Workflow validation failed: a block references observation_step "
+ f"{missing_step}, but {missing_reason}. "
+ "Call inspect_page_for_composition again for the reached page and pass the new "
+ "observation_step in block_observation_refs before composing page-dependent blocks. "
+ f"Offending blocks: {_format_page_block_findings([block])}"
+ )
return (
"Workflow validation failed: page-dependent build blocks need observed page evidence before they are "
f"authored. Call inspect_page_for_composition(target_url={target_url!r}) before composing page-dependent "
@@ -590,6 +888,9 @@ def _empty_evidence(inspected_url: str, current_url: str) -> dict[str, Any]:
"result_containers": [],
"anti_bot_indicators": [],
"challenge_controls": [],
+ "schema_empty_page": False,
+ "observed_empty_page": False,
+ "empty_page_visual_state": None,
"evidence_confidence": 0.0,
"source_tool": "inspect_page_for_composition",
**_evidence_metadata([]),
@@ -923,6 +1224,11 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
field_count = sum(len(form.get("fields") or []) for form in forms)
control_count = sum(len(form.get("submit_controls") or []) for form in forms)
+ body_text = _node_text(soup.body if soup.body is not None else soup).strip()
+ # Schema-empty means the page had content, but no bounded form/link/result/challenge structure.
+ schema_empty_page = bool((html or "").strip() or page_title or body_text) and not (
+ forms or navigation_targets or result_containers or challenge_controls or anti_bot_indicators
+ )
# Higher confidence means the parser saw a more complete form surface.
confidence = 0.85 if field_count and control_count else 0.6 if field_count else 0.3 if forms else 0.1
return {
@@ -934,6 +1240,9 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
"result_containers": result_containers,
"anti_bot_indicators": anti_bot_indicators,
"challenge_controls": challenge_controls,
+ "schema_empty_page": schema_empty_page,
+ "observed_empty_page": False,
+ "empty_page_visual_state": None,
"evidence_confidence": confidence,
"source_tool": "inspect_page_for_composition",
**_evidence_metadata(anti_bot_indicators, forms=forms),
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index e2fadacc4..47f149aac 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -204,7 +204,10 @@ def _merge_observed_acted_pages(prior: list[ObservedPage], flow_evidence: list[d
"""
by_url: dict[str, ObservedPage] = {page.url: page for page in prior if page.url}
for entry in flow_evidence:
+ evidence = entry.get("evidence")
url = entry.get("url")
+ if (not isinstance(url, str) or not url.strip()) and isinstance(evidence, dict):
+ url = evidence.get("current_url") or evidence.get("inspected_url")
if not isinstance(url, str) or not url.strip():
continue
existing = by_url.get(url)
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 1b05aaa85..ff0feac62 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -178,6 +178,12 @@ class AgentContext:
# while scouting the goal path, each tagged with how that state was reached.
# Feeds the per-acted-page composition gate; never persisted into workflow YAML.
flow_evidence: list[dict[str, Any]] = field(default_factory=list)
+ # In-turn side channel from workflow mutation calls: block label -> flow_evidence
+ # observation step used to ground the newly authored page-acting block.
+ block_observation_refs: dict[str, int] = field(default_factory=dict)
+ # Raw tool input for block_observation_refs, retained only for diagnostics
+ # when normalization drops malformed entries before composition validation.
+ raw_block_observation_refs: Any | None = None
# Hydrated at turn start from StructuredContext.observed_acted_pages; lets the
# composition gate credit a page observed on a prior turn when this turn's
# flow_evidence does not cover it (closes the spent-inspection-budget
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index c43d17a6c..8bb15ee49 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -12,7 +12,7 @@ from collections import Counter, defaultdict
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime
-from typing import Any, Literal
+from typing import Annotated, Any, Literal
from urllib.parse import urljoin, urlparse
import structlog
@@ -26,7 +26,7 @@ except ImportError: # pragma: no cover — bs4 is a transitive dep but discover
BeautifulSoup = None # type: ignore[assignment, misc]
from agents.run_context import RunContextWrapper
from jinja2.sandbox import SandboxedEnvironment
-from pydantic import ValidationError
+from pydantic import BaseModel, Field, ValidationError
from skyvern.config import settings
from skyvern.forge import app
@@ -60,6 +60,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
composition_page_evidence_error,
has_bounded_page_schema,
merge_visual_composition_evidence,
+ normalize_block_observation_refs,
page_evidence_needs_visual_fallback,
parse_composition_html,
workflow_target_url,
@@ -2022,6 +2023,11 @@ def _parameter_binding_invariant_error(
)
+class BlockObservationRef(BaseModel):
+ label: str
+ observation_step: Annotated[int, Field(ge=0)]
+
+
async def _update_workflow(
params: dict[str, Any],
ctx: AgentContext,
@@ -2033,6 +2039,10 @@ async def _update_workflow(
return {"ok": False, "error": authority_error}
workflow_yaml = params["workflow_yaml"]
+ # Tool wrappers run authority/loop guards before calling here. The composition
+ # gate below consumes these refs, so they must be visible before validation.
+ ctx.raw_block_observation_refs = params.get("raw_block_observation_refs", params.get("block_observation_refs"))
+ ctx.block_observation_refs = normalize_block_observation_refs(params.get("block_observation_refs"))
if allow_missing_credentials is None:
allow_missing_credentials = getattr(ctx, "allow_untested_workflow_draft", False) is True
if not allow_missing_credentials:
@@ -5661,6 +5671,7 @@ def _diagnosis_repair_tool_error(copilot_ctx: Any, source_tool: str, error: str)
async def update_workflow_tool(
ctx: RunContextWrapper,
workflow_yaml: str,
+ block_observation_refs: list[BlockObservationRef] | None = None,
) -> str:
"""Validate and update the workflow YAML definition.
Provide the complete workflow YAML as a string.
@@ -5675,9 +5686,14 @@ async def update_workflow_tool(
building or editing the workflow. Do not invent URL params, form fields,
result affordances, or page structure from memory; ground workflow blocks
in observed MCP evidence or information the user supplied.
+ When you compose no-url blocks from a page reached by prior clicks, include
+ `block_observation_refs` entries with each block label and the
+ `observation_step` returned by inspect_page_for_composition for the page
+ that block acts on.
"""
copilot_ctx = ctx.context
- arguments = {"workflow_yaml": workflow_yaml}
+ normalized_block_observation_refs = normalize_block_observation_refs(block_observation_refs)
+ arguments = {"workflow_yaml": workflow_yaml, "block_observation_refs": normalized_block_observation_refs}
loop_error = _tool_loop_error(copilot_ctx, "update_workflow", arguments)
if loop_error:
return json.dumps({"ok": False, "error": loop_error})
@@ -5710,7 +5726,7 @@ async def update_workflow_tool(
with copilot_span("update_workflow", data={"yaml_length": len(workflow_yaml)}):
result = await _update_workflow(
- arguments,
+ {**arguments, "raw_block_observation_refs": block_observation_refs},
copilot_ctx,
allow_missing_credentials=getattr(copilot_ctx, "allow_untested_workflow_draft", False) is True,
)
@@ -5927,6 +5943,7 @@ async def update_and_run_blocks_tool(
ctx: RunContextWrapper,
workflow_yaml: str,
block_labels: list[str],
+ block_observation_refs: list[BlockObservationRef] | None = None,
parameters: dict[str, Any] | None = None,
) -> Any:
"""Update the workflow YAML and immediately run the specified blocks in one step.
@@ -5965,6 +5982,10 @@ async def update_and_run_blocks_tool(
state or observed URL deltas.
Browser inspection is build-time context; add durable workflow blocks only
for the reusable actions/checks the workflow actually needs.
+ When you compose no-url blocks from a page reached by prior clicks, include
+ `block_observation_refs` entries with each block label and the
+ `observation_step` returned by inspect_page_for_composition for the page
+ that block acts on.
When inspected evidence shows an anti-bot challenge gating a disabled
submit/search control, account for challenge resolution before submit;
do not compose a click against a control observed as disabled.
@@ -5972,7 +5993,13 @@ async def update_and_run_blocks_tool(
copilot_ctx = ctx.context
copilot_ctx.completion_verification_result = None
handler_start = time.monotonic()
- arguments = {"workflow_yaml": workflow_yaml, "block_labels": block_labels, "parameters": parameters or {}}
+ normalized_block_observation_refs = normalize_block_observation_refs(block_observation_refs)
+ arguments = {
+ "workflow_yaml": workflow_yaml,
+ "block_labels": block_labels,
+ "block_observation_refs": normalized_block_observation_refs,
+ "parameters": parameters or {},
+ }
skip_run_after_update = _request_policy_allows_update_and_skip_run(copilot_ctx, "update_and_run_blocks")
authority_error = _authority_tool_error(
copilot_ctx,
@@ -5993,7 +6020,11 @@ async def update_and_run_blocks_tool(
# Step 1: Update the workflow
with copilot_span("update_workflow", data={"yaml_length": len(workflow_yaml)}):
update_result = await _update_workflow(
- {"workflow_yaml": workflow_yaml},
+ {
+ "workflow_yaml": workflow_yaml,
+ "block_observation_refs": normalized_block_observation_refs,
+ "raw_block_observation_refs": block_observation_refs,
+ },
copilot_ctx,
allow_missing_credentials=skip_run_after_update,
)
@@ -6558,15 +6589,21 @@ def _composition_visual_prompt(evidence: dict[str, Any]) -> str:
"challenge_state": evidence.get("challenge_state") or {},
"form_count": len(evidence.get("forms") or []),
"result_container_count": len(evidence.get("result_containers") or []),
+ "schema_empty_page": evidence.get("schema_empty_page") is True,
}
return (
"Summarize this screenshot for Workflow Copilot build-time page evidence. "
"Return JSON only with keys: summary, challenge_detected, challenge_kind, "
- "challenge_location, submit_blocked, blocked_submit_controls, omissions. Focus on "
+ "challenge_location, submit_blocked, blocked_submit_controls, empty_page_visible, "
+ "loading_state_visible, omissions. Focus on "
"visible anti-bot or human-verification state, whether it appears to gate a submit/search "
"control, and where it appears relative to the page controls. Do not include raw DOM, "
"code, selectors, personal data, or workflow instructions. If no challenge is visible, "
- "set challenge_detected to false and submit_blocked to false.\n\n"
+ "set challenge_detected to false and submit_blocked to false. If DOM context shows a "
+ "schema-empty page, set empty_page_visible to true only when the screenshot shows a "
+ "settled page with no visible forms, controls, result data, challenge, or loading/progress "
+ "state; set loading_state_visible to true when the page appears to be waiting, loading, "
+ "redirecting, or still rendering.\n\n"
f"DOM evidence context:\n{json.dumps(context, sort_keys=True)}"
)
@@ -6587,6 +6624,8 @@ def _normalize_visual_summary(value: Any) -> dict[str, Any] | None:
challenge_location = value.get("challenge_location")
submit_blocked = value.get("submit_blocked")
blocked_submit_controls = value.get("blocked_submit_controls")
+ empty_page_visible = value.get("empty_page_visible")
+ loading_state_visible = value.get("loading_state_visible")
omissions = value.get("omissions")
return {
"summary": summary if isinstance(summary, str) else "",
@@ -6594,6 +6633,8 @@ def _normalize_visual_summary(value: Any) -> dict[str, Any] | None:
"challenge_kind": challenge_kind if isinstance(challenge_kind, str) else "",
"challenge_location": challenge_location if isinstance(challenge_location, str) else "",
"submit_blocked": submit_blocked if isinstance(submit_blocked, bool) else None,
+ "empty_page_visible": empty_page_visible if isinstance(empty_page_visible, bool) else None,
+ "loading_state_visible": loading_state_visible if isinstance(loading_state_visible, bool) else None,
"blocked_submit_controls": [item for item in blocked_submit_controls if isinstance(item, str)]
if isinstance(blocked_submit_controls, list)
else [],
@@ -6709,7 +6750,11 @@ async def _composition_evidence_after_navigation_failure(
return evidence if evidence.get("screenshot_used") else None
-_MAX_FLOW_EVIDENCE = 12
+# Large enough for long single-turn reconnaissance: block_observation_refs are
+# chosen after scouting, so this cap must stay above the citation window a turn
+# can realistically compose against. Entries carry bounded parse summaries, not
+# raw HTML or screenshots.
+_MAX_FLOW_EVIDENCE = 64
def _inspection_reached_via(*, use_current_page: bool, post_run: bool) -> str:
@@ -6726,7 +6771,7 @@ def _inspection_reached_via(*, use_current_page: bool, post_run: bool) -> str:
return "interaction"
-def _append_flow_evidence(copilot_ctx: Any, evidence: dict[str, Any], *, reached_via: str) -> None:
+def _append_flow_evidence(copilot_ctx: Any, evidence: dict[str, Any], *, reached_via: str) -> int | None:
"""Append a typed entry to the bounded flow-evidence trajectory (SKY-10562).
One entry per scouted page: the page-evidence packet plus how it was reached
@@ -6735,19 +6780,30 @@ def _append_flow_evidence(copilot_ctx: Any, evidence: dict[str, Any], *, reached
"""
trajectory = getattr(copilot_ctx, "flow_evidence", None)
if not isinstance(trajectory, list):
- return
- url = evidence.get("current_url") or evidence.get("inspected_url") or ""
+ return None
+ prior_steps = [entry.get("step") for entry in trajectory if isinstance(entry, dict)]
+ step = (
+ max((value for value in prior_steps if isinstance(value, int) and not isinstance(value, bool)), default=-1) + 1
+ )
trajectory.append(
{
"evidence": evidence,
"reached_via": reached_via,
- "url": url if isinstance(url, str) else "",
"had_bounded_schema": has_bounded_page_schema(evidence),
- "step": len(trajectory),
+ "step": step,
}
)
if len(trajectory) > _MAX_FLOW_EVIDENCE:
+ overflow_entry_count = len(trajectory) - _MAX_FLOW_EVIDENCE
+ LOG.warning(
+ "copilot_flow_evidence_evicted",
+ overflow_entry_count=overflow_entry_count,
+ max_flow_evidence=_MAX_FLOW_EVIDENCE,
+ retained_window_size=_MAX_FLOW_EVIDENCE,
+ latest_step=step,
+ )
del trajectory[:-_MAX_FLOW_EVIDENCE]
+ return step
_COMPOSITION_HOLLOW_RECAPTURE_RETRIES = 2
@@ -6853,7 +6909,10 @@ async def _capture_composition_evidence(
await asyncio.sleep(_COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS)
if evidence is not None and html_truncated:
evidence = _composition_add_inspection_warning(evidence, "html_sliced_at_cap")
- if evidence is not None and page_evidence_needs_visual_fallback(evidence):
+ if evidence is not None and (
+ page_evidence_needs_visual_fallback(evidence)
+ or (evidence.get("schema_empty_page") is True and not has_bounded_page_schema(evidence))
+ ):
evidence = await _augment_composition_evidence_with_visual_fallback(copilot_ctx, evidence)
return evidence, None
@@ -7006,12 +7065,21 @@ async def _inspect_page_for_composition_impl(
copilot_ctx.post_run_current_page_inspection_workflow_run_id = run_id
copilot_ctx.composition_page_evidence = evidence
reached_via = _inspection_reached_via(use_current_page=use_current_page, post_run=bool(run_id))
- _append_flow_evidence(copilot_ctx, evidence, reached_via=reached_via)
+ observation_step = _append_flow_evidence(copilot_ctx, evidence, reached_via=reached_via)
+ if observation_step is None:
+ LOG.warning("copilot_flow_evidence_append_failed_no_trajectory")
_mark_page_inspected(copilot_ctx)
# Surface the reached page at the top level so the model registers that the
# inspection already navigated there and does not re-issue navigate_browser.
current_url = evidence.get("current_url") or evidence.get("inspected_url") or ""
- result = {"ok": True, "current_url": current_url, "reached_via": reached_via, "data": evidence}
+ result = {
+ "ok": True,
+ "current_url": current_url,
+ "reached_via": reached_via,
+ "data": evidence,
+ }
+ if observation_step is not None:
+ result["observation_step"] = observation_step
record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
return result
@@ -7415,7 +7483,9 @@ async def inspect_page_for_composition_tool(
Returns observed page evidence: current URL, title, navigation targets, form
fields with labels and selectors, submit/search controls, result containers,
compact visible text excerpts, anti-bot indicators, and bounded visual
- challenge evidence when DOM evidence shows challenge state. Do NOT paste the
+ challenge evidence when DOM evidence shows challenge state. The returned
+ `observation_step` is the side-channel id to pass in `block_observation_refs`
+ when a newly authored block acts on this observed page. Do NOT paste the
evidence into workflow YAML; use it to ground concise block prompts. If a
block run changes pages, inspect the reached page before authoring downstream
form/search/result blocks. If the
diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.py
index c62e95aa7..e95eef05f 100644
--- a/tests/unit/test_copilot_composition_evidence.py
+++ b/tests/unit/test_copilot_composition_evidence.py
@@ -3,6 +3,7 @@
from __future__ import annotations
from dataclasses import dataclass, field
+from unittest.mock import patch
import yaml
@@ -10,6 +11,7 @@ from skyvern.forge.sdk.copilot.build_phase import BuildPhase
from skyvern.forge.sdk.copilot.composition_evidence import (
composition_page_evidence_error,
merge_visual_composition_evidence,
+ normalize_block_observation_refs,
page_evidence_needs_visual_fallback,
parse_composition_html,
)
@@ -24,6 +26,9 @@ class _Ctx:
composition_page_evidence: dict | None = None
workflow_yaml: str | None = None
flow_evidence: list[dict] = field(default_factory=list)
+ # Looser than AgentContext so tests can feed malformed refs into the gate.
+ block_observation_refs: dict[str, object] = field(default_factory=dict)
+ raw_block_observation_refs: object | None = None
prior_observed_acted_pages: list[dict] = field(default_factory=list)
per_tool_budget_problem_block_labels: list[str] = field(default_factory=list)
workflow_verification_evidence: WorkflowVerificationEvidence = field(default_factory=WorkflowVerificationEvidence)
@@ -31,19 +36,29 @@ class _Ctx:
last_failure_category_top: str | None = None
-def _flow_entry(url: str, *, reached_via: str = "navigate", with_form: bool = True) -> dict:
+def _flow_entry(
+ url: str,
+ *,
+ reached_via: str = "navigate",
+ with_form: bool = True,
+ observed_empty_page: bool = False,
+ step: int = 0,
+) -> dict:
evidence: dict = {
"inspected_url": url,
"current_url": url,
"source_tool": "inspect_page_for_composition",
"forms": [{"fields": [_field("X", "x")], "submit_controls": []}] if with_form else [],
+ "navigation_targets": [],
+ "result_containers": [],
+ "challenge_controls": [],
+ "observed_empty_page": observed_empty_page,
}
return {
"evidence": evidence,
"reached_via": reached_via,
- "url": url,
- "had_bounded_schema": with_form,
- "step": 0,
+ "had_bounded_schema": with_form or observed_empty_page,
+ "step": step,
}
@@ -166,6 +181,97 @@ def test_composition_parse_html_adds_challenge_state_for_anti_bot_dom() -> None:
assert parsed["challenge_state"]["gated_submit_controls"] == []
+def test_composition_parse_html_reports_schema_empty_without_semantic_terminal_inference() -> None:
+ parsed = parse_composition_html(
+ """
+ <html><head><title>Done</title></head><body>
+ <main>Confirmation complete.</main>
+ </body></html>
+ """,
+ inspected_url="https://example.com/confirmation",
+ current_url="https://example.com/confirmation",
+ )
+
+ assert parsed["forms"] == []
+ assert parsed["navigation_targets"] == []
+ assert parsed["result_containers"] == []
+ assert parsed["schema_empty_page"] is True
+ assert parsed["observed_empty_page"] is False
+ assert parsed["empty_page_visual_state"] is None
+ assert "empty_page_state" not in parsed
+
+
+def test_composition_parse_html_keeps_loading_shell_unobserved_without_visual_confirmation() -> None:
+ parsed = parse_composition_html(
+ """
+ <html><head><title>Loading</title></head><body>
+ <main>Loading...</main>
+ </body></html>
+ """,
+ inspected_url="https://example.com/results",
+ current_url="https://example.com/results",
+ )
+
+ assert parsed["forms"] == []
+ assert parsed["navigation_targets"] == []
+ assert parsed["result_containers"] == []
+ assert parsed["schema_empty_page"] is True
+ assert parsed["observed_empty_page"] is False
+ assert parsed["empty_page_visual_state"] is None
+ assert "empty_page_state" not in parsed
+
+
+def test_visual_summary_marks_observed_empty_page_without_text_hints() -> None:
+ parsed = parse_composition_html(
+ """
+ <html><head><title>Receipt</title></head><body></body></html>
+ """,
+ inspected_url="https://example.com/receipt",
+ current_url="https://example.com/receipt",
+ )
+
+ marked = merge_visual_composition_evidence(
+ parsed,
+ visual_summary={
+ "summary": "A settled blank page is visible after the submit action.",
+ "challenge_detected": False,
+ "submit_blocked": False,
+ "empty_page_visible": True,
+ "loading_state_visible": False,
+ },
+ )
+
+ assert marked["observed_empty_page"] is True
+ assert marked["empty_page_observation_source"] == "vision_summary"
+ assert marked["empty_page_visual_state"] == "settled_empty"
+
+
+def test_visual_summary_keeps_loading_shell_unobserved() -> None:
+ parsed = parse_composition_html(
+ """
+ <html><head><title>Loading</title></head><body>
+ <main>Loading...</main>
+ </body></html>
+ """,
+ inspected_url="https://example.com/results",
+ current_url="https://example.com/results",
+ )
+
+ marked = merge_visual_composition_evidence(
+ parsed,
+ visual_summary={
+ "summary": "The page is still rendering and shows a wait state.",
+ "challenge_detected": False,
+ "submit_blocked": False,
+ "empty_page_visible": False,
+ "loading_state_visible": True,
+ },
+ )
+
+ assert marked["observed_empty_page"] is False
+ assert marked["empty_page_visual_state"] == "loading_or_progress"
+
+
def test_merge_visual_composition_evidence_keeps_screenshot_bounded_and_typed() -> None:
parsed = parse_composition_html(
"<html><head><title>Just a moment...</title></head><body>Human verification</body></html>",
@@ -1029,6 +1135,39 @@ def test_composition_gate_allows_separate_form_state_and_submit_blocks_from_one_
assert error is None
+def test_composition_gate_allows_navigation_split_blocks_sharing_entrypoint_observation_ref() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "prepare_lookup",
+ "navigation_goal": "Enter the observed First Name and Last Name fields without submitting.",
+ },
+ {
+ "block_type": "navigation",
+ "label": "submit_lookup",
+ "navigation_goal": "Click the observed Search button and wait for results.",
+ },
+ {
+ "block_type": "extraction",
+ "label": "extract_results",
+ "data_extraction_goal": "Extract the credential rows from the result page.",
+ },
+ )
+ ctx = _Ctx(
+ flow_evidence=[_flow_entry("https://example.com/lookup", reached_via="navigate", step=0)],
+ block_observation_refs={
+ "prepare_lookup": 0,
+ "submit_lookup": 0,
+ "extract_results": 0,
+ },
+ )
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is None
+
+
# ---------------- SKY-10562: block-type-agnostic, per-acted-page, multi-page gate ----------------
@@ -1086,6 +1225,223 @@ def test_composition_gate_multi_page_flow_evidence_grounds_each_acted_page() ->
assert composition_page_evidence_error(both, workflow_yaml) is None
+def test_composition_gate_requires_block_observation_refs_for_click_reached_pages() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {"block_type": "action", "label": "search_product", "navigation_goal": "Search for the product."},
+ {"block_type": "action", "label": "add_first_result", "navigation_goal": "Add the first result to the cart."},
+ {"block_type": "extraction", "label": "read_cart", "data_extraction_goal": "Read the cart contents."},
+ )
+ ctx = _Ctx(
+ flow_evidence=[_flow_entry("https://example.com/", step=0)],
+ block_observation_refs={"search_product": 0},
+ )
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is not None
+ assert "requires a block_observation_refs entry" in error
+ assert "Pass an interaction- or post_run-reached observation_step" in error
+ assert "add_first_result (action)" in error
+
+
+def test_composition_gate_rejects_click_reached_blocks_reusing_entrypoint_observation_ref() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {"block_type": "action", "label": "search_product", "navigation_goal": "Search for the product."},
+ {"block_type": "action", "label": "add_first_result", "navigation_goal": "Add the first result to the cart."},
+ {"block_type": "extraction", "label": "read_cart", "data_extraction_goal": "Read the cart contents."},
+ )
+ ctx = _Ctx(
+ flow_evidence=[_flow_entry("https://example.com/", reached_via="navigate", step=0)],
+ block_observation_refs={
+ "search_product": 0,
+ "add_first_result": 0,
+ "read_cart": 0,
+ },
+ )
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is not None
+ assert "references observation_step 0" in error
+ assert "reached via 'navigate'" in error
+ assert "add_first_result (action)" in error
+
+
+def test_composition_gate_reports_missing_referenced_observation_step() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {"block_type": "action", "label": "search_product", "navigation_goal": "Search for the product."},
+ {"block_type": "action", "label": "add_first_result", "navigation_goal": "Add the first result to the cart."},
+ )
+ ctx = _Ctx(
+ flow_evidence=[_flow_entry("https://example.com/", reached_via="navigate", step=0)],
+ block_observation_refs={
+ "search_product": 0,
+ "add_first_result": 9,
+ },
+ )
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is not None
+ assert "references observation_step 9" in error
+ assert "observation step was not found in flow evidence" in error
+ assert "add_first_result (action)" in error
+
+
+def test_composition_gate_reports_evicted_referenced_observation_step() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {"block_type": "action", "label": "search_product", "navigation_goal": "Search for the product."},
+ {"block_type": "action", "label": "add_first_result", "navigation_goal": "Add the first result to the cart."},
+ )
+ ctx = _Ctx(
+ flow_evidence=[_flow_entry("https://example.com/cart", reached_via="interaction", step=65)],
+ block_observation_refs={
+ "search_product": 65,
+ "add_first_result": 9,
+ },
+ )
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is not None
+ assert "references observation_step 9" in error
+ assert "no longer available in the flow-evidence window" in error
+
+
+def test_normalize_block_observation_refs_rejects_string_steps() -> None:
+ assert normalize_block_observation_refs(
+ [
+ {"label": "add_to_cart", "observation_step": 2},
+ {"label": "confirm_cart", "observation_step": "3"},
+ ]
+ ) == {"add_to_cart": 2}
+
+
+def test_normalize_block_observation_refs_warns_on_unexpected_container_type() -> None:
+ with patch("skyvern.forge.sdk.copilot.composition_evidence.LOG.warning") as warning:
+ assert normalize_block_observation_refs("add_to_cart:2") == {}
+
+ warning.assert_called_once_with(
+ "copilot_block_observation_refs_unexpected_type_ignored",
+ value_type="str",
+ )
+
+
+def test_composition_gate_reports_string_typed_observation_step_from_raw_refs() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {"block_type": "action", "label": "search_product", "navigation_goal": "Search for the product."},
+ {"block_type": "action", "label": "add_first_result", "navigation_goal": "Add the first result to the cart."},
+ )
+ ctx = _Ctx(
+ flow_evidence=[
+ _flow_entry("https://example.com/", reached_via="navigate", step=0),
+ _flow_entry("https://example.com/results", reached_via="interaction", step=1),
+ ],
+ block_observation_refs={
+ "search_product": 0,
+ },
+ raw_block_observation_refs=[
+ {"label": "search_product", "observation_step": 0},
+ {"label": "add_first_result", "observation_step": "1"},
+ ],
+ )
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is not None
+ assert "observation_step '1' as a string" in error
+ assert "Pass the integer observation_step" in error
+ assert "add_first_result (action)" in error
+
+
+def test_composition_gate_rejects_action_after_navigation_reusing_entrypoint_observation_ref() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {"block_type": "navigation", "label": "search_product", "navigation_goal": "Search for the product."},
+ {"block_type": "action", "label": "add_first_result", "navigation_goal": "Add the first result to the cart."},
+ )
+ ctx = _Ctx(
+ flow_evidence=[_flow_entry("https://example.com/", reached_via="navigate", step=0)],
+ block_observation_refs={
+ "search_product": 0,
+ "add_first_result": 0,
+ },
+ )
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is not None
+ assert "add_first_result (action)" in error
+
+
+def test_composition_gate_allows_click_reached_pages_with_block_observation_refs() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {"block_type": "action", "label": "search_product", "navigation_goal": "Search for the product."},
+ {"block_type": "action", "label": "add_first_result", "navigation_goal": "Add the first result to the cart."},
+ {"block_type": "extraction", "label": "read_cart", "data_extraction_goal": "Read the cart contents."},
+ )
+ ctx = _Ctx(
+ flow_evidence=[
+ _flow_entry("https://example.com/", reached_via="navigate", step=0),
+ _flow_entry("https://example.com/results", reached_via="interaction", step=1),
+ _flow_entry("https://example.com/cart", reached_via="interaction", step=2),
+ ],
+ block_observation_refs={
+ "search_product": 0,
+ "add_first_result": 1,
+ "read_cart": 2,
+ },
+ )
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is None
+
+
+def test_composition_gate_allows_truthfully_empty_observed_confirmation_page() -> None:
+ confirmation_evidence = merge_visual_composition_evidence(
+ parse_composition_html(
+ "<html><head><title>Blank receipt</title></head><body></body></html>",
+ inspected_url="https://example.com/confirmation",
+ current_url="https://example.com/confirmation",
+ ),
+ visual_summary={
+ "summary": "The browser shows a settled blank receipt page.",
+ "challenge_detected": False,
+ "submit_blocked": False,
+ "empty_page_visible": True,
+ "loading_state_visible": False,
+ },
+ )
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {"block_type": "action", "label": "submit_form", "navigation_goal": "Submit the form."},
+ {"block_type": "validation", "label": "confirm_done", "complete_criterion": "The confirmation page loaded."},
+ )
+ ctx = _Ctx(
+ flow_evidence=[
+ _flow_entry("https://example.com/", reached_via="navigate", step=0),
+ {
+ "evidence": confirmation_evidence,
+ "reached_via": "interaction",
+ "had_bounded_schema": True,
+ "step": 1,
+ },
+ ],
+ block_observation_refs={"submit_form": 0, "confirm_done": 1},
+ )
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is None
+
+
def test_composition_gate_regates_changed_block_url() -> None:
previous = _yaml(
{"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
@@ -1147,3 +1503,19 @@ def test_composition_gate_cross_turn_credit_requires_same_page_not_origin() -> N
)
exact.workflow_yaml = _yaml({"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"})
assert composition_page_evidence_error(exact, workflow_yaml) is None
+
+
+def test_composition_gate_matches_url_blocks_against_target_when_observation_ref_is_present() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {"block_type": "goto_url", "label": "open_cart", "url": "https://example.com/cart"},
+ )
+ ctx = _Ctx(
+ flow_evidence=[_flow_entry("https://example.com/")],
+ block_observation_refs={"open_cart": 0},
+ )
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is not None
+ assert "open_cart (goto_url)" in error
diff --git a/tests/unit/test_copilot_context.py b/tests/unit/test_copilot_context.py
index 33afb5f6c..75a79d102 100644
--- a/tests/unit/test_copilot_context.py
+++ b/tests/unit/test_copilot_context.py
@@ -4,7 +4,7 @@ from __future__ import annotations
from unittest.mock import MagicMock
-from skyvern.forge.sdk.copilot.context import StructuredContext
+from skyvern.forge.sdk.copilot.context import ObservedPage, StructuredContext, _merge_observed_acted_pages
def test_merge_turn_summary_caps_urls_visited() -> None:
@@ -34,6 +34,27 @@ def test_merge_turn_summary_caps_credentials_checked() -> None:
assert len(ctx.credentials_checked) == 40
+def test_merge_observed_acted_pages_uses_nested_evidence_url() -> None:
+ pages = _merge_observed_acted_pages(
+ [ObservedPage(url="https://example.com/old", had_bounded_schema=True, reached_via="navigate")],
+ [
+ {
+ "evidence": {
+ "current_url": "https://example.com/cart",
+ "inspected_url": "https://example.com/cart",
+ },
+ "had_bounded_schema": True,
+ "reached_via": "interaction",
+ "step": 3,
+ }
+ ],
+ )
+
+ by_url = {page.url: page for page in pages}
+ assert by_url["https://example.com/cart"].had_bounded_schema is True
+ assert by_url["https://example.com/cart"].reached_via == "interaction"
+
+
class TestCopilotContext:
def test_inherits_agent_context(self) -> None:
from skyvern.forge.sdk.copilot.context import CopilotContext
diff --git a/tests/unit/test_copilot_page_evidence_policy.py b/tests/unit/test_copilot_page_evidence_policy.py
index 15459217d..aae163f39 100644
--- a/tests/unit/test_copilot_page_evidence_policy.py
+++ b/tests/unit/test_copilot_page_evidence_policy.py
@@ -2,11 +2,15 @@
from __future__ import annotations
+import pytest
+from pydantic import ValidationError
+
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.copilot.config import DEFAULT_MAX_TURNS
from skyvern.forge.sdk.copilot.tools import (
_COMPOSITION_INSPECTION_PER_CHAT_BUDGET,
_COMPOSITION_INSPECTION_PER_TURN_BUDGET,
+ BlockObservationRef,
run_blocks_tool,
update_and_run_blocks_tool,
update_workflow_tool,
@@ -99,3 +103,8 @@ def test_inspection_budget_allows_multi_page_authoring_evidence_but_remains_boun
def test_default_loop_budget_allows_inspect_build_run_answer_trajectory() -> None:
assert DEFAULT_MAX_TURNS >= 35
+
+
+def test_block_observation_ref_rejects_negative_steps() -> None:
+ with pytest.raises(ValidationError):
+ BlockObservationRef(label="add_to_cart", observation_step=-1)