Skip to content

Automate browser based workflows with AI

SKY-10346: Harden page evidence boundaries (PR 3/12) (#6235)

Andrew Neilson3cbcd09b60df
diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.py
index 4f0f2e44c..3ead0018e 100644
--- a/skyvern/forge/sdk/copilot/composition_evidence.py
+++ b/skyvern/forge/sdk/copilot/composition_evidence.py
@@ -17,6 +17,9 @@ from skyvern.utils.yaml_loader import safe_load_no_dates
_BUILD_MODE_VALUES: frozenset[str] = frozenset({"build", "draft_only", "edit", "unknown"})
_PAGE_SHAPING_BLOCK_TYPES: frozenset[str] = frozenset({"navigation", "login"})
+_PAGE_READING_BLOCK_TYPES: frozenset[str] = frozenset({"extraction"})
+_SCHEMA_EVIDENCE_TOOL = "inspect_page_for_composition"
+_POST_RUN_CONTINUATION_EVIDENCE_TOOLS: frozenset[str] = frozenset({"inspect_page_for_composition"})
_RESULT_CONTAINER_HINTS: frozenset[str] = frozenset({"result", "results", "record", "records", "row", "rows"})
_MAX_FORMS = 5
_MAX_FIELDS_PER_FORM = 20
@@ -39,6 +42,7 @@ _ANTI_BOT_PATTERNS = (
)
_MAX_VISUAL_SUMMARY_CHARS = 500
_MAX_VISUAL_OMISSIONS = 5
+_ANTI_BOT_SCAN_BYTES = 250_000
def _bounded_string(value: Any, max_chars: int) -> str:
@@ -235,6 +239,27 @@ def workflow_target_url(workflow_yaml: str | None) -> str | None:
return None
+def _workflow_target_url_for_new_page_block(
+ workflow_yaml: str | None, previous_workflow_yaml: str | None
+) -> str | None:
+ previous = {
+ (str(block.get("label") or ""), str(block.get("block_type") or "").strip().lower())
+ for block in _parse_workflow_blocks(previous_workflow_yaml)
+ }
+ last_url: str | None = None
+ for block in _parse_workflow_blocks(workflow_yaml):
+ url = block.get("url")
+ if isinstance(url, str) and url.strip():
+ last_url = url.strip()
+ block_type = str(block.get("block_type") or "").strip().lower()
+ if block_type not in _PAGE_SHAPING_BLOCK_TYPES | _PAGE_READING_BLOCK_TYPES:
+ continue
+ label = str(block.get("label") or "")
+ if (label, block_type) not in previous:
+ return last_url
+ return workflow_target_url(workflow_yaml)
+
+
def _mode_requires_evidence(ctx: Any) -> bool:
mode_value = getattr(getattr(getattr(ctx, "turn_intent", None), "mode", None), "value", None)
phase = getattr(ctx, "build_phase", None)
@@ -266,7 +291,24 @@ def _new_page_shaping_blocks(workflow_yaml: str | None, previous_workflow_yaml:
return blocks
-def _format_page_shaping_findings(blocks: list[dict[str, str]]) -> str:
+def _new_page_reading_blocks(workflow_yaml: str | None, previous_workflow_yaml: str | None) -> list[dict[str, str]]:
+ previous = {
+ (str(block.get("label") or ""), str(block.get("block_type") or "").strip().lower())
+ for block in _parse_workflow_blocks(previous_workflow_yaml)
+ }
+ blocks: list[dict[str, str]] = []
+ for block in _parse_workflow_blocks(workflow_yaml):
+ block_type = str(block.get("block_type") or "").strip().lower()
+ if block_type not in _PAGE_READING_BLOCK_TYPES:
+ continue
+ label = str(block.get("label") or "<missing label>")
+ if (label, block_type) in previous:
+ continue
+ blocks.append({"label": label, "block_type": block_type})
+ return blocks
+
+
+def _format_page_block_findings(blocks: list[dict[str, str]]) -> str:
return ", ".join(f"{block['label']} ({block['block_type']})" for block in blocks[:5])
@@ -300,17 +342,29 @@ def _same_origin(left: str | None, right: str | None) -> bool:
return left_parsed.netloc.lower() == right_parsed.netloc.lower()
-def _evidence_matches_target(evidence: dict[str, Any] | None, target_url: str | None) -> bool:
+def _evidence_matches_target(
+ evidence: dict[str, Any] | None,
+ target_url: str | None,
+ *,
+ allow_post_run_browser_observation: bool = False,
+) -> bool:
if not evidence or not target_url:
return False
- if evidence.get("source_tool") != "inspect_page_for_composition":
- return False
+ source_tool = evidence.get("source_tool")
current_url = evidence.get("current_url")
inspected_url = evidence.get("inspected_url")
- return _same_page(current_url if isinstance(current_url, str) else None, target_url) or _same_page(
- inspected_url if isinstance(inspected_url, str) else None,
- target_url,
- )
+ current_url = current_url if isinstance(current_url, str) else None
+ inspected_url = inspected_url if isinstance(inspected_url, str) else None
+ if source_tool == _SCHEMA_EVIDENCE_TOOL:
+ if _same_page(current_url, target_url) or _same_page(inspected_url, target_url):
+ return True
+ if (
+ allow_post_run_browser_observation
+ and source_tool in _POST_RUN_CONTINUATION_EVIDENCE_TOOLS
+ and evidence.get("observed_after_workflow_run") is True
+ ):
+ return _same_origin(current_url, target_url) or _same_origin(inspected_url, target_url)
+ return False
def composition_page_evidence_error(ctx: Any, workflow_yaml: str | None) -> str | None:
@@ -324,21 +378,33 @@ def composition_page_evidence_error(ctx: Any, workflow_yaml: str | None) -> str
if not _mode_requires_evidence(ctx):
return None
- page_shaping_blocks = _new_page_shaping_blocks(workflow_yaml, getattr(ctx, "workflow_yaml", None))
- if not page_shaping_blocks:
+ previous_workflow_yaml = getattr(ctx, "workflow_yaml", None)
+ page_shaping_blocks = _new_page_shaping_blocks(workflow_yaml, previous_workflow_yaml)
+ page_reading_blocks = _new_page_reading_blocks(workflow_yaml, previous_workflow_yaml)
+ if not page_shaping_blocks and not page_reading_blocks:
return None
- target_url = workflow_target_url(workflow_yaml)
+ target_url = _workflow_target_url_for_new_page_block(workflow_yaml, previous_workflow_yaml)
evidence = getattr(ctx, "composition_page_evidence", None)
- if _evidence_matches_target(evidence, target_url):
- return None
- offending = _format_page_shaping_findings(page_shaping_blocks)
- return (
- "Workflow validation failed: page-dependent build blocks need observed page evidence before they are "
- f"authored. Call inspect_page_for_composition(target_url={target_url!r}) before composing navigation/login "
- "blocks, or save only the initial goto_url block and inspect the reached page before the next mutation. "
- f"Offending blocks: {offending}"
- )
+ allow_post_run_browser_observation = bool(previous_workflow_yaml)
+ if not _evidence_matches_target(
+ evidence,
+ target_url,
+ allow_post_run_browser_observation=allow_post_run_browser_observation,
+ ):
+ offending = _format_page_block_findings(page_shaping_blocks + page_reading_blocks)
+ return (
+ "Workflow validation failed: page-dependent build blocks need observed page evidence before they are "
+ f"authored. Call inspect_page_for_composition(target_url={target_url!r}) before composing page-dependent "
+ "blocks, or save only the initial goto_url block and inspect the reached page before the next mutation. "
+ f"Offending blocks: {offending}"
+ )
+
+ # Matched page evidence may ground a multi-block mutation. This gate enforces
+ # inspection before page-dependent composition; it does not prescribe a
+ # one-block-per-observation workflow construction style.
+
+ return None
def _empty_evidence(inspected_url: str, current_url: str) -> dict[str, Any]:
@@ -506,7 +572,11 @@ def _result_container_entry(node: Any) -> dict[str, Any]:
tag_name = str(getattr(node, "name", "") or "").lower()
node_id = str(node.get("id") or "")
selector = _selector_for(node)[:160]
- entry: dict[str, Any] = {"tag": tag_name, "id": node_id[:120], "selector": selector}
+ entry: dict[str, Any] = {
+ "tag": tag_name,
+ "id": node_id[:120],
+ "selector": selector,
+ }
if tag_name == "table":
entry["row_selector"] = f"{selector} tbody tr"
entry["expand_toggle_candidates"] = [
@@ -519,58 +589,60 @@ def _result_container_entry(node: Any) -> dict[str, Any]:
return entry
-def _challenge_identity(node: Any) -> str:
- parts = [
- str(getattr(node, "name", "") or ""),
- _attr_value(node, "id"),
- _attr_value(node, "name"),
- " ".join(_classes_for(node)),
- _attr_value(node, "src"),
- # Standard anti-bot widgets commonly expose this attribute.
- _attr_value(node, "data-sitekey"),
- _attr_value(node, "aria-label"),
- _attr_value(node, "title"),
- ]
- return " ".join(part for part in parts if part).lower()
-
-
-def _challenge_entry(node: Any) -> dict[str, Any]:
- return {
- "tag": str(getattr(node, "name", "") or "")[:40],
+def _challenge_control_entry(node: Any) -> dict[str, Any]:
+ tag_name = str(getattr(node, "name", "") or "").lower()
+ entry: dict[str, Any] = {
+ "tag": tag_name,
"id": _attr_value(node, "id")[:120],
"name": _attr_value(node, "name")[:120],
"class": " ".join(_classes_for(node)[:5])[:160],
"type": _attr_value(node, "type")[:40],
"selector": _selector_for(node)[:160],
+ "text": _schema_text(_node_text(node) or _attr_value(node, "aria-label"), 200),
}
+ for key in ("src", "title", "data-sitekey", "data-callback", "data-expired-callback", "data-error-callback"):
+ value = _attr_value(node, key)
+ if value:
+ entry[key.replace("-", "_")] = value[:300]
+ return {key: value for key, value in entry.items() if value}
-def _anti_bot_evidence(nodes: list[Any], html: str) -> tuple[list[str], list[dict[str, Any]]]:
- indicators: list[str] = []
-
- def add_indicator(value: str) -> None:
- if value not in indicators:
- indicators.append(value)
-
- html_prefix = (html or "").lower()[:4096]
- for pattern in _ANTI_BOT_PATTERNS:
- if pattern in html_prefix:
- add_indicator(pattern)
+def _challenge_controls(soup: Any) -> list[dict[str, Any]]:
+ controls: list[dict[str, Any]] = []
+ seen_selectors: set[str] = set()
+ for node in soup.find_all(True):
+ if len(controls) >= _MAX_CHALLENGE_CONTROLS:
+ break
+ identity = " ".join(
+ str(value or "")
+ for value in (
+ getattr(node, "name", ""),
+ _attr_value(node, "id"),
+ _attr_value(node, "name"),
+ _attr_value(node, "class"),
+ _attr_value(node, "src"),
+ _attr_value(node, "type"),
+ _attr_value(node, "data-sitekey"),
+ _attr_value(node, "data-callback"),
+ _attr_value(node, "data-expired-callback"),
+ _attr_value(node, "data-error-callback"),
+ _attr_value(node, "aria-label"),
+ _attr_value(node, "title"),
+ )
+ ).lower()
+ if not any(pattern in identity for pattern in _ANTI_BOT_PATTERNS):
+ continue
+ selector = _selector_for(node)[:160]
+ if selector in seen_selectors:
+ continue
+ seen_selectors.add(selector)
+ controls.append(_challenge_control_entry(node))
+ return controls
- challenge_controls: list[dict[str, Any]] = []
- for node in nodes:
- identity = _challenge_identity(node)
- matched = False
- for pattern in _ANTI_BOT_PATTERNS:
- if pattern in identity:
- add_indicator(pattern)
- matched = True
- tag_name = str(getattr(node, "name", "") or "").lower()
- if matched and tag_name in {"div", "iframe", "input", "button", "textarea"}:
- if len(challenge_controls) < _MAX_CHALLENGE_CONTROLS:
- challenge_controls.append(_challenge_entry(node))
- return indicators[:8], challenge_controls
+def _anti_bot_indicators(html: str, page_title: str) -> list[str]:
+ haystack = f"{page_title}\n{html[:_ANTI_BOT_SCAN_BYTES]}".lower()
+ return [pattern for pattern in _ANTI_BOT_PATTERNS if pattern in haystack]
def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -> dict[str, Any]:
@@ -582,6 +654,12 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
soup = BeautifulSoup(html or "", "html.parser")
except Exception:
return _empty_evidence(inspected_url, current_url)
+ page_title = _page_title(soup)
+ challenge_controls = _challenge_controls(soup)
+ anti_bot_indicators = _anti_bot_indicators(html or "", page_title)
+
+ for node in soup.find_all(["script", "style", "noscript"]):
+ node.decompose()
all_nodes = soup.find_all(True)
@@ -674,11 +752,10 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
control_count = sum(len(form.get("submit_controls") or []) for form in forms)
# Higher confidence means the parser saw a more complete form surface.
confidence = 0.85 if field_count and control_count else 0.6 if field_count else 0.3 if forms else 0.1
- anti_bot_indicators, challenge_controls = _anti_bot_evidence(all_nodes, html)
return {
"inspected_url": inspected_url,
"current_url": current_url,
- "page_title": _page_title(soup),
+ "page_title": page_title,
"forms": forms,
"navigation_targets": navigation_targets,
"result_containers": result_containers,
diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.py
index b38da94f7..09579a105 100644
--- a/tests/unit/test_copilot_composition_evidence.py
+++ b/tests/unit/test_copilot_composition_evidence.py
@@ -238,6 +238,24 @@ def test_composition_gate_requires_page_evidence_before_page_dependent_blocks()
assert "search_lookup" in error
+def test_composition_gate_names_extraction_only_blocks_missing_evidence() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_results", "url": "https://example.com/results"},
+ {
+ "block_type": "extraction",
+ "label": "extract_results",
+ "data_extraction_goal": "Extract the visible result rows.",
+ },
+ )
+
+ error = composition_page_evidence_error(_Ctx(), workflow_yaml)
+
+ assert error is not None
+ assert "page-dependent blocks" in error
+ assert "navigation/login" not in error
+ assert "extract_results (extraction)" in error
+
+
def test_composition_gate_rejects_stale_page_evidence_from_another_origin() -> None:
workflow_yaml = _yaml(
{"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
@@ -302,6 +320,343 @@ def test_composition_gate_rejects_untyped_browser_observation_evidence() -> None
assert "inspect_page_for_composition" in error
+def test_composition_gate_rejects_precompose_screenshot_evidence_outside_inspection_tool() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "search_lookup",
+ "navigation_goal": "Enter the observed first-name and last-name fields and submit.",
+ },
+ )
+ evidence = {
+ "inspected_url": "https://example.com/lookup",
+ "current_url": "https://example.com/lookup",
+ "forms": [],
+ "source_tool": "get_browser_screenshot",
+ "evidence_sources": ["screenshot", "vision_summary"],
+ "screenshot_used": True,
+ "visual_evidence_summary": "A search form is visible.",
+ }
+
+ error = composition_page_evidence_error(_Ctx(composition_page_evidence=evidence), workflow_yaml)
+
+ assert error is not None
+ assert "inspect_page_for_composition" in error
+
+
+def test_composition_gate_accepts_screenshot_evidence_from_inspection_tool() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "search_lookup",
+ "navigation_goal": "Enter the observed first-name and last-name fields and submit.",
+ },
+ )
+ evidence = {
+ **_first_last_evidence(),
+ "evidence_sources": ["dom_html", "screenshot", "vision_summary"],
+ "screenshot_used": True,
+ "visual_evidence_summary": "A challenge is visible below the search form.",
+ }
+
+ error = composition_page_evidence_error(_Ctx(composition_page_evidence=evidence), workflow_yaml)
+
+ assert error is None
+
+
+def test_composition_gate_rejects_post_run_browser_observation_outside_inspection_tool() -> None:
+ existing_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "search_lookup",
+ "navigation_goal": "Enter the observed First Name and Last Name fields and submit.",
+ },
+ )
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "search_lookup",
+ "navigation_goal": "Enter the observed First Name and Last Name fields and submit.",
+ },
+ {
+ "block_type": "navigation",
+ "label": "expand_result",
+ "navigation_goal": "Click the observed result-row expansion control.",
+ },
+ )
+ evidence = {
+ "inspected_url": "https://example.com/results?id=123",
+ "current_url": "https://example.com/results?id=123",
+ "forms": [],
+ "source_tool": "get_browser_screenshot",
+ "observed_after_workflow_run": True,
+ }
+ ctx = _Ctx(composition_page_evidence=evidence)
+ ctx.workflow_yaml = existing_yaml # type: ignore[attr-defined]
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is not None
+ assert "inspect_page_for_composition" in error
+
+
+def test_composition_gate_allows_post_run_current_page_schema_on_same_origin_continuation() -> None:
+ existing_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "search_lookup",
+ "navigation_goal": "Enter the observed First Name and Last Name fields and submit.",
+ },
+ )
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "search_lookup",
+ "navigation_goal": "Enter the observed First Name and Last Name fields and submit.",
+ },
+ {
+ "block_type": "navigation",
+ "label": "expand_result",
+ "navigation_goal": "Click the observed result-row expansion control.",
+ },
+ )
+ evidence = {
+ "inspected_url": "current_page",
+ "current_url": "https://example.com/results?id=123",
+ "forms": [],
+ "source_tool": "inspect_page_for_composition",
+ "observed_after_workflow_run": True,
+ }
+ ctx = _Ctx(composition_page_evidence=evidence)
+ ctx.workflow_yaml = existing_yaml # type: ignore[attr-defined]
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is None
+
+
+def test_composition_gate_allows_multiple_new_page_changing_blocks_from_one_observation() -> None:
+ existing_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "open_form",
+ "navigation_goal": "Open the observed lookup form.",
+ },
+ )
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "open_form",
+ "navigation_goal": "Open the observed lookup form.",
+ },
+ {
+ "block_type": "navigation",
+ "label": "submit_search",
+ "navigation_goal": "Fill the observed first-name and last-name fields and submit.",
+ },
+ {
+ "block_type": "navigation",
+ "label": "expand_result",
+ "navigation_goal": "Click the result-row expansion control.",
+ },
+ )
+ evidence = {
+ "inspected_url": "current_page",
+ "current_url": "https://example.com/lookup",
+ "forms": [],
+ "source_tool": "inspect_page_for_composition",
+ "observed_after_workflow_run": True,
+ }
+ ctx = _Ctx(composition_page_evidence=evidence)
+ ctx.workflow_yaml = existing_yaml # type: ignore[attr-defined]
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is None
+
+
+def test_composition_gate_allows_extraction_added_with_new_page_changing_block() -> None:
+ existing_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "submit_search",
+ "navigation_goal": "Fill the observed search form and submit.",
+ },
+ )
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "submit_search",
+ "navigation_goal": "Fill the observed search form and submit.",
+ },
+ {
+ "block_type": "navigation",
+ "label": "expand_result",
+ "navigation_goal": "Click the observed result-row expansion control.",
+ },
+ {
+ "block_type": "extraction",
+ "label": "extract_expanded_result",
+ "data_extraction_goal": "Extract the values visible in the expanded row.",
+ },
+ )
+ evidence = {
+ "inspected_url": "current_page",
+ "current_url": "https://example.com/results",
+ "result_containers": [{"selector": "#results"}],
+ "source_tool": "inspect_page_for_composition",
+ "observed_after_workflow_run": True,
+ }
+ ctx = _Ctx(composition_page_evidence=evidence)
+ ctx.workflow_yaml = existing_yaml # type: ignore[attr-defined]
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is None
+
+
+def test_composition_gate_allows_extraction_after_matching_current_page_evidence() -> None:
+ existing_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "submit_search",
+ "navigation_goal": "Fill the observed search form and submit.",
+ },
+ )
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "submit_search",
+ "navigation_goal": "Fill the observed search form and submit.",
+ },
+ {
+ "block_type": "extraction",
+ "label": "extract_visible_results",
+ "data_extraction_goal": "Extract the values visible on the observed results page.",
+ },
+ )
+ evidence = {
+ "inspected_url": "current_page",
+ "current_url": "https://example.com/results",
+ "result_containers": [{"selector": "#results"}],
+ "source_tool": "inspect_page_for_composition",
+ "observed_after_workflow_run": True,
+ }
+ ctx = _Ctx(composition_page_evidence=evidence)
+ ctx.workflow_yaml = existing_yaml # type: ignore[attr-defined]
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is None
+
+
+def test_composition_gate_targets_nearest_url_before_new_page_block() -> None:
+ existing_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {
+ "block_type": "goto_url",
+ "label": "open_find_record",
+ "url": "https://example.com/registry/",
+ },
+ )
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {
+ "block_type": "goto_url",
+ "label": "open_find_record",
+ "url": "https://example.com/registry/",
+ },
+ {
+ "block_type": "navigation",
+ "label": "search_standard_record",
+ "url": "https://example.com/registry/search",
+ "navigation_goal": "Enter the observed First Name and Last Name fields and submit.",
+ },
+ )
+ evidence = {
+ "inspected_url": "https://example.com/registry/search",
+ "current_url": "https://example.com/registry/search",
+ "forms": [],
+ "source_tool": "inspect_page_for_composition",
+ }
+ ctx = _Ctx(composition_page_evidence=evidence)
+ ctx.workflow_yaml = existing_yaml # type: ignore[attr-defined]
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is None
+
+
+def test_composition_gate_error_names_nearest_url_before_new_page_block() -> None:
+ existing_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {
+ "block_type": "goto_url",
+ "label": "open_find_record",
+ "url": "https://example.com/registry/",
+ },
+ )
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {
+ "block_type": "goto_url",
+ "label": "open_find_record",
+ "url": "https://example.com/registry/",
+ },
+ {
+ "block_type": "navigation",
+ "label": "search_standard_record",
+ "url": "https://example.com/registry/search",
+ "navigation_goal": "Enter the observed First Name and Last Name fields and submit.",
+ },
+ )
+ ctx = _Ctx(composition_page_evidence=None)
+ ctx.workflow_yaml = existing_yaml # type: ignore[attr-defined]
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is not None
+ assert "target_url='https://example.com/registry/search'" in error
+
+
+def test_composition_gate_rejects_same_origin_browser_observation_before_run_continuation() -> None:
+ existing_yaml = _yaml({"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"})
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "search_lookup",
+ "navigation_goal": "Enter the observed First Name and Last Name fields and submit.",
+ },
+ )
+ evidence = {
+ "inspected_url": "https://example.com/lookup",
+ "current_url": "https://example.com/lookup",
+ "forms": [],
+ "source_tool": "get_browser_screenshot",
+ "observed_after_workflow_run": False,
+ }
+ ctx = _Ctx(composition_page_evidence=evidence)
+ ctx.workflow_yaml = existing_yaml # type: ignore[attr-defined]
+
+ error = composition_page_evidence_error(ctx, workflow_yaml)
+
+ assert error is not None
+ assert "inspect_page_for_composition" in error
+
+
def test_composition_gate_applies_to_edit_turns_that_add_page_dependent_blocks() -> None:
existing_yaml = _yaml(
{"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
@@ -348,9 +703,37 @@ def test_composition_gate_allows_navigation_after_matching_evidence() -> None:
"block_type": "navigation",
"label": "search_lookup",
"navigation_goal": (
- "Enter the first name into the observed First Name field, enter the last name into the "
- "observed Last Name field, then click the observed Search button."
+ "Enter the observed First Name and Last Name fields, then click the observed Search button."
),
+ "complete_criterion": "The search results table is visible.",
+ },
+ )
+
+ error = composition_page_evidence_error(_Ctx(composition_page_evidence=_first_last_evidence()), workflow_yaml)
+
+ assert error is None
+
+
+def test_composition_gate_allows_separate_form_state_and_submit_blocks_from_one_observation() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},
+ {
+ "block_type": "navigation",
+ "label": "prepare_lookup",
+ "navigation_goal": (
+ "Enter the observed First Name and Last Name fields. Stop with the Search button visible; "
+ "do not submit the form."
+ ),
+ },
+ {
+ "block_type": "navigation",
+ "label": "submit_lookup",
+ "navigation_goal": "Click the observed Search button and wait for the result page.",
+ },
+ {
+ "block_type": "extraction",
+ "label": "extract_results",
+ "data_extraction_goal": "Extract the credential rows from the result page.",
},
)