Skip to content

Automate browser based workflows with AI

test(SKY-12085): prove dispatched terminal-page capture reaches judgment verdicts (#7263)

Andrew Neilson552813f8d2af
diff --git a/tests/unit/copilot_test_helpers.py b/tests/unit/copilot_test_helpers.py
index 5289b90c5..32937f65e 100644
--- a/tests/unit/copilot_test_helpers.py
+++ b/tests/unit/copilot_test_helpers.py
@@ -1,10 +1,15 @@
-"""Shared builders for copilot unit tests (CopilotContext + verified-goal contract)."""
+"""Shared builders for copilot unit tests."""
from __future__ import annotations
import json
+from datetime import datetime, timezone
+from types import SimpleNamespace
from unittest.mock import MagicMock
+import pytest
+
+from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisInput,
@@ -15,6 +20,93 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
VerificationResult,
)
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion
+from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
+
+DISPATCHED_LOGIN_GATE_HTML = (
+ "<html><head><title>Sign in</title></head><body><main>"
+ "<h1>Sign in to continue</h1>"
+ '<form id="signin" action="/session" method="post">'
+ '<label for="account-email">Email</label>'
+ '<input id="account-email" name="email" type="email" required />'
+ '<label for="account-password">Password</label>'
+ '<input id="account-password" name="password" type="password" required />'
+ '<button type="submit">Sign in</button>'
+ "</form></main></body></html>"
+)
+DISPATCHED_RESULTS_HTML = (
+ "<html><head><title>Available providers</title></head><body><main>"
+ "<h1>Available providers</h1>"
+ '<table id="provider-results"><tbody>'
+ "<tr><td>Example Fiber</td><td>up to 500 Mbps</td></tr>"
+ "<tr><td>Example Cable</td><td>up to 300 Mbps</td></tr>"
+ "</tbody></table></main></body></html>"
+)
+DISPATCHED_NAV_ONLY_HTML = (
+ "<html><head><title>Site map</title></head><body><main>"
+ '<a href="https://example.test/plans">Plans</a>'
+ '<a href="https://example.test/support">Support</a>'
+ "</main></body></html>"
+)
+
+
+def make_stub_artifact(
+ artifact_id: str,
+ file_name: str,
+ file_size: int | None,
+ artifact_type: ArtifactType = ArtifactType.DOWNLOAD,
+) -> SimpleNamespace:
+ return SimpleNamespace(
+ artifact_id=artifact_id,
+ uri=f"s3://bucket/{file_name}",
+ file_size=file_size,
+ artifact_type=artifact_type,
+ )
+
+
+def make_stub_html_artifact(
+ artifact_id: str,
+ artifact_type: ArtifactType,
+ file_size: int | None = 400,
+ created_at: datetime | None = None,
+) -> SimpleNamespace:
+ artifact = make_stub_artifact(artifact_id, f"{artifact_id}.html", file_size, artifact_type=artifact_type)
+ artifact.created_at = created_at or datetime(2026, 7, 9, tzinfo=timezone.utc)
+ return artifact
+
+
+def stub_artifact_app(
+ monkeypatch: pytest.MonkeyPatch,
+ artifacts: list[SimpleNamespace],
+ retrieved: dict[str, bytes],
+ *,
+ by_ids: list[SimpleNamespace] | None = None,
+) -> list[str]:
+ retrieved_ids: list[str] = []
+
+ async def fake_get_artifacts_for_run(
+ run_id: str, *, organization_id: str, artifact_types: object
+ ) -> list[SimpleNamespace]:
+ return artifacts
+
+ async def fake_get_artifacts_by_ids(artifact_ids: list[str], *, organization_id: str) -> list[SimpleNamespace]:
+ pool = {artifact.artifact_id: artifact for artifact in (by_ids if by_ids is not None else artifacts)}
+ return [pool[artifact_id] for artifact_id in artifact_ids if artifact_id in pool]
+
+ async def fake_retrieve_artifact(artifact: SimpleNamespace) -> bytes:
+ retrieved_ids.append(artifact.artifact_id)
+ return retrieved.get(artifact.artifact_id, b"")
+
+ fake_app = SimpleNamespace(
+ DATABASE=SimpleNamespace(
+ artifacts=SimpleNamespace(
+ get_artifacts_for_run=fake_get_artifacts_for_run,
+ get_artifacts_by_ids=fake_get_artifacts_by_ids,
+ )
+ ),
+ ARTIFACT_MANAGER=SimpleNamespace(retrieve_artifact=fake_retrieve_artifact),
+ )
+ monkeypatch.setattr(run_execution_module, "app", fake_app)
+ return retrieved_ids
def make_copilot_ctx(**overrides: object) -> CopilotContext:
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index f15923b01..edfd17018 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -12,6 +12,7 @@ import pytest
from structlog.testing import capture_logs
from skyvern.config import settings
+from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot.agent import (
_completion_contract_not_violated,
_rewrite_failed_test_response,
@@ -103,6 +104,7 @@ from skyvern.forge.sdk.copilot.tools import (
_tool_visible_result_after_completion_verification,
_watchdog_exit_allows_terminal_promotion,
)
+from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
from skyvern.forge.sdk.copilot.tools._shared import (
_TASK_ENVELOPE_BLOCK_TYPES,
_has_meaningful_registered_output_payload,
@@ -118,7 +120,16 @@ from skyvern.forge.sdk.copilot.tools.workflow_update import (
_apply_code_artifact_requested_output_evidence_sources,
_normalize_code_artifact_metadata,
)
+from tests.unit.copilot_test_helpers import (
+ DISPATCHED_LOGIN_GATE_HTML,
+ DISPATCHED_NAV_ONLY_HTML,
+ DISPATCHED_RESULTS_HTML,
+)
from tests.unit.copilot_test_helpers import make_completion_criterion as _criterion
+from tests.unit.copilot_test_helpers import (
+ make_stub_html_artifact,
+ stub_artifact_app,
+)
_STRUCTURED_RECORD_CRITERIA = (
("fallback_record_identity", "The returned record identifies the target record."),
@@ -10310,6 +10321,108 @@ def test_judgment_truth_condition_refutes_inverted_login_gate_packet() -> None:
assert verdicts[0].evidence_source == "independent_page_evidence"
+def _dispatched_judgment_criterion() -> CompletionCriterion:
+ return replace(
+ _criterion(
+ "c_login_gate",
+ "The returned record reports whether the login gate blocks the target.",
+ output_path="output.login_gate_blocks_target",
+ expected_output_value=True,
+ requested_output_evidence_source="independent_run_evidence",
+ ),
+ judgment_truth_condition=JudgmentTruthCondition(predicate="login_gate_blocks_target", polarity_when_holds=True),
+ )
+
+
+async def _dispatched_chain_snapshot(
+ monkeypatch: pytest.MonkeyPatch, ctx: CopilotContext, html: str
+) -> RunEvidenceSnapshot:
+ stub_artifact_app(
+ monkeypatch,
+ [make_stub_html_artifact("art_terminal", ArtifactType.HTML_ACTION)],
+ {"art_terminal": html.encode()},
+ )
+ await run_execution_module._capture_dispatched_terminal_page_evidence(
+ ctx, run_id="wr_requested_output", organization_id="o", current_url=""
+ )
+ return _build_run_evidence_snapshot(ctx, _requested_output_result({"login_gate_blocks_target": True}))
+
+
+async def test_dispatched_chain_login_gate_packet_confirms_judgment_boolean(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("login_gate_blocks_target")
+ snapshot = await _dispatched_chain_snapshot(monkeypatch, ctx, DISPATCHED_LOGIN_GATE_HTML)
+
+ with capture_logs() as logs:
+ verdicts = grade_requested_output_criteria(ctx, [_dispatched_judgment_criterion()], snapshot)
+
+ bound = snapshot.block_outputs[_POST_RUN_PAGE_OBSERVATION_LABEL]
+ assert snapshot.block_output_sources[_POST_RUN_PAGE_OBSERVATION_LABEL] == "independent_page_evidence"
+ assert bound["forms"]
+ assert verdicts[0].state == "satisfied"
+ assert verdicts[0].reason_code == "evidence_confirms"
+ assert verdicts[0].evidence_ref == f"block_outputs:{_POST_RUN_PAGE_OBSERVATION_LABEL}"
+ assert verdicts[0].evidence_source == "independent_page_evidence"
+ assert any(
+ log["event"] == "copilot_judgment_evidence_verdict"
+ and log["predicate"] == "login_gate_blocks_target"
+ and log["packet_label"] == _POST_RUN_PAGE_OBSERVATION_LABEL
+ and log["verdict"] == "evidence_confirms"
+ for log in logs
+ )
+
+
+async def test_dispatched_chain_result_container_packet_contradicts_login_gate(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("login_gate_blocks_target")
+ snapshot = await _dispatched_chain_snapshot(monkeypatch, ctx, DISPATCHED_RESULTS_HTML)
+
+ with capture_logs() as logs:
+ verdicts = grade_requested_output_criteria(ctx, [_dispatched_judgment_criterion()], snapshot)
+
+ bound = snapshot.block_outputs[_POST_RUN_PAGE_OBSERVATION_LABEL]
+ assert bound["result_containers"]
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "evidence_contradicts"
+ assert verdicts[0].evidence_ref == f"block_outputs:{_POST_RUN_PAGE_OBSERVATION_LABEL}"
+ assert verdicts[0].evidence_source == "independent_page_evidence"
+ assert all(verdict.reason_code != "structurally_abstained" for verdict in verdicts)
+ assert any(
+ log["event"] == "copilot_judgment_evidence_verdict"
+ and log["predicate"] == "login_gate_blocks_target"
+ and log["verdict"] == "evidence_contradicts"
+ for log in logs
+ )
+
+
+async def test_dispatched_chain_nav_only_page_abstains(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Dispatched captures parse with no current URL, so the same-origin filter drops every
+ # navigation target and a nav-only terminal page yields an undecidable hollow packet.
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("login_gate_blocks_target")
+ snapshot = await _dispatched_chain_snapshot(monkeypatch, ctx, DISPATCHED_NAV_ONLY_HTML)
+
+ verdicts = grade_requested_output_criteria(ctx, [_dispatched_judgment_criterion()], snapshot)
+
+ bound = snapshot.block_outputs[_POST_RUN_PAGE_OBSERVATION_LABEL]
+ assert bound["navigation_targets"] == []
+ assert bound["forms"] == []
+ assert bound["result_containers"] == []
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "structurally_abstained"
+ assert verdicts[0].evidence_source == "independent_page_evidence"
+
+
def test_judgment_truth_condition_hollow_packet_abstains_before_self_emitted_output() -> None:
ctx = _run_ctx()
ctx.code_artifact_metadata = _metadata_for_requested_paths("login_gate_blocks_target")
diff --git a/tests/unit/test_copilot_positive_evidence_carriers.py b/tests/unit/test_copilot_positive_evidence_carriers.py
index 2458337c9..46be5f78c 100644
--- a/tests/unit/test_copilot_positive_evidence_carriers.py
+++ b/tests/unit/test_copilot_positive_evidence_carriers.py
@@ -27,7 +27,13 @@ from skyvern.forge.sdk.copilot.runtime import (
)
from skyvern.forge.sdk.copilot.tools import completion as completion_module
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
-from tests.unit.copilot_test_helpers import make_completion_criterion
+from tests.unit.copilot_test_helpers import (
+ DISPATCHED_NAV_ONLY_HTML,
+ make_completion_criterion,
+)
+from tests.unit.copilot_test_helpers import make_stub_artifact as _artifact
+from tests.unit.copilot_test_helpers import make_stub_html_artifact as _html_artifact
+from tests.unit.copilot_test_helpers import stub_artifact_app as _stub_app
_POST_RUN_LABEL = "post_run_page_observation"
_ARTIFACT_LABEL = "registered_artifact_observation"
@@ -373,55 +379,6 @@ def test_parse_registered_artifact_text_pdf_cleans_temp_file_on_exception(monkey
assert not os.path.exists(captured["path"])
-def _artifact(
- artifact_id: str,
- file_name: str,
- file_size: int | None,
- artifact_type: ArtifactType = ArtifactType.DOWNLOAD,
-) -> SimpleNamespace:
- return SimpleNamespace(
- artifact_id=artifact_id,
- uri=f"s3://bucket/{file_name}",
- file_size=file_size,
- artifact_type=artifact_type,
- )
-
-
-def _stub_app(
- monkeypatch: pytest.MonkeyPatch,
- artifacts: list[SimpleNamespace],
- retrieved: dict[str, bytes],
- *,
- by_ids: list[SimpleNamespace] | None = None,
-) -> list[str]:
- retrieved_ids: list[str] = []
-
- async def fake_get_artifacts_for_run(
- run_id: str, *, organization_id: str, artifact_types: object
- ) -> list[SimpleNamespace]:
- return artifacts
-
- async def fake_get_artifacts_by_ids(artifact_ids: list[str], *, organization_id: str) -> list[SimpleNamespace]:
- pool = {artifact.artifact_id: artifact for artifact in (by_ids if by_ids is not None else artifacts)}
- return [pool[artifact_id] for artifact_id in artifact_ids if artifact_id in pool]
-
- async def fake_retrieve_artifact(artifact: SimpleNamespace) -> bytes:
- retrieved_ids.append(artifact.artifact_id)
- return retrieved.get(artifact.artifact_id, b"")
-
- fake_app = SimpleNamespace(
- DATABASE=SimpleNamespace(
- artifacts=SimpleNamespace(
- get_artifacts_for_run=fake_get_artifacts_for_run,
- get_artifacts_by_ids=fake_get_artifacts_by_ids,
- )
- ),
- ARTIFACT_MANAGER=SimpleNamespace(retrieve_artifact=fake_retrieve_artifact),
- )
- monkeypatch.setattr(run_execution_module, "app", fake_app)
- return retrieved_ids
-
-
@pytest.mark.asyncio
async def test_artifact_producer_binds_parsed_text(monkeypatch: pytest.MonkeyPatch) -> None:
artifacts = [_artifact("art_1", "receipt.txt", 20)]
@@ -554,17 +511,6 @@ _HTML_NO_VALUE = (
_HTML_SCRAPE_PREACTION = "<html><body><main><p>SCRAPEONLYTOKEN loading form</p></main></body></html>"
-def _html_artifact(
- artifact_id: str,
- artifact_type: ArtifactType,
- file_size: int | None = 400,
- created_at: datetime | None = None,
-) -> SimpleNamespace:
- artifact = _artifact(artifact_id, f"{artifact_id}.html", file_size, artifact_type=artifact_type)
- artifact.created_at = created_at or datetime(2026, 7, 9, tzinfo=timezone.utc)
- return artifact
-
-
def _producer_ctx(pre_run_prose: str | None = "Submit your request below.") -> SimpleNamespace:
baseline = {"visible_text_excerpt": pre_run_prose} if pre_run_prose is not None else None
return SimpleNamespace(
@@ -785,3 +731,72 @@ async def test_dispatched_producer_abstains_without_terminal_artifact(monkeypatc
ctx, run_id="wr_disp", organization_id="o_1", current_url=""
)
assert ctx.composition_page_evidence is None
+
+
+_HTML_FORM_AND_RESULTS = (
+ "<html><head><title>Find a provider</title></head><body><main>"
+ '<form id="finder" action="/find" method="get">'
+ '<label for="zip">ZIP code</label>'
+ '<input id="zip" name="zip" type="text" required />'
+ '<input id="account-password" name="password" type="password" />'
+ '<button type="submit">Search</button>'
+ "</form>"
+ '<table id="provider-results"><tbody>'
+ "<tr><td>Example Fiber</td><td>up to 500 Mbps</td></tr>"
+ "<tr><td>Example Cable</td><td>up to 300 Mbps</td></tr>"
+ "</tbody></table></main></body></html>"
+)
+_HTML_DISABLED_SUBMIT = (
+ "<html><body><main>"
+ '<form id="apply" action="/apply" method="post">'
+ '<input id="account-email" name="email" type="email" />'
+ '<input id="account-password" name="password" type="password" />'
+ '<button type="submit" disabled>Submit</button>'
+ "</form></main></body></html>"
+)
+
+
+async def _dispatched_packet(monkeypatch: pytest.MonkeyPatch, html: str) -> dict[str, object]:
+ _stub_app(monkeypatch, [_html_artifact("art_page", ArtifactType.HTML_ACTION)], {"art_page": html.encode()})
+ ctx = _producer_ctx()
+ await run_execution_module._capture_dispatched_terminal_page_evidence(
+ ctx, run_id="wr_disp", organization_id="o_1", current_url=""
+ )
+ assert ctx.composition_page_evidence is not None
+ return ctx.composition_page_evidence
+
+
+async def test_dispatched_packet_carries_forms_and_result_containers(monkeypatch: pytest.MonkeyPatch) -> None:
+ packet = await _dispatched_packet(monkeypatch, _HTML_FORM_AND_RESULTS)
+ forms = packet["forms"]
+ assert forms
+ assert any(field.get("type") == "password" for field in forms[0]["fields"])
+ assert any(control.get("type") == "submit" for control in forms[0]["submit_controls"])
+ containers = packet["result_containers"]
+ assert containers
+ assert any(container.get("selector") for container in containers)
+
+
+async def test_dispatched_packet_drops_navigation_targets_without_current_url(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Dispatched captures carry no current URL, so the same-origin link filter drops every target.
+ packet = await _dispatched_packet(monkeypatch, DISPATCHED_NAV_ONLY_HTML)
+ assert packet["navigation_targets"] == []
+ assert packet["forms"] == []
+ assert packet["result_containers"] == []
+
+
+async def test_dispatched_packet_carries_static_disabled_submit_control(monkeypatch: pytest.MonkeyPatch) -> None:
+ packet = await _dispatched_packet(monkeypatch, _HTML_DISABLED_SUBMIT)
+ controls = packet["forms"][0]["submit_controls"]
+ assert controls
+ assert controls[0]["disabled"] is True
+ # Without challenge indicators the static parse never claims challenge gating; the
+ # literal disabled attribute above is the only gating signal a dispatched packet carries.
+ challenge_state = packet["challenge_state"]
+ assert challenge_state["gates_submit_controls"] is False
+ assert challenge_state["gated_submit_controls"] == []