Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
test(SKY-12085): prove dispatched terminal-page capture reaches judgment verdicts (#7263)
Andrew Neilson552813f8d2af
diff --git a/tests/unit/copilot_test_helpers.py b/tests/unit/copilot_test_helpers.pyindex 5289b90c5..32937f65e 100644--- a/tests/unit/copilot_test_helpers.py+++ b/tests/unit/copilot_test_helpers.py@@ -1,10 +1,15 @@-"""Shared builders for copilot unit tests (CopilotContext + verified-goal contract)."""+"""Shared builders for copilot unit tests."""from __future__ import annotationsimport json+from datetime import datetime, timezone+from types import SimpleNamespacefrom unittest.mock import MagicMock+import pytest++from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (DiagnosisInput,@@ -15,6 +20,93 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (VerificationResult,)from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion+from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module++DISPATCHED_LOGIN_GATE_HTML = (+ "<html><head><title>Sign in</title></head><body><main>"+ "<h1>Sign in to continue</h1>"+ '<form id="signin" action="/session" method="post">'+ '<label for="account-email">Email</label>'+ '<input id="account-email" name="email" type="email" required />'+ '<label for="account-password">Password</label>'+ '<input id="account-password" name="password" type="password" required />'+ '<button type="submit">Sign in</button>'+ "</form></main></body></html>"+)+DISPATCHED_RESULTS_HTML = (+ "<html><head><title>Available providers</title></head><body><main>"+ "<h1>Available providers</h1>"+ '<table id="provider-results"><tbody>'+ "<tr><td>Example Fiber</td><td>up to 500 Mbps</td></tr>"+ "<tr><td>Example Cable</td><td>up to 300 Mbps</td></tr>"+ "</tbody></table></main></body></html>"+)+DISPATCHED_NAV_ONLY_HTML = (+ "<html><head><title>Site map</title></head><body><main>"+ '<a href="https://example.test/plans">Plans</a>'+ '<a href="https://example.test/support">Support</a>'+ "</main></body></html>"+)+++def make_stub_artifact(+ artifact_id: str,+ file_name: str,+ file_size: int | None,+ artifact_type: ArtifactType = ArtifactType.DOWNLOAD,+) -> SimpleNamespace:+ return SimpleNamespace(+ artifact_id=artifact_id,+ uri=f"s3://bucket/{file_name}",+ file_size=file_size,+ artifact_type=artifact_type,+ )+++def make_stub_html_artifact(+ artifact_id: str,+ artifact_type: ArtifactType,+ file_size: int | None = 400,+ created_at: datetime | None = None,+) -> SimpleNamespace:+ artifact = make_stub_artifact(artifact_id, f"{artifact_id}.html", file_size, artifact_type=artifact_type)+ artifact.created_at = created_at or datetime(2026, 7, 9, tzinfo=timezone.utc)+ return artifact+++def stub_artifact_app(+ monkeypatch: pytest.MonkeyPatch,+ artifacts: list[SimpleNamespace],+ retrieved: dict[str, bytes],+ *,+ by_ids: list[SimpleNamespace] | None = None,+) -> list[str]:+ retrieved_ids: list[str] = []++ async def fake_get_artifacts_for_run(+ run_id: str, *, organization_id: str, artifact_types: object+ ) -> list[SimpleNamespace]:+ return artifacts++ async def fake_get_artifacts_by_ids(artifact_ids: list[str], *, organization_id: str) -> list[SimpleNamespace]:+ pool = {artifact.artifact_id: artifact for artifact in (by_ids if by_ids is not None else artifacts)}+ return [pool[artifact_id] for artifact_id in artifact_ids if artifact_id in pool]++ async def fake_retrieve_artifact(artifact: SimpleNamespace) -> bytes:+ retrieved_ids.append(artifact.artifact_id)+ return retrieved.get(artifact.artifact_id, b"")++ fake_app = SimpleNamespace(+ DATABASE=SimpleNamespace(+ artifacts=SimpleNamespace(+ get_artifacts_for_run=fake_get_artifacts_for_run,+ get_artifacts_by_ids=fake_get_artifacts_by_ids,+ )+ ),+ ARTIFACT_MANAGER=SimpleNamespace(retrieve_artifact=fake_retrieve_artifact),+ )+ monkeypatch.setattr(run_execution_module, "app", fake_app)+ return retrieved_idsdef make_copilot_ctx(**overrides: object) -> CopilotContext:diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex f15923b01..edfd17018 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -12,6 +12,7 @@ import pytestfrom structlog.testing import capture_logsfrom skyvern.config import settings+from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.copilot.agent import (_completion_contract_not_violated,_rewrite_failed_test_response,@@ -103,6 +104,7 @@ from skyvern.forge.sdk.copilot.tools import (_tool_visible_result_after_completion_verification,_watchdog_exit_allows_terminal_promotion,)+from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_modulefrom skyvern.forge.sdk.copilot.tools._shared import (_TASK_ENVELOPE_BLOCK_TYPES,_has_meaningful_registered_output_payload,@@ -118,7 +120,16 @@ from skyvern.forge.sdk.copilot.tools.workflow_update import (_apply_code_artifact_requested_output_evidence_sources,_normalize_code_artifact_metadata,)+from tests.unit.copilot_test_helpers import (+ DISPATCHED_LOGIN_GATE_HTML,+ DISPATCHED_NAV_ONLY_HTML,+ DISPATCHED_RESULTS_HTML,+)from tests.unit.copilot_test_helpers import make_completion_criterion as _criterion+from tests.unit.copilot_test_helpers import (+ make_stub_html_artifact,+ stub_artifact_app,+)_STRUCTURED_RECORD_CRITERIA = (("fallback_record_identity", "The returned record identifies the target record."),@@ -10310,6 +10321,108 @@ def test_judgment_truth_condition_refutes_inverted_login_gate_packet() -> None:assert verdicts[0].evidence_source == "independent_page_evidence"+def _dispatched_judgment_criterion() -> CompletionCriterion:+ return replace(+ _criterion(+ "c_login_gate",+ "The returned record reports whether the login gate blocks the target.",+ output_path="output.login_gate_blocks_target",+ expected_output_value=True,+ requested_output_evidence_source="independent_run_evidence",+ ),+ judgment_truth_condition=JudgmentTruthCondition(predicate="login_gate_blocks_target", polarity_when_holds=True),+ )+++async def _dispatched_chain_snapshot(+ monkeypatch: pytest.MonkeyPatch, ctx: CopilotContext, html: str+) -> RunEvidenceSnapshot:+ stub_artifact_app(+ monkeypatch,+ [make_stub_html_artifact("art_terminal", ArtifactType.HTML_ACTION)],+ {"art_terminal": html.encode()},+ )+ await run_execution_module._capture_dispatched_terminal_page_evidence(+ ctx, run_id="wr_requested_output", organization_id="o", current_url=""+ )+ return _build_run_evidence_snapshot(ctx, _requested_output_result({"login_gate_blocks_target": True}))+++async def test_dispatched_chain_login_gate_packet_confirms_judgment_boolean(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("login_gate_blocks_target")+ snapshot = await _dispatched_chain_snapshot(monkeypatch, ctx, DISPATCHED_LOGIN_GATE_HTML)++ with capture_logs() as logs:+ verdicts = grade_requested_output_criteria(ctx, [_dispatched_judgment_criterion()], snapshot)++ bound = snapshot.block_outputs[_POST_RUN_PAGE_OBSERVATION_LABEL]+ assert snapshot.block_output_sources[_POST_RUN_PAGE_OBSERVATION_LABEL] == "independent_page_evidence"+ assert bound["forms"]+ assert verdicts[0].state == "satisfied"+ assert verdicts[0].reason_code == "evidence_confirms"+ assert verdicts[0].evidence_ref == f"block_outputs:{_POST_RUN_PAGE_OBSERVATION_LABEL}"+ assert verdicts[0].evidence_source == "independent_page_evidence"+ assert any(+ log["event"] == "copilot_judgment_evidence_verdict"+ and log["predicate"] == "login_gate_blocks_target"+ and log["packet_label"] == _POST_RUN_PAGE_OBSERVATION_LABEL+ and log["verdict"] == "evidence_confirms"+ for log in logs+ )+++async def test_dispatched_chain_result_container_packet_contradicts_login_gate(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("login_gate_blocks_target")+ snapshot = await _dispatched_chain_snapshot(monkeypatch, ctx, DISPATCHED_RESULTS_HTML)++ with capture_logs() as logs:+ verdicts = grade_requested_output_criteria(ctx, [_dispatched_judgment_criterion()], snapshot)++ bound = snapshot.block_outputs[_POST_RUN_PAGE_OBSERVATION_LABEL]+ assert bound["result_containers"]+ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "evidence_contradicts"+ assert verdicts[0].evidence_ref == f"block_outputs:{_POST_RUN_PAGE_OBSERVATION_LABEL}"+ assert verdicts[0].evidence_source == "independent_page_evidence"+ assert all(verdict.reason_code != "structurally_abstained" for verdict in verdicts)+ assert any(+ log["event"] == "copilot_judgment_evidence_verdict"+ and log["predicate"] == "login_gate_blocks_target"+ and log["verdict"] == "evidence_contradicts"+ for log in logs+ )+++async def test_dispatched_chain_nav_only_page_abstains(monkeypatch: pytest.MonkeyPatch) -> None:+ # Dispatched captures parse with no current URL, so the same-origin filter drops every+ # navigation target and a nav-only terminal page yields an undecidable hollow packet.+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("login_gate_blocks_target")+ snapshot = await _dispatched_chain_snapshot(monkeypatch, ctx, DISPATCHED_NAV_ONLY_HTML)++ verdicts = grade_requested_output_criteria(ctx, [_dispatched_judgment_criterion()], snapshot)++ bound = snapshot.block_outputs[_POST_RUN_PAGE_OBSERVATION_LABEL]+ assert bound["navigation_targets"] == []+ assert bound["forms"] == []+ assert bound["result_containers"] == []+ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "structurally_abstained"+ assert verdicts[0].evidence_source == "independent_page_evidence"++def test_judgment_truth_condition_hollow_packet_abstains_before_self_emitted_output() -> None:ctx = _run_ctx()ctx.code_artifact_metadata = _metadata_for_requested_paths("login_gate_blocks_target")diff --git a/tests/unit/test_copilot_positive_evidence_carriers.py b/tests/unit/test_copilot_positive_evidence_carriers.pyindex 2458337c9..46be5f78c 100644--- a/tests/unit/test_copilot_positive_evidence_carriers.py+++ b/tests/unit/test_copilot_positive_evidence_carriers.py@@ -27,7 +27,13 @@ from skyvern.forge.sdk.copilot.runtime import ()from skyvern.forge.sdk.copilot.tools import completion as completion_modulefrom skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module-from tests.unit.copilot_test_helpers import make_completion_criterion+from tests.unit.copilot_test_helpers import (+ DISPATCHED_NAV_ONLY_HTML,+ make_completion_criterion,+)+from tests.unit.copilot_test_helpers import make_stub_artifact as _artifact+from tests.unit.copilot_test_helpers import make_stub_html_artifact as _html_artifact+from tests.unit.copilot_test_helpers import stub_artifact_app as _stub_app_POST_RUN_LABEL = "post_run_page_observation"_ARTIFACT_LABEL = "registered_artifact_observation"@@ -373,55 +379,6 @@ def test_parse_registered_artifact_text_pdf_cleans_temp_file_on_exception(monkeyassert not os.path.exists(captured["path"])-def _artifact(- artifact_id: str,- file_name: str,- file_size: int | None,- artifact_type: ArtifactType = ArtifactType.DOWNLOAD,-) -> SimpleNamespace:- return SimpleNamespace(- artifact_id=artifact_id,- uri=f"s3://bucket/{file_name}",- file_size=file_size,- artifact_type=artifact_type,- )---def _stub_app(- monkeypatch: pytest.MonkeyPatch,- artifacts: list[SimpleNamespace],- retrieved: dict[str, bytes],- *,- by_ids: list[SimpleNamespace] | None = None,-) -> list[str]:- retrieved_ids: list[str] = []-- async def fake_get_artifacts_for_run(- run_id: str, *, organization_id: str, artifact_types: object- ) -> list[SimpleNamespace]:- return artifacts-- async def fake_get_artifacts_by_ids(artifact_ids: list[str], *, organization_id: str) -> list[SimpleNamespace]:- pool = {artifact.artifact_id: artifact for artifact in (by_ids if by_ids is not None else artifacts)}- return [pool[artifact_id] for artifact_id in artifact_ids if artifact_id in pool]-- async def fake_retrieve_artifact(artifact: SimpleNamespace) -> bytes:- retrieved_ids.append(artifact.artifact_id)- return retrieved.get(artifact.artifact_id, b"")-- fake_app = SimpleNamespace(- DATABASE=SimpleNamespace(- artifacts=SimpleNamespace(- get_artifacts_for_run=fake_get_artifacts_for_run,- get_artifacts_by_ids=fake_get_artifacts_by_ids,- )- ),- ARTIFACT_MANAGER=SimpleNamespace(retrieve_artifact=fake_retrieve_artifact),- )- monkeypatch.setattr(run_execution_module, "app", fake_app)- return retrieved_ids--@pytest.mark.asyncioasync def test_artifact_producer_binds_parsed_text(monkeypatch: pytest.MonkeyPatch) -> None:artifacts = [_artifact("art_1", "receipt.txt", 20)]@@ -554,17 +511,6 @@ _HTML_NO_VALUE = (_HTML_SCRAPE_PREACTION = "<html><body><main><p>SCRAPEONLYTOKEN loading form</p></main></body></html>"-def _html_artifact(- artifact_id: str,- artifact_type: ArtifactType,- file_size: int | None = 400,- created_at: datetime | None = None,-) -> SimpleNamespace:- artifact = _artifact(artifact_id, f"{artifact_id}.html", file_size, artifact_type=artifact_type)- artifact.created_at = created_at or datetime(2026, 7, 9, tzinfo=timezone.utc)- return artifact--def _producer_ctx(pre_run_prose: str | None = "Submit your request below.") -> SimpleNamespace:baseline = {"visible_text_excerpt": pre_run_prose} if pre_run_prose is not None else Nonereturn SimpleNamespace(@@ -785,3 +731,72 @@ async def test_dispatched_producer_abstains_without_terminal_artifact(monkeypatcctx, run_id="wr_disp", organization_id="o_1", current_url="")assert ctx.composition_page_evidence is None+++_HTML_FORM_AND_RESULTS = (+ "<html><head><title>Find a provider</title></head><body><main>"+ '<form id="finder" action="/find" method="get">'+ '<label for="zip">ZIP code</label>'+ '<input id="zip" name="zip" type="text" required />'+ '<input id="account-password" name="password" type="password" />'+ '<button type="submit">Search</button>'+ "</form>"+ '<table id="provider-results"><tbody>'+ "<tr><td>Example Fiber</td><td>up to 500 Mbps</td></tr>"+ "<tr><td>Example Cable</td><td>up to 300 Mbps</td></tr>"+ "</tbody></table></main></body></html>"+)+_HTML_DISABLED_SUBMIT = (+ "<html><body><main>"+ '<form id="apply" action="/apply" method="post">'+ '<input id="account-email" name="email" type="email" />'+ '<input id="account-password" name="password" type="password" />'+ '<button type="submit" disabled>Submit</button>'+ "</form></main></body></html>"+)+++async def _dispatched_packet(monkeypatch: pytest.MonkeyPatch, html: str) -> dict[str, object]:+ _stub_app(monkeypatch, [_html_artifact("art_page", ArtifactType.HTML_ACTION)], {"art_page": html.encode()})+ ctx = _producer_ctx()+ await run_execution_module._capture_dispatched_terminal_page_evidence(+ ctx, run_id="wr_disp", organization_id="o_1", current_url=""+ )+ assert ctx.composition_page_evidence is not None+ return ctx.composition_page_evidence+++async def test_dispatched_packet_carries_forms_and_result_containers(monkeypatch: pytest.MonkeyPatch) -> None:+ packet = await _dispatched_packet(monkeypatch, _HTML_FORM_AND_RESULTS)+ forms = packet["forms"]+ assert forms+ assert any(field.get("type") == "password" for field in forms[0]["fields"])+ assert any(control.get("type") == "submit" for control in forms[0]["submit_controls"])+ containers = packet["result_containers"]+ assert containers+ assert any(container.get("selector") for container in containers)+++async def test_dispatched_packet_drops_navigation_targets_without_current_url(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Dispatched captures carry no current URL, so the same-origin link filter drops every target.+ packet = await _dispatched_packet(monkeypatch, DISPATCHED_NAV_ONLY_HTML)+ assert packet["navigation_targets"] == []+ assert packet["forms"] == []+ assert packet["result_containers"] == []+++async def test_dispatched_packet_carries_static_disabled_submit_control(monkeypatch: pytest.MonkeyPatch) -> None:+ packet = await _dispatched_packet(monkeypatch, _HTML_DISABLED_SUBMIT)+ controls = packet["forms"][0]["submit_controls"]+ assert controls+ assert controls[0]["disabled"] is True+ # Without challenge indicators the static parse never claims challenge gating; the+ # literal disabled attribute above is the only gating signal a dispatched packet carries.+ challenge_state = packet["challenge_state"]+ assert challenge_state["gates_submit_controls"] is False+ assert challenge_state["gated_submit_controls"] == []