Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11468 Add typed download criterion reconciliation (#6861)
Andrew Neilson3fc3fe8912b8
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex aa1c4377c..33975afe0 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -42,6 +42,12 @@ _REASON_CODES = frozenset({"evidence_confirms", "no_evidence", "evidence_contrad_CONTINGENT_ABSTENTION_REASON_CODES = frozenset({"unknown", "no_evidence", "evidence_contradicts", "missing_exact_field", "unproducible"})+REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID = "__copilot_registered_download__downloaded_files_non_empty"+_REGISTERED_DOWNLOAD_COUNT_KEYS = (+ "downloaded_file_count",+ "downloaded_file_url_count",+ "downloaded_file_artifact_count",+)_PAGE_EVIDENCE_KEYS = ("current_url","page_title",@@ -248,6 +254,61 @@ class RunEvidenceSnapshot:_UNAVAILABLE = CompletionVerificationResult(status="unavailable")_MISSING_VERDICT_EVIDENCE = "judge did not return a verdict for this criterion"+_MISSING_REGISTERED_DOWNLOAD_EVIDENCE = "run output did not include a non-empty registered browser download"+++def registered_download_completion_criterion() -> CompletionCriterion:+ return CompletionCriterion(+ id=REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,+ outcome="A browser download is registered with a non-empty downloaded file surface.",+ )+++def is_registered_download_completion_criterion(criterion: CompletionCriterion) -> bool:+ return criterion.id == REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID+++def _is_positive_download_count(value: Any) -> bool:+ return isinstance(value, int) and not isinstance(value, bool) and value > 0+++def _registered_download_evidence_label(snapshot: RunEvidenceSnapshot) -> str | None:+ for label, payload in snapshot.block_outputs.items():+ if not isinstance(payload, dict) or payload.get("download_registered") is not True:+ continue+ if any(_is_positive_download_count(payload.get(key)) for key in _REGISTERED_DOWNLOAD_COUNT_KEYS):+ return str(label)+ return None+++def grade_registered_download_criteria(+ criteria: list[CompletionCriterion], snapshot: RunEvidenceSnapshot+) -> list[CriterionVerdict]:+ registered_criteria = [+ criterion for criterion in criteria if is_registered_download_completion_criterion(criterion)+ ]+ if not registered_criteria:+ return []+ label = _registered_download_evidence_label(snapshot)+ if label is not None:+ return [+ CriterionVerdict(+ criterion_id=criterion.id,+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref=f"block_outputs:{label}",+ )+ for criterion in registered_criteria+ ]+ return [+ CriterionVerdict(+ criterion_id=criterion.id,+ state="unsatisfied",+ reason_code="no_evidence",+ missing_evidence=_MISSING_REGISTERED_DOWNLOAD_EVIDENCE,+ )+ for criterion in registered_criteria+ ]def _clean_optional_text(value: Any, *, max_chars: int) -> str | None:diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 47295e15a..7763ab7bf 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -21,8 +21,11 @@ from skyvern.forge.sdk.copilot.completion_verification import (grade_fallback_floor_reached_end_state_criteria,grade_present_value_criteria,grade_record_semantic_consistency,+ grade_registered_download_criteria,grade_structured_record_criteria,grade_terminal_goal_record_criteria,+ is_registered_download_completion_criterion,+ registered_download_completion_criterion,structural_unfired_contingent_criterion_ids,summarize_unsatisfied_outcomes,verdict_missing_evidence,@@ -31,7 +34,14 @@ from skyvern.forge.sdk.copilot.enforcement import _goal_likely_needs_more_blocksfrom skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handlerfrom skyvern.forge.sdk.copilot.outcome_verification_trace import record_completion_verificationfrom skyvern.forge.sdk.copilot.output_utils import iter_failure_reasons-from skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOAD_OUTPUT_KEYS+from skyvern.forge.sdk.copilot.reached_download_target import (+ DOWNLOAD_KIND_ATTRIBUTE,+ DOWNLOAD_KIND_EXTENSION,+ DOWNLOAD_KIND_REGISTERED,+ REGISTERED_DOWNLOAD_OUTPUT_KEYS,+ ReachedDownloadTarget,+ derive_from_block_outputs,+)from skyvern.forge.sdk.copilot.request_policy import CompletionCriterionfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_span@@ -55,9 +65,82 @@ from .blockers import (LOG = structlog.get_logger()+_TYPED_DOWNLOAD_KINDS = frozenset({DOWNLOAD_KIND_REGISTERED, DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION})+++def _completion_request_policy(copilot_ctx: Any) -> Any | None:+ try:+ return copilot_ctx.request_policy+ except AttributeError:+ return None+++def _is_typed_download_target(value: object) -> bool:+ if isinstance(value, ReachedDownloadTarget):+ return value.download_kind in _TYPED_DOWNLOAD_KINDS+ if not isinstance(value, dict):+ return False+ download_kind = value.get("download_kind")+ return isinstance(download_kind, str) and download_kind in _TYPED_DOWNLOAD_KINDS+++def _ctx_reached_download_target(copilot_ctx: Any) -> object | None:+ try:+ return copilot_ctx.reached_download_target+ except AttributeError:+ return None+++def _result_data(result: dict[str, Any]) -> dict[str, Any]:+ data = result.get("data")+ return data if isinstance(data, dict) else {}+++def _result_block_outputs_by_label(result: dict[str, Any]) -> dict[str, Any]:+ data = _result_data(result)+ blocks = data.get("blocks")+ block_outputs: dict[str, Any] = {}+ if isinstance(blocks, list):+ for block in blocks:+ if not isinstance(block, dict):+ continue+ label = block.get("label")+ output = block.get("extracted_data")+ if isinstance(label, str) and isinstance(output, dict):+ block_outputs[label] = output+ # Signal detection reads raw download keys; evidence snapshots normalize/redact payloads before grading.+ for registered in _registered_output_parameter_payloads(data):+ label = registered.get("output_parameter_key") or registered.get("block_label")+ value = registered.get("value")+ if isinstance(label, str) and isinstance(value, dict):+ block_outputs[label] = value+ return block_outputs+++def _result_has_registered_download_block_output(result: dict[str, Any]) -> bool:+ return derive_from_block_outputs(_result_block_outputs_by_label(result)) is not None+++def _has_typed_download_signal(copilot_ctx: Any, result: dict[str, Any]) -> bool:+ if _is_typed_download_target(_ctx_reached_download_target(copilot_ctx)):+ return True+ if _is_typed_download_target(_result_data(result).get("reached_download_target")):+ return True+ return _result_has_registered_download_block_output(result)+++def _reconcile_download_completion_criterion(+ copilot_ctx: Any, result: dict[str, Any], criteria: list[CompletionCriterion]+) -> list[CompletionCriterion]:+ if any(is_registered_download_completion_criterion(criterion) for criterion in criteria):+ return criteria+ if not _has_typed_download_signal(copilot_ctx, result):+ return criteria+ return [*criteria, registered_download_completion_criterion()]+def _completion_verification_criteria(copilot_ctx: Any) -> list[CompletionCriterion]:- policy = getattr(copilot_ctx, "request_policy", None)+ policy = _completion_request_policy(copilot_ctx)# A method-mandated criterion asserts HOW the goal was reached; the outcome# judge sees only end-state evidence.return policy.graded_completion_criteria() if policy is not None else []@@ -499,7 +582,7 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Rublock_outputs[output_key] = output_valuefor registered in _registered_output_parameter_payloads(data):registered_output_key = registered.get("output_parameter_key")- registered_output_value = registered.get("value")+ registered_output_value = _completion_evidence_payload(registered.get("value"))registered_block_label = registered.get("block_label")if isinstance(registered_output_key, str) and registered_output_key:block_outputs[registered_output_key] = registered_output_value@@ -692,6 +775,8 @@ def _deterministic_run_verification_result(deterministic_by_id[verdict.criterion_id] = verdictfor verdict in grade_terminal_goal_record_criteria(run_criteria, snapshot):deterministic_by_id[verdict.criterion_id] = verdict+ for verdict in grade_registered_download_criteria(run_criteria, snapshot):+ deterministic_by_id[verdict.criterion_id] = verdictfor verdict in semantic_verdicts:deterministic_by_id[verdict.criterion_id] = verdict@@ -738,6 +823,7 @@ async def _maybe_run_completion_verification():return Nonecriteria = _completion_verification_criteria(copilot_ctx)+ criteria = _reconcile_download_completion_criterion(copilot_ctx, result, criteria)run_criteria, definition_criteria = _split_criteria_by_plane(criteria)criterion_ids = [criterion.id for criterion in criteria]contingent_ids, contingent_on_by_id, contingent_path_by_id = _contingent_metadata_for_criteria(criteria)diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 74d0bb1cb..90dde3b81 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.agent import (_verified_workflow_or_none,)from skyvern.forge.sdk.copilot.completion_verification import (+ REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,CompletionVerificationResult,CriterionVerdict,RunEvidenceSnapshot,@@ -26,8 +27,11 @@ from skyvern.forge.sdk.copilot.completion_verification import (grade_definition_criteria,grade_present_value_criteria,grade_record_semantic_consistency,+ grade_registered_download_criteria,grade_structured_record_criteria,grade_terminal_goal_record_criteria,+ registered_download_completion_criterion,+ run_plane_all_no_evidence,structural_unfired_contingent_criterion_ids,structured_record_has_goal_content,structured_record_has_identity,@@ -48,6 +52,7 @@ from skyvern.forge.sdk.copilot.enforcement import (verified_goal_satisfied_context,)from skyvern.forge.sdk.copilot.hooks import _tool_completion_satisfies_turn+from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTargetfrom skyvern.forge.sdk.copilot.request_policy import (CompletionCriterion,RequestPolicy,@@ -74,7 +79,10 @@ from skyvern.forge.sdk.copilot.tools import (_tool_visible_result_after_completion_verification,_watchdog_exit_allows_terminal_promotion,)-from skyvern.forge.sdk.copilot.tools.completion import _artifact_health_blocker_from_result+from skyvern.forge.sdk.copilot.tools.completion import (+ _artifact_health_blocker_from_result,+ _reconcile_download_completion_criterion,+)def _criterion(@@ -3595,6 +3603,303 @@ def test_snapshot_summarizes_registered_download_outputs() -> None:assert "RecordingLocator" not in rendered and "Download" not in rendered and "secret" not in rendered+def _download_result(output: dict[str, Any]) -> dict:+ return {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_download",+ "overall_status": "completed",+ "executed_block_labels": ["download_document"],+ "current_url": "https://example.test/documents",+ "blocks": [+ {+ "label": "download_document",+ "block_type": "CODE",+ "status": "completed",+ "extracted_data": output,+ }+ ],+ },+ }+++def _registered_download_output_parameter_result(value: dict[str, Any]) -> dict:+ return {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_download",+ "overall_status": "completed",+ "executed_block_labels": [],+ "current_url": "https://example.test/documents",+ "blocks": [],+ "registered_output_parameter_values": [+ {+ "workflow_run_id": "wr_download",+ "output_parameter_key": "download_document_output",+ "block_label": "download_document",+ "block_type": "CODE",+ "value": value,+ }+ ],+ },+ }+++async def test_download_registered_non_empty_injects_and_verifies_without_judge(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("registered download evidence must bypass the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "download_document")+ ctx.request_policy = RequestPolicy(completion_criteria=[])++ verification = await _maybe_run_completion_verification(+ ctx,+ _download_result(+ {+ "downloaded_files": [{"filename": "statement.pdf"}],+ "downloaded_file_urls": [],+ "downloaded_file_artifact_ids": [],+ }+ ),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is True+ assert verification.criterion_ids == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]+ assert verification.verdicts == [+ CriterionVerdict(+ criterion_id=REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="block_outputs:download_document",+ )+ ]+ assert ctx.request_policy.completion_criteria == []+++async def test_download_registered_output_parameter_injects_and_verifies_without_judge(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("registered output-parameter download evidence must bypass the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "download_document")+ ctx.request_policy = RequestPolicy(completion_criteria=[])++ verification = await _maybe_run_completion_verification(+ ctx,+ _registered_download_output_parameter_result(+ {+ "downloaded_files": [{"filename": "document.pdf"}],+ "downloaded_file_urls": [],+ "downloaded_file_artifact_ids": [],+ }+ ),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is True+ assert verification.criterion_ids == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]+ assert verification.verdicts == [+ CriterionVerdict(+ criterion_id=REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="block_outputs:download_document_output",+ )+ ]+ assert ctx.request_policy.completion_criteria == []+++async def test_download_typed_affordance_injects_and_fails_without_registered_output(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("missing registered download evidence must be deterministic")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ ctx.request_policy = RequestPolicy(completion_criteria=[])+ ctx.reached_download_target = ReachedDownloadTarget(+ selector='a[href="/files/statement.pdf"]',+ affordance_text="Download",+ download_kind="extension",+ source_step="trajectory_recency",+ already_registered=False,+ )++ verification = await _maybe_run_completion_verification(ctx, _goto_only_result(), time.monotonic())++ assert verification is not None+ assert verification.is_fully_satisfied() is False+ assert verification.criterion_ids == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]+ assert verification.verdicts == [+ CriterionVerdict(+ criterion_id=REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,+ state="unsatisfied",+ reason_code="no_evidence",+ missing_evidence="run output did not include a non-empty registered browser download",+ )+ ]+ assert ctx.request_policy.completion_criteria == []+++async def test_definition_only_non_download_remains_no_gradeable_run_plane(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("definition-only criteria must not reach the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="definition_inputs",+ outcome="The workflow accepts the account number as a reusable input.",+ level="definition",+ )+ ]+ )++ verification = await _maybe_run_completion_verification(ctx, _goto_only_result(), time.monotonic())++ assert verification is not None+ assert verification.criterion_ids == ["definition_inputs"]+ assert verification.verdicts == [+ CriterionVerdict(+ criterion_id="definition_inputs",+ state="unknown",+ reason_code="definition_parameters_absent",+ )+ ]+ assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID not in [+ criterion.id for criterion in ctx.request_policy.completion_criteria+ ]+++def test_download_reconciliation_is_idempotent_when_criterion_exists() -> None:+ ctx = _run_ctx()+ criterion = registered_download_completion_criterion()+ ctx.request_policy = RequestPolicy(completion_criteria=[criterion])+ criteria = [criterion]++ reconciled = _reconcile_download_completion_criterion(+ ctx,+ {"ok": True, "data": {"reached_download_target": {"download_kind": "extension"}}},+ criteria,+ )++ assert reconciled is criteria+ assert [existing.id for existing in ctx.request_policy.completion_criteria] == [+ REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID+ ]+++def test_download_reconciliation_does_not_mutate_request_policy() -> None:+ ctx = _run_ctx()+ ctx.request_policy = RequestPolicy(completion_criteria=[])++ reconciled = _reconcile_download_completion_criterion(+ ctx,+ {"ok": True, "data": {"reached_download_target": {"download_kind": "extension"}}},+ [],+ )++ assert [criterion.id for criterion in reconciled] == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]+ assert ctx.request_policy.completion_criteria == []+++def test_download_grader_requires_non_empty_registered_surface() -> None:+ criteria = [registered_download_completion_criterion()]++ missing = grade_registered_download_criteria(+ criteria,+ RunEvidenceSnapshot(block_outputs={"download_document": {"download_registered": True}}),+ )+ present = grade_registered_download_criteria(+ criteria,+ RunEvidenceSnapshot(+ block_outputs={+ "download_document": {+ "download_registered": True,+ "downloaded_file_count": 0,+ "downloaded_file_url_count": 1,+ "downloaded_file_artifact_count": 0,+ }+ }+ ),+ )++ assert missing == [+ CriterionVerdict(+ criterion_id=REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,+ state="unsatisfied",+ reason_code="no_evidence",+ missing_evidence="run output did not include a non-empty registered browser download",+ )+ ]+ assert (+ run_plane_all_no_evidence(+ CompletionVerificationResult(+ status="evaluated",+ criterion_ids=[REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID],+ verdicts=missing,+ )+ )+ is True+ )+ assert present == [+ CriterionVerdict(+ criterion_id=REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="block_outputs:download_document",+ )+ ]+++def test_snapshot_normalizes_registered_download_output_parameter_values() -> None:+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "download_document")++ snapshot = _build_run_evidence_snapshot(+ ctx,+ _registered_download_output_parameter_result(+ {+ "page": "<RecordingLocator>",+ "download": "<Download>",+ "downloaded_file_name": "document.pdf",+ "downloaded_files": [{"filename": "document.pdf"}],+ "downloaded_file_urls": ["https://example.test/downloads/document.pdf?token=secret"],+ "downloaded_file_artifact_ids": ["artifact_1"],+ }+ ),+ )+ rendered = snapshot.render_prompt_block()++ assert snapshot.block_outputs["download_document_output"] == {+ "download_registered": True,+ "downloaded_file_count": 1,+ "downloaded_file_url_count": 1,+ "downloaded_file_artifact_count": 1,+ "downloaded_file_names": ["document.pdf"],+ }+ assert "RecordingLocator" not in rendered and "Download" not in rendered and "secret" not in rendered++def test_snapshot_includes_verified_context_labels_without_prior_outputs() -> None:ctx = _run_ctx()labels = [