Skip to content

Automate browser based workflows with AI

SKY-11468 Add typed download criterion reconciliation (#6861)

Andrew Neilson3fc3fe8912b8
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index aa1c4377c..33975afe0 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -42,6 +42,12 @@ _REASON_CODES = frozenset({"evidence_confirms", "no_evidence", "evidence_contrad
_CONTINGENT_ABSTENTION_REASON_CODES = frozenset(
{"unknown", "no_evidence", "evidence_contradicts", "missing_exact_field", "unproducible"}
)
+REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID = "__copilot_registered_download__downloaded_files_non_empty"
+_REGISTERED_DOWNLOAD_COUNT_KEYS = (
+ "downloaded_file_count",
+ "downloaded_file_url_count",
+ "downloaded_file_artifact_count",
+)
_PAGE_EVIDENCE_KEYS = (
"current_url",
"page_title",
@@ -248,6 +254,61 @@ class RunEvidenceSnapshot:
_UNAVAILABLE = CompletionVerificationResult(status="unavailable")
_MISSING_VERDICT_EVIDENCE = "judge did not return a verdict for this criterion"
+_MISSING_REGISTERED_DOWNLOAD_EVIDENCE = "run output did not include a non-empty registered browser download"
+
+
+def registered_download_completion_criterion() -> CompletionCriterion:
+ return CompletionCriterion(
+ id=REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,
+ outcome="A browser download is registered with a non-empty downloaded file surface.",
+ )
+
+
+def is_registered_download_completion_criterion(criterion: CompletionCriterion) -> bool:
+ return criterion.id == REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID
+
+
+def _is_positive_download_count(value: Any) -> bool:
+ return isinstance(value, int) and not isinstance(value, bool) and value > 0
+
+
+def _registered_download_evidence_label(snapshot: RunEvidenceSnapshot) -> str | None:
+ for label, payload in snapshot.block_outputs.items():
+ if not isinstance(payload, dict) or payload.get("download_registered") is not True:
+ continue
+ if any(_is_positive_download_count(payload.get(key)) for key in _REGISTERED_DOWNLOAD_COUNT_KEYS):
+ return str(label)
+ return None
+
+
+def grade_registered_download_criteria(
+ criteria: list[CompletionCriterion], snapshot: RunEvidenceSnapshot
+) -> list[CriterionVerdict]:
+ registered_criteria = [
+ criterion for criterion in criteria if is_registered_download_completion_criterion(criterion)
+ ]
+ if not registered_criteria:
+ return []
+ label = _registered_download_evidence_label(snapshot)
+ if label is not None:
+ return [
+ CriterionVerdict(
+ criterion_id=criterion.id,
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref=f"block_outputs:{label}",
+ )
+ for criterion in registered_criteria
+ ]
+ return [
+ CriterionVerdict(
+ criterion_id=criterion.id,
+ state="unsatisfied",
+ reason_code="no_evidence",
+ missing_evidence=_MISSING_REGISTERED_DOWNLOAD_EVIDENCE,
+ )
+ for criterion in registered_criteria
+ ]
def _clean_optional_text(value: Any, *, max_chars: int) -> str | None:
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 47295e15a..7763ab7bf 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -21,8 +21,11 @@ from skyvern.forge.sdk.copilot.completion_verification import (
grade_fallback_floor_reached_end_state_criteria,
grade_present_value_criteria,
grade_record_semantic_consistency,
+ grade_registered_download_criteria,
grade_structured_record_criteria,
grade_terminal_goal_record_criteria,
+ is_registered_download_completion_criterion,
+ registered_download_completion_criterion,
structural_unfired_contingent_criterion_ids,
summarize_unsatisfied_outcomes,
verdict_missing_evidence,
@@ -31,7 +34,14 @@ from skyvern.forge.sdk.copilot.enforcement import _goal_likely_needs_more_blocks
from skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handler
from skyvern.forge.sdk.copilot.outcome_verification_trace import record_completion_verification
from skyvern.forge.sdk.copilot.output_utils import iter_failure_reasons
-from skyvern.forge.sdk.copilot.reached_download_target import REGISTERED_DOWNLOAD_OUTPUT_KEYS
+from skyvern.forge.sdk.copilot.reached_download_target import (
+ DOWNLOAD_KIND_ATTRIBUTE,
+ DOWNLOAD_KIND_EXTENSION,
+ DOWNLOAD_KIND_REGISTERED,
+ REGISTERED_DOWNLOAD_OUTPUT_KEYS,
+ ReachedDownloadTarget,
+ derive_from_block_outputs,
+)
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
@@ -55,9 +65,82 @@ from .blockers import (
LOG = structlog.get_logger()
+_TYPED_DOWNLOAD_KINDS = frozenset({DOWNLOAD_KIND_REGISTERED, DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION})
+
+
+def _completion_request_policy(copilot_ctx: Any) -> Any | None:
+ try:
+ return copilot_ctx.request_policy
+ except AttributeError:
+ return None
+
+
+def _is_typed_download_target(value: object) -> bool:
+ if isinstance(value, ReachedDownloadTarget):
+ return value.download_kind in _TYPED_DOWNLOAD_KINDS
+ if not isinstance(value, dict):
+ return False
+ download_kind = value.get("download_kind")
+ return isinstance(download_kind, str) and download_kind in _TYPED_DOWNLOAD_KINDS
+
+
+def _ctx_reached_download_target(copilot_ctx: Any) -> object | None:
+ try:
+ return copilot_ctx.reached_download_target
+ except AttributeError:
+ return None
+
+
+def _result_data(result: dict[str, Any]) -> dict[str, Any]:
+ data = result.get("data")
+ return data if isinstance(data, dict) else {}
+
+
+def _result_block_outputs_by_label(result: dict[str, Any]) -> dict[str, Any]:
+ data = _result_data(result)
+ blocks = data.get("blocks")
+ block_outputs: dict[str, Any] = {}
+ if isinstance(blocks, list):
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ label = block.get("label")
+ output = block.get("extracted_data")
+ if isinstance(label, str) and isinstance(output, dict):
+ block_outputs[label] = output
+ # Signal detection reads raw download keys; evidence snapshots normalize/redact payloads before grading.
+ for registered in _registered_output_parameter_payloads(data):
+ label = registered.get("output_parameter_key") or registered.get("block_label")
+ value = registered.get("value")
+ if isinstance(label, str) and isinstance(value, dict):
+ block_outputs[label] = value
+ return block_outputs
+
+
+def _result_has_registered_download_block_output(result: dict[str, Any]) -> bool:
+ return derive_from_block_outputs(_result_block_outputs_by_label(result)) is not None
+
+
+def _has_typed_download_signal(copilot_ctx: Any, result: dict[str, Any]) -> bool:
+ if _is_typed_download_target(_ctx_reached_download_target(copilot_ctx)):
+ return True
+ if _is_typed_download_target(_result_data(result).get("reached_download_target")):
+ return True
+ return _result_has_registered_download_block_output(result)
+
+
+def _reconcile_download_completion_criterion(
+ copilot_ctx: Any, result: dict[str, Any], criteria: list[CompletionCriterion]
+) -> list[CompletionCriterion]:
+ if any(is_registered_download_completion_criterion(criterion) for criterion in criteria):
+ return criteria
+ if not _has_typed_download_signal(copilot_ctx, result):
+ return criteria
+ return [*criteria, registered_download_completion_criterion()]
+
def _completion_verification_criteria(copilot_ctx: Any) -> list[CompletionCriterion]:
- policy = getattr(copilot_ctx, "request_policy", None)
+ policy = _completion_request_policy(copilot_ctx)
# A method-mandated criterion asserts HOW the goal was reached; the outcome
# judge sees only end-state evidence.
return policy.graded_completion_criteria() if policy is not None else []
@@ -499,7 +582,7 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru
block_outputs[output_key] = output_value
for registered in _registered_output_parameter_payloads(data):
registered_output_key = registered.get("output_parameter_key")
- registered_output_value = registered.get("value")
+ registered_output_value = _completion_evidence_payload(registered.get("value"))
registered_block_label = registered.get("block_label")
if isinstance(registered_output_key, str) and registered_output_key:
block_outputs[registered_output_key] = registered_output_value
@@ -692,6 +775,8 @@ def _deterministic_run_verification_result(
deterministic_by_id[verdict.criterion_id] = verdict
for verdict in grade_terminal_goal_record_criteria(run_criteria, snapshot):
deterministic_by_id[verdict.criterion_id] = verdict
+ for verdict in grade_registered_download_criteria(run_criteria, snapshot):
+ deterministic_by_id[verdict.criterion_id] = verdict
for verdict in semantic_verdicts:
deterministic_by_id[verdict.criterion_id] = verdict
@@ -738,6 +823,7 @@ async def _maybe_run_completion_verification(
):
return None
criteria = _completion_verification_criteria(copilot_ctx)
+ criteria = _reconcile_download_completion_criterion(copilot_ctx, result, criteria)
run_criteria, definition_criteria = _split_criteria_by_plane(criteria)
criterion_ids = [criterion.id for criterion in criteria]
contingent_ids, contingent_on_by_id, contingent_path_by_id = _contingent_metadata_for_criteria(criteria)
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 74d0bb1cb..90dde3b81 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.agent import (
_verified_workflow_or_none,
)
from skyvern.forge.sdk.copilot.completion_verification import (
+ REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,
CompletionVerificationResult,
CriterionVerdict,
RunEvidenceSnapshot,
@@ -26,8 +27,11 @@ from skyvern.forge.sdk.copilot.completion_verification import (
grade_definition_criteria,
grade_present_value_criteria,
grade_record_semantic_consistency,
+ grade_registered_download_criteria,
grade_structured_record_criteria,
grade_terminal_goal_record_criteria,
+ registered_download_completion_criterion,
+ run_plane_all_no_evidence,
structural_unfired_contingent_criterion_ids,
structured_record_has_goal_content,
structured_record_has_identity,
@@ -48,6 +52,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
verified_goal_satisfied_context,
)
from skyvern.forge.sdk.copilot.hooks import _tool_completion_satisfies_turn
+from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
RequestPolicy,
@@ -74,7 +79,10 @@ from skyvern.forge.sdk.copilot.tools import (
_tool_visible_result_after_completion_verification,
_watchdog_exit_allows_terminal_promotion,
)
-from skyvern.forge.sdk.copilot.tools.completion import _artifact_health_blocker_from_result
+from skyvern.forge.sdk.copilot.tools.completion import (
+ _artifact_health_blocker_from_result,
+ _reconcile_download_completion_criterion,
+)
def _criterion(
@@ -3595,6 +3603,303 @@ def test_snapshot_summarizes_registered_download_outputs() -> None:
assert "RecordingLocator" not in rendered and "Download" not in rendered and "secret" not in rendered
+def _download_result(output: dict[str, Any]) -> dict:
+ return {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_download",
+ "overall_status": "completed",
+ "executed_block_labels": ["download_document"],
+ "current_url": "https://example.test/documents",
+ "blocks": [
+ {
+ "label": "download_document",
+ "block_type": "CODE",
+ "status": "completed",
+ "extracted_data": output,
+ }
+ ],
+ },
+ }
+
+
+def _registered_download_output_parameter_result(value: dict[str, Any]) -> dict:
+ return {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_download",
+ "overall_status": "completed",
+ "executed_block_labels": [],
+ "current_url": "https://example.test/documents",
+ "blocks": [],
+ "registered_output_parameter_values": [
+ {
+ "workflow_run_id": "wr_download",
+ "output_parameter_key": "download_document_output",
+ "block_label": "download_document",
+ "block_type": "CODE",
+ "value": value,
+ }
+ ],
+ },
+ }
+
+
+async def test_download_registered_non_empty_injects_and_verifies_without_judge(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("registered download evidence must bypass the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "download_document")
+ ctx.request_policy = RequestPolicy(completion_criteria=[])
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _download_result(
+ {
+ "downloaded_files": [{"filename": "statement.pdf"}],
+ "downloaded_file_urls": [],
+ "downloaded_file_artifact_ids": [],
+ }
+ ),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ assert verification.criterion_ids == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]
+ assert verification.verdicts == [
+ CriterionVerdict(
+ criterion_id=REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:download_document",
+ )
+ ]
+ assert ctx.request_policy.completion_criteria == []
+
+
+async def test_download_registered_output_parameter_injects_and_verifies_without_judge(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("registered output-parameter download evidence must bypass the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "download_document")
+ ctx.request_policy = RequestPolicy(completion_criteria=[])
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _registered_download_output_parameter_result(
+ {
+ "downloaded_files": [{"filename": "document.pdf"}],
+ "downloaded_file_urls": [],
+ "downloaded_file_artifact_ids": [],
+ }
+ ),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ assert verification.criterion_ids == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]
+ assert verification.verdicts == [
+ CriterionVerdict(
+ criterion_id=REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:download_document_output",
+ )
+ ]
+ assert ctx.request_policy.completion_criteria == []
+
+
+async def test_download_typed_affordance_injects_and_fails_without_registered_output(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("missing registered download evidence must be deterministic")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[])
+ ctx.reached_download_target = ReachedDownloadTarget(
+ selector='a[href="/files/statement.pdf"]',
+ affordance_text="Download",
+ download_kind="extension",
+ source_step="trajectory_recency",
+ already_registered=False,
+ )
+
+ verification = await _maybe_run_completion_verification(ctx, _goto_only_result(), time.monotonic())
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is False
+ assert verification.criterion_ids == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]
+ assert verification.verdicts == [
+ CriterionVerdict(
+ criterion_id=REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,
+ state="unsatisfied",
+ reason_code="no_evidence",
+ missing_evidence="run output did not include a non-empty registered browser download",
+ )
+ ]
+ assert ctx.request_policy.completion_criteria == []
+
+
+async def test_definition_only_non_download_remains_no_gradeable_run_plane(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("definition-only criteria must not reach the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="definition_inputs",
+ outcome="The workflow accepts the account number as a reusable input.",
+ level="definition",
+ )
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(ctx, _goto_only_result(), time.monotonic())
+
+ assert verification is not None
+ assert verification.criterion_ids == ["definition_inputs"]
+ assert verification.verdicts == [
+ CriterionVerdict(
+ criterion_id="definition_inputs",
+ state="unknown",
+ reason_code="definition_parameters_absent",
+ )
+ ]
+ assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID not in [
+ criterion.id for criterion in ctx.request_policy.completion_criteria
+ ]
+
+
+def test_download_reconciliation_is_idempotent_when_criterion_exists() -> None:
+ ctx = _run_ctx()
+ criterion = registered_download_completion_criterion()
+ ctx.request_policy = RequestPolicy(completion_criteria=[criterion])
+ criteria = [criterion]
+
+ reconciled = _reconcile_download_completion_criterion(
+ ctx,
+ {"ok": True, "data": {"reached_download_target": {"download_kind": "extension"}}},
+ criteria,
+ )
+
+ assert reconciled is criteria
+ assert [existing.id for existing in ctx.request_policy.completion_criteria] == [
+ REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID
+ ]
+
+
+def test_download_reconciliation_does_not_mutate_request_policy() -> None:
+ ctx = _run_ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[])
+
+ reconciled = _reconcile_download_completion_criterion(
+ ctx,
+ {"ok": True, "data": {"reached_download_target": {"download_kind": "extension"}}},
+ [],
+ )
+
+ assert [criterion.id for criterion in reconciled] == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]
+ assert ctx.request_policy.completion_criteria == []
+
+
+def test_download_grader_requires_non_empty_registered_surface() -> None:
+ criteria = [registered_download_completion_criterion()]
+
+ missing = grade_registered_download_criteria(
+ criteria,
+ RunEvidenceSnapshot(block_outputs={"download_document": {"download_registered": True}}),
+ )
+ present = grade_registered_download_criteria(
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ "download_document": {
+ "download_registered": True,
+ "downloaded_file_count": 0,
+ "downloaded_file_url_count": 1,
+ "downloaded_file_artifact_count": 0,
+ }
+ }
+ ),
+ )
+
+ assert missing == [
+ CriterionVerdict(
+ criterion_id=REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,
+ state="unsatisfied",
+ reason_code="no_evidence",
+ missing_evidence="run output did not include a non-empty registered browser download",
+ )
+ ]
+ assert (
+ run_plane_all_no_evidence(
+ CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=[REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID],
+ verdicts=missing,
+ )
+ )
+ is True
+ )
+ assert present == [
+ CriterionVerdict(
+ criterion_id=REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:download_document",
+ )
+ ]
+
+
+def test_snapshot_normalizes_registered_download_output_parameter_values() -> None:
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "download_document")
+
+ snapshot = _build_run_evidence_snapshot(
+ ctx,
+ _registered_download_output_parameter_result(
+ {
+ "page": "<RecordingLocator>",
+ "download": "<Download>",
+ "downloaded_file_name": "document.pdf",
+ "downloaded_files": [{"filename": "document.pdf"}],
+ "downloaded_file_urls": ["https://example.test/downloads/document.pdf?token=secret"],
+ "downloaded_file_artifact_ids": ["artifact_1"],
+ }
+ ),
+ )
+ rendered = snapshot.render_prompt_block()
+
+ assert snapshot.block_outputs["download_document_output"] == {
+ "download_registered": True,
+ "downloaded_file_count": 1,
+ "downloaded_file_url_count": 1,
+ "downloaded_file_artifact_count": 1,
+ "downloaded_file_names": ["document.pdf"],
+ }
+ assert "RecordingLocator" not in rendered and "Download" not in rendered and "secret" not in rendered
+
+
def test_snapshot_includes_verified_context_labels_without_prior_outputs() -> None:
ctx = _run_ctx()
labels = [