Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11530): credit registered download output criteria (#6875)
Andrew Neilson5053de80d1a4
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2index 63f82c042..67d1fa256 100644--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2@@ -25,10 +25,11 @@ completion_contract:- Set to null when the user does not provide an explicit completion condition.completion_criteria:-- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, contingent_on, contingent_antecedent_output_path, implicit, method_mandated, level, kind, terminal_action_family}.+- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, contingent_on, contingent_antecedent_output_path, deliverable_kind, implicit, method_mandated, level, kind, terminal_action_family}.- outcome: a short statement of an end state the workflow must achieve (an END, not a step). Extract the goal as outcomes the user wants to be true, not the recipe they describe.- contingent_on: null for ordinary criteria. Use a short antecedent string only when this criterion should be graded only if that antecedent happens, for example {"contingent_on": "the provider site blocks online submission", "contingent_antecedent_output_path": "output.blocker", "outcome": "the blocker is reported to the user"}. The antecedent goes in contingent_on; the consequent to grade goes in outcome. Do not encode if/then criteria only in prose.- contingent_antecedent_output_path: null unless the antecedent is represented by a structured run output field. The only valid form is output.<field> (for example output.blocker). Do not emit transcript paths, selectors, regexes, JSONPath, screenshots, nested paths, or prose conditions here.+- deliverable_kind: null unless this criterion's requested output is satisfied by execution-owned browser-download registration rather than a user-authored extraction field. The only supported non-null value is "registered_download".- kind=outcome|terminal_action. Use terminal_action only when the criterion's end state is a terminal action in the external site for one of the supported families below. Use outcome for all other outcomes.- terminal_action_family=request|application|form|order|null. Set the matching family when kind=terminal_action; set null when kind=outcome.- Capture ALL distinct outcomes the goal requires, including implicit constraints the user did not state as a step (for example "added exactly once / not duplicated", "the item is present in the cart"). Mark such items implicit=true.diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.pyindex 837b30783..ca24fb2c2 100644--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py@@ -20,6 +20,7 @@ from skyvern.forge.sdk.copilot.request_policy import (CriterionKind,TerminalActionFamily,_normalize_contingent_antecedent_output_path,+ _normalize_deliverable_kind,is_fallback_floor_criterion,normalized_criterion_outcome_key,requested_output_path_for_field,@@ -117,6 +118,7 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion"outcome": criterion.outcome,"contingent_on": criterion.contingent_on,"contingent_antecedent_output_path": criterion.contingent_antecedent_output_path,+ "deliverable_kind": criterion.deliverable_kind,"implicit": criterion.implicit,"method_mandated": criterion.method_mandated,"level": criterion.level,@@ -159,6 +161,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:if isinstance(contingent_on, str) and contingent_on.strip()else None,contingent_antecedent_output_path=contingent_antecedent_output_path,+ deliverable_kind=_normalize_deliverable_kind(item.get("deliverable_kind")),implicit=bool(item.get("implicit")),method_mandated=bool(item.get("method_mandated")),level=level if isinstance(level, str) and level in _CRITERION_LEVELS else "run", # type: ignore[arg-type]@@ -173,13 +176,16 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:contingent_key = criterion.contingent_on or ""contingent_path_key = criterion.contingent_antecedent_output_path or ""+ deliverable_kind_key = criterion.deliverable_kind or ""if criterion.output_path:return (f"contingent:{contingent_key}\x1fantecedent_path:{contingent_path_key}"+ f"\x1fdeliverable_kind:{deliverable_kind_key}"f"\x1foutput_path:{criterion.output_path}")return (f"contingent:{contingent_key}\x1fantecedent_path:{contingent_path_key}"+ f"\x1fdeliverable_kind:{deliverable_kind_key}"f"\x1foutcome:{normalized_criterion_outcome_key(criterion.outcome)}")diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex 56ac68faa..93cac0a11 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -366,9 +366,10 @@ def _render_criteria(criteria: list[CompletionCriterion]) -> str:if criterion.contingent_antecedent_output_pathelse "")+ deliverable_kind = f" [deliverable_kind={criterion.deliverable_kind}]" if criterion.deliverable_kind else ""output_path = f" [required_output_path={criterion.output_path}]" if criterion.output_path else ""parts.append(- f"- {criterion.id}: {criterion.outcome}{contingent_on}{antecedent_output_path}{output_path}{flags}"+ f"- {criterion.id}: {criterion.outcome}{contingent_on}{antecedent_output_path}{deliverable_kind}{output_path}{flags}")return "\n".join(parts)diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex dd186c188..af84f8cd8 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -58,6 +58,13 @@ _CLASSIFICATION_RESPONSE_FIELDS = {"raw_secret_handling","clarification_reason",}+_REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS = frozenset(+ {+ "output.downloaded_files",+ "output.downloaded_file_urls",+ "output.downloaded_file_artifact_ids",+ }+)ClarificationReason = Literal["none","raw_secret",@@ -203,6 +210,7 @@ class CompletionCriterion:outcome: strcontingent_on: str | None = Nonecontingent_antecedent_output_path: str | None = None+ deliverable_kind: Literal["registered_download"] | None = Noneimplicit: bool = Falsemethod_mandated: bool = False# "definition": a property of the workflow definition itself, graded against the@@ -583,7 +591,7 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:if not isinstance(raw, list):return []criteria: list[CompletionCriterion] = []- seen: set[tuple[str, str, str]] = set()+ seen: set[tuple[str, str, str, str]] = set()for item in raw:if not isinstance(item, dict):continue@@ -605,10 +613,12 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:contingent_antecedent_output_path = _normalize_contingent_antecedent_output_path(item.get("contingent_antecedent_output_path"))+ deliverable_kind = _normalize_deliverable_kind(item.get("deliverable_kind"))key = (contingent_on or "",contingent_antecedent_output_path or "",output_path or normalized_criterion_outcome_key(outcome),+ deliverable_kind or "",)if key in seen:continue@@ -621,6 +631,7 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:outcome=outcome,contingent_on=contingent_on,contingent_antecedent_output_path=contingent_antecedent_output_path,+ deliverable_kind=deliverable_kind,implicit=bool(item.get("implicit")),method_mandated=bool(item.get("method_mandated")),level=cast(CriterionLevel, level_raw)@@ -643,6 +654,10 @@ def _normalize_contingent_antecedent_output_path(raw: Any) -> str | None:return path if _CONTINGENT_ANTECEDENT_OUTPUT_PATH_RE.fullmatch(path) else None+def _normalize_deliverable_kind(raw: Any) -> Literal["registered_download"] | None:+ return "registered_download" if raw == "registered_download" else None++def normalized_criterion_outcome_key(outcome: str) -> str:collapsed = " ".join((outcome or "").split())[:_COMPLETION_CRITERION_OUTCOME_MAX_CHARS]return collapsed.strip().lower().rstrip(".!?;:,").strip()@@ -924,17 +939,28 @@ def _apply_requested_output_completion_criteria(if not requested_output_paths:return- contingent_metadata_by_output_path: dict[str, tuple[str | None, str | None]] = {}+ metadata_by_output_path: dict[str, tuple[str | None, str | None, Literal["registered_download"] | None]] = {}for criterion in policy.completion_criteria:if criterion.level == "definition" or criterion.method_mandated:continue- if not criterion.contingent_on and not criterion.contingent_antecedent_output_path:+ if (+ not criterion.contingent_on+ and not criterion.contingent_antecedent_output_path+ and not criterion.deliverable_kind+ ):continuefor field_name, output_path, _field_label in requested_specs:if criterion.output_path == output_path or _criterion_text_covers_requested_output(criterion, field_name):- contingent_metadata_by_output_path.setdefault(+ deliverable_kind = (+ criterion.deliverable_kind if output_path in _REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS else None+ )+ metadata_by_output_path.setdefault(output_path,- (criterion.contingent_on, criterion.contingent_antecedent_output_path),+ (+ criterion.contingent_on,+ criterion.contingent_antecedent_output_path,+ deliverable_kind,+ ),)preserved_criteria = [@@ -953,8 +979,9 @@ def _apply_requested_output_completion_criteria(outcome=f"The returned record includes {field_label}.",level="run",output_path=output_path,- contingent_on=contingent_metadata_by_output_path.get(output_path, (None, None))[0],- contingent_antecedent_output_path=contingent_metadata_by_output_path.get(output_path, (None, None))[1],+ contingent_on=metadata_by_output_path.get(output_path, (None, None, None))[0],+ contingent_antecedent_output_path=metadata_by_output_path.get(output_path, (None, None, None))[1],+ deliverable_kind=metadata_by_output_path.get(output_path, (None, None, None))[2],)for _field_name, output_path, field_label in requested_specs]@@ -981,6 +1008,8 @@ def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | Nonitem["contingent_on"] = criterion.contingent_onif criterion.contingent_antecedent_output_path:item["contingent_antecedent_output_path"] = criterion.contingent_antecedent_output_path+ if criterion.deliverable_kind:+ item["deliverable_kind"] = criterion.deliverable_kindif criterion.output_path:item["output_path"] = criterion.output_pathrendered.append(item)diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 7763ab7bf..25dacce66 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -66,6 +66,7 @@ from .blockers import (LOG = structlog.get_logger()_TYPED_DOWNLOAD_KINDS = frozenset({DOWNLOAD_KIND_REGISTERED, DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION})+_REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS = frozenset(f"output.{key}" for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS)def _completion_request_policy(copilot_ctx: Any) -> Any | None:@@ -121,6 +122,15 @@ def _result_has_registered_download_block_output(result: dict[str, Any]) -> boolreturn derive_from_block_outputs(_result_block_outputs_by_label(result)) is not None+def _registered_download_requested_output_criterion(criterion: CompletionCriterion) -> bool:+ return (+ criterion.output_path is not None+ and criterion.level != "definition"+ and not criterion.method_mandated+ and criterion.output_path in _REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS+ )++def _has_typed_download_signal(copilot_ctx: Any, result: dict[str, Any]) -> bool:if _is_typed_download_target(_ctx_reached_download_target(copilot_ctx)):return True@@ -132,11 +142,17 @@ def _has_typed_download_signal(copilot_ctx: Any, result: dict[str, Any]) -> booldef _reconcile_download_completion_criterion(copilot_ctx: Any, result: dict[str, Any], criteria: list[CompletionCriterion]) -> list[CompletionCriterion]:- if any(is_registered_download_completion_criterion(criterion) for criterion in criteria):- return criteria+ has_registered_download_evidence = _result_has_registered_download_block_output(result)+ reconciled = (+ [criterion for criterion in criteria if not _registered_download_requested_output_criterion(criterion)]+ if has_registered_download_evidence+ else criteria+ )+ if any(is_registered_download_completion_criterion(criterion) for criterion in reconciled):+ return reconciledif not _has_typed_download_signal(copilot_ctx, result):- return criteria- return [*criteria, registered_download_completion_criterion()]+ return reconciled+ return [*reconciled, registered_download_completion_criterion()]def _completion_verification_criteria(copilot_ctx: Any) -> list[CompletionCriterion]:diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.pyindex 993a5f495..4d20d9718 100644--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py@@ -53,6 +53,7 @@ from skyvern.forge.sdk.copilot.request_policy import (CompletionCriterion,RequestPolicy,_parse_completion_criteria,+ _render_active_criteria_for_prompt,normalized_criterion_outcome_key,)from skyvern.forge.sdk.copilot.tools.completion import (@@ -62,8 +63,21 @@ from skyvern.forge.sdk.copilot.tools.completion import ()-def _criterion(cid: str, outcome: str, *, level: str = "run", output_path: str | None = None) -> CompletionCriterion:- return CompletionCriterion(id=cid, outcome=outcome, level=level, output_path=output_path) # type: ignore[arg-type]+def _criterion(+ cid: str,+ outcome: str,+ *,+ level: str = "run",+ output_path: str | None = None,+ deliverable_kind: str | None = None,+) -> CompletionCriterion:+ return CompletionCriterion(+ id=cid,+ outcome=outcome,+ level=level, # type: ignore[arg-type]+ output_path=output_path,+ deliverable_kind=deliverable_kind, # type: ignore[arg-type]+ )def _stored(@@ -273,6 +287,33 @@ def test_criteria_json_round_trip_preserves_terminal_action_fields() -> None:assert criteria_from_json(criteria_to_json(criteria)) == criteria+def test_criteria_json_round_trip_preserves_deliverable_kind() -> None:+ criteria = (+ _criterion(+ "c0",+ "The requested download is returned.",+ output_path="output.output_id",+ deliverable_kind="registered_download",+ ),+ )++ assert criteria_from_json(criteria_to_json(criteria)) == criteria+++def test_criteria_from_json_normalizes_unknown_deliverable_kind() -> None:+ (criterion,) = criteria_from_json(+ [+ {+ "id": "c0",+ "outcome": "The requested download is returned.",+ "deliverable_kind": "download",+ }+ ]+ )++ assert criterion.deliverable_kind is None++def test_criteria_from_json_coerces_invalid_level_to_run() -> None:(criterion,) = criteria_from_json([{"id": "c0", "outcome": "done", "level": "bogus"}])assert criterion.level == "run"@@ -408,6 +449,36 @@ def test_completion_verifier_render_includes_required_output_path() -> None:assert "required_output_path=output.id" in rendered+def test_active_criteria_rendering_includes_deliverable_kind() -> None:+ rendered = _render_active_criteria_for_prompt(+ [+ _criterion(+ "c0",+ "The requested download is returned.",+ output_path="output.output_id",+ deliverable_kind="registered_download",+ )+ ]+ )++ assert "registered_download" in rendered+++def test_completion_verifier_render_includes_deliverable_kind() -> None:+ rendered = _render_criteria(+ [+ _criterion(+ "c0",+ "The requested download is returned.",+ output_path="output.output_id",+ deliverable_kind="registered_download",+ )+ ]+ )++ assert "deliverable_kind=registered_download" in rendered++def test_required_output_path_missing_is_not_satisfied_by_evidence_text() -> None:id_criterion = _criterion("c1", "The returned record includes ID.", output_path="output.id")criteria = [@@ -581,6 +652,32 @@ def test_parse_completion_criteria_tolerates_unhashable_terminal_action_fields()assert [c.terminal_action_family for c in parsed] == [None, None]+def test_parse_completion_criteria_dedupes_by_deliverable_kind() -> None:+ parsed = _parse_completion_criteria(+ [+ {+ "outcome": "The returned record includes output id.",+ "output_path": "output.output_id",+ },+ {+ "outcome": "The returned record includes output id.",+ "output_path": "output.output_id",+ "deliverable_kind": "registered_download",+ },+ {+ "outcome": "The returned record includes output id.",+ "output_path": "output.output_id",+ "deliverable_kind": "registered_download",+ },+ ]+ )++ assert [(criterion.id, criterion.deliverable_kind) for criterion in parsed] == [+ ("c0", None),+ ("c1", "registered_download"),+ ]++def test_parse_completion_criteria_preserves_typed_terminal_action_fields() -> None:(criterion,) = _parse_completion_criteria([@@ -618,6 +715,49 @@ def test_parse_completion_criteria_defaults_invalid_terminal_action_fields() ->assert parsed[1].terminal_action_family is None+def test_adopted_active_criteria_retain_deliverable_kind() -> None:+ criterion = _criterion(+ "c0",+ "The requested download is returned.",+ output_path="output.output_id",+ deliverable_kind="registered_download",+ )+ stored = StoredCriteriaSet(set_id="wccs_1", goal_epoch=1, criteria=(criterion,))+ state = _adopted_turn_state(stored)++ assert state.decision is not None+ assert state.decision.criteria[0].deliverable_kind == "registered_download"+++def test_marked_deliverable_criterion_supersedes_unmarked_stored_criterion() -> None:+ stored = StoredCriteriaSet(+ set_id="wccs_1",+ goal_epoch=1,+ criteria=(+ _criterion(+ "s0",+ "The requested download is returned.",+ output_path="output.output_id",+ ),+ ),+ )+ fresh = [+ _criterion(+ "c0",+ "The requested download is returned.",+ output_path="output.output_id",+ deliverable_kind="registered_download",+ )+ ]++ decision = reconcile_completion_criteria(+ StoredCriteriaSnapshot(active=stored, next_epoch=2), fresh, actionable=True+ )++ assert decision.action == "create"+ assert decision.criteria[0].deliverable_kind == "registered_download"++def test_definition_grading_unknown_when_no_deterministic_check_applies() -> None:criteria = [_criterion("c0", "the summary email is well written", level="definition")](verdict,) = grade_definition_criteria(criteria, _PARAMETERIZED_YAML)diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 5a43bd55e..fd22a90d1 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -95,6 +95,7 @@ def _criterion(contingent_antecedent_output_path: str | None = None,kind: str = "outcome",terminal_action_family: str | None = None,+ deliverable_kind: str | None = None,) -> CompletionCriterion:return CompletionCriterion(id=cid,@@ -105,6 +106,7 @@ def _criterion(contingent_antecedent_output_path=contingent_antecedent_output_path,kind=kind,terminal_action_family=terminal_action_family,+ deliverable_kind=deliverable_kind, # type: ignore[arg-type])@@ -3993,6 +3995,251 @@ async def test_download_registered_output_parameter_injects_and_verifies_withoutassert ctx.request_policy.completion_criteria == []+async def test_marked_requested_output_id_still_requires_exact_field(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("marked requested-output failure must be deterministic")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "download_document")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("output_id")+ requested = _criterion(+ "c_output_id",+ "The returned record includes output id.",+ output_path="output.output_id",+ deliverable_kind="registered_download",+ )+ ctx.request_policy = RequestPolicy(completion_criteria=[requested])++ verification = await _maybe_run_completion_verification(+ ctx,+ _download_result(+ {+ "downloaded_files": [{"filename": "statement.pdf"}],+ "downloaded_file_urls": [],+ "downloaded_file_artifact_ids": [],+ }+ ),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is False+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}+ assert verdicts["c_output_id"].reason_code == "missing_exact_field"+ assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID in verdicts+ assert ctx.request_policy.completion_criteria == [requested]+++async def test_marked_requested_output_npi_still_requires_exact_field(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("marked requested-output failure must be deterministic")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "download_document")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")+ requested = _criterion(+ "c_npi",+ "The returned record includes NPI.",+ output_path="output.npi",+ deliverable_kind="registered_download",+ )+ ctx.request_policy = RequestPolicy(completion_criteria=[requested])++ verification = await _maybe_run_completion_verification(+ ctx,+ _download_result(+ {+ "downloaded_files": [{"filename": "statement.pdf"}],+ "downloaded_file_urls": [],+ "downloaded_file_artifact_ids": [],+ }+ ),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is False+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}+ assert verdicts["c_npi"].reason_code == "missing_exact_field"+ assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID in verdicts+ assert ctx.request_policy.completion_criteria == [requested]+++async def test_unmarked_requested_output_id_still_requires_exact_field(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("unmarked requested-output failure must be deterministic")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "download_document")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("output_id")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ _criterion(+ "c_output_id",+ "The returned record includes output id.",+ output_path="output.output_id",+ )+ ]+ )++ verification = await _maybe_run_completion_verification(+ ctx,+ _download_result(+ {+ "downloaded_files": [{"filename": "statement.pdf"}],+ "downloaded_file_urls": [],+ "downloaded_file_artifact_ids": [],+ }+ ),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is False+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}+ assert verdicts["c_output_id"].reason_code == "missing_exact_field"+++async def test_marked_download_deliverable_does_not_remove_mixed_extraction_fields(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("mixed requested-output failure must be deterministic")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "download_document")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("output_id", "npi")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ _criterion(+ "c_output_id",+ "The returned record includes output id.",+ output_path="output.output_id",+ deliverable_kind="registered_download",+ ),+ _criterion("c_npi", "The returned record includes NPI.", output_path="output.npi"),+ ]+ )++ verification = await _maybe_run_completion_verification(+ ctx,+ _download_result(+ {+ "downloaded_files": [{"filename": "statement.pdf"}],+ "downloaded_file_urls": [],+ "downloaded_file_artifact_ids": [],+ }+ ),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is False+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}+ assert verdicts["c_output_id"].reason_code == "missing_exact_field"+ assert verdicts["c_npi"].reason_code == "missing_exact_field"+ assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID in verdicts+++ ("output_path", "criterion_id"),+ [+ ("output.downloaded_files", "c_downloaded_files"),+ ("output.downloaded_file_urls", "c_downloaded_file_urls"),+ ("output.downloaded_file_artifact_ids", "c_downloaded_file_artifact_ids"),+ ],+)+async def test_download_output_path_reconciles_with_registered_download_evidence(+ monkeypatch: pytest.MonkeyPatch,+ output_path: str,+ criterion_id: str,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("registered download output path must bypass the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "download_document")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ _criterion(+ criterion_id,+ "The returned record includes registered download output.",+ output_path=output_path,+ )+ ]+ )++ verification = await _maybe_run_completion_verification(+ ctx,+ _download_result(+ {+ "downloaded_files": [{"filename": "statement.pdf"}],+ "downloaded_file_urls": [],+ "downloaded_file_artifact_ids": [],+ }+ ),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is True+ assert verification.criterion_ids == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]+++async def test_marked_download_deliverable_without_registered_evidence_fails_closed(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("missing registered download evidence must be deterministic")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "download_document")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("output_id")+ ctx.reached_download_target = ReachedDownloadTarget(+ selector='a[href="/files/statement.pdf"]',+ affordance_text="Download",+ download_kind="extension",+ source_step="trajectory_recency",+ already_registered=False,+ )+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ _criterion(+ "c_output_id",+ "The returned record includes output id.",+ output_path="output.output_id",+ deliverable_kind="registered_download",+ )+ ]+ )++ verification = await _maybe_run_completion_verification(ctx, _goto_only_result(), time.monotonic())++ assert verification is not None+ assert verification.is_fully_satisfied() is False+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}+ assert verdicts["c_output_id"].reason_code == "missing_exact_field"+ assert verdicts[REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID].reason_code == "no_evidence"++@pytest.mark.asyncioasync def test_download_typed_affordance_injects_and_fails_without_registered_output(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.pyindex 8373b74e7..4a86d060d 100644--- a/tests/unit/test_copilot_request_policy_completion_criteria.py+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py@@ -62,6 +62,7 @@ def _criterion(method_mandated: bool = False,contingent_on: str | None = None,contingent_antecedent_output_path: str | None = None,+ deliverable_kind: str | None = None,) -> CompletionCriterion:return CompletionCriterion(id=cid,@@ -71,6 +72,7 @@ def _criterion(method_mandated=method_mandated,contingent_on=contingent_on,contingent_antecedent_output_path=contingent_antecedent_output_path,+ deliverable_kind=deliverable_kind, # type: ignore[arg-type])@@ -421,6 +423,99 @@ def test_classifier_parse_preserves_contingent_antecedent_output_path_without_inassert criteria[3].contingent_antecedent_output_path is None+def test_classifier_parse_preserves_only_registered_download_deliverable_kind() -> None:+ criteria = _parse_completion_criteria(+ [+ {+ "outcome": "The requested download is returned.",+ "deliverable_kind": "registered_download",+ },+ {+ "outcome": "Unknown deliverable is ignored.",+ "deliverable_kind": "download",+ },+ {+ "outcome": "Absent deliverable stays empty.",+ },+ ]+ )++ assert [criterion.deliverable_kind for criterion in criteria] == ["registered_download", None, None]+++def test_requested_output_canonicalization_drops_marker_for_output_id() -> None:+ policy = RequestPolicy(+ completion_criteria=[+ _criterion(+ "c0",+ "The returned record includes output id.",+ output_path="output.output_id",+ deliverable_kind="registered_download",+ )+ ]+ )++ _apply_requested_output_completion_criteria(+ policy,+ "Return a final record with output id.",+ aliases={"output id": "output.output_id"},+ )++ criteria = _requested_output_subset(policy, {"output.output_id"})+ assert len(criteria) == 1+ assert criteria[0].id == "__copilot_requested_output__output_output_id"+ assert criteria[0].deliverable_kind is None+++def test_requested_output_canonicalization_drops_marker_for_npi() -> None:+ parsed = _parse_completion_criteria(+ [+ {+ "outcome": "The returned record includes NPI.",+ "output_path": "output.npi",+ "deliverable_kind": "registered_download",+ }+ ]+ )+ assert parsed[0].deliverable_kind == "registered_download"+ policy = RequestPolicy(completion_criteria=parsed)++ _apply_requested_output_completion_criteria(+ policy,+ "Return a final record with NPI.",+ aliases={"NPI": "output.npi"},+ )++ criteria = _requested_output_subset(policy, {"output.npi"})+ assert len(criteria) == 1+ assert criteria[0].id == "__copilot_requested_output__output_npi"+ assert criteria[0].deliverable_kind is None+++def test_requested_output_canonicalization_preserves_marker_for_approved_download_path() -> None:+ policy = RequestPolicy(+ completion_criteria=[+ _criterion(+ "c0",+ "The returned record includes download artifact.",+ output_path="output.output_id",+ deliverable_kind="registered_download",+ )+ ]+ )++ _apply_requested_output_completion_criteria(+ policy,+ "Return a final record with download artifact.",+ aliases={"download artifact": "output.downloaded_file_artifact_ids"},+ )++ criteria = _requested_output_subset(policy, {"output.downloaded_file_artifact_ids"})+ assert len(criteria) == 1+ assert criteria[0].id == "__copilot_requested_output__output_downloaded_file_artifact_ids"+ assert criteria[0].deliverable_kind == "registered_download"++def test_active_criteria_rendering_includes_contingent_on() -> None:rendered = _render_active_criteria_for_prompt([@@ -471,6 +566,32 @@ def test_active_criteria_rendering_includes_contingent_antecedent_output_path()]+def test_active_criteria_rendering_includes_deliverable_kind() -> None:+ rendered = _render_active_criteria_for_prompt(+ [+ _criterion(+ "c0",+ "The requested download is returned.",+ output_path="output.output_id",+ deliverable_kind="registered_download",+ )+ ]+ )++ assert json.loads(rendered) == [+ {+ "outcome": "The requested download is returned.",+ "implicit": False,+ "method_mandated": False,+ "level": "run",+ "kind": "outcome",+ "terminal_action_family": None,+ "deliverable_kind": "registered_download",+ "output_path": "output.output_id",+ }+ ]++def test_criteria_json_round_trips_contingent_on() -> None:criteria = (_criterion(diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.pyindex 188116864..0771a1de9 100644--- a/tests/unit/test_copilot_request_policy_prompt.py+++ b/tests/unit/test_copilot_request_policy_prompt.py@@ -88,10 +88,11 @@ class TestRequestPolicyPromptStructure:rendered = _render()assert ("{outcome, contingent_on, contingent_antecedent_output_path, "- "implicit, method_mandated, level, kind, terminal_action_family}"+ "deliverable_kind, implicit, method_mandated, level, kind, terminal_action_family}") in renderedassert "kind=outcome|terminal_action" in renderedassert "terminal_action_family=request|application|form|order|null" in rendered+ assert 'The only supported non-null value is "registered_download"' in rendereddef test_active_completion_criteria_render_typed_terminal_action_fields(self) -> None:active = _render_active_criteria_for_prompt(