Skip to content

Automate browser based workflows with AI

fix(SKY-11530): credit registered download output criteria (#6875)

Andrew Neilson5053de80d1a4
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
index 63f82c042..67d1fa256 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
@@ -25,10 +25,11 @@ completion_contract:
- Set to null when the user does not provide an explicit completion condition.
completion_criteria:
-- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, contingent_on, contingent_antecedent_output_path, implicit, method_mandated, level, kind, terminal_action_family}.
+- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, contingent_on, contingent_antecedent_output_path, deliverable_kind, implicit, method_mandated, level, kind, terminal_action_family}.
- outcome: a short statement of an end state the workflow must achieve (an END, not a step). Extract the goal as outcomes the user wants to be true, not the recipe they describe.
- contingent_on: null for ordinary criteria. Use a short antecedent string only when this criterion should be graded only if that antecedent happens, for example {"contingent_on": "the provider site blocks online submission", "contingent_antecedent_output_path": "output.blocker", "outcome": "the blocker is reported to the user"}. The antecedent goes in contingent_on; the consequent to grade goes in outcome. Do not encode if/then criteria only in prose.
- contingent_antecedent_output_path: null unless the antecedent is represented by a structured run output field. The only valid form is output.<field> (for example output.blocker). Do not emit transcript paths, selectors, regexes, JSONPath, screenshots, nested paths, or prose conditions here.
+- deliverable_kind: null unless this criterion's requested output is satisfied by execution-owned browser-download registration rather than a user-authored extraction field. The only supported non-null value is "registered_download".
- kind=outcome|terminal_action. Use terminal_action only when the criterion's end state is a terminal action in the external site for one of the supported families below. Use outcome for all other outcomes.
- terminal_action_family=request|application|form|order|null. Set the matching family when kind=terminal_action; set null when kind=outcome.
- Capture ALL distinct outcomes the goal requires, including implicit constraints the user did not state as a step (for example "added exactly once / not duplicated", "the item is present in the cart"). Mark such items implicit=true.
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index 837b30783..ca24fb2c2 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -20,6 +20,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
CriterionKind,
TerminalActionFamily,
_normalize_contingent_antecedent_output_path,
+ _normalize_deliverable_kind,
is_fallback_floor_criterion,
normalized_criterion_outcome_key,
requested_output_path_for_field,
@@ -117,6 +118,7 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion
"outcome": criterion.outcome,
"contingent_on": criterion.contingent_on,
"contingent_antecedent_output_path": criterion.contingent_antecedent_output_path,
+ "deliverable_kind": criterion.deliverable_kind,
"implicit": criterion.implicit,
"method_mandated": criterion.method_mandated,
"level": criterion.level,
@@ -159,6 +161,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
if isinstance(contingent_on, str) and contingent_on.strip()
else None,
contingent_antecedent_output_path=contingent_antecedent_output_path,
+ deliverable_kind=_normalize_deliverable_kind(item.get("deliverable_kind")),
implicit=bool(item.get("implicit")),
method_mandated=bool(item.get("method_mandated")),
level=level if isinstance(level, str) and level in _CRITERION_LEVELS else "run", # type: ignore[arg-type]
@@ -173,13 +176,16 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
contingent_key = criterion.contingent_on or ""
contingent_path_key = criterion.contingent_antecedent_output_path or ""
+ deliverable_kind_key = criterion.deliverable_kind or ""
if criterion.output_path:
return (
f"contingent:{contingent_key}\x1fantecedent_path:{contingent_path_key}"
+ f"\x1fdeliverable_kind:{deliverable_kind_key}"
f"\x1foutput_path:{criterion.output_path}"
)
return (
f"contingent:{contingent_key}\x1fantecedent_path:{contingent_path_key}"
+ f"\x1fdeliverable_kind:{deliverable_kind_key}"
f"\x1foutcome:{normalized_criterion_outcome_key(criterion.outcome)}"
)
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 56ac68faa..93cac0a11 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -366,9 +366,10 @@ def _render_criteria(criteria: list[CompletionCriterion]) -> str:
if criterion.contingent_antecedent_output_path
else ""
)
+ deliverable_kind = f" [deliverable_kind={criterion.deliverable_kind}]" if criterion.deliverable_kind else ""
output_path = f" [required_output_path={criterion.output_path}]" if criterion.output_path else ""
parts.append(
- f"- {criterion.id}: {criterion.outcome}{contingent_on}{antecedent_output_path}{output_path}{flags}"
+ f"- {criterion.id}: {criterion.outcome}{contingent_on}{antecedent_output_path}{deliverable_kind}{output_path}{flags}"
)
return "\n".join(parts)
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index dd186c188..af84f8cd8 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -58,6 +58,13 @@ _CLASSIFICATION_RESPONSE_FIELDS = {
"raw_secret_handling",
"clarification_reason",
}
+_REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS = frozenset(
+ {
+ "output.downloaded_files",
+ "output.downloaded_file_urls",
+ "output.downloaded_file_artifact_ids",
+ }
+)
ClarificationReason = Literal[
"none",
"raw_secret",
@@ -203,6 +210,7 @@ class CompletionCriterion:
outcome: str
contingent_on: str | None = None
contingent_antecedent_output_path: str | None = None
+ deliverable_kind: Literal["registered_download"] | None = None
implicit: bool = False
method_mandated: bool = False
# "definition": a property of the workflow definition itself, graded against the
@@ -583,7 +591,7 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
if not isinstance(raw, list):
return []
criteria: list[CompletionCriterion] = []
- seen: set[tuple[str, str, str]] = set()
+ seen: set[tuple[str, str, str, str]] = set()
for item in raw:
if not isinstance(item, dict):
continue
@@ -605,10 +613,12 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
contingent_antecedent_output_path = _normalize_contingent_antecedent_output_path(
item.get("contingent_antecedent_output_path")
)
+ deliverable_kind = _normalize_deliverable_kind(item.get("deliverable_kind"))
key = (
contingent_on or "",
contingent_antecedent_output_path or "",
output_path or normalized_criterion_outcome_key(outcome),
+ deliverable_kind or "",
)
if key in seen:
continue
@@ -621,6 +631,7 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
outcome=outcome,
contingent_on=contingent_on,
contingent_antecedent_output_path=contingent_antecedent_output_path,
+ deliverable_kind=deliverable_kind,
implicit=bool(item.get("implicit")),
method_mandated=bool(item.get("method_mandated")),
level=cast(CriterionLevel, level_raw)
@@ -643,6 +654,10 @@ def _normalize_contingent_antecedent_output_path(raw: Any) -> str | None:
return path if _CONTINGENT_ANTECEDENT_OUTPUT_PATH_RE.fullmatch(path) else None
+def _normalize_deliverable_kind(raw: Any) -> Literal["registered_download"] | None:
+ return "registered_download" if raw == "registered_download" else None
+
+
def normalized_criterion_outcome_key(outcome: str) -> str:
collapsed = " ".join((outcome or "").split())[:_COMPLETION_CRITERION_OUTCOME_MAX_CHARS]
return collapsed.strip().lower().rstrip(".!?;:,").strip()
@@ -924,17 +939,28 @@ def _apply_requested_output_completion_criteria(
if not requested_output_paths:
return
- contingent_metadata_by_output_path: dict[str, tuple[str | None, str | None]] = {}
+ metadata_by_output_path: dict[str, tuple[str | None, str | None, Literal["registered_download"] | None]] = {}
for criterion in policy.completion_criteria:
if criterion.level == "definition" or criterion.method_mandated:
continue
- if not criterion.contingent_on and not criterion.contingent_antecedent_output_path:
+ if (
+ not criterion.contingent_on
+ and not criterion.contingent_antecedent_output_path
+ and not criterion.deliverable_kind
+ ):
continue
for field_name, output_path, _field_label in requested_specs:
if criterion.output_path == output_path or _criterion_text_covers_requested_output(criterion, field_name):
- contingent_metadata_by_output_path.setdefault(
+ deliverable_kind = (
+ criterion.deliverable_kind if output_path in _REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS else None
+ )
+ metadata_by_output_path.setdefault(
output_path,
- (criterion.contingent_on, criterion.contingent_antecedent_output_path),
+ (
+ criterion.contingent_on,
+ criterion.contingent_antecedent_output_path,
+ deliverable_kind,
+ ),
)
preserved_criteria = [
@@ -953,8 +979,9 @@ def _apply_requested_output_completion_criteria(
outcome=f"The returned record includes {field_label}.",
level="run",
output_path=output_path,
- contingent_on=contingent_metadata_by_output_path.get(output_path, (None, None))[0],
- contingent_antecedent_output_path=contingent_metadata_by_output_path.get(output_path, (None, None))[1],
+ contingent_on=metadata_by_output_path.get(output_path, (None, None, None))[0],
+ contingent_antecedent_output_path=metadata_by_output_path.get(output_path, (None, None, None))[1],
+ deliverable_kind=metadata_by_output_path.get(output_path, (None, None, None))[2],
)
for _field_name, output_path, field_label in requested_specs
]
@@ -981,6 +1008,8 @@ def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | Non
item["contingent_on"] = criterion.contingent_on
if criterion.contingent_antecedent_output_path:
item["contingent_antecedent_output_path"] = criterion.contingent_antecedent_output_path
+ if criterion.deliverable_kind:
+ item["deliverable_kind"] = criterion.deliverable_kind
if criterion.output_path:
item["output_path"] = criterion.output_path
rendered.append(item)
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 7763ab7bf..25dacce66 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -66,6 +66,7 @@ from .blockers import (
LOG = structlog.get_logger()
_TYPED_DOWNLOAD_KINDS = frozenset({DOWNLOAD_KIND_REGISTERED, DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION})
+_REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS = frozenset(f"output.{key}" for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS)
def _completion_request_policy(copilot_ctx: Any) -> Any | None:
@@ -121,6 +122,15 @@ def _result_has_registered_download_block_output(result: dict[str, Any]) -> bool
return derive_from_block_outputs(_result_block_outputs_by_label(result)) is not None
+def _registered_download_requested_output_criterion(criterion: CompletionCriterion) -> bool:
+ return (
+ criterion.output_path is not None
+ and criterion.level != "definition"
+ and not criterion.method_mandated
+ and criterion.output_path in _REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS
+ )
+
+
def _has_typed_download_signal(copilot_ctx: Any, result: dict[str, Any]) -> bool:
if _is_typed_download_target(_ctx_reached_download_target(copilot_ctx)):
return True
@@ -132,11 +142,17 @@ def _has_typed_download_signal(copilot_ctx: Any, result: dict[str, Any]) -> bool
def _reconcile_download_completion_criterion(
copilot_ctx: Any, result: dict[str, Any], criteria: list[CompletionCriterion]
) -> list[CompletionCriterion]:
- if any(is_registered_download_completion_criterion(criterion) for criterion in criteria):
- return criteria
+ has_registered_download_evidence = _result_has_registered_download_block_output(result)
+ reconciled = (
+ [criterion for criterion in criteria if not _registered_download_requested_output_criterion(criterion)]
+ if has_registered_download_evidence
+ else criteria
+ )
+ if any(is_registered_download_completion_criterion(criterion) for criterion in reconciled):
+ return reconciled
if not _has_typed_download_signal(copilot_ctx, result):
- return criteria
- return [*criteria, registered_download_completion_criterion()]
+ return reconciled
+ return [*reconciled, registered_download_completion_criterion()]
def _completion_verification_criteria(copilot_ctx: Any) -> list[CompletionCriterion]:
diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.py
index 993a5f495..4d20d9718 100644
--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py
+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py
@@ -53,6 +53,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
RequestPolicy,
_parse_completion_criteria,
+ _render_active_criteria_for_prompt,
normalized_criterion_outcome_key,
)
from skyvern.forge.sdk.copilot.tools.completion import (
@@ -62,8 +63,21 @@ from skyvern.forge.sdk.copilot.tools.completion import (
)
-def _criterion(cid: str, outcome: str, *, level: str = "run", output_path: str | None = None) -> CompletionCriterion:
- return CompletionCriterion(id=cid, outcome=outcome, level=level, output_path=output_path) # type: ignore[arg-type]
+def _criterion(
+ cid: str,
+ outcome: str,
+ *,
+ level: str = "run",
+ output_path: str | None = None,
+ deliverable_kind: str | None = None,
+) -> CompletionCriterion:
+ return CompletionCriterion(
+ id=cid,
+ outcome=outcome,
+ level=level, # type: ignore[arg-type]
+ output_path=output_path,
+ deliverable_kind=deliverable_kind, # type: ignore[arg-type]
+ )
def _stored(
@@ -273,6 +287,33 @@ def test_criteria_json_round_trip_preserves_terminal_action_fields() -> None:
assert criteria_from_json(criteria_to_json(criteria)) == criteria
+def test_criteria_json_round_trip_preserves_deliverable_kind() -> None:
+ criteria = (
+ _criterion(
+ "c0",
+ "The requested download is returned.",
+ output_path="output.output_id",
+ deliverable_kind="registered_download",
+ ),
+ )
+
+ assert criteria_from_json(criteria_to_json(criteria)) == criteria
+
+
+def test_criteria_from_json_normalizes_unknown_deliverable_kind() -> None:
+ (criterion,) = criteria_from_json(
+ [
+ {
+ "id": "c0",
+ "outcome": "The requested download is returned.",
+ "deliverable_kind": "download",
+ }
+ ]
+ )
+
+ assert criterion.deliverable_kind is None
+
+
def test_criteria_from_json_coerces_invalid_level_to_run() -> None:
(criterion,) = criteria_from_json([{"id": "c0", "outcome": "done", "level": "bogus"}])
assert criterion.level == "run"
@@ -408,6 +449,36 @@ def test_completion_verifier_render_includes_required_output_path() -> None:
assert "required_output_path=output.id" in rendered
+def test_active_criteria_rendering_includes_deliverable_kind() -> None:
+ rendered = _render_active_criteria_for_prompt(
+ [
+ _criterion(
+ "c0",
+ "The requested download is returned.",
+ output_path="output.output_id",
+ deliverable_kind="registered_download",
+ )
+ ]
+ )
+
+ assert "registered_download" in rendered
+
+
+def test_completion_verifier_render_includes_deliverable_kind() -> None:
+ rendered = _render_criteria(
+ [
+ _criterion(
+ "c0",
+ "The requested download is returned.",
+ output_path="output.output_id",
+ deliverable_kind="registered_download",
+ )
+ ]
+ )
+
+ assert "deliverable_kind=registered_download" in rendered
+
+
def test_required_output_path_missing_is_not_satisfied_by_evidence_text() -> None:
id_criterion = _criterion("c1", "The returned record includes ID.", output_path="output.id")
criteria = [
@@ -581,6 +652,32 @@ def test_parse_completion_criteria_tolerates_unhashable_terminal_action_fields()
assert [c.terminal_action_family for c in parsed] == [None, None]
+def test_parse_completion_criteria_dedupes_by_deliverable_kind() -> None:
+ parsed = _parse_completion_criteria(
+ [
+ {
+ "outcome": "The returned record includes output id.",
+ "output_path": "output.output_id",
+ },
+ {
+ "outcome": "The returned record includes output id.",
+ "output_path": "output.output_id",
+ "deliverable_kind": "registered_download",
+ },
+ {
+ "outcome": "The returned record includes output id.",
+ "output_path": "output.output_id",
+ "deliverable_kind": "registered_download",
+ },
+ ]
+ )
+
+ assert [(criterion.id, criterion.deliverable_kind) for criterion in parsed] == [
+ ("c0", None),
+ ("c1", "registered_download"),
+ ]
+
+
def test_parse_completion_criteria_preserves_typed_terminal_action_fields() -> None:
(criterion,) = _parse_completion_criteria(
[
@@ -618,6 +715,49 @@ def test_parse_completion_criteria_defaults_invalid_terminal_action_fields() ->
assert parsed[1].terminal_action_family is None
+def test_adopted_active_criteria_retain_deliverable_kind() -> None:
+ criterion = _criterion(
+ "c0",
+ "The requested download is returned.",
+ output_path="output.output_id",
+ deliverable_kind="registered_download",
+ )
+ stored = StoredCriteriaSet(set_id="wccs_1", goal_epoch=1, criteria=(criterion,))
+ state = _adopted_turn_state(stored)
+
+ assert state.decision is not None
+ assert state.decision.criteria[0].deliverable_kind == "registered_download"
+
+
+def test_marked_deliverable_criterion_supersedes_unmarked_stored_criterion() -> None:
+ stored = StoredCriteriaSet(
+ set_id="wccs_1",
+ goal_epoch=1,
+ criteria=(
+ _criterion(
+ "s0",
+ "The requested download is returned.",
+ output_path="output.output_id",
+ ),
+ ),
+ )
+ fresh = [
+ _criterion(
+ "c0",
+ "The requested download is returned.",
+ output_path="output.output_id",
+ deliverable_kind="registered_download",
+ )
+ ]
+
+ decision = reconcile_completion_criteria(
+ StoredCriteriaSnapshot(active=stored, next_epoch=2), fresh, actionable=True
+ )
+
+ assert decision.action == "create"
+ assert decision.criteria[0].deliverable_kind == "registered_download"
+
+
def test_definition_grading_unknown_when_no_deterministic_check_applies() -> None:
criteria = [_criterion("c0", "the summary email is well written", level="definition")]
(verdict,) = grade_definition_criteria(criteria, _PARAMETERIZED_YAML)
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 5a43bd55e..fd22a90d1 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -95,6 +95,7 @@ def _criterion(
contingent_antecedent_output_path: str | None = None,
kind: str = "outcome",
terminal_action_family: str | None = None,
+ deliverable_kind: str | None = None,
) -> CompletionCriterion:
return CompletionCriterion(
id=cid,
@@ -105,6 +106,7 @@ def _criterion(
contingent_antecedent_output_path=contingent_antecedent_output_path,
kind=kind,
terminal_action_family=terminal_action_family,
+ deliverable_kind=deliverable_kind, # type: ignore[arg-type]
)
@@ -3993,6 +3995,251 @@ async def test_download_registered_output_parameter_injects_and_verifies_without
assert ctx.request_policy.completion_criteria == []
+async def test_marked_requested_output_id_still_requires_exact_field(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("marked requested-output failure must be deterministic")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "download_document")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("output_id")
+ requested = _criterion(
+ "c_output_id",
+ "The returned record includes output id.",
+ output_path="output.output_id",
+ deliverable_kind="registered_download",
+ )
+ ctx.request_policy = RequestPolicy(completion_criteria=[requested])
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _download_result(
+ {
+ "downloaded_files": [{"filename": "statement.pdf"}],
+ "downloaded_file_urls": [],
+ "downloaded_file_artifact_ids": [],
+ }
+ ),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is False
+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
+ assert verdicts["c_output_id"].reason_code == "missing_exact_field"
+ assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID in verdicts
+ assert ctx.request_policy.completion_criteria == [requested]
+
+
+async def test_marked_requested_output_npi_still_requires_exact_field(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("marked requested-output failure must be deterministic")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "download_document")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")
+ requested = _criterion(
+ "c_npi",
+ "The returned record includes NPI.",
+ output_path="output.npi",
+ deliverable_kind="registered_download",
+ )
+ ctx.request_policy = RequestPolicy(completion_criteria=[requested])
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _download_result(
+ {
+ "downloaded_files": [{"filename": "statement.pdf"}],
+ "downloaded_file_urls": [],
+ "downloaded_file_artifact_ids": [],
+ }
+ ),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is False
+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
+ assert verdicts["c_npi"].reason_code == "missing_exact_field"
+ assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID in verdicts
+ assert ctx.request_policy.completion_criteria == [requested]
+
+
+async def test_unmarked_requested_output_id_still_requires_exact_field(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("unmarked requested-output failure must be deterministic")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "download_document")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("output_id")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion(
+ "c_output_id",
+ "The returned record includes output id.",
+ output_path="output.output_id",
+ )
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _download_result(
+ {
+ "downloaded_files": [{"filename": "statement.pdf"}],
+ "downloaded_file_urls": [],
+ "downloaded_file_artifact_ids": [],
+ }
+ ),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is False
+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
+ assert verdicts["c_output_id"].reason_code == "missing_exact_field"
+
+
+async def test_marked_download_deliverable_does_not_remove_mixed_extraction_fields(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("mixed requested-output failure must be deterministic")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "download_document")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("output_id", "npi")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion(
+ "c_output_id",
+ "The returned record includes output id.",
+ output_path="output.output_id",
+ deliverable_kind="registered_download",
+ ),
+ _criterion("c_npi", "The returned record includes NPI.", output_path="output.npi"),
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _download_result(
+ {
+ "downloaded_files": [{"filename": "statement.pdf"}],
+ "downloaded_file_urls": [],
+ "downloaded_file_artifact_ids": [],
+ }
+ ),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is False
+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
+ assert verdicts["c_output_id"].reason_code == "missing_exact_field"
+ assert verdicts["c_npi"].reason_code == "missing_exact_field"
+ assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID in verdicts
+
+
+ ("output_path", "criterion_id"),
+ [
+ ("output.downloaded_files", "c_downloaded_files"),
+ ("output.downloaded_file_urls", "c_downloaded_file_urls"),
+ ("output.downloaded_file_artifact_ids", "c_downloaded_file_artifact_ids"),
+ ],
+)
+async def test_download_output_path_reconciles_with_registered_download_evidence(
+ monkeypatch: pytest.MonkeyPatch,
+ output_path: str,
+ criterion_id: str,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("registered download output path must bypass the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "download_document")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion(
+ criterion_id,
+ "The returned record includes registered download output.",
+ output_path=output_path,
+ )
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _download_result(
+ {
+ "downloaded_files": [{"filename": "statement.pdf"}],
+ "downloaded_file_urls": [],
+ "downloaded_file_artifact_ids": [],
+ }
+ ),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ assert verification.criterion_ids == [REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID]
+
+
+async def test_marked_download_deliverable_without_registered_evidence_fails_closed(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("missing registered download evidence must be deterministic")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "download_document")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("output_id")
+ ctx.reached_download_target = ReachedDownloadTarget(
+ selector='a[href="/files/statement.pdf"]',
+ affordance_text="Download",
+ download_kind="extension",
+ source_step="trajectory_recency",
+ already_registered=False,
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion(
+ "c_output_id",
+ "The returned record includes output id.",
+ output_path="output.output_id",
+ deliverable_kind="registered_download",
+ )
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(ctx, _goto_only_result(), time.monotonic())
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is False
+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
+ assert verdicts["c_output_id"].reason_code == "missing_exact_field"
+ assert verdicts[REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID].reason_code == "no_evidence"
+
+
@pytest.mark.asyncio
async def test_download_typed_affordance_injects_and_fails_without_registered_output(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.py
index 8373b74e7..4a86d060d 100644
--- a/tests/unit/test_copilot_request_policy_completion_criteria.py
+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py
@@ -62,6 +62,7 @@ def _criterion(
method_mandated: bool = False,
contingent_on: str | None = None,
contingent_antecedent_output_path: str | None = None,
+ deliverable_kind: str | None = None,
) -> CompletionCriterion:
return CompletionCriterion(
id=cid,
@@ -71,6 +72,7 @@ def _criterion(
method_mandated=method_mandated,
contingent_on=contingent_on,
contingent_antecedent_output_path=contingent_antecedent_output_path,
+ deliverable_kind=deliverable_kind, # type: ignore[arg-type]
)
@@ -421,6 +423,99 @@ def test_classifier_parse_preserves_contingent_antecedent_output_path_without_in
assert criteria[3].contingent_antecedent_output_path is None
+def test_classifier_parse_preserves_only_registered_download_deliverable_kind() -> None:
+ criteria = _parse_completion_criteria(
+ [
+ {
+ "outcome": "The requested download is returned.",
+ "deliverable_kind": "registered_download",
+ },
+ {
+ "outcome": "Unknown deliverable is ignored.",
+ "deliverable_kind": "download",
+ },
+ {
+ "outcome": "Absent deliverable stays empty.",
+ },
+ ]
+ )
+
+ assert [criterion.deliverable_kind for criterion in criteria] == ["registered_download", None, None]
+
+
+def test_requested_output_canonicalization_drops_marker_for_output_id() -> None:
+ policy = RequestPolicy(
+ completion_criteria=[
+ _criterion(
+ "c0",
+ "The returned record includes output id.",
+ output_path="output.output_id",
+ deliverable_kind="registered_download",
+ )
+ ]
+ )
+
+ _apply_requested_output_completion_criteria(
+ policy,
+ "Return a final record with output id.",
+ aliases={"output id": "output.output_id"},
+ )
+
+ criteria = _requested_output_subset(policy, {"output.output_id"})
+ assert len(criteria) == 1
+ assert criteria[0].id == "__copilot_requested_output__output_output_id"
+ assert criteria[0].deliverable_kind is None
+
+
+def test_requested_output_canonicalization_drops_marker_for_npi() -> None:
+ parsed = _parse_completion_criteria(
+ [
+ {
+ "outcome": "The returned record includes NPI.",
+ "output_path": "output.npi",
+ "deliverable_kind": "registered_download",
+ }
+ ]
+ )
+ assert parsed[0].deliverable_kind == "registered_download"
+ policy = RequestPolicy(completion_criteria=parsed)
+
+ _apply_requested_output_completion_criteria(
+ policy,
+ "Return a final record with NPI.",
+ aliases={"NPI": "output.npi"},
+ )
+
+ criteria = _requested_output_subset(policy, {"output.npi"})
+ assert len(criteria) == 1
+ assert criteria[0].id == "__copilot_requested_output__output_npi"
+ assert criteria[0].deliverable_kind is None
+
+
+def test_requested_output_canonicalization_preserves_marker_for_approved_download_path() -> None:
+ policy = RequestPolicy(
+ completion_criteria=[
+ _criterion(
+ "c0",
+ "The returned record includes download artifact.",
+ output_path="output.output_id",
+ deliverable_kind="registered_download",
+ )
+ ]
+ )
+
+ _apply_requested_output_completion_criteria(
+ policy,
+ "Return a final record with download artifact.",
+ aliases={"download artifact": "output.downloaded_file_artifact_ids"},
+ )
+
+ criteria = _requested_output_subset(policy, {"output.downloaded_file_artifact_ids"})
+ assert len(criteria) == 1
+ assert criteria[0].id == "__copilot_requested_output__output_downloaded_file_artifact_ids"
+ assert criteria[0].deliverable_kind == "registered_download"
+
+
def test_active_criteria_rendering_includes_contingent_on() -> None:
rendered = _render_active_criteria_for_prompt(
[
@@ -471,6 +566,32 @@ def test_active_criteria_rendering_includes_contingent_antecedent_output_path()
]
+def test_active_criteria_rendering_includes_deliverable_kind() -> None:
+ rendered = _render_active_criteria_for_prompt(
+ [
+ _criterion(
+ "c0",
+ "The requested download is returned.",
+ output_path="output.output_id",
+ deliverable_kind="registered_download",
+ )
+ ]
+ )
+
+ assert json.loads(rendered) == [
+ {
+ "outcome": "The requested download is returned.",
+ "implicit": False,
+ "method_mandated": False,
+ "level": "run",
+ "kind": "outcome",
+ "terminal_action_family": None,
+ "deliverable_kind": "registered_download",
+ "output_path": "output.output_id",
+ }
+ ]
+
+
def test_criteria_json_round_trips_contingent_on() -> None:
criteria = (
_criterion(
diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.py
index 188116864..0771a1de9 100644
--- a/tests/unit/test_copilot_request_policy_prompt.py
+++ b/tests/unit/test_copilot_request_policy_prompt.py
@@ -88,10 +88,11 @@ class TestRequestPolicyPromptStructure:
rendered = _render()
assert (
"{outcome, contingent_on, contingent_antecedent_output_path, "
- "implicit, method_mandated, level, kind, terminal_action_family}"
+ "deliverable_kind, implicit, method_mandated, level, kind, terminal_action_family}"
) in rendered
assert "kind=outcome|terminal_action" in rendered
assert "terminal_action_family=request|application|form|order|null" in rendered
+ assert 'The only supported non-null value is "registered_download"' in rendered
def test_active_completion_criteria_render_typed_terminal_action_fields(self) -> None:
active = _render_active_criteria_for_prompt(