Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11510): type terminal action criteria (#6863)
Andrew Neilson9dfafd5fccf2
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2index 314b26e5d..63f82c042 100644--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2@@ -25,10 +25,12 @@ completion_contract:- Set to null when the user does not provide an explicit completion condition.completion_criteria:-- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, contingent_on, contingent_antecedent_output_path, implicit, method_mandated, level}.+- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, contingent_on, contingent_antecedent_output_path, implicit, method_mandated, level, kind, terminal_action_family}.- outcome: a short statement of an end state the workflow must achieve (an END, not a step). Extract the goal as outcomes the user wants to be true, not the recipe they describe.- contingent_on: null for ordinary criteria. Use a short antecedent string only when this criterion should be graded only if that antecedent happens, for example {"contingent_on": "the provider site blocks online submission", "contingent_antecedent_output_path": "output.blocker", "outcome": "the blocker is reported to the user"}. The antecedent goes in contingent_on; the consequent to grade goes in outcome. Do not encode if/then criteria only in prose.- contingent_antecedent_output_path: null unless the antecedent is represented by a structured run output field. The only valid form is output.<field> (for example output.blocker). Do not emit transcript paths, selectors, regexes, JSONPath, screenshots, nested paths, or prose conditions here.+- kind=outcome|terminal_action. Use terminal_action only when the criterion's end state is a terminal action in the external site for one of the supported families below. Use outcome for all other outcomes.+- terminal_action_family=request|application|form|order|null. Set the matching family when kind=terminal_action; set null when kind=outcome.- Capture ALL distinct outcomes the goal requires, including implicit constraints the user did not state as a step (for example "added exactly once / not duplicated", "the item is present in the cart"). Mark such items implicit=true.- A prescribed HOW step is NOT a criterion. "search for X", "open the result lightbox", "click add to cart", "go to the cart page" are methods/navigation, not outcomes — do not list them. A different or shorter path that reaches the same end state must still satisfy the goal.- Telling/showing/reporting results to the user is NOT a criterion. "tell me what it extracted", "show me the data", "confirm to me it worked" describe the chat reply, which no workflow definition or run can evidence — capture only the underlying workflow outcome (the data is extracted into the run output, the form is submitted) and list nothing for the reporting itself.diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.pyindex cbd579bad..837b30783 100644--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py@@ -9,7 +9,7 @@ stays at the route/repository seam; everything here is side-effect free.from __future__ import annotationsfrom dataclasses import dataclass, field-from typing import Any, Literal+from typing import Any, Literal, castfrom skyvern.forge.sdk.copilot.completion_verification import (CompletionVerificationResult,@@ -17,6 +17,8 @@ from skyvern.forge.sdk.copilot.completion_verification import ()from skyvern.forge.sdk.copilot.request_policy import (CompletionCriterion,+ CriterionKind,+ TerminalActionFamily,_normalize_contingent_antecedent_output_path,is_fallback_floor_criterion,normalized_criterion_outcome_key,@@ -31,6 +33,8 @@ CRITERIA_SET_STATUS_ACTIVE = "active"CRITERIA_SET_STATUS_SUPERSEDED = "superseded"TRIPWIRE_CONSECUTIVE_ALL_NO_EVIDENCE = 2_CRITERION_LEVELS = ("definition", "run")+_CRITERION_KINDS = ("outcome", "terminal_action")+_TERMINAL_ACTION_FAMILIES = ("request", "application", "form", "order")@dataclass(frozen=True)@@ -117,6 +121,8 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion"method_mandated": criterion.method_mandated,"level": criterion.level,"output_path": criterion.output_path,+ "kind": criterion.kind,+ "terminal_action_family": criterion.terminal_action_family,}for criterion in criteria]@@ -139,6 +145,12 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:contingent_antecedent_output_path = _normalize_contingent_antecedent_output_path(item.get("contingent_antecedent_output_path"))+ kind_raw = item.get("kind")+ kind = kind_raw if isinstance(kind_raw, str) and kind_raw in _CRITERION_KINDS else "outcome"+ family_raw = item.get("terminal_action_family")+ terminal_action_family = (+ family_raw if kind == "terminal_action" and family_raw in _TERMINAL_ACTION_FAMILIES else None+ )criteria.append(CompletionCriterion(id=criterion_id,@@ -151,6 +163,8 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:method_mandated=bool(item.get("method_mandated")),level=level if isinstance(level, str) and level in _CRITERION_LEVELS else "run", # type: ignore[arg-type]output_path=output_path.strip() if isinstance(output_path, str) and output_path.strip() else None,+ kind=cast(CriterionKind, kind),+ terminal_action_family=cast(TerminalActionFamily | None, terminal_action_family),))return tuple(criteria)diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex 33975afe0..56ac68faa 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -25,6 +25,7 @@ from skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.copilot.output_utils import parse_final_responsefrom skyvern.forge.sdk.copilot.request_policy import (CompletionCriterion,+ TerminalActionFamily,is_fallback_floor_base_criterion,redact_raw_secrets_for_prompt,)@@ -852,19 +853,7 @@ _TERMINAL_ARTIFACT_KEY_TOKENS = ((("reference", "number"), "reference"),(("reference", "id"), "reference"),)-_TERMINAL_CRITERION_VERBS = frozenset({"submit", "submitted", "submission", "place", "placed"})-_TERMINAL_CRITERION_ABSTAIN_TOKENS = frozenset(- {- "lookup",- "looked",- "retrieve",- "retrieved",- "retrieval",- "return",- "returned",- "status",- }-)+_GENERIC_TERMINAL_SUCCESS_LEAF_TOKENS = frozenset({"submit", "submitted", "submission", "place", "placed"})_NEGATIVE_GUARD_TOKENS = frozenset({"blocker", "error", "failure", "failed", "challenge"})_NEGATIVE_TERMINAL_STATUS_VALUES = frozenset({@@ -882,13 +871,14 @@ _NEGATIVE_TERMINAL_STATUS_VALUES = frozenset("unable",})-_TERMINAL_RECORD_FAMILY_ARTIFACTS = {+_TERMINAL_RECORD_FAMILIES: tuple[TerminalActionFamily, ...] = ("request", "application", "form", "order")+_TERMINAL_RECORD_FAMILY_ARTIFACTS: dict[TerminalActionFamily, frozenset[str]] = {"request": frozenset({"confirmation", "reference", "request", "submission"}),"application": frozenset({"application", "confirmation", "reference", "submission"}),"form": frozenset({"confirmation", "reference", "submission"}),"order": frozenset({"order", "receipt"}),}-_TERMINAL_RECORD_FAMILY_ACTIONS = {+_TERMINAL_RECORD_FAMILY_ACTIONS: dict[TerminalActionFamily, frozenset[str]] = {"request": frozenset({"request", "submission"}),"application": frozenset({"application", "submission"}),"form": frozenset({"form", "submission"}),@@ -901,9 +891,9 @@ def grade_terminal_goal_record_criteria() -> list[CriterionVerdict]:verdicts: list[CriterionVerdict] = []eligible_criteria = [- (criterion, family)+ criterionfor criterion in criteria- if (family := _terminal_goal_criterion_family(criterion.outcome)) is not None+ if criterion.kind == "terminal_action" and criterion.terminal_action_family in _TERMINAL_RECORD_FAMILY_ARTIFACTS]if not eligible_criteria:return []@@ -911,8 +901,9 @@ def grade_terminal_goal_record_criteria(record = _structured_record_payload(payload)if record is None:continue- for criterion, family in eligible_criteria:- if _terminal_goal_record_confirmed(record, family):+ for criterion in eligible_criteria:+ family = criterion.terminal_action_family+ if family is not None and _terminal_goal_record_confirmed(record, family):verdicts.append(CriterionVerdict(criterion_id=criterion.id,@@ -951,29 +942,12 @@ def _fallback_floor_reached_end_state_evidence_ref(snapshot: RunEvidenceSnapshotrecord = _structured_record_payload(payload)if record is None:continue- if any(_terminal_goal_record_confirmed(record, family) for family in _TERMINAL_RECORD_FAMILY_ARTIFACTS):+ if any(_terminal_goal_record_confirmed(record, family) for family in _TERMINAL_RECORD_FAMILIES):return f"block_outputs:{label}"return None-def _terminal_goal_criterion_family(outcome: str) -> str | None:- if _extract_present_value_literals(outcome):- return None- tokens = set(_status_word_tokens(outcome))- if tokens & _TERMINAL_CRITERION_ABSTAIN_TOKENS:- return None- if not tokens & _TERMINAL_CRITERION_VERBS:- return None- families = [family for family in ("request", "application", "form", "order") if family in tokens]- if len(families) != 1:- return None- family = families[0]- if family == "order" and "placed" not in tokens:- return None- return family---def _terminal_goal_record_confirmed(record: dict[str, Any], family: str) -> bool:+def _terminal_goal_record_confirmed(record: dict[str, Any], family: TerminalActionFamily) -> bool:if _terminal_goal_record_has_negative_guard(record):return Falseif _structured_record_contradiction(record):@@ -1016,7 +990,7 @@ def _terminal_goal_record_has_generic_success_claim(record: dict[str, Any]) -> bfor key, value in _walk_record_scalars(record):leaf_tokens = _record_key_leaf_tokens(key)if value is True and _terminal_action_family_for_key(key) is None and leaf_tokens:- if leaf_tokens[-1] in _TERMINAL_CRITERION_VERBS:+ if leaf_tokens[-1] in _GENERIC_TERMINAL_SUCCESS_LEAF_TOKENS:return Trueif leaf_tokens in generic_success_keys and value is True:return True@@ -1044,7 +1018,7 @@ def _terminal_goal_record_has_negative_guard(record: dict[str, Any]) -> bool:return False-def _terminal_goal_record_has_artifact_for_family(record: dict[str, Any], family: str) -> bool:+def _terminal_goal_record_has_artifact_for_family(record: dict[str, Any], family: TerminalActionFamily) -> bool:allowed_artifacts = _TERMINAL_RECORD_FAMILY_ARTIFACTS[family]for key, value in _walk_record_scalars(record):if isinstance(value, bool) or not _is_meaningful_record_value(value):diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 8c83a700c..c1ab93c57 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -157,6 +157,10 @@ _FALLBACK_FLOOR_CREDENTIAL_OUTCOME = "The credentialed step authenticates and reCriterionLevel = Literal["definition", "run"]_CRITERION_LEVELS: frozenset[str] = frozenset({"definition", "run"})+CriterionKind = Literal["outcome", "terminal_action"]+TerminalActionFamily = Literal["request", "application", "form", "order"]+_CRITERION_KINDS: frozenset[str] = frozenset({"outcome", "terminal_action"})+_TERMINAL_ACTION_FAMILIES: frozenset[str] = frozenset({"request", "application", "form", "order"})_OUTPUT_INTENT_RE = re.compile(r"\b(?:read|capture|extract|output|return|returns|returned|include|includes|including|"@@ -205,6 +209,8 @@ class CompletionCriterion:# YAML; "run": an end state only a run can evidence. Invalid input coerces to "run".level: CriterionLevel = "run"output_path: str | None = None+ kind: CriterionKind = "outcome"+ terminal_action_family: TerminalActionFamily | None = None@dataclass@@ -486,6 +492,18 @@ def _coerce_raw_secret_handling(value: Any) -> RawSecretHandling:return "none"+def _coerce_criterion_kind(value: Any) -> CriterionKind:+ if isinstance(value, str) and value in _CRITERION_KINDS:+ return cast(CriterionKind, value)+ return "outcome"+++def _coerce_terminal_action_family(value: Any, kind: CriterionKind) -> TerminalActionFamily | None:+ if kind == "terminal_action" and isinstance(value, str) and value in _TERMINAL_ACTION_FAMILIES:+ return cast(TerminalActionFamily, value)+ return None++def _coerce_classifier_payload(raw: Any) -> dict[str, Any] | None:if isinstance(raw, str):raw = parse_final_response(raw)@@ -590,6 +608,7 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:continueseen.add(key)level_raw = item.get("level")+ kind = _coerce_criterion_kind(item.get("kind"))criteria.append(CompletionCriterion(id=f"c{len(criteria)}",@@ -602,6 +621,8 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:if isinstance(level_raw, str) and level_raw in _CRITERION_LEVELSelse "run",output_path=output_path,+ kind=kind,+ terminal_action_family=_coerce_terminal_action_family(item.get("terminal_action_family"), kind),))if len(criteria) >= _MAX_COMPLETION_CRITERIA:@@ -947,6 +968,8 @@ def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | Non"implicit": criterion.implicit,"method_mandated": criterion.method_mandated,"level": criterion.level,+ "kind": criterion.kind,+ "terminal_action_family": criterion.terminal_action_family,}if criterion.contingent_on:item["contingent_on"] = criterion.contingent_ondiff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.pyindex a25f6baa7..2d4401140 100644--- a/skyvern/forge/sdk/copilot/tools/_shared.py+++ b/skyvern/forge/sdk/copilot/tools/_shared.py@@ -202,8 +202,14 @@ def _registered_output_parameter_payloads(data: Mapping[str, Any]) -> list[Mappiaccumulated outputs cannot satisfy the current run's completion contract."""run_id = data.get("workflow_run_id")- registered = data.get("registered_output_parameter_values")- if not isinstance(registered, list):+ registered_values = data.get("registered_output_parameter_values")+ registered_outputs = data.get("workflow_run_output_parameters")+ registered = []+ if isinstance(registered_values, list):+ registered.extend(registered_values)+ if isinstance(registered_outputs, list):+ registered.extend(registered_outputs)+ if not registered:return []if not isinstance(run_id, str):# Without a current run id the payloads can't be attributed to this run;diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.pyindex 12e142e47..993a5f495 100644--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py@@ -260,6 +260,19 @@ def test_criteria_json_round_trip_preserves_level_and_flags() -> None:assert criteria_from_json(criteria_to_json(criteria)) == criteria+def test_criteria_json_round_trip_preserves_terminal_action_fields() -> None:+ criteria = (+ CompletionCriterion(+ id="c0",+ outcome="a commercial water service request is started",+ kind="terminal_action",+ terminal_action_family="request",+ ),+ )++ assert criteria_from_json(criteria_to_json(criteria)) == criteria++def test_criteria_from_json_coerces_invalid_level_to_run() -> None:(criterion,) = criteria_from_json([{"id": "c0", "outcome": "done", "level": "bogus"}])assert criterion.level == "run"@@ -548,6 +561,63 @@ def test_parse_completion_criteria_tolerates_unhashable_level() -> None:assert [c.level for c in parsed] == ["run", "run"]+def test_parse_completion_criteria_tolerates_unhashable_terminal_action_fields() -> None:+ parsed = _parse_completion_criteria(+ [+ {+ "outcome": "a commercial water service request is started",+ "kind": ["terminal_action"],+ "terminal_action_family": ["request"],+ },+ {+ "outcome": "a permit application is submitted",+ "kind": "terminal_action",+ "terminal_action_family": ["application"],+ },+ ]+ )++ assert [c.kind for c in parsed] == ["outcome", "terminal_action"]+ assert [c.terminal_action_family for c in parsed] == [None, None]+++def test_parse_completion_criteria_preserves_typed_terminal_action_fields() -> None:+ (criterion,) = _parse_completion_criteria(+ [+ {+ "outcome": "a commercial water service request is started",+ "kind": "terminal_action",+ "terminal_action_family": "request",+ }+ ]+ )++ assert criterion.kind == "terminal_action"+ assert criterion.terminal_action_family == "request"+++def test_parse_completion_criteria_defaults_invalid_terminal_action_fields() -> None:+ parsed = _parse_completion_criteria(+ [+ {+ "outcome": "a commercial water service request is started",+ "kind": "terminal_action",+ "terminal_action_family": "invoice",+ },+ {+ "outcome": "the item is in the cart",+ "kind": "done",+ "terminal_action_family": "request",+ },+ ]+ )++ assert parsed[0].kind == "terminal_action"+ assert parsed[0].terminal_action_family is None+ assert parsed[1].kind == "outcome"+ assert parsed[1].terminal_action_family is None++def test_definition_grading_unknown_when_no_deterministic_check_applies() -> None:criteria = [_criterion("c0", "the summary email is well written", level="definition")](verdict,) = grade_definition_criteria(criteria, _PARAMETERIZED_YAML)diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 90dde3b81..5a43bd55e 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -93,6 +93,8 @@ def _criterion(output_path: str | None = None,contingent_on: str | None = None,contingent_antecedent_output_path: str | None = None,+ kind: str = "outcome",+ terminal_action_family: str | None = None,) -> CompletionCriterion:return CompletionCriterion(id=cid,@@ -101,6 +103,8 @@ def _criterion(output_path=output_path,contingent_on=contingent_on,contingent_antecedent_output_path=contingent_antecedent_output_path,+ kind=kind,+ terminal_action_family=terminal_action_family,)@@ -769,7 +773,15 @@ def test_terminal_goal_record_satisfies_flat_submit_payload() -> None:snapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": _terminal_goal_payload()})verdicts = grade_terminal_goal_record_criteria(- [_criterion("c0", "a commercial water service request is submitted")], snapshot+ [+ _criterion(+ "c0",+ "a commercial water service request is submitted",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ],+ snapshot,)assert verdicts == [@@ -787,12 +799,133 @@ def test_terminal_goal_record_accepts_family_artifact_without_self_asserted_boolsnapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": payload})verdicts = grade_terminal_goal_record_criteria(- [_criterion("c0", "a commercial water service request is submitted")], snapshot+ [+ _criterion(+ "c0",+ "a commercial water service request is submitted",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ],+ snapshot,)assert _satisfied_criterion_ids(verdicts) == {"c0"}+def test_terminal_goal_record_accepts_started_request_when_typed_with_artifact() -> None:+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": _terminal_goal_payload()})++ verdicts = grade_terminal_goal_record_criteria(+ [+ _criterion(+ "c0",+ "a commercial water service request is started",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ],+ snapshot,+ )++ assert _satisfied_criterion_ids(verdicts) == {"c0"}+++def test_terminal_goal_record_accepts_typed_create_or_verify_with_identifier() -> None:+ payload = _terminal_goal_payload(+ submitted=None,+ confirmation_number=None,+ request_id="QC-2002-DEMO",+ submission_result="already_present",+ )+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})++ verdicts = grade_terminal_goal_record_criteria(+ [+ _criterion(+ "c0",+ "the service request is created or verified",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ],+ snapshot,+ )++ assert _satisfied_criterion_ids(verdicts) == {"c0"}+++def test_terminal_goal_record_submission_result_is_not_credit_source() -> None:+ payload = _terminal_goal_payload(+ submitted=None,+ confirmation_number=None,+ request_id=None,+ submission_result="already_present",+ )+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})++ assert (+ grade_terminal_goal_record_criteria(+ [+ _criterion(+ "c0",+ "the service request is created or verified",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ],+ snapshot,+ )+ == []+ )+++def test_terminal_goal_record_requires_typed_terminal_action_family() -> None:+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": _terminal_goal_payload()})++ assert grade_terminal_goal_record_criteria([_criterion("c0", "a request is submitted")], snapshot) == []+ assert (+ grade_terminal_goal_record_criteria(+ [_criterion("c1", "a request is submitted", kind="terminal_action")], snapshot+ )+ == []+ )+ assert (+ grade_terminal_goal_record_criteria(+ [+ _criterion(+ "c2",+ "a request is submitted",+ kind="terminal_action",+ terminal_action_family="invoice",+ )+ ],+ snapshot,+ )+ == []+ )+++def test_terminal_goal_record_self_asserted_boolean_without_identifier_remains_uncredited() -> None:+ payload = _terminal_goal_payload(submitted=True, confirmation_number=None)+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})++ assert (+ grade_terminal_goal_record_criteria(+ [+ _criterion(+ "c0",+ "a request is submitted",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ],+ snapshot,+ )+ == []+ )++@pytest.mark.parametrize("payload",[@@ -808,57 +941,87 @@ def test_terminal_goal_record_rejects_ordinary_identifiers(payload: dict[str, Anassert (grade_terminal_goal_record_criteria(- [_criterion("c0", "a commercial water service request is submitted")], snapshot+ [+ _criterion(+ "c0",+ "a commercial water service request is submitted",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ],+ snapshot,)== [])@pytest.mark.parametrize(- ("payload", "outcome"),+ ("payload", "outcome", "family"),[(_terminal_goal_payload(confirmation_number="WTR-1842-DEMO"),"a commercial water service request is submitted",+ "request",+ ),+ (+ _terminal_goal_payload(order_placed=True, submitted=None, order_number="ORD-1842"),+ "an order is placed",+ "order",),- (_terminal_goal_payload(order_placed=True, submitted=None, order_number="ORD-1842"), "an order is placed"),(_terminal_goal_payload(application_submitted=True, submitted=None, application_id="APP-1842"),"an application is submitted",+ "application",),(_terminal_goal_payload(form_submitted=True, submitted=None, submission_id="SUB-1842"),"the form is submitted",+ "form",),(_terminal_goal_payload(request_submitted=True, submitted=None, request_id="REQ-1842"),"a service request is submitted",+ "request",),],)-def test_terminal_goal_record_accepts_narrow_terminal_artifacts(payload: dict[str, Any], outcome: str) -> None:+def test_terminal_goal_record_accepts_narrow_terminal_artifacts(+ payload: dict[str, Any], outcome: str, family: str+) -> None:snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})- verdicts = grade_terminal_goal_record_criteria([_criterion("c0", outcome)], snapshot)+ verdicts = grade_terminal_goal_record_criteria(+ [_criterion("c0", outcome, kind="terminal_action", terminal_action_family=family)], snapshot+ )assert _satisfied_criterion_ids(verdicts) == {"c0"}@pytest.mark.parametrize(- ("payload", "outcome"),+ ("payload", "outcome", "family"),[- (_terminal_goal_payload(), "an order is placed"),- (_terminal_goal_payload(order_placed=True, submitted=None, order_number="ORD-1842"), "a request is submitted"),+ (_terminal_goal_payload(), "an order is placed", "order"),+ (+ _terminal_goal_payload(order_placed=True, submitted=None, order_number="ORD-1842"),+ "a request is submitted",+ "request",+ ),(_terminal_goal_payload(application_submitted=True, submitted=None, application_id="APP-1842"),"an order is placed",+ "order",),],)-def test_terminal_goal_record_rejects_mismatched_families(payload: dict[str, Any], outcome: str) -> None:+def test_terminal_goal_record_rejects_mismatched_families(payload: dict[str, Any], outcome: str, family: str) -> None:snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})- assert grade_terminal_goal_record_criteria([_criterion("c0", outcome)], snapshot) == []+ assert (+ grade_terminal_goal_record_criteria(+ [_criterion("c0", outcome, kind="terminal_action", terminal_action_family=family)], snapshot+ )+ == []+ )@pytest.mark.parametrize(@@ -873,7 +1036,20 @@ def test_terminal_goal_record_rejects_mismatched_families(payload: dict[str, Anydef test_terminal_goal_record_rejects_generic_success_synonyms(payload: dict[str, Any]) -> None:snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})- assert grade_terminal_goal_record_criteria([_criterion("c0", "a request is submitted")], snapshot) == []+ assert (+ grade_terminal_goal_record_criteria(+ [+ _criterion(+ "c0",+ "a request is submitted",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ],+ snapshot,+ )+ == []+ )@pytest.mark.parametrize(@@ -914,7 +1090,15 @@ def test_terminal_goal_record_negative_guards_abstain(payload: dict[str, Any]) -assert (grade_terminal_goal_record_criteria(- [_criterion("c0", "a commercial water service request is submitted")], snapshot+ [+ _criterion(+ "c0",+ "a commercial water service request is submitted",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ],+ snapshot,)== [])@@ -928,7 +1112,15 @@ def test_terminal_goal_record_rejects_negated_or_temporal_action_keys(key: str)assert (grade_terminal_goal_record_criteria(- [_criterion("c0", "a commercial water service request is submitted")], snapshot+ [+ _criterion(+ "c0",+ "a commercial water service request is submitted",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ],+ snapshot,)== [])@@ -2288,7 +2480,14 @@ async def test_page_observation_verification_does_not_apply_terminal_goal_record_patch_completion_handler(monkeypatch, handler)ctx = _run_ctx()ctx.request_policy = RequestPolicy(- completion_criteria=[_criterion("c0", "a commercial water service request is submitted")]+ completion_criteria=[+ _criterion(+ "c0",+ "a commercial water service request is submitted",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ])ctx.last_test_ok = Falsectx.last_run_blocks_workflow_run_id = "wr_cancel"@@ -2507,7 +2706,12 @@ async def test_mixed_completion_verification_preserves_structural_unfired_contin_set_workflow_labels(ctx, "submit_water_request")ctx.request_policy = RequestPolicy(completion_criteria=[- _criterion("c0", "a commercial water service request is submitted"),+ _criterion(+ "c0",+ "a commercial water service request is submitted",+ kind="terminal_action",+ terminal_action_family="request",+ ),_criterion("c1","A provider blocker is reported to the user.",@@ -3290,7 +3494,14 @@ async def test_maybe_run_completion_verification_terminal_goal_bypasses_judge(ctx = _run_ctx()_set_workflow_labels(ctx, "submit_water_request")ctx.request_policy = RequestPolicy(- completion_criteria=[_criterion("c0", "a commercial water service request is submitted")]+ completion_criteria=[+ _criterion(+ "c0",+ "a commercial water service request is submitted",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ])verification = await _maybe_run_completion_verification(ctx, _terminal_goal_output_result(), time.monotonic())@@ -3312,7 +3523,14 @@ async def test_maybe_run_completion_verification_terminal_goal_without_boolean_bctx = _run_ctx()_set_workflow_labels(ctx, "submit_water_request")ctx.request_policy = RequestPolicy(- completion_criteria=[_criterion("c0", "a commercial water service request is submitted")]+ completion_criteria=[+ _criterion(+ "c0",+ "a commercial water service request is submitted",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ])verification = await _maybe_run_completion_verification(@@ -3348,6 +3566,55 @@ async def test_maybe_run_completion_verification_fallback_floor_uses_recorded_teassert verification.verdicts[0].evidence_ref == "block_outputs:submit_water_request"+async def test_maybe_run_completion_verification_terminal_goal_uses_workflow_run_output_parameters(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("registered terminal-action evidence must bypass the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "submit_water_request")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ _criterion(+ "c1",+ "a commercial water service request is started",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ]+ )+ result = {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_terminal_goal",+ "overall_status": "completed",+ "executed_block_labels": ["submit_water_request"],+ "current_url": "https://example.test/confirmation",+ "blocks": [],+ "workflow_run_output_parameters": [+ {+ "workflow_run_id": "wr_terminal_goal",+ "output_parameter_id": "op_terminal",+ "output_parameter_key": "submit_water_request_output",+ "block_label": "submit_water_request",+ "block_type": "CODE",+ "value": _terminal_goal_payload(),+ }+ ],+ },+ }++ verification = await _maybe_run_completion_verification(ctx, result, time.monotonic())++ assert verification is not None+ assert verification.is_fully_satisfied() is True+ assert verification.verdicts[0].criterion_id == "c1"+ assert verification.verdicts[0].evidence_ref == "block_outputs:submit_water_request_output"++@pytest.mark.asyncioasync def test_maybe_run_completion_verification_fallback_floor_uses_page_end_state_evidence(monkeypatch: pytest.MonkeyPatch,@@ -3447,7 +3714,12 @@ async def test_maybe_run_completion_verification_mixed_terminal_goal_upgrades_ju_set_workflow_labels(ctx, "submit_water_request")ctx.request_policy = RequestPolicy(completion_criteria=[- _criterion("c0", "a commercial water service request is submitted"),+ _criterion(+ "c0",+ "a commercial water service request is submitted",+ kind="terminal_action",+ terminal_action_family="request",+ ),_criterion("c1", "the selected start date is reported"),])diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.pyindex a0257a8a0..9a3f62aca 100644--- a/tests/unit/test_copilot_request_policy_completion_criteria.py+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py@@ -416,6 +416,8 @@ def test_active_criteria_rendering_includes_contingent_on() -> None:"implicit": False,"method_mandated": False,"level": "run",+ "kind": "outcome",+ "terminal_action_family": None,"contingent_on": "the provider site blocks online submission",}]@@ -439,6 +441,8 @@ def test_active_criteria_rendering_includes_contingent_antecedent_output_path()"implicit": False,"method_mandated": False,"level": "run",+ "kind": "outcome",+ "terminal_action_family": None,"contingent_on": "the provider site blocks online submission","contingent_antecedent_output_path": "output.blocker",}diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.pyindex 45d97de88..188116864 100644--- a/tests/unit/test_copilot_request_policy_prompt.py+++ b/tests/unit/test_copilot_request_policy_prompt.py@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.request_policy import (_classify_request,_credential_ids,_raw_secret_detected,+ _render_active_criteria_for_prompt,contains_email_password_pair,is_fallback_floor_criterion,redact_raw_secrets_for_prompt,@@ -26,6 +27,7 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (def _render(**overrides: str) -> str:+ active_completion_criteria = overrides.get("active_completion_criteria", "")return prompt_engine.load_prompt(template=PROMPT_NAME,user_message=overrides.get("user_message", ""),@@ -35,6 +37,8 @@ def _render(**overrides: str) -> str:latest_assistant_turn=overrides.get("latest_assistant_turn", "(none)"),retained_history=overrides.get("retained_history", "(none)"),global_llm_context=overrides.get("global_llm_context", ""),+ raw_secret_present=overrides.get("raw_secret_present", "false"),+ active_completion_criteria=active_completion_criteria,)@@ -80,6 +84,32 @@ class TestRequestPolicyPromptStructure:assert "verbatim substring of the LATEST user message" in renderedassert "Do not cite a token that appears only in prior turns" in rendered+ def test_completion_criteria_schema_includes_typed_terminal_action_fields(self) -> None:+ rendered = _render()+ assert (+ "{outcome, contingent_on, contingent_antecedent_output_path, "+ "implicit, method_mandated, level, kind, terminal_action_family}"+ ) in rendered+ assert "kind=outcome|terminal_action" in rendered+ assert "terminal_action_family=request|application|form|order|null" in rendered++ def test_active_completion_criteria_render_typed_terminal_action_fields(self) -> None:+ active = _render_active_criteria_for_prompt(+ [+ CompletionCriterion(+ id="c0",+ outcome="a commercial water service request is started",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ]+ )++ rendered = _render(active_completion_criteria=active)++ assert '"kind": "terminal_action"' in rendered+ assert '"terminal_action_family": "request"' in rendered+class TestRawSecretBackstop:def test_keyvault_handle_does_not_trip_backstop(self) -> None: