Skip to content

Automate browser based workflows with AI

fix(SKY-11510): type terminal action criteria (#6863)

Andrew Neilson9dfafd5fccf2
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
index 314b26e5d..63f82c042 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
@@ -25,10 +25,12 @@ completion_contract:
- Set to null when the user does not provide an explicit completion condition.
completion_criteria:
-- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, contingent_on, contingent_antecedent_output_path, implicit, method_mandated, level}.
+- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, contingent_on, contingent_antecedent_output_path, implicit, method_mandated, level, kind, terminal_action_family}.
- outcome: a short statement of an end state the workflow must achieve (an END, not a step). Extract the goal as outcomes the user wants to be true, not the recipe they describe.
- contingent_on: null for ordinary criteria. Use a short antecedent string only when this criterion should be graded only if that antecedent happens, for example {"contingent_on": "the provider site blocks online submission", "contingent_antecedent_output_path": "output.blocker", "outcome": "the blocker is reported to the user"}. The antecedent goes in contingent_on; the consequent to grade goes in outcome. Do not encode if/then criteria only in prose.
- contingent_antecedent_output_path: null unless the antecedent is represented by a structured run output field. The only valid form is output.<field> (for example output.blocker). Do not emit transcript paths, selectors, regexes, JSONPath, screenshots, nested paths, or prose conditions here.
+- kind=outcome|terminal_action. Use terminal_action only when the criterion's end state is a terminal action in the external site for one of the supported families below. Use outcome for all other outcomes.
+- terminal_action_family=request|application|form|order|null. Set the matching family when kind=terminal_action; set null when kind=outcome.
- Capture ALL distinct outcomes the goal requires, including implicit constraints the user did not state as a step (for example "added exactly once / not duplicated", "the item is present in the cart"). Mark such items implicit=true.
- A prescribed HOW step is NOT a criterion. "search for X", "open the result lightbox", "click add to cart", "go to the cart page" are methods/navigation, not outcomes — do not list them. A different or shorter path that reaches the same end state must still satisfy the goal.
- Telling/showing/reporting results to the user is NOT a criterion. "tell me what it extracted", "show me the data", "confirm to me it worked" describe the chat reply, which no workflow definition or run can evidence — capture only the underlying workflow outcome (the data is extracted into the run output, the form is submitted) and list nothing for the reporting itself.
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index cbd579bad..837b30783 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -9,7 +9,7 @@ stays at the route/repository seam; everything here is side-effect free.
from __future__ import annotations
from dataclasses import dataclass, field
-from typing import Any, Literal
+from typing import Any, Literal, cast
from skyvern.forge.sdk.copilot.completion_verification import (
CompletionVerificationResult,
@@ -17,6 +17,8 @@ from skyvern.forge.sdk.copilot.completion_verification import (
)
from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
+ CriterionKind,
+ TerminalActionFamily,
_normalize_contingent_antecedent_output_path,
is_fallback_floor_criterion,
normalized_criterion_outcome_key,
@@ -31,6 +33,8 @@ CRITERIA_SET_STATUS_ACTIVE = "active"
CRITERIA_SET_STATUS_SUPERSEDED = "superseded"
TRIPWIRE_CONSECUTIVE_ALL_NO_EVIDENCE = 2
_CRITERION_LEVELS = ("definition", "run")
+_CRITERION_KINDS = ("outcome", "terminal_action")
+_TERMINAL_ACTION_FAMILIES = ("request", "application", "form", "order")
@dataclass(frozen=True)
@@ -117,6 +121,8 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion
"method_mandated": criterion.method_mandated,
"level": criterion.level,
"output_path": criterion.output_path,
+ "kind": criterion.kind,
+ "terminal_action_family": criterion.terminal_action_family,
}
for criterion in criteria
]
@@ -139,6 +145,12 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
contingent_antecedent_output_path = _normalize_contingent_antecedent_output_path(
item.get("contingent_antecedent_output_path")
)
+ kind_raw = item.get("kind")
+ kind = kind_raw if isinstance(kind_raw, str) and kind_raw in _CRITERION_KINDS else "outcome"
+ family_raw = item.get("terminal_action_family")
+ terminal_action_family = (
+ family_raw if kind == "terminal_action" and family_raw in _TERMINAL_ACTION_FAMILIES else None
+ )
criteria.append(
CompletionCriterion(
id=criterion_id,
@@ -151,6 +163,8 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
method_mandated=bool(item.get("method_mandated")),
level=level if isinstance(level, str) and level in _CRITERION_LEVELS else "run", # type: ignore[arg-type]
output_path=output_path.strip() if isinstance(output_path, str) and output_path.strip() else None,
+ kind=cast(CriterionKind, kind),
+ terminal_action_family=cast(TerminalActionFamily | None, terminal_action_family),
)
)
return tuple(criteria)
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 33975afe0..56ac68faa 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -25,6 +25,7 @@ from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.copilot.output_utils import parse_final_response
from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
+ TerminalActionFamily,
is_fallback_floor_base_criterion,
redact_raw_secrets_for_prompt,
)
@@ -852,19 +853,7 @@ _TERMINAL_ARTIFACT_KEY_TOKENS = (
(("reference", "number"), "reference"),
(("reference", "id"), "reference"),
)
-_TERMINAL_CRITERION_VERBS = frozenset({"submit", "submitted", "submission", "place", "placed"})
-_TERMINAL_CRITERION_ABSTAIN_TOKENS = frozenset(
- {
- "lookup",
- "looked",
- "retrieve",
- "retrieved",
- "retrieval",
- "return",
- "returned",
- "status",
- }
-)
+_GENERIC_TERMINAL_SUCCESS_LEAF_TOKENS = frozenset({"submit", "submitted", "submission", "place", "placed"})
_NEGATIVE_GUARD_TOKENS = frozenset({"blocker", "error", "failure", "failed", "challenge"})
_NEGATIVE_TERMINAL_STATUS_VALUES = frozenset(
{
@@ -882,13 +871,14 @@ _NEGATIVE_TERMINAL_STATUS_VALUES = frozenset(
"unable",
}
)
-_TERMINAL_RECORD_FAMILY_ARTIFACTS = {
+_TERMINAL_RECORD_FAMILIES: tuple[TerminalActionFamily, ...] = ("request", "application", "form", "order")
+_TERMINAL_RECORD_FAMILY_ARTIFACTS: dict[TerminalActionFamily, frozenset[str]] = {
"request": frozenset({"confirmation", "reference", "request", "submission"}),
"application": frozenset({"application", "confirmation", "reference", "submission"}),
"form": frozenset({"confirmation", "reference", "submission"}),
"order": frozenset({"order", "receipt"}),
}
-_TERMINAL_RECORD_FAMILY_ACTIONS = {
+_TERMINAL_RECORD_FAMILY_ACTIONS: dict[TerminalActionFamily, frozenset[str]] = {
"request": frozenset({"request", "submission"}),
"application": frozenset({"application", "submission"}),
"form": frozenset({"form", "submission"}),
@@ -901,9 +891,9 @@ def grade_terminal_goal_record_criteria(
) -> list[CriterionVerdict]:
verdicts: list[CriterionVerdict] = []
eligible_criteria = [
- (criterion, family)
+ criterion
for criterion in criteria
- if (family := _terminal_goal_criterion_family(criterion.outcome)) is not None
+ if criterion.kind == "terminal_action" and criterion.terminal_action_family in _TERMINAL_RECORD_FAMILY_ARTIFACTS
]
if not eligible_criteria:
return []
@@ -911,8 +901,9 @@ def grade_terminal_goal_record_criteria(
record = _structured_record_payload(payload)
if record is None:
continue
- for criterion, family in eligible_criteria:
- if _terminal_goal_record_confirmed(record, family):
+ for criterion in eligible_criteria:
+ family = criterion.terminal_action_family
+ if family is not None and _terminal_goal_record_confirmed(record, family):
verdicts.append(
CriterionVerdict(
criterion_id=criterion.id,
@@ -951,29 +942,12 @@ def _fallback_floor_reached_end_state_evidence_ref(snapshot: RunEvidenceSnapshot
record = _structured_record_payload(payload)
if record is None:
continue
- if any(_terminal_goal_record_confirmed(record, family) for family in _TERMINAL_RECORD_FAMILY_ARTIFACTS):
+ if any(_terminal_goal_record_confirmed(record, family) for family in _TERMINAL_RECORD_FAMILIES):
return f"block_outputs:{label}"
return None
-def _terminal_goal_criterion_family(outcome: str) -> str | None:
- if _extract_present_value_literals(outcome):
- return None
- tokens = set(_status_word_tokens(outcome))
- if tokens & _TERMINAL_CRITERION_ABSTAIN_TOKENS:
- return None
- if not tokens & _TERMINAL_CRITERION_VERBS:
- return None
- families = [family for family in ("request", "application", "form", "order") if family in tokens]
- if len(families) != 1:
- return None
- family = families[0]
- if family == "order" and "placed" not in tokens:
- return None
- return family
-
-
-def _terminal_goal_record_confirmed(record: dict[str, Any], family: str) -> bool:
+def _terminal_goal_record_confirmed(record: dict[str, Any], family: TerminalActionFamily) -> bool:
if _terminal_goal_record_has_negative_guard(record):
return False
if _structured_record_contradiction(record):
@@ -1016,7 +990,7 @@ def _terminal_goal_record_has_generic_success_claim(record: dict[str, Any]) -> b
for key, value in _walk_record_scalars(record):
leaf_tokens = _record_key_leaf_tokens(key)
if value is True and _terminal_action_family_for_key(key) is None and leaf_tokens:
- if leaf_tokens[-1] in _TERMINAL_CRITERION_VERBS:
+ if leaf_tokens[-1] in _GENERIC_TERMINAL_SUCCESS_LEAF_TOKENS:
return True
if leaf_tokens in generic_success_keys and value is True:
return True
@@ -1044,7 +1018,7 @@ def _terminal_goal_record_has_negative_guard(record: dict[str, Any]) -> bool:
return False
-def _terminal_goal_record_has_artifact_for_family(record: dict[str, Any], family: str) -> bool:
+def _terminal_goal_record_has_artifact_for_family(record: dict[str, Any], family: TerminalActionFamily) -> bool:
allowed_artifacts = _TERMINAL_RECORD_FAMILY_ARTIFACTS[family]
for key, value in _walk_record_scalars(record):
if isinstance(value, bool) or not _is_meaningful_record_value(value):
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 8c83a700c..c1ab93c57 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -157,6 +157,10 @@ _FALLBACK_FLOOR_CREDENTIAL_OUTCOME = "The credentialed step authenticates and re
CriterionLevel = Literal["definition", "run"]
_CRITERION_LEVELS: frozenset[str] = frozenset({"definition", "run"})
+CriterionKind = Literal["outcome", "terminal_action"]
+TerminalActionFamily = Literal["request", "application", "form", "order"]
+_CRITERION_KINDS: frozenset[str] = frozenset({"outcome", "terminal_action"})
+_TERMINAL_ACTION_FAMILIES: frozenset[str] = frozenset({"request", "application", "form", "order"})
_OUTPUT_INTENT_RE = re.compile(
r"\b(?:read|capture|extract|output|return|returns|returned|include|includes|including|"
@@ -205,6 +209,8 @@ class CompletionCriterion:
# YAML; "run": an end state only a run can evidence. Invalid input coerces to "run".
level: CriterionLevel = "run"
output_path: str | None = None
+ kind: CriterionKind = "outcome"
+ terminal_action_family: TerminalActionFamily | None = None
@dataclass
@@ -486,6 +492,18 @@ def _coerce_raw_secret_handling(value: Any) -> RawSecretHandling:
return "none"
+def _coerce_criterion_kind(value: Any) -> CriterionKind:
+ if isinstance(value, str) and value in _CRITERION_KINDS:
+ return cast(CriterionKind, value)
+ return "outcome"
+
+
+def _coerce_terminal_action_family(value: Any, kind: CriterionKind) -> TerminalActionFamily | None:
+ if kind == "terminal_action" and isinstance(value, str) and value in _TERMINAL_ACTION_FAMILIES:
+ return cast(TerminalActionFamily, value)
+ return None
+
+
def _coerce_classifier_payload(raw: Any) -> dict[str, Any] | None:
if isinstance(raw, str):
raw = parse_final_response(raw)
@@ -590,6 +608,7 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
continue
seen.add(key)
level_raw = item.get("level")
+ kind = _coerce_criterion_kind(item.get("kind"))
criteria.append(
CompletionCriterion(
id=f"c{len(criteria)}",
@@ -602,6 +621,8 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
if isinstance(level_raw, str) and level_raw in _CRITERION_LEVELS
else "run",
output_path=output_path,
+ kind=kind,
+ terminal_action_family=_coerce_terminal_action_family(item.get("terminal_action_family"), kind),
)
)
if len(criteria) >= _MAX_COMPLETION_CRITERIA:
@@ -947,6 +968,8 @@ def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | Non
"implicit": criterion.implicit,
"method_mandated": criterion.method_mandated,
"level": criterion.level,
+ "kind": criterion.kind,
+ "terminal_action_family": criterion.terminal_action_family,
}
if criterion.contingent_on:
item["contingent_on"] = criterion.contingent_on
diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.py
index a25f6baa7..2d4401140 100644
--- a/skyvern/forge/sdk/copilot/tools/_shared.py
+++ b/skyvern/forge/sdk/copilot/tools/_shared.py
@@ -202,8 +202,14 @@ def _registered_output_parameter_payloads(data: Mapping[str, Any]) -> list[Mappi
accumulated outputs cannot satisfy the current run's completion contract.
"""
run_id = data.get("workflow_run_id")
- registered = data.get("registered_output_parameter_values")
- if not isinstance(registered, list):
+ registered_values = data.get("registered_output_parameter_values")
+ registered_outputs = data.get("workflow_run_output_parameters")
+ registered = []
+ if isinstance(registered_values, list):
+ registered.extend(registered_values)
+ if isinstance(registered_outputs, list):
+ registered.extend(registered_outputs)
+ if not registered:
return []
if not isinstance(run_id, str):
# Without a current run id the payloads can't be attributed to this run;
diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.py
index 12e142e47..993a5f495 100644
--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py
+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py
@@ -260,6 +260,19 @@ def test_criteria_json_round_trip_preserves_level_and_flags() -> None:
assert criteria_from_json(criteria_to_json(criteria)) == criteria
+def test_criteria_json_round_trip_preserves_terminal_action_fields() -> None:
+ criteria = (
+ CompletionCriterion(
+ id="c0",
+ outcome="a commercial water service request is started",
+ kind="terminal_action",
+ terminal_action_family="request",
+ ),
+ )
+
+ assert criteria_from_json(criteria_to_json(criteria)) == criteria
+
+
def test_criteria_from_json_coerces_invalid_level_to_run() -> None:
(criterion,) = criteria_from_json([{"id": "c0", "outcome": "done", "level": "bogus"}])
assert criterion.level == "run"
@@ -548,6 +561,63 @@ def test_parse_completion_criteria_tolerates_unhashable_level() -> None:
assert [c.level for c in parsed] == ["run", "run"]
+def test_parse_completion_criteria_tolerates_unhashable_terminal_action_fields() -> None:
+ parsed = _parse_completion_criteria(
+ [
+ {
+ "outcome": "a commercial water service request is started",
+ "kind": ["terminal_action"],
+ "terminal_action_family": ["request"],
+ },
+ {
+ "outcome": "a permit application is submitted",
+ "kind": "terminal_action",
+ "terminal_action_family": ["application"],
+ },
+ ]
+ )
+
+ assert [c.kind for c in parsed] == ["outcome", "terminal_action"]
+ assert [c.terminal_action_family for c in parsed] == [None, None]
+
+
+def test_parse_completion_criteria_preserves_typed_terminal_action_fields() -> None:
+ (criterion,) = _parse_completion_criteria(
+ [
+ {
+ "outcome": "a commercial water service request is started",
+ "kind": "terminal_action",
+ "terminal_action_family": "request",
+ }
+ ]
+ )
+
+ assert criterion.kind == "terminal_action"
+ assert criterion.terminal_action_family == "request"
+
+
+def test_parse_completion_criteria_defaults_invalid_terminal_action_fields() -> None:
+ parsed = _parse_completion_criteria(
+ [
+ {
+ "outcome": "a commercial water service request is started",
+ "kind": "terminal_action",
+ "terminal_action_family": "invoice",
+ },
+ {
+ "outcome": "the item is in the cart",
+ "kind": "done",
+ "terminal_action_family": "request",
+ },
+ ]
+ )
+
+ assert parsed[0].kind == "terminal_action"
+ assert parsed[0].terminal_action_family is None
+ assert parsed[1].kind == "outcome"
+ assert parsed[1].terminal_action_family is None
+
+
def test_definition_grading_unknown_when_no_deterministic_check_applies() -> None:
criteria = [_criterion("c0", "the summary email is well written", level="definition")]
(verdict,) = grade_definition_criteria(criteria, _PARAMETERIZED_YAML)
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 90dde3b81..5a43bd55e 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -93,6 +93,8 @@ def _criterion(
output_path: str | None = None,
contingent_on: str | None = None,
contingent_antecedent_output_path: str | None = None,
+ kind: str = "outcome",
+ terminal_action_family: str | None = None,
) -> CompletionCriterion:
return CompletionCriterion(
id=cid,
@@ -101,6 +103,8 @@ def _criterion(
output_path=output_path,
contingent_on=contingent_on,
contingent_antecedent_output_path=contingent_antecedent_output_path,
+ kind=kind,
+ terminal_action_family=terminal_action_family,
)
@@ -769,7 +773,15 @@ def test_terminal_goal_record_satisfies_flat_submit_payload() -> None:
snapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": _terminal_goal_payload()})
verdicts = grade_terminal_goal_record_criteria(
- [_criterion("c0", "a commercial water service request is submitted")], snapshot
+ [
+ _criterion(
+ "c0",
+ "a commercial water service request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ],
+ snapshot,
)
assert verdicts == [
@@ -787,12 +799,133 @@ def test_terminal_goal_record_accepts_family_artifact_without_self_asserted_bool
snapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": payload})
verdicts = grade_terminal_goal_record_criteria(
- [_criterion("c0", "a commercial water service request is submitted")], snapshot
+ [
+ _criterion(
+ "c0",
+ "a commercial water service request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ],
+ snapshot,
)
assert _satisfied_criterion_ids(verdicts) == {"c0"}
+def test_terminal_goal_record_accepts_started_request_when_typed_with_artifact() -> None:
+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": _terminal_goal_payload()})
+
+ verdicts = grade_terminal_goal_record_criteria(
+ [
+ _criterion(
+ "c0",
+ "a commercial water service request is started",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ],
+ snapshot,
+ )
+
+ assert _satisfied_criterion_ids(verdicts) == {"c0"}
+
+
+def test_terminal_goal_record_accepts_typed_create_or_verify_with_identifier() -> None:
+ payload = _terminal_goal_payload(
+ submitted=None,
+ confirmation_number=None,
+ request_id="QC-2002-DEMO",
+ submission_result="already_present",
+ )
+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})
+
+ verdicts = grade_terminal_goal_record_criteria(
+ [
+ _criterion(
+ "c0",
+ "the service request is created or verified",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ],
+ snapshot,
+ )
+
+ assert _satisfied_criterion_ids(verdicts) == {"c0"}
+
+
+def test_terminal_goal_record_submission_result_is_not_credit_source() -> None:
+ payload = _terminal_goal_payload(
+ submitted=None,
+ confirmation_number=None,
+ request_id=None,
+ submission_result="already_present",
+ )
+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})
+
+ assert (
+ grade_terminal_goal_record_criteria(
+ [
+ _criterion(
+ "c0",
+ "the service request is created or verified",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ],
+ snapshot,
+ )
+ == []
+ )
+
+
+def test_terminal_goal_record_requires_typed_terminal_action_family() -> None:
+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_water_request": _terminal_goal_payload()})
+
+ assert grade_terminal_goal_record_criteria([_criterion("c0", "a request is submitted")], snapshot) == []
+ assert (
+ grade_terminal_goal_record_criteria(
+ [_criterion("c1", "a request is submitted", kind="terminal_action")], snapshot
+ )
+ == []
+ )
+ assert (
+ grade_terminal_goal_record_criteria(
+ [
+ _criterion(
+ "c2",
+ "a request is submitted",
+ kind="terminal_action",
+ terminal_action_family="invoice",
+ )
+ ],
+ snapshot,
+ )
+ == []
+ )
+
+
+def test_terminal_goal_record_self_asserted_boolean_without_identifier_remains_uncredited() -> None:
+ payload = _terminal_goal_payload(submitted=True, confirmation_number=None)
+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})
+
+ assert (
+ grade_terminal_goal_record_criteria(
+ [
+ _criterion(
+ "c0",
+ "a request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ],
+ snapshot,
+ )
+ == []
+ )
+
+
@pytest.mark.parametrize(
"payload",
[
@@ -808,57 +941,87 @@ def test_terminal_goal_record_rejects_ordinary_identifiers(payload: dict[str, An
assert (
grade_terminal_goal_record_criteria(
- [_criterion("c0", "a commercial water service request is submitted")], snapshot
+ [
+ _criterion(
+ "c0",
+ "a commercial water service request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ],
+ snapshot,
)
== []
)
@pytest.mark.parametrize(
- ("payload", "outcome"),
+ ("payload", "outcome", "family"),
[
(
_terminal_goal_payload(confirmation_number="WTR-1842-DEMO"),
"a commercial water service request is submitted",
+ "request",
+ ),
+ (
+ _terminal_goal_payload(order_placed=True, submitted=None, order_number="ORD-1842"),
+ "an order is placed",
+ "order",
),
- (_terminal_goal_payload(order_placed=True, submitted=None, order_number="ORD-1842"), "an order is placed"),
(
_terminal_goal_payload(application_submitted=True, submitted=None, application_id="APP-1842"),
"an application is submitted",
+ "application",
),
(
_terminal_goal_payload(form_submitted=True, submitted=None, submission_id="SUB-1842"),
"the form is submitted",
+ "form",
),
(
_terminal_goal_payload(request_submitted=True, submitted=None, request_id="REQ-1842"),
"a service request is submitted",
+ "request",
),
],
)
-def test_terminal_goal_record_accepts_narrow_terminal_artifacts(payload: dict[str, Any], outcome: str) -> None:
+def test_terminal_goal_record_accepts_narrow_terminal_artifacts(
+ payload: dict[str, Any], outcome: str, family: str
+) -> None:
snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})
- verdicts = grade_terminal_goal_record_criteria([_criterion("c0", outcome)], snapshot)
+ verdicts = grade_terminal_goal_record_criteria(
+ [_criterion("c0", outcome, kind="terminal_action", terminal_action_family=family)], snapshot
+ )
assert _satisfied_criterion_ids(verdicts) == {"c0"}
@pytest.mark.parametrize(
- ("payload", "outcome"),
+ ("payload", "outcome", "family"),
[
- (_terminal_goal_payload(), "an order is placed"),
- (_terminal_goal_payload(order_placed=True, submitted=None, order_number="ORD-1842"), "a request is submitted"),
+ (_terminal_goal_payload(), "an order is placed", "order"),
+ (
+ _terminal_goal_payload(order_placed=True, submitted=None, order_number="ORD-1842"),
+ "a request is submitted",
+ "request",
+ ),
(
_terminal_goal_payload(application_submitted=True, submitted=None, application_id="APP-1842"),
"an order is placed",
+ "order",
),
],
)
-def test_terminal_goal_record_rejects_mismatched_families(payload: dict[str, Any], outcome: str) -> None:
+def test_terminal_goal_record_rejects_mismatched_families(payload: dict[str, Any], outcome: str, family: str) -> None:
snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})
- assert grade_terminal_goal_record_criteria([_criterion("c0", outcome)], snapshot) == []
+ assert (
+ grade_terminal_goal_record_criteria(
+ [_criterion("c0", outcome, kind="terminal_action", terminal_action_family=family)], snapshot
+ )
+ == []
+ )
@pytest.mark.parametrize(
@@ -873,7 +1036,20 @@ def test_terminal_goal_record_rejects_mismatched_families(payload: dict[str, Any
def test_terminal_goal_record_rejects_generic_success_synonyms(payload: dict[str, Any]) -> None:
snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": payload})
- assert grade_terminal_goal_record_criteria([_criterion("c0", "a request is submitted")], snapshot) == []
+ assert (
+ grade_terminal_goal_record_criteria(
+ [
+ _criterion(
+ "c0",
+ "a request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ],
+ snapshot,
+ )
+ == []
+ )
@pytest.mark.parametrize(
@@ -914,7 +1090,15 @@ def test_terminal_goal_record_negative_guards_abstain(payload: dict[str, Any]) -
assert (
grade_terminal_goal_record_criteria(
- [_criterion("c0", "a commercial water service request is submitted")], snapshot
+ [
+ _criterion(
+ "c0",
+ "a commercial water service request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ],
+ snapshot,
)
== []
)
@@ -928,7 +1112,15 @@ def test_terminal_goal_record_rejects_negated_or_temporal_action_keys(key: str)
assert (
grade_terminal_goal_record_criteria(
- [_criterion("c0", "a commercial water service request is submitted")], snapshot
+ [
+ _criterion(
+ "c0",
+ "a commercial water service request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ],
+ snapshot,
)
== []
)
@@ -2288,7 +2480,14 @@ async def test_page_observation_verification_does_not_apply_terminal_goal_record
_patch_completion_handler(monkeypatch, handler)
ctx = _run_ctx()
ctx.request_policy = RequestPolicy(
- completion_criteria=[_criterion("c0", "a commercial water service request is submitted")]
+ completion_criteria=[
+ _criterion(
+ "c0",
+ "a commercial water service request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ]
)
ctx.last_test_ok = False
ctx.last_run_blocks_workflow_run_id = "wr_cancel"
@@ -2507,7 +2706,12 @@ async def test_mixed_completion_verification_preserves_structural_unfired_contin
_set_workflow_labels(ctx, "submit_water_request")
ctx.request_policy = RequestPolicy(
completion_criteria=[
- _criterion("c0", "a commercial water service request is submitted"),
+ _criterion(
+ "c0",
+ "a commercial water service request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ ),
_criterion(
"c1",
"A provider blocker is reported to the user.",
@@ -3290,7 +3494,14 @@ async def test_maybe_run_completion_verification_terminal_goal_bypasses_judge(
ctx = _run_ctx()
_set_workflow_labels(ctx, "submit_water_request")
ctx.request_policy = RequestPolicy(
- completion_criteria=[_criterion("c0", "a commercial water service request is submitted")]
+ completion_criteria=[
+ _criterion(
+ "c0",
+ "a commercial water service request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ]
)
verification = await _maybe_run_completion_verification(ctx, _terminal_goal_output_result(), time.monotonic())
@@ -3312,7 +3523,14 @@ async def test_maybe_run_completion_verification_terminal_goal_without_boolean_b
ctx = _run_ctx()
_set_workflow_labels(ctx, "submit_water_request")
ctx.request_policy = RequestPolicy(
- completion_criteria=[_criterion("c0", "a commercial water service request is submitted")]
+ completion_criteria=[
+ _criterion(
+ "c0",
+ "a commercial water service request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ]
)
verification = await _maybe_run_completion_verification(
@@ -3348,6 +3566,55 @@ async def test_maybe_run_completion_verification_fallback_floor_uses_recorded_te
assert verification.verdicts[0].evidence_ref == "block_outputs:submit_water_request"
+async def test_maybe_run_completion_verification_terminal_goal_uses_workflow_run_output_parameters(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("registered terminal-action evidence must bypass the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "submit_water_request")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion(
+ "c1",
+ "a commercial water service request is started",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ]
+ )
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_terminal_goal",
+ "overall_status": "completed",
+ "executed_block_labels": ["submit_water_request"],
+ "current_url": "https://example.test/confirmation",
+ "blocks": [],
+ "workflow_run_output_parameters": [
+ {
+ "workflow_run_id": "wr_terminal_goal",
+ "output_parameter_id": "op_terminal",
+ "output_parameter_key": "submit_water_request_output",
+ "block_label": "submit_water_request",
+ "block_type": "CODE",
+ "value": _terminal_goal_payload(),
+ }
+ ],
+ },
+ }
+
+ verification = await _maybe_run_completion_verification(ctx, result, time.monotonic())
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ assert verification.verdicts[0].criterion_id == "c1"
+ assert verification.verdicts[0].evidence_ref == "block_outputs:submit_water_request_output"
+
+
@pytest.mark.asyncio
async def test_maybe_run_completion_verification_fallback_floor_uses_page_end_state_evidence(
monkeypatch: pytest.MonkeyPatch,
@@ -3447,7 +3714,12 @@ async def test_maybe_run_completion_verification_mixed_terminal_goal_upgrades_ju
_set_workflow_labels(ctx, "submit_water_request")
ctx.request_policy = RequestPolicy(
completion_criteria=[
- _criterion("c0", "a commercial water service request is submitted"),
+ _criterion(
+ "c0",
+ "a commercial water service request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ ),
_criterion("c1", "the selected start date is reported"),
]
)
diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.py
index a0257a8a0..9a3f62aca 100644
--- a/tests/unit/test_copilot_request_policy_completion_criteria.py
+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py
@@ -416,6 +416,8 @@ def test_active_criteria_rendering_includes_contingent_on() -> None:
"implicit": False,
"method_mandated": False,
"level": "run",
+ "kind": "outcome",
+ "terminal_action_family": None,
"contingent_on": "the provider site blocks online submission",
}
]
@@ -439,6 +441,8 @@ def test_active_criteria_rendering_includes_contingent_antecedent_output_path()
"implicit": False,
"method_mandated": False,
"level": "run",
+ "kind": "outcome",
+ "terminal_action_family": None,
"contingent_on": "the provider site blocks online submission",
"contingent_antecedent_output_path": "output.blocker",
}
diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.py
index 45d97de88..188116864 100644
--- a/tests/unit/test_copilot_request_policy_prompt.py
+++ b/tests/unit/test_copilot_request_policy_prompt.py
@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
_classify_request,
_credential_ids,
_raw_secret_detected,
+ _render_active_criteria_for_prompt,
contains_email_password_pair,
is_fallback_floor_criterion,
redact_raw_secrets_for_prompt,
@@ -26,6 +27,7 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (
def _render(**overrides: str) -> str:
+ active_completion_criteria = overrides.get("active_completion_criteria", "")
return prompt_engine.load_prompt(
template=PROMPT_NAME,
user_message=overrides.get("user_message", ""),
@@ -35,6 +37,8 @@ def _render(**overrides: str) -> str:
latest_assistant_turn=overrides.get("latest_assistant_turn", "(none)"),
retained_history=overrides.get("retained_history", "(none)"),
global_llm_context=overrides.get("global_llm_context", ""),
+ raw_secret_present=overrides.get("raw_secret_present", "false"),
+ active_completion_criteria=active_completion_criteria,
)
@@ -80,6 +84,32 @@ class TestRequestPolicyPromptStructure:
assert "verbatim substring of the LATEST user message" in rendered
assert "Do not cite a token that appears only in prior turns" in rendered
+ def test_completion_criteria_schema_includes_typed_terminal_action_fields(self) -> None:
+ rendered = _render()
+ assert (
+ "{outcome, contingent_on, contingent_antecedent_output_path, "
+ "implicit, method_mandated, level, kind, terminal_action_family}"
+ ) in rendered
+ assert "kind=outcome|terminal_action" in rendered
+ assert "terminal_action_family=request|application|form|order|null" in rendered
+
+ def test_active_completion_criteria_render_typed_terminal_action_fields(self) -> None:
+ active = _render_active_criteria_for_prompt(
+ [
+ CompletionCriterion(
+ id="c0",
+ outcome="a commercial water service request is started",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ]
+ )
+
+ rendered = _render(active_completion_criteria=active)
+
+ assert '"kind": "terminal_action"' in rendered
+ assert '"terminal_action_family": "request"' in rendered
+
class TestRawSecretBackstop:
def test_keyvault_handle_does_not_trip_backstop(self) -> None: