Skip to content

Automate browser based workflows with AI

SKY-11503 Gate redundant output-field asks (#6866)

Andrew Neilsoncb3a4aedaccf
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 1ad3fe6b1..67a7810e3 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -67,6 +67,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
looks_like_workflow_delivery_claim,
parse_final_response,
)
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, request_policy_has_present_completion_contract
from skyvern.forge.sdk.copilot.run_outcome import (
TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,
TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,
@@ -86,6 +87,7 @@ from skyvern.forge.sdk.copilot.turn_halt import (
stash_repair_ceiling_turn_halt,
stash_turn_halt_from_blocker_signal,
)
+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentMode
from skyvern.utils.token_counter import count_tokens
if TYPE_CHECKING:
@@ -177,6 +179,13 @@ _PROGRESS_NARRATION_PATTERNS = [
re.compile(r"\bi\s+have\s+not\s+yet\b", re.IGNORECASE),
]
+PRESENT_COMPLETION_CONTRACT_ASK_RETRY = (
+ "The final ASK_QUESTION is not an allowed terminal response for this turn: the request already has a typed "
+ "completion contract / completion criteria and no separate required clarification is active. Continue authoring "
+ "the workflow from the existing contract, then run/test it before responding. Only ask the user if a separate "
+ "required input is missing under RequestPolicy or TurnIntent."
+)
+
def _is_progress_narration(user_response: Any) -> bool:
if not isinstance(user_response, str) or not user_response:
@@ -844,6 +853,63 @@ def _completion_contract_unknown_due_to_policy_fallback(ctx: Any) -> bool:
return _request_completion_contract_status(ctx) == "unknown"
+_AUTHORING_TURN_INTENT_MODES = frozenset({TurnIntentMode.BUILD, TurnIntentMode.EDIT, TurnIntentMode.DRAFT_ONLY})
+
+
+def _turn_intent_can_author_without_user_input(turn_intent: Any) -> bool:
+ if not isinstance(turn_intent, TurnIntent):
+ return False
+ if turn_intent.mode not in _AUTHORING_TURN_INTENT_MODES:
+ return False
+ if turn_intent.authority.requires_user_input:
+ return False
+ return turn_intent.authority.may_update_workflow
+
+
+def _has_current_turn_update_or_test_marker(ctx: Any) -> bool:
+ if bool(getattr(ctx, "update_workflow_called", False)):
+ return True
+ if bool(getattr(ctx, "test_after_update_done", False)):
+ return True
+ if getattr(ctx, "last_update_block_count", None) is not None:
+ return True
+ if getattr(ctx, "last_test_ok", None) is not None:
+ return True
+ for field_name in (
+ "last_run_blocks_workflow_run_id",
+ "last_successful_run_blocks_workflow_run_id",
+ "last_outcome_gate_workflow_run_id",
+ ):
+ marker = getattr(ctx, field_name, None)
+ if isinstance(marker, str) and marker.strip():
+ return True
+ return False
+
+
+def _present_completion_contract_ask_retry(ctx: Any, parsed: dict[str, Any]) -> str | None:
+ if parsed.get("type") != "ASK_QUESTION":
+ return None
+ request_policy = getattr(ctx, "request_policy", None)
+ if not isinstance(request_policy, RequestPolicy):
+ return None
+ if not request_policy_has_present_completion_contract(request_policy):
+ return None
+ if request_policy.user_response_policy == "ask_clarification":
+ return None
+ if request_policy.clarification_reason not in (None, "none"):
+ return None
+ if not _turn_intent_can_author_without_user_input(getattr(ctx, "turn_intent", None)):
+ return None
+ if _has_current_turn_update_or_test_marker(ctx):
+ return None
+ LOG.info(
+ "copilot.present_completion_contract_ask_retry",
+ reason_code="present_completion_contract_ask_internal_retry",
+ turn_intent_mode=getattr(getattr(ctx, "turn_intent", None), "mode", None),
+ )
+ return PRESENT_COMPLETION_CONTRACT_ASK_RETRY
+
+
def _nudge(config: CopilotConfig | None, key: str) -> str:
if config is None:
return DEFAULT_ENFORCEMENT_NUDGES[key]
@@ -981,6 +1047,10 @@ def _response_coverage_nudge(ctx: Any, parsed: dict[str, Any], config: CopilotCo
if discovery_entrypoint_nudge is not None:
return discovery_entrypoint_nudge
+ present_contract_retry = _present_completion_contract_ask_retry(ctx, parsed)
+ if present_contract_retry is not None:
+ return present_contract_retry
+
if response_type not in ("REPLY", "REPLACE_WORKFLOW"):
return None
diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.py
index 7b9c225d2..b29e0c308 100644
--- a/skyvern/forge/sdk/copilot/output_policy.py
+++ b/skyvern/forge/sdk/copilot/output_policy.py
@@ -14,6 +14,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
from skyvern.forge.sdk.copilot.request_policy import (
RequestPolicy,
contains_email_password_pair,
+ request_policy_has_present_completion_contract,
)
from skyvern.forge.sdk.copilot.secret_redaction import (
RAW_SECRET_PATTERNS,
@@ -436,7 +437,7 @@ def evaluate_output_policy(
if (
response_type == "ASK_QUESTION"
and request_policy.user_response_policy != "ask_clarification"
- and _request_policy_has_present_completion_contract(request_policy)
+ and request_policy_has_present_completion_contract(request_policy)
and not has_workflow_proposal
and not workflow_attempted
and _asks_to_confirm_output_fields(user_response)
@@ -452,10 +453,6 @@ def evaluate_output_policy(
return verdict
-def _request_policy_has_present_completion_contract(request_policy: RequestPolicy) -> bool:
- return request_policy.completion_contract_status == "present" or bool(request_policy.completion_criteria)
-
-
def _asks_to_confirm_output_fields(user_response: str | None) -> bool:
if not isinstance(user_response, str) or not user_response.strip():
return False
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index c1ab93c57..dd186c188 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -303,6 +303,12 @@ class RequestPolicy:
return "\n".join(lines)
+def request_policy_has_present_completion_contract(request_policy: RequestPolicy | None) -> bool:
+ if request_policy is None:
+ return False
+ return request_policy.completion_contract_status == "present" or bool(request_policy.completion_criteria)
+
+
_TRANSCRIPT_TOTAL_CHAR_BUDGET = 2048
TRANSCRIPT_ANCHOR_CHAR_CAP = 512
_TRANSCRIPT_RETAINED_MIN_CHARS = 512
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 852c6fa43..4e88f47e8 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -1807,7 +1807,7 @@ class TestTranslateToAgentResultGating:
completion_criteria=[
CompletionCriterion(id="provider", outcome="The returned record identifies the provider."),
],
- )
+ ),
)
result = _fake_run_result({"type": "ASK_QUESTION", "user_response": "Which saved credential should I use?"})
diff --git a/tests/unit/test_copilot_pre_discovery_url_nudge.py b/tests/unit/test_copilot_pre_discovery_url_nudge.py
index 5c7d297fb..ee5f367b7 100644
--- a/tests/unit/test_copilot_pre_discovery_url_nudge.py
+++ b/tests/unit/test_copilot_pre_discovery_url_nudge.py
@@ -14,13 +14,18 @@ from __future__ import annotations
from types import SimpleNamespace
+import pytest
+
from skyvern.forge.sdk.copilot.build_phase import BuildPhase
from skyvern.forge.sdk.copilot.enforcement import (
MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES,
PRE_DISCOVERY_URL_QUESTION_NUDGE,
+ PRESENT_COMPLETION_CONTRACT_ASK_RETRY,
_pre_discovery_url_question_nudge,
_response_coverage_nudge,
)
+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
class _Ctx:
@@ -38,6 +43,13 @@ class _Ctx:
self.no_workflow_nudge_count = 0
self.coverage_nudge_count = 0
self.format_nudge_count = 0
+ self.test_after_update_done = False
+ self.workflow_persisted = False
+ self.last_update_block_count = None
+ self.last_test_ok = None
+ self.last_run_blocks_workflow_run_id = None
+ self.last_successful_run_blocks_workflow_run_id = None
+ self.last_outcome_gate_workflow_run_id = None
_URL_ASK = {
@@ -45,6 +57,45 @@ _URL_ASK = {
"user_response": "I can build that, but I need the exact URL to use as the entry point. Please provide the URL.",
}
+_OUTPUT_CONFIRMATION_ASK = {
+ "type": "ASK_QUESTION",
+ "user_response": "Please confirm the output fields before I build this workflow.",
+}
+
+
+def _authoring_intent(*, mode: TurnIntentMode = TurnIntentMode.BUILD, requires_user_input: bool = False) -> TurnIntent:
+ return TurnIntent(
+ mode=mode,
+ authority=TurnIntentAuthority(
+ may_update_workflow=mode in {TurnIntentMode.BUILD, TurnIntentMode.EDIT, TurnIntentMode.DRAFT_ONLY},
+ may_run_blocks=mode in {TurnIntentMode.BUILD, TurnIntentMode.EDIT, TurnIntentMode.DRAFT_ONLY},
+ requires_user_input=requires_user_input,
+ ),
+ )
+
+
+def _present_contract_policy(**overrides: object) -> RequestPolicy:
+ defaults = dict(
+ user_response_policy="proceed",
+ clarification_reason="none",
+ completion_contract_status="present",
+ completion_criteria=[
+ CompletionCriterion(id="record_id", outcome="The returned record includes the requested id."),
+ ],
+ )
+ defaults.update(overrides)
+ return RequestPolicy(**defaults)
+
+
+def _present_contract_ctx(**overrides: object) -> _Ctx:
+ ctx = _Ctx()
+ ctx.build_phase = BuildPhase.COMPOSING
+ ctx.request_policy = _present_contract_policy()
+ ctx.turn_intent = _authoring_intent()
+ for name, value in overrides.items():
+ setattr(ctx, name, value)
+ return ctx
+
def test_pre_discovery_url_ask_in_initial_phase_steers_to_discovery() -> None:
ctx = _Ctx()
@@ -142,3 +193,68 @@ def test_pre_discovery_nudge_is_bounded() -> None:
assert ctx.pre_discovery_url_question_nudge_count == expected_count
assert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is None
assert ctx.pre_discovery_url_question_nudge_count == MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES
+
+
+def test_present_completion_contract_ask_returns_internal_retry() -> None:
+ ctx = _present_contract_ctx()
+
+ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) == PRESENT_COMPLETION_CONTRACT_ASK_RETRY
+
+
+def test_present_completion_contract_ask_has_no_per_rule_cap() -> None:
+ ctx = _present_contract_ctx()
+
+ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) == PRESENT_COMPLETION_CONTRACT_ASK_RETRY
+ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) == PRESENT_COMPLETION_CONTRACT_ASK_RETRY
+
+
+def test_present_completion_contract_ask_allows_request_policy_clarification() -> None:
+ ctx = _present_contract_ctx(
+ request_policy=_present_contract_policy(
+ user_response_policy="ask_clarification",
+ clarification_reason="credential_name_unresolved",
+ ),
+ turn_intent=_authoring_intent(mode=TurnIntentMode.CLARIFY, requires_user_input=True),
+ )
+
+ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) is None
+
+
+def test_present_completion_contract_ask_allows_non_none_clarification_reason() -> None:
+ ctx = _present_contract_ctx(
+ request_policy=_present_contract_policy(clarification_reason="credential_name_unresolved"),
+ )
+
+ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) is None
+
+
+def test_present_completion_contract_ask_allows_clarify_turn_intent() -> None:
+ ctx = _present_contract_ctx(
+ turn_intent=TurnIntent(
+ mode=TurnIntentMode.CLARIFY,
+ authority=TurnIntentAuthority(requires_user_input=True),
+ ),
+ )
+
+ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) is None
+
+
+def test_present_completion_contract_ask_allows_turn_intent_requiring_user_input() -> None:
+ ctx = _present_contract_ctx(turn_intent=_authoring_intent(requires_user_input=True))
+
+ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) is None
+
+
+ "marker",
+ [
+ {"update_workflow_called": True},
+ {"last_update_block_count": 1},
+ {"last_test_ok": False},
+ {"last_run_blocks_workflow_run_id": "wr_test"},
+ ],
+)
+def test_present_completion_contract_ask_allows_after_update_or_test_marker(marker: dict[str, object]) -> None:
+ ctx = _present_contract_ctx(**marker)
+
+ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) is None
diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.py
index 9a3f62aca..8373b74e7 100644
--- a/tests/unit/test_copilot_request_policy_completion_criteria.py
+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py
@@ -21,6 +21,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
_classify_request,
_parse_completion_criteria,
_render_active_criteria_for_prompt,
+ request_policy_has_present_completion_contract,
)
@@ -96,6 +97,27 @@ def _requested_output_subset(policy: RequestPolicy, requested_output_paths: set[
]
+def test_present_completion_contract_helper_accepts_present_status() -> None:
+ policy = RequestPolicy(completion_contract_status="present")
+
+ assert request_policy_has_present_completion_contract(policy) is True
+
+
+def test_present_completion_contract_helper_accepts_criteria_backed_policy() -> None:
+ policy = RequestPolicy(
+ completion_contract_status="absent",
+ completion_criteria=[CompletionCriterion(id="record_id", outcome="The returned record includes an id.")],
+ )
+
+ assert request_policy_has_present_completion_contract(policy) is True
+
+
+def test_present_completion_contract_helper_rejects_absent_policy() -> None:
+ policy = RequestPolicy(completion_contract_status="absent", completion_criteria=[])
+
+ assert request_policy_has_present_completion_contract(policy) is False
+
+
@pytest.mark.asyncio
async def test_classifier_output_is_augmented_with_generic_requested_outputs() -> None:
policy = await _policy_for_message(