Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11503 Gate redundant output-field asks (#6866)
Andrew Neilsoncb3a4aedaccf
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 1ad3fe6b1..67a7810e3 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -67,6 +67,7 @@ from skyvern.forge.sdk.copilot.output_utils import (looks_like_workflow_delivery_claim,parse_final_response,)+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, request_policy_has_present_completion_contractfrom skyvern.forge.sdk.copilot.run_outcome import (TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,@@ -86,6 +87,7 @@ from skyvern.forge.sdk.copilot.turn_halt import (stash_repair_ceiling_turn_halt,stash_turn_halt_from_blocker_signal,)+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentModefrom skyvern.utils.token_counter import count_tokensif TYPE_CHECKING:@@ -177,6 +179,13 @@ _PROGRESS_NARRATION_PATTERNS = [re.compile(r"\bi\s+have\s+not\s+yet\b", re.IGNORECASE),]+PRESENT_COMPLETION_CONTRACT_ASK_RETRY = (+ "The final ASK_QUESTION is not an allowed terminal response for this turn: the request already has a typed "+ "completion contract / completion criteria and no separate required clarification is active. Continue authoring "+ "the workflow from the existing contract, then run/test it before responding. Only ask the user if a separate "+ "required input is missing under RequestPolicy or TurnIntent."+)+def _is_progress_narration(user_response: Any) -> bool:if not isinstance(user_response, str) or not user_response:@@ -844,6 +853,63 @@ def _completion_contract_unknown_due_to_policy_fallback(ctx: Any) -> bool:return _request_completion_contract_status(ctx) == "unknown"+_AUTHORING_TURN_INTENT_MODES = frozenset({TurnIntentMode.BUILD, TurnIntentMode.EDIT, TurnIntentMode.DRAFT_ONLY})+++def _turn_intent_can_author_without_user_input(turn_intent: Any) -> bool:+ if not isinstance(turn_intent, TurnIntent):+ return False+ if turn_intent.mode not in _AUTHORING_TURN_INTENT_MODES:+ return False+ if turn_intent.authority.requires_user_input:+ return False+ return turn_intent.authority.may_update_workflow+++def _has_current_turn_update_or_test_marker(ctx: Any) -> bool:+ if bool(getattr(ctx, "update_workflow_called", False)):+ return True+ if bool(getattr(ctx, "test_after_update_done", False)):+ return True+ if getattr(ctx, "last_update_block_count", None) is not None:+ return True+ if getattr(ctx, "last_test_ok", None) is not None:+ return True+ for field_name in (+ "last_run_blocks_workflow_run_id",+ "last_successful_run_blocks_workflow_run_id",+ "last_outcome_gate_workflow_run_id",+ ):+ marker = getattr(ctx, field_name, None)+ if isinstance(marker, str) and marker.strip():+ return True+ return False+++def _present_completion_contract_ask_retry(ctx: Any, parsed: dict[str, Any]) -> str | None:+ if parsed.get("type") != "ASK_QUESTION":+ return None+ request_policy = getattr(ctx, "request_policy", None)+ if not isinstance(request_policy, RequestPolicy):+ return None+ if not request_policy_has_present_completion_contract(request_policy):+ return None+ if request_policy.user_response_policy == "ask_clarification":+ return None+ if request_policy.clarification_reason not in (None, "none"):+ return None+ if not _turn_intent_can_author_without_user_input(getattr(ctx, "turn_intent", None)):+ return None+ if _has_current_turn_update_or_test_marker(ctx):+ return None+ LOG.info(+ "copilot.present_completion_contract_ask_retry",+ reason_code="present_completion_contract_ask_internal_retry",+ turn_intent_mode=getattr(getattr(ctx, "turn_intent", None), "mode", None),+ )+ return PRESENT_COMPLETION_CONTRACT_ASK_RETRY++def _nudge(config: CopilotConfig | None, key: str) -> str:if config is None:return DEFAULT_ENFORCEMENT_NUDGES[key]@@ -981,6 +1047,10 @@ def _response_coverage_nudge(ctx: Any, parsed: dict[str, Any], config: CopilotCoif discovery_entrypoint_nudge is not None:return discovery_entrypoint_nudge+ present_contract_retry = _present_completion_contract_ask_retry(ctx, parsed)+ if present_contract_retry is not None:+ return present_contract_retry+if response_type not in ("REPLY", "REPLACE_WORKFLOW"):return Nonediff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.pyindex 7b9c225d2..b29e0c308 100644--- a/skyvern/forge/sdk/copilot/output_policy.py+++ b/skyvern/forge/sdk/copilot/output_policy.py@@ -14,6 +14,7 @@ from skyvern.forge.sdk.copilot.output_utils import (from skyvern.forge.sdk.copilot.request_policy import (RequestPolicy,contains_email_password_pair,+ request_policy_has_present_completion_contract,)from skyvern.forge.sdk.copilot.secret_redaction import (RAW_SECRET_PATTERNS,@@ -436,7 +437,7 @@ def evaluate_output_policy(if (response_type == "ASK_QUESTION"and request_policy.user_response_policy != "ask_clarification"- and _request_policy_has_present_completion_contract(request_policy)+ and request_policy_has_present_completion_contract(request_policy)and not has_workflow_proposaland not workflow_attemptedand _asks_to_confirm_output_fields(user_response)@@ -452,10 +453,6 @@ def evaluate_output_policy(return verdict-def _request_policy_has_present_completion_contract(request_policy: RequestPolicy) -> bool:- return request_policy.completion_contract_status == "present" or bool(request_policy.completion_criteria)--def _asks_to_confirm_output_fields(user_response: str | None) -> bool:if not isinstance(user_response, str) or not user_response.strip():return Falsediff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex c1ab93c57..dd186c188 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -303,6 +303,12 @@ class RequestPolicy:return "\n".join(lines)+def request_policy_has_present_completion_contract(request_policy: RequestPolicy | None) -> bool:+ if request_policy is None:+ return False+ return request_policy.completion_contract_status == "present" or bool(request_policy.completion_criteria)++_TRANSCRIPT_TOTAL_CHAR_BUDGET = 2048TRANSCRIPT_ANCHOR_CHAR_CAP = 512_TRANSCRIPT_RETAINED_MIN_CHARS = 512diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 852c6fa43..4e88f47e8 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -1807,7 +1807,7 @@ class TestTranslateToAgentResultGating:completion_criteria=[CompletionCriterion(id="provider", outcome="The returned record identifies the provider."),],- )+ ),)result = _fake_run_result({"type": "ASK_QUESTION", "user_response": "Which saved credential should I use?"})diff --git a/tests/unit/test_copilot_pre_discovery_url_nudge.py b/tests/unit/test_copilot_pre_discovery_url_nudge.pyindex 5c7d297fb..ee5f367b7 100644--- a/tests/unit/test_copilot_pre_discovery_url_nudge.py+++ b/tests/unit/test_copilot_pre_discovery_url_nudge.py@@ -14,13 +14,18 @@ from __future__ import annotationsfrom types import SimpleNamespace+import pytest+from skyvern.forge.sdk.copilot.build_phase import BuildPhasefrom skyvern.forge.sdk.copilot.enforcement import (MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES,PRE_DISCOVERY_URL_QUESTION_NUDGE,+ PRESENT_COMPLETION_CONTRACT_ASK_RETRY,_pre_discovery_url_question_nudge,_response_coverage_nudge,)+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentModeclass _Ctx:@@ -38,6 +43,13 @@ class _Ctx:self.no_workflow_nudge_count = 0self.coverage_nudge_count = 0self.format_nudge_count = 0+ self.test_after_update_done = False+ self.workflow_persisted = False+ self.last_update_block_count = None+ self.last_test_ok = None+ self.last_run_blocks_workflow_run_id = None+ self.last_successful_run_blocks_workflow_run_id = None+ self.last_outcome_gate_workflow_run_id = None_URL_ASK = {@@ -45,6 +57,45 @@ _URL_ASK = {"user_response": "I can build that, but I need the exact URL to use as the entry point. Please provide the URL.",}+_OUTPUT_CONFIRMATION_ASK = {+ "type": "ASK_QUESTION",+ "user_response": "Please confirm the output fields before I build this workflow.",+}+++def _authoring_intent(*, mode: TurnIntentMode = TurnIntentMode.BUILD, requires_user_input: bool = False) -> TurnIntent:+ return TurnIntent(+ mode=mode,+ authority=TurnIntentAuthority(+ may_update_workflow=mode in {TurnIntentMode.BUILD, TurnIntentMode.EDIT, TurnIntentMode.DRAFT_ONLY},+ may_run_blocks=mode in {TurnIntentMode.BUILD, TurnIntentMode.EDIT, TurnIntentMode.DRAFT_ONLY},+ requires_user_input=requires_user_input,+ ),+ )+++def _present_contract_policy(**overrides: object) -> RequestPolicy:+ defaults = dict(+ user_response_policy="proceed",+ clarification_reason="none",+ completion_contract_status="present",+ completion_criteria=[+ CompletionCriterion(id="record_id", outcome="The returned record includes the requested id."),+ ],+ )+ defaults.update(overrides)+ return RequestPolicy(**defaults)+++def _present_contract_ctx(**overrides: object) -> _Ctx:+ ctx = _Ctx()+ ctx.build_phase = BuildPhase.COMPOSING+ ctx.request_policy = _present_contract_policy()+ ctx.turn_intent = _authoring_intent()+ for name, value in overrides.items():+ setattr(ctx, name, value)+ return ctx+def test_pre_discovery_url_ask_in_initial_phase_steers_to_discovery() -> None:ctx = _Ctx()@@ -142,3 +193,68 @@ def test_pre_discovery_nudge_is_bounded() -> None:assert ctx.pre_discovery_url_question_nudge_count == expected_countassert _pre_discovery_url_question_nudge(ctx, _URL_ASK) is Noneassert ctx.pre_discovery_url_question_nudge_count == MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES+++def test_present_completion_contract_ask_returns_internal_retry() -> None:+ ctx = _present_contract_ctx()++ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) == PRESENT_COMPLETION_CONTRACT_ASK_RETRY+++def test_present_completion_contract_ask_has_no_per_rule_cap() -> None:+ ctx = _present_contract_ctx()++ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) == PRESENT_COMPLETION_CONTRACT_ASK_RETRY+ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) == PRESENT_COMPLETION_CONTRACT_ASK_RETRY+++def test_present_completion_contract_ask_allows_request_policy_clarification() -> None:+ ctx = _present_contract_ctx(+ request_policy=_present_contract_policy(+ user_response_policy="ask_clarification",+ clarification_reason="credential_name_unresolved",+ ),+ turn_intent=_authoring_intent(mode=TurnIntentMode.CLARIFY, requires_user_input=True),+ )++ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) is None+++def test_present_completion_contract_ask_allows_non_none_clarification_reason() -> None:+ ctx = _present_contract_ctx(+ request_policy=_present_contract_policy(clarification_reason="credential_name_unresolved"),+ )++ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) is None+++def test_present_completion_contract_ask_allows_clarify_turn_intent() -> None:+ ctx = _present_contract_ctx(+ turn_intent=TurnIntent(+ mode=TurnIntentMode.CLARIFY,+ authority=TurnIntentAuthority(requires_user_input=True),+ ),+ )++ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) is None+++def test_present_completion_contract_ask_allows_turn_intent_requiring_user_input() -> None:+ ctx = _present_contract_ctx(turn_intent=_authoring_intent(requires_user_input=True))++ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) is None+++ "marker",+ [+ {"update_workflow_called": True},+ {"last_update_block_count": 1},+ {"last_test_ok": False},+ {"last_run_blocks_workflow_run_id": "wr_test"},+ ],+)+def test_present_completion_contract_ask_allows_after_update_or_test_marker(marker: dict[str, object]) -> None:+ ctx = _present_contract_ctx(**marker)++ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) is Nonediff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.pyindex 9a3f62aca..8373b74e7 100644--- a/tests/unit/test_copilot_request_policy_completion_criteria.py+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py@@ -21,6 +21,7 @@ from skyvern.forge.sdk.copilot.request_policy import (_classify_request,_parse_completion_criteria,_render_active_criteria_for_prompt,+ request_policy_has_present_completion_contract,)@@ -96,6 +97,27 @@ def _requested_output_subset(policy: RequestPolicy, requested_output_paths: set[]+def test_present_completion_contract_helper_accepts_present_status() -> None:+ policy = RequestPolicy(completion_contract_status="present")++ assert request_policy_has_present_completion_contract(policy) is True+++def test_present_completion_contract_helper_accepts_criteria_backed_policy() -> None:+ policy = RequestPolicy(+ completion_contract_status="absent",+ completion_criteria=[CompletionCriterion(id="record_id", outcome="The returned record includes an id.")],+ )++ assert request_policy_has_present_completion_contract(policy) is True+++def test_present_completion_contract_helper_rejects_absent_policy() -> None:+ policy = RequestPolicy(completion_contract_status="absent", completion_criteria=[])++ assert request_policy_has_present_completion_contract(policy) is False++@pytest.mark.asyncioasync def test_classifier_output_is_augmented_with_generic_requested_outputs() -> None:policy = await _policy_for_message(