Skip to content

Automate browser based workflows with AI

fix(SKY-11996): restore copilot actuation-obligation feature reverted by #13210 (#7230)

Shuchang Zhengbf3624d788e3
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 090cfffc1..b4156db24 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -10,7 +10,7 @@ import contextlib
import json
import re
import uuid
-from collections.abc import Callable, Iterator
+from collections.abc import Callable, Iterator, Mapping
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
@@ -57,6 +57,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
is_optional_dismissal_only_trajectory,
render_synthesized_offer_text,
synthesize_code_block,
+ trajectory_has_browser_fill_interaction,
)
from skyvern.forge.sdk.copilot.completion_criteria_store import (
StoredCriteriaSnapshot,
@@ -106,19 +107,33 @@ from skyvern.forge.sdk.copilot.outcome_verification_trace import (
record_criteria_lifecycle,
record_gate_decision,
)
+from skyvern.forge.sdk.copilot.output_contracts import (
+ OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,
+ OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,
+)
from skyvern.forge.sdk.copilot.output_policy import (
+ ACTUATION_OBLIGATION_STEER_REASON_CODE,
+ ACTUATION_OBLIGATION_UNMET_REASON_CODE,
UNVALIDATED_DISCLOSURE_PHRASES,
WORKFLOW_PRESENT_SENTINEL,
+ ActuationObligationEvaluation,
+ ActuationObligationStatus,
+ CannotActReason,
CopilotOutputKind,
OutputPolicyReason,
OutputPolicyVerdict,
+ actuation_obligation_key,
build_output_policy_diagnostics,
derive_output_kind,
+ evaluate_actuation_obligation,
evaluate_output_policy,
hard_block_output_policy_verdict,
normalize_response_scaffolding,
output_policy_verdict_from_trace_data,
output_policy_verdict_to_trace_data,
+ prior_turn_satisfies_actuation_terminal_condition,
+ request_policy_requires_durable_fill,
+ turn_intent_requires_actuation,
)
from skyvern.forge.sdk.copilot.output_utils import (
extract_final_text,
@@ -481,16 +496,7 @@ def _store_request_policy_on_context(
chat_history_text=policy_inputs.chat_history_text,
previous_user_message=policy_inputs.previous_user_message,
)
- _reconcile_completion_criteria_on_context(ctx, policy, policy_inputs)
- ctx.request_policy = policy
- ctx.allow_untested_workflow_draft = policy.testing_intent == "skip_test"
- ctx.user_message = agent_user_message
- ctx.block_goal_main_goal = _build_block_goal_main_goal(
- user_message=agent_user_message,
- chat_history_text=policy_chat_history_text,
- global_llm_context=policy_inputs.global_llm_context,
- )
- ctx.turn_intent = build_turn_intent(
+ turn_intent = build_turn_intent(
user_message=policy_inputs.user_message,
workflow_yaml=policy_inputs.workflow_yaml,
chat_history=policy_inputs.chat_history_messages,
@@ -503,6 +509,16 @@ def _store_request_policy_on_context(
classifier_result=turn_intent_classifier_result,
fix_origin=policy_inputs.fix_origin,
)
+ _reconcile_completion_criteria_on_context(ctx, policy, policy_inputs)
+ ctx.request_policy = policy
+ ctx.allow_untested_workflow_draft = policy.testing_intent == "skip_test"
+ ctx.user_message = agent_user_message
+ ctx.block_goal_main_goal = _build_block_goal_main_goal(
+ user_message=agent_user_message,
+ chat_history_text=policy_chat_history_text,
+ global_llm_context=policy_inputs.global_llm_context,
+ )
+ ctx.turn_intent = turn_intent
def _turn_intent_log_fields(intent: TurnIntent | None) -> dict[str, Any]:
@@ -1237,6 +1253,18 @@ _DRAFT_ONLY_MCP_TOOL_ALLOWLIST = frozenset({"get_block_schema", "validate_block"
_DRAFT_ONLY_NATIVE_TOOL_DENYLIST = frozenset(
{"discover_workflow_entrypoint", "inspect_page_for_composition", "fill_credential_field"}
)
+_MUTATING_BROWSER_SCOUT_TOOLS = frozenset({"click", "press_key", "type_text", "select_option", "fill_credential_field"})
+_STRUCTURAL_CANNOT_ACT_REASON_CODES = frozenset(
+ {OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE, OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE}
+)
+_FinalActionDataValue = str | int | float | bool | None
+_ACTUATION_OBLIGATION_STEER_REPLY = (
+ "I need to act in the browser for this request. Please let me try the browser action, or tell me what blocks it."
+)
+_ACTUATION_OBLIGATION_UNMET_REPLY = (
+ "I still did not act in the browser for this request, so I cannot treat the task as completed. "
+ "Please retry or tell me what prevented the browser action."
+)
def _request_policy_disables_browser_scout_tools(request_policy: RequestPolicy | None) -> bool:
@@ -1248,6 +1276,108 @@ def _request_policy_disables_browser_scout_tools(request_policy: RequestPolicy |
)
+def _turn_intent_requires_browser_state(turn_intent: TurnIntent | None) -> bool:
+ return isinstance(turn_intent, TurnIntent) and RequiredContextKey.BROWSER_STATE in turn_intent.required_context
+
+
+def _turn_intent_browser_only_tools(turn_intent: TurnIntent | None) -> bool:
+ return (
+ isinstance(turn_intent, TurnIntent)
+ and turn_intent.mode == TurnIntentMode.BUILD
+ and RequiredContextKey.BROWSER_STATE in turn_intent.required_context
+ and not turn_intent.authority.may_update_workflow
+ and not turn_intent.authority.may_run_blocks
+ )
+
+
+def _successful_mutating_browser_action_count(ctx: CopilotContext) -> int:
+ return sum(
+ 1
+ for interaction in ctx.scout_trajectory
+ if isinstance(interaction, Mapping) and interaction.get("tool_name") in _MUTATING_BROWSER_SCOUT_TOOLS
+ )
+
+
+def _successful_durable_fill_count(ctx: CopilotContext) -> int:
+ return int(trajectory_has_browser_fill_interaction(ctx.scout_trajectory))
+
+
+def _typed_cannot_act_reason(ctx: CopilotContext) -> CannotActReason | None:
+ signal = ctx.blocker_signal if isinstance(ctx.blocker_signal, CopilotToolBlockerSignal) else None
+ if signal is None:
+ return None
+ if signal.blocker_kind == "authority_denied":
+ return None
+ if signal.blocker_kind == "missing_required_context":
+ return CannotActReason.MISSING_FIELD_VALUE
+ if signal.recovery_hint == "ask_user_clarifying":
+ return CannotActReason.AMBIGUOUS_TARGET
+ if signal.internal_reason_code in _STRUCTURAL_CANNOT_ACT_REASON_CODES:
+ return CannotActReason.STRUCTURAL_BLOCKER
+ return None
+
+
+def _evaluate_actuation_obligation_for_output(
+ ctx: CopilotContext,
+ action_data: Mapping[str, _FinalActionDataValue],
+ response_type: str,
+ output_kind: CopilotOutputKind,
+) -> ActuationObligationEvaluation:
+ cannot_act_reason = _typed_cannot_act_reason(ctx)
+ evaluation = evaluate_actuation_obligation(
+ turn_intent=ctx.turn_intent,
+ response_type=response_type,
+ output_kind=output_kind,
+ successful_mutating_browser_actions=_successful_mutating_browser_action_count(ctx),
+ cannot_act_reason=cannot_act_reason,
+ prior_turn_outcome=ctx.prior_turn_outcome,
+ )
+ if evaluation.status != ActuationObligationStatus.ALLOWED:
+ return evaluation
+ if (
+ not _turn_intent_browser_only_tools(ctx.turn_intent)
+ or not request_policy_requires_durable_fill(ctx.request_policy)
+ or response_type != "REPLY"
+ or output_kind != CopilotOutputKind.INFORMATIONAL_ANSWER
+ ):
+ return evaluation
+ if _successful_durable_fill_count(ctx) > 0 or cannot_act_reason is not None:
+ return evaluation
+ obligation_key = actuation_obligation_key(ctx.turn_intent)
+ if prior_turn_satisfies_actuation_terminal_condition(ctx.prior_turn_outcome, obligation_key):
+ return ActuationObligationEvaluation(
+ status=ActuationObligationStatus.TERMINAL,
+ reason_code=ACTUATION_OBLIGATION_UNMET_REASON_CODE,
+ obligation_key=obligation_key,
+ )
+ return ActuationObligationEvaluation(
+ status=ActuationObligationStatus.STEER,
+ reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
+ obligation_key=obligation_key,
+ )
+
+
+def _actuation_obligation_diagnostics(
+ ctx: CopilotContext,
+ evaluation: ActuationObligationEvaluation,
+) -> dict[str, str | int | bool | None]:
+ if (
+ not turn_intent_requires_actuation(ctx.turn_intent)
+ and evaluation.status == ActuationObligationStatus.ALLOWED
+ and evaluation.cannot_act_reason is None
+ ):
+ return {}
+ return {
+ "actuation_obligation_status": evaluation.status.value,
+ "actuation_obligation_reason_code": evaluation.reason_code or None,
+ "actuation_obligation_key": evaluation.obligation_key or None,
+ "cannot_act_reason": evaluation.cannot_act_reason.value if evaluation.cannot_act_reason is not None else None,
+ "successful_mutating_browser_actions": _successful_mutating_browser_action_count(ctx),
+ "durable_fill_required": request_policy_requires_durable_fill(ctx.request_policy),
+ "successful_durable_browser_fills": _successful_durable_fill_count(ctx),
+ }
+
+
def _mcp_tool_surface_for_turn(
alias_map: dict[str, str],
overlays: dict[str, Any],
@@ -3303,6 +3433,15 @@ async def _translate_to_agent_result(
unvalidated=unvalidated,
output_kind=output_kind,
)
+ actuation_obligation = _evaluate_actuation_obligation_for_output(ctx, action_data, resp_type, output_kind)
+ if actuation_obligation.status == ActuationObligationStatus.STEER:
+ raw_output_policy_verdict.add(OutputPolicyReason.ACTUATION_OBLIGATION_STEER)
+ actuation_obligation_terminal = actuation_obligation.status == ActuationObligationStatus.TERMINAL
+ if actuation_obligation_terminal:
+ resp_type = "REPLY"
+ user_response = _ACTUATION_OBLIGATION_UNMET_REPLY
+ last_workflow = None
+ last_workflow_yaml = None
output_policy_verdict = _copy_output_policy_verdict(raw_output_policy_verdict)
soft_rewrite_reasons: list[OutputPolicyReason] = []
unbacked_workflow_delivery_rewritten = False
@@ -3356,6 +3495,7 @@ async def _translate_to_agent_result(
hard_block_reason_codes=list(output_policy_verdict.reason_codes),
soft_rewrite_reason_codes=soft_rewrite_reasons,
)
+ output_policy_diagnostics.update(_actuation_obligation_diagnostics(ctx, actuation_obligation))
trace_data = output_policy_verdict_to_trace_data(
output_policy_verdict,
surface="final_translation",
@@ -3386,15 +3526,22 @@ async def _translate_to_agent_result(
else []
)
reason_code = ",".join(reason_codes)
+ terminal_reason = None
+ if actuation_obligation_terminal:
+ reason_code = ACTUATION_OBLIGATION_UNMET_REASON_CODE
+ terminal_reason = ACTUATION_OBLIGATION_UNMET_REASON_CODE
final_user_response, turn_outcome = apply_repeated_reply_guard(
final_text=final_user_response,
attempted_kind=attempted_kind,
blocked_signatures=ctx.blocked_reply_signatures,
reason_code=reason_code,
+ terminal_reason=terminal_reason,
turn_intent=ctx.turn_intent,
tool_calls=[name for name in tool_call_names if name],
)
+ if actuation_obligation.reason_code:
+ turn_outcome = turn_outcome.model_copy(update={"actuation_obligation_key": actuation_obligation.obligation_key})
return _finalize_result_with_blocker_override(
ctx,
@@ -3595,6 +3742,7 @@ def _blocked_final_output_kind(verdict: OutputPolicyVerdict) -> CopilotOutputKin
OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS,
OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE,
OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED,
+ OutputPolicyReason.ACTUATION_OBLIGATION_STEER,
}
if any(reason in clarification_reasons for reason in verdict.reason_codes):
return CopilotOutputKind.CLARIFICATION_REQUEST
@@ -3661,6 +3809,16 @@ def _evaluate_copilot_final_output_policy(
workflow_attempted=workflow_attempted,
output_kind=output_kind,
)
+ actuation_obligation = _evaluate_actuation_obligation_for_output(
+ ctx,
+ action_data,
+ policy_response_type,
+ output_kind,
+ )
+ if actuation_obligation.status == ActuationObligationStatus.STEER:
+ raw_verdict.add(OutputPolicyReason.ACTUATION_OBLIGATION_STEER)
+ elif actuation_obligation.status == ActuationObligationStatus.TERMINAL:
+ raw_verdict.add(OutputPolicyReason.ACTUATION_OBLIGATION_UNMET)
hard_verdict = hard_block_output_policy_verdict(raw_verdict)
deferred_reason_codes = _defer_avoidable_ask_to_recycle(ctx, hard_verdict, response_type)
if deferred_reason_codes is not None:
@@ -3674,6 +3832,7 @@ def _evaluate_copilot_final_output_policy(
hard_block_reason_codes=list(hard_verdict.reason_codes),
soft_rewrite_reason_codes=[],
)
+ diagnostics.update(_actuation_obligation_diagnostics(ctx, actuation_obligation))
if deferred_reason_codes is not None:
diagnostics["deferred_to_recycle"] = True
diagnostics["deferred_reason_codes"] = [reason.value for reason in deferred_reason_codes]
@@ -3878,6 +4037,10 @@ def _build_output_policy_blocked_result(
"workflow without a tracked URL, re-select it so Copilot can confirm its URL scope."
)
add_saved_draft_copy = True
+ elif OutputPolicyReason.ACTUATION_OBLIGATION_UNMET in verdict.reason_codes:
+ user_response = _ACTUATION_OBLIGATION_UNMET_REPLY
+ elif OutputPolicyReason.ACTUATION_OBLIGATION_STEER in verdict.reason_codes:
+ user_response = _ACTUATION_OBLIGATION_STEER_REPLY
elif preserved_workflow is not None:
user_response = (
"I could not safely return that chat reply, but the workflow draft is still saved. "
@@ -3908,13 +4071,37 @@ def _build_output_policy_blocked_result(
composed_from_recorded_evidence = True
if preserved_workflow is not None and add_saved_draft_copy:
user_response = f"{user_response} {_SAVED_DRAFT_OUTPUT_POLICY_SUFFIX}"
+ has_non_actuation_hard_block = any(
+ reason
+ not in {
+ OutputPolicyReason.ACTUATION_OBLIGATION_STEER,
+ OutputPolicyReason.ACTUATION_OBLIGATION_UNMET,
+ }
+ for reason in verdict.reason_codes
+ )
+ if has_non_actuation_hard_block:
+ blocked_reason_code = "output_policy_block"
+ blocked_terminal_reason: str | None = "output_policy_block"
+ elif OutputPolicyReason.ACTUATION_OBLIGATION_UNMET in verdict.reason_codes:
+ blocked_reason_code = ACTUATION_OBLIGATION_UNMET_REASON_CODE
+ blocked_terminal_reason = ACTUATION_OBLIGATION_UNMET_REASON_CODE
+ else:
+ blocked_reason_code = ACTUATION_OBLIGATION_STEER_REASON_CODE
+ blocked_terminal_reason = None
final_user_response, output_policy_outcome = apply_repeated_reply_guard(
final_text=user_response,
attempted_kind=ResponseKind.CLARIFY,
blocked_signatures=ctx.blocked_reply_signatures,
- reason_code="output_policy_block",
- terminal_reason="output_policy_block",
+ reason_code=blocked_reason_code,
+ terminal_reason=blocked_terminal_reason,
)
+ if blocked_reason_code in {ACTUATION_OBLIGATION_STEER_REASON_CODE, ACTUATION_OBLIGATION_UNMET_REASON_CODE}:
+ key = ""
+ if output_policy_diagnostics is not None:
+ key = str(output_policy_diagnostics.get("actuation_obligation_key") or "")
+ output_policy_outcome = output_policy_outcome.model_copy(
+ update={"actuation_obligation_key": key or actuation_obligation_key(ctx.turn_intent)}
+ )
if composed_from_recorded_evidence and fallback_user_response is not None:
composed_verdict = evaluate_output_policy(
request_policy=ctx.request_policy,
@@ -3997,6 +4184,7 @@ async def run_copilot_agent(
prior_copilot_workflow_yaml: str | None = None,
prior_block_count: int | None = None,
stored_completion_criteria: StoredCriteriaSnapshot | None = None,
+ prior_turn_outcome: TurnOutcome | None = None,
) -> AgentResult:
# One id per turn — passed to every downstream AgentResult and
# CopilotContext so the envelope and terminal frames correlate. The
@@ -4034,6 +4222,7 @@ async def run_copilot_agent(
prior_block_count=prior_block_count,
ctx_sink=ctx_sink,
stored_completion_criteria=stored_completion_criteria,
+ prior_turn_outcome=prior_turn_outcome,
)
except Exception as exc:
LOG.error(
@@ -4102,6 +4291,7 @@ async def _run_copilot_turn_impl(
prior_block_count: int | None = None,
ctx_sink: list[CopilotContext] | None = None,
stored_completion_criteria: StoredCriteriaSnapshot | None = None,
+ prior_turn_outcome: TurnOutcome | None = None,
) -> AgentResult:
copilot_config = config or CopilotConfig(security_rules=security_rules)
chat_history_text = _format_chat_history(chat_history)
@@ -4166,6 +4356,7 @@ async def _run_copilot_turn_impl(
turn_index=turn_index,
prior_block_count=prior_block_count,
prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,
+ prior_turn_outcome=prior_turn_outcome,
block_authoring_policy=copilot_config.block_authoring_policy,
impose_synthesized_code_block=copilot_config.impose_synthesized_code_block,
copilot_config=copilot_config,
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index 4ba8d5c5e..567a0d1de 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -1101,6 +1101,21 @@ def trajectory_has_credential_fill(trajectory: Sequence[Mapping[str, Any]]) -> b
return False
+def trajectory_has_browser_fill_interaction(trajectory: Sequence[Mapping[str, Any]]) -> bool:
+ for interaction in trajectory:
+ tool_name = str(interaction.get("tool_name") or "")
+ typed_length = interaction.get("typed_length")
+ if tool_name == "type_text" and (
+ (isinstance(typed_length, int) and typed_length > 0) or str(interaction.get("typed_value") or "").strip()
+ ):
+ return True
+ if tool_name == "select_option" and str(interaction.get("value") or "").strip():
+ return True
+ if tool_name == CREDENTIAL_FILL_TOOL_NAME and str(interaction.get("credential_field") or "").strip():
+ return True
+ return False
+
+
def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
return build_artifact_metadata_skeleton(trajectory, block_label=_SYNTHESIZED_BLOCK_LABEL)
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index bfced6eae..b518762c3 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -643,6 +643,7 @@ class CopilotContext(AgentContext):
target_block_label: str | None = None
turn_intent: TurnIntent | None = None
turn_context_packet: TurnContextPacket | None = None
+ prior_turn_outcome: TurnOutcome | None = None
latest_diagnosis_repair_contract: DiagnosisRepairContract | None = None
blocked_reply_signatures: list[str] = field(default_factory=list)
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 8ce3dfa31..3376e3e31 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -37,6 +37,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
is_optional_dismissal_only_trajectory,
render_synthesized_offer_text,
synthesize_code_block,
+ trajectory_has_browser_fill_interaction,
)
from skyvern.forge.sdk.copilot.completion_criteria_store import requested_output_paths
from skyvern.forge.sdk.copilot.completion_verification import only_structural_requested_output_abstentions
@@ -76,7 +77,10 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY, normalize_failure_reason
from skyvern.forge.sdk.copilot.narration import TransitionKind
from skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryState
-from skyvern.forge.sdk.copilot.output_policy import normalize_response_scaffolding
+from skyvern.forge.sdk.copilot.output_policy import (
+ completion_criterion_requires_browser_fill_delivery,
+ normalize_response_scaffolding,
+)
from skyvern.forge.sdk.copilot.output_utils import (
extract_final_text,
looks_like_workflow_delivery_claim,
@@ -116,7 +120,7 @@ from skyvern.forge.sdk.copilot.turn_halt import (
stash_repair_ceiling_turn_halt,
stash_turn_halt_from_blocker_signal,
)
-from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentMode
+from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentMode
from skyvern.utils.token_counter import count_tokens
if TYPE_CHECKING:
@@ -188,6 +192,7 @@ SYNTHESIZED_BLOCK_PERSISTENCE_TOOL = "update_and_run_blocks"
_SYNTHESIZED_BLOCK_PERSISTENCE_ALLOWED_TOOLS = frozenset(
{SYNTHESIZED_BLOCK_PERSISTENCE_TOOL, "fill_credential_field", "update_workflow"}
)
+_ACTUATION_OBLIGATION_REQUIRED_FILL_TOOL = "type_text"
_SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS = frozenset(
{"click", "press_key", "type_text", "select_option", "navigate_browser"}
)
@@ -2394,6 +2399,39 @@ def _uncovered_output_reject_admits_evaluate(ctx: CopilotContext, tool_name: str
return tool_name == "evaluate" and bool(_active_uncovered_output_reject_paths(ctx))
+def _actuation_obligation_live_fill_delivery_required(ctx: CopilotContext) -> bool:
+ turn_intent = getattr(ctx, "turn_intent", None)
+ if (
+ not isinstance(turn_intent, TurnIntent)
+ or turn_intent.mode != TurnIntentMode.BUILD
+ or RequiredContextKey.BROWSER_STATE not in turn_intent.required_context
+ ):
+ return False
+ if normalize_block_authoring_policy(ctx.block_authoring_policy) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return False
+ request_policy = getattr(ctx, "request_policy", None)
+ criteria: list[CompletionCriterion] = []
+ if isinstance(request_policy, RequestPolicy):
+ criteria.extend(request_policy.completion_criteria)
+ turn_state = getattr(ctx, "completion_criteria_turn_state", None)
+ if turn_state is not None and turn_state.decision is not None:
+ criteria.extend(turn_state.decision.criteria)
+ if any(completion_criterion_requires_browser_fill_delivery(criterion) for criterion in criteria):
+ return True
+ trajectory = getattr(ctx, "scout_trajectory", None)
+ return trajectory_has_browser_fill_interaction(trajectory) if isinstance(trajectory, list) else False
+
+
+def _actuation_obligation_required_fill_tool(ctx: CopilotContext) -> str | None:
+ if _actuation_obligation_live_fill_delivery_required(ctx):
+ return _ACTUATION_OBLIGATION_REQUIRED_FILL_TOOL
+ return None
+
+
+def _actuation_obligation_admits_required_fill_tool(ctx: CopilotContext, tool_name: str) -> bool:
+ return tool_name == _actuation_obligation_required_fill_tool(ctx)
+
+
def consume_uncovered_output_reopen_event(ctx: CopilotContext) -> bool:
"""Arm a one-shot scout-window reopen for the first author-time reject citing an uncovered
requested-output path. Returns True only on that first reject per structural identity; a
@@ -2501,6 +2539,8 @@ def synthesized_block_persistence_signal(ctx: Any, tool_name: str) -> CopilotToo
return None
if _uncovered_output_reject_admits_evaluate(ctx, tool_name):
return None
+ if _actuation_obligation_admits_required_fill_tool(ctx, tool_name):
+ return None
if (
ambiguous_selector_rescout_state != "block"
and not _should_force_synthesized_block_persistence(ctx)
diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.py
index 08581034f..caa8b04ea 100644
--- a/skyvern/forge/sdk/copilot/output_policy.py
+++ b/skyvern/forge/sdk/copilot/output_policy.py
@@ -13,6 +13,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
looks_like_workflow_yaml_in_chat,
)
from skyvern.forge.sdk.copilot.request_policy import (
+ CompletionCriterion,
RequestPolicy,
contains_email_password_pair,
request_policy_has_present_completion_contract,
@@ -21,6 +22,7 @@ from skyvern.forge.sdk.copilot.secret_redaction import (
RAW_SECRET_PATTERNS,
SECRET_KEYWORD_ASSIGNMENT_PATTERN,
)
+from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentMode
from skyvern.forge.sdk.copilot.workflow_credential_utils import (
block_credential_ids,
credential_param_ids,
@@ -30,8 +32,12 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import (
workflow_credential_ids_from_parsed,
workflow_credential_origins_from_parsed,
)
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome
WORKFLOW_PRESENT_SENTINEL = object()
+ACTUATION_OBLIGATION_STEER_REASON_CODE = "actuation_obligation_steer"
+ACTUATION_OBLIGATION_UNMET_REASON_CODE = "actuation_obligation_unmet"
+ACTUATION_OBLIGATION_BROWSER_ACTION_KEY = "browser_state:build:no_update:no_run"
_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")
_PLACEHOLDER_MARKERS = ("{{", "{%", "[REDACTED_SECRET]")
# RHS of a secret-keyword assignment that references a bound value instead of carrying one:
@@ -183,6 +189,26 @@ class CopilotOutputKind(StrEnum):
WORKFLOW_RUN_RESULT = "workflow_run_result"
+class CannotActReason(StrEnum):
+ MISSING_FIELD_VALUE = "missing_field_value"
+ AMBIGUOUS_TARGET = "ambiguous_target"
+ STRUCTURAL_BLOCKER = "structural_blocker"
+
+
+class ActuationObligationStatus(StrEnum):
+ ALLOWED = "allowed"
+ STEER = "steer"
+ TERMINAL = "terminal"
+
+
+@dataclass(frozen=True)
+class ActuationObligationEvaluation:
+ status: ActuationObligationStatus = ActuationObligationStatus.ALLOWED
+ reason_code: str = ""
+ cannot_act_reason: CannotActReason | None = None
+ obligation_key: str = ""
+
+
class OutputPolicyReason(StrEnum):
RAW_SECRET_LEAK = "raw_secret_leak"
REQUEST_POLICY_CLARIFICATION_BYPASS = "request_policy_clarification_bypass"
@@ -199,6 +225,8 @@ class OutputPolicyReason(StrEnum):
SELF_PRESCRIPTIVE_PHRASE_LEAK = "self_prescriptive_phrase_leak"
WORKFLOW_YAML_IN_REPLY = "workflow_yaml_in_reply"
AVOIDABLE_OUTPUT_FIELD_CONFIRMATION = "avoidable_output_field_confirmation"
+ ACTUATION_OBLIGATION_STEER = ACTUATION_OBLIGATION_STEER_REASON_CODE
+ ACTUATION_OBLIGATION_UNMET = ACTUATION_OBLIGATION_UNMET_REASON_CODE
@dataclass
@@ -232,10 +260,97 @@ _FINAL_OUTPUT_HARD_BLOCK_REASONS: frozenset[OutputPolicyReason] = frozenset(
OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK,
OutputPolicyReason.OUTPUT_POLICY_CONTEXT_MISSING,
OutputPolicyReason.AVOIDABLE_OUTPUT_FIELD_CONFIRMATION,
+ OutputPolicyReason.ACTUATION_OBLIGATION_STEER,
+ OutputPolicyReason.ACTUATION_OBLIGATION_UNMET,
}
)
+def coerce_cannot_act_reason(value: str | None) -> CannotActReason | None:
+ if value is None:
+ return None
+ try:
+ return CannotActReason(value)
+ except ValueError:
+ return None
+
+
+def turn_intent_requires_actuation(turn_intent: TurnIntent | None) -> bool:
+ return (
+ isinstance(turn_intent, TurnIntent)
+ and turn_intent.mode in {TurnIntentMode.BUILD, TurnIntentMode.UNKNOWN}
+ and RequiredContextKey.BROWSER_STATE in turn_intent.required_context
+ and not turn_intent.authority.may_update_workflow
+ and not turn_intent.authority.may_run_blocks
+ )
+
+
+def actuation_obligation_key(turn_intent: TurnIntent | None) -> str:
+ if not turn_intent_requires_actuation(turn_intent):
+ return ""
+ if turn_intent is not None and turn_intent.mode == TurnIntentMode.UNKNOWN:
+ return "browser_state:unknown:no_update:no_run"
+ return ACTUATION_OBLIGATION_BROWSER_ACTION_KEY
+
+
+def completion_criterion_requires_browser_fill_delivery(criterion: CompletionCriterion) -> bool:
+ if criterion.kind == "terminal_action" and criterion.terminal_action_family == "form":
+ return True
+ return criterion.kind == "outcome" and criterion.level == "run" and criterion.method_mandated
+
+
+def request_policy_requires_durable_fill(request_policy: RequestPolicy | None) -> bool:
+ return isinstance(request_policy, RequestPolicy) and any(
+ completion_criterion_requires_browser_fill_delivery(criterion)
+ for criterion in request_policy.completion_criteria
+ )
+
+
+def prior_turn_satisfies_actuation_terminal_condition(
+ prior_turn_outcome: TurnOutcome | None,
+ obligation_key: str,
+) -> bool:
+ # One key covers the turn's actuation obligation; a later partial action
+ # still failed the same browser-fill contract.
+ return (
+ prior_turn_outcome is not None
+ and prior_turn_outcome.reason_code
+ in {ACTUATION_OBLIGATION_STEER_REASON_CODE, ACTUATION_OBLIGATION_UNMET_REASON_CODE}
+ and prior_turn_outcome.actuation_obligation_key == obligation_key
+ )
+
+
+def evaluate_actuation_obligation(
+ *,
+ turn_intent: TurnIntent | None,
+ response_type: str,
+ output_kind: CopilotOutputKind,
+ successful_mutating_browser_actions: int,
+ cannot_act_reason: CannotActReason | None,
+ prior_turn_outcome: TurnOutcome | None,
+) -> ActuationObligationEvaluation:
+ obligation_key = actuation_obligation_key(turn_intent)
+ if (
+ not turn_intent_requires_actuation(turn_intent)
+ or response_type != "REPLY"
+ or output_kind != CopilotOutputKind.INFORMATIONAL_ANSWER
+ ):
+ return ActuationObligationEvaluation(cannot_act_reason=cannot_act_reason)
+ if successful_mutating_browser_actions > 0 or cannot_act_reason is not None:
+ return ActuationObligationEvaluation(cannot_act_reason=cannot_act_reason, obligation_key=obligation_key)
+ if prior_turn_satisfies_actuation_terminal_condition(prior_turn_outcome, obligation_key):
+ return ActuationObligationEvaluation(
+ status=ActuationObligationStatus.TERMINAL,
+ reason_code=ACTUATION_OBLIGATION_UNMET_REASON_CODE,
+ obligation_key=obligation_key,
+ )
+ return ActuationObligationEvaluation(
+ status=ActuationObligationStatus.STEER,
+ reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
+ obligation_key=obligation_key,
+ )
+
+
def hard_block_output_policy_verdict(verdict: OutputPolicyVerdict) -> OutputPolicyVerdict:
hard_reasons = [reason for reason in verdict.reason_codes if reason in _FINAL_OUTPUT_HARD_BLOCK_REASONS]
return OutputPolicyVerdict(
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 960a333ff..72fc572b7 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -2069,6 +2069,7 @@ async def _new_copilot_chat_post(
prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,
prior_block_count=prior_block_count,
stored_completion_criteria=stored_completion_criteria,
+ prior_turn_outcome=prior_turn_outcome,
)
agent_result.turn_outcome = _with_current_copilot_code_mode_metadata(
diff --git a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
index 23b55a413..8e4d50a48 100644
--- a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
+++ b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
@@ -29,6 +29,7 @@ class TurnOutcome(BaseModel):
turn_intent_summary: dict[str, Any] = Field(default_factory=dict)
response_kind: ResponseKind
reason_code: str = ""
+ actuation_obligation_key: str = ""
normalized_reply_signature: str = ""
tool_calls: list[str] = Field(default_factory=list)
terminal_reason: str | None = None
diff --git a/tests/unit/test_copilot_agent_blocker_renderer.py b/tests/unit/test_copilot_agent_blocker_renderer.py
index 399ccfc98..fc306f105 100644
--- a/tests/unit/test_copilot_agent_blocker_renderer.py
+++ b/tests/unit/test_copilot_agent_blocker_renderer.py
@@ -27,7 +27,12 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
)
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext
-from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, OutputPolicyReason, OutputPolicyVerdict
+from skyvern.forge.sdk.copilot.output_policy import (
+ ACTUATION_OBLIGATION_STEER_REASON_CODE,
+ CopilotOutputKind,
+ OutputPolicyReason,
+ OutputPolicyVerdict,
+)
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcome
from skyvern.forge.sdk.copilot.turn_halt import TurnHalt, TurnHaltKind
@@ -338,6 +343,18 @@ def test_output_policy_specific_branches_bypass_recorded_terminal_fallback() ->
assert result.narrative_payload["responseType"] == "ASK_QUESTION"
+def test_output_policy_hard_block_priority_over_actuation_steer() -> None:
+ ctx = _ctx()
+
+ result = _blocked_result(ctx, OutputPolicyReason.RAW_SECRET_LEAK, OutputPolicyReason.ACTUATION_OBLIGATION_STEER)
+
+ assert result.user_response == _RAW_SECRET_LEAK_REFUSAL
+ assert result.turn_outcome is not None
+ assert result.turn_outcome.reason_code == "output_policy_block"
+ assert result.turn_outcome.terminal_reason == "output_policy_block"
+ assert result.turn_outcome.reason_code != ACTUATION_OBLIGATION_STEER_REASON_CODE
+
+
def test_shim_preserves_workflow_draft_when_signal_opts_in() -> None:
"""Late-block-running blockers say 'wrap up with what you have' — they
surface the saved draft alongside the rendered chat reply."""
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 070c962b0..f9fd3a836 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -60,6 +60,11 @@ from skyvern.forge.sdk.copilot.enforcement import (
verified_goal_satisfied_context,
)
from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
+from skyvern.forge.sdk.copilot.output_policy import (
+ ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
+ ACTUATION_OBLIGATION_STEER_REASON_CODE,
+ ACTUATION_OBLIGATION_UNMET_REASON_CODE,
+)
from skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failure
from skyvern.forge.sdk.copilot.request_policy import (
_MAX_COMPLETION_CRITERIA,
@@ -90,12 +95,14 @@ from skyvern.forge.sdk.copilot.turn_halt import (
stash_repair_ceiling_turn_halt,
)
from skyvern.forge.sdk.copilot.turn_intent import (
+ RequiredContextKey,
TurnIntent,
TurnIntentAuthority,
TurnIntentMode,
TurnIntentReasonCode,
)
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatHistoryMessage,
WorkflowCopilotChatSender,
@@ -1640,7 +1647,6 @@ class TestRequestPolicyInputGuardrail:
request_policy_handler=request_policy_handler,
turn_intent_handler=turn_intent_handler,
)
-
guardrails = agent_module._build_copilot_input_guardrails(
InputGuardrail,
GuardrailFunctionOutput,
@@ -1886,6 +1892,30 @@ def _chat_request() -> SimpleNamespace:
)
+def _browser_actuation_intent() -> TurnIntent:
+ return TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ required_context=[RequiredContextKey.BROWSER_STATE],
+ authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
+ )
+
+
+def _unknown_browser_actuation_intent() -> TurnIntent:
+ return TurnIntent(
+ mode=TurnIntentMode.UNKNOWN,
+ required_context=[RequiredContextKey.BROWSER_STATE],
+ authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
+ )
+
+
+def _browser_actuation_intent_without_browser_state(*, mode: TurnIntentMode) -> TurnIntent:
+ return TurnIntent(
+ mode=mode,
+ required_context=[],
+ authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
+ )
+
+
class TestBlockGoalMainGoal:
def test_empty_message_returns_empty(self) -> None:
assert agent_module._build_block_goal_main_goal("", chat_history_text="", global_llm_context=None) == ""
@@ -2268,6 +2298,69 @@ class TestTranslateToAgentResultGating:
assert agent_result.response_type == "ASK_QUESTION"
assert agent_result.user_response == "Which account should I use?"
+ def test_actuation_required_reply_without_browser_mutation_returns_steer(self) -> None:
+ ctx = _ctx(turn_intent=_browser_actuation_intent())
+ result = _fake_run_result({"type": "REPLY", "user_response": "I can fill those fields for you."})
+
+ agent_result = asyncio.run(
+ agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+ )
+
+ assert agent_result.response_type == "ASK_QUESTION"
+ assert agent_result.turn_outcome is not None
+ assert agent_result.turn_outcome.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE
+ assert agent_result.turn_outcome.actuation_obligation_key == ACTUATION_OBLIGATION_BROWSER_ACTION_KEY
+
+ def test_repeated_actuation_required_reply_without_browser_mutation_returns_terminal(self) -> None:
+ ctx = _ctx(
+ turn_intent=_browser_actuation_intent(),
+ prior_turn_outcome=TurnOutcome(
+ response_kind=ResponseKind.CLARIFY,
+ reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
+ actuation_obligation_key=ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
+ ),
+ )
+ result = _fake_run_result({"type": "REPLY", "user_response": "I can fill those fields for you."})
+
+ agent_result = asyncio.run(
+ agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+ )
+
+ assert agent_result.response_type == "REPLY"
+ assert agent_result.updated_workflow is None
+ assert agent_result.turn_outcome is not None
+ assert agent_result.turn_outcome.reason_code == ACTUATION_OBLIGATION_UNMET_REASON_CODE
+ assert agent_result.turn_outcome.terminal_reason == ACTUATION_OBLIGATION_UNMET_REASON_CODE
+ assert agent_result.turn_outcome.actuation_obligation_key == ACTUATION_OBLIGATION_BROWSER_ACTION_KEY
+
+ def test_unknown_click_with_authority_denied_blocker_returns_reply(self) -> None:
+ ctx = _ctx(turn_intent=_unknown_browser_actuation_intent())
+ ctx.scout_trajectory.append({"tool_name": "click"})
+ ctx.blocker_signal = CopilotToolBlockerSignal(
+ blocker_kind="authority_denied",
+ agent_steering_text="Use browser tools.",
+ user_facing_reason="I'll respond with the information I already have.",
+ recovery_hint="report_blocker_to_user",
+ internal_reason_code="turn_intent_no_mutation_run_blocked",
+ blocked_tool="update_and_run_blocks",
+ classifier_mode="unknown",
+ )
+ result = _fake_run_result({"type": "REPLY", "user_response": "I'll respond with the information I have."})
+
+ agent_result = asyncio.run(
+ agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+ )
+
+ assert agent_result.response_type == "REPLY"
+ assert agent_result.turn_outcome is not None
+ assert agent_result.turn_outcome.reason_code != ACTUATION_OBLIGATION_STEER_REASON_CODE
+
def test_verified_terminal_state_surfaces_workflow_despite_weak_final_reply(self) -> None:
workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))
ctx = _ctx(
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index fb7f5d4dc..8c227dee8 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -65,7 +65,7 @@ from skyvern.forge.sdk.copilot.output_contracts import (
OutputContractAdvisoryState,
)
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
-from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion
+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
from skyvern.forge.sdk.copilot.streaming_adapter import _update_enforcement_from_tool
from skyvern.forge.sdk.copilot.tools import (
_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
@@ -75,7 +75,7 @@ from skyvern.forge.sdk.copilot.tools import (
_record_workflow_update_result,
)
from skyvern.forge.sdk.copilot.turn_halt import stash_turn_halt_from_blocker_signal
-from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
+from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentAuthority, TurnIntentMode
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
from tests.unit.conftest import make_copilot_context
@@ -128,6 +128,7 @@ class _Ctx:
self.completion_criteria_turn_state = None
self.reached_download_target: ReachedDownloadTarget | None = None
self.author_time_gate_ablation_events = []
+ self.request_policy = None
class TestSynthesizedOfferPersistenceGate:
@@ -340,6 +341,185 @@ class TestSynthesizedOfferPersistenceGate:
assert signal.cleared_by_tools == frozenset({"update_and_run_blocks"})
assert signal.renders_final_reply is False
+ def test_actuation_obligation_admits_required_fill_tool_during_persistence_offer(self) -> None:
+ ctx = _Ctx()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ required_context={RequiredContextKey.BROWSER_STATE},
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="form-submit",
+ outcome="form fields are filled",
+ kind="terminal_action",
+ terminal_action_family="form",
+ )
+ ],
+ )
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.synthesized_block_offered = True
+ ctx.synthesized_block_offered_trajectory_len = 1
+ ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}]
+
+ assert synthesized_block_persistence_signal(ctx, "type_text") is None
+ click_signal = synthesized_block_persistence_signal(ctx, "click")
+ assert isinstance(click_signal, CopilotToolBlockerSignal)
+ assert click_signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
+
+ def test_actuation_obligation_admits_required_fill_tool_for_method_mandated_run_contract(self) -> None:
+ ctx = _Ctx()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ required_context={RequiredContextKey.BROWSER_STATE},
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="visible-fill",
+ outcome="fields are visibly filled on the live page",
+ method_mandated=True,
+ level="run",
+ )
+ ],
+ )
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.synthesized_block_offered = True
+ ctx.synthesized_block_offered_trajectory_len = 1
+ ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}]
+
+ assert synthesized_block_persistence_signal(ctx, "type_text") is None
+ click_signal = synthesized_block_persistence_signal(ctx, "click")
+ assert isinstance(click_signal, CopilotToolBlockerSignal)
+ assert click_signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
+
+ def test_actuation_obligation_blocks_type_text_for_definition_method_contract(self) -> None:
+ ctx = _Ctx()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ required_context={RequiredContextKey.BROWSER_STATE},
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="definition-contract",
+ outcome="inputs are reusable",
+ method_mandated=True,
+ level="definition",
+ )
+ ],
+ )
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.synthesized_block_offered = True
+ ctx.synthesized_block_offered_trajectory_len = 1
+ ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}]
+
+ signal = synthesized_block_persistence_signal(ctx, "type_text")
+
+ assert isinstance(signal, CopilotToolBlockerSignal)
+ assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
+
+ def test_actuation_obligation_admits_required_fill_tool_after_turn_state_reconcile(self) -> None:
+ ctx = _Ctx()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ required_context={RequiredContextKey.BROWSER_STATE},
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="form-submit",
+ outcome="form fields are filled",
+ kind="terminal_action",
+ terminal_action_family="form",
+ )
+ ],
+ )
+ ctx.completion_criteria_turn_state = SimpleNamespace(
+ decision=SimpleNamespace(
+ criteria=(
+ CompletionCriterion(
+ id="workflow-run",
+ outcome="workflow has been tested",
+ kind="terminal_action",
+ terminal_action_family="workflow_run",
+ ),
+ )
+ )
+ )
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.synthesized_block_offered = True
+ ctx.synthesized_block_offered_trajectory_len = 2
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "button.start", "accessible_name": "Start"},
+ {"tool_name": "type_text", "selector": "#company", "accessible_name": "Company"},
+ ]
+
+ assert synthesized_block_persistence_signal(ctx, "type_text") is None
+
+ def test_persistence_offer_blocks_type_text_without_actuation_obligation(self) -> None:
+ ctx = _Ctx()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ required_context={RequiredContextKey.BROWSER_STATE},
+ )
+ ctx.request_policy = RequestPolicy()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.synthesized_block_offered = True
+ ctx.synthesized_block_offered_trajectory_len = 1
+ ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}]
+
+ signal = synthesized_block_persistence_signal(ctx, "type_text")
+
+ assert isinstance(signal, CopilotToolBlockerSignal)
+ assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
+ assert signal.blocked_tool == "type_text"
+
+ def test_actuation_obligation_admits_required_fill_tool_from_fill_trajectory(self) -> None:
+ ctx = _Ctx()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ required_context={RequiredContextKey.BROWSER_STATE},
+ )
+ ctx.request_policy = RequestPolicy()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.synthesized_block_offered = True
+ ctx.synthesized_block_offered_trajectory_len = 1
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#company",
+ "typed_value": "Example Realty Labs Inc",
+ "accessible_name": "Company",
+ }
+ ]
+
+ assert synthesized_block_persistence_signal(ctx, "type_text") is None
+
+ def test_fill_less_trajectory_does_not_arm_actuation_obligation(self) -> None:
+ ctx = _Ctx()
+ ctx.turn_intent = TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ required_context={RequiredContextKey.BROWSER_STATE},
+ )
+ ctx.request_policy = RequestPolicy()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.synthesized_block_offered = True
+ ctx.synthesized_block_offered_trajectory_len = 1
+ ctx.scout_trajectory = [{"tool_name": "click", "selector": "button.start", "accessible_name": "Start"}]
+
+ signal = synthesized_block_persistence_signal(ctx, "type_text")
+
+ assert isinstance(signal, CopilotToolBlockerSignal)
+ assert signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
+
def test_prerun_ambiguous_bare_selector_repair_allows_one_evaluate(self) -> None:
ctx = _Ctx()
ctx.turn_intent = TurnIntent(
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index befb08ec0..898ea3e3a 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -21,12 +21,19 @@ from skyvern.forge.sdk.copilot.enforcement import (
PRESENT_COMPLETION_CONTRACT_ASK_RETRY,
_response_coverage_nudge,
)
+from skyvern.forge.sdk.copilot.output_contracts import OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE
from skyvern.forge.sdk.copilot.output_policy import (
+ ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
+ ACTUATION_OBLIGATION_STEER_REASON_CODE,
+ ACTUATION_OBLIGATION_UNMET_REASON_CODE,
+ ActuationObligationStatus,
+ CannotActReason,
CopilotOutputKind,
OutputPolicyReason,
OutputPolicyVerdict,
_contains_internal_tool_vocab_leak,
derive_output_kind,
+ evaluate_actuation_obligation,
evaluate_output_policy,
hard_block_output_policy_verdict,
normalize_response_scaffolding,
@@ -36,7 +43,8 @@ from skyvern.forge.sdk.copilot.tools import (
_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL,
NATIVE_TOOLS,
)
-from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
+from skyvern.forge.sdk.copilot.turn_intent import RequiredContextKey, TurnIntent, TurnIntentAuthority, TurnIntentMode
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
def _credential(credential_id: str = "cred_safe", tested_url: str = "https://login.example.test/login") -> object:
@@ -63,6 +71,43 @@ def _ctx(**overrides: object) -> CopilotContext:
return CopilotContext(**defaults)
+def _browser_actuation_intent() -> TurnIntent:
+ return TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ required_context=[RequiredContextKey.BROWSER_STATE],
+ authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
+ )
+
+
+def _live_fill_policy() -> RequestPolicy:
+ return _policy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c0",
+ outcome="The form is completed.",
+ kind="terminal_action",
+ terminal_action_family="form",
+ )
+ ]
+ )
+
+
+def _unknown_browser_actuation_intent() -> TurnIntent:
+ return TurnIntent(
+ mode=TurnIntentMode.UNKNOWN,
+ required_context=[RequiredContextKey.BROWSER_STATE],
+ authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
+ )
+
+
+def _browser_actuation_intent_without_browser_state(*, mode: TurnIntentMode) -> TurnIntent:
+ return TurnIntent(
+ mode=mode,
+ required_context=[],
+ authority=TurnIntentAuthority(may_update_workflow=False, may_run_blocks=False),
+ )
+
+
def _fake_run_result(payload: dict) -> SimpleNamespace:
return SimpleNamespace(final_output=json.dumps(payload), new_items=[])
@@ -1384,6 +1429,384 @@ def test_sdk_output_guardrail_hard_blocks_raw_secret_final_text() -> None:
}
+def test_actuation_obligation_steers_zero_action_informational_reply() -> None:
+ evaluation = evaluate_actuation_obligation(
+ turn_intent=_browser_actuation_intent(),
+ response_type="REPLY",
+ output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
+ successful_mutating_browser_actions=0,
+ cannot_act_reason=None,
+ prior_turn_outcome=None,
+ )
+
+ assert evaluation.status == ActuationObligationStatus.STEER
+ assert evaluation.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE
+
+
+def test_actuation_obligation_steers_unknown_browser_only_intent() -> None:
+ evaluation = evaluate_actuation_obligation(
+ turn_intent=_unknown_browser_actuation_intent(),
+ response_type="REPLY",
+ output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
+ successful_mutating_browser_actions=0,
+ cannot_act_reason=None,
+ prior_turn_outcome=None,
+ )
+
+ assert evaluation.status == ActuationObligationStatus.STEER
+ assert evaluation.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE
+
+
+def test_actuation_obligation_allows_typed_cannot_act_reason() -> None:
+ evaluation = evaluate_actuation_obligation(
+ turn_intent=_browser_actuation_intent(),
+ response_type="REPLY",
+ output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
+ successful_mutating_browser_actions=0,
+ cannot_act_reason=CannotActReason.MISSING_FIELD_VALUE,
+ prior_turn_outcome=None,
+ )
+
+ assert evaluation.status == ActuationObligationStatus.ALLOWED
+ assert evaluation.cannot_act_reason == CannotActReason.MISSING_FIELD_VALUE
+
+
+def test_actuation_obligation_repeats_as_terminal() -> None:
+ evaluation = evaluate_actuation_obligation(
+ turn_intent=_browser_actuation_intent(),
+ response_type="REPLY",
+ output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
+ successful_mutating_browser_actions=0,
+ cannot_act_reason=None,
+ prior_turn_outcome=TurnOutcome(
+ response_kind=ResponseKind.CLARIFY,
+ reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
+ actuation_obligation_key=ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
+ ),
+ )
+
+ assert evaluation.status == ActuationObligationStatus.TERMINAL
+ assert evaluation.reason_code == ACTUATION_OBLIGATION_UNMET_REASON_CODE
+
+
+def test_actuation_obligation_prior_unmet_stays_terminal() -> None:
+ evaluation = evaluate_actuation_obligation(
+ turn_intent=_browser_actuation_intent(),
+ response_type="REPLY",
+ output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
+ successful_mutating_browser_actions=0,
+ cannot_act_reason=None,
+ prior_turn_outcome=TurnOutcome(
+ response_kind=ResponseKind.CLARIFY,
+ reason_code=ACTUATION_OBLIGATION_UNMET_REASON_CODE,
+ terminal_reason=ACTUATION_OBLIGATION_UNMET_REASON_CODE,
+ actuation_obligation_key=ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
+ ),
+ )
+
+ assert evaluation.status == ActuationObligationStatus.TERMINAL
+ assert evaluation.reason_code == ACTUATION_OBLIGATION_UNMET_REASON_CODE
+
+
+def test_actuation_obligation_unrelated_prior_steer_stays_recoverable() -> None:
+ evaluation = evaluate_actuation_obligation(
+ turn_intent=_browser_actuation_intent(),
+ response_type="REPLY",
+ output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
+ successful_mutating_browser_actions=0,
+ cannot_act_reason=None,
+ prior_turn_outcome=TurnOutcome(
+ response_kind=ResponseKind.CLARIFY,
+ reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
+ actuation_obligation_key="other",
+ ),
+ )
+
+ assert evaluation.status == ActuationObligationStatus.STEER
+ assert evaluation.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE
+
+
[email protected]("mode", [TurnIntentMode.DOCS_ANSWER, TurnIntentMode.DIAGNOSE])
+def test_actuation_obligation_allows_non_actuation_intents(mode: TurnIntentMode) -> None:
+ evaluation = evaluate_actuation_obligation(
+ turn_intent=TurnIntent(mode=mode, authority=TurnIntentAuthority(may_update_workflow=False)),
+ response_type="REPLY",
+ output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
+ successful_mutating_browser_actions=0,
+ cannot_act_reason=None,
+ prior_turn_outcome=None,
+ )
+
+ assert evaluation.status == ActuationObligationStatus.ALLOWED
+
+
+def test_sdk_output_guardrail_steers_actuation_required_zero_action_reply() -> None:
+ evaluation = evaluate_actuation_obligation(
+ turn_intent=_browser_actuation_intent(),
+ response_type="REPLY",
+ output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
+ successful_mutating_browser_actions=0,
+ cannot_act_reason=None,
+ prior_turn_outcome=None,
+ )
+
+ assert evaluation.status == ActuationObligationStatus.STEER
+ assert evaluation.reason_code == ACTUATION_OBLIGATION_STEER_REASON_CODE
+
+
+def test_sdk_output_guardrail_steers_actuation_required_reply() -> None:
+ ctx = _ctx(
+ turn_intent=_browser_actuation_intent(),
+ tool_activity=[{"tool": "evaluate"}, {"tool": "get_browser_screenshot"}],
+ )
+
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
+ ctx,
+ {"type": "REPLY", "user_response": "I can fill those fields for you."},
+ )
+
+ assert response_type == "REPLY"
+ assert not verdict.allowed
+ assert verdict.reason_codes == [OutputPolicyReason.ACTUATION_OBLIGATION_STEER]
+ assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE
+ assert diagnostics["successful_mutating_browser_actions"] == 0
+ assert diagnostics.get("deferred_to_recycle", False) is False
+
+
+async def test_sdk_output_guardrail_trips_actuation_steer() -> None:
+ ctx = _ctx(turn_intent=_browser_actuation_intent())
+ output_guardrails = agent_module._build_copilot_output_guardrails(OutputGuardrail, GuardrailFunctionOutput)
+ result = await output_guardrails[0].run(
+ RunContextWrapper(context=ctx),
+ SimpleNamespace(),
+ {"type": "REPLY", "user_response": "I can fill those fields for you."},
+ )
+
+ assert result.output.tripwire_triggered is True
+ assert result.output.output_info["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE
+
+
+def test_sdk_output_guardrail_repeated_actuation_steer_terminalizes() -> None:
+ ctx = _ctx(
+ turn_intent=_browser_actuation_intent(),
+ prior_turn_outcome=TurnOutcome(
+ response_kind=ResponseKind.CLARIFY,
+ reason_code=ACTUATION_OBLIGATION_STEER_REASON_CODE,
+ actuation_obligation_key=ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
+ ),
+ )
+ output = {"type": "REPLY", "user_response": "I can fill those fields for you."}
+
+ verdict, _, diagnostics = agent_module._evaluate_copilot_final_output_policy(ctx, output)
+
+ assert not verdict.allowed
+ assert verdict.reason_codes == [OutputPolicyReason.ACTUATION_OBLIGATION_UNMET]
+ assert diagnostics["actuation_obligation_status"] == ActuationObligationStatus.TERMINAL.value
+ assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_UNMET_REASON_CODE
+
+
+def test_sdk_output_guardrail_rejects_uncorroborated_model_typed_cannot_act_reason() -> None:
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
+ _ctx(turn_intent=_browser_actuation_intent()),
+ {
+ "type": "REPLY",
+ "user_response": "I need the field value before I can continue.",
+ "cannot_act_reason": "missing_field_value",
+ },
+ )
+
+ assert response_type == "REPLY"
+ assert not verdict.allowed
+ assert diagnostics["cannot_act_reason"] is None
+ assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE
+
+
+def test_sdk_output_guardrail_allows_structural_blocker_reason() -> None:
+ ctx = _ctx(turn_intent=_browser_actuation_intent())
+ ctx.blocker_signal = CopilotToolBlockerSignal(
+ blocker_kind="missing_required_context",
+ agent_steering_text="Ask which field value to use.",
+ user_facing_reason="I need the field value before I can continue.",
+ recovery_hint="ask_user_clarifying",
+ )
+
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
+ ctx,
+ {"type": "REPLY", "user_response": "I need the field value before I can continue."},
+ )
+
+ assert response_type == "REPLY"
+ assert verdict.allowed
+ assert diagnostics["cannot_act_reason"] == "missing_field_value"
+
+
+def test_sdk_output_guardrail_allows_explicit_structural_blocker_reason_code() -> None:
+ ctx = _ctx(turn_intent=_browser_actuation_intent())
+ ctx.blocker_signal = CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text="Report the unavailable output source.",
+ user_facing_reason="The values are not observable on this page.",
+ recovery_hint="report_blocker_to_user",
+ internal_reason_code=OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,
+ )
+
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
+ ctx,
+ {"type": "REPLY", "user_response": "The values are not observable on this page."},
+ )
+
+ assert response_type == "REPLY"
+ assert verdict.allowed
+ assert diagnostics["cannot_act_reason"] == "structural_blocker"
+
+
+def test_sdk_output_guardrail_rejects_generic_report_blocker_reason() -> None:
+ ctx = _ctx(turn_intent=_browser_actuation_intent())
+ ctx.blocker_signal = CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text="Report the blocker.",
+ user_facing_reason="The page is not ready.",
+ recovery_hint="report_blocker_to_user",
+ internal_reason_code="tool_error_late_block_running",
+ )
+
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
+ ctx,
+ {"type": "REPLY", "user_response": "The page is not ready."},
+ )
+
+ assert response_type == "REPLY"
+ assert not verdict.allowed
+ assert verdict.reason_codes == [OutputPolicyReason.ACTUATION_OBLIGATION_STEER]
+ assert diagnostics["cannot_act_reason"] is None
+
+
+def test_sdk_output_guardrail_allows_successful_mutating_browser_action() -> None:
+ ctx = _ctx(turn_intent=_browser_actuation_intent())
+ ctx.scout_trajectory.append({"tool_name": "click"})
+
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
+ ctx,
+ {"type": "REPLY", "user_response": "Clicked the field."},
+ )
+
+ assert response_type == "REPLY"
+ assert verdict.allowed
+ assert diagnostics["successful_mutating_browser_actions"] == 1
+
+
+def test_sdk_output_guardrail_rejects_form_fill_click_only_reply() -> None:
+ ctx = _ctx(turn_intent=_browser_actuation_intent(), request_policy=_live_fill_policy())
+ ctx.scout_trajectory.append({"tool_name": "click"})
+
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
+ ctx,
+ {"type": "REPLY", "user_response": "The requested form interaction was completed."},
+ )
+
+ assert response_type == "REPLY"
+ assert not verdict.allowed
+ assert verdict.reason_codes == [OutputPolicyReason.ACTUATION_OBLIGATION_STEER]
+ assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE
+ assert diagnostics["successful_mutating_browser_actions"] == 1
+ assert diagnostics["successful_durable_browser_fills"] == 0
+ assert diagnostics["durable_fill_required"] is True
+
+
+def test_sdk_output_guardrail_requires_browser_state_for_form_fill_obligation() -> None:
+ ctx = _ctx(
+ turn_intent=_browser_actuation_intent_without_browser_state(mode=TurnIntentMode.BUILD),
+ request_policy=_live_fill_policy(),
+ )
+ ctx.scout_trajectory.append({"tool_name": "click"})
+
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
+ ctx,
+ {"type": "REPLY", "user_response": "The requested form interaction was completed."},
+ )
+
+ assert response_type == "REPLY"
+ assert verdict.allowed
+ assert diagnostics.get("deferred_to_recycle", False) is False
+ assert "durable_fill_required" not in diagnostics
+
+
+def test_sdk_output_guardrail_allows_non_form_click_reply() -> None:
+ ctx = _ctx(
+ turn_intent=_browser_actuation_intent_without_browser_state(mode=TurnIntentMode.UNKNOWN),
+ )
+ ctx.scout_trajectory.append({"tool_name": "click"})
+
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
+ ctx,
+ {"type": "REPLY", "user_response": "Clicked the requested target."},
+ )
+
+ assert response_type == "REPLY"
+ assert verdict.allowed
+ assert diagnostics.get("deferred_to_recycle", False) is False
+ assert "durable_fill_required" not in diagnostics
+
+
+def test_sdk_output_guardrail_allows_live_fill_after_durable_fill() -> None:
+ ctx = _ctx(turn_intent=_browser_actuation_intent(), request_policy=_live_fill_policy())
+ ctx.scout_trajectory.append({"tool_name": "type_text", "typed_length": 3})
+
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
+ ctx,
+ {"type": "REPLY", "user_response": "The requested form interaction was completed."},
+ )
+
+ assert response_type == "REPLY"
+ assert verdict.allowed
+ assert diagnostics.get("deferred_to_recycle", False) is False
+ assert diagnostics["actuation_obligation_status"] == ActuationObligationStatus.ALLOWED.value
+ assert diagnostics["successful_durable_browser_fills"] == 1
+
+
+def test_sdk_output_guardrail_rejects_live_fill_after_noop_type_text() -> None:
+ ctx = _ctx(turn_intent=_browser_actuation_intent(), request_policy=_live_fill_policy())
+ ctx.scout_trajectory.append({"tool_name": "type_text", "typed_length": 0, "typed_value": ""})
+
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
+ ctx,
+ {"type": "REPLY", "user_response": "The requested form interaction was completed."},
+ )
+
+ assert response_type == "REPLY"
+ assert not verdict.allowed
+ assert diagnostics["actuation_obligation_reason_code"] == ACTUATION_OBLIGATION_STEER_REASON_CODE
+ assert diagnostics["successful_durable_browser_fills"] == 0
+ assert diagnostics["durable_fill_required"] is True
+
+
+def test_sdk_output_guardrail_allows_unknown_click_with_authority_denied_blocker() -> None:
+ ctx = _ctx(turn_intent=_unknown_browser_actuation_intent())
+ ctx.scout_trajectory.append({"tool_name": "click"})
+ ctx.blocker_signal = CopilotToolBlockerSignal(
+ blocker_kind="authority_denied",
+ agent_steering_text="Use browser tools.",
+ user_facing_reason="I'll respond with the information I already have.",
+ recovery_hint="report_blocker_to_user",
+ internal_reason_code="turn_intent_no_mutation_run_blocked",
+ blocked_tool="update_and_run_blocks",
+ classifier_mode="unknown",
+ )
+
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
+ ctx,
+ {"type": "REPLY", "user_response": "I'll respond with the information I already have."},
+ )
+
+ assert response_type == "REPLY"
+ assert verdict.allowed
+ assert diagnostics.get("deferred_to_recycle", False) is False
+ assert diagnostics["actuation_obligation_status"] == ActuationObligationStatus.ALLOWED.value
+ assert diagnostics["cannot_act_reason"] is None
+ assert diagnostics["successful_mutating_browser_actions"] == 1
+
+
@pytest.mark.parametrize(
("reason", "expected_terms"),
[