Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-12957 Copilot: stop asking the user to confirm outputs the request contract already pinned (#7715)
Andrew Neilsonfdeb748b46c5
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2index 63feed6aa..7383d795e 100644--- a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2@@ -263,9 +263,13 @@ When `goal_reached` is `true`, you are asserting "the workflow I built and teste{"type": "ASK_QUESTION","user_response": "A clear, specific question to ask the user — aim for 1-2 sentences (~200 characters), but never drop the observed field/control or a required part of a multi-part question to hit that length; specificity wins over brevity.",+ "ask_subject": "One of: output_schema | credentials | target_url | disambiguation | other. The category of information the question is about.",+ "refs": ["output.number_of_website_visitors"],"global_llm_context": { ...same structured object as above... }}+`refs` are output-path identifiers copied verbatim from the `requested_output_paths` list in the request policy summary, when `ask_subject` is `output_schema`. Omit `refs` for other subjects.+IMPORTANT: Always preserve and extend global_llm_context across turns using the structured JSON format above. Each slot has a specific purpose:- user_goal: The user's high-level objective. Set this on the first turn, update if the goal evolves.- urls_visited: URLs you navigated to and what you found. Append new entries, do not repeat existing ones.diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex c893b8f0e..a9fc49e98 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -97,7 +97,9 @@ from skyvern.forge.sdk.copilot.context import (StructuredContext,TurnNarrativePayload,adopt_model_authored_context,+ coerce_ask_subject,finalize_discovery_counter_in_global_llm_context,+ parsed_ask_refs,record_approved_credentials_in_global_llm_context,render_loaded_result_context_for_prompt,sanitize_global_llm_context_for_prompt,@@ -3951,6 +3953,12 @@ async def _translate_to_agent_result(resp_type = action_data.get("type", "REPLY")if resp_type not in COPILOT_RESPONSE_TYPES:resp_type = "REPLY"+ if resp_type == "ASK_QUESTION":+ LOG.info(+ "copilot_ask_subject",+ subject=coerce_ask_subject(action_data.get("ask_subject")),+ refs=parsed_ask_refs(action_data.get("refs")),+ )normalized_scaffolding = normalize_response_scaffolding(resp_type, str(user_response))resp_type = normalized_scaffolding.response_typeuser_response = normalized_scaffolding.user_response or "Done."diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 6e611def9..390b752e4 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -30,6 +30,22 @@ ResponseType = Literal["REPLY", "ASK_QUESTION", "REPLACE_WORKFLOW"]COPILOT_RESPONSE_TYPES: tuple[ResponseType, ...] = get_args(ResponseType)ProposalDisposition = Literal["no_proposal", "auto_applicable", "review_untested", "review_tested"]+AskSubject = Literal["output_schema", "credentials", "target_url", "disambiguation", "other"]+COPILOT_ASK_SUBJECTS: tuple[AskSubject, ...] = get_args(AskSubject)+++def coerce_ask_subject(value: object) -> AskSubject | None:+ for subject in COPILOT_ASK_SUBJECTS:+ if value == subject:+ return subject+ return None+++def parsed_ask_refs(value: object) -> list[str]:+ if not isinstance(value, list):+ return []+ return [ref for ref in value if isinstance(ref, str) and ref]+class DeliveredUnverifiedPublicOutputs(dict[str, Any]):"""Run-output values explicitly selected for terminal presentation.diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex cf4080088..e4f5c3a6b 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -93,7 +93,12 @@ from skyvern.forge.sdk.copilot.config import (CopilotConfig,normalize_block_authoring_policy,)-from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext+from skyvern.forge.sdk.copilot.context import (+ AskSubject,+ CodeAuthoringRepairContext,+ coerce_ask_subject,+ parsed_ask_refs,+)from skyvern.forge.sdk.copilot.credential_pause import credential_pause_would_fire, maybe_credential_pausefrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (DiagnosisRepairContract,@@ -121,6 +126,7 @@ from skyvern.forge.sdk.copilot.request_policy import (REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS,CompletionCriterion,RequestPolicy,+ floor_rekeyed_requested_output_paths,request_policy_has_present_completion_contract,requested_output_path_for_field,schema_output_path_aliases_from_criteria,@@ -1151,7 +1157,7 @@ def _turn_intent_can_update_and_run_without_user_input(turn_intent: Any) -> boolreturn bool(turn_intent.authority.may_run_blocks)-def recycle_admits_present_completion_contract_ask(ctx: CopilotContext) -> bool:+def _present_completion_contract_ask_admission_base(ctx: CopilotContext) -> bool:request_policy = ctx.request_policyif not isinstance(request_policy, RequestPolicy):return False@@ -1161,17 +1167,35 @@ def recycle_admits_present_completion_contract_ask(ctx: CopilotContext) -> bool:return Falseif request_policy.clarification_reason not in (None, "none"):return False- if not _turn_intent_can_author_without_user_input(ctx.turn_intent):- return False- if ctx.has_genuine_workflow_attempt():+ return _turn_intent_can_author_without_user_input(ctx.turn_intent)+++def recycle_admits_present_completion_contract_ask(ctx: CopilotContext) -> bool:+ if not _present_completion_contract_ask_admission_base(ctx):return False- return True+ return not ctx.has_genuine_workflow_attempt()def _present_completion_contract_ask_retry(ctx: CopilotContext, parsed: dict[str, Any]) -> str | None:if parsed.get("type") != "ASK_QUESTION":return None- if not recycle_admits_present_completion_contract_ask(ctx):+ ask_subject = coerce_ask_subject(parsed.get("ask_subject"))+ if ask_subject is not None:+ # A schema ask the contract already answers is redundant whether or not the turn has+ # built anything yet, so it resolves on the base admission without the attempt check.+ if _present_completion_contract_ask_admission_base(ctx):+ auto_answer = _typed_ask_subject_auto_answer(ctx, ask_subject, parsed)+ if auto_answer is not None:+ return auto_answer+ retry_admitted = recycle_admits_present_completion_contract_ask(ctx)+ if ask_subject is not None:+ LOG.info(+ "copilot_ask_subject_passed_through",+ subject=ask_subject,+ outcome="build_first_retry" if retry_admitted else "reached_user",+ **ctx.genuine_attempt_parity_fields(),+ )+ if not retry_admitted:return NoneLOG.info("copilot.present_completion_contract_ask_retry",@@ -1182,6 +1206,35 @@ def _present_completion_contract_ask_retry(ctx: CopilotContext, parsed: dict[strreturn PRESENT_COMPLETION_CONTRACT_ASK_RETRY+def _typed_ask_subject_auto_answer(ctx: CopilotContext, ask_subject: AskSubject, parsed: dict[str, Any]) -> str | None:+ if ask_subject != "output_schema":+ return None+ refs = parsed_ask_refs(parsed.get("refs"))+ if not refs:+ return None+ # Reads the raw policy criteria rather than the turn-active set the other requested-output+ # consumers use: floor-rekey annotations are baked in at request-policy build time, and what+ # the model may cite as refs is rendered from this same set in `prompt_summary`.+ policy = ctx.request_policy+ if not isinstance(policy, RequestPolicy):+ return None+ criteria = policy.graded_completion_criteria()+ requested = requested_output_paths(criteria) | floor_rekeyed_requested_output_paths(criteria)+ if not requested or not set(refs) <= requested:+ return None+ resolved = sorted(set(refs))+ LOG.info(+ "copilot_ask_subject_auto_answered",+ subject=ask_subject,+ resolved_refs=resolved,+ )+ return (+ "The outputs you asked to confirm are already pinned by this request's completion contract. "+ f"Requested output paths: {', '.join(resolved)}. Author and test the workflow to produce them, "+ "choosing a reasonable representation for each, instead of asking the user to re-confirm."+ )++def _nudge(config: CopilotConfig | None, key: str) -> str:if config is None:return DEFAULT_ENFORCEMENT_NUDGES[key]@@ -2411,18 +2464,7 @@ def _pre_run_gated_completion_criteria(ctx: AgentContext) -> tuple[CompletionCridef _floor_rekeyed_requested_output_paths(ctx: AgentContext) -> set[str]:- """Identity of runtime-output criteria whose slot rekey cleared ``output_path``; the rekey- preserves it in ``floor_rekeyed_from_path``, and without it they vanish from the requested set."""- return {- criterion.floor_rekeyed_from_path- for criterion in _pre_run_gated_completion_criteria(ctx)- if criterion.requested_output_floor_rekeyed- and criterion.floor_rekeyed_from_path- and criterion.kind == "outcome"- and criterion.level != "definition"- and not criterion.method_mandated- and criterion.requested_output_evidence_source == "runtime_output"- }+ return floor_rekeyed_requested_output_paths(_pre_run_gated_completion_criteria(ctx))def pre_run_gated_outputs_without_path(ctx: AgentContext) -> tuple[CompletionCriterion, ...]:diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 05e891f6b..c3a87ede2 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -695,6 +695,18 @@ class RequestPolicy:lines.append(f"completion_contract: {self.completion_contract}")if self.raw_secret_detected:lines.append(f"raw_secret_detected: {self.raw_secret_detected}")+ graded_criteria = self.graded_completion_criteria()+ requested_output_path_literals = sorted(+ {+ criterion.output_path+ for criterion in graded_criteria+ if criterion.output_path and criterion.level != "definition"+ }+ | floor_rekeyed_requested_output_paths(graded_criteria)+ )+ if requested_output_path_literals:+ lines.append("requested_output_paths:")+ lines += [f"- {path}" for path in requested_output_path_literals]validation_classification_criteria = [criterionfor criterion in self.graded_completion_criteria()@@ -727,6 +739,23 @@ def request_policy_has_present_completion_contract(request_policy: RequestPolicyreturn request_policy.completion_contract_status == "present" or bool(request_policy.completion_criteria)+def floor_rekeyed_requested_output_paths(+ criteria: tuple[CompletionCriterion, ...] | list[CompletionCriterion],+) -> set[str]:+ """Identity of runtime-output criteria whose slot rekey cleared ``output_path``; the rekey+ preserves it in ``floor_rekeyed_from_path``, and without it they vanish from the requested set."""+ return {+ criterion.floor_rekeyed_from_path+ for criterion in criteria+ if criterion.requested_output_floor_rekeyed+ and criterion.floor_rekeyed_from_path+ and criterion.kind == "outcome"+ and criterion.level != "definition"+ and not criterion.method_mandated+ and criterion.requested_output_evidence_source == "runtime_output"+ }++def is_defer_authoring_durable_fill_criterion(criterion: CompletionCriterion) -> bool:return criterion.id == DEFER_AUTHORING_DURABLE_FILL_CRITERION_IDdiff --git a/tests/unit/test_copilot_pre_discovery_url_nudge.py b/tests/unit/test_copilot_pre_discovery_url_nudge.pyindex e0a5250fc..3889ef61e 100644--- a/tests/unit/test_copilot_pre_discovery_url_nudge.py+++ b/tests/unit/test_copilot_pre_discovery_url_nudge.py@@ -12,9 +12,12 @@ untouched.from __future__ import annotations+import json+from collections.abc import Callablefrom types import SimpleNamespaceimport pytest+from structlog.testing import capture_logsfrom skyvern.forge.sdk.copilot.build_phase import DISCOVERY_FAILURE_STREAK_ESCAPE_THRESHOLD, BuildPhasefrom skyvern.forge.sdk.copilot.context import CopilotContext@@ -22,6 +25,7 @@ from skyvern.forge.sdk.copilot.enforcement import (MAX_PRE_DISCOVERY_URL_QUESTION_NUDGES,PRE_DISCOVERY_URL_QUESTION_NUDGE,PRESENT_COMPLETION_CONTRACT_ASK_RETRY,+ _check_enforcement,_pre_discovery_url_question_nudge,_response_coverage_nudge,)@@ -289,6 +293,148 @@ def test_present_completion_contract_ask_suppressed_by_genuine_attempt(marker: dassert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) is None+_REQUESTED_OUTPUT_PATHS = ("output.number_of_new_signups", "output.number_of_website_visitors")+++def _output_path_contract_policy(**overrides: object) -> RequestPolicy:+ defaults = dict(+ user_response_policy="proceed",+ clarification_reason="none",+ completion_contract_status="present",+ completion_criteria=[+ CompletionCriterion(+ id="visitors", outcome="The website visitor count is returned.", output_path=_REQUESTED_OUTPUT_PATHS[1]+ ),+ CompletionCriterion(+ id="signups", outcome="The new signup count is returned.", output_path=_REQUESTED_OUTPUT_PATHS[0]+ ),+ ],+ )+ defaults.update(overrides)+ return RequestPolicy(**defaults)+++def _output_schema_ask(refs: list[str]) -> dict[str, object]:+ return {+ "type": "ASK_QUESTION",+ "user_response": "Should the output be website visitors and new signups as integers?",+ "ask_subject": "output_schema",+ "refs": refs,+ }+++def test_output_schema_ask_with_covering_refs_auto_answers() -> None:+ ctx = _present_contract_ctx(request_policy=_output_path_contract_policy())++ with capture_logs() as logs:+ nudge = _response_coverage_nudge(ctx, _output_schema_ask(list(_REQUESTED_OUTPUT_PATHS)))++ assert nudge is not None+ assert nudge != PRESENT_COMPLETION_CONTRACT_ASK_RETRY+ for path in _REQUESTED_OUTPUT_PATHS:+ assert path in nudge+ events = [entry for entry in logs if entry["event"] == "copilot_ask_subject_auto_answered"]+ assert len(events) == 1+ assert events[0]["subject"] == "output_schema"+ assert set(events[0]["resolved_refs"]) == set(_REQUESTED_OUTPUT_PATHS)+++ "refs",+ [+ pytest.param([], id="empty"),+ pytest.param(["output.number_of_website_visitors", "output.unknown_field"], id="partial"),+ pytest.param(["output.unknown_field"], id="unresolvable"),+ ],+)+def test_output_schema_ask_without_full_coverage_passes_through(refs: list[str]) -> None:+ ctx = _mid_build_ctx(_output_path_contract_policy())++ with capture_logs() as logs:+ nudge = _response_coverage_nudge(ctx, _output_schema_ask(refs))++ assert nudge is None+ assert not [entry for entry in logs if entry["event"] == "copilot_ask_subject_auto_answered"]+++_CREDENTIALS_ASK = {+ "type": "ASK_QUESTION",+ "user_response": "Which saved credential should I use?",+ "ask_subject": "credentials",+}+++def test_non_output_schema_subject_ask_passes_through() -> None:+ ctx = _mid_build_ctx(_output_path_contract_policy())++ assert _response_coverage_nudge(ctx, _CREDENTIALS_ASK) is None+++ "ask",+ [+ pytest.param(_CREDENTIALS_ASK, id="credentials"),+ pytest.param({**_CREDENTIALS_ASK, "ask_subject": "other"}, id="other"),+ pytest.param(_output_schema_ask(["output.unknown_field"]), id="output_schema_unresolvable"),+ ],+)+def test_typed_subject_ask_before_a_genuine_attempt_keeps_the_legacy_retry(ask: dict[str, object]) -> None:+ """A typed subject the contract cannot answer must not buy the ask an early exit past the+ build-first retry; only a resolved auto-answer skips it."""+ ctx = _present_contract_ctx(request_policy=_output_path_contract_policy())++ assert _response_coverage_nudge(ctx, ask) == PRESENT_COMPLETION_CONTRACT_ASK_RETRY+++ ("ctx_factory", "expected_outcome"),+ [+ pytest.param(+ lambda: _present_contract_ctx(request_policy=_output_path_contract_policy()),+ "build_first_retry",+ id="retried",+ ),+ pytest.param(+ lambda: _mid_build_ctx(_output_path_contract_policy()),+ "reached_user",+ id="reached_user",+ ),+ ],+)+def test_unresolved_typed_ask_logs_which_outcome_it_got(ctx_factory: Callable[[], _Ctx], expected_outcome: str) -> None:+ with capture_logs() as logs:+ _response_coverage_nudge(ctx_factory(), _CREDENTIALS_ASK)++ events = [entry for entry in logs if entry["event"] == "copilot_ask_subject_passed_through"]+ assert len(events) == 1+ assert events[0]["outcome"] == expected_outcome+++def test_absent_subject_present_contract_ask_keeps_legacy_retry() -> None:+ ctx = _present_contract_ctx(request_policy=_output_path_contract_policy())++ assert _response_coverage_nudge(ctx, _OUTPUT_CONFIRMATION_ASK) == PRESENT_COMPLETION_CONTRACT_ASK_RETRY+++def test_definition_level_output_path_does_not_count_as_coverage() -> None:+ policy = _output_path_contract_policy(+ completion_criteria=[+ CompletionCriterion(+ id="def_visitors",+ outcome="The website visitor count is defined.",+ output_path="output.number_of_website_visitors",+ level="definition",+ ),+ CompletionCriterion(+ id="signups", outcome="The new signup count is returned.", output_path="output.number_of_new_signups"+ ),+ ],+ )+ ctx = _mid_build_ctx(policy)++ assert _response_coverage_nudge(ctx, _output_schema_ask(["output.number_of_website_visitors"])) is None++def test_present_completion_contract_ask_admits_after_scout_only_marker() -> None:ctx = _present_contract_ctx(test_after_update_done=True)@@ -320,3 +466,151 @@ def test_recycle_admission_is_superset_of_backstop_block(marker: dict[str, objecassert recycle_admitselse:assert not recycle_admits+++def _rekeyed_criterion(criterion_id: str, outcome: str, path: str, **overrides: object) -> CompletionCriterion:+ defaults = dict(+ id=criterion_id,+ outcome=outcome,+ output_path=None,+ requested_output_floor_rekeyed=True,+ floor_rekeyed_from_path=path,+ kind="outcome",+ level="run",+ pinability="shapeless_valid",+ requested_output_evidence_source="runtime_output",+ )+ defaults.update(overrides)+ return CompletionCriterion(**defaults)+++def _rekeyed_contract_policy(**overrides: object) -> RequestPolicy:+ return _output_path_contract_policy(+ completion_criteria=[+ _rekeyed_criterion("visitors", "The website visitor count is returned.", _REQUESTED_OUTPUT_PATHS[1]),+ _rekeyed_criterion("signups", "The new signup count is returned.", _REQUESTED_OUTPUT_PATHS[0]),+ ],+ **overrides,+ )+++def _mid_build_ctx(policy: RequestPolicy) -> _Ctx:+ return _present_contract_ctx(request_policy=policy, update_workflow_called=True)+++def test_output_schema_ask_auto_answers_after_genuine_attempt() -> None:+ ctx = _mid_build_ctx(_output_path_contract_policy())++ nudge = _response_coverage_nudge(ctx, _output_schema_ask(list(_REQUESTED_OUTPUT_PATHS)))++ assert nudge is not None+ assert nudge != PRESENT_COMPLETION_CONTRACT_ASK_RETRY+ for path in _REQUESTED_OUTPUT_PATHS:+ assert path in nudge+++def test_floor_rekeyed_contract_covers_output_schema_ask() -> None:+ ctx = _mid_build_ctx(_rekeyed_contract_policy())++ with capture_logs() as logs:+ nudge = _response_coverage_nudge(ctx, _output_schema_ask(list(_REQUESTED_OUTPUT_PATHS)))++ assert nudge is not None+ events = [entry for entry in logs if entry["event"] == "copilot_ask_subject_auto_answered"]+ assert len(events) == 1+ assert set(events[0]["resolved_refs"]) == set(_REQUESTED_OUTPUT_PATHS)+++def test_floor_rekeyed_paths_are_quotable_from_the_prompt_summary() -> None:+ summary = _rekeyed_contract_policy().prompt_summary()++ assert "requested_output_paths:" in summary+ for path in _REQUESTED_OUTPUT_PATHS:+ assert f"- {path}" in summary+++def test_credentials_ask_after_genuine_attempt_passes_through() -> None:+ ctx = _mid_build_ctx(_rekeyed_contract_policy())+ ask = {+ "type": "ASK_QUESTION",+ "user_response": "Which saved credential should I use?",+ "ask_subject": "credentials",+ }++ assert _response_coverage_nudge(ctx, ask) is None+++def test_uncovered_output_schema_ask_after_genuine_attempt_passes_through() -> None:+ ctx = _mid_build_ctx(_rekeyed_contract_policy())++ with capture_logs() as logs:+ nudge = _response_coverage_nudge(ctx, _output_schema_ask(["output.unknown_field"]))++ assert nudge is None+ assert not [entry for entry in logs if entry["event"] == "copilot_ask_subject_auto_answered"]+++def test_definition_level_rekeyed_path_does_not_count_as_coverage() -> None:+ policy = _output_path_contract_policy(+ completion_criteria=[+ _rekeyed_criterion(+ "visitors",+ "The website visitor count is defined.",+ _REQUESTED_OUTPUT_PATHS[1],+ level="definition",+ ),+ ],+ )+ ctx = _mid_build_ctx(policy)++ assert _response_coverage_nudge(ctx, _output_schema_ask([_REQUESTED_OUTPUT_PATHS[1]])) is None+++def test_clarification_reason_keeps_output_schema_ask_with_the_user() -> None:+ ctx = _mid_build_ctx(_rekeyed_contract_policy(clarification_reason="credentials"))++ assert _response_coverage_nudge(ctx, _output_schema_ask(list(_REQUESTED_OUTPUT_PATHS))) is None+++def _enforcement_ctx(policy: RequestPolicy) -> CopilotContext:+ ctx = CopilotContext(+ organization_id="o_test",+ workflow_id="w_test",+ workflow_permanent_id="wpid_test",+ workflow_yaml="",+ browser_session_id=None,+ stream=None,+ )+ ctx.build_phase = BuildPhase.COMPOSING+ ctx.request_policy = policy+ ctx.turn_intent = _authoring_intent()+ ctx.update_workflow_called = True+ ctx.test_after_update_done = True+ ctx.discovery_calls_this_turn = 1+ return ctx+++def test_enforcement_auto_answers_rekeyed_output_schema_ask_mid_build() -> None:+ ctx = _enforcement_ctx(_rekeyed_contract_policy())+ result = SimpleNamespace(final_output=json.dumps(_output_schema_ask(list(_REQUESTED_OUTPUT_PATHS))))++ nudge = _check_enforcement(ctx, result)++ assert nudge is not None+ for path in _REQUESTED_OUTPUT_PATHS:+ assert path in nudge+++def test_definition_level_rekeyed_path_stays_out_of_the_prompt_summary() -> None:+ policy = _output_path_contract_policy(+ completion_criteria=[+ _rekeyed_criterion(+ "visitors",+ "The website visitor count is defined.",+ _REQUESTED_OUTPUT_PATHS[1],+ level="definition",+ ),+ ],+ )++ assert "requested_output_paths:" not in policy.prompt_summary()