Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10335: copilot BuildPhase machine + AgentResult writeback factory (#6177)
Andrew Neilson8915cc049bf8
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 900a2feec..a92ef888b 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -32,7 +32,13 @@ from pydantic import ValidationErrorfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.copilot.config import CopilotConfig-from skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES, AgentResult, CopilotContext, StructuredContext+from skyvern.forge.sdk.copilot.context import (+ COPILOT_RESPONSE_TYPES,+ AgentResult,+ CopilotContext,+ StructuredContext,+ finalize_discovery_counter_in_global_llm_context,+)from skyvern.forge.sdk.copilot.output_policy import (UNVALIDATED_DISCLOSURE_PHRASES,WORKFLOW_PRESENT_SENTINEL,@@ -672,6 +678,28 @@ def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:return None, None+def _make_agent_result(+ ctx: CopilotContext | None,+ *,+ global_llm_context: str | None = None,+ turn_outcome: TurnOutcome | None = None,+ **kwargs: Any,+) -> AgentResult:+ """Sole ``AgentResult`` constructor in this module.++ Routes every ``AgentResult`` through the discovery-counter finalizer so+ the per-chat budget survives every exit path (timeout, cancel, max-turns,+ output-policy block, clarification helpers, non-retriable nav error,+ normal translate-result, missing-SDK fallback, unexpected-error fallback).+ """+ final_context = (+ finalize_discovery_counter_in_global_llm_context(ctx, global_llm_context)+ if ctx is not None+ else global_llm_context+ )+ return AgentResult(global_llm_context=final_context, turn_outcome=turn_outcome, **kwargs)++def _build_exit_result(ctx: CopilotContext,user_response: str,@@ -687,7 +715,8 @@ def _build_exit_result(blocked_signatures=ctx.blocked_reply_signatures,terminal_reason=terminal_reason or ("cancel" if cancelled else None),)- return AgentResult(+ return _make_agent_result(+ ctx,user_response=final_text,updated_workflow=verified_workflow,global_llm_context=global_llm_context,@@ -979,7 +1008,8 @@ def _build_wip_exit_result():reply = _last_good_failure_reply(ctx, tested_reply) if recorded_failure_reply else tested_replyfinal_text, outcome = _guard(reply)- return AgentResult(+ return _make_agent_result(+ ctx,user_response=final_text,updated_workflow=ctx.last_good_workflow,global_llm_context=global_llm_context,@@ -1002,7 +1032,8 @@ def _build_wip_exit_result(else:reply = unvalidated_reply if unvalidated else tested_replyfinal_text, outcome = _guard(reply)- return AgentResult(+ return _make_agent_result(+ ctx,user_response=final_text,updated_workflow=ctx.last_workflow,global_llm_context=global_llm_context,@@ -1391,7 +1422,8 @@ def _translate_to_agent_result(tool_calls=[name for name in tool_call_names if name],)- return AgentResult(+ return _make_agent_result(+ ctx,user_response=final_user_response,updated_workflow=last_workflow,global_llm_context=enriched_context or None,@@ -1440,7 +1472,8 @@ def _build_feasibility_clarification_result(blocked_signatures=list(ctx.blocked_reply_signatures),reason_code="feasibility_clarification",)- return AgentResult(+ return _make_agent_result(+ ctx,user_response=final_text,updated_workflow=None,global_llm_context=enriched_context,@@ -1516,7 +1549,8 @@ def _build_request_policy_clarification_result(blocked_signatures=list(ctx.blocked_reply_signatures),reason_code="request_policy_clarification",)- return AgentResult(+ return _make_agent_result(+ ctx,user_response=final_text,updated_workflow=None,global_llm_context=structured.to_json_str(),@@ -1811,7 +1845,8 @@ def _build_output_policy_blocked_result(reason_code="output_policy_block",terminal_reason="output_policy_block",)- return AgentResult(+ return _make_agent_result(+ ctx,user_response=final_user_response,updated_workflow=preserved_workflow,global_llm_context=structured.to_json_str(),@@ -1970,7 +2005,8 @@ async def _run_copilot_turn_impl(blocked_signatures=(),terminal_reason="missing_sdk",)- return AgentResult(+ return _make_agent_result(+ None,user_response=final_missing_text,updated_workflow=None,global_llm_context=global_llm_context,@@ -2045,6 +2081,12 @@ async def _run_copilot_turn_impl(previous_user_message=previous_user_message,)+ # Hydrate the per-chat discovery counter from the inbound global_llm_context.+ # `ctx.build_phase` stays at the safe default and `initial_build_phase` is+ # not yet called from the agent loop — that activates in the discovery PR.+ prior_structured_context = StructuredContext.from_json_str(global_llm_context)+ ctx.prior_discovery_calls_made = prior_structured_context.discovery_calls_made+# Preflight feasibility classifier — fires on every turn so mid-session pivots# to impossible targets are caught the same as first-turn structural mismatches.from skyvern.forge.sdk.copilot.feasibility_gate import run_feasibility_gate@@ -2294,7 +2336,8 @@ async def _run_copilot_turn_impl(blocked_signatures=ctx.blocked_reply_signatures,terminal_reason="non_retriable_nav",)- return AgentResult(+ return _make_agent_result(+ ctx,user_response=final_nav_text,updated_workflow=None,global_llm_context=global_llm_context,diff --git a/skyvern/forge/sdk/copilot/build_phase.py b/skyvern/forge/sdk/copilot/build_phase.pynew file mode 100644index 000000000..4c041c49d--- /dev/null+++ b/skyvern/forge/sdk/copilot/build_phase.py@@ -0,0 +1,220 @@+"""Build-phase machinery for the copilot agent loop.++The phase machine separates build-time discovery from workflow composition.+TurnIntent.mode stays BUILD; phase is loop runtime state on ctx. The+deterministic orchestrator owns transitions — an agent-emitted phase signal+does not by itself unlock mutation authority.++Phases:+- INITIAL: BUILD turn with no known entrypoint URL. Discovery is available;+ mutation and direct browser primitives are gated off.+- DISCOVERING: discover_workflow_entrypoint is running. Same gate as INITIAL.+- COMPOSING: entrypoint resolved. Mutation tools and direct browser primitives+ are available; discovery is no longer available.+- TESTING: post-update; same authority as COMPOSING.+"""++from __future__ import annotations++import re+import time+from enum import StrEnum+from typing import TYPE_CHECKING, Any++import structlog+import yaml++from skyvern.utils.yaml_loader import safe_load_no_dates++if TYPE_CHECKING:+ from skyvern.forge.sdk.copilot.context import CopilotContext+ from skyvern.forge.sdk.copilot.turn_intent import TurnIntent++LOG = structlog.get_logger()++# Mode values that explicitly rule out discovery. Other modes (BUILD,+# DRAFT_ONLY, UNKNOWN) may enter INITIAL phase when no URL signal exists —+# the TurnIntent keyword classifier gates NEW_BROWSER_TASK_TERMS on+# `not has_workflow`, so a fresh build turn with an empty-blocks YAML can+# legitimately classify as UNKNOWN even though discovery is the right next+# step. The phase gate keeps mutation blocked until discovery returns a+# candidate (or the model ASK_QUESTIONs for a URL).+_PHASE_NON_BUILD_MODE_VALUES: frozenset[str] = frozenset({"edit", "diagnose", "docs_answer", "clarify", "refuse"})+++class BuildPhase(StrEnum):+ INITIAL = "initial"+ DISCOVERING = "discovering"+ COMPOSING = "composing"+ TESTING = "testing"+++DISCOVERY_PERMITTED_PHASES: frozenset[BuildPhase] = frozenset({BuildPhase.INITIAL, BuildPhase.DISCOVERING})+MUTATION_PERMITTED_PHASES: frozenset[BuildPhase] = frozenset({BuildPhase.COMPOSING, BuildPhase.TESTING})++# Tools whose call paths must be gated by phase. Names match the+# function_tool name_override / MCP overlay names registered elsewhere.+_BROWSER_PRIMITIVE_TOOLS: frozenset[str] = frozenset(+ {+ "navigate_browser",+ "evaluate",+ "click",+ "type_text",+ "scroll",+ "select_option",+ "press_key",+ "console_messages",+ "get_browser_screenshot",+ }+)+_MUTATION_TOOLS: frozenset[str] = frozenset(+ {"update_workflow", "update_and_run_blocks", "run_blocks_and_collect_debug"}+)+_DISCOVERY_TOOLS: frozenset[str] = frozenset({"discover_workflow_entrypoint"})++_URL_IN_TEXT_RE = re.compile(r"https?://[^\s<>\"\)]+", re.IGNORECASE)+++def _yaml_has_target_url(workflow_yaml: str | None) -> bool:+ """True when the YAML carries a goto_url or navigation block with a non-empty url."""+ if not workflow_yaml:+ return False+ try:+ parsed = safe_load_no_dates(workflow_yaml)+ except yaml.YAMLError:+ return False+ if not isinstance(parsed, dict):+ return False+ definition = parsed.get("workflow_definition")+ if not isinstance(definition, dict):+ return False+ blocks = definition.get("blocks")+ if not isinstance(blocks, list):+ return False+ for block in blocks:+ if not isinstance(block, dict):+ continue+ block_type = block.get("block_type")+ if block_type not in {"goto_url", "navigation"}:+ continue+ url = block.get("url")+ if isinstance(url, str) and url.strip():+ return True+ return False+++def initial_build_phase(+ turn_intent: TurnIntent | None,+ user_message: str,+ agent_user_message: str,+ workflow_yaml: str | None,+) -> BuildPhase:+ """Decide the initial build phase for this turn.++ Returns COMPOSING (sentinel — harmless because the existing TurnIntent gate+ already blocks mutation for non-BUILD modes) when the mode is one of the+ explicitly-non-build values. For BUILD / DRAFT_ONLY / UNKNOWN, returns+ COMPOSING when this turn already has a URL signal, else INITIAL.++ UNKNOWN deliberately enters INITIAL when no URL is present: a fresh chat+ whose latest message dodges every keyword heuristic (e.g. "go to X" with+ an empty-blocks scaffold workflow that flips `has_workflow=True` and+ suppresses NEW_BROWSER_TASK_TERMS) should still get discovery — the+ alternative is to let mutation through on an ambiguous turn, which is+ worse than asking discovery to resolve or falling through to ASK_QUESTION.++ Signals considered (this-turn only — never prior visited URLs or chat+ history, which can leak stale URLs across unrelated turns):+ - The raw latest user message.+ - The rewritten agent input (request-policy may carry the prior request).+ - The current workflow YAML's first goto_url/navigation block.+ """+ mode_value = getattr(getattr(turn_intent, "mode", None), "value", None)+ if mode_value in _PHASE_NON_BUILD_MODE_VALUES:+ return BuildPhase.COMPOSING+ if _URL_IN_TEXT_RE.search(user_message or ""):+ return BuildPhase.COMPOSING+ if _URL_IN_TEXT_RE.search(agent_user_message or ""):+ return BuildPhase.COMPOSING+ if _yaml_has_target_url(workflow_yaml):+ return BuildPhase.COMPOSING+ return BuildPhase.INITIAL+++def _log_transition(ctx: CopilotContext, *, prev: BuildPhase, new: BuildPhase, reason: str) -> None:+ LOG.info(+ "copilot.build_phase_transition",+ prev_phase=prev.value,+ new_phase=new.value,+ transition_reason=reason,+ workflow_permanent_id=getattr(ctx, "workflow_permanent_id", None),+ )+++def advance_to_discovering(ctx: CopilotContext) -> None:+ if ctx.build_phase != BuildPhase.INITIAL:+ raise ValueError(f"advance_to_discovering called from {ctx.build_phase.value}; expected INITIAL")+ if ctx.discovery_started_monotonic is None:+ ctx.discovery_started_monotonic = time.monotonic()+ prev = ctx.build_phase+ ctx.build_phase = BuildPhase.DISCOVERING+ _log_transition(ctx, prev=prev, new=ctx.build_phase, reason="discovery_started")+++def advance_to_composing(ctx: CopilotContext, *, reason: str) -> None:+ if ctx.build_phase not in (BuildPhase.INITIAL, BuildPhase.DISCOVERING):+ raise ValueError(f"advance_to_composing called from {ctx.build_phase.value}; expected INITIAL or DISCOVERING")+ prev = ctx.build_phase+ ctx.build_phase = BuildPhase.COMPOSING+ _log_transition(ctx, prev=prev, new=ctx.build_phase, reason=reason)+++def advance_to_testing(ctx: CopilotContext) -> None:+ if ctx.build_phase == BuildPhase.TESTING:+ # Already in TESTING (or post-TESTING composer follow-up) is benign.+ return+ if ctx.build_phase != BuildPhase.COMPOSING:+ raise ValueError(f"advance_to_testing called from {ctx.build_phase.value}; expected COMPOSING or TESTING")+ prev = ctx.build_phase+ ctx.build_phase = BuildPhase.TESTING+ _log_transition(ctx, prev=prev, new=ctx.build_phase, reason="update_workflow_succeeded")+++def _phase_tool_error(ctx: Any, tool_name: str) -> str | None:+ """Shared phase-aware authority error. Returns the steering error string+ when the (phase, tool) pair is forbidden; None otherwise.++ Called from both the native gate (`_authority_tool_error` in tools.py)+ and the MCP-adapter gate (`SkyvernOverlayMCPServer.call_tool` pre-pre-hook).+ Errors flow back to the LLM as next-step steering, per the ai-agents+ 'errors as steering' guidance.+ """+ phase = getattr(ctx, "build_phase", None)+ if not isinstance(phase, BuildPhase):+ return None++ in_discovery = phase in DISCOVERY_PERMITTED_PHASES+ in_mutation = phase in MUTATION_PERMITTED_PHASES++ if tool_name in _DISCOVERY_TOOLS and in_mutation:+ return (+ "discover_workflow_entrypoint is only available before composition. "+ "The workflow already has a target URL — proceed with update_workflow or update_and_run_blocks. "+ "safe_reason_code=build_phase_discovery_disallowed_post_compose."+ )++ if tool_name in _BROWSER_PRIMITIVE_TOOLS and in_discovery:+ return (+ "Direct browser tools are not callable before composition. "+ "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL. "+ "safe_reason_code=build_phase_browser_blocked_pre_compose."+ )++ if tool_name in _MUTATION_TOOLS and in_discovery:+ return (+ "Workflow mutation is gated to composition. "+ "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL first. "+ "safe_reason_code=build_phase_mutation_blocked_pre_compose."+ )++ return Nonediff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex e80879b7a..2bf689b4f 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Literal, get_argsfrom pydantic import BaseModel, Field+from skyvern.forge.sdk.copilot.build_phase import BuildPhasefrom skyvern.forge.sdk.copilot.runtime import AgentContextfrom skyvern.forge.sdk.workflow.models.workflow import Workflow@@ -48,6 +49,10 @@ class StructuredContext(BaseModel):credentials_checked: list[CredentialCheck] = Field(default_factory=list)decisions_made: list[str] = Field(default_factory=list)workflow_state: str = ""+ # Per-chat discovery budget. Survives turn boundaries via+ # AgentResult.global_llm_context — finalized deterministically at every+ # AgentResult exit by `finalize_discovery_counter_in_global_llm_context`.+ discovery_calls_made: int = 0def to_json_str(self) -> str:return self.model_dump_json(indent=2)@@ -118,6 +123,27 @@ class StructuredContext(BaseModel):self.credentials_checked = self.credentials_checked[-40:]+def finalize_discovery_counter_in_global_llm_context(ctx: Any, raw_context: str | None) -> str | None:+ """Fold the per-chat discovery counter into the outgoing global_llm_context.++ Called from agent.py's `_make_agent_result` factory so every AgentResult+ exit path — timeout, cancel, max-turns, output-policy block, request-+ policy clarification, feasibility clarification, non-retriable nav error,+ normal translate-result, missing-SDK fallback, unexpected-error fallback —+ carries the updated count.++ Returns None when there is nothing to record and no prior context, so the+ pre-existing 'no global_llm_context' behaviour is preserved.+ """+ prior = int(getattr(ctx, "prior_discovery_calls_made", 0) or 0)+ this_turn = int(getattr(ctx, "discovery_calls_this_turn", 0) or 0)+ if not raw_context and this_turn == 0:+ return None+ sc = StructuredContext.from_json_str(raw_context)+ sc.discovery_calls_made = prior + this_turn+ return sc.to_json_str()++@dataclassclass AgentResult:user_response: str@@ -290,3 +316,16 @@ class CopilotContext(AgentContext):# retries. Declared here (rather than attached dynamically) so future# refactors can't strip it silently.narrator_state: NarratorState | None = None++ # Default COMPOSING is the safe non-BUILD value; the orchestrator+ # overrides at turn start via `initial_build_phase`.+ build_phase: BuildPhase = BuildPhase.COMPOSING+ # Hydrated from inbound StructuredContext.discovery_calls_made at turn start.+ prior_discovery_calls_made: int = 0+ discovery_calls_this_turn: int = 0+ discovery_step_count: int = 0+ discovery_started_monotonic: float | None = None+ discovery_evidence_trail: list[dict[str, Any]] = field(default_factory=list)+ # Set in `_run_attempt` after SkyvernOverlayMCPServer is constructed.+ # The discovery tool reaches the connected FastMCP client through this.+ discovery_mcp_server: Any | None = Nonediff --git a/tests/unit/test_copilot_build_phase.py b/tests/unit/test_copilot_build_phase.pynew file mode 100644index 000000000..8a39fc3c6--- /dev/null+++ b/tests/unit/test_copilot_build_phase.py@@ -0,0 +1,255 @@+"""Unit tests for the copilot ``BuildPhase`` machinery."""++from __future__ import annotations++import pytest++from skyvern.forge.sdk.copilot.build_phase import (+ DISCOVERY_PERMITTED_PHASES,+ MUTATION_PERMITTED_PHASES,+ BuildPhase,+ _phase_tool_error,+ _yaml_has_target_url,+ advance_to_composing,+ advance_to_discovering,+ advance_to_testing,+ initial_build_phase,+)+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentMode+++class _Ctx:+ """Lightweight ctx stand-in — the helpers only read/write a small surface."""++ def __init__(self, phase: BuildPhase = BuildPhase.COMPOSING) -> None:+ self.build_phase = phase+ self.discovery_started_monotonic: float | None = None+ self.workflow_permanent_id = "wpid_test"+++def _ti(mode: TurnIntentMode) -> TurnIntent:+ return TurnIntent(mode=mode)+++# ---------------- initial_build_phase ----------------+++ "mode,user_message,agent_message,workflow_yaml,expected",+ [+ # Explicitly-non-build modes return the COMPOSING sentinel.+ (TurnIntentMode.EDIT, "no url", "no url", "", BuildPhase.COMPOSING),+ (TurnIntentMode.DOCS_ANSWER, "what is a workflow?", "what is a workflow?", "", BuildPhase.COMPOSING),+ (TurnIntentMode.DIAGNOSE, "the run failed", "the run failed", "", BuildPhase.COMPOSING),+ (TurnIntentMode.CLARIFY, "anything", "anything", "", BuildPhase.COMPOSING),+ (TurnIntentMode.REFUSE, "anything", "anything", "", BuildPhase.COMPOSING),+ # BUILD with URL in latest user_message -> COMPOSING.+ (TurnIntentMode.BUILD, "go to https://example.com/login", "", "", BuildPhase.COMPOSING),+ # BUILD with URL only in rewritten agent message (request-policy continuation) -> COMPOSING.+ (TurnIntentMode.BUILD, "and download it", "earlier: go to https://example.com/file", "", BuildPhase.COMPOSING),+ # BUILD with no URL anywhere -> INITIAL.+ (TurnIntentMode.BUILD, "go to example", "go to example", "", BuildPhase.INITIAL),+ # DRAFT_ONLY behaves like BUILD for phase init.+ (+ TurnIntentMode.DRAFT_ONLY,+ "draft a workflow for example.com",+ "draft a workflow for example.com",+ "",+ BuildPhase.INITIAL,+ ),+ # DRAFT_ONLY with URL -> COMPOSING.+ (TurnIntentMode.DRAFT_ONLY, "draft a workflow for https://example.com", "", "", BuildPhase.COMPOSING),+ # UNKNOWN with no URL also enters INITIAL — a fresh "go to X" turn+ # with an empty-blocks scaffold workflow flips `has_workflow=True`+ # in the keyword classifier and suppresses NEW_BROWSER_TASK_TERMS,+ # producing UNKNOWN. Discovery is still the right next step there.+ (TurnIntentMode.UNKNOWN, "go to example and find sortable tables", "", "", BuildPhase.INITIAL),+ # UNKNOWN with a URL in the message -> COMPOSING (URL signal wins).+ (TurnIntentMode.UNKNOWN, "go to https://example.com/x", "", "", BuildPhase.COMPOSING),+ ],+)+def test_initial_build_phase_returns_expected(+ mode: TurnIntentMode,+ user_message: str,+ agent_message: str,+ workflow_yaml: str,+ expected: BuildPhase,+) -> None:+ assert initial_build_phase(_ti(mode), user_message, agent_message, workflow_yaml) == expected+++def test_initial_build_phase_yaml_goto_url_unlocks_composing() -> None:+ yaml_text = """+title: T+workflow_definition:+ parameters: []+ blocks:+ - block_type: goto_url+ label: open+ url: https://example.com/page+"""+ assert (+ initial_build_phase(_ti(TurnIntentMode.BUILD), "do that thing", "do that thing", yaml_text)+ == BuildPhase.COMPOSING+ )+++def test_initial_build_phase_yaml_with_empty_url_does_not_unlock() -> None:+ yaml_text = """+title: T+workflow_definition:+ parameters: []+ blocks:+ - block_type: goto_url+ label: open+ url: ""+"""+ assert initial_build_phase(_ti(TurnIntentMode.BUILD), "do thing", "do thing", yaml_text) == BuildPhase.INITIAL+++def test_initial_build_phase_none_turn_intent_acts_like_unknown_mode() -> None:+ # turn_intent=None is treated as if mode is UNKNOWN — eligible for INITIAL+ # when no URL signal exists. The phase gate then blocks mutation until+ # discovery resolves an entrypoint or the model ASK_QUESTIONs.+ assert initial_build_phase(None, "go to example", "go to example", "") == BuildPhase.INITIAL+ # URL present -> COMPOSING regardless of None turn_intent.+ assert initial_build_phase(None, "go to https://example.com", "", "") == BuildPhase.COMPOSING+++def test_initial_build_phase_yaml_malformed_falls_back_to_no_url() -> None:+ assert initial_build_phase(_ti(TurnIntentMode.BUILD), "open it", "open it", "[ unbalanced") == BuildPhase.INITIAL+++# ---------------- _yaml_has_target_url ----------------+++def test_yaml_has_target_url_detects_navigation_block_url() -> None:+ yaml_text = """+title: T+workflow_definition:+ parameters: []+ blocks:+ - block_type: navigation+ label: visit+ url: https://example.com/x+ navigation_goal: visit it+"""+ assert _yaml_has_target_url(yaml_text) is True+++def test_yaml_has_target_url_ignores_non_navigation_blocks() -> None:+ yaml_text = """+title: T+workflow_definition:+ parameters: []+ blocks:+ - block_type: code+ label: c+ code: "x = 1"+"""+ assert _yaml_has_target_url(yaml_text) is False+++def test_yaml_has_target_url_handles_empty_and_none() -> None:+ assert _yaml_has_target_url(None) is False+ assert _yaml_has_target_url("") is False+++# ---------------- transition helpers ----------------+++def test_advance_to_discovering_only_from_initial() -> None:+ ctx = _Ctx(BuildPhase.INITIAL)+ advance_to_discovering(ctx)+ assert ctx.build_phase == BuildPhase.DISCOVERING+ assert ctx.discovery_started_monotonic is not None+++def test_advance_to_discovering_rejects_other_phases() -> None:+ for phase in (BuildPhase.DISCOVERING, BuildPhase.COMPOSING, BuildPhase.TESTING):+ ctx = _Ctx(phase)+ with pytest.raises(ValueError):+ advance_to_discovering(ctx)+++def test_advance_to_composing_accepts_initial_and_discovering() -> None:+ for phase in (BuildPhase.INITIAL, BuildPhase.DISCOVERING):+ ctx = _Ctx(phase)+ advance_to_composing(ctx, reason="test")+ assert ctx.build_phase == BuildPhase.COMPOSING+++def test_advance_to_composing_rejects_from_composing_or_testing() -> None:+ for phase in (BuildPhase.COMPOSING, BuildPhase.TESTING):+ ctx = _Ctx(phase)+ with pytest.raises(ValueError):+ advance_to_composing(ctx, reason="test")+++def test_advance_to_testing_only_from_composing() -> None:+ ctx = _Ctx(BuildPhase.COMPOSING)+ advance_to_testing(ctx)+ assert ctx.build_phase == BuildPhase.TESTING+++def test_advance_to_testing_is_noop_from_testing() -> None:+ ctx = _Ctx(BuildPhase.TESTING)+ advance_to_testing(ctx)+ assert ctx.build_phase == BuildPhase.TESTING+++def test_advance_to_testing_rejects_pre_composition_phases() -> None:+ for phase in (BuildPhase.INITIAL, BuildPhase.DISCOVERING):+ ctx = _Ctx(phase)+ with pytest.raises(ValueError):+ advance_to_testing(ctx)+++def test_phase_sets_are_disjoint() -> None:+ assert DISCOVERY_PERMITTED_PHASES.isdisjoint(MUTATION_PERMITTED_PHASES)+++# ---------------- _phase_tool_error ----------------+++ "tool_name,phase,should_block",+ [+ # Discovery tool: allowed in INITIAL/DISCOVERING, blocked in COMPOSING/TESTING.+ ("discover_workflow_entrypoint", BuildPhase.INITIAL, False),+ ("discover_workflow_entrypoint", BuildPhase.DISCOVERING, False),+ ("discover_workflow_entrypoint", BuildPhase.COMPOSING, True),+ ("discover_workflow_entrypoint", BuildPhase.TESTING, True),+ # Browser primitives: blocked in INITIAL/DISCOVERING, allowed in COMPOSING/TESTING.+ ("navigate_browser", BuildPhase.INITIAL, True),+ ("navigate_browser", BuildPhase.DISCOVERING, True),+ ("navigate_browser", BuildPhase.COMPOSING, False),+ ("evaluate", BuildPhase.INITIAL, True),+ ("type_text", BuildPhase.DISCOVERING, True),+ ("click", BuildPhase.COMPOSING, False),+ # Mutation tools: blocked in INITIAL/DISCOVERING, allowed in COMPOSING/TESTING.+ ("update_workflow", BuildPhase.INITIAL, True),+ ("update_workflow", BuildPhase.DISCOVERING, True),+ ("update_workflow", BuildPhase.COMPOSING, False),+ ("update_and_run_blocks", BuildPhase.INITIAL, True),+ ("run_blocks_and_collect_debug", BuildPhase.DISCOVERING, True),+ # Unknown tool name -> no error.+ ("list_credentials", BuildPhase.INITIAL, False),+ ("list_credentials", BuildPhase.COMPOSING, False),+ ],+)+def test_phase_tool_error_matrix(tool_name: str, phase: BuildPhase, should_block: bool) -> None:+ ctx = _Ctx(phase)+ error = _phase_tool_error(ctx, tool_name)+ if should_block:+ assert error is not None+ assert "safe_reason_code=" in error+ else:+ assert error is None+++def test_phase_tool_error_returns_none_when_phase_attr_missing() -> None:+ class _NoPhase:+ pass++ assert _phase_tool_error(_NoPhase(), "navigate_browser") is Nonediff --git a/tests/unit/test_copilot_discovery_counter_persistence.py b/tests/unit/test_copilot_discovery_counter_persistence.pynew file mode 100644index 000000000..97da4d378--- /dev/null+++ b/tests/unit/test_copilot_discovery_counter_persistence.py@@ -0,0 +1,89 @@+"""Round-trip tests for the per-chat discovery counter.++The counter ships in ``StructuredContext.discovery_calls_made``, which is+serialized into ``AgentResult.global_llm_context`` and re-parsed at the next+turn's start. ``finalize_discovery_counter_in_global_llm_context`` is the+single writeback site — called by ``_make_agent_result`` for every exit path.+"""++from __future__ import annotations++import json+from dataclasses import dataclass++from skyvern.forge.sdk.copilot.context import (+ StructuredContext,+ finalize_discovery_counter_in_global_llm_context,+)+++@dataclass+class _Ctx:+ prior_discovery_calls_made: int = 0+ discovery_calls_this_turn: int = 0+++def test_structured_context_default_discovery_calls_made_is_zero() -> None:+ assert StructuredContext().discovery_calls_made == 0+++def test_structured_context_round_trip_preserves_discovery_calls_made() -> None:+ sc = StructuredContext(user_goal="x", discovery_calls_made=2)+ raw = sc.to_json_str()+ parsed = StructuredContext.from_json_str(raw)+ assert parsed.discovery_calls_made == 2+++def test_finalize_writes_summed_counter_into_outgoing_context() -> None:+ inbound = StructuredContext(user_goal="x", discovery_calls_made=1).to_json_str()+ ctx = _Ctx(prior_discovery_calls_made=1, discovery_calls_this_turn=1)+ out = finalize_discovery_counter_in_global_llm_context(ctx, inbound)+ assert out is not None+ sc = StructuredContext.from_json_str(out)+ assert sc.discovery_calls_made == 2+ assert sc.user_goal == "x"+++def test_finalize_writes_zero_when_no_calls_made_and_no_prior() -> None:+ ctx = _Ctx(prior_discovery_calls_made=0, discovery_calls_this_turn=0)+ # No prior context + no this-turn activity -> no need to invent a context.+ assert finalize_discovery_counter_in_global_llm_context(ctx, None) is None+++def test_finalize_writes_prior_when_this_turn_is_zero_and_prior_context_exists() -> None:+ inbound = StructuredContext(user_goal="g", discovery_calls_made=2).to_json_str()+ ctx = _Ctx(prior_discovery_calls_made=2, discovery_calls_this_turn=0)+ out = finalize_discovery_counter_in_global_llm_context(ctx, inbound)+ assert out is not None+ sc = StructuredContext.from_json_str(out)+ assert sc.discovery_calls_made == 2+++def test_finalize_handles_string_only_inbound_context() -> None:+ """Legacy `global_llm_context` was a plain string. The migration path in+ `StructuredContext.from_json_str` should preserve the string in+ user_goal and zero the counter."""+ ctx = _Ctx(prior_discovery_calls_made=0, discovery_calls_this_turn=1)+ out = finalize_discovery_counter_in_global_llm_context(ctx, "legacy string context")+ assert out is not None+ sc = StructuredContext.from_json_str(out)+ assert sc.discovery_calls_made == 1+ assert sc.user_goal == "legacy string context"+++def test_finalize_handles_invalid_json_inbound() -> None:+ ctx = _Ctx(prior_discovery_calls_made=0, discovery_calls_this_turn=1)+ out = finalize_discovery_counter_in_global_llm_context(ctx, "{not valid json")+ assert out is not None+ sc = StructuredContext.from_json_str(out)+ assert sc.discovery_calls_made == 1+++def test_finalize_treats_none_ctx_as_passthrough_in_factory() -> None:+ """The factory in agent.py passes ctx=None for very-early errors (before+ CopilotContext is constructed). The finalizer itself isn't called in that+ branch — _make_agent_result skips it — but verify that the StructuredContext+ round-trip itself still preserves a counter set by an earlier turn."""+ inbound = StructuredContext(discovery_calls_made=2).to_json_str()+ parsed = json.loads(inbound)+ assert parsed["discovery_calls_made"] == 2diff --git a/tests/unit/test_copilot_make_agent_result_sole_use.py b/tests/unit/test_copilot_make_agent_result_sole_use.pynew file mode 100644index 000000000..1b0e96124--- /dev/null+++ b/tests/unit/test_copilot_make_agent_result_sole_use.py@@ -0,0 +1,69 @@+"""Regression guard: `_make_agent_result` is the only allowed+``AgentResult(...)`` constructor in `skyvern/forge/sdk/copilot/agent.py`.++The factory routes every ``AgentResult`` through the discovery counter+finalizer, so the per-chat budget survives every exit path of the agent+loop. Route-level constructors (e.g.+`skyvern/forge/sdk/routes/workflow_copilot.py`'s recoverable-failure path)+are intentionally outside this scope — they run before the agent loop or+on a route-side failure where the agent's counter is by definition zero.+"""++from __future__ import annotations++import ast+from pathlib import Path++_AGENT_MODULE = Path(__file__).resolve().parents[2] / "skyvern/forge/sdk/copilot/agent.py"+++def _agent_result_call_sites() -> list[tuple[str, int]]:+ """Return (function_name, line) for every ``AgentResult(...)`` call in agent.py."""+ source = _AGENT_MODULE.read_text(encoding="utf-8")+ tree = ast.parse(source, filename=str(_AGENT_MODULE))++ sites: list[tuple[str, int]] = []++ class _Visitor(ast.NodeVisitor):+ def __init__(self) -> None:+ self._fn_stack: list[str] = []++ def visit_FunctionDef(self, node: ast.FunctionDef) -> None:+ self._fn_stack.append(node.name)+ self.generic_visit(node)+ self._fn_stack.pop()++ def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:+ self._fn_stack.append(node.name)+ self.generic_visit(node)+ self._fn_stack.pop()++ def visit_Call(self, node: ast.Call) -> None:+ func = node.func+ if isinstance(func, ast.Name) and func.id == "AgentResult":+ enclosing = self._fn_stack[-1] if self._fn_stack else "<module>"+ sites.append((enclosing, node.lineno))+ self.generic_visit(node)++ _Visitor().visit(tree)+ return sites+++def test_agent_result_is_only_constructed_inside_make_agent_result() -> None:+ sites = _agent_result_call_sites()+ offending = [(fn, ln) for fn, ln in sites if fn != "_make_agent_result"]+ assert not offending, (+ "Every AgentResult construction in agent.py must go through "+ "_make_agent_result(ctx, ...) so the discovery counter writeback fires "+ "on every exit. Offending sites:\n "+ + "\n ".join(f"line {ln} inside {fn or '<module>'}" for fn, ln in offending)+ )+++def test_make_agent_result_actually_constructs_agent_result() -> None:+ """Belt-and-braces: ensure we didn't accidentally remove the one AgentResult call."""+ sites = _agent_result_call_sites()+ inside_factory = [site for site in sites if site[0] == "_make_agent_result"]+ assert len(inside_factory) >= 1, (+ f"_make_agent_result must construct exactly one AgentResult; found {len(inside_factory)} calls inside it"+ )