Skip to content

Automate browser based workflows with AI

SKY-10335: copilot BuildPhase machine + AgentResult writeback factory (#6177)

Andrew Neilson8915cc049bf8
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 900a2feec..a92ef888b 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -32,7 +32,13 @@ from pydantic import ValidationError
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.copilot.config import CopilotConfig
-from skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES, AgentResult, CopilotContext, StructuredContext
+from skyvern.forge.sdk.copilot.context import (
+ COPILOT_RESPONSE_TYPES,
+ AgentResult,
+ CopilotContext,
+ StructuredContext,
+ finalize_discovery_counter_in_global_llm_context,
+)
from skyvern.forge.sdk.copilot.output_policy import (
UNVALIDATED_DISCLOSURE_PHRASES,
WORKFLOW_PRESENT_SENTINEL,
@@ -672,6 +678,28 @@ def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:
return None, None
+def _make_agent_result(
+ ctx: CopilotContext | None,
+ *,
+ global_llm_context: str | None = None,
+ turn_outcome: TurnOutcome | None = None,
+ **kwargs: Any,
+) -> AgentResult:
+ """Sole ``AgentResult`` constructor in this module.
+
+ Routes every ``AgentResult`` through the discovery-counter finalizer so
+ the per-chat budget survives every exit path (timeout, cancel, max-turns,
+ output-policy block, clarification helpers, non-retriable nav error,
+ normal translate-result, missing-SDK fallback, unexpected-error fallback).
+ """
+ final_context = (
+ finalize_discovery_counter_in_global_llm_context(ctx, global_llm_context)
+ if ctx is not None
+ else global_llm_context
+ )
+ return AgentResult(global_llm_context=final_context, turn_outcome=turn_outcome, **kwargs)
+
+
def _build_exit_result(
ctx: CopilotContext,
user_response: str,
@@ -687,7 +715,8 @@ def _build_exit_result(
blocked_signatures=ctx.blocked_reply_signatures,
terminal_reason=terminal_reason or ("cancel" if cancelled else None),
)
- return AgentResult(
+ return _make_agent_result(
+ ctx,
user_response=final_text,
updated_workflow=verified_workflow,
global_llm_context=global_llm_context,
@@ -979,7 +1008,8 @@ def _build_wip_exit_result(
):
reply = _last_good_failure_reply(ctx, tested_reply) if recorded_failure_reply else tested_reply
final_text, outcome = _guard(reply)
- return AgentResult(
+ return _make_agent_result(
+ ctx,
user_response=final_text,
updated_workflow=ctx.last_good_workflow,
global_llm_context=global_llm_context,
@@ -1002,7 +1032,8 @@ def _build_wip_exit_result(
else:
reply = unvalidated_reply if unvalidated else tested_reply
final_text, outcome = _guard(reply)
- return AgentResult(
+ return _make_agent_result(
+ ctx,
user_response=final_text,
updated_workflow=ctx.last_workflow,
global_llm_context=global_llm_context,
@@ -1391,7 +1422,8 @@ def _translate_to_agent_result(
tool_calls=[name for name in tool_call_names if name],
)
- return AgentResult(
+ return _make_agent_result(
+ ctx,
user_response=final_user_response,
updated_workflow=last_workflow,
global_llm_context=enriched_context or None,
@@ -1440,7 +1472,8 @@ def _build_feasibility_clarification_result(
blocked_signatures=list(ctx.blocked_reply_signatures),
reason_code="feasibility_clarification",
)
- return AgentResult(
+ return _make_agent_result(
+ ctx,
user_response=final_text,
updated_workflow=None,
global_llm_context=enriched_context,
@@ -1516,7 +1549,8 @@ def _build_request_policy_clarification_result(
blocked_signatures=list(ctx.blocked_reply_signatures),
reason_code="request_policy_clarification",
)
- return AgentResult(
+ return _make_agent_result(
+ ctx,
user_response=final_text,
updated_workflow=None,
global_llm_context=structured.to_json_str(),
@@ -1811,7 +1845,8 @@ def _build_output_policy_blocked_result(
reason_code="output_policy_block",
terminal_reason="output_policy_block",
)
- return AgentResult(
+ return _make_agent_result(
+ ctx,
user_response=final_user_response,
updated_workflow=preserved_workflow,
global_llm_context=structured.to_json_str(),
@@ -1970,7 +2005,8 @@ async def _run_copilot_turn_impl(
blocked_signatures=(),
terminal_reason="missing_sdk",
)
- return AgentResult(
+ return _make_agent_result(
+ None,
user_response=final_missing_text,
updated_workflow=None,
global_llm_context=global_llm_context,
@@ -2045,6 +2081,12 @@ async def _run_copilot_turn_impl(
previous_user_message=previous_user_message,
)
+ # Hydrate the per-chat discovery counter from the inbound global_llm_context.
+ # `ctx.build_phase` stays at the safe default and `initial_build_phase` is
+ # not yet called from the agent loop — that activates in the discovery PR.
+ prior_structured_context = StructuredContext.from_json_str(global_llm_context)
+ ctx.prior_discovery_calls_made = prior_structured_context.discovery_calls_made
+
# Preflight feasibility classifier — fires on every turn so mid-session pivots
# to impossible targets are caught the same as first-turn structural mismatches.
from skyvern.forge.sdk.copilot.feasibility_gate import run_feasibility_gate
@@ -2294,7 +2336,8 @@ async def _run_copilot_turn_impl(
blocked_signatures=ctx.blocked_reply_signatures,
terminal_reason="non_retriable_nav",
)
- return AgentResult(
+ return _make_agent_result(
+ ctx,
user_response=final_nav_text,
updated_workflow=None,
global_llm_context=global_llm_context,
diff --git a/skyvern/forge/sdk/copilot/build_phase.py b/skyvern/forge/sdk/copilot/build_phase.py
new file mode 100644
index 000000000..4c041c49d
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/build_phase.py
@@ -0,0 +1,220 @@
+"""Build-phase machinery for the copilot agent loop.
+
+The phase machine separates build-time discovery from workflow composition.
+TurnIntent.mode stays BUILD; phase is loop runtime state on ctx. The
+deterministic orchestrator owns transitions — an agent-emitted phase signal
+does not by itself unlock mutation authority.
+
+Phases:
+- INITIAL: BUILD turn with no known entrypoint URL. Discovery is available;
+ mutation and direct browser primitives are gated off.
+- DISCOVERING: discover_workflow_entrypoint is running. Same gate as INITIAL.
+- COMPOSING: entrypoint resolved. Mutation tools and direct browser primitives
+ are available; discovery is no longer available.
+- TESTING: post-update; same authority as COMPOSING.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from enum import StrEnum
+from typing import TYPE_CHECKING, Any
+
+import structlog
+import yaml
+
+from skyvern.utils.yaml_loader import safe_load_no_dates
+
+if TYPE_CHECKING:
+ from skyvern.forge.sdk.copilot.context import CopilotContext
+ from skyvern.forge.sdk.copilot.turn_intent import TurnIntent
+
+LOG = structlog.get_logger()
+
+# Mode values that explicitly rule out discovery. Other modes (BUILD,
+# DRAFT_ONLY, UNKNOWN) may enter INITIAL phase when no URL signal exists —
+# the TurnIntent keyword classifier gates NEW_BROWSER_TASK_TERMS on
+# `not has_workflow`, so a fresh build turn with an empty-blocks YAML can
+# legitimately classify as UNKNOWN even though discovery is the right next
+# step. The phase gate keeps mutation blocked until discovery returns a
+# candidate (or the model ASK_QUESTIONs for a URL).
+_PHASE_NON_BUILD_MODE_VALUES: frozenset[str] = frozenset({"edit", "diagnose", "docs_answer", "clarify", "refuse"})
+
+
+class BuildPhase(StrEnum):
+ INITIAL = "initial"
+ DISCOVERING = "discovering"
+ COMPOSING = "composing"
+ TESTING = "testing"
+
+
+DISCOVERY_PERMITTED_PHASES: frozenset[BuildPhase] = frozenset({BuildPhase.INITIAL, BuildPhase.DISCOVERING})
+MUTATION_PERMITTED_PHASES: frozenset[BuildPhase] = frozenset({BuildPhase.COMPOSING, BuildPhase.TESTING})
+
+# Tools whose call paths must be gated by phase. Names match the
+# function_tool name_override / MCP overlay names registered elsewhere.
+_BROWSER_PRIMITIVE_TOOLS: frozenset[str] = frozenset(
+ {
+ "navigate_browser",
+ "evaluate",
+ "click",
+ "type_text",
+ "scroll",
+ "select_option",
+ "press_key",
+ "console_messages",
+ "get_browser_screenshot",
+ }
+)
+_MUTATION_TOOLS: frozenset[str] = frozenset(
+ {"update_workflow", "update_and_run_blocks", "run_blocks_and_collect_debug"}
+)
+_DISCOVERY_TOOLS: frozenset[str] = frozenset({"discover_workflow_entrypoint"})
+
+_URL_IN_TEXT_RE = re.compile(r"https?://[^\s<>\"\)]+", re.IGNORECASE)
+
+
+def _yaml_has_target_url(workflow_yaml: str | None) -> bool:
+ """True when the YAML carries a goto_url or navigation block with a non-empty url."""
+ if not workflow_yaml:
+ return False
+ try:
+ parsed = safe_load_no_dates(workflow_yaml)
+ except yaml.YAMLError:
+ return False
+ if not isinstance(parsed, dict):
+ return False
+ definition = parsed.get("workflow_definition")
+ if not isinstance(definition, dict):
+ return False
+ blocks = definition.get("blocks")
+ if not isinstance(blocks, list):
+ return False
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ block_type = block.get("block_type")
+ if block_type not in {"goto_url", "navigation"}:
+ continue
+ url = block.get("url")
+ if isinstance(url, str) and url.strip():
+ return True
+ return False
+
+
+def initial_build_phase(
+ turn_intent: TurnIntent | None,
+ user_message: str,
+ agent_user_message: str,
+ workflow_yaml: str | None,
+) -> BuildPhase:
+ """Decide the initial build phase for this turn.
+
+ Returns COMPOSING (sentinel — harmless because the existing TurnIntent gate
+ already blocks mutation for non-BUILD modes) when the mode is one of the
+ explicitly-non-build values. For BUILD / DRAFT_ONLY / UNKNOWN, returns
+ COMPOSING when this turn already has a URL signal, else INITIAL.
+
+ UNKNOWN deliberately enters INITIAL when no URL is present: a fresh chat
+ whose latest message dodges every keyword heuristic (e.g. "go to X" with
+ an empty-blocks scaffold workflow that flips `has_workflow=True` and
+ suppresses NEW_BROWSER_TASK_TERMS) should still get discovery — the
+ alternative is to let mutation through on an ambiguous turn, which is
+ worse than asking discovery to resolve or falling through to ASK_QUESTION.
+
+ Signals considered (this-turn only — never prior visited URLs or chat
+ history, which can leak stale URLs across unrelated turns):
+ - The raw latest user message.
+ - The rewritten agent input (request-policy may carry the prior request).
+ - The current workflow YAML's first goto_url/navigation block.
+ """
+ mode_value = getattr(getattr(turn_intent, "mode", None), "value", None)
+ if mode_value in _PHASE_NON_BUILD_MODE_VALUES:
+ return BuildPhase.COMPOSING
+ if _URL_IN_TEXT_RE.search(user_message or ""):
+ return BuildPhase.COMPOSING
+ if _URL_IN_TEXT_RE.search(agent_user_message or ""):
+ return BuildPhase.COMPOSING
+ if _yaml_has_target_url(workflow_yaml):
+ return BuildPhase.COMPOSING
+ return BuildPhase.INITIAL
+
+
+def _log_transition(ctx: CopilotContext, *, prev: BuildPhase, new: BuildPhase, reason: str) -> None:
+ LOG.info(
+ "copilot.build_phase_transition",
+ prev_phase=prev.value,
+ new_phase=new.value,
+ transition_reason=reason,
+ workflow_permanent_id=getattr(ctx, "workflow_permanent_id", None),
+ )
+
+
+def advance_to_discovering(ctx: CopilotContext) -> None:
+ if ctx.build_phase != BuildPhase.INITIAL:
+ raise ValueError(f"advance_to_discovering called from {ctx.build_phase.value}; expected INITIAL")
+ if ctx.discovery_started_monotonic is None:
+ ctx.discovery_started_monotonic = time.monotonic()
+ prev = ctx.build_phase
+ ctx.build_phase = BuildPhase.DISCOVERING
+ _log_transition(ctx, prev=prev, new=ctx.build_phase, reason="discovery_started")
+
+
+def advance_to_composing(ctx: CopilotContext, *, reason: str) -> None:
+ if ctx.build_phase not in (BuildPhase.INITIAL, BuildPhase.DISCOVERING):
+ raise ValueError(f"advance_to_composing called from {ctx.build_phase.value}; expected INITIAL or DISCOVERING")
+ prev = ctx.build_phase
+ ctx.build_phase = BuildPhase.COMPOSING
+ _log_transition(ctx, prev=prev, new=ctx.build_phase, reason=reason)
+
+
+def advance_to_testing(ctx: CopilotContext) -> None:
+ if ctx.build_phase == BuildPhase.TESTING:
+ # Already in TESTING (or post-TESTING composer follow-up) is benign.
+ return
+ if ctx.build_phase != BuildPhase.COMPOSING:
+ raise ValueError(f"advance_to_testing called from {ctx.build_phase.value}; expected COMPOSING or TESTING")
+ prev = ctx.build_phase
+ ctx.build_phase = BuildPhase.TESTING
+ _log_transition(ctx, prev=prev, new=ctx.build_phase, reason="update_workflow_succeeded")
+
+
+def _phase_tool_error(ctx: Any, tool_name: str) -> str | None:
+ """Shared phase-aware authority error. Returns the steering error string
+ when the (phase, tool) pair is forbidden; None otherwise.
+
+ Called from both the native gate (`_authority_tool_error` in tools.py)
+ and the MCP-adapter gate (`SkyvernOverlayMCPServer.call_tool` pre-pre-hook).
+ Errors flow back to the LLM as next-step steering, per the ai-agents
+ 'errors as steering' guidance.
+ """
+ phase = getattr(ctx, "build_phase", None)
+ if not isinstance(phase, BuildPhase):
+ return None
+
+ in_discovery = phase in DISCOVERY_PERMITTED_PHASES
+ in_mutation = phase in MUTATION_PERMITTED_PHASES
+
+ if tool_name in _DISCOVERY_TOOLS and in_mutation:
+ return (
+ "discover_workflow_entrypoint is only available before composition. "
+ "The workflow already has a target URL — proceed with update_workflow or update_and_run_blocks. "
+ "safe_reason_code=build_phase_discovery_disallowed_post_compose."
+ )
+
+ if tool_name in _BROWSER_PRIMITIVE_TOOLS and in_discovery:
+ return (
+ "Direct browser tools are not callable before composition. "
+ "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL. "
+ "safe_reason_code=build_phase_browser_blocked_pre_compose."
+ )
+
+ if tool_name in _MUTATION_TOOLS and in_discovery:
+ return (
+ "Workflow mutation is gated to composition. "
+ "Call discover_workflow_entrypoint to resolve the entrypoint URL, or ASK_QUESTION for a URL first. "
+ "safe_reason_code=build_phase_mutation_blocked_pre_compose."
+ )
+
+ return None
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index e80879b7a..2bf689b4f 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Literal, get_args
from pydantic import BaseModel, Field
+from skyvern.forge.sdk.copilot.build_phase import BuildPhase
from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.workflow.models.workflow import Workflow
@@ -48,6 +49,10 @@ class StructuredContext(BaseModel):
credentials_checked: list[CredentialCheck] = Field(default_factory=list)
decisions_made: list[str] = Field(default_factory=list)
workflow_state: str = ""
+ # Per-chat discovery budget. Survives turn boundaries via
+ # AgentResult.global_llm_context — finalized deterministically at every
+ # AgentResult exit by `finalize_discovery_counter_in_global_llm_context`.
+ discovery_calls_made: int = 0
def to_json_str(self) -> str:
return self.model_dump_json(indent=2)
@@ -118,6 +123,27 @@ class StructuredContext(BaseModel):
self.credentials_checked = self.credentials_checked[-40:]
+def finalize_discovery_counter_in_global_llm_context(ctx: Any, raw_context: str | None) -> str | None:
+ """Fold the per-chat discovery counter into the outgoing global_llm_context.
+
+ Called from agent.py's `_make_agent_result` factory so every AgentResult
+ exit path — timeout, cancel, max-turns, output-policy block, request-
+ policy clarification, feasibility clarification, non-retriable nav error,
+ normal translate-result, missing-SDK fallback, unexpected-error fallback —
+ carries the updated count.
+
+ Returns None when there is nothing to record and no prior context, so the
+ pre-existing 'no global_llm_context' behaviour is preserved.
+ """
+ prior = int(getattr(ctx, "prior_discovery_calls_made", 0) or 0)
+ this_turn = int(getattr(ctx, "discovery_calls_this_turn", 0) or 0)
+ if not raw_context and this_turn == 0:
+ return None
+ sc = StructuredContext.from_json_str(raw_context)
+ sc.discovery_calls_made = prior + this_turn
+ return sc.to_json_str()
+
+
@dataclass
class AgentResult:
user_response: str
@@ -290,3 +316,16 @@ class CopilotContext(AgentContext):
# retries. Declared here (rather than attached dynamically) so future
# refactors can't strip it silently.
narrator_state: NarratorState | None = None
+
+ # Default COMPOSING is the safe non-BUILD value; the orchestrator
+ # overrides at turn start via `initial_build_phase`.
+ build_phase: BuildPhase = BuildPhase.COMPOSING
+ # Hydrated from inbound StructuredContext.discovery_calls_made at turn start.
+ prior_discovery_calls_made: int = 0
+ discovery_calls_this_turn: int = 0
+ discovery_step_count: int = 0
+ discovery_started_monotonic: float | None = None
+ discovery_evidence_trail: list[dict[str, Any]] = field(default_factory=list)
+ # Set in `_run_attempt` after SkyvernOverlayMCPServer is constructed.
+ # The discovery tool reaches the connected FastMCP client through this.
+ discovery_mcp_server: Any | None = None
diff --git a/tests/unit/test_copilot_build_phase.py b/tests/unit/test_copilot_build_phase.py
new file mode 100644
index 000000000..8a39fc3c6
--- /dev/null
+++ b/tests/unit/test_copilot_build_phase.py
@@ -0,0 +1,255 @@
+"""Unit tests for the copilot ``BuildPhase`` machinery."""
+
+from __future__ import annotations
+
+import pytest
+
+from skyvern.forge.sdk.copilot.build_phase import (
+ DISCOVERY_PERMITTED_PHASES,
+ MUTATION_PERMITTED_PHASES,
+ BuildPhase,
+ _phase_tool_error,
+ _yaml_has_target_url,
+ advance_to_composing,
+ advance_to_discovering,
+ advance_to_testing,
+ initial_build_phase,
+)
+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentMode
+
+
+class _Ctx:
+ """Lightweight ctx stand-in — the helpers only read/write a small surface."""
+
+ def __init__(self, phase: BuildPhase = BuildPhase.COMPOSING) -> None:
+ self.build_phase = phase
+ self.discovery_started_monotonic: float | None = None
+ self.workflow_permanent_id = "wpid_test"
+
+
+def _ti(mode: TurnIntentMode) -> TurnIntent:
+ return TurnIntent(mode=mode)
+
+
+# ---------------- initial_build_phase ----------------
+
+
+ "mode,user_message,agent_message,workflow_yaml,expected",
+ [
+ # Explicitly-non-build modes return the COMPOSING sentinel.
+ (TurnIntentMode.EDIT, "no url", "no url", "", BuildPhase.COMPOSING),
+ (TurnIntentMode.DOCS_ANSWER, "what is a workflow?", "what is a workflow?", "", BuildPhase.COMPOSING),
+ (TurnIntentMode.DIAGNOSE, "the run failed", "the run failed", "", BuildPhase.COMPOSING),
+ (TurnIntentMode.CLARIFY, "anything", "anything", "", BuildPhase.COMPOSING),
+ (TurnIntentMode.REFUSE, "anything", "anything", "", BuildPhase.COMPOSING),
+ # BUILD with URL in latest user_message -> COMPOSING.
+ (TurnIntentMode.BUILD, "go to https://example.com/login", "", "", BuildPhase.COMPOSING),
+ # BUILD with URL only in rewritten agent message (request-policy continuation) -> COMPOSING.
+ (TurnIntentMode.BUILD, "and download it", "earlier: go to https://example.com/file", "", BuildPhase.COMPOSING),
+ # BUILD with no URL anywhere -> INITIAL.
+ (TurnIntentMode.BUILD, "go to example", "go to example", "", BuildPhase.INITIAL),
+ # DRAFT_ONLY behaves like BUILD for phase init.
+ (
+ TurnIntentMode.DRAFT_ONLY,
+ "draft a workflow for example.com",
+ "draft a workflow for example.com",
+ "",
+ BuildPhase.INITIAL,
+ ),
+ # DRAFT_ONLY with URL -> COMPOSING.
+ (TurnIntentMode.DRAFT_ONLY, "draft a workflow for https://example.com", "", "", BuildPhase.COMPOSING),
+ # UNKNOWN with no URL also enters INITIAL — a fresh "go to X" turn
+ # with an empty-blocks scaffold workflow flips `has_workflow=True`
+ # in the keyword classifier and suppresses NEW_BROWSER_TASK_TERMS,
+ # producing UNKNOWN. Discovery is still the right next step there.
+ (TurnIntentMode.UNKNOWN, "go to example and find sortable tables", "", "", BuildPhase.INITIAL),
+ # UNKNOWN with a URL in the message -> COMPOSING (URL signal wins).
+ (TurnIntentMode.UNKNOWN, "go to https://example.com/x", "", "", BuildPhase.COMPOSING),
+ ],
+)
+def test_initial_build_phase_returns_expected(
+ mode: TurnIntentMode,
+ user_message: str,
+ agent_message: str,
+ workflow_yaml: str,
+ expected: BuildPhase,
+) -> None:
+ assert initial_build_phase(_ti(mode), user_message, agent_message, workflow_yaml) == expected
+
+
+def test_initial_build_phase_yaml_goto_url_unlocks_composing() -> None:
+ yaml_text = """
+title: T
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: goto_url
+ label: open
+ url: https://example.com/page
+"""
+ assert (
+ initial_build_phase(_ti(TurnIntentMode.BUILD), "do that thing", "do that thing", yaml_text)
+ == BuildPhase.COMPOSING
+ )
+
+
+def test_initial_build_phase_yaml_with_empty_url_does_not_unlock() -> None:
+ yaml_text = """
+title: T
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: goto_url
+ label: open
+ url: ""
+"""
+ assert initial_build_phase(_ti(TurnIntentMode.BUILD), "do thing", "do thing", yaml_text) == BuildPhase.INITIAL
+
+
+def test_initial_build_phase_none_turn_intent_acts_like_unknown_mode() -> None:
+ # turn_intent=None is treated as if mode is UNKNOWN — eligible for INITIAL
+ # when no URL signal exists. The phase gate then blocks mutation until
+ # discovery resolves an entrypoint or the model ASK_QUESTIONs.
+ assert initial_build_phase(None, "go to example", "go to example", "") == BuildPhase.INITIAL
+ # URL present -> COMPOSING regardless of None turn_intent.
+ assert initial_build_phase(None, "go to https://example.com", "", "") == BuildPhase.COMPOSING
+
+
+def test_initial_build_phase_yaml_malformed_falls_back_to_no_url() -> None:
+ assert initial_build_phase(_ti(TurnIntentMode.BUILD), "open it", "open it", "[ unbalanced") == BuildPhase.INITIAL
+
+
+# ---------------- _yaml_has_target_url ----------------
+
+
+def test_yaml_has_target_url_detects_navigation_block_url() -> None:
+ yaml_text = """
+title: T
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: navigation
+ label: visit
+ url: https://example.com/x
+ navigation_goal: visit it
+"""
+ assert _yaml_has_target_url(yaml_text) is True
+
+
+def test_yaml_has_target_url_ignores_non_navigation_blocks() -> None:
+ yaml_text = """
+title: T
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: code
+ label: c
+ code: "x = 1"
+"""
+ assert _yaml_has_target_url(yaml_text) is False
+
+
+def test_yaml_has_target_url_handles_empty_and_none() -> None:
+ assert _yaml_has_target_url(None) is False
+ assert _yaml_has_target_url("") is False
+
+
+# ---------------- transition helpers ----------------
+
+
+def test_advance_to_discovering_only_from_initial() -> None:
+ ctx = _Ctx(BuildPhase.INITIAL)
+ advance_to_discovering(ctx)
+ assert ctx.build_phase == BuildPhase.DISCOVERING
+ assert ctx.discovery_started_monotonic is not None
+
+
+def test_advance_to_discovering_rejects_other_phases() -> None:
+ for phase in (BuildPhase.DISCOVERING, BuildPhase.COMPOSING, BuildPhase.TESTING):
+ ctx = _Ctx(phase)
+ with pytest.raises(ValueError):
+ advance_to_discovering(ctx)
+
+
+def test_advance_to_composing_accepts_initial_and_discovering() -> None:
+ for phase in (BuildPhase.INITIAL, BuildPhase.DISCOVERING):
+ ctx = _Ctx(phase)
+ advance_to_composing(ctx, reason="test")
+ assert ctx.build_phase == BuildPhase.COMPOSING
+
+
+def test_advance_to_composing_rejects_from_composing_or_testing() -> None:
+ for phase in (BuildPhase.COMPOSING, BuildPhase.TESTING):
+ ctx = _Ctx(phase)
+ with pytest.raises(ValueError):
+ advance_to_composing(ctx, reason="test")
+
+
+def test_advance_to_testing_only_from_composing() -> None:
+ ctx = _Ctx(BuildPhase.COMPOSING)
+ advance_to_testing(ctx)
+ assert ctx.build_phase == BuildPhase.TESTING
+
+
+def test_advance_to_testing_is_noop_from_testing() -> None:
+ ctx = _Ctx(BuildPhase.TESTING)
+ advance_to_testing(ctx)
+ assert ctx.build_phase == BuildPhase.TESTING
+
+
+def test_advance_to_testing_rejects_pre_composition_phases() -> None:
+ for phase in (BuildPhase.INITIAL, BuildPhase.DISCOVERING):
+ ctx = _Ctx(phase)
+ with pytest.raises(ValueError):
+ advance_to_testing(ctx)
+
+
+def test_phase_sets_are_disjoint() -> None:
+ assert DISCOVERY_PERMITTED_PHASES.isdisjoint(MUTATION_PERMITTED_PHASES)
+
+
+# ---------------- _phase_tool_error ----------------
+
+
+ "tool_name,phase,should_block",
+ [
+ # Discovery tool: allowed in INITIAL/DISCOVERING, blocked in COMPOSING/TESTING.
+ ("discover_workflow_entrypoint", BuildPhase.INITIAL, False),
+ ("discover_workflow_entrypoint", BuildPhase.DISCOVERING, False),
+ ("discover_workflow_entrypoint", BuildPhase.COMPOSING, True),
+ ("discover_workflow_entrypoint", BuildPhase.TESTING, True),
+ # Browser primitives: blocked in INITIAL/DISCOVERING, allowed in COMPOSING/TESTING.
+ ("navigate_browser", BuildPhase.INITIAL, True),
+ ("navigate_browser", BuildPhase.DISCOVERING, True),
+ ("navigate_browser", BuildPhase.COMPOSING, False),
+ ("evaluate", BuildPhase.INITIAL, True),
+ ("type_text", BuildPhase.DISCOVERING, True),
+ ("click", BuildPhase.COMPOSING, False),
+ # Mutation tools: blocked in INITIAL/DISCOVERING, allowed in COMPOSING/TESTING.
+ ("update_workflow", BuildPhase.INITIAL, True),
+ ("update_workflow", BuildPhase.DISCOVERING, True),
+ ("update_workflow", BuildPhase.COMPOSING, False),
+ ("update_and_run_blocks", BuildPhase.INITIAL, True),
+ ("run_blocks_and_collect_debug", BuildPhase.DISCOVERING, True),
+ # Unknown tool name -> no error.
+ ("list_credentials", BuildPhase.INITIAL, False),
+ ("list_credentials", BuildPhase.COMPOSING, False),
+ ],
+)
+def test_phase_tool_error_matrix(tool_name: str, phase: BuildPhase, should_block: bool) -> None:
+ ctx = _Ctx(phase)
+ error = _phase_tool_error(ctx, tool_name)
+ if should_block:
+ assert error is not None
+ assert "safe_reason_code=" in error
+ else:
+ assert error is None
+
+
+def test_phase_tool_error_returns_none_when_phase_attr_missing() -> None:
+ class _NoPhase:
+ pass
+
+ assert _phase_tool_error(_NoPhase(), "navigate_browser") is None
diff --git a/tests/unit/test_copilot_discovery_counter_persistence.py b/tests/unit/test_copilot_discovery_counter_persistence.py
new file mode 100644
index 000000000..97da4d378
--- /dev/null
+++ b/tests/unit/test_copilot_discovery_counter_persistence.py
@@ -0,0 +1,89 @@
+"""Round-trip tests for the per-chat discovery counter.
+
+The counter ships in ``StructuredContext.discovery_calls_made``, which is
+serialized into ``AgentResult.global_llm_context`` and re-parsed at the next
+turn's start. ``finalize_discovery_counter_in_global_llm_context`` is the
+single writeback site — called by ``_make_agent_result`` for every exit path.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+
+from skyvern.forge.sdk.copilot.context import (
+ StructuredContext,
+ finalize_discovery_counter_in_global_llm_context,
+)
+
+
+@dataclass
+class _Ctx:
+ prior_discovery_calls_made: int = 0
+ discovery_calls_this_turn: int = 0
+
+
+def test_structured_context_default_discovery_calls_made_is_zero() -> None:
+ assert StructuredContext().discovery_calls_made == 0
+
+
+def test_structured_context_round_trip_preserves_discovery_calls_made() -> None:
+ sc = StructuredContext(user_goal="x", discovery_calls_made=2)
+ raw = sc.to_json_str()
+ parsed = StructuredContext.from_json_str(raw)
+ assert parsed.discovery_calls_made == 2
+
+
+def test_finalize_writes_summed_counter_into_outgoing_context() -> None:
+ inbound = StructuredContext(user_goal="x", discovery_calls_made=1).to_json_str()
+ ctx = _Ctx(prior_discovery_calls_made=1, discovery_calls_this_turn=1)
+ out = finalize_discovery_counter_in_global_llm_context(ctx, inbound)
+ assert out is not None
+ sc = StructuredContext.from_json_str(out)
+ assert sc.discovery_calls_made == 2
+ assert sc.user_goal == "x"
+
+
+def test_finalize_writes_zero_when_no_calls_made_and_no_prior() -> None:
+ ctx = _Ctx(prior_discovery_calls_made=0, discovery_calls_this_turn=0)
+ # No prior context + no this-turn activity -> no need to invent a context.
+ assert finalize_discovery_counter_in_global_llm_context(ctx, None) is None
+
+
+def test_finalize_writes_prior_when_this_turn_is_zero_and_prior_context_exists() -> None:
+ inbound = StructuredContext(user_goal="g", discovery_calls_made=2).to_json_str()
+ ctx = _Ctx(prior_discovery_calls_made=2, discovery_calls_this_turn=0)
+ out = finalize_discovery_counter_in_global_llm_context(ctx, inbound)
+ assert out is not None
+ sc = StructuredContext.from_json_str(out)
+ assert sc.discovery_calls_made == 2
+
+
+def test_finalize_handles_string_only_inbound_context() -> None:
+ """Legacy `global_llm_context` was a plain string. The migration path in
+ `StructuredContext.from_json_str` should preserve the string in
+ user_goal and zero the counter."""
+ ctx = _Ctx(prior_discovery_calls_made=0, discovery_calls_this_turn=1)
+ out = finalize_discovery_counter_in_global_llm_context(ctx, "legacy string context")
+ assert out is not None
+ sc = StructuredContext.from_json_str(out)
+ assert sc.discovery_calls_made == 1
+ assert sc.user_goal == "legacy string context"
+
+
+def test_finalize_handles_invalid_json_inbound() -> None:
+ ctx = _Ctx(prior_discovery_calls_made=0, discovery_calls_this_turn=1)
+ out = finalize_discovery_counter_in_global_llm_context(ctx, "{not valid json")
+ assert out is not None
+ sc = StructuredContext.from_json_str(out)
+ assert sc.discovery_calls_made == 1
+
+
+def test_finalize_treats_none_ctx_as_passthrough_in_factory() -> None:
+ """The factory in agent.py passes ctx=None for very-early errors (before
+ CopilotContext is constructed). The finalizer itself isn't called in that
+ branch — _make_agent_result skips it — but verify that the StructuredContext
+ round-trip itself still preserves a counter set by an earlier turn."""
+ inbound = StructuredContext(discovery_calls_made=2).to_json_str()
+ parsed = json.loads(inbound)
+ assert parsed["discovery_calls_made"] == 2
diff --git a/tests/unit/test_copilot_make_agent_result_sole_use.py b/tests/unit/test_copilot_make_agent_result_sole_use.py
new file mode 100644
index 000000000..1b0e96124
--- /dev/null
+++ b/tests/unit/test_copilot_make_agent_result_sole_use.py
@@ -0,0 +1,69 @@
+"""Regression guard: `_make_agent_result` is the only allowed
+``AgentResult(...)`` constructor in `skyvern/forge/sdk/copilot/agent.py`.
+
+The factory routes every ``AgentResult`` through the discovery counter
+finalizer, so the per-chat budget survives every exit path of the agent
+loop. Route-level constructors (e.g.
+`skyvern/forge/sdk/routes/workflow_copilot.py`'s recoverable-failure path)
+are intentionally outside this scope — they run before the agent loop or
+on a route-side failure where the agent's counter is by definition zero.
+"""
+
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+_AGENT_MODULE = Path(__file__).resolve().parents[2] / "skyvern/forge/sdk/copilot/agent.py"
+
+
+def _agent_result_call_sites() -> list[tuple[str, int]]:
+ """Return (function_name, line) for every ``AgentResult(...)`` call in agent.py."""
+ source = _AGENT_MODULE.read_text(encoding="utf-8")
+ tree = ast.parse(source, filename=str(_AGENT_MODULE))
+
+ sites: list[tuple[str, int]] = []
+
+ class _Visitor(ast.NodeVisitor):
+ def __init__(self) -> None:
+ self._fn_stack: list[str] = []
+
+ def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+ self._fn_stack.append(node.name)
+ self.generic_visit(node)
+ self._fn_stack.pop()
+
+ def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
+ self._fn_stack.append(node.name)
+ self.generic_visit(node)
+ self._fn_stack.pop()
+
+ def visit_Call(self, node: ast.Call) -> None:
+ func = node.func
+ if isinstance(func, ast.Name) and func.id == "AgentResult":
+ enclosing = self._fn_stack[-1] if self._fn_stack else "<module>"
+ sites.append((enclosing, node.lineno))
+ self.generic_visit(node)
+
+ _Visitor().visit(tree)
+ return sites
+
+
+def test_agent_result_is_only_constructed_inside_make_agent_result() -> None:
+ sites = _agent_result_call_sites()
+ offending = [(fn, ln) for fn, ln in sites if fn != "_make_agent_result"]
+ assert not offending, (
+ "Every AgentResult construction in agent.py must go through "
+ "_make_agent_result(ctx, ...) so the discovery counter writeback fires "
+ "on every exit. Offending sites:\n "
+ + "\n ".join(f"line {ln} inside {fn or '<module>'}" for fn, ln in offending)
+ )
+
+
+def test_make_agent_result_actually_constructs_agent_result() -> None:
+ """Belt-and-braces: ensure we didn't accidentally remove the one AgentResult call."""
+ sites = _agent_result_call_sites()
+ inside_factory = [site for site in sites if site[0] == "_make_agent_result"]
+ assert len(inside_factory) >= 1, (
+ f"_make_agent_result must construct exactly one AgentResult; found {len(inside_factory)} calls inside it"
+ )