Skip to content

Automate browser based workflows with AI

fix(SKY-10137): annotate _workflow_with_runtime_block_goal_context's ctx as CopilotContext (#6115)

Celal Zamanoğlu4cf197f2abfd
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 630f57860..6f1e8b5ba 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -31,7 +31,6 @@ from pydantic import ValidationError
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
-from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_block_goals
from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES, AgentResult, CopilotContext, StructuredContext
from skyvern.forge.sdk.copilot.output_policy import (
@@ -151,13 +150,6 @@ class RequestPolicyGuardrailInputs:
browser_session_id: str | None = None
-_BLOCK_GOAL_CONTEXT_PREAMBLE = (
- "Interpret the latest user message in the context of the prior conversation. If it is a correction, "
- "refinement, or continuation, preserve the established website, actions, filters, and output requirements "
- "except where the latest message changes them."
-)
-
-
async def _resolve_live_browser_session_id(
chat_request: WorkflowCopilotChatRequest,
organization_id: str,
@@ -236,24 +228,7 @@ def _build_block_goal_main_goal(
raw_current_message = (user_message or "").strip()
if not raw_current_message:
return ""
- current_message = escape_code_fences(raw_current_message)
- structured_context = StructuredContext.from_json_str(global_llm_context)
- context_sections: list[str] = []
- if structured_context.user_goal.strip():
- prior_goal = escape_code_fences(structured_context.user_goal.strip())
- context_sections.append(f"Prior high-level goal:\n{prior_goal}")
- if chat_history_text.strip():
- escaped_chat_history = escape_code_fences(chat_history_text.strip())
- context_sections.append(f"Recent chat history:\n{escaped_chat_history}")
- if not context_sections:
- return current_message
-
- return (
- _BLOCK_GOAL_CONTEXT_PREAMBLE
- + "\n\n"
- + "\n\n".join(context_sections)
- + f"\n\nLatest user message:\n{current_message}"
- )
+ return escape_code_fences(raw_current_message)
def _request_policy_agent_inputs(
@@ -704,13 +679,7 @@ _RAW_SECRET_LEAK_REFUSAL = (
"Store credentials in the Skyvern Credentials UI and reply with the saved credential name or a "
f"credential ID beginning with cred_. {RAW_SECRET_REFUSAL_SENTINEL}."
)
-_SAVED_DRAFT_OUTPUT_POLICY_SUFFIX = " I only blocked the chat reply; the workflow draft is still saved."
-
-
-def _with_saved_draft_output_policy_copy(user_response: str) -> str:
- return f"{user_response}{_SAVED_DRAFT_OUTPUT_POLICY_SUFFIX}"
-
-
+_SAVED_DRAFT_OUTPUT_POLICY_SUFFIX = "I only blocked the chat reply; the workflow draft is still saved."
_CANCEL_REPLY_DEFAULT = "Cancelled by user."
_CANCEL_REPLY_UNVALIDATED = (
"Cancelled. I have a draft workflow you can keep — accept it to save "
@@ -1018,11 +987,6 @@ def _translate_to_agent_result(
ctx.last_test_ok = None
workflow_yaml = ""
if workflow_yaml:
- block_goal_main_goal = ctx.block_goal_main_goal or ctx.user_message
- if block_goal_main_goal:
- workflow_yaml = wrap_block_goals(workflow_yaml, block_goal_main_goal)
- else:
- LOG.warning("REPLACE_WORKFLOW inline path missing block-goal context; skipping block-goal wrap")
try:
last_workflow = _process_workflow_yaml(
workflow_id=chat_request.workflow_id,
@@ -1588,7 +1552,7 @@ def _build_output_policy_blocked_result(
else:
user_response = "I could not safely return that chat reply. Please adjust the request and try again."
if preserved_workflow is not None and add_saved_draft_copy:
- user_response = _with_saved_draft_output_policy_copy(user_response)
+ user_response = f"{user_response} {_SAVED_DRAFT_OUTPUT_POLICY_SUFFIX}"
return AgentResult(
user_response=user_response,
updated_workflow=preserved_workflow,
diff --git a/skyvern/forge/sdk/copilot/block_goal_wrapping.py b/skyvern/forge/sdk/copilot/block_goal_wrapping.py
index 9202aece5..05834d4c6 100644
--- a/skyvern/forge/sdk/copilot/block_goal_wrapping.py
+++ b/skyvern/forge/sdk/copilot/block_goal_wrapping.py
@@ -1,7 +1,7 @@
-"""Wrap copilot-v2-generated block intent fields (``navigation_goal``,
+"""Wrap runtime block intent fields (``navigation_goal``,
``complete_criterion``, ``terminate_criterion``) with the effective
"big goal" context, mirroring the TaskV2 pattern that applies
-``MINI_GOAL_TEMPLATE`` at every mini-goal construction site.
+``MINI_GOAL_TEMPLATE`` at mini-goal execution sites.
Without this wrap:
- The Skyvern verifier (``complete_verify``) has no user-intent context
@@ -14,13 +14,18 @@ Without this wrap:
from __future__ import annotations
import re
-from typing import Any
+from typing import TYPE_CHECKING, Any, TypeVar
import yaml
from skyvern.constants import MINI_GOAL_TEMPLATE
from skyvern.utils.yaml_loader import safe_load_no_dates
+if TYPE_CHECKING:
+ from skyvern.forge.sdk.workflow.models.workflow import Workflow
+
+WorkflowT = TypeVar("WorkflowT", bound="Workflow")
+
# Block fields whose value expresses the LLM's "mini goal" — what it should
# do or what it should check for. Wrapped in MINI_GOAL_TEMPLATE alongside the
# effective main goal so the downstream LLMs can reason about intent.
@@ -101,22 +106,9 @@ def _wrap_blocks_in_place(blocks: list[Any], user_message: str) -> bool:
continue
for field_name in _WRAPPABLE_FIELDS:
value = block.get(field_name)
- if not isinstance(value, str) or not value:
- continue
- wrapped_parts = _extract_wrapped_goal(value)
- if wrapped_parts is not None:
- mini_goal, existing_main_goal = wrapped_parts
- main_goal = existing_main_goal if existing_main_goal == user_message else user_message
- next_value = MINI_GOAL_TEMPLATE.format(mini_goal=mini_goal, main_goal=main_goal)
- if next_value != value:
- block[field_name] = next_value
- mutated = True
- continue
- if _WRAPPED_PREFIX not in value:
- block[field_name] = MINI_GOAL_TEMPLATE.format(
- mini_goal=value,
- main_goal=user_message,
- )
+ next_value = _wrapped_field_value(value, user_message)
+ if next_value is not None:
+ block[field_name] = next_value
mutated = True
loop_blocks = block.get("loop_blocks")
if isinstance(loop_blocks, list):
@@ -144,3 +136,60 @@ def _extract_wrapped_goal(value: str) -> tuple[str, str] | None:
last = (mini_goal, main_goal)
current = mini_goal
return last
+
+
+def _wrapped_field_value(value: Any, main_goal: str) -> str | None:
+ if not isinstance(value, str) or not value:
+ return None
+ wrapped_parts = _extract_wrapped_goal(value)
+ if wrapped_parts is not None:
+ mini_goal, _existing_main_goal = wrapped_parts
+ next_value = MINI_GOAL_TEMPLATE.format(mini_goal=mini_goal, main_goal=main_goal)
+ return next_value if next_value != value else None
+ if _WRAPPED_PREFIX not in value:
+ return MINI_GOAL_TEMPLATE.format(mini_goal=value, main_goal=main_goal)
+ return None
+
+
+def wrap_workflow_block_goals(workflow: WorkflowT, main_goal: str) -> WorkflowT:
+ """Return a workflow copy with block intent fields wrapped for one runtime run."""
+ if not main_goal:
+ return workflow
+ workflow_definition = getattr(workflow, "workflow_definition", None)
+ blocks = getattr(workflow_definition, "blocks", None)
+ if not isinstance(blocks, list):
+ return workflow
+ if not _block_models_need_wrapping(blocks, main_goal):
+ return workflow
+
+ wrapped = workflow.model_copy(deep=True)
+ wrapped_blocks = getattr(wrapped.workflow_definition, "blocks", None)
+ if not isinstance(wrapped_blocks, list):
+ return workflow
+ return wrapped if _wrap_block_models_in_place(wrapped_blocks, main_goal) else workflow
+
+
+def _block_models_need_wrapping(blocks: list[Any], main_goal: str) -> bool:
+ for block in blocks:
+ for field_name in _WRAPPABLE_FIELDS:
+ if _wrapped_field_value(getattr(block, field_name, None), main_goal) is not None:
+ return True
+ loop_blocks = getattr(block, "loop_blocks", None)
+ if isinstance(loop_blocks, list) and _block_models_need_wrapping(loop_blocks, main_goal):
+ return True
+ return False
+
+
+def _wrap_block_models_in_place(blocks: list[Any], main_goal: str) -> bool:
+ mutated = False
+ for block in blocks:
+ for field_name in _WRAPPABLE_FIELDS:
+ value = getattr(block, field_name, None)
+ next_value = _wrapped_field_value(value, main_goal)
+ if next_value is not None:
+ setattr(block, field_name, next_value)
+ mutated = True
+ loop_blocks = getattr(block, "loop_blocks", None)
+ if isinstance(loop_blocks, list):
+ mutated = _wrap_block_models_in_place(loop_blocks, main_goal) or mutated
+ return mutated
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index 0386c5550..9c749e02b 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -23,7 +23,7 @@ from skyvern.forge import app
from skyvern.forge.failure_classifier import classify_from_failure_reason
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp
-from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_block_goals
+from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_workflow_block_goals
from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
@@ -2102,6 +2102,14 @@ def _watchdog_user_failure_reason(
return f"{body} Run ID: {workflow_run_id}. Outcome is uncertain."
+def _workflow_with_runtime_block_goal_context(workflow: Workflow, ctx: CopilotContext) -> Workflow:
+ block_goal_main_goal = ctx.block_goal_main_goal or ctx.user_message or ""
+ if not block_goal_main_goal:
+ LOG.warning("run_blocks invoked without block-goal context; using persisted workflow goals unchanged")
+ return workflow
+ return wrap_workflow_block_goals(workflow, block_goal_main_goal)
+
+
async def _run_blocks_and_collect_debug(
params: dict[str, Any],
ctx: CopilotContext,
@@ -2160,6 +2168,7 @@ async def _run_blocks_and_collect_debug(
return {"ok": False, "error": "Organization not found"}
organization = Organization.model_validate(org)
+ runtime_workflow = _workflow_with_runtime_block_goal_context(workflow, ctx)
user_params: dict[str, Any] = params.get("parameters") or {}
all_workflow_params, all_output_params = await asyncio.gather(
@@ -2251,6 +2260,7 @@ async def _run_blocks_and_collect_debug(
browser_session_id=ctx.browser_session_id,
block_labels=labels_to_execute,
block_outputs=block_outputs_to_seed or None,
+ workflow_override=runtime_workflow,
)
)
@@ -4318,15 +4328,6 @@ async def update_and_run_blocks_tool(
# the new one — we need the pre-update state to diff against.
prior_definition = await _get_prior_workflow_definition(copilot_ctx)
- # Wrap each block's navigation_goal / complete_criterion / terminate_criterion
- # with the effective "big goal" context so downstream LLMs (verifier,
- # validation-block prompt) have user-intent framing — mirrors TaskV2.
- block_goal_main_goal = copilot_ctx.block_goal_main_goal or copilot_ctx.user_message
- if block_goal_main_goal:
- workflow_yaml = wrap_block_goals(workflow_yaml, block_goal_main_goal)
- else:
- LOG.warning("update_and_run_blocks invoked without block-goal context; skipping block-goal wrap")
-
# Step 1: Update the workflow
with copilot_span("update_workflow", data={"yaml_length": len(workflow_yaml)}):
update_result = await _update_workflow(
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 69ccb6e07..25bbeba33 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -330,6 +330,10 @@ async def _persist_proposed_workflow_state(chat: Any, agent_result: AgentResult,
# auto-accept toggle.
await _clear_proposed_workflow(chat)
elif (
+ # This intentionally checks the raw setting, not
+ # ``auto_accept_effective``: no-proposal OutputPolicy blocks are not
+ # auto-applicable, but ordinary auto-accept turns still need to clear a
+ # stale unvalidated card because the UI has no review panel for it.
chat.auto_accept is True
and chat.proposed_workflow is not None
and chat.proposed_workflow.get("_copilot_unvalidated") is True
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 9acd3e4a6..b36c06672 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -1323,6 +1323,7 @@ class WorkflowService:
block_outputs: dict[str, Any] | None = None,
browser_session_id: str | None = None,
need_call_webhook: bool = True,
+ workflow_override: Workflow | None = None,
) -> WorkflowRun:
"""Execute a workflow.
@@ -1344,7 +1345,9 @@ class WorkflowService:
block_outputs=block_outputs,
)
workflow_run = await self.get_workflow_run(workflow_run_id=workflow_run_id, organization_id=organization_id)
- workflow = await self.get_workflow_by_permanent_id(workflow_permanent_id=workflow_run.workflow_permanent_id)
+ workflow = workflow_override or await self.get_workflow_by_permanent_id(
+ workflow_permanent_id=workflow_run.workflow_permanent_id
+ )
has_conditionals = workflow_script_service.workflow_has_conditionals(workflow)
browser_profile_id = workflow_run.browser_profile_id
close_browser_on_completion = browser_session_id is None and not workflow_run.browser_address
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 9e30fdcdf..18fe9c7c1 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -542,7 +542,7 @@ class TestBlockGoalMainGoal:
assert goal == "Use ` ` `this` ` ` safely."
- def test_structured_user_goal_added_as_prior_high_level_goal(self) -> None:
+ def test_correction_message_wins_over_structured_user_goal(self) -> None:
global_context = json.dumps(
{"user_goal": "Locate research about gravitational waves this week.", "workflow_state": "draft"}
)
@@ -553,43 +553,48 @@ class TestBlockGoalMainGoal:
global_llm_context=global_context,
)
- assert "Prior high-level goal:\nLocate research about gravitational waves this week." in goal
- assert "Latest user message:\nI meant black holes" in goal
- assert "Recent chat history" not in goal
+ assert goal == "I meant black holes"
- def test_plain_global_context_becomes_prior_high_level_goal(self) -> None:
+ def test_bare_confirmation_does_not_infer_structured_user_goal(self) -> None:
+ global_context = json.dumps({"user_goal": "Locate research about gravitational waves this week."})
+
+ goal = agent_module._build_block_goal_main_goal(
+ user_message="Yes, please.",
+ chat_history_text="user: Locate research about gravitational waves this week.",
+ global_llm_context=global_context,
+ )
+
+ assert goal == "Yes, please."
+
+ def test_current_message_wins_over_plain_global_context(self) -> None:
goal = agent_module._build_block_goal_main_goal(
user_message="I meant black holes",
chat_history_text="",
global_llm_context="Legacy goal with ```fenced``` context.",
)
- assert "Prior high-level goal:\nLegacy goal with ` ` `fenced` ` ` context." in goal
- assert "Latest user message:\nI meant black holes" in goal
+ assert goal == "I meant black holes"
- def test_chat_history_added_when_present(self) -> None:
+ def test_chat_history_is_not_denormalized_into_goal(self) -> None:
goal = agent_module._build_block_goal_main_goal(
user_message="I meant black holes",
chat_history_text="user: Search arXiv for recent papers.\nai: Drafted workflow.",
global_llm_context=None,
)
- assert "Recent chat history:\nuser: Search arXiv for recent papers." in goal
- assert "ai: Drafted workflow." in goal
- assert "Latest user message:\nI meant black holes" in goal
+ assert goal == "I meant black holes"
- def test_chat_history_and_latest_message_escape_code_fences(self) -> None:
+ def test_latest_message_escapes_code_fences_without_chat_history(self) -> None:
goal = agent_module._build_block_goal_main_goal(
user_message="I meant ```black holes```",
chat_history_text="user: Search ```arXiv``` for recent papers.",
global_llm_context=None,
)
- assert "user: Search ` ` `arXiv` ` ` for recent papers." in goal
- assert "Latest user message:\nI meant ` ` `black holes` ` `" in goal
+ assert goal == "I meant ` ` `black holes` ` `"
assert "```" not in goal
- def test_includes_both_structured_goal_and_chat_history(self) -> None:
+ def test_current_message_wins_over_chat_history_and_structured_goal(self) -> None:
global_context = json.dumps({"user_goal": "Find papers about gravitational waves."})
goal = agent_module._build_block_goal_main_goal(
@@ -598,10 +603,70 @@ class TestBlockGoalMainGoal:
global_llm_context=global_context,
)
- assert "Prior high-level goal:\nFind papers about gravitational waves." in goal
- assert "Recent chat history:" in goal
- assert "Latest user message:\nI meant neutron stars" in goal
- assert goal.find("Prior high-level goal") < goal.find("Recent chat history") < goal.find("Latest user message")
+ assert goal == "I meant neutron stars"
+
+
+class TestRuntimeBlockGoalPersistenceBoundary:
+ @pytest.mark.asyncio
+ async def test_update_and_run_blocks_persists_clean_yaml(self, monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot import tools as tools_module
+ from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
+
+ clean_yaml = """
+title: Test workflow
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: navigation
+ label: submit
+ navigation_goal: Submit the contact form.
+"""
+ captured: dict[str, str] = {}
+
+ async def fake_update_workflow(payload, ctx, allow_missing_credentials=False):
+ captured["workflow_yaml"] = payload["workflow_yaml"]
+ ctx.workflow_yaml = payload["workflow_yaml"]
+ workflow = _process_workflow_yaml(
+ workflow_id=ctx.workflow_id,
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ organization_id=ctx.organization_id,
+ workflow_yaml=payload["workflow_yaml"],
+ )
+ return {"ok": True, "_workflow": workflow, "data": {"block_count": 1}}
+
+ async def fake_run_blocks(params, ctx, **kwargs):
+ return {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr-1",
+ "overall_status": "completed",
+ "blocks": [],
+ },
+ }
+
+ monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: False)
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
+ monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)
+ monkeypatch.setattr(tools_module, "_pre_run_workflow_coverage_error", lambda *args: None)
+ monkeypatch.setattr(tools_module, "_plan_frontier", lambda *args: (["submit"], {}, "submit"))
+ monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", fake_run_blocks)
+ monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "enqueue_screenshot_from_result", lambda *args, **kwargs: None)
+
+ ctx = _ctx(
+ user_message="Submit a contact form.",
+ block_goal_main_goal="Submit a contact form.",
+ )
+ result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
+ json.dumps({"workflow_yaml": clean_yaml, "block_labels": ["submit"], "parameters": {}}),
+ )
+
+ assert json.loads(result)["ok"] is True
+ assert captured["workflow_yaml"] == clean_yaml
+ assert "Achieve the following mini goal" not in captured["workflow_yaml"]
class TestTranslateToAgentResultGating:
@@ -1030,23 +1095,14 @@ workflow_definition:
assert "test failed" in agent_result.user_response.lower()
assert "keep the draft" in agent_result.user_response.lower()
- def test_inline_replace_workflow_wraps_block_goals_with_user_message(self, monkeypatch) -> None:
- # SKY-9174 parity: update_and_run_blocks_tool wraps block goals with
- # the user's chat message as big-goal context. The REPLACE_WORKFLOW
- # inline path must do the same, otherwise the untested yaml latches
- # onto ctx without user-intent framing and any downstream block run
- # hits the verifier-on-confirmation-surface bug this PR fixes.
+ def test_inline_replace_workflow_persists_clean_agent_yaml(self, monkeypatch) -> None:
captured: dict[str, str] = {}
def fake_process(**kwargs):
captured["yaml"] = kwargs["workflow_yaml"]
return SimpleNamespace(name="new-wf")
- def fake_wrap(workflow_yaml: str, user_message: str) -> str:
- return f"WRAPPED::{user_message}::{workflow_yaml}"
-
monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", fake_process)
- monkeypatch.setattr("skyvern.forge.sdk.copilot.agent.wrap_block_goals", fake_wrap)
ctx = _ctx(user_message="Submit a contact form on example.com.")
result = _fake_run_result(
@@ -1056,21 +1112,17 @@ workflow_definition:
result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
)
- assert captured["yaml"] == "WRAPPED::Submit a contact form on example.com.::raw: yaml"
- assert ctx.last_workflow_yaml == "WRAPPED::Submit a contact form on example.com.::raw: yaml"
+ assert captured["yaml"] == "raw: yaml"
+ assert ctx.last_workflow_yaml == "raw: yaml"
- def test_inline_replace_workflow_prefers_resolved_block_goal_main_goal(self, monkeypatch) -> None:
+ def test_inline_replace_workflow_does_not_denormalize_resolved_goal(self, monkeypatch) -> None:
captured: dict[str, str] = {}
def fake_process(**kwargs):
captured["yaml"] = kwargs["workflow_yaml"]
return SimpleNamespace(name="new-wf")
- def fake_wrap(workflow_yaml: str, user_message: str) -> str:
- return f"WRAPPED::{user_message}::{workflow_yaml}"
-
monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", fake_process)
- monkeypatch.setattr("skyvern.forge.sdk.copilot.agent.wrap_block_goals", fake_wrap)
ctx = _ctx(
user_message="I meant black holes",
@@ -1083,7 +1135,7 @@ workflow_definition:
result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
)
- assert captured["yaml"] == "WRAPPED::Go to arXiv and find research about black holes.::raw: yaml"
+ assert captured["yaml"] == "raw: yaml"
def test_ask_question_with_verified_workflow_suppresses_and_clears(self) -> None:
# A verified-but-non-terminal workflow built this turn must not surface
diff --git a/tests/unit/test_copilot_block_goal_wrapping.py b/tests/unit/test_copilot_block_goal_wrapping.py
index 652ec2a84..20f5e1aa0 100644
--- a/tests/unit/test_copilot_block_goal_wrapping.py
+++ b/tests/unit/test_copilot_block_goal_wrapping.py
@@ -7,11 +7,13 @@ Covers wrapping of ``navigation_goal``, ``complete_criterion``, and
from __future__ import annotations
import textwrap
+from types import SimpleNamespace
import yaml
from skyvern.constants import MINI_GOAL_TEMPLATE
-from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_block_goals
+from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_block_goals, wrap_workflow_block_goals
+from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
USER_MESSAGE = "Submit a contact form on example.com with my details."
@@ -20,7 +22,7 @@ def _yaml_with_blocks(*blocks: dict) -> str:
return yaml.safe_dump(
{
"title": "test workflow",
- "workflow_definition": {"blocks": list(blocks)},
+ "workflow_definition": {"parameters": [], "blocks": list(blocks)},
},
sort_keys=False,
)
@@ -343,3 +345,104 @@ def test_returns_input_unchanged_when_blocks_not_list() -> None:
out = wrap_block_goals(src, USER_MESSAGE)
assert out == src
+
+
+def test_wrap_workflow_block_goals_wraps_runtime_copy_only() -> None:
+ workflow = _process_workflow_yaml(
+ workflow_id="w_test",
+ workflow_permanent_id="wpid_test",
+ organization_id="o_test",
+ workflow_yaml=_yaml_with_blocks(
+ {
+ "block_type": "navigation",
+ "label": "submit",
+ "navigation_goal": "Submit the contact form.",
+ "complete_criterion": "The confirmation page is visible.",
+ }
+ ),
+ )
+
+ wrapped = wrap_workflow_block_goals(workflow, USER_MESSAGE)
+
+ original_block = workflow.workflow_definition.blocks[0]
+ wrapped_block = wrapped.workflow_definition.blocks[0]
+ assert original_block.navigation_goal == "Submit the contact form."
+ assert original_block.complete_criterion == "The confirmation page is visible."
+ assert wrapped_block.navigation_goal == _wrapped("Submit the contact form.")
+ assert wrapped_block.complete_criterion == _wrapped("The confirmation page is visible.")
+ assert wrapped is not workflow
+
+
+def test_wrap_workflow_block_goals_recurses_into_loop_blocks() -> None:
+ workflow = _process_workflow_yaml(
+ workflow_id="w_test",
+ workflow_permanent_id="wpid_test",
+ organization_id="o_test",
+ workflow_yaml=yaml.safe_dump(
+ {
+ "title": "loop workflow",
+ "workflow_definition": {
+ "parameters": [
+ {
+ "parameter_type": "workflow",
+ "key": "items",
+ "workflow_parameter_type": "json",
+ "default_value": '["alpha"]',
+ }
+ ],
+ "blocks": [
+ {
+ "block_type": "for_loop",
+ "label": "loop",
+ "loop_over_parameter_key": "items",
+ "loop_blocks": [
+ {
+ "block_type": "navigation",
+ "label": "inner_nav",
+ "navigation_goal": "Process each item.",
+ },
+ {
+ "block_type": "validation",
+ "label": "inner_check",
+ "complete_criterion": "The item is processed.",
+ },
+ ],
+ }
+ ],
+ },
+ },
+ sort_keys=False,
+ ),
+ )
+
+ wrapped = wrap_workflow_block_goals(workflow, USER_MESSAGE)
+
+ original_loop_blocks = workflow.workflow_definition.blocks[0].loop_blocks
+ wrapped_loop_blocks = wrapped.workflow_definition.blocks[0].loop_blocks
+ assert original_loop_blocks[0].navigation_goal == "Process each item."
+ assert original_loop_blocks[1].complete_criterion == "The item is processed."
+ assert wrapped_loop_blocks[0].navigation_goal == _wrapped("Process each item.")
+ assert wrapped_loop_blocks[1].complete_criterion == _wrapped("The item is processed.")
+
+
+def test_wrap_workflow_block_goals_skips_copy_when_no_runtime_mutation_needed() -> None:
+ parsed_workflow = _process_workflow_yaml(
+ workflow_id="w_test",
+ workflow_permanent_id="wpid_test",
+ organization_id="o_test",
+ workflow_yaml=_yaml_with_blocks(
+ {
+ "block_type": "navigation",
+ "label": "submit",
+ "navigation_goal": _wrapped("Submit the contact form."),
+ "complete_criterion": _wrapped("The confirmation page is visible."),
+ }
+ ),
+ )
+
+ def fail_model_copy(*_args: object, **_kwargs: object) -> None:
+ raise AssertionError("model_copy should not run when block goals do not need wrapping")
+
+ workflow = SimpleNamespace(workflow_definition=parsed_workflow.workflow_definition, model_copy=fail_model_copy)
+
+ assert wrap_workflow_block_goals(workflow, USER_MESSAGE) is workflow