Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Stop MCP model-key hallucinations in workflow creation (#SKY-8302) (#5061)
Marc Kelechava87f18f4ee092
diff --git a/skyvern/cli/mcp_tools/__init__.py b/skyvern/cli/mcp_tools/__init__.pyindex 7385c9bd8..093825f29 100644--- a/skyvern/cli/mcp_tools/__init__.py+++ b/skyvern/cli/mcp_tools/__init__.py@@ -198,6 +198,7 @@ Validate blocks with skyvern_block_validate() before submitting.Split workflows into multiple blocks — one block per logical step — rather than cramming everything into a single block.Use **navigation** blocks for actions (filling forms, clicking buttons) and **extraction** blocks for pulling data.Do NOT use the deprecated "task" or "task_v2" block types — use "navigation" for actions and "extraction" for data extraction. These replacements give clearer semantics and are what the Skyvern UI uses. Existing workflows with task/task_v2 blocks will continue to work — do not convert them unless the user asks. New workflows must use navigation/extraction.+For **text_prompt** blocks, default to Skyvern Optimized by omitting both `model` and `llm_key`. If an explicit model is required, use `model: {"model_name": "<value from /models>"}`. Do not invent internal `llm_key` strings.GOOD (4 blocks, each with clear single responsibility):Block 1 (navigation): "Select Sole Proprietor and click Continue"diff --git a/skyvern/cli/mcp_tools/blocks.py b/skyvern/cli/mcp_tools/blocks.pyindex c0f0f4af8..d90bffc77 100644--- a/skyvern/cli/mcp_tools/blocks.py+++ b/skyvern/cli/mcp_tools/blocks.py@@ -187,6 +187,18 @@ BLOCK_EXAMPLES: dict[str, dict[str, Any]] = {"label": "wait_for_processing","wait_sec": 30,},+ "text_prompt": {+ "block_type": "text_prompt",+ "label": "summarize_results",+ "prompt": "Summarize {{ raw_results }} into a short customer-facing update",+ "parameter_keys": ["raw_results"],+ "json_schema": {+ "type": "object",+ "properties": {+ "summary": {"type": "string"},+ },+ },+ },"goto_url": {"block_type": "goto_url","label": "open_cart",diff --git a/skyvern/cli/mcp_tools/prompts.py b/skyvern/cli/mcp_tools/prompts.pyindex 0d55ecab2..d28446303 100644--- a/skyvern/cli/mcp_tools/prompts.py+++ b/skyvern/cli/mcp_tools/prompts.py@@ -90,6 +90,16 @@ Keep engine 1.0 (default, omit field) when:When in doubt, split into multiple 1.0 blocks rather than using one 2.0 block — it's cheaper andgives you per-block observability. Only navigation blocks support engine 2.0.+### Model selection for text_prompt blocks++Default to Skyvern Optimized for text_prompt blocks by omitting both `model` and `llm_key`.++If the user explicitly asks for a specific model, use the public `model` field:+ `"model": {"model_name": "<one of the values returned by /models>"}`++Do NOT invent internal `llm_key` strings like `ANTHROPIC_CLAUDE_3_5_SONNET`.+Only use `llm_key` when the user explicitly provides an exact internal key and wants that advanced override.+### One block per logical stepSplit workflows into small, focused blocks. Each block should do ONE thing.diff --git a/skyvern/cli/mcp_tools/workflow.py b/skyvern/cli/mcp_tools/workflow.pyindex 81d89ca87..deb01c6a9 100644--- a/skyvern/cli/mcp_tools/workflow.py+++ b/skyvern/cli/mcp_tools/workflow.py@@ -17,6 +17,7 @@ from pydantic import Fieldfrom skyvern.client.errors import NotFoundErrorfrom skyvern.client.types import WorkflowCreateYamlRequestfrom skyvern.schemas.runs import ProxyLocation+from skyvern.schemas.workflows import WorkflowCreateYAMLRequest as WorkflowCreateYAMLRequestSchemafrom ._common import ErrorCode, Timer, make_error, make_resultfrom ._session import get_skyvern@@ -206,6 +207,63 @@ _CODE_V2_DEFAULTS: dict[str, Any] = {}+def _deep_merge(base: Any, override: Any) -> Any:+ """Recursively merge normalized JSON-like data over the raw payload.++ Unknown fields should survive normalization. Lists are merged by index so+ overlapping items keep raw unknown keys even if normalization changes the+ list length.+ """++ if isinstance(base, dict) and isinstance(override, dict):+ result = dict(base)+ for key, value in override.items():+ if key in result:+ result[key] = _deep_merge(result[key], value)+ else:+ result[key] = value+ return result++ if isinstance(base, list) and isinstance(override, list):+ merged: list[Any] = []+ for idx in range(max(len(base), len(override))):+ if idx < len(base) and idx < len(override):+ merged.append(_deep_merge(base[idx], override[idx]))+ elif idx < len(override):+ merged.append(override[idx])+ else:+ merged.append(base[idx])+ return merged++ return override+++def _normalize_json_definition(raw: Any) -> WorkflowCreateYamlRequest:+ """Normalize JSON workflow definitions through the shared backend schema."""++ if not isinstance(raw, dict):+ raise TypeError("Workflow definition JSON must be an object")++ try:+ normalized = WorkflowCreateYAMLRequestSchema.model_validate(raw)+ except Exception as exc:+ # Internal schema is stricter than the Fern SDK — skip normalization so+ # unknown/future fields are not rejected.+ LOG.warning("Skipping text-prompt normalization; internal schema rejected payload", error=str(exc))+ return WorkflowCreateYamlRequest(**raw)++ merged = _deep_merge(raw, normalized.model_dump(mode="json"))+ return WorkflowCreateYamlRequest(**merged)+++def _make_invalid_json_definition_error(exc: Exception) -> dict[str, Any]:+ return make_error(+ ErrorCode.INVALID_INPUT,+ f"Invalid JSON definition: {exc}",+ "Provide a valid JSON object for the workflow definition",+ )++def _inject_code_v2_defaults(definition: str, fmt: str) -> str:"""Inject Code 2.0 defaults into a JSON definition string when not explicitly set.@@ -237,42 +295,28 @@ def _parse_definition(Exactly one of the first two will be set on success, or error on failure.JSON input is parsed into a WorkflowCreateYamlRequest (the type the SDK expects)."""+if fmt == "json":try:raw = json.loads(definition)- return WorkflowCreateYamlRequest(**raw), None, Noneexcept (json.JSONDecodeError, TypeError) as e:- return (- None,- None,- make_error(- ErrorCode.INVALID_INPUT,- f"Invalid JSON definition: {e}",- "Provide a valid JSON object for the workflow definition",- ),- )+ return None, None, _make_invalid_json_definition_error(e)+ try:+ return _normalize_json_definition(raw), None, Noneexcept Exception as e:- return (- None,- None,- make_error(- ErrorCode.INVALID_INPUT,- f"Invalid workflow definition: {e}",- "Check the workflow definition fields (title, workflow_definition with blocks)",- ),- )+ return None, None, _make_invalid_json_definition_error(e)elif fmt == "yaml":return None, definition, Noneelse:# auto: try JSON first, fall back to YAMLtry:raw = json.loads(definition)- return WorkflowCreateYamlRequest(**raw), None, Noneexcept (json.JSONDecodeError, TypeError):return None, definition, None- except Exception:- # JSON parsed but failed model validation — treat as YAML- return None, definition, None+ try:+ return _normalize_json_definition(raw), None, None+ except Exception as e:+ return None, None, _make_invalid_json_definition_error(e)# ---------------------------------------------------------------------------diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.pyindex 3daf40fb0..2a1e524aa 100644--- a/skyvern/schemas/workflows.py+++ b/skyvern/schemas/workflows.py@@ -1,4 +1,5 @@import abc+import functoolsfrom dataclasses import dataclassfrom enum import StrEnumfrom typing import Annotated, Any, Literal@@ -7,6 +8,8 @@ import structlogfrom pydantic import BaseModel, Field, field_validator, model_validatorfrom skyvern.config import settings+from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry+from skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType, WorkflowParameterTypefrom skyvern.schemas.runs import GeoTarget, ProxyLocation, RunEnginefrom skyvern.utils.strings import sanitize_identifier@@ -44,6 +47,26 @@ def sanitize_parameter_key(value: str) -> str:return sanitize_identifier(value, default="parameter")+def _has_jinja_syntax(value: str) -> bool:+ return "{{" in value or "{%" in value++[email protected]_cache(maxsize=1)+def _get_text_prompt_model_name_by_llm_key() -> dict[str, str]:+ """Build a reverse mapping from internal llm_key to public model_name.++ Cached because settings don't change at runtime. Tests that monkeypatch+ settings must call ``_get_text_prompt_model_name_by_llm_key.cache_clear()``+ to avoid cross-test pollution.+ """+ reverse_mapping: dict[str, str] = {}+ for model_name, metadata in SettingsManager.get_settings().get_model_name_to_llm_key().items():+ llm_key = metadata.get("llm_key")+ if llm_key and llm_key not in reverse_mapping:+ reverse_mapping[llm_key] = model_name+ return reverse_mapping++def _replace_references_in_value(value: Any, old_key: str, new_key: str) -> Any:"""Recursively replaces Jinja references in a value (string, dict, or list)."""if isinstance(value, str):@@ -679,6 +702,42 @@ class TextPromptBlockYAML(BlockYAML):parameter_keys: list[str] | None = Nonejson_schema: dict[str, Any] | None = None+ @model_validator(mode="after")+ def normalize_llm_selection(self) -> "TextPromptBlockYAML":+ raw_llm_key = self.llm_key.strip() if self.llm_key else None++ if self.model:+ # `model` is the stable public contract; ignore any raw llm_key override+ # once a model has been selected.+ self.llm_key = None+ return self++ if not raw_llm_key:+ self.llm_key = None+ return self++ if _has_jinja_syntax(raw_llm_key):+ self.llm_key = raw_llm_key+ return self++ model_name = _get_text_prompt_model_name_by_llm_key().get(raw_llm_key)+ if model_name:+ self.model = {"model_name": model_name}+ self.llm_key = None+ return self++ if raw_llm_key in LLMConfigRegistry.get_model_names():+ self.llm_key = raw_llm_key+ return self++ LOG.warning(+ "Unrecognized text prompt llm_key; defaulting to Skyvern Optimized/default model path",+ label=self.label,+ llm_key=raw_llm_key,+ )+ self.llm_key = None+ return self+class DownloadToS3BlockYAML(BlockYAML):# There is a mypy bug with Literal. Without the type: ignore, mypy will raise an error:diff --git a/skyvern/services/workflow_script_service.py b/skyvern/services/workflow_script_service.pyindex e1c12387d..72f37c097 100644--- a/skyvern/services/workflow_script_service.py+++ b/skyvern/services/workflow_script_service.py@@ -495,7 +495,8 @@ _FOR_LOOP_RE = re.compile(# nested quantifiers would cause exponential backtracking._PAGE_CLICK_START_RE = re.compile(r"await page\.click\(")# Matches a prompt='...' or prompt="..." keyword argument inside a function call.-_PROMPT_KWARG_RE = re.compile(r"""prompt\s*=\s*(['"])(.*?)\1""", re.DOTALL)+# Uses (?:\\.|…) to skip backslash-escaped quotes so apostrophes don't truncate the match.+_PROMPT_KWARG_RE = re.compile(r"""prompt\s*=\s*(['"])((?:\\.|(?!\1).)*?)\1""", re.DOTALL)def _find_click_calls(text: str) -> list[tuple[int, int, str]]:@@ -622,8 +623,10 @@ def _patch_static_clicks_in_block(body: str) -> str:if prompt_match:quote = prompt_match.group(1)original_prompt = prompt_match.group(2)- # Use an f-string so current_value is evaluated at runtime- new_prompt = f"prompt=f{quote}{original_prompt} Target: {{current_value}}{quote}"+ # Use an f-string so current_value is evaluated at runtime.+ # Escape existing braces so they are literal in the f-string.+ escaped_prompt = original_prompt.replace("{", "{{").replace("}", "}}")+ new_prompt = f"prompt=f{quote}{escaped_prompt} Target: {{current_value}}{quote}"patched = patched[: prompt_match.start()] + new_prompt + patched[prompt_match.end() :]else:# No prompt= kwarg — add one with current_value contextdiff --git a/tests/unit/test_fix_static_actions_in_for_loops.py b/tests/unit/test_fix_static_actions_in_for_loops.pyindex 602e62b5a..15246a469 100644--- a/tests/unit/test_fix_static_actions_in_for_loops.py+++ b/tests/unit/test_fix_static_actions_in_for_loops.py@@ -378,3 +378,31 @@ class TestFixStaticActionsInForLoops:breakelse:raise AssertionError("Expected a prompt=f line with current_value")++ def test_prompt_with_curly_braces_are_escaped(self) -> None:+ """Existing braces in prompt text must be escaped to avoid f-string evaluation."""+ body = textwrap.dedent("""\+ await page.click(+ selector='a.download-link',+ ai='fallback',+ prompt='Extract items matching {pattern}',+ )""")+ result = _patch_static_clicks_in_block(body)+ assert "ai='proactive'" in result+ # Original braces must be doubled so they're literal in the f-string+ assert "{{pattern}}" in result+ # The injected Target should still reference current_value+ assert "{current_value}" in result++ def test_two_level_nested_selector_is_matched(self) -> None:+ """Selectors like tr:has(td:has-text("Report")) must not break the regex."""+ body = textwrap.dedent("""\+ await page.click(+ selector='tr:has(td:has-text("Report"))',+ ai='fallback',+ prompt='Click row',+ )""")+ result = _patch_static_clicks_in_block(body)+ assert "ai='proactive'" in result+ assert "ai='fallback'" not in result+ assert "current_value" in resultdiff --git a/tests/unit/test_mcp_workflow_guidance.py b/tests/unit/test_mcp_workflow_guidance.pynew file mode 100644index 000000000..cec0a27d4--- /dev/null+++ b/tests/unit/test_mcp_workflow_guidance.py@@ -0,0 +1,29 @@+from __future__ import annotations++import pytest++from skyvern.cli.mcp_tools import mcp+from skyvern.cli.mcp_tools.blocks import skyvern_block_schema+from skyvern.cli.mcp_tools.prompts import BUILD_WORKFLOW_CONTENT+++def test_build_workflow_prompt_guides_text_prompt_defaults() -> None:+ assert "Default to Skyvern Optimized for text_prompt blocks by omitting both `model` and `llm_key`." in (+ BUILD_WORKFLOW_CONTENT+ )+ assert "Do NOT invent internal `llm_key` strings like `ANTHROPIC_CLAUDE_3_5_SONNET`." in BUILD_WORKFLOW_CONTENT+++def test_mcp_instructions_guide_text_prompt_defaults() -> None:+ assert "For **text_prompt** blocks, default to Skyvern Optimized by omitting both `model` and `llm_key`." in (+ mcp.instructions+ )+++async def test_text_prompt_block_schema_example_omits_raw_llm_key() -> None:+ result = await skyvern_block_schema(block_type="text_prompt")++ assert result["ok"] is True+ assert "llm_key" not in result["data"]["example"]+ assert "model" not in result["data"]["example"]diff --git a/tests/unit/test_mcp_workflow_tools.py b/tests/unit/test_mcp_workflow_tools.pynew file mode 100644index 000000000..793a3ff1d--- /dev/null+++ b/tests/unit/test_mcp_workflow_tools.py@@ -0,0 +1,335 @@+from __future__ import annotations++import json+from datetime import datetime, timezone+from types import SimpleNamespace+from unittest.mock import AsyncMock, patch++import pytest++import skyvern.cli.mcp_tools.workflow as workflow_tools+++def _fake_workflow_response() -> SimpleNamespace:+ now = datetime.now(timezone.utc)+ return SimpleNamespace(+ workflow_permanent_id="wpid_test",+ workflow_id="wf_test",+ title="Example Workflow",+ version=1,+ status="published",+ description=None,+ is_saved_task=False,+ folder_id=None,+ created_at=now,+ modified_at=now,+ )+++async def test_workflow_create_normalizes_invalid_text_prompt_llm_key(monkeypatch: pytest.MonkeyPatch) -> None:+ fake_client = SimpleNamespace(create_workflow=AsyncMock(return_value=_fake_workflow_response()))+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)++ definition = {+ "title": "Normalize invalid llm_key",+ "workflow_definition": {+ "blocks": [+ {+ "block_type": "text_prompt",+ "label": "summarize",+ "prompt": "Summarize the result",+ "llm_key": "ANTHROPIC_CLAUDE_3_5_SONNET",+ }+ ],+ "parameters": [],+ },+ }++ result = await workflow_tools.skyvern_workflow_create(definition=json.dumps(definition), format="json")++ sent_definition = fake_client.create_workflow.await_args.kwargs["json_definition"]+ sent_block = sent_definition.workflow_definition.blocks[0]++ assert result["ok"] is True+ assert sent_block.llm_key is None+ assert sent_block.model is None+ assert "ANTHROPIC_CLAUDE_3_5_SONNET" not in json.dumps(sent_definition.model_dump(mode="json"))+++async def test_workflow_create_preserves_explicit_internal_text_prompt_llm_key(monkeypatch: pytest.MonkeyPatch) -> None:+ fake_client = SimpleNamespace(create_workflow=AsyncMock(return_value=_fake_workflow_response()))+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)++ definition = {+ "title": "Preserve explicit internal llm_key",+ "workflow_definition": {+ "blocks": [+ {+ "block_type": "text_prompt",+ "label": "summarize",+ "prompt": "Summarize the result",+ "llm_key": "SPECIAL_INTERNAL_KEY",+ }+ ],+ "parameters": [],+ },+ }++ with patch(+ "skyvern.schemas.workflows.LLMConfigRegistry.get_model_names",+ return_value=["SPECIAL_INTERNAL_KEY"],+ ):+ result = await workflow_tools.skyvern_workflow_create(definition=json.dumps(definition), format="json")++ sent_definition = fake_client.create_workflow.await_args.kwargs["json_definition"]+ sent_block = sent_definition.workflow_definition.blocks[0]++ assert result["ok"] is True+ assert sent_block.model is None+ assert sent_block.llm_key == "SPECIAL_INTERNAL_KEY"+++# ---------------------------------------------------------------------------+# Prove the exact Slack scenario: MCP agent hallucinates various model strings+# ---------------------------------------------------------------------------+++ "hallucinated_key",+ [+ "ANTHROPIC_CLAUDE_3_5_SONNET", # exact key from the Slack thread+ "ANTHROPIC_CLAUDE3.5_SONNET", # the "correct" key Pedro mentioned — still not public+ "ANTHROPIC_CLAUDE_3_5_HAIKU",+ "OPENAI_GPT4_TURBO",+ "VERTEX_GEMINI_2_FLASH",+ "claude-3-opus-20240229",+ "gpt-4o-mini",+ "gemini-pro",+ ],+)+async def test_mcp_strips_all_common_hallucinated_llm_keys(+ monkeypatch: pytest.MonkeyPatch, hallucinated_key: str+) -> None:+ """MCP workflow creation must strip ANY hallucinated llm_key and default to Skyvern Optimized (null)."""+ fake_client = SimpleNamespace(create_workflow=AsyncMock(return_value=_fake_workflow_response()))+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)++ definition = {+ "title": "Agent-generated workflow",+ "workflow_definition": {+ "blocks": [+ {+ "block_type": "text_prompt",+ "label": "limit_pages",+ "prompt": "Extract only the first 10 results",+ "llm_key": hallucinated_key,+ }+ ],+ "parameters": [],+ },+ }++ with patch(+ "skyvern.schemas.workflows.LLMConfigRegistry.get_model_names",+ return_value=[], # simulate: none of these are registered+ ):+ result = await workflow_tools.skyvern_workflow_create(definition=json.dumps(definition), format="auto")++ sent_def = fake_client.create_workflow.await_args.kwargs["json_definition"]+ sent_block = sent_def.workflow_definition.blocks[0]++ assert result["ok"] is True+ assert sent_block.llm_key is None, f"hallucinated key {hallucinated_key!r} was NOT stripped"+ assert sent_block.model is None, "should default to Skyvern Optimized (null model)"+++async def test_workflow_create_preserves_unknown_fields(monkeypatch: pytest.MonkeyPatch) -> None:+ """Fields not in the internal schema should survive normalization.++ The Fern-generated WorkflowCreateYamlRequest uses extra='allow', so unknown+ fields are accepted. Our normalization deep-merges the original raw dict with+ the normalized output so that future SDK fields not yet mirrored in the+ internal schema are preserved at any nesting depth.+ """+ fake_client = SimpleNamespace(create_workflow=AsyncMock(return_value=_fake_workflow_response()))+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)++ definition = {+ "title": "Unknown fields test",+ "some_future_sdk_field": "should_survive",+ "workflow_definition": {+ "parameters": [],+ "some_nested_future_field": "also_survives",+ "blocks": [+ {+ "block_type": "text_prompt",+ "label": "summarize",+ "prompt": "Summarize",+ }+ ],+ },+ }++ result = await workflow_tools.skyvern_workflow_create(definition=json.dumps(definition), format="json")+ assert result["ok"] is True++ sent = fake_client.create_workflow.await_args.kwargs["json_definition"]+ # Top-level unknown field preserved+ assert sent.some_future_sdk_field == "should_survive"+ # Nested unknown field inside workflow_definition also preserved via deep merge+ wd = sent.workflow_definition+ wd_dict = wd.model_dump(mode="json") if hasattr(wd, "model_dump") else wd.__dict__+ assert wd_dict.get("some_nested_future_field") == "also_survives"+++async def test_workflow_create_preserves_block_level_unknown_fields(monkeypatch: pytest.MonkeyPatch) -> None:+ """Unknown fields inside individual block dicts survive normalization via deep merge."""+ fake_client = SimpleNamespace(create_workflow=AsyncMock(return_value=_fake_workflow_response()))+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)++ definition = {+ "title": "Block-level unknown fields test",+ "workflow_definition": {+ "parameters": [],+ "blocks": [+ {+ "block_type": "text_prompt",+ "label": "summarize",+ "prompt": "Summarize",+ "some_future_block_field": 42,+ }+ ],+ },+ }++ result = await workflow_tools.skyvern_workflow_create(definition=json.dumps(definition), format="json")+ assert result["ok"] is True++ sent = fake_client.create_workflow.await_args.kwargs["json_definition"]+ sent_block = sent.workflow_definition.blocks[0]+ block_dict = sent_block.model_dump(mode="json") if hasattr(sent_block, "model_dump") else sent_block.__dict__+ assert block_dict.get("some_future_block_field") == 42+++async def test_workflow_create_falls_back_on_schema_validation_error(monkeypatch: pytest.MonkeyPatch) -> None:+ """If the internal schema rejects the payload, normalization is skipped and the raw dict is forwarded."""+ fake_client = SimpleNamespace(create_workflow=AsyncMock(return_value=_fake_workflow_response()))+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)++ definition = {+ "title": "Schema rejection test",+ "workflow_definition": {+ "parameters": [],+ "blocks": [+ {+ "block_type": "text_prompt",+ "label": "summarize",+ "prompt": "Summarize",+ "llm_key": "HALLUCINATED_KEY",+ }+ ],+ },+ }++ with patch(+ "skyvern.cli.mcp_tools.workflow.WorkflowCreateYAMLRequestSchema.model_validate",+ side_effect=Exception("schema rejected"),+ ):+ result = await workflow_tools.skyvern_workflow_create(definition=json.dumps(definition), format="json")++ assert result["ok"] is True+ sent = fake_client.create_workflow.await_args.kwargs["json_definition"]+ # Normalization was skipped, so the hallucinated key passes through to the SDK+ sent_block = sent.workflow_definition.blocks[0]+ block_dict = sent_block.model_dump(mode="json") if hasattr(sent_block, "model_dump") else sent_block.__dict__+ assert block_dict.get("llm_key") == "HALLUCINATED_KEY"++[email protected]("format_name", ["json", "auto"])+def test_parse_definition_returns_invalid_input_when_schema_fallback_still_fails(format_name: str) -> None:+ """If both the internal schema and Fern fallback reject the JSON, return a structured INVALID_INPUT error."""++ json_def, yaml_def, err = workflow_tools._parse_definition(+ json.dumps({"title": "Missing workflow_definition"}),+ format_name,+ )++ assert json_def is None+ assert yaml_def is None+ assert err is not None+ assert err["code"] == workflow_tools.ErrorCode.INVALID_INPUT+ assert "Invalid JSON definition" in err["message"]+++def test_deep_merge_preserves_block_unknown_fields_when_list_lengths_differ() -> None:+ """Overlapping block data should still merge when normalization changes list length."""++ raw = {+ "workflow_definition": {+ "blocks": [+ {+ "block_type": "text_prompt",+ "label": "summarize",+ "prompt": "Summarize",+ "some_future_block_field": 42,+ }+ ]+ }+ }+ normalized = {+ "workflow_definition": {+ "blocks": [+ {+ "block_type": "text_prompt",+ "label": "summarize",+ "prompt": "Summarize",+ },+ {+ "block_type": "text_prompt",+ "label": "synthetic_followup",+ "prompt": "Follow up",+ },+ ]+ }+ }++ merged = workflow_tools._deep_merge(raw, normalized)+ merged_blocks = merged["workflow_definition"]["blocks"]++ assert len(merged_blocks) == 2+ assert merged_blocks[0]["some_future_block_field"] == 42+ assert merged_blocks[1]["label"] == "synthetic_followup"+++async def test_mcp_text_prompt_without_llm_key_stays_null(monkeypatch: pytest.MonkeyPatch) -> None:+ """When MCP correctly omits llm_key (Skyvern Optimized), it stays null through the whole pipeline."""+ fake_client = SimpleNamespace(create_workflow=AsyncMock(return_value=_fake_workflow_response()))+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)++ definition = {+ "title": "Well-behaved MCP workflow",+ "workflow_definition": {+ "blocks": [+ {+ "block_type": "text_prompt",+ "label": "summarize",+ "prompt": "Summarize the extracted data",+ }+ ],+ "parameters": [],+ },+ }++ result = await workflow_tools.skyvern_workflow_create(definition=json.dumps(definition), format="json")+ sent_block = fake_client.create_workflow.await_args.kwargs["json_definition"].workflow_definition.blocks[0]++ assert result["ok"] is True+ assert sent_block.llm_key is None+ assert sent_block.model is Nonediff --git a/tests/unit/test_text_prompt_block.py b/tests/unit/test_text_prompt_block.pyindex 18bed8ff5..f1f661cdc 100644--- a/tests/unit/test_text_prompt_block.py+++ b/tests/unit/test_text_prompt_block.py@@ -9,6 +9,7 @@ from skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.workflow.models.block import TextPromptBlockfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType+from skyvern.schemas.workflows import TextPromptBlockYAML, WorkflowRequestblock_module = sys.modules["skyvern.forge.sdk.workflow.models.block"]@@ -212,3 +213,177 @@ async def test_text_prompt_block_prefers_prompt_type_config_over_secondary(monkeassert captured["default_handler"] == prompt_config_handlerprompt_config_handler.assert_awaited_once()assert response == {"llm_response": "config"}+++async def test_text_prompt_block_bad_llm_key_uses_same_runtime_path_as_no_override(monkeypatch):+ monkeypatch.setattr(SettingsManager, "_SettingsManager__instance", base_settings)+ now = datetime.now(timezone.utc)++ normalized_bad = TextPromptBlockYAML(+ label="bad_key",+ prompt="Summarize status.",+ llm_key="ANTHROPIC_CLAUDE_3_5_SONNET",+ )+ no_override = TextPromptBlockYAML(+ label="no_override",+ prompt="Summarize status.",+ llm_key=None,+ )++ blocks = []+ for idx, yaml_block in enumerate((normalized_bad, no_override), start=1):+ output_parameter = OutputParameter(+ parameter_type=ParameterType.OUTPUT,+ key=f"text_prompt_output_{idx}",+ description=None,+ output_parameter_id=f"output-{idx}",+ workflow_id="workflow-1",+ created_at=now,+ modified_at=now,+ deleted_at=None,+ )+ blocks.append(+ TextPromptBlock(+ label=yaml_block.label,+ llm_key=yaml_block.llm_key,+ prompt=yaml_block.prompt,+ parameters=[],+ json_schema=None,+ output_parameter=output_parameter,+ model=yaml_block.model,+ )+ )++ captured: list[tuple[str | None, object]] = []+ fake_secondary_handler = AsyncMock(return_value={"llm_response": "secondary"})++ async def fake_prompt_type_handler(*args, **kwargs):+ return None++ def fake_get_override_handler(llm_key: str | None, *, default):+ captured.append((llm_key, default))+ return default++ block_module.app.SECONDARY_LLM_API_HANDLER = fake_secondary_handler+ block_module.app.LLM_API_HANDLER = AsyncMock()+ LLMAPIHandlerFactory = block_module.LLMAPIHandlerFactory+ monkeypatch.setattr(+ LLMAPIHandlerFactory,+ "get_override_llm_api_handler",+ fake_get_override_handler,+ raising=False,+ )+ monkeypatch.setattr(+ block_module,+ "get_llm_handler_for_prompt_type",+ fake_prompt_type_handler,+ raising=False,+ )+ monkeypatch.setattr(+ prompt_engine,+ "load_prompt_from_string",+ lambda template, **kwargs: template,+ )++ for block in blocks:+ response = await block.send_prompt(block.prompt, {}, workflow_run_id="workflow-run", organization_id="org-1")+ assert response == {"llm_response": "secondary"}++ assert captured == [+ (None, fake_secondary_handler),+ (None, fake_secondary_handler),+ ]+++async def test_text_prompt_block_uses_explicit_internal_llm_key_override(monkeypatch):+ now = datetime.now(timezone.utc)+ output_parameter = OutputParameter(+ parameter_type=ParameterType.OUTPUT,+ key="text_prompt_output_internal",+ description=None,+ output_parameter_id="output-internal",+ workflow_id="workflow-1",+ created_at=now,+ modified_at=now,+ deleted_at=None,+ )++ block = TextPromptBlock(+ label="text-block",+ llm_key="SPECIAL_INTERNAL_KEY",+ prompt="Summarize status.",+ parameters=[],+ json_schema=None,+ output_parameter=output_parameter,+ model=None,+ )++ captured: dict[str, object] = {}+ fake_default_handler = AsyncMock()+ fake_override_handler = AsyncMock(return_value={"llm_response": "override"})++ async def fake_resolve_default_llm_handler(*args, **kwargs):+ return fake_default_handler++ def fake_get_override_handler(llm_key: str | None, *, default):+ captured["llm_key"] = llm_key+ captured["default_handler"] = default+ return fake_override_handler++ block_module.app.LLM_API_HANDLER = fake_default_handler+ LLMAPIHandlerFactory = block_module.LLMAPIHandlerFactory+ monkeypatch.setattr(+ LLMAPIHandlerFactory,+ "get_override_llm_api_handler",+ fake_get_override_handler,+ raising=False,+ )+ monkeypatch.setattr(+ TextPromptBlock,+ "_resolve_default_llm_handler",+ fake_resolve_default_llm_handler,+ raising=False,+ )+ monkeypatch.setattr(+ prompt_engine,+ "load_prompt_from_string",+ lambda template, **kwargs: template,+ )++ response = await block.send_prompt(block.prompt, {}, workflow_run_id="workflow-run", organization_id="org-1")++ assert captured["llm_key"] == "SPECIAL_INTERNAL_KEY"+ assert captured["default_handler"] == fake_default_handler+ fake_override_handler.assert_awaited_once()+ assert response == {"llm_response": "override"}+++def test_workflow_request_deserialization_strips_invalid_text_prompt_llm_key() -> None:+ """Verify FastAPI deserialization (not just explicit model_validate) strips bad keys.++ Moved from tests/scenario/ — this test only validates Pydantic behavior and+ does not require a database connection.+ """+ raw_request = {+ "json_definition": {+ "title": "Deserialization test",+ "workflow_definition": {+ "blocks": [+ {+ "block_type": "text_prompt",+ "label": "summarize",+ "prompt": "Summarize the result",+ "llm_key": "ANTHROPIC_CLAUDE_3_5_SONNET",+ }+ ],+ "parameters": [],+ },+ }+ }+ workflow_request = WorkflowRequest.model_validate(raw_request)++ block = workflow_request.json_definition.workflow_definition.blocks[0]+ assert block.llm_key is None+ assert block.model is Nonediff --git a/tests/unit/test_text_prompt_block_yaml.py b/tests/unit/test_text_prompt_block_yaml.pynew file mode 100644index 000000000..6235df0cd--- /dev/null+++ b/tests/unit/test_text_prompt_block_yaml.py@@ -0,0 +1,88 @@+from __future__ import annotations++from unittest.mock import patch++import pytest++from skyvern.config import settings as base_settings+from skyvern.forge.sdk.settings_manager import SettingsManager+from skyvern.schemas.workflows import TextPromptBlockYAML, _get_text_prompt_model_name_by_llm_key++[email protected](autouse=True)+def _clear_llm_key_cache():+ """Clear the lru_cache before each test to prevent cross-test pollution."""+ _get_text_prompt_model_name_by_llm_key.cache_clear()+ yield+ _get_text_prompt_model_name_by_llm_key.cache_clear()+++class TestTextPromptBlockYAMLNormalization:+ def test_converts_known_llm_key_to_model(self, monkeypatch) -> None:+ monkeypatch.setattr(SettingsManager, "_SettingsManager__instance", base_settings)++ block = TextPromptBlockYAML(+ label="summarize",+ prompt="Summarize the data.",+ llm_key="VERTEX_GEMINI_2.5_FLASH",+ )++ assert block.model == {"model_name": "gemini-2.5-flash"}+ assert block.llm_key is None++ def test_clears_invalid_llm_key_to_use_default_model(self, monkeypatch) -> None:+ monkeypatch.setattr(SettingsManager, "_SettingsManager__instance", base_settings)++ with patch(+ "skyvern.schemas.workflows.LLMConfigRegistry.get_model_names",+ return_value=[],+ ):+ block = TextPromptBlockYAML(+ label="summarize",+ prompt="Summarize the data.",+ llm_key="ANTHROPIC_CLAUDE_3_5_SONNET",+ )++ assert block.model is None+ assert block.llm_key is None++ def test_preserves_registered_advanced_llm_key(self, monkeypatch) -> None:+ monkeypatch.setattr(SettingsManager, "_SettingsManager__instance", base_settings)++ with patch(+ "skyvern.schemas.workflows.LLMConfigRegistry.get_model_names",+ return_value=["SPECIAL_INTERNAL_KEY"],+ ):+ block = TextPromptBlockYAML(+ label="summarize",+ prompt="Summarize the data.",+ llm_key="SPECIAL_INTERNAL_KEY",+ )++ assert block.model is None+ assert block.llm_key == "SPECIAL_INTERNAL_KEY"++ def test_preserves_templated_llm_key(self, monkeypatch) -> None:+ monkeypatch.setattr(SettingsManager, "_SettingsManager__instance", base_settings)++ block = TextPromptBlockYAML(+ label="summarize",+ prompt="Summarize the data.",+ llm_key="{{ prompt_block_llm_key }}",+ )++ assert block.model is None+ assert block.llm_key == "{{ prompt_block_llm_key }}"++ def test_model_override_clears_raw_llm_key(self, monkeypatch) -> None:+ monkeypatch.setattr(SettingsManager, "_SettingsManager__instance", base_settings)++ block = TextPromptBlockYAML(+ label="summarize",+ prompt="Summarize the data.",+ llm_key="ANTHROPIC_CLAUDE_3_5_SONNET",+ model={"model_name": "gemini-3-pro-preview"},+ )++ assert block.model == {"model_name": "gemini-3-pro-preview"}+ assert block.llm_key is None