Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-10075): tiered slim LLM output prompts (slim_safe / slim_terse) (#6392)
pedrohsdb634dc11e0ab1
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 2ea3eb067..a00ca0bd1 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -99,6 +99,7 @@ from skyvern.forge.sdk.event.factory import EventStrategyFactoryfrom skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_contextfrom skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handlerfrom skyvern.forge.sdk.experimentation.screenshot_downscale import resolve_screenshot_downscale_for_context+from skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_valuefrom skyvern.forge.sdk.log_artifacts import save_step_logs, save_task_logsfrom skyvern.forge.sdk.models import SpeculativeLLMMetadata, Step, StepStatusfrom skyvern.forge.sdk.schemas.files import FileInfo@@ -2811,6 +2812,8 @@ class ForgeAgent:template_name = "check-user-goal-with-termination" if use_termination_prompt else "check-user-goal"prompt_name = "check-user-goal-with-termination" if use_termination_prompt else "check-user-goal"+ slim_output = await get_slim_output_template_value(template_name)+# SKY-9718 Layer 1: gate the lean recipe on the PostHog flag at the# call site. complete_verify only needs the visual page state to# decide is_complete / is_terminate / continue — no element-id refs,@@ -2827,6 +2830,7 @@ class ForgeAgent:complete_criterion=task.complete_criterion,terminate_criterion=task.terminate_criterion,action_history=actions_and_results_str,+ slim_output=slim_output,local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),without_screenshots=not llm_screenshots_enabled,html_need_skyvern_attrs=False,@@ -3458,11 +3462,12 @@ class ForgeAgent:complete_criterion: str | None,enriched_tree_enabled: bool = False,llm_screenshots_enabled: bool = True,+ slim_output: str | None = None,) -> str:"""Build a short-but-unique cache variant identifier so extract-action prompts that- differ meaningfully (OTP, close-page availability, complete criteria) do not reuse- the same Vertex cache object.+ differ meaningfully (OTP, close-page availability, complete criteria, slim output)+ do not reuse the same Vertex cache object."""variant_parts: list[str] = []if verification_code_check:@@ -3477,6 +3482,8 @@ class ForgeAgent:normalized = " ".join(complete_criterion.split())digest = hashlib.sha256(normalized.encode("utf-8"), usedforsecurity=False).hexdigest()[:6]variant_parts.append(f"cc{digest}")+ if slim_output:+ variant_parts.append(f"slim_{slim_output}")return "-".join(variant_parts) if variant_parts else "std"async def _create_vertex_cache_for_task(@@ -3692,6 +3699,8 @@ class ForgeAgent:context = skyvern_context.ensure_context()+ slim_output = await get_slim_output_template_value(template)+# Reset cached prompt and cache reference by default; we will set them below if caching is enabled.# This prevents extract-action cache from being attached to other prompts like decisive-criterion-validate.context.cached_static_prompt = None@@ -3767,6 +3776,7 @@ class ForgeAgent:"recent_dialog_messages_str": recent_dialog_messages_str,"llm_screenshots_enabled": llm_screenshots_enabled,"enriched_tree_enabled": enriched_tree_enabled,+ "slim_output": slim_output,}cache_variant = self._build_extract_action_cache_variant(verification_code_check=verification_code_check,@@ -3774,6 +3784,7 @@ class ForgeAgent:complete_criterion=task.complete_criterion.strip() if task.complete_criterion else None,enriched_tree_enabled=enriched_tree_enabled,llm_screenshots_enabled=llm_screenshots_enabled,+ slim_output=slim_output,)static_prompt = prompt_engine.load_prompt(f"{template}-static", **prompt_kwargs)dynamic_prompt = prompt_engine.load_prompt(@@ -3863,6 +3874,7 @@ class ForgeAgent:recent_dialog_messages_str=recent_dialog_messages_str,llm_screenshots_enabled=llm_screenshots_enabled,enriched_tree_enabled=enriched_tree_enabled,+ slim_output=slim_output,lean_compress_long_href=False,lean_compress_image_src=context.enable_lean_element_tree,lean_strip_url_query_strings=context.enable_lean_element_tree,diff --git a/skyvern/forge/prompts/skyvern/auto-completion-choose-option.j2 b/skyvern/forge/prompts/skyvern/auto-completion-choose-option.j2index 367d524f4..bd808fa36 100644--- a/skyvern/forge/prompts/skyvern/auto-completion-choose-option.j2+++ b/skyvern/forge/prompts/skyvern/auto-completion-choose-option.j2@@ -16,10 +16,12 @@ Each interactable element is tagged with an ID.Reply in JSON format with the following keys:{- "thought": str, // Think step by step. Describe your thought about how you achieve the {{ "three" if is_search else "two" }} goals with convincing evidence.{% if is_search %}+{%- if slim_output not in ('safe', 'terse') %}+ "thought": str, // Think step by step. Describe your thought about how you achieve the {{ "three" if is_search else "two" }} goals with convincing evidence.{% endif %}{% if is_search %}"direct_searching": bool, // True if direct searching is a better way compared to all suggestions, otherwise False.{% endif %}"auto_completion_attempt": bool, // True if there's any auto completion attempt based on the rules. Otherwise, it should be False.- "reasoning": str, // The reasoning behind the decision. Be specific, referencing the value and the element id in your reasoning. Mention why you chose the element id. Keep the reasoning short and to the point.+{%- if slim_output not in ('safe', 'terse') %}+ "reasoning": str, // The reasoning behind the decision. Be specific, referencing the value and the element id in your reasoning. Mention why you chose the element id. Keep the reasoning short and to the point.{% endif %}"confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence."relevance_float": float, // The relative between the selected element and the provided information. You should consider how much the selected option is related to the user goal, the user details and the context. Pick a number between 0.00 and 1.00. 0.00 means no relevance, 1.00 means full relevance, the precision is 0.01."value": str, // The value to select.diff --git a/skyvern/forge/prompts/skyvern/check-user-goal-with-termination.j2 b/skyvern/forge/prompts/skyvern/check-user-goal-with-termination.j2index 2861c2a60..01f713c76 100644--- a/skyvern/forge/prompts/skyvern/check-user-goal-with-termination.j2+++ b/skyvern/forge/prompts/skyvern/check-user-goal-with-termination.j2@@ -3,8 +3,9 @@ You are here to help the user determine if the user has completed their goal onMake sure to ONLY return the JSON object in this format with no additional text before or after it:```json{- "page_info": str, // Think step by step. Describe all the useful information in the page related to the user goal.- "thoughts": str, // Think step by step. Explain your reasoning for the status you selected.+{%- if slim_output not in ('safe', 'terse') %}+ "page_info": str, // Think step by step. Describe all the useful information in the page related to the user goal.{% endif %}+ "thoughts": str, // Think step by step. Explain your reasoning for the status you selected.{{ " Maximum 15 words, telegraphic style — drop articles and filler words." if slim_output == 'terse' else "" }}"status": str, // Must be one of three values: "complete", "terminate", or "continue". Use "complete" ONLY if the user goal has been fully achieved{{ " according to the complete criterion" if complete_criterion else "" }}. Use "terminate" ONLY if the goal CANNOT ever be achieved (e.g., a file doesn't exist, an error modal blocks progress permanently, or an explicit termination condition is met in the user goal). Use "continue" if the goal is not yet achieved but more steps could potentially achieve it (this is the most common case - use this when you need to wait, navigate, or try different actions)."failure_categories": array // Only populate when status is "terminate". Classify the root cause. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), INFRASTRUCTURE_ERROR (network, OOM), UNKNOWN (cannot determine). Return one or more categories ordered by likelihood.[{diff --git a/skyvern/forge/prompts/skyvern/check-user-goal.j2 b/skyvern/forge/prompts/skyvern/check-user-goal.j2index e47327e40..dc4f92d22 100644--- a/skyvern/forge/prompts/skyvern/check-user-goal.j2+++ b/skyvern/forge/prompts/skyvern/check-user-goal.j2@@ -3,8 +3,9 @@ Your are here to help the user determine if the user has completed their goal onMake sure to ONLY return the JSON object in this format with no additional text before or after it:```json{- "page_info": str, // Think step by step. Describe all the useful information in the page related to the user goal.- "thoughts": str, // Think step by step. What information makes you believe whether user goal has completed or not. Use information you see on the site to explain.+{%- if slim_output not in ('safe', 'terse') %}+ "page_info": str, // Think step by step. Describe all the useful information in the page related to the user goal.{% endif %}+ "thoughts": str, // Think step by step. What information makes you believe whether user goal has completed or not. Use information you see on the site to explain.{{ " Maximum 15 words, telegraphic style — drop articles and filler words." if slim_output == 'terse' else "" }}"user_goal_achieved": bool // True if the user goal has been completed{{ " according to the complete criterion" if complete_criterion else "" }}, false otherwise.}```diff --git a/skyvern/forge/prompts/skyvern/extract-action-static.j2 b/skyvern/forge/prompts/skyvern/extract-action-static.j2index e8c656cda..cb3e4d401 100644--- a/skyvern/forge/prompts/skyvern/extract-action-static.j2+++ b/skyvern/forge/prompts/skyvern/extract-action-static.j2@@ -11,12 +11,13 @@ If you see a popup in the page screenshot, prioritize actions on the popup.Reply in JSON format with the following keys:{+{%- if slim_output not in ('safe', 'terse') %}"user_goal_stage": str, // A string to describe the reasoning whether user goal has been achieved or not."user_goal_achieved": bool, // True if the user goal has been completed, otherwise False.- "action_plan": str, // A string that describes the plan of actions you're going to take. Be specific and to the point. Use this as a quick summary of the actions you're going to take, and what order you're going to take them in, and how that moves you towards your overall goal. Output "COMPLETE" action in the "actions" if user_goal_achieved is True. Output "TERMINATE" action in the "actions" if your plan is to terminate the process.+ "action_plan": str, // A string that describes the plan of actions you're going to take. Be specific and to the point. Use this as a quick summary of the actions you're going to take, and what order you're going to take them in, and how that moves you towards your overall goal. Output "COMPLETE" action in the "actions" if user_goal_achieved is True. Output "TERMINATE" action in the "actions" if your plan is to terminate the process.{% endif %}"actions": array // An array of actions. Here's the format of each action:[{- "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.+ "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.{{ " Maximum 15 words, telegraphic style — drop articles and filler words." if slim_output == 'terse' else "" }}"user_detail_query": str, // Think of this value as a Jeopardy question and the intention behind the action. Ask the user for the details you need for executing this action. IMPORTANT: This question must be user information agnostic and must NOT contain specific user data (like names, IDs, specific values, etc.). Ask what information is needed generically. Ask the question even if the details are disclosed in user goal or user details. If it's a text field, ask for the text. If it's a file upload, ask for the file. If it's a dropdown, ask for the relevant information. If you are clicking on something specific, ask about what the intention is behind the click and what to click on. If you're downloading a file and you have multiple options, ask the user which one to download. Examples are: "What product ID should I input into the search bar?", "What file should I upload?", "What is the previous insurance provider of the user?", "Which invoice should I download?", "Does the user have any pets?", "Which member should I take action on?". NEVER include specific user data in this question - keep it generic so it can be answered with different user contexts."user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user goal or user details. This answer CAN contain specific user data."confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence@@ -36,11 +37,13 @@ Reply in JSON format with the following keys:"value": str // the value of the option. MAKE SURE YOU USE THIS VALUE TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION VALUE HERE},"click_context": { // The context for CLICK action only. null if not CLICK action- "thought": str, // Describe how you decided that this action is a single choice option or multi-choice option.+{%- if slim_output not in ('safe', 'terse') %}+ "thought": str, // Describe how you decided that this action is a single choice option or multi-choice option.{% endif %}"single_option_click": bool, // True if the click is the only choice to proceed towards the goal, regardless of different user context or input. False if there are multiple valid options that depend on user input. Examples: clicking a login button to login is True (it's the only way to login); clicking a radio button for a multi-choice question (e.g., selecting "male", "female", or "other" for gender) is False (the choice depends on user input). When clicking on radio buttons, dropdown options, or any element that represents one of multiple possible selections, this should be False. If the click is intended to open the dropdown menu in order to select an option from the dropdown, this should be False.},"context": { // The context for INPUT_TEXT or SELECT_OPTION action only. null if not INPUT_TEXT or SELECT_OPTION action. Extract the following detailed information from the "reasoning", and double-check the information by analysing the HTML elements.- "thought": str, // A string to describe how you double-check the context information to ensure the accuracy.+{%- if slim_output not in ('safe', 'terse') %}+ "thought": str, // A string to describe how you double-check the context information to ensure the accuracy.{% endif %}"field": str, // Which field is this action intended to fill out?"is_required": bool, // True if this is a required field, otherwise false."is_search_bar": bool, // True if the element to take the action is a search bar, otherwise false.diff --git a/skyvern/forge/prompts/skyvern/extract-action.j2 b/skyvern/forge/prompts/skyvern/extract-action.j2index a69fe89fa..1f1158502 100644--- a/skyvern/forge/prompts/skyvern/extract-action.j2+++ b/skyvern/forge/prompts/skyvern/extract-action.j2@@ -11,12 +11,13 @@ If you see a popup in the page screenshot, prioritize actions on the popup.Reply in JSON format with the following keys:{+{%- if slim_output not in ('safe', 'terse') %}"user_goal_stage": str, // A string to describe the reasoning whether user goal has been achieved or not."user_goal_achieved": bool, // True if the user goal has been completed, otherwise False.- "action_plan": str, // A string that describes the plan of actions you're going to take. Be specific and to the point. Use this as a quick summary of the actions you're going to take, and what order you're going to take them in, and how that moves you towards your overall goal. Output "COMPLETE" action in the "actions" if user_goal_achieved is True. Output "TERMINATE" action in the "actions" if your plan is to terminate the process.+ "action_plan": str, // A string that describes the plan of actions you're going to take. Be specific and to the point. Use this as a quick summary of the actions you're going to take, and what order you're going to take them in, and how that moves you towards your overall goal. Output "COMPLETE" action in the "actions" if user_goal_achieved is True. Output "TERMINATE" action in the "actions" if your plan is to terminate the process.{% endif %}"actions": array // An array of actions. Here's the format of each action:[{- "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.+ "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.{{ " Maximum 15 words, telegraphic style — drop articles and filler words." if slim_output == 'terse' else "" }}"user_detail_query": str, // Think of this value as a Jeopardy question and the intention behind the action. Ask the user for the details you need for executing this action. IMPORTANT: This question must be user information agnostic and must NOT contain specific user data (like names, IDs, specific values, etc.). Ask what information is needed generically. Ask the question even if the details are disclosed in user goal or user details. If it's a text field, ask for the text. If it's a file upload, ask for the file. If it's a dropdown, ask for the relevant information. If you are clicking on something specific, ask about what the intention is behind the click and what to click on. If you're downloading a file and you have multiple options, ask the user which one to download. Examples are: "What product ID should I input into the search bar?", "What file should I upload?", "What is the previous insurance provider of the user?", "Which invoice should I download?", "Does the user have any pets?", "Which member should I take action on?". NEVER include specific user data in this question - keep it generic so it can be answered with different user contexts."user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user goal or user details. This answer CAN contain specific user data."confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence@@ -36,11 +37,13 @@ Reply in JSON format with the following keys:"value": str // the value of the option. MAKE SURE YOU USE THIS VALUE TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION VALUE HERE},"click_context": { // The context for CLICK action only. null if not CLICK action- "thought": str, // Describe how you decided that this action is a single choice option or multi-choice option.+{%- if slim_output not in ('safe', 'terse') %}+ "thought": str, // Describe how you decided that this action is a single choice option or multi-choice option.{% endif %}"single_option_click": bool, // True if the click is the only choice to proceed towards the goal, regardless of different user context or input. False if there are multiple valid options that depend on user input. Examples: clicking a login button to login is True (it's the only way to login); clicking a radio button for a multi-choice question (e.g., selecting "male", "female", or "other" for gender) is False (the choice depends on user input). When clicking on radio buttons, dropdown options, or any element that represents one of multiple possible selections, this should be False. If the click is intended to open the dropdown menu in order to select an option from the dropdown, this should be False.},"context": { // The context for INPUT_TEXT or SELECT_OPTION action only. null if not INPUT_TEXT or SELECT_OPTION action. Extract the following detailed information from the "reasoning", and double-check the information by analysing the HTML elements.- "thought": str, // A string to describe how you double-check the context information to ensure the accuracy.+{%- if slim_output not in ('safe', 'terse') %}+ "thought": str, // A string to describe how you double-check the context information to ensure the accuracy.{% endif %}"field": str, // Which field is this action intended to fill out?"is_required": bool, // True if this is a required field, otherwise false."is_search_bar": bool, // True if the element to take the action is a search bar, otherwise false.diff --git a/skyvern/forge/prompts/skyvern/parse-input-or-select-context.j2 b/skyvern/forge/prompts/skyvern/parse-input-or-select-context.j2index 9c15d6f41..99d20218b 100644--- a/skyvern/forge/prompts/skyvern/parse-input-or-select-context.j2+++ b/skyvern/forge/prompts/skyvern/parse-input-or-select-context.j2@@ -4,7 +4,8 @@ MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commaReply in the following JSON format:{- "thought": str, // A string to describe how you double-check the information to ensure the accuracy.+{%- if slim_output not in ('safe', 'terse') %}+ "thought": str, // A string to describe how you double-check the information to ensure the accuracy.{% endif %}"field": str, // Which field is this action intended to fill out?"is_required": bool, // True if this is a required field, otherwise false."is_search_bar": bool, // True if the element to take the action is a search bar, otherwise false.diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 7f191cc58..acf0a8827 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -45,6 +45,7 @@ from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import EnrichTreeMode, SkyvernContextfrom skyvern.forge.sdk.db.enums import WorkflowRunTriggerType+from skyvern.forge.sdk.experimentation.prompt_families import effective_prompt_schema_variantfrom skyvern.forge.sdk.experimentation.screenshot_downscale import effective_downscale_heightfrom skyvern.forge.sdk.models import SpeculativeLLMMetadata, Stepfrom skyvern.forge.sdk.schemas.ai_suggestions import AISuggestion@@ -252,6 +253,14 @@ def _enrich_tree_log_fields(context: SkyvernContext | None, step: Step | None =}+def _slim_log_fields(context: SkyvernContext | None, prompt_name: str | None) -> dict[str, Any]:+ assigned = context.slim_output_variant_assigned if context else None+ return {+ "prompt_schema_variant": effective_prompt_schema_variant(assigned, prompt_name),+ "prompt_schema_variant_assigned": assigned,+ }++def _screenshot_downscale_log_fields(context: SkyvernContext | None) -> dict[str, Any]:# variant is the raw A/B arm ("control"/"treatment"/None); max_height is the effective applied# height (also non-null when the static kill-switch downscales outside the flag).@@ -1502,6 +1511,7 @@ class LLMAPIHandlerFactory:image_tokens_source=image_source,service_tier=getattr(response, "service_tier", None),llm_screenshots_enabled=llm_screenshots_enabled,+ **_slim_log_fields(context, prompt_name),**_enrich_tree_log_fields(context, step),**_screenshot_downscale_log_fields(context),**_consume_prompt_breakdown(context),@@ -2055,6 +2065,7 @@ class LLMAPIHandlerFactory:image_tokens_source=image_source,service_tier=getattr(response, "service_tier", None),llm_screenshots_enabled=llm_screenshots_enabled,+ **_slim_log_fields(context, prompt_name),**_enrich_tree_log_fields(context, step),**_screenshot_downscale_log_fields(context),**_consume_prompt_breakdown(context),@@ -2542,6 +2553,7 @@ class LLMCaller:image_cost=image_cost if image_cost > 0 else None,image_tokens_source=image_source,llm_screenshots_enabled=llm_screenshots_enabled,+ **_slim_log_fields(context, prompt_name),**_enrich_tree_log_fields(context, step),**_screenshot_downscale_log_fields(context),**_consume_prompt_breakdown(context),diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex 5cb95f82f..11053ca7e 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -1,5 +1,6 @@from __future__ import annotations+import asynciofrom contextlib import contextmanagerfrom contextvars import ContextVar, Tokenfrom dataclasses import dataclass, field@@ -122,6 +123,12 @@ class SkyvernContext:screenshot_downscale_max_height: int | None = Nonestep_retry_index: int = 0+ # Run-level SLIM_LLM_OUTPUT_PROMPTS assignment, resolved once by slim_llm_output.+ # The lock makes first-use resolution single-flight under parallel prompt builds.+ slim_output_variant_assigned: str | None = None+ slim_output_variant_resolved: bool = False+ slim_output_variant_lock: asyncio.Lock = field(default_factory=asyncio.Lock)+# Trigger type of the enclosing workflow run (manual/api/scheduled/webhook).# Routed through SkyvernContext so non-API entry points (workers, scripts) can populate it# without taking a dependency on the public-API request shape.diff --git a/skyvern/forge/sdk/experimentation/prompt_families.py b/skyvern/forge/sdk/experimentation/prompt_families.pynew file mode 100644index 000000000..8600d032f--- /dev/null+++ b/skyvern/forge/sdk/experimentation/prompt_families.py@@ -0,0 +1,83 @@+from __future__ import annotations++from enum import StrEnum++SLIM_LLM_OUTPUT_PROMPTS_FLAG = "SLIM_LLM_OUTPUT_PROMPTS"++SLIM_VARIANT_SAFE = "slim_safe"+SLIM_VARIANT_TERSE = "slim_terse"+SLIM_VARIANTS = frozenset({SLIM_VARIANT_SAFE, SLIM_VARIANT_TERSE})+++class PromptFamily(StrEnum):+ EXTRACT_ACTIONS = "extract-actions"+ CHECK_USER_GOAL = "check-user-goal"+ AUTO_COMPLETION_CHOOSE_OPTION = "auto-completion-choose-option"+ PARSE_INPUT_OR_SELECT_CONTEXT = "parse-input-or-select-context"+++# Template names and prompt_names are different namespaces (e.g. template+# "extract-action" logs as prompt_name "extract-actions"). Both the render-time+# helper and the metrics emitter resolve through these maps so they cannot diverge.+# extract-action-dynamic is mapped for completeness but carries no output schema;+# the slim conditionals live in extract-action(-static), and the split render is+# resolved once at the base-template call site.+TEMPLATES_BY_FAMILY: dict[PromptFamily, frozenset[str]] = {+ PromptFamily.EXTRACT_ACTIONS: frozenset({"extract-action", "extract-action-static", "extract-action-dynamic"}),+ PromptFamily.CHECK_USER_GOAL: frozenset({"check-user-goal", "check-user-goal-with-termination"}),+ PromptFamily.AUTO_COMPLETION_CHOOSE_OPTION: frozenset({"auto-completion-choose-option"}),+ PromptFamily.PARSE_INPUT_OR_SELECT_CONTEXT: frozenset({"parse-input-or-select-context"}),+}++PROMPT_NAMES_BY_FAMILY: dict[PromptFamily, frozenset[str]] = {+ PromptFamily.EXTRACT_ACTIONS: frozenset({"extract-actions"}),+ PromptFamily.CHECK_USER_GOAL: frozenset(+ {"check-user-goal", "check-user-goal-after-click", "check-user-goal-with-termination"}+ ),+ PromptFamily.AUTO_COMPLETION_CHOOSE_OPTION: frozenset({"auto-completion-choose-option"}),+ PromptFamily.PARSE_INPUT_OR_SELECT_CONTEXT: frozenset({"parse-input-or-select-context"}),+}++# Families whose templates honor the slim variants at runtime. A family is added+# only after passing its offline backtest gate (SKY-10075).+SLIM_ENABLED_FAMILIES: frozenset[PromptFamily] = frozenset({PromptFamily.EXTRACT_ACTIONS, PromptFamily.CHECK_USER_GOAL})++_FAMILY_BY_TEMPLATE: dict[str, PromptFamily] = {+ template: family for family, templates in TEMPLATES_BY_FAMILY.items() for template in templates+}+_FAMILY_BY_PROMPT_NAME: dict[str, PromptFamily] = {+ prompt_name: family for family, prompt_names in PROMPT_NAMES_BY_FAMILY.items() for prompt_name in prompt_names+}+++def family_for_template(template_name: str | None) -> PromptFamily | None:+ if not template_name:+ return None+ return _FAMILY_BY_TEMPLATE.get(template_name)+++def family_for_prompt_name(prompt_name: str | None) -> PromptFamily | None:+ if not prompt_name:+ return None+ return _FAMILY_BY_PROMPT_NAME.get(prompt_name)+++def slim_variant_for_family(assigned: str | None, family: PromptFamily | None) -> str | None:+ """Single gating rule shared by rendering and telemetry: the assigned variant+ applies only to known, slim-enabled families; everything else is control."""+ if assigned is None or assigned not in SLIM_VARIANTS:+ return None+ if family is None or family not in SLIM_ENABLED_FAMILIES:+ return None+ return assigned+++def effective_prompt_schema_variant(assigned: str | None, prompt_name: str | None) -> str | None:+ """Per-call cohort label: the run's assigned variant if this prompt's family renders+ slim, else None. Unknown prompt_names fail safe to None (control).++ Logged alongside the raw run-level assignment: a row with (assigned="slim_*",+ effective=None) means the run is in a treatment cohort but this prompt family+ rendered the control schema. Run-level analysis keys on assigned (intent-to-treat);+ per-prompt token/cost deltas key on effective."""+ return slim_variant_for_family(assigned, family_for_prompt_name(prompt_name))diff --git a/skyvern/forge/sdk/experimentation/slim_llm_output.py b/skyvern/forge/sdk/experimentation/slim_llm_output.pynew file mode 100644index 000000000..e85540d4b--- /dev/null+++ b/skyvern/forge/sdk/experimentation/slim_llm_output.py@@ -0,0 +1,66 @@+from __future__ import annotations++import structlog++from skyvern.forge import app+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.experimentation.prompt_families import (+ SLIM_LLM_OUTPUT_PROMPTS_FLAG,+ SLIM_VARIANT_SAFE,+ SLIM_VARIANT_TERSE,+ SLIM_VARIANTS,+ family_for_template,+ slim_variant_for_family,+)++LOG = structlog.get_logger()++# Template contract: slim_output is None (control) or one of these strings — never+# "off", which is truthy in Jinja and would slim the control cohort.+TEMPLATE_VALUE_BY_VARIANT: dict[str, str] = {+ SLIM_VARIANT_SAFE: "safe",+ SLIM_VARIANT_TERSE: "terse",+}+++async def get_slim_output_template_value(template_name: str) -> str | None:+ """Resolve the run's assigned slim variant (once per run, memoized on context) and+ map it to the template var for this template's family; None = control schema."""+ context = skyvern_context.current()+ if context is None:+ return None+ if not context.slim_output_variant_resolved:+ # Single-flight: parallel first-use callers (speculative extract-actions ++ # verification) must all observe the same one-shot resolution.+ async with context.slim_output_variant_lock:+ if not context.slim_output_variant_resolved:+ await _resolve_assigned_variant(context)+ variant = slim_variant_for_family(context.slim_output_variant_assigned, family_for_template(template_name))+ if variant is None:+ return None+ return TEMPLATE_VALUE_BY_VARIANT[variant]+++async def _resolve_assigned_variant(context: skyvern_context.SkyvernContext) -> None:+ distinct_id = context.workflow_run_id or context.task_id or context.task_v2_id or context.run_id+ if not distinct_id:+ # No run identity to bucket on — pin the context to control instead of+ # re-entering the lock on every subsequent render.+ context.slim_output_variant_resolved = True+ return+ try:+ variant = await app.EXPERIMENTATION_PROVIDER.get_value_cached(+ SLIM_LLM_OUTPUT_PROMPTS_FLAG,+ distinct_id,+ properties={+ "organization_id": context.organization_id,+ "workflow_permanent_id": context.workflow_permanent_id,+ },+ )+ except Exception:+ # Flag-evaluation failure must never alter prompt content; pin the whole run+ # to control so cohorts stay internally consistent.+ LOG.warning("Failed to resolve SLIM_LLM_OUTPUT_PROMPTS; defaulting to control", exc_info=True)+ variant = None+ context.slim_output_variant_assigned = variant if variant in SLIM_VARIANTS else None+ context.slim_output_variant_resolved = Truediff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 19a9d60fc..79c8f8559 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -78,6 +78,7 @@ from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import PendingFileChooserListener, ensure_contextfrom skyvern.forge.sdk.event.factory import EventStrategyFactoryfrom skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler+from skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_valuefrom skyvern.forge.sdk.models import Stepfrom skyvern.forge.sdk.schemas.tasks import Taskfrom skyvern.forge.sdk.services.bitwarden import BitwardenConstants@@ -1451,6 +1452,7 @@ async def handle_sequential_click_for_dropdown(# rendered. Gate lean on the PostHog flag._ctx = skyvern_context.current()lean_enabled = bool(_ctx and _ctx.enable_lean_element_tree)+ slim_output = await get_slim_output_template_value("check-user-goal")prompt = load_prompt_with_elements(element_tree_builder=scraped_page_after_open,prompt_engine=prompt_engine,@@ -1460,6 +1462,7 @@ async def handle_sequential_click_for_dropdown(new_elements_ids=new_element_ids,without_screenshots=True,action_history=action_history_str,+ slim_output=slim_output,local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),lean_compress_long_href=lean_enabled,lean_compress_image_src=lean_enabled,@@ -3711,6 +3714,7 @@ async def choose_auto_completion_dropdown()html = scraped_page_after_open.build_element_tree()+ slim_output = await get_slim_output_template_value("auto-completion-choose-option")auto_completion_confirm_prompt = prompt_engine.load_prompt("auto-completion-choose-option",is_search=context.is_search_bar,@@ -3721,6 +3725,7 @@ async def choose_auto_completion_dropdown(elements=html,new_elements_ids=new_interactable_element_ids,local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),+ slim_output=slim_output,)LOG.info("Confirm if it's an auto completion dropdown")json_response = await app.AUTO_COMPLETION_LLM_API_HANDLER(@@ -4083,6 +4088,7 @@ async def discover_and_select_from_full_dropdown(new_element_ids = [e.get("id", "") for e in cleaned_elements if e.get("id")]field_information = context.field if not context.intention else context.intention+ slim_output = await get_slim_output_template_value("auto-completion-choose-option")prompt = prompt_engine.load_prompt("auto-completion-choose-option",is_search=context.is_search_bar,@@ -4093,6 +4099,7 @@ async def discover_and_select_from_full_dropdown(elements=html,new_elements_ids=new_element_ids,local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),+ slim_output=slim_output,)LOG.info(@@ -5717,12 +5724,14 @@ async def _get_input_or_select_context(except Exception:LOG.warning("Failed to get sub element tree, using the original element tree", exc_info=True, path=path)+ slim_output = await get_slim_output_template_value("parse-input-or-select-context")prompt = load_prompt_with_elements(element_tree_builder=element_tree_builder,prompt_engine=prompt_engine,template_name="parse-input-or-select-context",action_reasoning=action.reasoning,element_id=action.element_id,+ slim_output=slim_output,)# Use centralized parse-select handler (set at init or via scripts)json_response = await app.PARSE_SELECT_LLM_API_HANDLER(diff --git a/tests/unit/golden_prompts/auto-completion-choose-option.control.txt b/tests/unit/golden_prompts/auto-completion-choose-option.control.txtnew file mode 100644index 000000000..726b2bfbc--- /dev/null+++ b/tests/unit/golden_prompts/auto-completion-choose-option.control.txt@@ -0,0 +1,55 @@+There is an input element on an HTML page. Based on the context and information provided, you have two goals:+ - Confirm if an auto-completion attempt appears after the user inputs the current value.+ - If auto-completion suggestions appear, assist the user in selecting the most appropriate element based on the user's goal, details, and the context.++You can confirm an auto-completion attempt based on the following rules:+ - Several auto-completion suggestions appear for the input value.+ - Although messages like "No results" and "No match" mean no option was matched, they still indicate an attempt to generate auto-completion suggestions.++You must identify a potential auto-completion suggestion based on the following rules:+ - The option must be an element with an ID from the provided "HTML elements". Do not create or assume options outside of these elements.+ - The content of the option must be meaningful. Do not consider non-message indicators like "No results" or "No match" as valid options.++MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.+Each interactable element is tagged with an ID.++Reply in JSON format with the following keys:+{+ "thought": str, // Think step by step. Describe your thought about how you achieve the two goals with convincing evidence.+ "auto_completion_attempt": bool, // True if there's any auto completion attempt based on the rules. Otherwise, it should be False.+ "reasoning": str, // The reasoning behind the decision. Be specific, referencing the value and the element id in your reasoning. Mention why you chose the element id. Keep the reasoning short and to the point.+ "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence.+ "relevance_float": float, // The relative between the selected element and the provided information. You should consider how much the selected option is related to the user goal, the user details and the context. Pick a number between 0.00 and 1.00. 0.00 means no relevance, 1.00 means full relevance, the precision is 0.01.+ "value": str, // The value to select.+ "id": str, // The id of the most relevant and interactable element to take the action. The id must be from "HTML elements". It should be null if no element is relative or there's no auto completion suggestion.+}++Context:+```+Choose an auto-completion suggestion for "name"+```++Input value:+```+John+```++User goal:+```+test goal+```++User details:+```+{}+```++HTML elements:+```+<html></html>+```++Current datetime, ISO format:+```+2025-01-01T00:00:00+```\ No newline at end of filediff --git a/tests/unit/golden_prompts/check-user-goal-with-termination.control.txt b/tests/unit/golden_prompts/check-user-goal-with-termination.control.txtnew file mode 100644index 000000000..75be5e1bc--- /dev/null+++ b/tests/unit/golden_prompts/check-user-goal-with-termination.control.txt@@ -0,0 +1,46 @@+You are here to help the user determine if the user has completed their goal on the web. Use the content of the elements parsed from the page, the screenshots of the page, the user goal and user details to determine the status of the task.++Make sure to ONLY return the JSON object in this format with no additional text before or after it:+```json+{+ "page_info": str, // Think step by step. Describe all the useful information in the page related to the user goal.+ "thoughts": str, // Think step by step. Explain your reasoning for the status you selected.+ "status": str, // Must be one of three values: "complete", "terminate", or "continue". Use "complete" ONLY if the user goal has been fully achieved. Use "terminate" ONLY if the goal CANNOT ever be achieved (e.g., a file doesn't exist, an error modal blocks progress permanently, or an explicit termination condition is met in the user goal). Use "continue" if the goal is not yet achieved but more steps could potentially achieve it (this is the most common case - use this when you need to wait, navigate, or try different actions).+ "failure_categories": array // Only populate when status is "terminate". Classify the root cause. Categories: ANTI_BOT_DETECTION (CAPTCHA, Cloudflare, bot detection), BROWSER_ERROR (crash, page closed), NAVIGATION_FAILURE (404, redirect), PAGE_LOAD_TIMEOUT (timeout), AUTH_FAILURE (login failed, MFA), LLM_REASONING_ERROR (wrong element, wrong action), CREDENTIAL_ERROR (missing credentials), ELEMENT_NOT_FOUND (target missing), WRONG_PAGE_STATE (unexpected page), DATA_EXTRACTION_FAILURE (extraction failed), INFRASTRUCTURE_ERROR (network, OOM), UNKNOWN (cannot determine). Return one or more categories ordered by likelihood.+ [{+ "category": str, // The category from the list above+ "confidence_float": float, // 0.0 to 1.0+ "reasoning": str // Brief explanation+ }]+}+```++Important: Think carefully about the difference between "terminate" and "continue":+- "terminate" = impossible to achieve, stop trying (e.g., "account does not exist", "file unavailable", permanent error)+- "continue" = not done yet, but achievable with more steps (e.g., page is loading, need to click something, need to wait)++User Goal:+```+test goal+```++User Details:+```+{}+```+++Action History:+```+[]+```++Elements on the page:+```+<html></html>+```++Current datetime, ISO format:+```+2025-01-01T00:00:00+```diff --git a/tests/unit/golden_prompts/check-user-goal.control.txt b/tests/unit/golden_prompts/check-user-goal.control.txtnew file mode 100644index 000000000..5cbbfc4ad--- /dev/null+++ b/tests/unit/golden_prompts/check-user-goal.control.txt@@ -0,0 +1,36 @@+Your are here to help the user determine if the user has completed their goal on the web. Use the content of the elements parsed from the page, the screenshots of the page, the user goal and user details to determine whether the user goal has been completed or not.++Make sure to ONLY return the JSON object in this format with no additional text before or after it:+```json+{+ "page_info": str, // Think step by step. Describe all the useful information in the page related to the user goal.+ "thoughts": str, // Think step by step. What information makes you believe whether user goal has completed or not. Use information you see on the site to explain.+ "user_goal_achieved": bool // True if the user goal has been completed, false otherwise.+}+```++User Goal:+```+test goal+```++User Details:+```+{}+```+++Action History:+```+[]+```++Elements on the page:+```+<html></html>+```++Current datetime, ISO format:+```+2025-01-01T00:00:00+```\ No newline at end of filediff --git a/tests/unit/golden_prompts/extract-action-static.control.txt b/tests/unit/golden_prompts/extract-action-static.control.txtnew file mode 100644index 000000000..711c21609--- /dev/null+++ b/tests/unit/golden_prompts/extract-action-static.control.txt@@ -0,0 +1,59 @@+Identify actions to help user progress towards the user goal using the DOM elements given in the list and the screenshot of the website.+Include only the elements that are relevant to the user goal, without altering or imagining new elements.+Accurately interpret and understand the functional significance of SVG elements based on their shapes and context within the webpage.+Use the user details to fill in necessary values. Always satisfy required fields if the field isn't already filled in. Don't return any action for the same field, if this field is already filled in and the value is the same as the one you would have filled in.+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.+Each interactable element is tagged with an ID. Avoid taking action on a disabled element when there is an alternative action available.+If you see any information in red in the page screenshot, this means a condition wasn't satisfied. prioritize actions with the red information.+If you see a popup in the page screenshot, prioritize actions on the popup.+++Reply in JSON format with the following keys:+{+ "user_goal_stage": str, // A string to describe the reasoning whether user goal has been achieved or not.+ "user_goal_achieved": bool, // True if the user goal has been completed, otherwise False.+ "action_plan": str, // A string that describes the plan of actions you're going to take. Be specific and to the point. Use this as a quick summary of the actions you're going to take, and what order you're going to take them in, and how that moves you towards your overall goal. Output "COMPLETE" action in the "actions" if user_goal_achieved is True. Output "TERMINATE" action in the "actions" if your plan is to terminate the process.+ "actions": array // An array of actions. Here's the format of each action:+ [{+ "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.+ "user_detail_query": str, // Think of this value as a Jeopardy question and the intention behind the action. Ask the user for the details you need for executing this action. IMPORTANT: This question must be user information agnostic and must NOT contain specific user data (like names, IDs, specific values, etc.). Ask what information is needed generically. Ask the question even if the details are disclosed in user goal or user details. If it's a text field, ask for the text. If it's a file upload, ask for the file. If it's a dropdown, ask for the relevant information. If you are clicking on something specific, ask about what the intention is behind the click and what to click on. If you're downloading a file and you have multiple options, ask the user which one to download. Examples are: "What product ID should I input into the search bar?", "What file should I upload?", "What is the previous insurance provider of the user?", "Which invoice should I download?", "Does the user have any pets?", "Which member should I take action on?". NEVER include specific user data in this question - keep it generic so it can be answered with different user contexts.+ "user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user goal or user details. This answer CAN contain specific user data.+ "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence+ "action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE", "KEYPRESS", "SCROLL". "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element without clicking, particularly when revealing hover-only menus or buttons before clicking, or when the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a visible captcha to solve on the screen. Captcha tokens expire quickly (~2 minutes), so fill all form fields first, then SOLVE_CAPTCHA immediately before the final submit or protected click — do not place SOLVE_CAPTCHA before a long form-fill sequence. "COMPLETE" is used when the user goal has been achieved AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the user goal is achieved. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned. "KEYPRESS" is used to press a keyboard key when no clickable button or element achieves the same result. Only use KEYPRESS when pressing a key is the sole way to proceed (e.g., pressing Enter to submit a search with no search button, or Escape to close a modal with no close button). KEYPRESS does not require an element id. Requires the "key" field. "SCROLL" is used to scroll within a specific scrollable container on the page (not the page itself). Only use SCROLL when a required action is blocked because the target element is hidden, disabled, or unreachable until the container is scrolled (e.g., an "Agree" button that only becomes enabled after scrolling to the bottom of a terms and conditions box). Do not use SCROLL for general page navigation. Provide the element id of an interactable element within or near the scrollable container. Requires the "direction" field. SCROLL does not require a "text", "option", or "file_url" field.+ "id": str, // The id of the element to take action on. The id has to be one from the elements list. For target-less actions (SOLVE_CAPTCHA, WAIT, COMPLETE, TERMINATE, KEYPRESS), set this to null.+ "captcha_type": str, // The type of captcha for SOLVE_CAPTCHA action only. null if not SOLVE_CAPTCHA action. It's a string enum: "TEXT_CAPTCHA", "RECAPTCHA", "HCAPTCHA", "MTCAPTCHA", "FUNCAPTCHA", "CLOUDFLARE", "OTHER".+ "text": str, // Text for INPUT_TEXT action only+ "key": str, // The keyboard key to press for KEYPRESS action only. Allowed values: "Enter", "Tab", "Escape", "ArrowDown", "ArrowUp". For "0"-"9" and "a"-"z", use KEYPRESS only when no suitable element is available for INPUT_TEXT. null if not KEYPRESS action.+ "repeat": int, // For KEYPRESS only: how many times to press the key in rapid succession without re-evaluating the page between presses. Use this when navigating transient menus (e.g., ArrowDown 3 times to reach an item). Defaults to 1. null if not KEYPRESS action.+ "direction": str, // The direction to scroll for SCROLL action only. Allowed values: "up", "down". null if not SCROLL action.+ "file_url": str, // The url of the file to upload if applicable. This field must be present for UPLOAD_FILE but can also be present for CLICK only if the click is to upload the file. It should be null otherwise.+ "double_click": bool, // For CLICK action only. True only when the element requires a double-click to activate (e.g., opening a file in a file manager, entering edit mode in a spreadsheet cell, or expanding a tree node). Defaults to false for normal clicks. Do not use double-click for regular buttons, links, form elements, clickable grid rows, icons, or dashboard cards.+ "download": bool, // Can only be true for CLICK or SELECT_OPTION actions. If true, the browser will trigger a download by clicking the element. If false, the browser will click the element without triggering a download.+ "option": { // The option to select for SELECT_OPTION action only. null if not SELECT_OPTION action+ "label": str, // the label of the option if any. MAKE SURE YOU USE THIS LABEL TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION LABEL HERE+ "index": int, // the index corresponding to the option index under the select element.+ "value": str // the value of the option. MAKE SURE YOU USE THIS VALUE TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION VALUE HERE+ },+ "click_context": { // The context for CLICK action only. null if not CLICK action+ "thought": str, // Describe how you decided that this action is a single choice option or multi-choice option.+ "single_option_click": bool, // True if the click is the only choice to proceed towards the goal, regardless of different user context or input. False if there are multiple valid options that depend on user input. Examples: clicking a login button to login is True (it's the only way to login); clicking a radio button for a multi-choice question (e.g., selecting "male", "female", or "other" for gender) is False (the choice depends on user input). When clicking on radio buttons, dropdown options, or any element that represents one of multiple possible selections, this should be False. If the click is intended to open the dropdown menu in order to select an option from the dropdown, this should be False.+ },+ "context": { // The context for INPUT_TEXT or SELECT_OPTION action only. null if not INPUT_TEXT or SELECT_OPTION action. Extract the following detailed information from the "reasoning", and double-check the information by analysing the HTML elements.+ "thought": str, // A string to describe how you double-check the context information to ensure the accuracy.+ "field": str, // Which field is this action intended to fill out?+ "is_required": bool, // True if this is a required field, otherwise false.+ "is_search_bar": bool, // True if the element to take the action is a search bar, otherwise false.+ "is_location_input": bool, // True if the element is asking user to input where he lives, otherwise false. For example, it is asking for location, or address, or other similar information. Output False if it only requires ZIP code or postal code.+ "is_date_related": bool, // True if the field is related to date input or select, otherwise false.+ "date_format": str, // The format of the date or datetime to be input. For example YYYY-MM-DD, YYYY-MM-DD HH:MM:SS, DD.MM.YYYY, MM/DD/YYYY, etc. If the field is not related to date input or select, this should be null.+ "is_text_captcha": bool, // True if the field is asking for a text captcha, otherwise false. Do not confuse it with the verification code. Text CAPTCHAs are typically displayed alongside an image of distorted letters or numbers.+ }+ }],+ "verification_code_reasoning": str, // Let's think step by step. Describe what you see and think if there is somewhere on the current page where you must enter the verification code now for login or any verification step. Explain why you believe a verification code needs to be entered somewhere or not. Do not imagine any place to enter the code if the code has not been sent yet.+ "place_to_enter_verification_code": bool, // Whether there is a place on the current page to enter the verification code now.+ "should_enter_verification_code": bool, // Whether the user should proceed to enter the verification code.+ "should_verify_by_magic_link": bool // Whether the page instructs the user to check their email for a magic link to verify the login.+}++Consider the action history from the last step and the screenshot together, if actions from the last step don't yield positive impact, try other actions or other action combinations.+Action history from previous steps: (note: even if the action history suggests goal is achieved, check the screenshot and the DOM elements to make sure the goal is achieved)\ No newline at end of filediff --git a/tests/unit/golden_prompts/extract-action.control.txt b/tests/unit/golden_prompts/extract-action.control.txtnew file mode 100644index 000000000..3f633ea9d--- /dev/null+++ b/tests/unit/golden_prompts/extract-action.control.txt@@ -0,0 +1,89 @@+Identify actions to help user progress towards the user goal using the DOM elements given in the list and the screenshot of the website.+Include only the elements that are relevant to the user goal, without altering or imagining new elements.+Accurately interpret and understand the functional significance of SVG elements based on their shapes and context within the webpage.+Use the user details to fill in necessary values. Always satisfy required fields if the field isn't already filled in. Don't return any action for the same field, if this field is already filled in and the value is the same as the one you would have filled in.+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.+Each interactable element is tagged with an ID. Avoid taking action on a disabled element when there is an alternative action available.+If you see any information in red in the page screenshot, this means a condition wasn't satisfied. prioritize actions with the red information.+If you see a popup in the page screenshot, prioritize actions on the popup.+++Reply in JSON format with the following keys:+{+ "user_goal_stage": str, // A string to describe the reasoning whether user goal has been achieved or not.+ "user_goal_achieved": bool, // True if the user goal has been completed, otherwise False.+ "action_plan": str, // A string that describes the plan of actions you're going to take. Be specific and to the point. Use this as a quick summary of the actions you're going to take, and what order you're going to take them in, and how that moves you towards your overall goal. Output "COMPLETE" action in the "actions" if user_goal_achieved is True. Output "TERMINATE" action in the "actions" if your plan is to terminate the process.+ "actions": array // An array of actions. Here's the format of each action:+ [{+ "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.+ "user_detail_query": str, // Think of this value as a Jeopardy question and the intention behind the action. Ask the user for the details you need for executing this action. IMPORTANT: This question must be user information agnostic and must NOT contain specific user data (like names, IDs, specific values, etc.). Ask what information is needed generically. Ask the question even if the details are disclosed in user goal or user details. If it's a text field, ask for the text. If it's a file upload, ask for the file. If it's a dropdown, ask for the relevant information. If you are clicking on something specific, ask about what the intention is behind the click and what to click on. If you're downloading a file and you have multiple options, ask the user which one to download. Examples are: "What product ID should I input into the search bar?", "What file should I upload?", "What is the previous insurance provider of the user?", "Which invoice should I download?", "Does the user have any pets?", "Which member should I take action on?". NEVER include specific user data in this question - keep it generic so it can be answered with different user contexts.+ "user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user goal or user details. This answer CAN contain specific user data.+ "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence+ "action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE", "KEYPRESS", "SCROLL". "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element without clicking, particularly when revealing hover-only menus or buttons before clicking, or when the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a visible captcha to solve on the screen. Captcha tokens expire quickly (~2 minutes), so fill all form fields first, then SOLVE_CAPTCHA immediately before the final submit or protected click — do not place SOLVE_CAPTCHA before a long form-fill sequence. "COMPLETE" is used when the user goal has been achieved AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the user goal is achieved. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned. "KEYPRESS" is used to press a keyboard key when no clickable button or element achieves the same result. Only use KEYPRESS when pressing a key is the sole way to proceed (e.g., pressing Enter to submit a search with no search button, or Escape to close a modal with no close button). KEYPRESS does not require an element id. Requires the "key" field. "SCROLL" is used to scroll within a specific scrollable container on the page (not the page itself). Only use SCROLL when a required action is blocked because the target element is hidden, disabled, or unreachable until the container is scrolled (e.g., an "Agree" button that only becomes enabled after scrolling to the bottom of a terms and conditions box). Do not use SCROLL for general page navigation. Provide the element id of an interactable element within or near the scrollable container. Requires the "direction" field. SCROLL does not require a "text", "option", or "file_url" field.+ "id": str, // The id of the element to take action on. The id has to be one from the elements list. For target-less actions (SOLVE_CAPTCHA, WAIT, COMPLETE, TERMINATE, KEYPRESS), set this to null.+ "captcha_type": str, // The type of captcha for SOLVE_CAPTCHA action only. null if not SOLVE_CAPTCHA action. It's a string enum: "TEXT_CAPTCHA", "RECAPTCHA", "HCAPTCHA", "MTCAPTCHA", "FUNCAPTCHA", "CLOUDFLARE", "OTHER".+ "text": str, // Text for INPUT_TEXT action only+ "key": str, // The keyboard key to press for KEYPRESS action only. Allowed values: "Enter", "Tab", "Escape", "ArrowDown", "ArrowUp". For "0"-"9" and "a"-"z", use KEYPRESS only when no suitable element is available for INPUT_TEXT. null if not KEYPRESS action.+ "repeat": int, // For KEYPRESS only: how many times to press the key in rapid succession without re-evaluating the page between presses. Use this when navigating transient menus (e.g., ArrowDown 3 times to reach an item). Defaults to 1. null if not KEYPRESS action.+ "direction": str, // The direction to scroll for SCROLL action only. Allowed values: "up", "down". null if not SCROLL action.+ "file_url": str, // The url of the file to upload if applicable. This field must be present for UPLOAD_FILE but can also be present for CLICK only if the click is to upload the file. It should be null otherwise.+ "double_click": bool, // For CLICK action only. True only when the element requires a double-click to activate (e.g., opening a file in a file manager, entering edit mode in a spreadsheet cell, or expanding a tree node). Defaults to false for normal clicks. Do not use double-click for regular buttons, links, form elements, clickable grid rows, icons, or dashboard cards.+ "download": bool, // Can only be true for CLICK or SELECT_OPTION actions. If true, the browser will trigger a download by clicking the element. If false, the browser will click the element without triggering a download.+ "option": { // The option to select for SELECT_OPTION action only. null if not SELECT_OPTION action+ "label": str, // the label of the option if any. MAKE SURE YOU USE THIS LABEL TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION LABEL HERE+ "index": int, // the index corresponding to the option index under the select element.+ "value": str // the value of the option. MAKE SURE YOU USE THIS VALUE TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION VALUE HERE+ },+ "click_context": { // The context for CLICK action only. null if not CLICK action+ "thought": str, // Describe how you decided that this action is a single choice option or multi-choice option.+ "single_option_click": bool, // True if the click is the only choice to proceed towards the goal, regardless of different user context or input. False if there are multiple valid options that depend on user input. Examples: clicking a login button to login is True (it's the only way to login); clicking a radio button for a multi-choice question (e.g., selecting "male", "female", or "other" for gender) is False (the choice depends on user input). When clicking on radio buttons, dropdown options, or any element that represents one of multiple possible selections, this should be False. If the click is intended to open the dropdown menu in order to select an option from the dropdown, this should be False.+ },+ "context": { // The context for INPUT_TEXT or SELECT_OPTION action only. null if not INPUT_TEXT or SELECT_OPTION action. Extract the following detailed information from the "reasoning", and double-check the information by analysing the HTML elements.+ "thought": str, // A string to describe how you double-check the context information to ensure the accuracy.+ "field": str, // Which field is this action intended to fill out?+ "is_required": bool, // True if this is a required field, otherwise false.+ "is_search_bar": bool, // True if the element to take the action is a search bar, otherwise false.+ "is_location_input": bool, // True if the element is asking user to input where he lives, otherwise false. For example, it is asking for location, or address, or other similar information. Output False if it only requires ZIP code or postal code.+ "is_date_related": bool, // True if the field is related to date input or select, otherwise false.+ "date_format": str, // The format of the date or datetime to be input. For example YYYY-MM-DD, YYYY-MM-DD HH:MM:SS, DD.MM.YYYY, MM/DD/YYYY, etc. If the field is not related to date input or select, this should be null.+ "is_text_captcha": bool, // True if the field is asking for a text captcha, otherwise false. Do not confuse it with the verification code. Text CAPTCHAs are typically displayed alongside an image of distorted letters or numbers.+ }+ }],+ "verification_code_reasoning": str, // Let's think step by step. Describe what you see and think if there is somewhere on the current page where you must enter the verification code now for login or any verification step. Explain why you believe a verification code needs to be entered somewhere or not. Do not imagine any place to enter the code if the code has not been sent yet.+ "place_to_enter_verification_code": bool, // Whether there is a place on the current page to enter the verification code now.+ "should_enter_verification_code": bool, // Whether the user should proceed to enter the verification code.+ "should_verify_by_magic_link": bool // Whether the page instructs the user to check their email for a magic link to verify the login.+}++Consider the action history from the last step and the screenshot together, if actions from the last step don't yield positive impact, try other actions or other action combinations.+Action history from previous steps: (note: even if the action history suggests goal is achieved, check the screenshot and the DOM elements to make sure the goal is achieved)+```+[]+```++++User goal:+```+test goal+```++++User details:+```+{}+```+++Clickable elements from `https://example.com`:+```+<html></html>+```++The URL of the page you're on right now is `https://example.com`.++Current datetime, ISO format:+```+2025-01-01T00:00:00+```\ No newline at end of filediff --git a/tests/unit/golden_prompts/parse-input-or-select-context.control.txt b/tests/unit/golden_prompts/parse-input-or-select-context.control.txtnew file mode 100644index 000000000..018090e51--- /dev/null+++ b/tests/unit/golden_prompts/parse-input-or-select-context.control.txt@@ -0,0 +1,25 @@+You are a browser agent performing actions on the web. You are instructed to take an INPUT or SELECT action on the element(id: "elem_1"). Extract some detailed information from the context/reasoning, and double-check the information by analysing the HTML elements.++MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.++Reply in the following JSON format:+{+ "thought": str, // A string to describe how you double-check the information to ensure the accuracy.+ "field": str, // Which field is this action intended to fill out?+ "is_required": bool, // True if this is a required field, otherwise false.+ "is_search_bar": bool, // True if the element to take the action is a search bar, otherwise false.+ "is_location_input": bool, // True if the element is asking user to input where he lives, otherwise false. For example, it is asking for location, or address, or other similar information. Output False if it only requires ZIP code or postal code.+ "is_date_related": bool, // True if the field is related to date input or select, otherwise false.+ "date_format": str, // The format of the date or datetime to be input. For example YYYY-MM-DD, YYYY-MM-DD HH:MM:SS, DD.MM.YYYY, MM/DD/YYYY, etc. If the field is not related to date input or select, this should be null.+ "is_text_captcha": bool, // True if the field is asking for a text captcha, otherwise false. Do not confuse it with the verification code. Text CAPTCHAs are typically displayed alongside an image of distorted letters or numbers.+}++Existing reasoning context:+```+test reasoning+```++HTML elements:+```+<html></html>+```diff --git a/tests/unit/golden_prompts/regenerate.py b/tests/unit/golden_prompts/regenerate.pynew file mode 100644index 000000000..e0bee5cff--- /dev/null+++ b/tests/unit/golden_prompts/regenerate.py@@ -0,0 +1,83 @@+"""Regenerate the golden control renders for the slim-output templates.++Run this ONLY when intentionally changing the control (flag-off) rendering of an+in-scope template, and review the golden diff like production code — it is the+byte-level contract that the SLIM_LLM_OUTPUT_PROMPTS control cohort is unchanged:++ uv run python tests/unit/golden_prompts/regenerate.py+"""++from __future__ import annotations++from pathlib import Path+from typing import Any++from skyvern.forge.sdk.prompting import PromptEngine++GOLDEN_DIR = Path(__file__).parent++EXTRACT_ACTION_KWARGS: dict[str, Any] = {+ "navigation_goal": "test goal",+ "navigation_payload_str": "{}",+ "starting_url": "https://example.com",+ "current_url": "https://example.com",+ "data_extraction_goal": None,+ "action_history": "[]",+ "error_code_mapping_str": None,+ "local_datetime": "2025-01-01T00:00:00",+ "verification_code_check": True,+ "complete_criterion": None,+ "terminate_criterion": None,+ "show_close_page_action": False,+ "open_tabs_context": None,+ "recent_dialog_messages_str": None,+ "llm_screenshots_enabled": True,+ "enriched_tree_enabled": False,+ "elements": "<html></html>",+}+CHECK_USER_GOAL_KWARGS: dict[str, Any] = {+ "navigation_goal": "test goal",+ "navigation_payload": "{}",+ "complete_criterion": None,+ "action_history": "[]",+ "new_elements_ids": None,+ "without_screenshots": False,+ "local_datetime": "2025-01-01T00:00:00",+ "elements": "<html></html>",+}++# Must stay in sync with _TEMPLATE_CASES in tests/unit/test_slim_llm_output.py.+CASES: dict[str, dict[str, Any]] = {+ "extract-action": EXTRACT_ACTION_KWARGS,+ "extract-action-static": EXTRACT_ACTION_KWARGS,+ "check-user-goal": CHECK_USER_GOAL_KWARGS,+ "check-user-goal-with-termination": {**CHECK_USER_GOAL_KWARGS, "terminate_criterion": None},+ "auto-completion-choose-option": {+ "is_search": False,+ "field_information": "name",+ "filled_value": "John",+ "navigation_goal": "test goal",+ "navigation_payload_str": "{}",+ "elements": "<html></html>",+ "new_elements_ids": None,+ "local_datetime": "2025-01-01T00:00:00",+ },+ "parse-input-or-select-context": {+ "element_id": "elem_1",+ "action_reasoning": "test reasoning",+ "elements": "<html></html>",+ },+}+++def main() -> None:+ engine = PromptEngine(model="skyvern")+ for name, kwargs in CASES.items():+ rendered = engine.load_prompt(name, **kwargs)+ path = GOLDEN_DIR / f"{name}.control.txt"+ path.write_text(rendered)+ print(f"wrote {path} ({len(rendered)} chars)")+++if __name__ == "__main__":+ main()diff --git a/tests/unit/test_slim_llm_output.py b/tests/unit/test_slim_llm_output.pynew file mode 100644index 000000000..ad16ab9f2--- /dev/null+++ b/tests/unit/test_slim_llm_output.py@@ -0,0 +1,447 @@+"""Tests for SKY-10075 / SKY-9717: SLIM_LLM_OUTPUT_PROMPTS tiered variants.++Covers the strict template contract (slim_output is None | 'safe' | 'terse'),+golden byte-identity of the control render, per-variant static-prefix contract,+the run-level variant resolver, the canonical prompt-family maps, the effective+per-call telemetry label, and the cache-variant key.++When intentionally changing the control render of an in-scope template, refresh+the snapshots with `uv run python tests/unit/golden_prompts/regenerate.py` and+review the golden diff like production code.+"""++from __future__ import annotations++import re+from pathlib import Path+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest++from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.experimentation import slim_llm_output+from skyvern.forge.sdk.experimentation.prompt_families import (+ PROMPT_NAMES_BY_FAMILY,+ SLIM_ENABLED_FAMILIES,+ SLIM_VARIANT_SAFE,+ SLIM_VARIANT_TERSE,+ TEMPLATES_BY_FAMILY,+ PromptFamily,+ effective_prompt_schema_variant,+ family_for_prompt_name,+ family_for_template,+)+from skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_value+from skyvern.forge.sdk.prompting import PromptEngine++GOLDEN_DIR = Path(__file__).parent / "golden_prompts"+TEMPLATE_DIR = Path(__file__).parent.parent.parent / "skyvern" / "forge" / "prompts" / "skyvern"++TERSE_MARKER = "Maximum 15 words"+++def prompt_engine() -> PromptEngine:+ return PromptEngine(model="skyvern")+++def run_context() -> Any:+ context = SkyvernContext(workflow_run_id="wr_123", task_id="tsk_123", organization_id="org_456")+ skyvern_context.set(context)+ yield context+ skyvern_context.reset()+++def _mock_provider(monkeypatch: pytest.MonkeyPatch, variant: Any) -> MagicMock:+ provider = MagicMock()+ if isinstance(variant, Exception):+ provider.get_value_cached = AsyncMock(side_effect=variant)+ else:+ provider.get_value_cached = AsyncMock(return_value=variant)+ mock_app = MagicMock()+ mock_app.EXPERIMENTATION_PROVIDER = provider+ monkeypatch.setattr(slim_llm_output, "app", mock_app)+ return provider+++_EXTRACT_ACTION_KWARGS: dict[str, Any] = {+ "navigation_goal": "test goal",+ "navigation_payload_str": "{}",+ "starting_url": "https://example.com",+ "current_url": "https://example.com",+ "data_extraction_goal": None,+ "action_history": "[]",+ "error_code_mapping_str": None,+ "local_datetime": "2025-01-01T00:00:00",+ "verification_code_check": True,+ "complete_criterion": None,+ "terminate_criterion": None,+ "show_close_page_action": False,+ "open_tabs_context": None,+ "recent_dialog_messages_str": None,+ "llm_screenshots_enabled": True,+ "enriched_tree_enabled": False,+ "elements": "<html></html>",+}+_CHECK_USER_GOAL_KWARGS: dict[str, Any] = {+ "navigation_goal": "test goal",+ "navigation_payload": "{}",+ "complete_criterion": None,+ "action_history": "[]",+ "new_elements_ids": None,+ "without_screenshots": False,+ "local_datetime": "2025-01-01T00:00:00",+ "elements": "<html></html>",+}++# template -> (render kwargs, fields dropped in slim, fields kept in every variant, has terse marker)+_TEMPLATE_CASES: dict[str, tuple[dict[str, Any], list[str], list[str], bool]] = {+ "extract-action": (+ _EXTRACT_ACTION_KWARGS,+ [+ '"user_goal_stage":',+ '"user_goal_achieved": bool, // True if the user goal has been completed, otherwise False.',+ '"action_plan":',+ '"thought":',+ ],+ [+ '"reasoning":',+ '"user_detail_query":',+ '"user_detail_answer":',+ '"confidence_float":',+ '"verification_code_reasoning":',+ '"place_to_enter_verification_code":',+ '"should_verify_by_magic_link":',+ ],+ True,+ ),+ "extract-action-static": (+ _EXTRACT_ACTION_KWARGS,+ [+ '"user_goal_stage":',+ '"user_goal_achieved": bool, // True if the user goal has been completed, otherwise False.',+ '"action_plan":',+ '"thought":',+ ],+ ['"reasoning":', '"verification_code_reasoning":'],+ True,+ ),+ "check-user-goal": (+ _CHECK_USER_GOAL_KWARGS,+ ['"page_info":'],+ ['"thoughts":', '"user_goal_achieved":'],+ True,+ ),+ "check-user-goal-with-termination": (+ {**_CHECK_USER_GOAL_KWARGS, "terminate_criterion": None},+ ['"page_info":'],+ ['"thoughts":', '"status":', '"failure_categories":'],+ True,+ ),+ "auto-completion-choose-option": (+ {+ "is_search": False,+ "field_information": "name",+ "filled_value": "John",+ "navigation_goal": "test goal",+ "navigation_payload_str": "{}",+ "elements": "<html></html>",+ "new_elements_ids": None,+ "local_datetime": "2025-01-01T00:00:00",+ },+ ['"thought":', '"reasoning":'],+ ['"auto_completion_attempt":', '"confidence_float":', '"value":', '"id":'],+ False,+ ),+ "parse-input-or-select-context": (+ {"element_id": "elem_1", "action_reasoning": "test reasoning", "elements": "<html></html>"},+ ['"thought":'],+ ['"field":', '"is_required":', '"is_text_captcha":'],+ False,+ ),+}+++# ---------------------------------------------------------------------------+# Template contract+# ---------------------------------------------------------------------------++[email protected]("template_name", _TEMPLATE_CASES)+def test_control_render_is_byte_identical_to_golden(prompt_engine: PromptEngine, template_name: str) -> None:+ # The golden files were generated from the pre-slim production templates; any+ # control-render drift means the flag-off path changed behavior.+ kwargs, _, _, _ = _TEMPLATE_CASES[template_name]+ golden = (GOLDEN_DIR / f"{template_name}.control.txt").read_text()+ assert prompt_engine.load_prompt(template_name, **kwargs) == golden+ assert prompt_engine.load_prompt(template_name, slim_output=None, **kwargs) == golden++[email protected]("template_name", _TEMPLATE_CASES)[email protected]("variant", ["safe", "terse"])+def test_slim_render_drops_dead_fields_and_keeps_consumed_fields(+ prompt_engine: PromptEngine, template_name: str, variant: str+) -> None:+ kwargs, drops, keeps, _ = _TEMPLATE_CASES[template_name]+ control = prompt_engine.load_prompt(template_name, **kwargs)+ slim = prompt_engine.load_prompt(template_name, slim_output=variant, **kwargs)+ for field in drops:+ assert field in control, f"{field} missing from {template_name} control render"+ assert field not in slim, f"{field} should be dropped from {template_name} when slim_output={variant}"+ for field in keeps:+ assert field in slim, f"{field} must be kept in {template_name} when slim_output={variant}"+ assert len(slim) < len(control)++[email protected]("template_name", _TEMPLATE_CASES)+def test_terse_marker_only_in_terse_renders(prompt_engine: PromptEngine, template_name: str) -> None:+ kwargs, _, _, has_terse = _TEMPLATE_CASES[template_name]+ assert TERSE_MARKER not in prompt_engine.load_prompt(template_name, **kwargs)+ assert TERSE_MARKER not in prompt_engine.load_prompt(template_name, slim_output="safe", **kwargs)+ assert (TERSE_MARKER in prompt_engine.load_prompt(template_name, slim_output="terse", **kwargs)) == has_terse++[email protected]("variant", [None, "safe", "terse"])+def test_extract_action_static_is_verbatim_prefix_in_every_variant(+ prompt_engine: PromptEngine, variant: str | None+) -> None:+ # The cached prompt path renders static + dynamic separately and joins them; if the+ # static file stops being a verbatim prefix, the cached path silently diverges.+ kwargs = dict(_EXTRACT_ACTION_KWARGS)+ if variant is not None:+ kwargs["slim_output"] = variant+ full = prompt_engine.load_prompt("extract-action", **kwargs)+ static = prompt_engine.load_prompt("extract-action-static", **kwargs)+ assert full.startswith(static.rstrip()), f"static is not a prefix of complete when slim_output={variant}"+++def test_templates_never_use_bare_slim_output_truthiness() -> None:+ # "off" (or any string) is truthy in Jinja: a bare {% if slim_output %} would slim+ # the control cohort the moment a caller passes a string. Only explicit+ # membership/equality checks are allowed.+ bare_truthiness = re.compile(r"\{%-?\s*if\s+(not\s+)?slim_output\s*[-]?%\}")+ for template_path in TEMPLATE_DIR.glob("*.j2"):+ content = template_path.read_text()+ assert not bare_truthiness.search(content), f"bare slim_output truthiness check in {template_path.name}"+ for usage in re.findall(r"\{%-?\s*if[^%]*slim_output[^%]*%\}", content):+ assert ("not in" in usage) or ("==" in usage), (+ f"non-explicit slim_output check in {template_path.name}: {usage}"+ )+++# ---------------------------------------------------------------------------+# Canonical family maps+# ---------------------------------------------------------------------------+++def test_family_maps_are_consistent_and_disjoint() -> None:+ assert set(TEMPLATES_BY_FAMILY) == set(PromptFamily)+ assert set(PROMPT_NAMES_BY_FAMILY) == set(PromptFamily)+ all_templates = [t for templates in TEMPLATES_BY_FAMILY.values() for t in templates]+ all_prompt_names = [p for prompt_names in PROMPT_NAMES_BY_FAMILY.values() for p in prompt_names]+ assert len(all_templates) == len(set(all_templates)), "a template maps to two families"+ assert len(all_prompt_names) == len(set(all_prompt_names)), "a prompt_name maps to two families"+ assert SLIM_ENABLED_FAMILIES <= set(PromptFamily)+++def test_family_lookups() -> None:+ assert family_for_template("extract-action") == PromptFamily.EXTRACT_ACTIONS+ assert family_for_template("extract-action-static") == PromptFamily.EXTRACT_ACTIONS+ assert family_for_template("check-user-goal-with-termination") == PromptFamily.CHECK_USER_GOAL+ assert family_for_prompt_name("extract-actions") == PromptFamily.EXTRACT_ACTIONS+ assert family_for_prompt_name("check-user-goal-after-click") == PromptFamily.CHECK_USER_GOAL+ assert family_for_template("decisive-criterion-validate") is None+ assert family_for_prompt_name("extract-information") is None+ assert family_for_template(None) is None+ assert family_for_prompt_name(None) is None+++def test_effective_prompt_schema_variant() -> None:+ assert effective_prompt_schema_variant(SLIM_VARIANT_SAFE, "extract-actions") == SLIM_VARIANT_SAFE+ assert effective_prompt_schema_variant(SLIM_VARIANT_TERSE, "check-user-goal-after-click") == SLIM_VARIANT_TERSE+ # Families outside the allowlist render control, so their calls must be labeled control.+ assert effective_prompt_schema_variant(SLIM_VARIANT_SAFE, "auto-completion-choose-option") is None+ assert effective_prompt_schema_variant(SLIM_VARIANT_SAFE, "parse-input-or-select-context") is None+ # Unknown prompt_names and non-slim assignments fail safe to control.+ assert effective_prompt_schema_variant(SLIM_VARIANT_SAFE, "extract-information") is None+ assert effective_prompt_schema_variant(None, "extract-actions") is None+ assert effective_prompt_schema_variant("control", "extract-actions") is None+ assert effective_prompt_schema_variant("garbage", "extract-actions") is None+++# ---------------------------------------------------------------------------+# Run-level variant resolver+# ---------------------------------------------------------------------------+++ "flag_variant,expected_template_value",+ [(SLIM_VARIANT_SAFE, "safe"), (SLIM_VARIANT_TERSE, "terse"), ("control", None), (None, None), ("garbage", None)],+)+async def test_resolver_maps_flag_variant_to_template_value(+ monkeypatch: pytest.MonkeyPatch, run_context: SkyvernContext, flag_variant: Any, expected_template_value: Any+) -> None:+ provider = _mock_provider(monkeypatch, flag_variant)+ assert await get_slim_output_template_value("extract-action") == expected_template_value+ provider.get_value_cached.assert_awaited_once_with(+ "SLIM_LLM_OUTPUT_PROMPTS",+ "wr_123",+ properties={"organization_id": "org_456", "workflow_permanent_id": None},+ )+++async def test_resolver_resolves_once_per_run(monkeypatch: pytest.MonkeyPatch, run_context: SkyvernContext) -> None:+ provider = _mock_provider(monkeypatch, SLIM_VARIANT_SAFE)+ assert await get_slim_output_template_value("extract-action") == "safe"+ assert await get_slim_output_template_value("check-user-goal") == "safe"+ assert await get_slim_output_template_value("check-user-goal-with-termination") == "safe"+ assert provider.get_value_cached.await_count == 1+ assert run_context.slim_output_variant_assigned == SLIM_VARIANT_SAFE+ assert run_context.slim_output_variant_resolved is True+++async def test_resolver_excludes_families_outside_allowlist(+ monkeypatch: pytest.MonkeyPatch, run_context: SkyvernContext+) -> None:+ _mock_provider(monkeypatch, SLIM_VARIANT_SAFE)+ assert await get_slim_output_template_value("auto-completion-choose-option") is None+ assert await get_slim_output_template_value("parse-input-or-select-context") is None+ # The assignment is still recorded for telemetry even though these render control.+ assert run_context.slim_output_variant_assigned == SLIM_VARIANT_SAFE+++async def test_resolver_returns_none_for_unknown_template(+ monkeypatch: pytest.MonkeyPatch, run_context: SkyvernContext+) -> None:+ _mock_provider(monkeypatch, SLIM_VARIANT_SAFE)+ assert await get_slim_output_template_value("decisive-criterion-validate") is None+ assert await get_slim_output_template_value("single-click-action") is None+++async def test_resolver_pins_run_to_control_on_flag_error(+ monkeypatch: pytest.MonkeyPatch, run_context: SkyvernContext+) -> None:+ provider = _mock_provider(monkeypatch, RuntimeError("posthog down"))+ assert await get_slim_output_template_value("extract-action") is None+ assert run_context.slim_output_variant_assigned is None+ assert run_context.slim_output_variant_resolved is True+ # Resolution is pinned for the whole run — no per-call re-evaluation flapping.+ assert await get_slim_output_template_value("extract-action") is None+ assert provider.get_value_cached.await_count == 1+++async def test_resolver_returns_none_without_context(monkeypatch: pytest.MonkeyPatch) -> None:+ _mock_provider(monkeypatch, SLIM_VARIANT_SAFE)+ skyvern_context.reset()+ assert await get_slim_output_template_value("extract-action") is None+++async def test_concurrent_first_use_resolves_exactly_once(+ monkeypatch: pytest.MonkeyPatch, run_context: SkyvernContext+) -> None:+ # Parallel prompt builds (speculative extract-actions + verification) can hit the+ # resolver before any resolution exists; single-flight must guarantee one flag+ # evaluation and one consistent cohort for the whole run.+ import asyncio++ provider = MagicMock()++ async def _slow_get_value(*args: Any, **kwargs: Any) -> str:+ await asyncio.sleep(0.02)+ return SLIM_VARIANT_SAFE++ provider.get_value_cached = AsyncMock(side_effect=_slow_get_value)+ mock_app = MagicMock()+ mock_app.EXPERIMENTATION_PROVIDER = provider+ monkeypatch.setattr(slim_llm_output, "app", mock_app)++ results = await asyncio.gather(+ get_slim_output_template_value("extract-action"),+ get_slim_output_template_value("check-user-goal"),+ get_slim_output_template_value("extract-action"),+ )+ assert results == ["safe", "safe", "safe"]+ assert provider.get_value_cached.await_count == 1+ assert run_context.slim_output_variant_assigned == SLIM_VARIANT_SAFE+++async def test_concurrent_first_use_with_flag_error_pins_all_callers_to_control(+ monkeypatch: pytest.MonkeyPatch, run_context: SkyvernContext+) -> None:+ import asyncio++ provider = MagicMock()++ async def _slow_failure(*args: Any, **kwargs: Any) -> str:+ await asyncio.sleep(0.02)+ raise RuntimeError("posthog down")++ provider.get_value_cached = AsyncMock(side_effect=_slow_failure)+ mock_app = MagicMock()+ mock_app.EXPERIMENTATION_PROVIDER = provider+ monkeypatch.setattr(slim_llm_output, "app", mock_app)++ results = await asyncio.gather(+ get_slim_output_template_value("extract-action"),+ get_slim_output_template_value("check-user-goal"),+ )+ assert results == [None, None]+ assert provider.get_value_cached.await_count == 1+ assert run_context.slim_output_variant_assigned is None+ assert run_context.slim_output_variant_resolved is True+++async def test_resolver_pins_control_without_run_identifiers(monkeypatch: pytest.MonkeyPatch) -> None:+ provider = _mock_provider(monkeypatch, SLIM_VARIANT_SAFE)+ context = SkyvernContext(organization_id="org_456")+ skyvern_context.set(context)+ try:+ assert await get_slim_output_template_value("extract-action") is None+ assert await get_slim_output_template_value("extract-action") is None+ provider.get_value_cached.assert_not_awaited()+ # Pinned resolved so later renders skip the lock instead of re-entering forever.+ assert context.slim_output_variant_resolved is True+ assert context.slim_output_variant_assigned is None+ finally:+ skyvern_context.reset()+++# ---------------------------------------------------------------------------+# Cache-variant key+# ---------------------------------------------------------------------------+++def test_cache_variant_encodes_slim_arm_and_control_key_is_unchanged() -> None:+ from skyvern.forge.agent import ForgeAgent++ build = ForgeAgent._build_extract_action_cache_variant+ control = build(verification_code_check=False, show_close_page_action=False, complete_criterion=None)+ safe = build(+ verification_code_check=False, show_close_page_action=False, complete_criterion=None, slim_output="safe"+ )+ terse = build(+ verification_code_check=False, show_close_page_action=False, complete_criterion=None, slim_output="terse"+ )+ assert control == "std"+ assert safe == "slim_safe"+ assert terse == "slim_terse"+ assert len({control, safe, terse}) == 3++ otp_control = build(verification_code_check=True, show_close_page_action=False, complete_criterion=None)+ otp_safe = build(+ verification_code_check=True, show_close_page_action=False, complete_criterion=None, slim_output="safe"+ )+ assert otp_control == "vc"+ assert otp_safe == "vc-slim_safe"