Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
ai_adapt_value for text input (#3354)
Shuchang Zheng55d847461ecb
diff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.pyindex ab14134cc..f74ad07dc 100644--- a/skyvern/core/script_generations/generate_script.py+++ b/skyvern/core/script_generations/generate_script.py@@ -126,12 +126,12 @@ def _generate_text_call(text_value: str, intention: str, parameter_key: str) ->last_line=cst.SimpleWhitespace(DOUBLE_INDENT),),args=[- # First positional argument: context.generated_parameters['parameter_key']+ # First positional argument: context.parameters['parameter_key']cst.Arg(value=cst.Subscript(value=cst.Attribute(value=cst.Name("context"),- attr=cst.Name("generated_parameters"),+ attr=cst.Name("parameters"),),slice=[cst.SubscriptElement(slice=cst.Index(value=_value(parameter_key)))],),@@ -247,20 +247,21 @@ def _action_to_stmt(act: dict[str, Any], assign_to_output: bool = False) -> cst.)if method in ["type", "fill"]:- # Get intention from action- intention = act.get("intention") or act.get("reasoning") or ""-- # Use generate_text call if field_name is available, otherwise fallback to direct value+ # Use context.parameters if field_name is available, otherwise fallback to direct valueif act.get("field_name"):- text_value = _generate_text_call(- text_value=act["text"], intention=intention, parameter_key=act["field_name"]+ text_value = cst.Subscript(+ value=cst.Attribute(+ value=cst.Name("context"),+ attr=cst.Name("parameters"),+ ),+ slice=[cst.SubscriptElement(slice=cst.Index(value=_value(act["field_name"])))],)else:text_value = _value(act["text"])args.append(cst.Arg(- keyword=cst.Name("text"),+ keyword=cst.Name("value"),value=text_value,whitespace_after_arg=cst.ParenthesizedWhitespace(indent=True,@@ -268,6 +269,16 @@ def _action_to_stmt(act: dict[str, Any], assign_to_output: bool = False) -> cst.),))+ args.append(+ cst.Arg(+ keyword=cst.Name("ai_adapt_value"),+ value=cst.Name("True"),+ whitespace_after_arg=cst.ParenthesizedWhitespace(+ indent=True,+ last_line=cst.SimpleWhitespace(INDENT),+ ),+ )+ )elif method == "select_option":args.append(cst.Arg(diff --git a/skyvern/core/script_generations/run_initializer.py b/skyvern/core/script_generations/run_initializer.pyindex 724d6a2f8..e8d2fb18d 100644--- a/skyvern/core/script_generations/run_initializer.py+++ b/skyvern/core/script_generations/run_initializer.py@@ -2,8 +2,7 @@ from typing import Anyfrom pydantic import BaseModel-from skyvern.core.script_generations.script_run_context_manager import script_run_context_manager-from skyvern.core.script_generations.skyvern_page import RunContext, SkyvernPage+from skyvern.core.script_generations.skyvern_page import RunContext, SkyvernPage, script_run_context_managerfrom skyvern.forge import appfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.workflow.models.parameter import WorkflowParameterTypediff --git a/skyvern/core/script_generations/script_run_context_manager.py b/skyvern/core/script_generations/script_run_context_manager.pydeleted file mode 100644index 3c96dcbd8..000000000--- a/skyvern/core/script_generations/script_run_context_manager.py+++ /dev/null@@ -1,34 +0,0 @@-from typing import Callable--from skyvern.core.script_generations.skyvern_page import RunContext---class ScriptRunContextManager:- """- Manages the run context for code runs.- """-- def __init__(self) -> None:- # self.run_contexts: dict[str, RunContext] = {}- self.run_context: RunContext | None = None- self.cached_fns: dict[str, Callable] = {}-- def get_run_context(self) -> RunContext | None:- return self.run_context-- def set_run_context(self, run_context: RunContext) -> None:- self.run_context = run_context-- def ensure_run_context(self) -> RunContext:- if not self.run_context:- raise Exception("Run context not found")- return self.run_context-- def set_cached_fn(self, cache_key: str, fn: Callable) -> None:- self.cached_fns[cache_key] = fn-- def get_cached_fn(self, cache_key: str) -> Callable | None:- return self.cached_fns.get(cache_key)---script_run_context_manager = ScriptRunContextManager()diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex 25c87a522..b587d6da6 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -8,6 +8,7 @@ from datetime import datetime, timezonefrom enum import StrEnumfrom typing import Any, Callable, Literal+import structlogfrom playwright.async_api import Pagefrom skyvern.config import settings@@ -24,6 +25,8 @@ from skyvern.webeye.actions.actions import Action, ActionStatus, ExtractAction,from skyvern.webeye.browser_factory import BrowserStatefrom skyvern.webeye.scraper.scraper import ScrapedPage, scrape_website+LOG = structlog.get_logger()+class Driver(StrEnum):PLAYWRIGHT = "playwright"@@ -196,7 +199,8 @@ class SkyvernPage:# Create action record. TODO: store more action fieldskwargs = kwargs or {}- text = kwargs.get("text")+ # we're using "value" instead of "text" for input text actions interface+ text = kwargs.get("value", "")option_value = kwargs.get("option")select_option = SelectOption(value=option_value) if option_value else Noneresponse: str | None = kwargs.get("response")@@ -314,7 +318,7 @@ class SkyvernPage:current_url=self.page.url,elements=element_tree,local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),- user_context=getattr(context, "prompt", None),+ # user_context=getattr(context, "prompt", None),)json_response = await app.SINGLE_CLICK_AGENT_LLM_API_HANDLER(prompt=single_click_prompt,@@ -334,28 +338,31 @@ class SkyvernPage:async def fill(self,xpath: str,- text: str,+ value: str,+ ai_adapt_value: bool = False,intention: str | None = None,data: str | dict[str, Any] | None = None,timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,) -> None:- await self._input_text(xpath, text, intention, data, timeout)+ await self._input_text(xpath, value, ai_adapt_value, intention, data, timeout)@action_wrap(ActionType.INPUT_TEXT)async def type(self,xpath: str,- text: str,+ value: str,+ ai_adapt_value: bool = False,intention: str | None = None,data: str | dict[str, Any] | None = None,timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,) -> None:- await self._input_text(xpath, text, intention, data, timeout)+ await self._input_text(xpath, value, ai_adapt_value, intention, data, timeout)async def _input_text(self,xpath: str,- text: str,+ value: str,+ ai_adapt_value: bool = False,intention: str | None = None,data: str | dict[str, Any] | None = None,timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,@@ -372,11 +379,33 @@ class SkyvernPage:"""# format the text with the actual value of the parameter if it's a secret when running a workflowcontext = skyvern_context.current()+ value = value or ""if context and context.workflow_run_id:- text = await _get_actual_value_of_parameter_if_secret(context.workflow_run_id, text)+ value = await _get_actual_value_of_parameter_if_secret(context.workflow_run_id, value)++ if ai_adapt_value and intention:+ try:+ prompt = context.prompt if context else None+ # Build the element tree of the current page for the prompt+ # clean up empty data values+ data = {k: v for k, v in data.items() if v} if isinstance(data, dict) else (data or "")+ payload_str = json.dumps(data) if isinstance(data, (dict, list)) else (data or "")+ script_generation_input_text_prompt = prompt_engine.load_prompt(+ template="script-generation-input-text-generatiion",+ intention=intention,+ data=payload_str,+ goal=prompt,+ )+ json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(+ prompt=script_generation_input_text_prompt,+ prompt_name="script-generation-input-text-generatiion",+ )+ value = json_response.get("answer", value)+ except Exception:+ LOG.exception(f"Failed to adapt value for input text action on xpath={xpath}, value={value}")locator = self.page.locator(f"xpath={xpath}")- await handler_utils.input_sequentially(locator, text, timeout=timeout)+ await handler_utils.input_sequentially(locator, value, timeout=timeout)@action_wrap(ActionType.UPLOAD_FILE)async def upload_file(@@ -542,11 +571,13 @@ class RunContext:self.original_parameters = parametersself.generated_parameters = generated_parametersself.parameters = copy.deepcopy(parameters)- # if generated_parameters:- # self.parameters.update(generated_parameters)+ if generated_parameters:+ # hydrate the generated parameter fields in the run context parameters+ for key, value in generated_parameters.items():+ if key not in self.parameters:+ self.parameters[key] = valueself.page = pageself.trace: list[ActionCall] = []- self.prompt: str | None = Noneasync def _get_actual_value_of_parameter_if_secret(workflow_run_id: str, parameter: str) -> Any:@@ -560,3 +591,34 @@ async def _get_actual_value_of_parameter_if_secret(workflow_run_id: str, parametworkflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(workflow_run_id)secret_value = workflow_run_context.get_original_secret_value_or_none(parameter)return secret_value if secret_value is not None else parameter+++class ScriptRunContextManager:+ """+ Manages the run context for code runs.+ """++ def __init__(self) -> None:+ # self.run_contexts: dict[str, RunContext] = {}+ self.run_context: RunContext | None = None+ self.cached_fns: dict[str, Callable] = {}++ def get_run_context(self) -> RunContext | None:+ return self.run_context++ def set_run_context(self, run_context: RunContext) -> None:+ self.run_context = run_context++ def ensure_run_context(self) -> RunContext:+ if not self.run_context:+ raise Exception("Run context not found")+ return self.run_context++ def set_cached_fn(self, cache_key: str, fn: Callable) -> None:+ self.cached_fns[cache_key] = fn++ def get_cached_fn(self, cache_key: str) -> Callable | None:+ return self.cached_fns.get(cache_key)+++script_run_context_manager = ScriptRunContextManager()diff --git a/skyvern/core/script_generations/workflow_wrappers.py b/skyvern/core/script_generations/workflow_wrappers.pyindex db4b792bc..e7bb0c6ed 100644--- a/skyvern/core/script_generations/workflow_wrappers.py+++ b/skyvern/core/script_generations/workflow_wrappers.py@@ -1,7 +1,6 @@from typing import Any, Callable-from skyvern import RunContext, SkyvernPage-from skyvern.core.script_generations.script_run_context_manager import script_run_context_manager+from skyvern.core.script_generations.skyvern_page import RunContext, SkyvernPage, script_run_context_manager# Build a dummy workflow decoratordiff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex bc602729b..298f600bc 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -30,6 +30,7 @@ class SkyvernContext:script_id: str | None = Nonescript_revision_id: str | None = Noneaction_order: int = 0+ prompt: str | None = Nonedef __repr__(self) -> str:return f"SkyvernContext(request_id={self.request_id}, organization_id={self.organization_id}, task_id={self.task_id}, step_id={self.step_id}, workflow_id={self.workflow_id}, workflow_run_id={self.workflow_run_id}, task_v2_id={self.task_v2_id}, max_steps_override={self.max_steps_override}, run_id={self.run_id})"diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 7c15a3999..ed963360c 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -16,7 +16,7 @@ from skyvern.config import settingsfrom skyvern.constants import GET_DOWNLOADED_FILES_TIMEOUTfrom skyvern.core.script_generations.constants import SCRIPT_TASK_BLOCKSfrom skyvern.core.script_generations.generate_script import _build_block_fn, create_script_block-from skyvern.core.script_generations.script_run_context_manager import script_run_context_manager+from skyvern.core.script_generations.skyvern_page import script_run_context_managerfrom skyvern.exceptions import ScriptNotFound, WorkflowRunNotFoundfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine@@ -942,8 +942,8 @@ async def run_task(url=url,)# set the prompt in the RunContext- run_context = script_run_context_manager.ensure_run_context()- run_context.prompt = prompt+ context = skyvern_context.ensure_context()+ context.prompt = promptif cache_key:try:@@ -972,7 +972,7 @@ async def run_task()finally:# clear the prompt in the RunContext- run_context.prompt = None+ context.prompt = Noneelse:if workflow_run_block_id:await _update_workflow_block(@@ -984,7 +984,7 @@ async def run_task(step_status=StepStatus.failed,failure_reason="Cache key is required",)- run_context.prompt = None+ context.prompt = Noneraise Exception("Cache key is required to run task block in a script")@@ -1001,8 +1001,8 @@ async def download(url=url,)# set the prompt in the RunContext- run_context = script_run_context_manager.ensure_run_context()- run_context.prompt = prompt+ context = skyvern_context.ensure_context()+ context.prompt = promptif cache_key:try:@@ -1031,7 +1031,7 @@ async def download(workflow_run_block_id=workflow_run_block_id,)finally:- run_context.prompt = None+ context.prompt = Noneelse:if workflow_run_block_id:await _update_workflow_block(@@ -1043,7 +1043,7 @@ async def download(step_status=StepStatus.failed,failure_reason="Cache key is required",)- run_context.prompt = None+ context.prompt = Noneraise Exception("Cache key is required to run task block in a script")@@ -1060,8 +1060,8 @@ async def action(url=url,)# set the prompt in the RunContext- run_context = script_run_context_manager.ensure_run_context()- run_context.prompt = prompt+ context = skyvern_context.ensure_context()+ context.prompt = promptif cache_key:try:@@ -1089,7 +1089,7 @@ async def action(workflow_run_block_id=workflow_run_block_id,)finally:- run_context.prompt = None+ context.prompt = Noneelse:if workflow_run_block_id:await _update_workflow_block(@@ -1101,7 +1101,7 @@ async def action(step_status=StepStatus.failed,failure_reason="Cache key is required",)- run_context.prompt = None+ context.prompt = Noneraise Exception("Cache key is required to run task block in a script")@@ -1118,8 +1118,8 @@ async def login(url=url,)# set the prompt in the RunContext- run_context = script_run_context_manager.ensure_run_context()- run_context.prompt = prompt+ context = skyvern_context.ensure_context()+ context.prompt = promptif cache_key:try:@@ -1147,7 +1147,7 @@ async def login(workflow_run_block_id=workflow_run_block_id,)finally:- run_context.prompt = None+ context.prompt = Noneelse:if workflow_run_block_id:await _update_workflow_block(@@ -1159,7 +1159,7 @@ async def login(step_status=StepStatus.failed,failure_reason="Cache key is required",)- run_context.prompt = None+ context.prompt = Noneraise Exception("Cache key is required to run task block in a script")@@ -1178,8 +1178,8 @@ async def extract(url=url,)# set the prompt in the RunContext- run_context = script_run_context_manager.ensure_run_context()- run_context.prompt = prompt+ context = skyvern_context.ensure_context()+ context.prompt = promptoutput: dict[str, Any] | list | str | None = Noneif cache_key:@@ -1213,7 +1213,7 @@ async def extract()raisefinally:- run_context.prompt = None+ context.prompt = Noneelse:if workflow_run_block_id:await _update_workflow_block(@@ -1225,7 +1225,7 @@ async def extract(step_status=StepStatus.failed,failure_reason="Cache key is required",)- run_context.prompt = None+ context.prompt = Noneraise Exception("Cache key is required to run task block in a script")@@ -1296,8 +1296,8 @@ async def generate_text(new_text = text or ""if intention and data:try:- run_context = script_run_context_manager.ensure_run_context()- prompt = run_context.prompt+ context = skyvern_context.ensure_context()+ prompt = context.prompt# Build the element tree of the current page for the promptpayload_str = json.dumps(data) if isinstance(data, (dict, list)) else (data or "")script_generation_input_text_prompt = prompt_engine.load_prompt(