Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9344): guard against missing parent/children in element tree builder (#5736)
LawyZheng4ffca27f6928
diff --git a/skyvern/config.py b/skyvern/config.pyindex c784961f9..34249e404 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -250,6 +250,7 @@ class Settings(BaseSettings):# OPENAIOPENAI_API_KEY: str | None = NoneGPT5_REASONING_EFFORT: str | None = "medium"+ OPENAI_CUA_MODEL: str = "computer-use-preview"# ANTHROPICANTHROPIC_API_KEY: str | None = NoneANTHROPIC_CUA_LLM_KEY: str = "ANTHROPIC_CLAUDE4.6_SONNET"diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex e33a7a686..abe60a277 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -1841,10 +1841,11 @@ class ForgeAgent:previous_response: OpenAIResponse | None = None,engine: RunEngine = RunEngine.openai_cua,) -> tuple[list[Action], OpenAIResponse | None]:+ cua_model = app.OPENAI_CUA_MODELif not previous_response:# this is the first stepfirst_response: OpenAIResponse = await app.OPENAI_CLIENT.responses.create(- model="computer-use-preview",+ model=cua_model,tools=[{"type": "computer_use_preview",@@ -1931,7 +1932,7 @@ class ForgeAgent:if not resp_content:resp_content = "I don't know. Can you help me make the best decision to achieve the goal?"current_response = await app.OPENAI_CLIENT.responses.create(- model="computer-use-preview",+ model=cua_model,previous_response_id=previous_response.id,tools=[{@@ -1963,7 +1964,7 @@ class ForgeAgent:computer_call_input["acknowledged_safety_checks"] = pending_checkscurrent_response = await app.OPENAI_CLIENT.responses.create(- model="computer-use-preview",+ model=cua_model,previous_response_id=previous_response.id,tools=[{diff --git a/skyvern/forge/forge_app.py b/skyvern/forge/forge_app.pyindex 3e0aa80b1..5f2d8e35d 100644--- a/skyvern/forge/forge_app.py+++ b/skyvern/forge/forge_app.py@@ -56,6 +56,7 @@ class ForgeApp:RATE_LIMITER: RateLimiterLLM_API_HANDLER: LLMAPIHandlerOPENAI_CLIENT: AsyncOpenAI | AsyncAzureOpenAI+ OPENAI_CUA_MODEL: strANTHROPIC_CLIENT: AsyncAnthropic | AsyncAnthropicBedrockUI_TARS_CLIENT: AsyncOpenAI | NoneAZURE_CLIENT_FACTORY: AzureClientFactory@@ -119,6 +120,7 @@ def create_forge_app() -> ForgeApp:app.RATE_LIMITER = NoopRateLimiter()app.LLM_API_HANDLER = LLMAPIHandlerFactory.get_llm_api_handler(settings.LLM_KEY)+ app.OPENAI_CUA_MODEL = settings.OPENAI_CUA_MODELapp.OPENAI_CLIENT = AsyncOpenAI(api_key=settings.OPENAI_API_KEY or "",http_client=ForgeAsyncHttpxClientWrapper(),diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex 720eb1cc8..f8c6553e8 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -1863,14 +1863,30 @@ async function buildElementTree(// under itif (parentId === null) {resultArray.push(elementObj);- }- // If the element is interactable and has an interactable parent,- // then add it to the children of the parent- else {- // TODO: use dict/object so that we access these in O(1) instead- elements- .find((element) => element.id === parentId)- .children.push(elementObj);+ } else {+ const parentElement = elements.find(+ (element) => element.id === parentId,+ );+ if (parentElement) {+ if (!parentElement.children) {+ _jsConsoleWarn(+ "parent element missing children array, reinitializing",+ "parentId=" + parentId,+ "parentTag=" + parentElement.tagName,+ "childId=" + elementObj.id,+ );+ parentElement.children = [];+ }+ parentElement.children.push(elementObj);+ } else {+ _jsConsoleWarn(+ "parent element not found in elements array, treating as root",+ "parentId=" + parentId,+ "childId=" + elementObj.id,+ "childTag=" + elementObj.tagName,+ );+ resultArray.push(elementObj);+ }}parentId = elementObj.id;}