Skip to content

Automate browser based workflows with AI

fix(SKY-9344): guard against missing parent/children in element tree builder (#5736)

LawyZheng4ffca27f6928
diff --git a/skyvern/config.py b/skyvern/config.py
index c784961f9..34249e404 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -250,6 +250,7 @@ class Settings(BaseSettings):
# OPENAI
OPENAI_API_KEY: str | None = None
GPT5_REASONING_EFFORT: str | None = "medium"
+ OPENAI_CUA_MODEL: str = "computer-use-preview"
# ANTHROPIC
ANTHROPIC_API_KEY: str | None = None
ANTHROPIC_CUA_LLM_KEY: str = "ANTHROPIC_CLAUDE4.6_SONNET"
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index e33a7a686..abe60a277 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -1841,10 +1841,11 @@ class ForgeAgent:
previous_response: OpenAIResponse | None = None,
engine: RunEngine = RunEngine.openai_cua,
) -> tuple[list[Action], OpenAIResponse | None]:
+ cua_model = app.OPENAI_CUA_MODEL
if not previous_response:
# this is the first step
first_response: OpenAIResponse = await app.OPENAI_CLIENT.responses.create(
- model="computer-use-preview",
+ model=cua_model,
tools=[
{
"type": "computer_use_preview",
@@ -1931,7 +1932,7 @@ class ForgeAgent:
if not resp_content:
resp_content = "I don't know. Can you help me make the best decision to achieve the goal?"
current_response = await app.OPENAI_CLIENT.responses.create(
- model="computer-use-preview",
+ model=cua_model,
previous_response_id=previous_response.id,
tools=[
{
@@ -1963,7 +1964,7 @@ class ForgeAgent:
computer_call_input["acknowledged_safety_checks"] = pending_checks
current_response = await app.OPENAI_CLIENT.responses.create(
- model="computer-use-preview",
+ model=cua_model,
previous_response_id=previous_response.id,
tools=[
{
diff --git a/skyvern/forge/forge_app.py b/skyvern/forge/forge_app.py
index 3e0aa80b1..5f2d8e35d 100644
--- a/skyvern/forge/forge_app.py
+++ b/skyvern/forge/forge_app.py
@@ -56,6 +56,7 @@ class ForgeApp:
RATE_LIMITER: RateLimiter
LLM_API_HANDLER: LLMAPIHandler
OPENAI_CLIENT: AsyncOpenAI | AsyncAzureOpenAI
+ OPENAI_CUA_MODEL: str
ANTHROPIC_CLIENT: AsyncAnthropic | AsyncAnthropicBedrock
UI_TARS_CLIENT: AsyncOpenAI | None
AZURE_CLIENT_FACTORY: AzureClientFactory
@@ -119,6 +120,7 @@ def create_forge_app() -> ForgeApp:
app.RATE_LIMITER = NoopRateLimiter()
app.LLM_API_HANDLER = LLMAPIHandlerFactory.get_llm_api_handler(settings.LLM_KEY)
+ app.OPENAI_CUA_MODEL = settings.OPENAI_CUA_MODEL
app.OPENAI_CLIENT = AsyncOpenAI(
api_key=settings.OPENAI_API_KEY or "",
http_client=ForgeAsyncHttpxClientWrapper(),
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index 720eb1cc8..f8c6553e8 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -1863,14 +1863,30 @@ async function buildElementTree(
// under it
if (parentId === null) {
resultArray.push(elementObj);
- }
- // If the element is interactable and has an interactable parent,
- // then add it to the children of the parent
- else {
- // TODO: use dict/object so that we access these in O(1) instead
- elements
- .find((element) => element.id === parentId)
- .children.push(elementObj);
+ } else {
+ const parentElement = elements.find(
+ (element) => element.id === parentId,
+ );
+ if (parentElement) {
+ if (!parentElement.children) {
+ _jsConsoleWarn(
+ "parent element missing children array, reinitializing",
+ "parentId=" + parentId,
+ "parentTag=" + parentElement.tagName,
+ "childId=" + elementObj.id,
+ );
+ parentElement.children = [];
+ }
+ parentElement.children.push(elementObj);
+ } else {
+ _jsConsoleWarn(
+ "parent element not found in elements array, treating as root",
+ "parentId=" + parentId,
+ "childId=" + elementObj.id,
+ "childTag=" + elementObj.tagName,
+ );
+ resultArray.push(elementObj);
+ }
}
parentId = elementObj.id;
}