Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
ai element by prompt
Gregor Žunič18b6145ba22f
diff --git a/browser_use/actor/playground/flights.py b/browser_use/actor/playground/flights.pynew file mode 100644index 000000000..9f5b68d25--- /dev/null+++ b/browser_use/actor/playground/flights.py@@ -0,0 +1,36 @@+import asyncio++from browser_use import Browser, ChatOpenAI++llm = ChatOpenAI('gpt-4.1-mini')+++async def main():+ """+ Main function demonstrating mixed automation with Browser-Use and Playwright.+ """+ print('🚀 Mixed Automation with Browser-Use and Actor API')++ browser = Browser(keep_alive=True)+ await browser.start()++ target = await browser.get_current_target() or await browser.newTarget()++ # Go to apple wikipedia page+ await target.goto('https://www.google.com/travel/flights')++ await asyncio.sleep(2)++ round_trip_button = await target.mustGetElementByPrompt('round trip button', llm)+ await round_trip_button.click()++ one_way_button = await target.mustGetElementByPrompt('one way button', llm)+ await one_way_button.click()++ await asyncio.sleep(0.5)++ await browser.stop()+++if __name__ == '__main__':+ asyncio.run(main())diff --git a/browser_use/actor/playground/mixed_automation.py b/browser_use/actor/playground/mixed_automation.pyindex 1c8f28807..f164e7249 100644--- a/browser_use/actor/playground/mixed_automation.py+++ b/browser_use/actor/playground/mixed_automation.py@@ -2,7 +2,7 @@ import asynciofrom pydantic import BaseModel-from browser_use import Agent, Browser+from browser_use import Browser, ChatOpenAITASK = """On the current wikipedia page, find the latest huge edit and tell me what is was about.@@ -19,6 +19,9 @@ class LatestEditFinder(BaseModel):edit_url: str+llm = ChatOpenAI('gpt-4.1-mini')++async def main():"""Main function demonstrating mixed automation with Browser-Use and Playwright.@@ -28,31 +31,22 @@ async def main():browser = Browser(keep_alive=True)await browser.start()- apple_target = await browser.get_current_target() or await browser.newTarget()+ target = await browser.get_current_target() or await browser.newTarget()# Go to apple wikipedia page- await apple_target.goto('https://en.wikipedia.org/wiki/Apple_Inc.')+ await target.goto('https://browser-use.github.io/stress-tests/challenges/angularjs-form.html')await asyncio.sleep(1)- element = await apple_target.getElementsByCSSSelector(- '#mw-content-text > div.mw-content-ltr.mw-parser-output > p:nth-child(8) > a:nth-child(3)'- )+ element = await target.getElementByPrompt('zip code input', llm)++ print('Element found', element)- if first_element := element[0]:- print('Element found', first_element)- await first_element.click()+ if element:+ await element.click()else:print('No element found')- agent = Agent(- task='clic on the button that says "technology company"',- browser=browser,- )- output = await agent.run(max_steps=1)-- await asyncio.sleep(10)-await browser.stop()diff --git a/browser_use/actor/target.py b/browser_use/actor/target.pyindex c9fe349d9..554714b58 100644--- a/browser_use/actor/target.py+++ b/browser_use/actor/target.py@@ -2,6 +2,12 @@from typing import TYPE_CHECKING+from pydantic import BaseModel++from browser_use.dom.serializer.serializer import DOMTreeSerializer+from browser_use.dom.service import DomService+from browser_use.llm.messages import SystemMessage, UserMessage+if TYPE_CHECKING:from cdp_use.cdp.dom.commands import (DescribeNodeParameters,@@ -38,6 +44,8 @@ class Target:self._session_id: str | None = session_idself._mouse: 'Mouse | None' = None+ self._llm = llm+async def _ensure_session(self) -> str:"""Ensure we have a session ID for this target."""if not self._session_id:@@ -340,3 +348,100 @@ class Target:elements.append(Element(self._browser_session, backend_node_id, session_id))return elements++ # AI METHODS++ @property+ def dom_service(self) -> 'DomService':+ """Get the DOM service for this target."""+ return DomService(self._browser_session)++ async def getElementByPrompt(self, prompt: str, llm: 'BaseChatModel | None' = None) -> 'Element | None':+ """Get an element by a prompt."""+ await self._ensure_session()+ llm = llm or self._llm++ if not llm:+ raise ValueError('LLM not provided')++ dom_service = self.dom_service++ enhanced_dom_tree = await dom_service.get_dom_tree(target_id=self._target_id)++ serialized_dom_state, _ = DOMTreeSerializer(+ enhanced_dom_tree, None, paint_order_filtering=True+ ).serialize_accessible_elements()++ llm_representation = serialized_dom_state.llm_representation()++ system_message = SystemMessage(+ content="""You are an AI created to find an element on a page by a prompt.++<browser_state>+Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where+- index: Numeric identifier for interaction+- type: HTML element type (button, input, etc.)+- text: Element description++Examples:+[33]<div>User form</div>+[35]<button aria-label='Submit form'>Submit</button>++Note that:+- Only elements with numeric indexes in [] are interactive+- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)+- Pure text elements without [] are not interactive.+</browser_state>++Your task is to find an element index (if any) that matches the prompt (written in <prompt> tag).++If non of the elements matches the, return None.++Before you return the element index, reason about the state and elements for a sentence or two."""+ )++ state_message = UserMessage(+ content=f"""+ <browser_state>+ {llm_representation}+ </browser_state>++ <prompt>+ {prompt}+ </prompt>+ """+ )++ class ElementResponse(BaseModel):+ # thinking: str+ element_highlight_index: int | None++ llm_response = await llm.ainvoke(+ [+ system_message,+ state_message,+ ],+ output_format=ElementResponse,+ )++ element_highlight_index = llm_response.completion.element_highlight_index++ if element_highlight_index is None or element_highlight_index not in serialized_dom_state.selector_map:+ return None++ element = serialized_dom_state.selector_map[element_highlight_index]++ from .element import Element++ return Element(self._browser_session, element.backend_node_id, self._session_id)++ async def mustGetElementByPrompt(self, prompt: str, llm: 'BaseChatModel | None' = None) -> 'Element':+ """Get an element by a prompt.++ @dev LLM can still return None, this just raises an error if the element is not found.+ """+ element = await self.getElementByPrompt(prompt, llm)+ if element is None:+ raise ValueError(f'No element found for prompt: {prompt}')++ return element