Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
rename ClickElementEvent.new_tab to while_holding_ctrl
Nick Sweeting99a0e5a3bd93
diff --git a/browser_use/browser/default_action_watchdog.py b/browser_use/browser/default_action_watchdog.pyindex 1a256efd9..0dfc68875 100644--- a/browser_use/browser/default_action_watchdog.py+++ b/browser_use/browser/default_action_watchdog.py@@ -62,7 +62,7 @@ class DefaultActionWatchdog(BaseWatchdog):)# Perform the actual click using internal implementation- await self._click_element_node_impl(element_node, new_tab=event.new_tab)+ await self._click_element_node_impl(element_node, while_holding_ctrl=event.while_holding_ctrl)download_path = None # moved to downloads_watchdog.py# Build success message@@ -97,12 +97,12 @@ class DefaultActionWatchdog(BaseWatchdog):msg += f' - {new_tab_msg}'self.logger.info(f'π {new_tab_msg}')- if not event.new_tab:- # if new_tab=False it means agent was not expecting a new tab to be opened+ if not event.while_holding_ctrl:+ # if while_holding_ctrl=False it means agent was not expecting a new tab to be opened# so we need to switch to the new tab to make the agent aware of the surprise new tab that was opened.- # slightly counter-intuitive, when new_tab=True we dont actually want to switch to it,- # the agent is instructed that new_tab=True is equivalent to ctrl+click which opens in the background,- # so in multi_act it usually already sends [click_element_by_index(123, new_tab=True), switch_tab(-1)] anyway+ # when while_holding_ctrl=True we dont actually want to switch to it,+ # we should match human expectations of ctrl+click which opens in the background,+ # so in multi_act it usually already sends [click_element_by_index(123, while_holding_ctrl=True), switch_tab(tab_id=None)] anywayfrom browser_use.browser.events import SwitchTabEventnew_target_id = new_target_ids.pop()@@ -220,7 +220,7 @@ class DefaultActionWatchdog(BaseWatchdog):# ========== Implementation Methods ==========- async def _click_element_node_impl(self, element_node, new_tab: bool = False) -> str | None:+ async def _click_element_node_impl(self, element_node, while_holding_ctrl: bool = False) -> str | None:"""Click an element using pure CDP with multiple fallback methods for getting element geometry.@@ -452,7 +452,7 @@ class DefaultActionWatchdog(BaseWatchdog):# Calculate modifier bitmask for CDP# CDP Modifier bits: Alt=1, Control=2, Meta/Command=4, Shift=8modifiers = 0- if new_tab:+ if while_holding_ctrl:# Use platform-appropriate modifier for "open in new tab"if platform.system() == 'Darwin':modifiers = 4 # Meta/Cmd keydiff --git a/browser_use/browser/dom_watchdog.py b/browser_use/browser/dom_watchdog.pyindex 4038add4d..dc0992715 100644--- a/browser_use/browser/dom_watchdog.py+++ b/browser_use/browser/dom_watchdog.py@@ -107,7 +107,7 @@ class DOMWatchdog(BaseWatchdog):# Re-raise other errorsraise- def _get_recent_events_csv(self, limit: int = 10) -> str | None:+ def _get_recent_events_str(self, limit: int = 10) -> str | None:"""Get the most recent event names from the event bus as CSV.Args:@@ -124,6 +124,9 @@ class DOMWatchdog(BaseWatchdog):# Take the most recent events and get their namesrecent_event_names = [event.event_type for event in all_events[:limit]]+ # TODO: in the future dump these as JSON instead of a CSV of the event names only+ # some_event.model_dump(mode='json', exclude=some_event.event_builtin_fields)+ # include event_results summarized / truncated to some reasonable lengthif recent_event_names:return ', '.join(recent_event_names)@@ -173,6 +176,7 @@ class DOMWatchdog(BaseWatchdog):self.logger.debug('π DOMWatchdog.on_BrowserStateRequestEvent: Getting tabs info...')tabs_info = await self.browser_session.get_tabs()self.logger.debug(f'π DOMWatchdog.on_BrowserStateRequestEvent: Got {len(tabs_info)} tabs')+ self.logger.debug(f'π DOMWatchdog.on_BrowserStateRequestEvent: Tabs info: {tabs_info}')# Get viewport / scroll position info, remember changing scroll position should invalidate selector_map cache because it only includes visible elements# cdp_session = await self.browser_session.get_or_create_cdp_session(focus=True)@@ -225,7 +229,7 @@ class DOMWatchdog(BaseWatchdog):pixels_below=0,browser_errors=[],is_pdf_viewer=False,- recent_events=self._get_recent_events_csv() if event.include_recent_events else None,+ recent_events=self._get_recent_events_str() if event.include_recent_events else None,)# Normal path: Build DOM tree if requested@@ -347,7 +351,7 @@ class DOMWatchdog(BaseWatchdog):pixels_below=0,browser_errors=[],is_pdf_viewer=is_pdf_viewer,- recent_events=self._get_recent_events_csv() if event.include_recent_events else None,+ recent_events=self._get_recent_events_str() if event.include_recent_events else None,)# Cache the statediff --git a/browser_use/browser/events.py b/browser_use/browser/events.pyindex bbb812a38..702721efc 100644--- a/browser_use/browser/events.py+++ b/browser_use/browser/events.py@@ -1,15 +1,15 @@"""Event definitions for browser communication."""import inspect-from typing import Any, Generic, Literal+from typing import Any, Literalfrom bubus import BaseEventfrom bubus.models import T_EventResultType+from cdp_use.cdp.target import TargetIDfrom pydantic import BaseModel, Field, field_validatorfrom browser_use.browser.views import BrowserStateSummaryfrom browser_use.dom.views import EnhancedDOMTreeNode-from cdp_use.cdp.target import TargetID# ============================================================================# Agent/Controller -> BrowserSession Events (High-level browser actions)@@ -95,9 +95,9 @@ class ClickElementEvent(ElementSelectedEvent[None]):node: 'EnhancedDOMTreeNode'button: Literal['left', 'right', 'middle'] = 'left'- new_tab: bool = Field(+ while_holding_ctrl: bool = Field(default=False,- description='Set True to open any link clicked in a new tab in the background, can use switch_tab(tab_id=-1) after to focus it',+ description='Set True to open any link clicked in a new tab in the background, can use switch_tab(tab_id=None) after to focus it',)# click_count: int = 1 # TODO# expect_download: bool = False # moved to downloads_watchdog.py@@ -125,7 +125,7 @@ class ScrollEvent(ElementSelectedEvent[None]):event_timeout: float | None = 8.0 # seconds-class SwitchTabEvent(BaseEvent[None]):+class SwitchTabEvent(BaseEvent[TargetID]):"""Switch to a different tab."""target_id: TargetID | None = Field(default=None, description='None means switch to the most recently opened tab')diff --git a/browser_use/browser/session.py b/browser_use/browser/session.pyindex 3af623ebd..0fe4afb85 100644--- a/browser_use/browser/session.py+++ b/browser_use/browser/session.py@@ -8,7 +8,7 @@ import httpxfrom bubus import EventBusfrom cdp_use import CDPClientfrom cdp_use.cdp.network import Cookie-from cdp_use.cdp.target import TargetID, SessionID+from cdp_use.cdp.target import SessionID, TargetIDfrom pydantic import BaseModel, ConfigDict, Field, PrivateAttrfrom uuid_extensions import uuid7str@@ -462,7 +462,9 @@ class BrowserSession(BaseModel):await self.event_bus.dispatch(SwitchTabEvent(target_id=target_id))# which does this for us:# self.agent_focus = await self.get_or_create_cdp_session(target_id)- assert self.agent_focus is not None and self.agent_focus.target_id == target_id, 'Agent focus not updated to new target_id after SwitchTabEvent should have switched to it'+ assert self.agent_focus is not None and self.agent_focus.target_id == target_id, (+ 'Agent focus not updated to new target_id after SwitchTabEvent should have switched to it'+ )# Dispatch navigation startedawait self.event_bus.dispatch(NavigationStartedEvent(target_id=target_id, url=event.url))@@ -489,7 +491,9 @@ class BrowserSession(BaseModel):status=None, # CDP doesn't provide status directly))- await self.event_bus.dispatch(AgentFocusChangedEvent(target_id=target_id, url=event.url)) # do not await! AgentFocusChangedEvent calls SwitchTabEvent and it will deadlock, dispatch to enqueue and return+ await self.event_bus.dispatch(+ AgentFocusChangedEvent(target_id=target_id, url=event.url)+ ) # do not await! AgentFocusChangedEvent calls SwitchTabEvent and it will deadlock, dispatch to enqueue and return# Note: These should be handled by dedicated watchdogs:# - Security checks (security_watchdog)@@ -510,12 +514,12 @@ class BrowserSession(BaseModel):await self.event_bus.dispatch(AgentFocusChangedEvent(target_id=target_id, url=event.url))raise- async def on_SwitchTabEvent(self, event: SwitchTabEvent) -> None:+ async def on_SwitchTabEvent(self, event: SwitchTabEvent) -> TargetID:"""Handle tab switching - core browser functionality."""if not self.agent_focus:self.logger.warning('Cannot switch tabs - browser not connected')return-+all_pages = await self._cdp_get_all_pages()if event.target_id is None:# most recently opened page@@ -524,7 +528,9 @@ class BrowserSession(BaseModel):event.target_id = all_pages[-1]['targetId']else:# no pages open at all, create a new one (handles switching to it automatically)- self.event_bus.dispatch(NavigateToUrlEvent(url='about:blank', new_tab=True)) # do not await! NavigateToUrlEvent calls SwitchTabEvent and it will deadlock, dispatch to enqueue and return+ self.event_bus.dispatch(+ NavigateToUrlEvent(url='about:blank', new_tab=True)+ ) # do not await! NavigateToUrlEvent calls SwitchTabEvent and it will deadlock, dispatch to enqueue and returnreturn# switch to the target@@ -537,7 +543,7 @@ class BrowserSession(BaseModel):url=self.agent_focus.url,))-+ return self.agent_focus.target_idasync def on_TabClosedEvent(self, event: TabClosedEvent) -> None:"""Handle tab closure - update focus if needed."""@@ -591,8 +597,10 @@ class BrowserSession(BaseModel):else:raise Exception('β Failed to execute test JS expression with Page.evaluate')except Exception as e:- self.logger.error(f'π β Target {self.agent_focus.target_id} seems closed/crashed, switching to fallback page {all_targets[0]}: {type(e).__name__}: {e}')- all_pages = (await self._cdp_get_all_pages())+ self.logger.error(+ f'π β Target {self.agent_focus.target_id} seems closed/crashed, switching to fallback page {all_targets[0]}: {type(e).__name__}: {e}'+ )+ all_pages = await self._cdp_get_all_pages()last_target_id = all_pages[-1]['targetId'] if all_pages else Noneself.agent_focus = await self.get_or_create_cdp_session(target_id=last_target_id, focus=True)raise@@ -1152,14 +1160,41 @@ class BrowserSession(BaseModel):async def get_element_by_index(self, index: int) -> EnhancedDOMTreeNode | None:"""Alias for get_dom_element_by_index for backwards compatibility."""return await self.get_dom_element_by_index(index)-- def get_target_id_from_tab_id(self, tab_id: str) -> TargetID:++ async def get_target_id_from_tab_id(self, tab_id: str) -> TargetID:"""Get the full-length TargetID from the truncated 4-char tab_id."""- for session in self._cdp_session_pool.values():- if session.target_id[-4:] == tab_id:- return session.target_id+ for full_target_id in self._cdp_session_pool.keys():+ if full_target_id.endswith(tab_id):+ return full_target_id++ # may not have a cached session, so we need to get all pages and find the target id+ all_targets = await self.cdp_client.send.Target.getTargets()+ # Filter for valid page/tab targets only+ for target in all_targets.get('targetInfos', []):+ if target['targetId'].endswith(tab_id):+ return target['targetId']+raise ValueError(f'No TargetID found ending in tab_id=...{tab_id}')+ async def get_target_id_from_url(self, url: str) -> TargetID:+ """Get the TargetID from a URL."""+ all_targets = await self.cdp_client.send.Target.getTargets()+ for target in all_targets.get('targetInfos', []):+ if target['url'] == url:+ return target['targetId']++ # still not found, try substring match as fallback+ for target in all_targets.get('targetInfos', []):+ if url in target['url']:+ return target['targetId']++ raise ValueError(f'No TargetID found for url={url}')++ async def get_most_recently_opened_target_id(self) -> TargetID:+ """Get the most recently opened target ID."""+ all_targets = await self.cdp_client.send.Target.getTargets()+ return (await self._cdp_get_all_pages())[-1]['targetId']+def is_file_input(self, element: Any) -> bool:"""Check if element is a file input.diff --git a/browser_use/controller/service.py b/browser_use/controller/service.pyindex cfc2b2760..f8e580139 100644--- a/browser_use/controller/service.py+++ b/browser_use/controller/service.py@@ -263,7 +263,7 @@ class Controller(Generic[Context]):# Element Interaction Actions@self.registry.action(- 'Click element by index, set new_tab=True to open any resulting navigation in a new tab. Only click on indices that are inside your current browser_state. Never click or assume not existing indices.',+ 'Click element by index, set while_holding_ctrl=True to open any resulting navigation in a new tab. Only click on indices that are inside your current browser_state. Never click or assume not existing indices.',param_model=ClickElementAction,)async def click_element_by_index(params: ClickElementAction, browser_session: BrowserSession):@@ -278,7 +278,7 @@ class Controller(Generic[Context]):if node is None:raise ValueError(f'Element index {params.index} not found in DOM')- event = browser_session.event_bus.dispatch(ClickElementEvent(node=node, new_tab=params.new_tab))+ event = browser_session.event_bus.dispatch(ClickElementEvent(node=node, while_holding_ctrl=params.while_holding_ctrl))await event# Wait for handler to complete and get any exception (None is expected on success)await event.event_result(raise_if_any=True, raise_if_none=False)@@ -472,23 +472,30 @@ class Controller(Generic[Context]):async def switch_tab(params: SwitchTabAction, browser_session: BrowserSession):# Dispatch switch tab eventtry:- target_id = browser_session.get_target_id_from_tab_id(params.tab_id)+ if params.tab_id:+ target_id = await browser_session.get_target_id_from_tab_id(params.tab_id)+ elif params.url:+ target_id = await browser_session.get_target_id_from_url(params.url)+ else:+ target_id = await browser_session.get_most_recently_opened_target_id()+event = browser_session.event_bus.dispatch(SwitchTabEvent(target_id=target_id))await event- await event.event_result(raise_if_any=True, raise_if_none=False)- memory = f'Switched to tab #{params.tab_id}'+ new_target_id = await event.event_result(raise_if_any=True, raise_if_none=False)+ assert new_target_id, 'SwitchTabEvent did not return a TargetID for the new tab that was switched to'+ memory = f'Switched to Tab with ID {new_target_id[-4:]}'logger.info(f'π {memory}')return ActionResult(extracted_content=memory, include_in_memory=True, long_term_memory=memory)except Exception as e:logger.error(f'Failed to switch tab: {type(e).__name__}: {e}')clean_msg = extract_llm_error_message(e)- return ActionResult(error=f'Failed to switch to tab {params.tab_id}: {clean_msg}')+ return ActionResult(error=f'Failed to switch to tab {params.tab_id or params.url}: {clean_msg}')@self.registry.action('Close an existing tab', param_model=CloseTabAction)async def close_tab(params: CloseTabAction, browser_session: BrowserSession):# Dispatch close tab eventtry:- target_id = browser_session.get_target_id_from_tab_id(params.tab_id)+ target_id = await browser_session.get_target_id_from_tab_id(params.tab_id)event = browser_session.event_bus.dispatch(CloseTabEvent(target_id=target_id))await eventawait event.event_result(raise_if_any=True, raise_if_none=False)diff --git a/browser_use/controller/views.py b/browser_use/controller/views.pyindex 8db6fbbe0..cca08361f 100644--- a/browser_use/controller/views.py+++ b/browser_use/controller/views.py@@ -15,7 +15,7 @@ class GoToUrlAction(BaseModel):class ClickElementAction(BaseModel):index: int = Field(ge=1, description='index of the element to click')- new_tab: bool = Field(default=False, description='set True to open any resulting navigation in a new tab, False otherwise')+ while_holding_ctrl: bool = Field(default=False, description='set True to open any resulting navigation in a new background tab, False otherwise')# expect_download: bool = Field(default=False, description='set True if expecting a download, False otherwise') # moved to downloads_watchdog.py# click_count: int = 1 # TODO@@ -41,7 +41,16 @@ class StructuredOutputAction(BaseModel, Generic[T]):class SwitchTabAction(BaseModel):- tab_id: str = Field(min_length=4, max_length=4, description='4 character Tab ID') # last 4 chars of TargetID+ url: str | None = Field(+ default=None,+ description='URL or URL substring of the tab to switch to, if not provided, the tab_id or most recently opened tab will be used',+ )+ tab_id: str | None = Field(+ default=None,+ min_length=4,+ max_length=4,+ description='exact 4 character Tab ID to match instead of URL, prefer using this if known',+ ) # last 4 chars of TargetIDclass CloseTabAction(BaseModel):diff --git a/browser_use/mcp/server.py b/browser_use/mcp/server.pyindex 39f39d24a..a077d5cbe 100644--- a/browser_use/mcp/server.py+++ b/browser_use/mcp/server.py@@ -794,7 +794,7 @@ class BrowserUseServer:from browser_use.browser.events import SwitchTabEvent- target_id = self.browser_session.get_target_id_from_tab_id(tab_id)+ target_id = await self.browser_session.get_target_id_from_tab_id(tab_id)event = self.browser_session.event_bus.dispatch(SwitchTabEvent(target_id=target_id))await eventstate = await self.browser_session.get_browser_state_summary()@@ -805,16 +805,14 @@ class BrowserUseServer:if not self.browser_session:return 'Error: No browser session active'-from browser_use.browser.events import CloseTabEvent- target_id = self.browser_session.get_target_id_from_tab_id(tab_id)+ target_id = await self.browser_session.get_target_id_from_tab_id(tab_id)event = self.browser_session.event_bus.dispatch(CloseTabEvent(target_id=target_id))await eventcurrent_url = await self.browser_session.get_current_page_url()return f'Closed tab # {tab_id}, now on {current_url}'-async def run(self):"""Run the MCP server."""async with mcp.server.stdio.stdio_server() as (read_stream, write_stream):diff --git a/tests/ci/test_browser_event_ClickElementEvent.py b/tests/ci/test_browser_event_ClickElementEvent.pyindex c3585f382..b95446ef0 100644--- a/tests/ci/test_browser_event_ClickElementEvent.py+++ b/tests/ci/test_browser_event_ClickElementEvent.py@@ -203,7 +203,7 @@ class TestClickElementEvent:assert result_text == expected_result_text, f"Expected result text '{expected_result_text}', got '{result_text}'"async def test_click_element_new_tab(self, controller, browser_session, base_url, http_server):- """Test that click_element_by_index with new_tab=True opens links in new tabs."""+ """Test that click_element_by_index with while_holding_ctrl=True opens links in new tabs."""# Add route for new tab test pagehttp_server.expect_request('/newTab').respond_with_data("""@@ -249,8 +249,8 @@ class TestClickElementEvent:assert link_index is not None, 'Could not find link element'- # Click the link with new_tab=True- click_action = {'click_element_by_index': ClickElementAction(index=link_index, new_tab=True)}+ # Click the link with while_holding_ctrl=True+ click_action = {'click_element_by_index': ClickElementAction(index=link_index, while_holding_ctrl=True)}class ClickActionModel(ActionModel):click_element_by_index: ClickElementAction | None = None@@ -285,7 +285,7 @@ class TestClickElementEvent:assert f'{base_url}/page1' in new_tab.url, f'New tab should have page1 URL, but got {new_tab.url}'async def test_click_element_normal_vs_new_tab(self, controller, browser_session, base_url, http_server):- """Test that click_element_by_index behaves differently with new_tab=False vs new_tab=True."""+ """Test that click_element_by_index behaves differently with while_holding_ctrl=False vs while_holding_ctrl=True."""# Add route for comparison test pagehttp_server.expect_request('/comparison').respond_with_data("""@@ -327,8 +327,8 @@ class TestClickElementEvent:assert len(link_indices) >= 2, 'Need at least 2 links for comparison test'- # Test normal click (new_tab=False) - should navigate in current tab- click_action_normal = {'click_element_by_index': ClickElementAction(index=link_indices[0], new_tab=False)}+ # Test normal click (while_holding_ctrl=False) - should navigate in current tab+ click_action_normal = {'click_element_by_index': ClickElementAction(index=link_indices[0], while_holding_ctrl=False)}class ClickActionModel(ActionModel):click_element_by_index: ClickElementAction | None = None@@ -344,8 +344,8 @@ class TestClickElementEvent:await controller.act(GoToUrlActionModel(**goto_action), browser_session)await asyncio.sleep(1)- # Test new tab click (new_tab=True) - should open in new tab- click_action_new_tab = {'click_element_by_index': ClickElementAction(index=link_indices[1], new_tab=True)}+ # Test new tab click (while_holding_ctrl=True) - should open in new background tab+ click_action_new_tab = {'click_element_by_index': ClickElementAction(index=link_indices[1], while_holding_ctrl=True)}result = await controller.act(ClickActionModel(**click_action_new_tab), browser_session)await asyncio.sleep(1)diff --git a/tests/ci/test_navigation_events.py b/tests/ci/test_navigation_events.pyindex 27c79c2e2..cef92214c 100644--- a/tests/ci/test_navigation_events.py+++ b/tests/ci/test_navigation_events.py@@ -272,7 +272,7 @@ async def test_navigation_events_history_pushstate(httpserver):new_tab_link_found = True# Use new_tab=True parameter to properly trigger new tab behaviorclick_element = await session.get_dom_element_by_index(idx)- session.event_bus.dispatch(ClickElementEvent(node=click_element, new_tab=True))+ session.event_bus.dispatch(ClickElementEvent(node=click_element, while_holding_ctrl=True))breakif new_tab_link_found:@@ -397,7 +397,7 @@ async def test_navigation_events_link_clicks(httpserver):if hasattr(element, 'attributes') and element.attributes.get('id') == 'new-tab-link':new_tab_link_found = Trueclick_element = await session.get_dom_element_by_index(idx)- session.event_bus.dispatch(ClickElementEvent(node=click_element, new_tab=True))+ session.event_bus.dispatch(ClickElementEvent(node=click_element, while_holding_ctrl=True))breakif new_tab_link_found: