Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Add Rust agent action replay helper parity
Magnus Mülleredb79e9616d2
diff --git a/RUST_AGENT_INTEGRATION_FEATURES.md b/RUST_AGENT_INTEGRATION_FEATURES.mdindex f640235c7..08d0f9d7e 100644--- a/RUST_AGENT_INTEGRATION_FEATURES.md+++ b/RUST_AGENT_INTEGRATION_FEATURES.md@@ -496,10 +496,15 @@ Terminal core branch: `magnus/browser-use-rust-integration` at terminal main `ee- The helpers preserve failure bookkeeping, done-result logging, screenshot-backed history item creation, file-system state persistence, step counting, and done-only output switching for Rust-backed helper execution.- Proof: `test_rust_agent_exposes_step_finalization_helper_methods`.+98. Rust Agent action replay helper parity+ - The Rust-backed `Agent` now exposes Browser Use's `_execute_initial_actions()`, `_execute_history_step(...)`, and `_update_action_indices(...)` replay helpers.+ - Initial actions execute through `multi_act` and are recorded as step 0, while history replay remaps moved DOM indices by element hash before routing actions through the Rust-backed action path.+ - Proof: `test_rust_agent_exposes_action_replay_helper_methods`.+## Current Verification- `python3 -m py_compile browser_use/rust/service.py browser_use/rust/__init__.py browser_use/__init__.py tests/ci/test_rust_agent.py examples/rust_agent/basic.py examples/rust_agent/real_v8_smoke.py`-- `uv run pytest -q tests/ci/test_rust_agent.py` (71 tests)+- `uv run pytest -q tests/ci/test_rust_agent.py` (72 tests)- `cargo build -q -p browser-use-cli`- `CARGO_INCREMENTAL=0 cargo test -q -p browser-use-agent selected_remote_cdp_mode_allows_remote_cdp_connect -- --nocapture`- `CARGO_INCREMENTAL=0 cargo test -q -p browser-use-agent bare_browser_connect_resolves_to_selected_managed_mode_with_launch_args -- --nocapture`diff --git a/browser_use/rust/service.py b/browser_use/rust/service.pyindex abf4319aa..a94b4bb43 100644--- a/browser_use/rust/service.py+++ b/browser_use/rust/service.py@@ -1964,6 +1964,102 @@ class Agent(Generic[AgentStructuredOutput]):self.task = original_taskreturn history.action_results()+ async def _execute_initial_actions(self) -> None:+ """Execute configured Browser Use initial actions through the Rust-backed action path."""+ if not self.initial_actions or self.state.follow_up_task:+ return+ result = await self.multi_act(self.initial_actions)+ if result and self.initial_url and result[0].long_term_memory:+ result[0].long_term_memory = f'Found initial url and automatically loaded it. {result[0].long_term_memory}'+ self.state.last_result = result++ if self.settings.flash_mode:+ model_output = self.AgentOutput(+ evaluation_previous_goal=None,+ memory='Initial navigation',+ next_goal=None,+ action=self.initial_actions,+ )+ else:+ model_output = self.AgentOutput(+ evaluation_previous_goal='Start',+ memory=None,+ next_goal='Initial navigation',+ action=self.initial_actions,+ )++ metadata = StepMetadata(+ step_number=0,+ step_start_time=time.time(),+ step_end_time=time.time(),+ )+ state_history = BrowserStateHistory(+ url=self.initial_url or '',+ title='Initial Actions',+ tabs=[],+ interacted_element=[None] * len(self.initial_actions),+ screenshot_path=None,+ )+ self.history.add_item(+ AgentHistory(+ model_output=model_output,+ result=result,+ state=state_history,+ metadata=metadata,+ )+ )++ async def _execute_history_step(self, history_item: AgentHistory, delay: float) -> list[ActionResult]:+ """Replay a Browser Use history step when Python action models are available."""+ if self.browser_session is None:+ raise ValueError('BrowserSession is not set up')+ get_state = getattr(self.browser_session, 'get_browser_state_summary', None)+ if not callable(get_state):+ raise ValueError('BrowserSession does not expose get_browser_state_summary')+ state = await get_state(include_screenshot=False)+ if not state or not history_item.model_output:+ raise ValueError('Invalid state or model output')++ updated_actions = []+ for index, action in enumerate(history_item.model_output.action):+ historical_element = history_item.state.interacted_element[index]+ updated_action = await self._update_action_indices(historical_element, action, state)+ if updated_action is None:+ raise ValueError(f'Could not find matching element {index} in current page')+ updated_actions.append(updated_action)++ result = await self.multi_act(updated_actions)+ await asyncio.sleep(delay)+ return result++ async def _update_action_indices(+ self,+ historical_element: Any | None,+ action: Any,+ browser_state_summary: Any,+ ) -> Any | None:+ """Update an action index when a historical DOM element moved."""+ selector_map = getattr(getattr(browser_state_summary, 'dom_state', None), 'selector_map', {})+ if not historical_element or not selector_map:+ return action+ historical_hash = getattr(historical_element, 'element_hash', None)+ highlight_index = None+ for candidate_index, element in selector_map.items():+ if getattr(element, 'element_hash', None) == historical_hash:+ highlight_index = candidate_index+ break+ if highlight_index is None:+ return None++ get_index = getattr(action, 'get_index', None)+ set_index = getattr(action, 'set_index', None)+ if callable(get_index) and callable(set_index):+ old_index = get_index()+ if old_index != highlight_index:+ set_index(highlight_index)+ self.logger.info(f'Element moved in DOM, updated index from {old_index} to {highlight_index}')+ return action+def add_new_task(self, new_task: str) -> None:"""Add a follow-up task while keeping the same Browser Use-style agent object."""self.task = _task_with_schema(new_task, self.output_model_schema)diff --git a/tests/ci/test_rust_agent.py b/tests/ci/test_rust_agent.pyindex b16a100ec..6290f244e 100644--- a/tests/ci/test_rust_agent.py+++ b/tests/ci/test_rust_agent.py@@ -1732,6 +1732,82 @@ async def test_rust_agent_multi_act_routes_actions_to_followup():assert results[0].extracted_content == 'actions applied'+async def test_rust_agent_exposes_action_replay_helper_methods():+ from types import SimpleNamespace++ from browser_use.agent.views import ActionResult+ from browser_use.rust import Agent++ class BrowserProfile:+ downloads_path = None++ class BrowserSession:+ browser_profile = BrowserProfile()++ async def get_browser_state_summary(self, include_screenshot=False):+ assert include_screenshot is False+ return SimpleNamespace(+ dom_state=SimpleNamespace(selector_map={7: SimpleNamespace(element_hash='matching-hash')})+ )++ class FakeAction:+ def __init__(self):+ self.index = 2++ def get_index(self):+ return self.index++ def set_index(self, index):+ self.index = index++ def model_dump(self, **kwargs):+ return {'click_element': {'index': self.index}}++ agent = Agent(+ task='Replay actions.',+ browser_session=BrowserSession(),+ initial_actions=[{'navigate': {'url': 'https://example.com', 'new_tab': False}}],+ )+ agent.initial_url = 'https://example.com'+ initial_calls = []++ async def fake_initial_multi_act(actions):+ initial_calls.append(actions)+ return [ActionResult(long_term_memory='loaded')]++ agent.multi_act = fake_initial_multi_act+ await agent._execute_initial_actions()++ assert initial_calls == [agent.initial_actions]+ assert agent.state.last_result[0].long_term_memory == 'Found initial url and automatically loaded it. loaded'+ assert agent.history.history[0].metadata.step_number == 0+ assert agent.history.history[0].state.url == 'https://example.com'+ assert agent.history.history[0].model_output.action == agent.initial_actions++ action = FakeAction()+ replay_calls = []++ async def fake_replay_multi_act(actions):+ replay_calls.append(actions)+ return [ActionResult(extracted_content='replayed')]++ agent.multi_act = fake_replay_multi_act+ history_item = SimpleNamespace(+ model_output=SimpleNamespace(action=[action]),+ state=SimpleNamespace(interacted_element=[SimpleNamespace(element_hash='matching-hash')]),+ )+ results = await agent._execute_history_step(history_item, delay=0)++ assert action.index == 7+ assert replay_calls == [[action]]+ assert results[0].extracted_content == 'replayed'+ assert await agent._update_action_indices(+ SimpleNamespace(element_hash='missing-hash'),+ FakeAction(),+ SimpleNamespace(dom_state=SimpleNamespace(selector_map={1: SimpleNamespace(element_hash='other-hash')})),+ ) is None++async def test_rust_agent_trace_and_cloud_auth_helpers():from browser_use.rust import Agentfrom browser_use.rust.service import _history_from_events