Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Add Rust agent action replay helper parity

Magnus Mülleredb79e9616d2
diff --git a/RUST_AGENT_INTEGRATION_FEATURES.md b/RUST_AGENT_INTEGRATION_FEATURES.md
index f640235c7..08d0f9d7e 100644
--- a/RUST_AGENT_INTEGRATION_FEATURES.md
+++ b/RUST_AGENT_INTEGRATION_FEATURES.md
@@ -496,10 +496,15 @@ Terminal core branch: `magnus/browser-use-rust-integration` at terminal main `ee
- The helpers preserve failure bookkeeping, done-result logging, screenshot-backed history item creation, file-system state persistence, step counting, and done-only output switching for Rust-backed helper execution.
- Proof: `test_rust_agent_exposes_step_finalization_helper_methods`.
+98. Rust Agent action replay helper parity
+ - The Rust-backed `Agent` now exposes Browser Use's `_execute_initial_actions()`, `_execute_history_step(...)`, and `_update_action_indices(...)` replay helpers.
+ - Initial actions execute through `multi_act` and are recorded as step 0, while history replay remaps moved DOM indices by element hash before routing actions through the Rust-backed action path.
+ - Proof: `test_rust_agent_exposes_action_replay_helper_methods`.
+
## Current Verification
- `python3 -m py_compile browser_use/rust/service.py browser_use/rust/__init__.py browser_use/__init__.py tests/ci/test_rust_agent.py examples/rust_agent/basic.py examples/rust_agent/real_v8_smoke.py`
-- `uv run pytest -q tests/ci/test_rust_agent.py` (71 tests)
+- `uv run pytest -q tests/ci/test_rust_agent.py` (72 tests)
- `cargo build -q -p browser-use-cli`
- `CARGO_INCREMENTAL=0 cargo test -q -p browser-use-agent selected_remote_cdp_mode_allows_remote_cdp_connect -- --nocapture`
- `CARGO_INCREMENTAL=0 cargo test -q -p browser-use-agent bare_browser_connect_resolves_to_selected_managed_mode_with_launch_args -- --nocapture`
diff --git a/browser_use/rust/service.py b/browser_use/rust/service.py
index abf4319aa..a94b4bb43 100644
--- a/browser_use/rust/service.py
+++ b/browser_use/rust/service.py
@@ -1964,6 +1964,102 @@ class Agent(Generic[AgentStructuredOutput]):
self.task = original_task
return history.action_results()
+ async def _execute_initial_actions(self) -> None:
+ """Execute configured Browser Use initial actions through the Rust-backed action path."""
+ if not self.initial_actions or self.state.follow_up_task:
+ return
+ result = await self.multi_act(self.initial_actions)
+ if result and self.initial_url and result[0].long_term_memory:
+ result[0].long_term_memory = f'Found initial url and automatically loaded it. {result[0].long_term_memory}'
+ self.state.last_result = result
+
+ if self.settings.flash_mode:
+ model_output = self.AgentOutput(
+ evaluation_previous_goal=None,
+ memory='Initial navigation',
+ next_goal=None,
+ action=self.initial_actions,
+ )
+ else:
+ model_output = self.AgentOutput(
+ evaluation_previous_goal='Start',
+ memory=None,
+ next_goal='Initial navigation',
+ action=self.initial_actions,
+ )
+
+ metadata = StepMetadata(
+ step_number=0,
+ step_start_time=time.time(),
+ step_end_time=time.time(),
+ )
+ state_history = BrowserStateHistory(
+ url=self.initial_url or '',
+ title='Initial Actions',
+ tabs=[],
+ interacted_element=[None] * len(self.initial_actions),
+ screenshot_path=None,
+ )
+ self.history.add_item(
+ AgentHistory(
+ model_output=model_output,
+ result=result,
+ state=state_history,
+ metadata=metadata,
+ )
+ )
+
+ async def _execute_history_step(self, history_item: AgentHistory, delay: float) -> list[ActionResult]:
+ """Replay a Browser Use history step when Python action models are available."""
+ if self.browser_session is None:
+ raise ValueError('BrowserSession is not set up')
+ get_state = getattr(self.browser_session, 'get_browser_state_summary', None)
+ if not callable(get_state):
+ raise ValueError('BrowserSession does not expose get_browser_state_summary')
+ state = await get_state(include_screenshot=False)
+ if not state or not history_item.model_output:
+ raise ValueError('Invalid state or model output')
+
+ updated_actions = []
+ for index, action in enumerate(history_item.model_output.action):
+ historical_element = history_item.state.interacted_element[index]
+ updated_action = await self._update_action_indices(historical_element, action, state)
+ if updated_action is None:
+ raise ValueError(f'Could not find matching element {index} in current page')
+ updated_actions.append(updated_action)
+
+ result = await self.multi_act(updated_actions)
+ await asyncio.sleep(delay)
+ return result
+
+ async def _update_action_indices(
+ self,
+ historical_element: Any | None,
+ action: Any,
+ browser_state_summary: Any,
+ ) -> Any | None:
+ """Update an action index when a historical DOM element moved."""
+ selector_map = getattr(getattr(browser_state_summary, 'dom_state', None), 'selector_map', {})
+ if not historical_element or not selector_map:
+ return action
+ historical_hash = getattr(historical_element, 'element_hash', None)
+ highlight_index = None
+ for candidate_index, element in selector_map.items():
+ if getattr(element, 'element_hash', None) == historical_hash:
+ highlight_index = candidate_index
+ break
+ if highlight_index is None:
+ return None
+
+ get_index = getattr(action, 'get_index', None)
+ set_index = getattr(action, 'set_index', None)
+ if callable(get_index) and callable(set_index):
+ old_index = get_index()
+ if old_index != highlight_index:
+ set_index(highlight_index)
+ self.logger.info(f'Element moved in DOM, updated index from {old_index} to {highlight_index}')
+ return action
+
def add_new_task(self, new_task: str) -> None:
"""Add a follow-up task while keeping the same Browser Use-style agent object."""
self.task = _task_with_schema(new_task, self.output_model_schema)
diff --git a/tests/ci/test_rust_agent.py b/tests/ci/test_rust_agent.py
index b16a100ec..6290f244e 100644
--- a/tests/ci/test_rust_agent.py
+++ b/tests/ci/test_rust_agent.py
@@ -1732,6 +1732,82 @@ async def test_rust_agent_multi_act_routes_actions_to_followup():
assert results[0].extracted_content == 'actions applied'
+async def test_rust_agent_exposes_action_replay_helper_methods():
+ from types import SimpleNamespace
+
+ from browser_use.agent.views import ActionResult
+ from browser_use.rust import Agent
+
+ class BrowserProfile:
+ downloads_path = None
+
+ class BrowserSession:
+ browser_profile = BrowserProfile()
+
+ async def get_browser_state_summary(self, include_screenshot=False):
+ assert include_screenshot is False
+ return SimpleNamespace(
+ dom_state=SimpleNamespace(selector_map={7: SimpleNamespace(element_hash='matching-hash')})
+ )
+
+ class FakeAction:
+ def __init__(self):
+ self.index = 2
+
+ def get_index(self):
+ return self.index
+
+ def set_index(self, index):
+ self.index = index
+
+ def model_dump(self, **kwargs):
+ return {'click_element': {'index': self.index}}
+
+ agent = Agent(
+ task='Replay actions.',
+ browser_session=BrowserSession(),
+ initial_actions=[{'navigate': {'url': 'https://example.com', 'new_tab': False}}],
+ )
+ agent.initial_url = 'https://example.com'
+ initial_calls = []
+
+ async def fake_initial_multi_act(actions):
+ initial_calls.append(actions)
+ return [ActionResult(long_term_memory='loaded')]
+
+ agent.multi_act = fake_initial_multi_act
+ await agent._execute_initial_actions()
+
+ assert initial_calls == [agent.initial_actions]
+ assert agent.state.last_result[0].long_term_memory == 'Found initial url and automatically loaded it. loaded'
+ assert agent.history.history[0].metadata.step_number == 0
+ assert agent.history.history[0].state.url == 'https://example.com'
+ assert agent.history.history[0].model_output.action == agent.initial_actions
+
+ action = FakeAction()
+ replay_calls = []
+
+ async def fake_replay_multi_act(actions):
+ replay_calls.append(actions)
+ return [ActionResult(extracted_content='replayed')]
+
+ agent.multi_act = fake_replay_multi_act
+ history_item = SimpleNamespace(
+ model_output=SimpleNamespace(action=[action]),
+ state=SimpleNamespace(interacted_element=[SimpleNamespace(element_hash='matching-hash')]),
+ )
+ results = await agent._execute_history_step(history_item, delay=0)
+
+ assert action.index == 7
+ assert replay_calls == [[action]]
+ assert results[0].extracted_content == 'replayed'
+ assert await agent._update_action_indices(
+ SimpleNamespace(element_hash='missing-hash'),
+ FakeAction(),
+ SimpleNamespace(dom_state=SimpleNamespace(selector_map={1: SimpleNamespace(element_hash='other-hash')})),
+ ) is None
+
+
async def test_rust_agent_trace_and_cloud_auth_helpers():
from browser_use.rust import Agent
from browser_use.rust.service import _history_from_events