Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Deduplicate Rust SDK history usage

Magnus Müller474a06b0c1eb
diff --git a/RUST_AGENT_INTEGRATION_FEATURES.md b/RUST_AGENT_INTEGRATION_FEATURES.md
index 0c82aa32f..accc38751 100644
--- a/RUST_AGENT_INTEGRATION_FEATURES.md
+++ b/RUST_AGENT_INTEGRATION_FEATURES.md
@@ -187,6 +187,18 @@ This branch keeps the Python `Agent` unchanged unless callers explicitly import
browser runs, proxy country is passed to `browser remote start`, so Rust
SDK sessions can use Browser Use Cloud browser proxies through the same
Python-facing `browser_profile`/`browser_session` options.
+ - Browser-use now deduplicates SDK response/projected events before
+ reconstructing history and usage. Duplicate `token_count` and
+ `session.done` records no longer inflate eval history, dashboard usage, or
+ telemetry payloads when the terminal server returns both live and projected
+ event streams.
+ - Reconstructed browser history ignores internal browser-control endpoint
+ URLs such as `http://127.0.0.1:<port>` while still preserving genuine local
+ page URLs. Eval traces therefore show user-visible pages instead of CDP
+ helper endpoints.
+ - The Rust-only default LLM now resolves to Claude Sonnet 4.6 via
+ browser-use-style model names, while `DEFAULT_LLM` can still override it.
+ The normal Python `browser_use.Agent` default remains unchanged.
## Current Proof
@@ -248,6 +260,8 @@ This branch keeps the Python `Agent` unchanged unless callers explicitly import
- terminal `cargo test -p browser-use-cli sdk_provider_run_config_maps_browser_use_options_to_rust_core -- --nocapture`
- terminal `cargo test -p browser-use-agent stored_cloud_profile_uses_sdk_proxy_country_env_when_connecting -- --nocapture`
- browser-use `uv run pytest tests/ci/test_rust_agent.py::test_rust_agent_sdk_browser_payload_includes_profile_domains_window_and_proxy -q`
+- browser-use `uv run python -m py_compile browser_use/rust/service.py`
+- browser-use `uv run pytest tests/ci/test_rust_agent.py::test_rust_sdk_event_dedupe_removes_projected_usage_duplicates tests/ci/test_rust_agent.py::test_rust_history_ignores_internal_browser_connection_url tests/ci/test_rust_agent.py::test_rust_agent_default_llm_is_rust_only_anthropic_sonnet tests/ci/test_rust_agent.py::test_rust_agent_default_llm_respects_default_llm_env tests/ci/test_rust_agent.py::test_rust_agent_exposes_logging_helper_methods tests/ci/test_rust_agent.py::test_rust_agent_telemetry_filters_empty_reconstructed_urls tests/ci/models/test_llm_model_factory.py::test_get_llm_by_name_resolves_anthropic_sonnet_4_6_from_env -q`
- evaluations-internal `uv run python -m py_compile eval/service.py`
- evaluations-internal `python -m py_compile eval/task_types.py`
- evaluations-internal `PYTHONPATH=. uv run pytest tests/test_service_cli.py -q -k 'usage_aliases or trims_oversized_history_fields or rust_eval_uses_adapter_initial_navigation_default or rust_eval_preserves_explicit_direct_initial_navigation_override'`
diff --git a/browser_use/rust/service.py b/browser_use/rust/service.py
index 00bf9bdac..8a78f1355 100644
--- a/browser_use/rust/service.py
+++ b/browser_use/rust/service.py
@@ -1,7 +1,7 @@
from __future__ import annotations
-import asyncio
import ast
+import asyncio
import base64
import hashlib
import inspect
@@ -34,8 +34,8 @@ from browser_use.agent.cloud_events import (
UpdateAgentTaskEvent,
)
from browser_use.agent.judge import construct_judge_messages
-from browser_use.agent.message_manager.utils import save_conversation
from browser_use.agent.message_manager.service import MessageManager
+from browser_use.agent.message_manager.utils import save_conversation
from browser_use.agent.prompts import SystemPrompt
from browser_use.agent.views import (
ActionResult,
@@ -69,8 +69,8 @@ from browser_use.tokens.views import ModelUsageStats, UsageSummary
from browser_use.tools.registry.views import ActionModel
from browser_use.tools.service import Tools
from browser_use.utils import (
- SignalHandler,
URL_PATTERN,
+ SignalHandler,
_log_pretty_path,
check_latest_browser_use_version,
get_browser_use_version,
@@ -79,7 +79,6 @@ from browser_use.utils import (
sanitize_url_candidate,
)
-
Context = TypeVar('Context')
AgentHookFunc = Callable[['Agent'], Awaitable[None]]
AgentNewStepCallback = (
@@ -89,6 +88,7 @@ AgentDoneCallback = Callable[[AgentHistoryList], Awaitable[None]] | Callable[[Ag
logger = logging.getLogger(__name__)
DEFAULT_RUST_SDK_TOOL_ALLOWLIST = ['browser', 'browser_script', 'done']
+DEFAULT_RUST_AGENT_LLM = 'anthropic_claude_sonnet_4_6'
try:
from lmnr import Laminar # type: ignore
@@ -511,7 +511,41 @@ def _sdk_notification_events(sdk: Any) -> list[dict[str, Any]]:
continue
seen.add(identity)
events.append(event)
- return events
+ return _dedupe_sdk_events(events)
+
+
+def _event_payload_fingerprint(event: dict[str, Any]) -> str:
+ try:
+ return json.dumps(_event_payload(event), sort_keys=True, default=str)
+ except Exception:
+ return repr(_event_payload(event))
+
+
+def _sdk_event_dedupe_identity(event: dict[str, Any], fallback_index: int) -> tuple[Any, ...]:
+ event_type = _event_type(event)
+ seq = event.get('seq')
+ if seq is not None:
+ return ('seq', seq, event_type, _event_payload_fingerprint(event))
+ event_id = event.get('id') or event.get('event_id')
+ if event_id is not None:
+ return ('id', event_id, event_type)
+ return ('fallback', fallback_index)
+
+
+def _dedupe_sdk_events(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """Remove SDK response/projected duplicates while preserving first-seen order."""
+ deduped: dict[tuple[Any, ...], dict[str, Any]] = {}
+ order: list[tuple[Any, ...]] = []
+ for index, event in enumerate(events):
+ key = _sdk_event_dedupe_identity(event, index)
+ existing = deduped.get(key)
+ if existing is None:
+ deduped[key] = event
+ order.append(key)
+ continue
+ if not (existing.get('id') or existing.get('event_id')) and (event.get('id') or event.get('event_id')):
+ deduped[key] = event
+ return [deduped[key] for key in order]
def _sdk_events_truncated_for_transport(events: list[dict[str, Any]]) -> bool:
@@ -566,9 +600,9 @@ def _resolve_default_llm(llm: BaseChatModel | None) -> BaseChatModel:
from browser_use.llm.models import get_llm_by_name
return get_llm_by_name(default_llm_name)
- from browser_use import ChatBrowserUse
+ from browser_use.llm.models import get_llm_by_name
- return ChatBrowserUse()
+ return get_llm_by_name(os.environ.get('BROWSER_USE_RUST_DEFAULT_LLM', DEFAULT_RUST_AGENT_LLM))
def _extract_cdp_url(browser_session: BrowserSession | None) -> str | None:
@@ -2580,6 +2614,14 @@ def _browser_url(value: Any) -> str:
return ''
+def _internal_browser_endpoint_url(url: str) -> bool:
+ parsed = urlparse(url)
+ return parsed.scheme in {'http', 'https'} and parsed.hostname in {'127.0.0.1', 'localhost', '::1'} and parsed.path in {
+ '',
+ '/',
+ }
+
+
def _browser_state_candidates(value: Any) -> list[tuple[str, str, str]]:
candidates: list[tuple[str, str, str]] = []
if isinstance(value, dict):
@@ -2753,6 +2795,8 @@ def _browser_state_from_events(events: list[dict[str, Any]]) -> BrowserStateHist
else:
candidates = _browser_state_candidates(payload)
for candidate_url, candidate_title, candidate_target_id in candidates:
+ if payload.get('name') == 'browser' and _internal_browser_endpoint_url(candidate_url):
+ continue
url = candidate_url
title = candidate_title or title
tabs = [TabInfo(url=url, title=title, target_id=candidate_target_id)]
@@ -2767,7 +2811,10 @@ def _browser_state_from_events(events: list[dict[str, Any]]) -> BrowserStateHist
):
continue
payload = _event_payload(event)
- url = _browser_url(payload.get('live_url')) or _browser_url(payload.get('url')) or url
+ next_url = _browser_url(payload.get('live_url')) or _browser_url(payload.get('url'))
+ if event_type in {'browser.connected', 'browser.reconnected'} and _internal_browser_endpoint_url(next_url):
+ next_url = ''
+ url = next_url or url
title = str(payload.get('title') or title)
raw_tabs = payload.get('tabs')
if isinstance(raw_tabs, list):
@@ -4576,8 +4623,8 @@ class Agent(Generic[Context, AgentStructuredOutput]):
agent_type=None,
action_errors=self.history.errors(),
action_history=action_history_data,
- urls_visited=self.history.urls(),
- steps=self.state.n_steps,
+ urls_visited=[url for url in self.history.urls() if url],
+ steps=self.history.number_of_steps(),
total_input_tokens=total_input_tokens,
total_output_tokens=total_output_tokens,
prompt_cached_tokens=prompt_cached_tokens,
@@ -5059,16 +5106,16 @@ class Agent(Generic[Context, AgentStructuredOutput]):
if isinstance(history_payload, dict):
raw_events = history_payload.get('events')
if isinstance(raw_events, list):
- events = [event for event in raw_events if isinstance(event, dict)]
+ events = _dedupe_sdk_events([event for event in raw_events if isinstance(event, dict)])
errors = history_payload.get('errors')
if process_error is None and history_payload.get('success') is False and isinstance(errors, list) and errors:
process_error = '\n'.join(str(error) for error in errors if error)
raw_child_events = history_payload.get('child_events')
if isinstance(raw_child_events, list):
- child_events = [event for event in raw_child_events if isinstance(event, dict)]
+ child_events = _dedupe_sdk_events([event for event in raw_child_events if isinstance(event, dict)])
raw_usage_events = history_payload.get('usage_events')
if isinstance(raw_usage_events, list):
- usage_events = [event for event in raw_usage_events if isinstance(event, dict)]
+ usage_events = _dedupe_sdk_events([event for event in raw_usage_events if isinstance(event, dict)])
response_usage_event = _usage_event_from_sdk_history_usage(history_payload.get('usage'))
elif process_error is None:
process_error = 'Rust SDK server returned an invalid response.'
diff --git a/tests/ci/models/test_llm_model_factory.py b/tests/ci/models/test_llm_model_factory.py
index 7643de28d..2d3e23909 100644
--- a/tests/ci/models/test_llm_model_factory.py
+++ b/tests/ci/models/test_llm_model_factory.py
@@ -10,3 +10,13 @@ def test_get_llm_by_name_resolves_anthropic_from_env(monkeypatch):
assert isinstance(llm, ChatAnthropic)
assert llm.model == 'claude-sonnet-4-0'
assert llm.api_key == 'anthropic-test-key'
+
+
+def test_get_llm_by_name_resolves_anthropic_sonnet_4_6_from_env(monkeypatch):
+ monkeypatch.setenv('ANTHROPIC_API_KEY', 'anthropic-test-key')
+
+ llm = get_llm_by_name('anthropic_claude_sonnet_4_6')
+
+ assert isinstance(llm, ChatAnthropic)
+ assert llm.model == 'claude-sonnet-4-6'
+ assert llm.api_key == 'anthropic-test-key'
diff --git a/tests/ci/test_rust_agent.py b/tests/ci/test_rust_agent.py
index cfc5f9c08..32c5c9db2 100644
--- a/tests/ci/test_rust_agent.py
+++ b/tests/ci/test_rust_agent.py
@@ -5,7 +5,7 @@ import inspect
import json
import sys
from pathlib import Path
-from typing import get_args, get_origin, get_type_hints
+from typing import Any, cast, get_args, get_origin, get_type_hints
import pytest
from pydantic import BaseModel
@@ -262,6 +262,41 @@ def test_rust_events_reconstruct_browser_use_history():
assert history.usage.total_completion_tokens == 7
+def test_rust_history_ignores_internal_browser_connection_url():
+ from browser_use.rust.service import _history_from_events
+
+ history = _history_from_events(
+ [
+ {'event_type': 'browser.connected', 'payload': {'url': 'http://127.0.0.1:41335'}},
+ {
+ 'event_type': 'tool.output',
+ 'payload': {'name': 'browser', 'text': 'Connected to browser at http://127.0.0.1:41335'},
+ },
+ {'event_type': 'browser.state', 'payload': {'url': 'https://example.com', 'title': 'Example'}},
+ {'event_type': 'session.done', 'payload': {'result': 'final answer'}},
+ ],
+ model='gpt-test',
+ started=1.0,
+ finished=2.0,
+ output_model_schema=None,
+ process_error=None,
+ )
+ local_page_history = _history_from_events(
+ [
+ {'event_type': 'browser.state', 'payload': {'url': 'http://127.0.0.1:8000/', 'title': 'Local app'}},
+ {'event_type': 'session.done', 'payload': {'result': 'final answer'}},
+ ],
+ model='gpt-test',
+ started=1.0,
+ finished=2.0,
+ output_model_schema=None,
+ process_error=None,
+ )
+
+ assert history.urls() == ['https://example.com']
+ assert local_page_history.urls() == ['http://127.0.0.1:8000/']
+
+
def test_rust_history_reconstructs_terminal_browser_script_urls():
from browser_use.rust.service import _history_from_events
@@ -2146,6 +2181,48 @@ def test_rust_terminal_usage_mixed_events_do_not_shrink_totals():
assert summary.total_tokens == 570
+def test_rust_sdk_event_dedupe_removes_projected_usage_duplicates():
+ from browser_use.rust.service import _dedupe_sdk_events, _usage_from_events
+
+ usage_payload = {
+ 'info': {
+ 'last_token_usage': {
+ 'input_tokens': 100,
+ 'cached_input_tokens': 40,
+ 'input_cache_creation_tokens': 10,
+ 'output_tokens': 5,
+ 'total_tokens': 115,
+ },
+ 'total_token_usage': {
+ 'input_tokens': 100,
+ 'cached_input_tokens': 40,
+ 'input_cache_creation_tokens': 10,
+ 'output_tokens': 5,
+ 'total_tokens': 115,
+ },
+ }
+ }
+ events = [
+ {'seq': 1, 'id': 'usage-id', 'event_type': 'token_count', 'payload': usage_payload},
+ {'seq': 1, 'event_type': 'token_count', 'payload': usage_payload},
+ {'seq': 2, 'event_type': 'session.done', 'payload': {'result': 'done'}},
+ {'seq': 2, 'id': 'done-id', 'event_type': 'session.done', 'payload': {'result': 'done'}},
+ ]
+
+ deduped = _dedupe_sdk_events(events)
+ summary = _usage_from_events(deduped, 'claude-sonnet-4-6')
+
+ assert len(deduped) == 2
+ assert deduped[0]['id'] == 'usage-id'
+ assert deduped[1]['id'] == 'done-id'
+ assert summary.entry_count == 1
+ assert summary.total_prompt_tokens == 100
+ assert summary.total_prompt_cached_tokens == 40
+ assert summary.total_prompt_cache_creation_tokens == 10
+ assert summary.total_completion_tokens == 5
+ assert summary.total_tokens == 115
+
+
async def test_rust_terminal_priced_usage_prefers_model_usage_over_token_count(monkeypatch):
from browser_use.rust.service import _usage_from_events_with_costs
from browser_use.tokens.service import TokenCost
@@ -4149,21 +4226,35 @@ def test_rust_agent_defaults_page_extraction_llm_to_main_llm():
assert str(id(extraction_llm)) in override_agent.token_cost_service.registered_llms
-def test_rust_agent_defaults_llm_like_browser_use():
+def test_rust_agent_default_llm_is_rust_only_anthropic_sonnet(monkeypatch):
from browser_use.agent.service import _PythonAgent as BrowserUseAgent
from browser_use.rust import Agent as RustAgent
+ monkeypatch.setenv('ANTHROPIC_API_KEY', 'test-anthropic-key')
+
browser_use_agent = BrowserUseAgent(task='Use the default model.', directly_open_url=False)
rust_agent = RustAgent(task='Use the default model.', directly_open_url=False)
- assert type(rust_agent.llm) is type(browser_use_agent.llm)
- assert rust_agent.llm.provider == browser_use_agent.llm.provider == 'browser-use'
- assert rust_agent.llm.model == browser_use_agent.llm.model
- assert rust_agent.settings.flash_mode is True
+ assert browser_use_agent.llm.provider == 'browser-use'
+ assert rust_agent.llm.provider == 'anthropic'
+ assert rust_agent.llm.model == 'claude-sonnet-4-6'
+ assert rust_agent.settings.flash_mode is False
assert rust_agent.settings.page_extraction_llm is rust_agent.llm
assert rust_agent.model == rust_agent.llm.model
+def test_rust_agent_default_llm_respects_default_llm_env(monkeypatch):
+ from browser_use.rust import Agent
+
+ monkeypatch.setenv('OPENAI_API_KEY', 'test-openai-key')
+ monkeypatch.setenv('DEFAULT_LLM', 'openai_gpt_4_1_mini')
+
+ agent = Agent(task='Use configured default model.', directly_open_url=False)
+
+ assert agent.llm.provider == 'openai'
+ assert agent.llm.model == 'gpt-4.1-mini'
+
+
def test_rust_agent_enables_flash_mode_for_browser_use_llm_provider():
from browser_use.agent.service import _PythonAgent as BrowserUseAgent
from browser_use.rust import Agent as RustAgent
@@ -4657,6 +4748,7 @@ async def test_rust_agent_exposes_logging_helper_methods(monkeypatch):
process_error=None,
)
agent._sync_state_from_history()
+ agent.state.n_steps = 99
agent._log_final_outcome_messages()
agent._log_agent_event(max_steps=9, agent_run_error='manual-error')
@@ -4666,6 +4758,7 @@ async def test_rust_agent_exposes_logging_helper_methods(monkeypatch):
assert captured_events[0].cdp_url == 'browser.example'
assert captured_events[0].action_history == [[{'done': {'text': 'done', 'success': True}}]]
assert captured_events[0].urls_visited == ['https://example.com']
+ assert captured_events[0].steps == 1
assert captured_events[0].total_input_tokens == 11
assert captured_events[0].total_output_tokens == 7
assert captured_events[0].prompt_cached_tokens == 3
@@ -4673,6 +4766,45 @@ async def test_rust_agent_exposes_logging_helper_methods(monkeypatch):
assert captured_events[0].error_message == 'manual-error'
+def test_rust_agent_telemetry_filters_empty_reconstructed_urls():
+ import browser_use.rust.service as rust_service
+ from browser_use.rust import Agent
+
+ class LLM:
+ model = 'gpt-test'
+ provider = 'test-provider'
+
+ captured_events = []
+
+ class Telemetry:
+ def capture(self, event):
+ captured_events.append(event)
+
+ agent = Agent(task='Log telemetry URLs.', llm=cast(Any, LLM()), directly_open_url=False)
+ agent.telemetry = Telemetry()
+ agent.history = rust_service._history_from_events(
+ [
+ {'event_type': 'model.turn.request', 'payload': {'model': 'gpt-test'}},
+ {
+ 'event_type': 'tool.output',
+ 'payload': {'name': 'browser', 'text': 'Connected to browser at http://127.0.0.1:41335'},
+ },
+ {'event_type': 'model.turn.request', 'payload': {'model': 'gpt-test'}},
+ {'event_type': 'browser.state', 'payload': {'url': 'https://example.com', 'title': 'Example'}},
+ {'event_type': 'session.done', 'payload': {'result': 'done'}},
+ ],
+ model='gpt-test',
+ started=1.0,
+ finished=2.0,
+ output_model_schema=None,
+ process_error=None,
+ )
+
+ agent._log_agent_event(max_steps=3)
+
+ assert captured_events[0].urls_visited == ['https://example.com']
+
+
async def test_rust_agent_run_records_terminal_telemetry(monkeypatch):
from browser_use.rust import Agent
@@ -8741,8 +8873,8 @@ async def test_rust_agent_trace_and_cloud_auth_helpers():
async def test_rust_agent_run_exposes_laminar_trace_id_for_eval_links(monkeypatch):
- from browser_use.rust import Agent
import browser_use.rust.service as rust_service
+ from browser_use.rust import Agent
class FakeLaminar:
@staticmethod
@@ -8794,8 +8926,8 @@ def test_rust_laminar_replay_flush_avoids_context_reset(monkeypatch):
def test_rust_agent_laminar_run_summary_populates_current_span(monkeypatch):
- from browser_use.rust import Agent
import browser_use.rust.service as rust_service
+ from browser_use.rust import Agent
from browser_use.rust.service import _history_from_events
class FakeLaminar:
@@ -9151,8 +9283,8 @@ async def test_rust_agent_authenticate_cloud_sync_logs_browser_use_warning(monke
def test_rust_agent_trace_metadata_matches_browser_use_helpers(monkeypatch):
- from browser_use.rust import Agent
import browser_use.rust.service as rust_service
+ from browser_use.rust import Agent
from browser_use.rust.service import _history_from_events
monkeypatch.setattr(rust_service, 'get_browser_use_version', lambda: '9.9.9-test')