Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Expose Rust agent Laminar request payloads

Magnus Müller551fe77bce74
diff --git a/browser_use/rust/service.py b/browser_use/rust/service.py
index 6571c7136..32ea40223 100644
--- a/browser_use/rust/service.py
+++ b/browser_use/rust/service.py
@@ -1,11 +1,13 @@
from __future__ import annotations
import asyncio
+import base64
import hashlib
import inspect
import json
import keyword
import logging
+import mimetypes
import os
import re
import shutil
@@ -2750,7 +2752,15 @@ def _terminal_laminar_tools_for_span(span_input: dict[str, Any]) -> list[dict[st
def _terminal_laminar_span_input_payload(span_input: dict[str, Any]) -> Any:
- return _terminal_laminar_span_input_messages(span_input)
+ payload: dict[str, Any] = {
+ 'model': span_input.get('model'),
+ 'provider': span_input.get('provider'),
+ 'messages': _terminal_laminar_span_input_messages(span_input),
+ }
+ tools = _terminal_laminar_tools_for_span(span_input)
+ if tools:
+ payload['tools'] = tools
+ return payload
def _terminal_laminar_span_output_messages(span_output: dict[str, Any]) -> list[dict[str, Any]]:
@@ -2974,6 +2984,54 @@ def _terminal_laminar_tool_result_payload(events: list[dict[str, Any]], tool_cal
return None
+def _terminal_laminar_image_part_from_path(path: str, mime_type: str | None = None) -> dict[str, Any] | None:
+ image_path = Path(path)
+ if not image_path.exists() or not image_path.is_file():
+ return None
+ resolved_mime_type = mime_type or mimetypes.guess_type(str(image_path))[0] or 'image/png'
+ try:
+ data = base64.b64encode(image_path.read_bytes()).decode('ascii')
+ except OSError:
+ return None
+ return {'type': 'image_url', 'image_url': {'url': f'data:{resolved_mime_type};base64,{data}'}}
+
+
+def _terminal_laminar_image_part_for_span(image: Any) -> dict[str, Any] | None:
+ if isinstance(image, str):
+ if image.startswith(('http://', 'https://', 'data:image/')):
+ return {'type': 'image_url', 'image_url': {'url': image}}
+ return _terminal_laminar_image_part_from_path(image)
+ if not isinstance(image, dict):
+ return None
+ mime_type = image.get('mime_type') or image.get('mimeType') or image.get('media_type') or image.get('mediaType')
+ mime_type = mime_type if isinstance(mime_type, str) and mime_type else None
+ data = image.get('data') or image.get('base64')
+ if isinstance(data, str) and data:
+ resolved_mime_type = mime_type or 'image/png'
+ return {'type': 'image_url', 'image_url': {'url': f'data:{resolved_mime_type};base64,{data}'}}
+ url = image.get('url') or image.get('image_url')
+ if isinstance(url, str) and url:
+ if url.startswith('file://'):
+ return _terminal_laminar_image_part_from_path(url.removeprefix('file://'), mime_type)
+ return {'type': 'image_url', 'image_url': {'url': url}}
+ path = image.get('path')
+ if isinstance(path, str) and path:
+ return _terminal_laminar_image_part_from_path(path, mime_type)
+ return None
+
+
+def _terminal_laminar_image_parts_for_span(payload: dict[str, Any]) -> list[dict[str, Any]]:
+ images = payload.get('images')
+ if not isinstance(images, list):
+ return []
+ parts: list[dict[str, Any]] = []
+ for image in images:
+ part = _terminal_laminar_image_part_for_span(image)
+ if part is not None:
+ parts.append(part)
+ return parts
+
+
def _terminal_laminar_tool_input_message(tool_call: dict[str, Any]) -> list[dict[str, Any]]:
return [
{
@@ -2997,7 +3055,13 @@ def _terminal_laminar_tool_output_message(event_type: str | None, payload: dict[
text = _tool_result_text(payload)
if not text and event_type == 'tool.finished':
text = _synthetic_tool_result_text(str(payload.get('name') or 'tool'))
- parts = [{'type': 'text', 'text': text or ''}]
+ parts = []
+ if text:
+ parts.append({'type': 'text', 'text': text})
+ image_parts = _terminal_laminar_image_parts_for_span(payload)
+ parts.extend(image_parts)
+ if not parts:
+ parts = [{'type': 'text', 'text': ''}]
role = 'tool'
return [{'role': role, 'content': parts}]
diff --git a/tests/ci/test_rust_agent.py b/tests/ci/test_rust_agent.py
index bff499eff..ae5ccd196 100644
--- a/tests/ci/test_rust_agent.py
+++ b/tests/ci/test_rust_agent.py
@@ -7634,17 +7634,22 @@ def test_rust_agent_laminar_run_summary_populates_current_span(monkeypatch):
assert FakeLaminar.events[-1][0] == 'agent.run.terminal_summary'
assert FakeLaminar.spans[0]['name'] == 'rust_core.llm'
assert FakeLaminar.spans[0]['span_type'] == 'LLM'
- assert isinstance(FakeLaminar.spans[0]['input'], list)
- assert FakeLaminar.spans[0]['input'][0]['role'] == 'system'
- assert FakeLaminar.spans[0]['input'][0]['content'][0]['text'] == 'System prompt'
- assert FakeLaminar.spans[0]['input'][1]['content'][0]['text'] == 'Find the title on example.com'
- assert FakeLaminar.spans[0]['input'][1]['content'][1] == {
+ llm_input = FakeLaminar.spans[0]['input']
+ assert llm_input['model'] == 'claude-sonnet-4-6'
+ assert llm_input['provider'] == 'test-provider'
+ assert llm_input['messages'][0]['role'] == 'system'
+ assert llm_input['messages'][0]['content'][0]['text'] == 'System prompt'
+ assert llm_input['messages'][1]['content'][0]['text'] == 'Find the title on example.com'
+ assert llm_input['messages'][1]['content'][1] == {
'type': 'image_url',
'image_url': {
'url': 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB',
'detail': 'low',
},
}
+ assert llm_input['tools'][0]['name'] == 'browser_script'
+ assert 'click_at_xy' in llm_input['tools'][0]['description']
+ assert llm_input['tools'][0]['input_schema']['properties']['code']['type'] == 'string'
span_attrs = {}
for attributes in FakeLaminar.spans[0]['attributes']:
span_attrs.update(attributes)
@@ -7695,6 +7700,78 @@ def test_rust_agent_laminar_run_summary_populates_current_span(monkeypatch):
}
+def test_rust_agent_laminar_tool_span_preserves_image_only_outputs(tmp_path):
+ import browser_use.rust.service as rust_service
+
+ image_path = tmp_path / 'tool.png'
+ image_path.write_bytes(b'\x89PNG\r\n\x1a\n')
+
+ events = [
+ {
+ 'event_type': 'tool.started',
+ 'payload': {'name': 'browser_script', 'tool_call_id': 'call-1', 'arguments': {'code': 'observe()'}},
+ },
+ {
+ 'event_type': 'tool.output',
+ 'payload': {
+ 'name': 'browser_script',
+ 'tool_call_id': 'call-1',
+ 'ok': True,
+ 'text': '',
+ 'images': [{'path': str(image_path), 'mime_type': 'image/png'}],
+ },
+ },
+ ]
+
+ class FakeLaminar:
+ spans = []
+ current_span = None
+
+ @staticmethod
+ def is_initialized():
+ return True
+
+ @classmethod
+ def set_span_attributes(cls, attributes):
+ if cls.current_span is not None:
+ cls.current_span['attributes'].append(attributes)
+
+ @classmethod
+ def set_span_output(cls, output):
+ if cls.current_span is not None:
+ cls.current_span['outputs'].append(output)
+
+ @classmethod
+ def event(cls, name, attributes=None):
+ return None
+
+ @classmethod
+ def start_as_current_span(cls, name, input=None, span_type='DEFAULT'):
+ record = {'name': name, 'input': input, 'span_type': span_type, 'attributes': [], 'outputs': []}
+
+ class Span:
+ def __enter__(self):
+ cls.current_span = record
+ cls.spans.append(record)
+ return self
+
+ def __exit__(self, exc_type, exc, tb):
+ cls.current_span = None
+ return False
+
+ return Span()
+
+ original = rust_service.Laminar
+ try:
+ rust_service.Laminar = FakeLaminar
+ rust_service._record_laminar_terminal_tool_spans(events, max_spans=10)
+ finally:
+ rust_service.Laminar = original
+
+ output = FakeLaminar.spans[0]['outputs'][-1]
+ assert output[0]['content'] == [{'type': 'image_url', 'image_url': {'url': 'data:image/png;base64,iVBORw0KGgo='}}]
+
+
async def test_rust_agent_authenticate_cloud_sync_logs_browser_use_warning(monkeypatch):
from browser_use.agent.service import _PythonAgent as BrowserUseAgent
from browser_use.rust import Agent as RustAgent