Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
allow reading images such as jpg/png, as well as docx, fixes to the flash system prompt
mertunsall03c9e94dc68d
diff --git a/browser_use/agent/message_manager/service.py b/browser_use/agent/message_manager/service.pyindex f16583ee7..25e25670e 100644--- a/browser_use/agent/message_manager/service.py+++ b/browser_use/agent/message_manager/service.py@@ -184,6 +184,7 @@ class MessageManager:step_number = step_info.step_number if step_info else Noneself.state.read_state_description = ''+ self.state.read_state_images = [] # Clear images from previous stepaction_results = ''result_len = len(result)@@ -197,6 +198,11 @@ class MessageManager:read_state_idx += 1logger.debug(f'Added extracted_content to read_state_description: {action_result.extracted_content}')+ # Store images for one-time inclusion in the next message+ if action_result.images:+ self.state.read_state_images.extend(action_result.images)+ logger.debug(f'Added {len(action_result.images)} image(s) to read_state_images')+if action_result.long_term_memory:action_results += f'{action_result.long_term_memory}\n'logger.debug(f'Added long_term_memory to action_results: {action_result.long_term_memory}')@@ -354,6 +360,7 @@ class MessageManager:vision_detail_level=self.vision_detail_level,include_recent_events=self.include_recent_events,sample_images=self.sample_images,+ read_state_images=self.state.read_state_images,).get_user_message(effective_use_vision)# Store state message text for historydiff --git a/browser_use/agent/message_manager/views.py b/browser_use/agent/message_manager/views.pyindex d82dcd06c..88076ba52 100644--- a/browser_use/agent/message_manager/views.py+++ b/browser_use/agent/message_manager/views.py@@ -1,6 +1,6 @@from __future__ import annotations-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyfrom pydantic import BaseModel, ConfigDict, Field@@ -92,5 +92,7 @@ class MessageManagerState(BaseModel):default_factory=lambda: [HistoryItem(step_number=0, system_message='Agent initialized')])read_state_description: str = ''+ # Images to include in the next state message (cleared after each step)+ read_state_images: list[dict[str, Any]] = Field(default_factory=list)model_config = ConfigDict(arbitrary_types_allowed=True)diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.pyindex 829e3d4c2..c01a40413 100644--- a/browser_use/agent/prompts.py+++ b/browser_use/agent/prompts.py@@ -84,6 +84,7 @@ class AgentMessagePrompt:vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',include_recent_events: bool = False,sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,+ read_state_images: list[dict] | None = None,):self.browser_state: 'BrowserStateSummary' = browser_state_summaryself.file_system: 'FileSystem | None' = file_system@@ -100,6 +101,7 @@ class AgentMessagePrompt:self.vision_detail_level = vision_detail_levelself.include_recent_events = include_recent_eventsself.sample_images = sample_images or []+ self.read_state_images = read_state_images or []assert self.browser_statedef _extract_page_statistics(self) -> dict[str, int]:@@ -346,7 +348,10 @@ Available tabs:state_description += self.page_filtered_actions + '\n'state_description += '</page_specific_actions>\n'- if use_vision is True and self.screenshots:+ # Check if we have images to include (from read_file action)+ has_images = bool(self.read_state_images)++ if (use_vision is True and self.screenshots) or has_images:# Start with text descriptioncontent_parts: list[ContentPartTextParam | ContentPartImageParam] = [ContentPartTextParam(text=state_description)]@@ -375,6 +380,34 @@ Available tabs:))+ # Add read_state images (from read_file action) before screenshots+ for img_data in self.read_state_images:+ img_name = img_data.get('name', 'unknown')+ img_base64 = img_data.get('data', '')++ if not img_base64:+ continue++ # Detect image format from name+ if img_name.lower().endswith('.png'):+ media_type = 'image/png'+ else:+ media_type = 'image/jpeg'++ # Add label+ content_parts.append(ContentPartTextParam(text=f'Image from file: {img_name}'))++ # Add the image+ content_parts.append(+ ContentPartImageParam(+ image_url=ImageURL(+ url=f'data:{media_type};base64,{img_base64}',+ media_type=media_type,+ detail=self.vision_detail_level,+ ),+ )+ )+return UserMessage(content=content_parts, cache=True)return UserMessage(content=state_description, cache=True)diff --git a/browser_use/agent/system_prompt_flash.md b/browser_use/agent/system_prompt_flash.mdindex e5254fddc..c420cd0c6 100644--- a/browser_use/agent/system_prompt_flash.md+++ b/browser_use/agent/system_prompt_flash.md@@ -2,7 +2,7 @@ You are an AI agent designed to operate in an iterative loop to automate browser<language_settings>Default: English. Match user's language.</language_settings><user_request>Ultimate objective. Specific tasks: follow each step. Open-ended: plan approach.</user_request><browser_state>Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.</browser_state>-<file_system>- PDFs auto-download to available_file_paths. Read file or scroll viewer. Persistent file system for progress tracking. Long tasks <10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. CSV: use double quotes for commas. available_file_paths: downloaded/user files (read/upload only).</file_system>+<file_system>- PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or scroll and look at screenshot. You have access to persistent file system for progress tracking. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. When writing CSV, use double quotes for commas. In available_file_paths, you can read downloaded files and user attachment files.</file_system><output>You must respond with a valid JSON in this exact format:{{"memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its opvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",diff --git a/browser_use/agent/views.py b/browser_use/agent/views.pyindex a45db7e64..0d2744d6f 100644--- a/browser_use/agent/views.py+++ b/browser_use/agent/views.py@@ -112,6 +112,9 @@ class ActionResult(BaseModel):# Filesattachments: list[str] | None = None # Files to display in the done message+ # Images (base64 encoded) - separate from text content for efficient handling+ images: list[dict[str, Any]] | None = None # [{"name": "file.jpg", "data": "base64_string"}]+# Always include in long term memorylong_term_memory: str | None = None # Memory of this actiondiff --git a/browser_use/filesystem/file_system.py b/browser_use/filesystem/file_system.pyindex 24102ffe1..2579e62fa 100644--- a/browser_use/filesystem/file_system.py+++ b/browser_use/filesystem/file_system.py@@ -1,4 +1,5 @@import asyncio+import base64import osimport reimport shutil@@ -165,6 +166,46 @@ class PdfFile(BaseFile):await asyncio.get_event_loop().run_in_executor(executor, lambda: self.sync_to_disk_sync(path))+class DocxFile(BaseFile):+ """DOCX file implementation"""++ @property+ def extension(self) -> str:+ return 'docx'++ def sync_to_disk_sync(self, path: Path) -> None:+ file_path = path / self.full_name+ try:+ from docx import Document++ doc = Document()++ # Convert content to DOCX paragraphs+ content_lines = self.content.split('\n')++ for line in content_lines:+ if line.strip():+ # Handle basic markdown headers+ if line.startswith('# '):+ doc.add_heading(line[2:], level=1)+ elif line.startswith('## '):+ doc.add_heading(line[3:], level=2)+ elif line.startswith('### '):+ doc.add_heading(line[4:], level=3)+ else:+ doc.add_paragraph(line)+ else:+ doc.add_paragraph() # Empty paragraph for spacing++ doc.save(str(file_path))+ except Exception as e:+ raise FileSystemError(f"Error: Could not write to file '{self.full_name}'. {str(e)}")++ async def sync_to_disk(self, path: Path) -> None:+ with ThreadPoolExecutor() as executor:+ await asyncio.get_event_loop().run_in_executor(executor, lambda: self.sync_to_disk_sync(path))++class FileSystemState(BaseModel):"""Serializable state of the file system"""@@ -195,6 +236,7 @@ class FileSystem:'jsonl': JsonlFile,'csv': CsvFile,'pdf': PdfFile,+ 'docx': DocxFile,}self.files = {}@@ -264,20 +306,42 @@ class FileSystem:return file_obj.read()- async def read_file(self, full_filename: str, external_file: bool = False) -> str:- """Read file content using file-specific read method and return appropriate message to LLM"""+ async def read_file_structured(self, full_filename: str, external_file: bool = False) -> dict[str, Any]:+ """Read file and return structured data including images if applicable.++ Returns:+ dict with keys:+ - 'message': str - The message to display+ - 'images': list[dict] | None - Image data if file is an image: [{"name": str, "data": base64_str}]+ """+ result: dict[str, Any] = {'message': '', 'images': None}+if external_file:try:try:_, extension = self._parse_filename(full_filename)except Exception:- return f'Error: Invalid filename format {full_filename}. Must be alphanumeric with a supported extension.'+ result['message'] = (+ f'Error: Invalid filename format {full_filename}. Must be alphanumeric with a supported extension.'+ )+ return result+if extension in ['md', 'txt', 'json', 'jsonl', 'csv']:import anyioasync with await anyio.open_file(full_filename, 'r') as f:content = await f.read()- return f'Read from file {full_filename}.\n<content>\n{content}\n</content>'+ result['message'] = f'Read from file {full_filename}.\n<content>\n{content}\n</content>'+ return result++ elif extension == 'docx':+ from docx import Document++ doc = Document(full_filename)+ content = '\n'.join([para.text for para in doc.paragraphs])+ result['message'] = f'Read from file {full_filename}.\n<content>\n{content}\n</content>'+ return result+elif extension == 'pdf':import pypdf@@ -289,30 +353,66 @@ class FileSystem:for page in reader.pages[:MAX_PDF_PAGES]:extracted_text += page.extract_text()extra_pages_text = f'{extra_pages} more pages...' if extra_pages > 0 else ''- return f'Read from file {full_filename}.\n<content>\n{extracted_text}\n{extra_pages_text}</content>'+ result['message'] = (+ f'Read from file {full_filename}.\n<content>\n{extracted_text}\n{extra_pages_text}</content>'+ )+ return result++ elif extension in ['jpg', 'jpeg', 'png']:+ import anyio++ # Read image file and convert to base64+ async with await anyio.open_file(full_filename, 'rb') as f:+ img_data = await f.read()++ base64_str = base64.b64encode(img_data).decode('utf-8')++ result['message'] = f'Read image file {full_filename}.'+ result['images'] = [{'name': os.path.basename(full_filename), 'data': base64_str}]+ return result+else:- return f'Error: Cannot read file {full_filename} as {extension} extension is not supported.'+ result['message'] = f'Error: Cannot read file {full_filename} as {extension} extension is not supported.'+ return result+except FileNotFoundError:- return f"Error: File '{full_filename}' not found."+ result['message'] = f"Error: File '{full_filename}' not found."+ return resultexcept PermissionError:- return f"Error: Permission denied to read file '{full_filename}'."+ result['message'] = f"Error: Permission denied to read file '{full_filename}'."+ return resultexcept Exception as e:- return f"Error: Could not read file '{full_filename}'."+ result['message'] = f"Error: Could not read file '{full_filename}'. {str(e)}"+ return result+ # For internal files, only non-image types are supportedif not self._is_valid_filename(full_filename):- return INVALID_FILENAME_ERROR_MESSAGE+ result['message'] = INVALID_FILENAME_ERROR_MESSAGE+ return resultfile_obj = self.get_file(full_filename)if not file_obj:- return f"File '{full_filename}' not found."+ result['message'] = f"File '{full_filename}' not found."+ return resulttry:content = file_obj.read()- return f'Read from file {full_filename}.\n<content>\n{content}\n</content>'+ result['message'] = f'Read from file {full_filename}.\n<content>\n{content}\n</content>'+ return resultexcept FileSystemError as e:- return str(e)- except Exception:- return f"Error: Could not read file '{full_filename}'."+ result['message'] = str(e)+ return result+ except Exception as e:+ result['message'] = f"Error: Could not read file '{full_filename}'. {str(e)}"+ return result++ async def read_file(self, full_filename: str, external_file: bool = False) -> str:+ """Read file content using file-specific read method and return appropriate message to LLM.++ Note: For image files, use read_file_structured() to get image data.+ """+ result = await self.read_file_structured(full_filename, external_file)+ return result['message']async def write_file(self, full_filename: str, content: str) -> str:"""Write content to file using file-specific write method"""@@ -506,6 +606,8 @@ class FileSystem:file_obj = CsvFile(**file_info)elif file_type == 'PdfFile':file_obj = PdfFile(**file_info)+ elif file_type == 'DocxFile':+ file_obj = DocxFile(**file_info)else:# Skip unknown file typescontinuediff --git a/browser_use/tools/service.py b/browser_use/tools/service.pyindex dfd381a92..7af985474 100644--- a/browser_use/tools/service.py+++ b/browser_use/tools/service.py@@ -976,16 +976,22 @@ You will be given a query and the markdown of a webpage that has been filtered treturn ActionResult(extracted_content=result, long_term_memory=result)@self.registry.action(- 'Read the complete content of a file. Use this to view file contents before editing or to retrieve data from files.'+ 'Read the complete content of a file. Use this to view file contents before editing or to retrieve data from files. Supports text files (txt, md, json, csv, jsonl), documents (pdf, docx), and images (jpg, png).')async def read_file(file_name: str, available_file_paths: list[str], file_system: FileSystem):if available_file_paths and file_name in available_file_paths:- result = await file_system.read_file(file_name, external_file=True)+ structured_result = await file_system.read_file_structured(file_name, external_file=True)else:- result = await file_system.read_file(file_name)+ structured_result = await file_system.read_file_structured(file_name)++ result = structured_result['message']+ images = structured_result.get('images')MAX_MEMORY_SIZE = 1000- if len(result) > MAX_MEMORY_SIZE:+ # For images, create a shorter memory message+ if images:+ memory = f'Read image file {file_name}'+ elif len(result) > MAX_MEMORY_SIZE:lines = result.splitlines()display = ''lines_count = 0@@ -1003,6 +1009,7 @@ You will be given a query and the markdown of a webpage that has been filtered treturn ActionResult(extracted_content=result,long_term_memory=memory,+ images=images,include_extracted_content_only_once=True,)diff --git a/pyproject.toml b/pyproject.tomlindex 284451946..b2643e1a2 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -46,6 +46,7 @@ dependencies = ["pillow>=11.2.1","cloudpickle>=3.1.1","markdownify>=1.2.0",+ "python-docx>=1.2.0",]# google-api-core: only used for Google LLM APIs# pyperclip: only used for examples that use copy/pastediff --git a/tests/ci/test_file_system_docx.py b/tests/ci/test_file_system_docx.pynew file mode 100644index 000000000..cc71d500d--- /dev/null+++ b/tests/ci/test_file_system_docx.py@@ -0,0 +1,192 @@+"""Tests for DOCX file support in the FileSystem."""++from pathlib import Path++import pytest++from browser_use.filesystem.file_system import (+ DocxFile,+ FileSystem,+)+++class TestDocxFile:+ """Test DOCX file operations."""++ @pytest.mark.asyncio+ async def test_create_docx_file(self, tmp_path: Path):+ """Test creating a DOCX file."""+ fs = FileSystem(tmp_path)+ content = """# Heading 1+## Heading 2+### Heading 3+Regular paragraph text.++Another paragraph."""++ result = await fs.write_file('test.docx', content)+ assert 'successfully' in result.lower()+ assert 'test.docx' in fs.list_files()++ @pytest.mark.asyncio+ async def test_read_docx_file_internal(self, tmp_path: Path):+ """Test reading internal DOCX file."""+ fs = FileSystem(tmp_path)+ content = """# Title+Some content here."""++ await fs.write_file('test.docx', content)+ result = await fs.read_file('test.docx')++ assert 'test.docx' in result+ assert 'Title' in result or 'content' in result++ @pytest.mark.asyncio+ async def test_read_docx_file_external(self, tmp_path: Path):+ """Test reading external DOCX file."""+ from docx import Document++ # Create an external DOCX file+ external_file = tmp_path / 'external.docx'+ doc = Document()+ doc.add_heading('Test Heading', level=1)+ doc.add_paragraph('Test paragraph content.')+ doc.save(str(external_file))++ fs = FileSystem(tmp_path / 'workspace')+ structured_result = await fs.read_file_structured(str(external_file), external_file=True)++ assert 'message' in structured_result+ assert 'Test Heading' in structured_result['message']+ assert 'Test paragraph content' in structured_result['message']++ def test_docx_file_extension(self):+ """Test DOCX file extension property."""+ docx_file = DocxFile(name='test')+ assert docx_file.extension == 'docx'+ assert docx_file.full_name == 'test.docx'++ @pytest.mark.asyncio+ async def test_docx_with_unicode_characters(self, tmp_path: Path):+ """Test DOCX with unicode and emoji content."""+ fs = FileSystem(tmp_path)+ content = """# Unicode Test 🚀+Chinese: 你好世界+Arabic: مرحبا بالعالم+Emoji: 😀 👍 🎉"""++ result = await fs.write_file('unicode.docx', content)+ assert 'successfully' in result.lower()++ read_result = await fs.read_file('unicode.docx')+ assert 'Unicode Test' in read_result+ # Note: Emoji may not be preserved in all systems++ @pytest.mark.asyncio+ async def test_empty_docx_file(self, tmp_path: Path):+ """Test creating an empty DOCX file."""+ fs = FileSystem(tmp_path)+ result = await fs.write_file('empty.docx', '')+ assert 'successfully' in result.lower()++ @pytest.mark.asyncio+ async def test_large_docx_file(self, tmp_path: Path):+ """Test creating a large DOCX file."""+ fs = FileSystem(tmp_path)+ # Create content with 1000 lines+ lines = [f'Line {i}: This is a test line with some content.' for i in range(1000)]+ content = '\n'.join(lines)++ result = await fs.write_file('large.docx', content)+ assert 'successfully' in result.lower()++ # Verify it can be read back+ read_result = await fs.read_file('large.docx')+ assert 'Line 0:' in read_result+ assert 'Line 999:' in read_result++ @pytest.mark.asyncio+ async def test_corrupted_docx_file(self, tmp_path: Path):+ """Test reading a corrupted DOCX file."""+ # Create a corrupted DOCX file+ external_file = tmp_path / 'corrupted.docx'+ external_file.write_bytes(b'This is not a valid DOCX file')++ fs = FileSystem(tmp_path / 'workspace')+ structured_result = await fs.read_file_structured(str(external_file), external_file=True)++ assert 'message' in structured_result+ assert 'error' in structured_result['message'].lower() or 'could not' in structured_result['message'].lower()++ @pytest.mark.asyncio+ async def test_docx_with_multiple_paragraphs(self, tmp_path: Path):+ """Test DOCX with various paragraph styles."""+ fs = FileSystem(tmp_path)+ content = """# Main Title+## Subtitle+This is a regular paragraph.++This is another paragraph with some text.++### Section 3+Final paragraph here."""++ await fs.write_file('multi.docx', content)+ result = await fs.read_file('multi.docx')++ # Should contain all the text (headings converted to paragraphs)+ assert 'Main Title' in result+ assert 'Subtitle' in result+ assert 'regular paragraph' in result+ assert 'Final paragraph' in result+++class TestFileSystemDocxIntegration:+ """Integration tests for DOCX file type."""++ @pytest.mark.asyncio+ async def test_multiple_file_types_with_docx(self, tmp_path: Path):+ """Test working with DOCX alongside other file types."""+ fs = FileSystem(tmp_path)++ # Create different file types+ await fs.write_file('doc.docx', '# Document\nContent here')+ await fs.write_file('data.json', '{"key": "value"}')+ await fs.write_file('notes.txt', 'Some notes')++ # Verify all files exist+ files = fs.list_files()+ assert 'doc.docx' in files+ assert 'data.json' in files+ assert 'notes.txt' in files+ assert 'todo.md' in files # Default file++ @pytest.mark.asyncio+ async def test_file_system_state_with_docx(self, tmp_path: Path):+ """Test FileSystem state serialization with DOCX files."""+ fs = FileSystem(tmp_path)++ # Create files+ await fs.write_file('test.docx', '# Title\nContent')+ await fs.write_file('data.txt', 'Some text')++ # Get state+ state = fs.get_state()+ assert 'test.docx' in state.files+ assert 'data.txt' in state.files++ # Restore from state+ fs2 = FileSystem.from_state(state)+ assert 'test.docx' in fs2.list_files()+ assert 'data.txt' in fs2.list_files()++ def test_allowed_extensions_include_docx(self, tmp_path: Path):+ """Test that DOCX is in allowed extensions."""+ fs = FileSystem(tmp_path)+ allowed = fs.get_allowed_extensions()++ assert 'docx' in allowed+++if __name__ == '__main__':+ pytest.main([__file__, '-v'])diff --git a/tests/ci/test_file_system_images.py b/tests/ci/test_file_system_images.pynew file mode 100644index 000000000..8325853a6--- /dev/null+++ b/tests/ci/test_file_system_images.py@@ -0,0 +1,249 @@+"""Tests for image file support in the FileSystem."""++import base64+import io+from pathlib import Path++import pytest+from PIL import Image++from browser_use.filesystem.file_system import FileSystem+++class TestImageFiles:+ """Test image file operations - only external reading supported."""++ def create_test_image(self, width: int = 100, height: int = 100, format: str = 'PNG') -> bytes:+ """Create a test image and return bytes."""+ img = Image.new('RGB', (width, height), color='red')+ buffer = io.BytesIO()+ img.save(buffer, format=format)+ buffer.seek(0)+ return buffer.read()++ @pytest.mark.asyncio+ async def test_read_external_png_image(self, tmp_path: Path):+ """Test reading external PNG image file."""+ # Create an external image file+ external_file = tmp_path / 'test.png'+ img_bytes = self.create_test_image(width=300, height=200, format='PNG')+ external_file.write_bytes(img_bytes)++ fs = FileSystem(tmp_path / 'workspace')+ structured_result = await fs.read_file_structured(str(external_file), external_file=True)++ assert 'message' in structured_result+ assert 'Read image file' in structured_result['message']+ assert 'images' in structured_result+ assert structured_result['images'] is not None+ assert len(structured_result['images']) == 1++ img_data = structured_result['images'][0]+ assert img_data['name'] == 'test.png'+ assert 'data' in img_data+ # Verify base64 is valid+ decoded = base64.b64decode(img_data['data'])+ assert decoded == img_bytes++ @pytest.mark.asyncio+ async def test_read_external_jpg_image(self, tmp_path: Path):+ """Test reading external JPG image file."""+ # Create an external image file+ external_file = tmp_path / 'photo.jpg'+ img_bytes = self.create_test_image(width=150, height=100, format='JPEG')+ external_file.write_bytes(img_bytes)++ fs = FileSystem(tmp_path / 'workspace')+ structured_result = await fs.read_file_structured(str(external_file), external_file=True)++ assert 'message' in structured_result+ assert 'images' in structured_result+ assert structured_result['images'] is not None++ img_data = structured_result['images'][0]+ assert img_data['name'] == 'photo.jpg'+ decoded = base64.b64decode(img_data['data'])+ assert len(decoded) > 0++ @pytest.mark.asyncio+ async def test_read_jpeg_extension(self, tmp_path: Path):+ """Test reading .jpeg extension (not just .jpg)."""+ external_file = tmp_path / 'test.jpeg'+ img_bytes = self.create_test_image(format='JPEG')+ external_file.write_bytes(img_bytes)++ fs = FileSystem(tmp_path / 'workspace')+ structured_result = await fs.read_file_structured(str(external_file), external_file=True)++ assert structured_result['images'] is not None+ assert structured_result['images'][0]['name'] == 'test.jpeg'++ @pytest.mark.asyncio+ async def test_read_nonexistent_image(self, tmp_path: Path):+ """Test reading a nonexistent image file."""+ fs = FileSystem(tmp_path / 'workspace')+ structured_result = await fs.read_file_structured('/path/to/nonexistent.png', external_file=True)++ assert 'message' in structured_result+ assert 'not found' in structured_result['message'].lower()+ assert structured_result['images'] is None++ @pytest.mark.asyncio+ async def test_corrupted_image_file(self, tmp_path: Path):+ """Test reading a corrupted image file."""+ external_file = tmp_path / 'corrupted.png'+ # Write invalid PNG data+ external_file.write_bytes(b'Not a valid PNG file')++ fs = FileSystem(tmp_path / 'workspace')+ structured_result = await fs.read_file_structured(str(external_file), external_file=True)++ # Should still return base64 data (we don't validate image format)+ assert 'message' in structured_result+ assert 'Read image file' in structured_result['message']+ # Base64 encoding will succeed even for invalid image data+ assert structured_result['images'] is not None++ @pytest.mark.asyncio+ async def test_large_image_file(self, tmp_path: Path):+ """Test reading a large image file."""+ # Create a large image (2000x2000)+ external_file = tmp_path / 'large.png'+ img = Image.new('RGB', (2000, 2000), color='blue')+ img.save(str(external_file), format='PNG')++ fs = FileSystem(tmp_path / 'workspace')+ structured_result = await fs.read_file_structured(str(external_file), external_file=True)++ assert 'images' in structured_result+ assert structured_result['images'] is not None+ # Verify base64 data is present and substantial+ assert len(structured_result['images'][0]['data']) > 10000++ @pytest.mark.asyncio+ async def test_multiple_images_in_sequence(self, tmp_path: Path):+ """Test reading multiple images in sequence."""+ fs = FileSystem(tmp_path / 'workspace')++ # Create three different images+ for i, color in enumerate(['red', 'green', 'blue']):+ img_file = tmp_path / f'image_{i}.png'+ img = Image.new('RGB', (100, 100), color=color)+ img.save(str(img_file), format='PNG')++ # Read them all+ results = []+ for i in range(3):+ img_file = tmp_path / f'image_{i}.png'+ result = await fs.read_file_structured(str(img_file), external_file=True)+ results.append(result)++ # Verify all were read successfully+ for i, result in enumerate(results):+ assert result['images'] is not None+ assert result['images'][0]['name'] == f'image_{i}.png'++ @pytest.mark.asyncio+ async def test_different_image_formats(self, tmp_path: Path):+ """Test reading different image format variations."""+ fs = FileSystem(tmp_path / 'workspace')++ # Test .jpg+ jpg_file = tmp_path / 'test.jpg'+ img = Image.new('RGB', (50, 50), color='yellow')+ img.save(str(jpg_file), format='JPEG')+ result_jpg = await fs.read_file_structured(str(jpg_file), external_file=True)+ assert result_jpg['images'] is not None++ # Test .jpeg+ jpeg_file = tmp_path / 'test.jpeg'+ img.save(str(jpeg_file), format='JPEG')+ result_jpeg = await fs.read_file_structured(str(jpeg_file), external_file=True)+ assert result_jpeg['images'] is not None++ # Test .png+ png_file = tmp_path / 'test.png'+ img.save(str(png_file), format='PNG')+ result_png = await fs.read_file_structured(str(png_file), external_file=True)+ assert result_png['images'] is not None++ @pytest.mark.asyncio+ async def test_image_with_transparency(self, tmp_path: Path):+ """Test reading PNG with transparency (RGBA)."""+ external_file = tmp_path / 'transparent.png'+ # Create RGBA image with transparency+ img = Image.new('RGBA', (100, 100), color=(255, 0, 0, 128))+ img.save(str(external_file), format='PNG')++ fs = FileSystem(tmp_path / 'workspace')+ structured_result = await fs.read_file_structured(str(external_file), external_file=True)++ assert structured_result['images'] is not None+ assert len(structured_result['images'][0]['data']) > 0+++class TestActionResultImages:+ """Test ActionResult with images field."""++ def test_action_result_with_images(self):+ """Test creating ActionResult with images."""+ from browser_use.agent.views import ActionResult++ images = [{'name': 'test.png', 'data': 'base64_encoded_data_here'}]++ result = ActionResult(+ extracted_content='Read image file test.png',+ long_term_memory='Read image file test.png',+ images=images,+ include_extracted_content_only_once=True,+ )++ assert result.images is not None+ assert len(result.images) == 1+ assert result.images[0]['name'] == 'test.png'+ assert result.images[0]['data'] == 'base64_encoded_data_here'++ def test_action_result_without_images(self):+ """Test ActionResult without images (default behavior)."""+ from browser_use.agent.views import ActionResult++ result = ActionResult(extracted_content='Some text', long_term_memory='Memory')++ assert result.images is None++ def test_action_result_with_multiple_images(self):+ """Test ActionResult with multiple images."""+ from browser_use.agent.views import ActionResult++ images = [+ {'name': 'image1.png', 'data': 'base64_data_1'},+ {'name': 'image2.jpg', 'data': 'base64_data_2'},+ ]++ result = ActionResult(+ extracted_content='Read multiple images',+ long_term_memory='Read image files',+ images=images,+ include_extracted_content_only_once=True,+ )++ assert result.images is not None+ assert len(result.images) == 2+ assert result.images[0]['name'] == 'image1.png'+ assert result.images[1]['name'] == 'image2.jpg'++ def test_action_result_with_empty_images_list(self):+ """Test ActionResult with empty images list."""+ from browser_use.agent.views import ActionResult++ result = ActionResult(+ extracted_content='No images',+ images=[],+ )++ # Empty list is still valid+ assert result.images == []+++if __name__ == '__main__':+ pytest.main([__file__, '-v'])diff --git a/tests/ci/test_file_system_llm_integration.py b/tests/ci/test_file_system_llm_integration.pynew file mode 100644index 000000000..3962843b3--- /dev/null+++ b/tests/ci/test_file_system_llm_integration.py@@ -0,0 +1,386 @@+"""Integration tests for DOCX and image file support in LLM messages."""++import base64+import io+from pathlib import Path++import pytest+from PIL import Image++from browser_use.agent.message_manager.service import MessageManager+from browser_use.agent.prompts import AgentMessagePrompt+from browser_use.agent.views import ActionResult, AgentStepInfo+from browser_use.browser.views import BrowserStateSummary, TabInfo+from browser_use.dom.views import SerializedDOMState+from browser_use.filesystem.file_system import FileSystem+from browser_use.llm.messages import ContentPartImageParam, ContentPartTextParam, SystemMessage+++class TestImageInLLMMessages:+ """Test that images flow correctly through to LLM messages."""++ def create_test_image(self, width: int = 100, height: int = 100) -> bytes:+ """Create a test image and return bytes."""+ img = Image.new('RGB', (width, height), color='red')+ buffer = io.BytesIO()+ img.save(buffer, format='PNG')+ buffer.seek(0)+ return buffer.read()++ @pytest.mark.asyncio+ async def test_image_stored_in_message_manager(self, tmp_path: Path):+ """Test that images are stored in MessageManager state."""+ fs = FileSystem(tmp_path)+ system_message = SystemMessage(content='Test system message')+ mm = MessageManager(task='test', system_message=system_message, file_system=fs)++ # Create ActionResult with images+ images = [{'name': 'test.png', 'data': 'base64_test_data'}]+ action_results = [+ ActionResult(+ extracted_content='Read image file test.png',+ long_term_memory='Read image file test.png',+ images=images,+ include_extracted_content_only_once=True,+ )+ ]++ # Update message manager with results+ step_info = AgentStepInfo(step_number=1, max_steps=10)+ mm._update_agent_history_description(model_output=None, result=action_results, step_info=step_info)++ # Verify images are stored+ assert mm.state.read_state_images is not None+ assert len(mm.state.read_state_images) == 1+ assert mm.state.read_state_images[0]['name'] == 'test.png'+ assert mm.state.read_state_images[0]['data'] == 'base64_test_data'++ @pytest.mark.asyncio+ async def test_images_cleared_after_step(self, tmp_path: Path):+ """Test that images are cleared after each step."""+ fs = FileSystem(tmp_path)+ system_message = SystemMessage(content='Test system message')+ mm = MessageManager(task='test', system_message=system_message, file_system=fs)++ # First step with images+ images = [{'name': 'test.png', 'data': 'base64_data'}]+ action_results = [ActionResult(images=images, include_extracted_content_only_once=True)]+ step_info = AgentStepInfo(step_number=1, max_steps=10)+ mm._update_agent_history_description(model_output=None, result=action_results, step_info=step_info)++ assert len(mm.state.read_state_images) == 1++ # Second step without images - should clear+ action_results_2 = [ActionResult(extracted_content='No images')]+ step_info_2 = AgentStepInfo(step_number=2, max_steps=10)+ mm._update_agent_history_description(model_output=None, result=action_results_2, step_info=step_info_2)++ assert len(mm.state.read_state_images) == 0++ @pytest.mark.asyncio+ async def test_multiple_images_accumulated(self, tmp_path: Path):+ """Test that multiple images in one step are accumulated."""+ fs = FileSystem(tmp_path)+ system_message = SystemMessage(content='Test system message')+ mm = MessageManager(task='test', system_message=system_message, file_system=fs)++ # Multiple action results with images+ action_results = [+ ActionResult(images=[{'name': 'img1.png', 'data': 'data1'}], include_extracted_content_only_once=True),+ ActionResult(images=[{'name': 'img2.jpg', 'data': 'data2'}], include_extracted_content_only_once=True),+ ]+ step_info = AgentStepInfo(step_number=1, max_steps=10)+ mm._update_agent_history_description(model_output=None, result=action_results, step_info=step_info)++ assert len(mm.state.read_state_images) == 2+ assert mm.state.read_state_images[0]['name'] == 'img1.png'+ assert mm.state.read_state_images[1]['name'] == 'img2.jpg'++ def test_agent_message_prompt_includes_images(self, tmp_path: Path):+ """Test that AgentMessagePrompt includes images in message content."""+ fs = FileSystem(tmp_path)++ # Create browser state+ browser_state = BrowserStateSummary(+ url='https://example.com',+ title='Test',+ tabs=[TabInfo(target_id='test-0', url='https://example.com', title='Test')],+ screenshot=None,+ dom_state=SerializedDOMState(_root=None, selector_map={}),+ )++ # Create images+ read_state_images = [{'name': 'test.png', 'data': 'base64_image_data_here'}]++ # Create message prompt+ prompt = AgentMessagePrompt(+ browser_state_summary=browser_state,+ file_system=fs,+ read_state_images=read_state_images,+ )++ # Get user message with vision enabled+ user_message = prompt.get_user_message(use_vision=True)++ # Verify message has content parts (not just string)+ assert isinstance(user_message.content, list)++ # Find image content parts+ image_parts = [part for part in user_message.content if isinstance(part, ContentPartImageParam)]+ text_parts = [part for part in user_message.content if isinstance(part, ContentPartTextParam)]++ # Should have at least one image+ assert len(image_parts) >= 1++ # Should have text label+ image_labels = [part.text for part in text_parts if 'test.png' in part.text]+ assert len(image_labels) >= 1++ # Verify image data URL format+ img_part = image_parts[0]+ assert 'data:image/' in img_part.image_url.url+ assert 'base64,base64_image_data_here' in img_part.image_url.url++ def test_agent_message_prompt_png_vs_jpg_media_type(self, tmp_path: Path):+ """Test that AgentMessagePrompt correctly detects PNG vs JPG media types."""+ fs = FileSystem(tmp_path)++ browser_state = BrowserStateSummary(+ url='https://example.com',+ title='Test',+ tabs=[TabInfo(target_id='test-0', url='https://example.com', title='Test')],+ screenshot=None,+ dom_state=SerializedDOMState(_root=None, selector_map={}),+ )++ # Test PNG+ read_state_images_png = [{'name': 'test.png', 'data': 'data'}]+ prompt_png = AgentMessagePrompt(+ browser_state_summary=browser_state,+ file_system=fs,+ read_state_images=read_state_images_png,+ )+ message_png = prompt_png.get_user_message(use_vision=True)+ image_parts_png = [part for part in message_png.content if isinstance(part, ContentPartImageParam)]+ assert 'data:image/png;base64' in image_parts_png[0].image_url.url++ # Test JPG+ read_state_images_jpg = [{'name': 'photo.jpg', 'data': 'data'}]+ prompt_jpg = AgentMessagePrompt(+ browser_state_summary=browser_state,+ file_system=fs,+ read_state_images=read_state_images_jpg,+ )+ message_jpg = prompt_jpg.get_user_message(use_vision=True)+ image_parts_jpg = [part for part in message_jpg.content if isinstance(part, ContentPartImageParam)]+ assert 'data:image/jpeg;base64' in image_parts_jpg[0].image_url.url++ def test_agent_message_prompt_no_images(self, tmp_path: Path):+ """Test that message works correctly when no images are present."""+ fs = FileSystem(tmp_path)++ browser_state = BrowserStateSummary(+ url='https://example.com',+ title='Test',+ tabs=[TabInfo(target_id='test-0', url='https://example.com', title='Test')],+ screenshot=None,+ dom_state=SerializedDOMState(_root=None, selector_map={}),+ )++ # No images+ prompt = AgentMessagePrompt(+ browser_state_summary=browser_state,+ file_system=fs,+ read_state_images=[],+ )++ # Get user message without vision+ user_message = prompt.get_user_message(use_vision=False)++ # Should be plain text, not content parts+ assert isinstance(user_message.content, str)++ def test_agent_message_prompt_empty_base64_skipped(self, tmp_path: Path):+ """Test that images with empty base64 data are skipped."""+ fs = FileSystem(tmp_path)++ browser_state = BrowserStateSummary(+ url='https://example.com',+ title='Test',+ tabs=[TabInfo(target_id='test-0', url='https://example.com', title='Test')],+ screenshot=None,+ dom_state=SerializedDOMState(_root=None, selector_map={}),+ )++ # Image with empty data field+ read_state_images = [+ {'name': 'empty.png', 'data': ''}, # Empty - should be skipped+ {'name': 'valid.png', 'data': 'valid_data'}, # Valid+ ]++ prompt = AgentMessagePrompt(+ browser_state_summary=browser_state,+ file_system=fs,+ read_state_images=read_state_images,+ )++ user_message = prompt.get_user_message(use_vision=True)+ image_parts = [part for part in user_message.content if isinstance(part, ContentPartImageParam)]++ # Should only have 1 image (the valid one)+ assert len(image_parts) == 1+ assert 'valid_data' in image_parts[0].image_url.url+++class TestDocxInLLMMessages:+ """Test that DOCX content flows correctly through to LLM messages."""++ @pytest.mark.asyncio+ async def test_docx_in_extracted_content(self, tmp_path: Path):+ """Test that DOCX text appears in extracted_content."""+ fs = FileSystem(tmp_path)++ # Create DOCX file+ content = """# Title+Some important content here."""+ await fs.write_file('test.docx', content)++ # Read it+ result = await fs.read_file('test.docx')++ # Verify content is in the result+ assert 'Title' in result+ assert 'important content' in result++ @pytest.mark.asyncio+ async def test_docx_in_message_manager(self, tmp_path: Path):+ """Test that DOCX content appears in message manager state."""+ fs = FileSystem(tmp_path)+ system_message = SystemMessage(content='Test system message')+ mm = MessageManager(task='test', system_message=system_message, file_system=fs)++ # Simulate read_file action result+ docx_content = """Read from file test.docx.+<content>+Title+Some content here.+</content>"""++ action_results = [+ ActionResult(+ extracted_content=docx_content,+ long_term_memory='Read file test.docx',+ include_extracted_content_only_once=True,+ )+ ]++ step_info = AgentStepInfo(step_number=1, max_steps=10)+ mm._update_agent_history_description(model_output=None, result=action_results, step_info=step_info)++ # Verify it's in read_state_description+ assert 'Title' in mm.state.read_state_description+ assert 'Some content' in mm.state.read_state_description+++class TestEndToEndIntegration:+ """End-to-end tests for file reading and LLM message creation."""++ def create_test_image(self) -> bytes:+ """Create a test image."""+ img = Image.new('RGB', (50, 50), color='blue')+ buffer = io.BytesIO()+ img.save(buffer, format='PNG')+ buffer.seek(0)+ return buffer.read()++ @pytest.mark.asyncio+ async def test_image_end_to_end(self, tmp_path: Path):+ """Test complete flow: external image → FileSystem → ActionResult → MessageManager → Prompt."""+ # Step 1: Create external image+ external_file = tmp_path / 'photo.png'+ img_bytes = self.create_test_image()+ external_file.write_bytes(img_bytes)++ # Step 2: Read via FileSystem+ fs = FileSystem(tmp_path / 'workspace')+ structured_result = await fs.read_file_structured(str(external_file), external_file=True)++ assert structured_result['images'] is not None++ # Step 3: Create ActionResult (simulating tools/service.py)+ action_result = ActionResult(+ extracted_content=structured_result['message'],+ long_term_memory='Read image file photo.png',+ images=structured_result['images'],+ include_extracted_content_only_once=True,+ )++ # Step 4: Process in MessageManager+ system_message = SystemMessage(content='Test system message')+ mm = MessageManager(task='test', system_message=system_message, file_system=fs)+ step_info = AgentStepInfo(step_number=1, max_steps=10)+ mm._update_agent_history_description(model_output=None, result=[action_result], step_info=step_info)++ # Verify images stored+ assert len(mm.state.read_state_images) == 1+ assert mm.state.read_state_images[0]['name'] == 'photo.png'++ # Step 5: Create message with AgentMessagePrompt+ browser_state = BrowserStateSummary(+ url='https://example.com',+ title='Test',+ tabs=[TabInfo(target_id='test-0', url='https://example.com', title='Test')],+ screenshot=None,+ dom_state=SerializedDOMState(_root=None, selector_map={}),+ )++ prompt = AgentMessagePrompt(+ browser_state_summary=browser_state,+ file_system=fs,+ read_state_images=mm.state.read_state_images,+ )++ user_message = prompt.get_user_message(use_vision=True)++ # Verify image is in message+ assert isinstance(user_message.content, list)+ image_parts = [part for part in user_message.content if isinstance(part, ContentPartImageParam)]+ assert len(image_parts) >= 1++ # Verify image data is correct+ base64_str = base64.b64encode(img_bytes).decode('utf-8')+ assert base64_str in image_parts[0].image_url.url++ @pytest.mark.asyncio+ async def test_docx_end_to_end(self, tmp_path: Path):+ """Test complete flow: DOCX file → FileSystem → ActionResult → MessageManager."""+ # Step 1: Create DOCX+ fs = FileSystem(tmp_path)+ docx_content = """# Important Document+This is critical information."""++ await fs.write_file('important.docx', docx_content)++ # Step 2: Read it+ read_result = await fs.read_file('important.docx')++ # Step 3: Create ActionResult (simulating tools/service.py)+ action_result = ActionResult(+ extracted_content=read_result,+ long_term_memory=read_result[:100] if len(read_result) > 100 else read_result,+ include_extracted_content_only_once=True,+ )++ # Step 4: Process in MessageManager+ system_message = SystemMessage(content='Test system message')+ mm = MessageManager(task='test', system_message=system_message, file_system=fs)+ step_info = AgentStepInfo(step_number=1, max_steps=10)+ mm._update_agent_history_description(model_output=None, result=[action_result], step_info=step_info)++ # Verify content is in read_state+ assert 'Important Document' in mm.state.read_state_description+ assert 'critical information' in mm.state.read_state_description+++if __name__ == '__main__':+ pytest.main([__file__, '-v'])