Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Add error on out of range for open_files (#3214)
Matthew Zhou4c8e9af4bd02
diff --git a/letta/services/file_processor/chunker/line_chunker.py b/letta/services/file_processor/chunker/line_chunker.pyindex 3b49a43f9..c06f024b3 100644--- a/letta/services/file_processor/chunker/line_chunker.py+++ b/letta/services/file_processor/chunker/line_chunker.py@@ -99,7 +99,12 @@ class LineChunker:return [line for line in lines if line.strip()]def chunk_text(- self, file_metadata: FileMetadata, start: Optional[int] = None, end: Optional[int] = None, add_metadata: bool = True+ self,+ file_metadata: FileMetadata,+ start: Optional[int] = None,+ end: Optional[int] = None,+ add_metadata: bool = True,+ validate_range: bool = False,) -> List[str]:"""Content-aware text chunking based on file type"""strategy = self._determine_chunking_strategy(file_metadata)@@ -116,11 +121,31 @@ class LineChunker:content_lines = self._chunk_by_lines(text, preserve_indentation=False)total_chunks = len(content_lines)+ chunk_type = (+ "sentences" if strategy == ChunkingStrategy.DOCUMENTATION else "chunks" if strategy == ChunkingStrategy.PROSE else "lines"+ )++ # Validate range if requested+ if validate_range and (start is not None or end is not None):+ if start is not None and start >= total_chunks:+ # Convert to 1-indexed for user-friendly error message+ start_display = start + 1+ raise ValueError(+ f"File {file_metadata.file_name} has only {total_chunks} lines, but requested offset {start_display} is out of range"+ )++ if start is not None and end is not None and end > total_chunks:+ # Convert to 1-indexed for user-friendly error message+ start_display = start + 1+ end_display = end+ raise ValueError(+ f"File {file_metadata.file_name} has only {total_chunks} lines, but requested range {start_display} to {end_display} extends beyond file bounds"+ )# Handle start/end slicing- if start is not None and end is not None:+ if start is not None or end is not None:content_lines = content_lines[start:end]- line_offset = start+ line_offset = start if start is not None else 0else:line_offset = 0@@ -129,14 +154,15 @@ class LineChunker:# Add metadata about total chunksif add_metadata:- chunk_type = (- "sentences" if strategy == ChunkingStrategy.DOCUMENTATION else "chunks" if strategy == ChunkingStrategy.PROSE else "lines"- )if start is not None and end is not None:# Display 1-indexed ranges for usersstart_display = start + 1end_display = endcontent_lines.insert(0, f"[Viewing {chunk_type} {start_display} to {end_display} (out of {total_chunks} {chunk_type})]")+ elif start is not None:+ # Only start specified - viewing from start to end+ start_display = start + 1+ content_lines.insert(0, f"[Viewing {chunk_type} {start_display} to end (out of {total_chunks} {chunk_type})]")else:content_lines.insert(0, f"[Viewing file start (out of {total_chunks} {chunk_type})]")diff --git a/letta/services/tool_executor/files_tool_executor.py b/letta/services/tool_executor/files_tool_executor.pyindex 1de53724e..4815243a4 100644--- a/letta/services/tool_executor/files_tool_executor.py+++ b/letta/services/tool_executor/files_tool_executor.py@@ -175,7 +175,7 @@ class LettaFileToolExecutor(ToolExecutor):file = await self.file_manager.get_file_by_id(file_id=file_id, actor=self.actor, include_content=True)# Process file content- content_lines = LineChunker().chunk_text(file_metadata=file, start=start, end=end)+ content_lines = LineChunker().chunk_text(file_metadata=file, start=start, end=end, validate_range=True)visible_content = "\n".join(content_lines)# Handle LRU eviction and file openingdiff --git a/tests/test_utils.py b/tests/test_utils.pyindex 1af23e62c..5b0e724cb 100644--- a/tests/test_utils.py+++ b/tests/test_utils.py@@ -2,6 +2,8 @@ import pytestfrom letta.constants import MAX_FILENAME_LENGTHfrom letta.functions.ast_parsers import coerce_dict_args_by_annotations, get_function_annotations_from_source+from letta.schemas.file import FileMetadata+from letta.services.file_processor.chunker.line_chunker import LineChunkerfrom letta.services.helpers.agent_manager_helper import safe_formatfrom letta.utils import sanitize_filename@@ -407,3 +409,103 @@ def test_formatter():"""assert UNUSED_AND_EMPRY_VAR_SOL == safe_format(UNUSED_AND_EMPRY_VAR, VARS_DICT)+++# ---------------------- LineChunker TESTS ---------------------- #+++def test_line_chunker_valid_range():+ """Test that LineChunker works correctly with valid ranges"""+ file = FileMetadata(file_name="test.py", source_id="test_source", content="line1\nline2\nline3\nline4")+ chunker = LineChunker()++ # Test valid range with validation+ result = chunker.chunk_text(file, start=1, end=3, validate_range=True)+ # Should return lines 2 and 3 (0-indexed 1:3)+ assert "[Viewing lines 2 to 3 (out of 4 lines)]" in result[0]+ assert "2: line2" in result[1]+ assert "3: line3" in result[2]+++def test_line_chunker_valid_range_no_validation():+ """Test that LineChunker works the same without validation for valid ranges"""+ file = FileMetadata(file_name="test.py", source_id="test_source", content="line1\nline2\nline3\nline4")+ chunker = LineChunker()++ # Test same range without validation+ result = chunker.chunk_text(file, start=1, end=3, validate_range=False)+ assert "[Viewing lines 2 to 3 (out of 4 lines)]" in result[0]+ assert "2: line2" in result[1]+ assert "3: line3" in result[2]+++def test_line_chunker_out_of_range_start():+ """Test that LineChunker throws error when start is out of range"""+ file = FileMetadata(file_name="test.py", source_id="test_source", content="line1\nline2\nline3")+ chunker = LineChunker()++ # Test with start beyond file length (3 lines, requesting start=5 which is 0-indexed 4)+ with pytest.raises(ValueError, match="File test.py has only 3 lines, but requested offset 6 is out of range"):+ chunker.chunk_text(file, start=5, end=6, validate_range=True)+++def test_line_chunker_out_of_range_end():+ """Test that LineChunker throws error when end extends beyond file bounds"""+ file = FileMetadata(file_name="test.py", source_id="test_source", content="line1\nline2\nline3")+ chunker = LineChunker()++ # Test with end beyond file length (3 lines, requesting 1 to 10)+ with pytest.raises(ValueError, match="File test.py has only 3 lines, but requested range 1 to 10 extends beyond file bounds"):+ chunker.chunk_text(file, start=0, end=10, validate_range=True)+++def test_line_chunker_edge_case_empty_file():+ """Test that LineChunker handles empty files correctly"""+ file = FileMetadata(file_name="empty.py", source_id="test_source", content="")+ chunker = LineChunker()++ # Test requesting lines from empty file+ with pytest.raises(ValueError, match="File empty.py has only 0 lines, but requested offset 1 is out of range"):+ chunker.chunk_text(file, start=0, end=1, validate_range=True)+++def test_line_chunker_edge_case_single_line():+ """Test that LineChunker handles single line files correctly"""+ file = FileMetadata(file_name="single.py", source_id="test_source", content="only line")+ chunker = LineChunker()++ # Test valid single line access+ result = chunker.chunk_text(file, start=0, end=1, validate_range=True)+ assert "1: only line" in result[1]++ # Test out of range for single line file+ with pytest.raises(ValueError, match="File single.py has only 1 lines, but requested offset 2 is out of range"):+ chunker.chunk_text(file, start=1, end=2, validate_range=True)+++def test_line_chunker_validation_disabled_allows_out_of_range():+ """Test that when validation is disabled, out of range silently returns partial results"""+ file = FileMetadata(file_name="test.py", source_id="test_source", content="line1\nline2\nline3")+ chunker = LineChunker()++ # Test with validation disabled - should not raise error+ result = chunker.chunk_text(file, start=5, end=10, validate_range=False)+ # Should return empty content (except metadata header) since slice is out of bounds+ assert len(result) == 1 # Only metadata header+ assert "[Viewing lines 6 to 10 (out of 3 lines)]" in result[0]+++def test_line_chunker_only_start_parameter():+ """Test validation with only start parameter specified"""+ file = FileMetadata(file_name="test.py", source_id="test_source", content="line1\nline2\nline3")+ chunker = LineChunker()++ # Test valid start only+ result = chunker.chunk_text(file, start=1, validate_range=True)+ assert "[Viewing lines 2 to end (out of 3 lines)]" in result[0]+ assert "2: line2" in result[1]+ assert "3: line3" in result[2]++ # Test invalid start only+ with pytest.raises(ValueError, match="File test.py has only 3 lines, but requested offset 4 is out of range"):+ chunker.chunk_text(file, start=3, validate_range=True)