Your lightweight open-source personal agent.
Python agent-frameworkai-agentai-agentschatbotchatopsdiscord-botllm-agentsllmslocal-llmmcpmodel-context-protocolmulti-agentopenai-compatibleopenclawpersonal-ai-assistantpythonself-hostedtelegram-bot-ai-assistantwebuiworkflow-automation
fix(tools): bound find_files scans
chengyongru649e3958c58b
diff --git a/nanobot/agent/tools/search.py b/nanobot/agent/tools/search.pyindex 2599938c2..a74d85760 100644--- a/nanobot/agent/tools/search.py+++ b/nanobot/agent/tools/search.py@@ -4,9 +4,13 @@from __future__ import annotations+import asyncioimport fnmatch+import heapqimport osimport re+import threading+import timefrom collections import dequefrom contextlib import suppressfrom dataclasses import dataclass@@ -57,6 +61,43 @@ class _PendingContextMatch:remaining_after: int+@dataclass(slots=True)+class _FindFilesEntry:+ path: Path+ rel_path: str+ display_path: str+ name: str+ is_dir: bool+++class _FindFilesCancelledError(Exception):+ """Stop a worker scan after its owning async task was cancelled."""+++class _FindFilesBudgetExceededError(Exception):+ """Stop an unbounded filesystem scan at its configured budget."""+++@dataclass(slots=True)+class _FindFilesBudget:+ cancelled: threading.Event+ deadline: float+ max_paths: int+ scanned_paths: int = 0++ def checkpoint(self) -> None:+ if self.cancelled.is_set():+ raise _FindFilesCancelledError+ if time.monotonic() >= self.deadline:+ raise _FindFilesBudgetExceededError("time")++ def visit_path(self) -> None:+ self.checkpoint()+ self.scanned_paths += 1+ if self.scanned_paths > self.max_paths:+ raise _FindFilesBudgetExceededError("paths")++def _normalize_pattern(pattern: str) -> str:return pattern.strip().replace("\\", "/")@@ -150,6 +191,8 @@ class _SearchTool(_FsTool):class FindFilesTool(_SearchTool):"""Find files by path fragment, glob, or type."""_scopes = {"core", "subagent"}+ _MAX_SCAN_PATHS = 500_000+ _MAX_SCAN_SECONDS = 30.0@propertydef name(self) -> str:@@ -211,19 +254,101 @@ class FindFilesTool(_SearchTool):},}- def _iter_paths(self, root: Path, *, include_dirs: bool) -> Iterable[Path]:+ def _entry(self, path: Path, root: Path, *, is_dir: bool) -> _FindFilesEntry:+ display_path = self._display_path(path, root)+ return _FindFilesEntry(+ path=path,+ rel_path=path.relative_to(root).as_posix(),+ display_path=display_path,+ name=path.name,+ is_dir=is_dir,+ )++ def _push_directory_entries(+ self,+ directory: Path,+ root: Path,+ frontier: list[tuple[str, int, _FindFilesEntry]],+ sequence: int,+ budget: _FindFilesBudget,+ ) -> int:+ budget.checkpoint()+ try:+ with os.scandir(directory) as entries:+ for raw_entry in entries:+ budget.visit_path()+ try:+ is_dir = raw_entry.is_dir(follow_symlinks=False)+ # os.walk yields special files and broken file symlinks,+ # but does not descend into directory symlinks by default.+ if not is_dir and raw_entry.is_symlink() and raw_entry.is_dir():+ continue+ except OSError:+ continue+ if is_dir and raw_entry.name in self._IGNORE_DIRS:+ continue++ entry = self._entry(Path(raw_entry.path), root, is_dir=is_dir)+ sort_path = entry.display_path + ("/" if is_dir else "")+ heapq.heappush(frontier, (sort_path, sequence, entry))+ sequence += 1+ except OSError:+ # os.walk silently skips directories that cannot be listed. Preserve+ # that behavior while still allowing cancellation and budget errors+ # to propagate from the explicit checkpoints above.+ pass+ return sequence++ def _iter_paths(+ self,+ root: Path,+ *,+ include_dirs: bool,+ budget: _FindFilesBudget,+ ) -> Iterable[_FindFilesEntry]:+ budget.checkpoint()if root.is_file():- yield root+ budget.visit_path()+ yield self._entry(root, root.parent, is_dir=False)return+if include_dirs:- yield root- for dirpath, dirnames, filenames in os.walk(root):- dirnames[:] = sorted(d for d in dirnames if d not in self._IGNORE_DIRS)- current = Path(dirpath)- if include_dirs and current != root:- yield current- for filename in sorted(filenames):- yield current / filename+ yield self._entry(root, root, is_dir=True)++ frontier: list[tuple[str, int, _FindFilesEntry]] = []+ sequence = self._push_directory_entries(root, root, frontier, 0, budget)+ while frontier:+ budget.checkpoint()+ _, _, entry = heapq.heappop(frontier)+ if entry.is_dir:+ if include_dirs:+ yield entry+ sequence = self._push_directory_entries(+ entry.path,+ root,+ frontier,+ sequence,+ budget,+ )+ else:+ yield entry++ @staticmethod+ def _matches_entry(+ entry: _FindFilesEntry,+ *,+ query: str | None,+ glob: str | None,+ file_type: str | None,+ ) -> bool:+ if glob and not _match_glob(entry.rel_path, entry.name, glob):+ return False+ if entry.is_dir:+ if file_type:+ return False+ elif not _matches_type(entry.name, file_type):+ return False+ return _matches_query(entry.display_path, query)async def execute(self,@@ -237,65 +362,126 @@ class FindFilesTool(_SearchTool):offset: int = 0,**kwargs: Any,) -> str:+ cancelled = threading.Event()try:- target = self._resolve(path or ".")- if not target.exists():- return ToolResult.error(f"Error: Path not found: {path}")- if not (target.is_dir() or target.is_file()):- return ToolResult.error(f"Error: Unsupported path: {path}")-- if sort not in {"path", "modified"}:- return ToolResult.error("Error: sort must be 'path' or 'modified'")-- limit = (- _DEFAULT_FILE_HEAD_LIMIT- if head_limit is None- else None if head_limit == 0 else head_limit+ return await asyncio.to_thread(+ self._execute_sync,+ path=path,+ query=query,+ glob=glob,+ file_type=type,+ include_dirs=include_dirs,+ sort=sort,+ head_limit=head_limit,+ offset=offset,+ cancelled=cancelled,)- root = target if target.is_dir() else target.parent- matches: list[tuple[str, float]] = []+ except asyncio.CancelledError:+ cancelled.set()+ raise+ except PermissionError as e:+ return ToolResult.error(f"Error: {e}")+ except Exception as e:+ return ToolResult.error(f"Error finding files: {e}")- for candidate in self._iter_paths(target, include_dirs=include_dirs):- if candidate.is_dir() and not include_dirs:- continue- rel_path = candidate.relative_to(root).as_posix()- display_path = self._display_path(candidate, root)- name = candidate.name+ def _execute_sync(+ self,+ *,+ path: str,+ query: str | None,+ glob: str | None,+ file_type: str | None,+ include_dirs: bool,+ sort: str,+ head_limit: int | None,+ offset: int,+ cancelled: threading.Event,+ ) -> str:+ started_at = time.monotonic()+ if cancelled.is_set():+ raise _FindFilesCancelledError+ target = self._resolve(path or ".")+ if not target.exists():+ return ToolResult.error(f"Error: Path not found: {path}")+ if not (target.is_dir() or target.is_file()):+ return ToolResult.error(f"Error: Unsupported path: {path}")++ if sort not in {"path", "modified"}:+ return ToolResult.error("Error: sort must be 'path' or 'modified'")++ limit = (+ _DEFAULT_FILE_HEAD_LIMIT+ if head_limit is None+ else None if head_limit == 0 else head_limit+ )+ budget = _FindFilesBudget(+ cancelled=cancelled,+ deadline=started_at + self._MAX_SCAN_SECONDS,+ max_paths=self._MAX_SCAN_PATHS,+ )- if glob and not _match_glob(rel_path, name, glob):- continue- if candidate.is_file() and not _matches_type(name, type):+ def matching_entries() -> Iterator[tuple[str, float]]:+ for entry in self._iter_paths(+ target,+ include_dirs=include_dirs,+ budget=budget,+ ):+ if not self._matches_entry(+ entry,+ query=query,+ glob=glob,+ file_type=file_type,+ ):continue- if candidate.is_dir() and type:- continue- if not _matches_query(display_path, query):- continue- try:- mtime = candidate.stat().st_mtime- except OSError:- mtime = 0.0- suffix = "/" if candidate.is_dir() else ""- matches.append((display_path + suffix, mtime))-+ mtime = 0.0+ if sort == "modified":+ try:+ mtime = entry.path.stat().st_mtime+ except OSError:+ pass+ suffix = "/" if entry.is_dir else ""+ yield entry.display_path + suffix, mtime++ matches: list[tuple[str, float]]+ try:if sort == "modified":- matches.sort(key=lambda item: (-item[1], item[0]))+ if limit is None:+ matches = sorted(matching_entries(), key=lambda item: (-item[1], item[0]))+ else:+ selection_size = offset + limit + 1+ matches = heapq.nsmallest(+ selection_size,+ matching_entries(),+ key=lambda item: (-item[1], item[0]),+ )+ else:+ selection_size = None if limit is None else offset + limit + 1+ matches = []+ for match in matching_entries():+ matches.append(match)+ if selection_size is not None and len(matches) >= selection_size:+ break+ budget.checkpoint()+ except _FindFilesBudgetExceededError as exc:+ if str(exc) == "paths":+ detail = f"{self._MAX_SCAN_PATHS} paths"else:- matches.sort(key=lambda item: item[0])+ detail = f"{self._MAX_SCAN_SECONDS:g} seconds"+ return ToolResult.error(+ f"Error: find_files scan exceeded {detail}; "+ "narrow path, query, glob, or type and retry."+ )- paths = [item[0] for item in matches]- paged, truncated = _paginate(paths, limit, offset)- if not paged:- return "No files found"+ paths = [item[0] for item in matches]+ paged, truncated = _paginate(paths, limit, offset)+ if not paged:+ return "No files found"- result = "\n".join(paged)- note = _pagination_note(limit, offset, truncated)- if note:- result += "\n\n" + note- return result- except PermissionError as e:- return ToolResult.error(f"Error: {e}")- except Exception as e:- return ToolResult.error(f"Error finding files: {e}")+ result = "\n".join(paged)+ note = _pagination_note(limit, offset, truncated)+ if note:+ result += "\n\n" + note+ return resultclass GrepTool(_SearchTool):diff --git a/tests/tools/test_search_tools.py b/tests/tools/test_search_tools.pyindex 9fe06b44b..61dd8302b 100644--- a/tests/tools/test_search_tools.py+++ b/tests/tools/test_search_tools.py@@ -2,8 +2,10 @@from __future__ import annotations+import asyncioimport osimport re+import threadingimport timefrom pathlib import Pathfrom types import SimpleNamespace@@ -18,6 +20,11 @@ from nanobot.agent.tools.web import WebSearchToolfrom nanobot.bus.queue import MessageBusfrom nanobot.config.schema import WebSearchConfigfrom nanobot.providers.base import GenerationSettings+from nanobot.security.workspace_access import (+ bind_workspace_scope,+ default_workspace_scope,+ reset_workspace_scope,+)from nanobot.utils.llm_runtime import LLMRuntime@@ -98,6 +105,152 @@ async def test_find_files_rejects_paths_outside_workspace(tmp_path: Path) -> Nonassert result.startswith("Error:")+async def test_find_files_worker_preserves_current_workspace_scope(tmp_path: Path) -> None:+ project = tmp_path / "project"+ project.mkdir()+ (project / "inside.txt").write_text("ok\n", encoding="utf-8")+ (tmp_path / "outside.txt").write_text("nope\n", encoding="utf-8")+ tool = FindFilesTool(workspace=tmp_path, restrict_to_workspace=False)+ token = bind_workspace_scope(default_workspace_scope(project, True))+ try:+ result = await tool.execute(path=".")+ finally:+ reset_workspace_scope(token)++ assert result == "inside.txt"+++async def test_find_files_scan_keeps_event_loop_responsive(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ target = tmp_path / "match.txt"+ target.write_text("ok\n", encoding="utf-8")+ tool = FindFilesTool(workspace=tmp_path, allowed_dir=tmp_path)+ original_iter_paths = tool._iter_paths+ started = threading.Event()+ release = threading.Event()++ def blocking_iter_paths(root: Path, *, include_dirs: bool, budget):+ started.set()+ if not release.wait(timeout=1):+ raise TimeoutError("test did not release find_files traversal")+ yield from original_iter_paths(root, include_dirs=include_dirs, budget=budget)++ monkeypatch.setattr(tool, "_iter_paths", blocking_iter_paths)+ task = asyncio.create_task(tool.execute(path="."))+ try:+ assert await asyncio.to_thread(started.wait, 0.5)+ for _ in range(3):+ await asyncio.sleep(0.01)+ assert not task.done()+ finally:+ release.set()++ assert await asyncio.wait_for(task, timeout=0.5) == "match.txt"+++async def test_find_files_cancellation_stops_worker_scan(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ tool = FindFilesTool(workspace=tmp_path, allowed_dir=tmp_path)+ started = threading.Event()+ stopped = threading.Event()++ def cancellable_iter_paths(root: Path, *, include_dirs: bool, budget):+ del root, include_dirs+ started.set()+ if not budget.cancelled.wait(timeout=1):+ raise TimeoutError("find_files worker did not receive cancellation")+ stopped.set()+ if False:+ yield++ monkeypatch.setattr(tool, "_iter_paths", cancellable_iter_paths)+ task = asyncio.create_task(tool.execute(path="."))+ assert await asyncio.to_thread(started.wait, 0.5)++ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await asyncio.wait_for(task, timeout=0.2)++ assert await asyncio.to_thread(stopped.wait, 0.5)+++async def test_find_files_path_limit_stops_after_pagination_lookahead(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ paths = [tmp_path / name for name in ("a.txt", "b.txt", "c.txt")]+ for path in paths:+ path.write_text("ok\n", encoding="utf-8")+ tool = FindFilesTool(workspace=tmp_path, allowed_dir=tmp_path)+ visited: list[str] = []++ def ordered_iter_paths(root: Path, *, include_dirs: bool, budget):+ del include_dirs+ for path in paths:+ budget.checkpoint()+ visited.append(path.name)+ yield tool._entry(path, root, is_dir=False)++ monkeypatch.setattr(tool, "_iter_paths", ordered_iter_paths)+ result = await tool.execute(path=".", head_limit=1)++ assert result.splitlines()[0] == "a.txt"+ assert "pagination: limit=1, offset=0" in result+ assert visited == ["a.txt", "b.txt"]+++async def test_find_files_path_budget_counts_directories(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ (tmp_path / "one").mkdir()+ (tmp_path / "two").mkdir()+ monkeypatch.setattr(FindFilesTool, "_MAX_SCAN_PATHS", 1)+ tool = FindFilesTool(workspace=tmp_path, allowed_dir=tmp_path)++ result = await tool.execute(path=".")++ assert result.startswith("Error: find_files scan exceeded 1 paths")+++async def test_find_files_enforces_time_budget(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ (tmp_path / "match.txt").write_text("ok\n", encoding="utf-8")+ monkeypatch.setattr(FindFilesTool, "_MAX_SCAN_SECONDS", 0.0)+ tool = FindFilesTool(workspace=tmp_path, allowed_dir=tmp_path)++ result = await tool.execute(path=".")++ assert result.startswith("Error: find_files scan exceeded 0 seconds")+++async def test_find_files_path_sort_matches_existing_lexicographic_contract(+ tmp_path: Path,+) -> None:+ (tmp_path / "a").mkdir()+ (tmp_path / "a" / "inside.txt").write_text("ok\n", encoding="utf-8")+ (tmp_path / "a+").write_text("ok\n", encoding="utf-8")+ (tmp_path / "a.py").write_text("ok\n", encoding="utf-8")+ tool = FindFilesTool(workspace=tmp_path, allowed_dir=tmp_path)++ result = await tool.execute(path=".", head_limit=0)++ assert result.splitlines() == ["a+", "a.py", "a/inside.txt"]++@pytest.mark.asyncioasync def test_grep_respects_glob_filter_and_context(tmp_path: Path) -> None:(tmp_path / "src").mkdir()