The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(search): bound native rg stderr and name non-permission diagnostics
Tekniumaeaaa468ddff
diff --git a/tests/tools/test_search_permission_degradation.py b/tests/tools/test_search_permission_degradation.pyindex b8cd9fe438f..ddd8d598d97 100644--- a/tests/tools/test_search_permission_degradation.py+++ b/tests/tools/test_search_permission_degradation.py@@ -249,8 +249,9 @@ def test_file_search_permission_degrade_keeps_timeout_limit_reason():# --- content search ---------------------------------------------------------------def test_content_search_with_unreadable_dir_returns_partial_matches():- output = EACCES_DIR + "/repo/a.py:3:needle\n"[email protected]("other", ["", "rg: /repo/gone: No such file or directory (os error 2)\n"])+def test_content_search_with_unreadable_dir_returns_partial_matches(other):+ output = EACCES_DIR + other + "/repo/a.py:3:needle\n"env = RecordingEnvironment(rg_output=output, rg_code=2)result = ShellFileOperations(env).search(@@ -261,6 +262,8 @@ def test_content_search_with_unreadable_dir_returns_partial_matches():("/repo/a.py", 3, "needle")]assert "unreadable path" in (result.warning or "")assert "/srv/locked" in (result.warning or "")+ if other:+ assert other.strip() in (result.warning or "")class CountEnvironment(RecordingEnvironment):@@ -361,9 +364,14 @@ def test_grep_content_search_non_permission_error_still_fails_closed():@pytest.mark.parametrize("order", ["discovery", "modified"])-def test_find_exit1_with_payload_degrades_to_partial(order):[email protected]("diagnostic,expected", [+ ("", ["unreadable", "partial"]),+ ("find: '/narrow/gone': No such file or directory\n",+ ["Some paths could not be read: find: '/narrow/gone': No such file or directory"]),+])+def test_find_exit1_with_payload_degrades_to_partial(order, diagnostic, expected):output = "/narrow/partial.py\n" if order == "discovery" else "10 /narrow/partial.py\n"- env = GrepOnlyEnvironment(rg_output=output, rg_code=1)+ env = GrepOnlyEnvironment(rg_output=diagnostic + output, rg_code=1)result = ShellFileOperations(env).search("*.py", path="/narrow", target="files", order=order)@@ -371,9 +379,10 @@ def test_find_exit1_with_payload_degrades_to_partial(order):assert result.error is Noneassert result.files == ["/narrow/partial.py"]assert result.total_count == 1- assert result.warning is not None- assert "unreadable" in (result.warning or "")- assert "partial" in (result.warning or "")+ for text in expected:+ assert text in (result.warning or "")+ if diagnostic:+ assert "permission" not in (result.warning or "").lower()@pytest.mark.parametrize("order", ["discovery", "modified"])diff --git a/tests/tools/test_search_unreadable_bound.py b/tests/tools/test_search_unreadable_bound.pyindex 3c8a53adfe5..f97980243d4 100644--- a/tests/tools/test_search_unreadable_bound.py+++ b/tests/tools/test_search_unreadable_bound.py@@ -11,6 +11,7 @@ import shutilimport pytest+from tools import file_operations_searchfrom tools.environments.local import LocalEnvironmentfrom tools.file_operations import ShellFileOperations@@ -41,17 +42,30 @@ def locked_tree(tmp_path):("content", {"output_mode": "files_only"}),("content", {}),])-def test_many_unreadable_dirs_do_not_eat_the_result_budget(locked_tree, target, kwargs):+def test_many_unreadable_dirs_do_not_eat_the_result_budget(locked_tree, target, kwargs, monkeypatch):if not shutil.which("rg"):pytest.skip("needs ripgrep")ops = ShellFileOperations(LocalEnvironment(cwd=str(locked_tree)), cwd=str(locked_tree))pattern = "*.log" if target == "files" else "needle"- result = ops.search(pattern, path=str(locked_tree), target=target, limit=3, **kwargs)- assert result.error is None- found = result.files or [m.path for m in result.matches]++ # Bounded run: rg is stopped at the limit, so how many skips it reported first+ # depends on traversal order; only the budget and the warning's presence are fixed.+ bounded = ops.search(pattern, path=str(locked_tree), target=target, limit=3, **kwargs)+ assert bounded.error is None+ found = bounded.files or [m.path for m in bounded.matches]assert len(found) == 3assert all(p.endswith(".log") for p in found)- assert "Skipped 60 unreadable path(s)" in (result.warning or "")+ assert "unreadable path(s) with restricted permissions" in (bounded.warning or "")++ # Full traversal under a one-line stderr cap: every skip is still counted, the+ # ones past the cap as "+N more".+ monkeypatch.setattr(file_operations_search, "_STDERR_RETAIN_BYTES", 1)+ full = ops.search(pattern, path=str(locked_tree), target=target, limit=50, **kwargs)+ assert full.error is None+ found = full.files or [m.path for m in full.matches]+ assert sorted(os.path.basename(p) for p in found) == [f"f{i}.log" for i in range(5)]+ assert "Skipped 60 unreadable path(s)" in (full.warning or "")+ assert "(+59 more)" in (full.warning or "")def test_find_fallback_names_the_unreadable_dir(locked_tree, monkeypatch):diff --git a/tools/file_operations_search.py b/tools/file_operations_search.pyindex 009a505f10e..6be12a7750f 100644--- a/tools/file_operations_search.py+++ b/tools/file_operations_search.py@@ -172,18 +172,35 @@ _PERMISSION_DENIED_RE = re.compile(_FIND_QUOTES_RE = re.compile(r"^['\u2018]|['\u2019]$")+# The native rg runner keeps at most this much stderr (a huge tree of unreadable dirs+# would otherwise be held in memory whole) and appends one tally line for the rest.+_STDERR_RETAIN_BYTES = 64 * 1024+_OMITTED_DIAGNOSTICS_RE = re.compile(r"^rg: \.\.\. (\d+) more diagnostic line\(s\) omitted$")+++def _omitted_diagnostics_line(count: int) -> str:+ return f"rg: ... {count} more diagnostic line(s) omitted"+++def _omitted_diagnostic_count(diagnostics: str) -> int:+ return sum(int(m.group(1)) for line in diagnostics.splitlines()+ if (m := _OMITTED_DIAGNOSTICS_RE.match(line.strip())))+def _is_permission_only_error(diagnostics: str) -> bool:"""True when every diagnostic line reports an unreadable path (rg or grep).That is a partial-success condition — the tool still emits results for the- readable remainder — not a broken invocation like a bad glob or regex."""- lines = [line.strip() for line in diagnostics.splitlines() if line.strip()]+ readable remainder — not a broken invocation like a bad glob or regex. The+ native runner's omitted-lines tally is judged by the lines it kept."""+ lines = [line.strip() for line in diagnostics.splitlines()+ if line.strip() and not _OMITTED_DIAGNOSTICS_RE.match(line.strip())]return bool(lines) and all(_PERMISSION_DENIED_RE.match(line) for line in lines)def _extract_unreadable_paths(diagnostics: str) -> list[str]:"""Paths reported unreadable by rg or grep, deduplicated in first-seen order."""paths: list[str] = []+ seen: set[str] = set()for line in diagnostics.splitlines():match = _PERMISSION_DENIED_RE.match(line.strip())if not match:@@ -191,20 +208,22 @@ def _extract_unreadable_paths(diagnostics: str) -> list[str]:path = match.group("path").strip()if match.group("tool") == "find":path = _FIND_QUOTES_RE.sub("", path)- if path and path not in paths:+ if path and path not in seen:+ seen.add(path)paths.append(path)return paths-def _compact_unreadable_paths(paths: list[str], keep: int = 3) -> str:+def _compact_unreadable_paths(paths: list[str], keep: int = 3, omitted: int = 0) -> str:shown = ", ".join(paths[:keep])- extra = len(paths) - keep+ extra = max(len(paths) - keep, 0) + omittedreturn shown + (f" (+{extra} more)" if extra > 0 else "")-def _skipped_unreadable_warning(paths: list[str]) -> str:- return ("Skipped " + str(len(paths)) + " unreadable path(s) with restricted "- "permissions: " + _compact_unreadable_paths(paths)+def _skipped_unreadable_warning(paths: list[str], omitted: int = 0) -> str:+ """``omitted`` counts diagnostics the native runner dropped past its retention cap."""+ return ("Skipped " + str(len(paths) + omitted) + " unreadable path(s) with restricted "+ "permissions: " + _compact_unreadable_paths(paths, omitted=omitted)+ ". Results may be incomplete; fix permissions or run as a user that ""can read them to include them.")@@ -214,7 +233,7 @@ def _prepend_skipped_paths_warning(diagnostics: str, existing: Optional[str]) ->paths = _extract_unreadable_paths(diagnostics)if not paths:return existing- warning = _skipped_unreadable_warning(paths)+ warning = _skipped_unreadable_warning(paths, _omitted_diagnostic_count(diagnostics))return warning if not existing else f"{warning} {existing}"@@ -268,10 +287,29 @@ def _find_exit_verdict(exit_code: int, raw_files: list[str], diagnostics: list[sreturn SearchResult(error=_FIND_MODIFIED_ORDER_ERROR), Nonedetail = f": {diagnostics[0]}" if diagnostics else "."return SearchResult(error=f"File search failed while running bounded find traversal{detail}"), None+ # A non-permission diagnostic (a dir removed mid-walk) must not be reported as a permission+ # problem: name it, plus any permission skips, so the user chases the real cause.+ other = next((d.strip() for d in diagnostics if d.strip() and not _PERMISSION_DENIED_RE.match(d.strip())), None)+ if other:+ return None, f"Some paths could not be read: {other}. Results may be incomplete." + (+ f" {warning}" if warning else "")return None, warning or ("find traversal skipped one or more unreadable paths (restricted ""permissions); results are partial.")+def _drain_stderr_capped(pipe, chunks: list[bytes], omitted: list[int]) -> None:+ """Read ``pipe`` to EOF (rg must never block on a full stderr pipe) but keep only the+ first ``_STDERR_RETAIN_BYTES``; the first line is always kept so a lone huge diagnostic+ still names the failure. ``omitted[0]`` counts the dropped lines."""+ kept = 0+ for raw in pipe or ():+ if chunks and kept + len(raw) > _STDERR_RETAIN_BYTES:+ omitted[0] += 1+ continue+ chunks.append(raw)+ kept += len(raw)++def _parse_search_context_line(line: str) -> tuple[str, int, str] | None:"""Parse a ``path-line-content`` context line using the RIGHTMOST numericseparator (filenames may contain ``-<digits>-`` segments):@@ -346,9 +384,15 @@ def _parse_search_output(result, output_mode: str, limit: int, offset: int,return SearchResult(total_count=0, warning=warning)return SearchResult(error=f"Search failed: {error_msg}", total_count=0)skipped_warning = _prepend_skipped_paths_warning(diagnostics, None)- tool_errors = [ln.strip() for ln in diagnostics.splitlines() if ln.lstrip().startswith(("rg: ", "grep: "))]- if not skipped_warning and result.exit_code == 2 and tool_errors:- skipped_warning = f"The search reported errors for some paths; results may be incomplete ({tool_errors[0]})."+ # A non-permission error beside the skips (a path removed mid-search) is named too, not+ # folded into the permission warning.+ other_errors = [ln.strip() for ln in diagnostics.splitlines()+ if ln.lstrip().startswith(("rg: ", "grep: "))+ and not _PERMISSION_DENIED_RE.match(ln.strip())+ and not _OMITTED_DIAGNOSTICS_RE.match(ln.strip())]+ if result.exit_code == 2 and other_errors:+ error_warning = f"The search reported errors for some paths; results may be incomplete ({other_errors[0]})."+ skipped_warning = f"{error_warning} {skipped_warning}" if skipped_warning else error_warningif skipped_warning:warning = skipped_warning if not warning else f"{skipped_warning} {warning}"lines = [ln for ln in payload.strip().split('\n') if ln]@@ -531,11 +575,12 @@ class SearchMixin:drainer = threading.Thread(target=_drain, daemon=True)drainer.start()err_chunks: list[bytes] = []+ err_omitted = [0]err_drainer = Noneif merge_stderr:err_pipe = proc.stderrerr_drainer = threading.Thread( # health: allow HX012 -- pipe drainer, reads no profile-scoped state- target=lambda: err_chunks.extend(err_pipe or ()), daemon=True)+ target=_drain_stderr_capped, args=(err_pipe, err_chunks, err_omitted), daemon=True)err_drainer.start()deadline = time.monotonic() + timeoutexit_code: Optional[int] = None@@ -559,7 +604,10 @@ class SearchMixin:err_drainer.join()if proc.stderr is not None:proc.stderr.close()- stdout = b"".join(err_chunks).decode("utf-8", errors="replace") + stdout+ err_text = b"".join(err_chunks).decode("utf-8", errors="replace")+ if err_omitted[0]:+ err_text = err_text.rstrip("\n") + "\n" + _omitted_diagnostics_line(err_omitted[0]) + "\n"+ stdout = err_text + stdoutif exit_code == 124:return ExecuteResult(stdout=stdout + f"\n[Command timed out after {timeout}s]", exit_code=124)if exit_code == 130:@@ -1125,7 +1173,7 @@ class SearchMixin:skipped = _extract_unreadable_paths(diagnostics) \if _is_permission_only_error(diagnostics) else []permission_only = result.exit_code == 2 and bool(skipped)- warning = _skipped_unreadable_warning(skipped) if skipped else None+ warning = _skipped_unreadable_warning(skipped, _omitted_diagnostic_count(diagnostics)) if skipped else Noneif scoped_common:all_files = [f if posixpath.isabs(f) else posixpath.normpath(posixpath.join(scoped_common, f))