Skip to content

The agent that grows with you

fix(search): bound native rg stderr and name non-permission diagnostics

Tekniumaeaaa468ddff
diff --git a/tests/tools/test_search_permission_degradation.py b/tests/tools/test_search_permission_degradation.py
index b8cd9fe438f..ddd8d598d97 100644
--- a/tests/tools/test_search_permission_degradation.py
+++ b/tests/tools/test_search_permission_degradation.py
@@ -249,8 +249,9 @@ def test_file_search_permission_degrade_keeps_timeout_limit_reason():
# --- content search --------------------------------------------------------------
-def test_content_search_with_unreadable_dir_returns_partial_matches():
- output = EACCES_DIR + "/repo/a.py:3:needle\n"
[email protected]("other", ["", "rg: /repo/gone: No such file or directory (os error 2)\n"])
+def test_content_search_with_unreadable_dir_returns_partial_matches(other):
+ output = EACCES_DIR + other + "/repo/a.py:3:needle\n"
env = RecordingEnvironment(rg_output=output, rg_code=2)
result = ShellFileOperations(env).search(
@@ -261,6 +262,8 @@ def test_content_search_with_unreadable_dir_returns_partial_matches():
("/repo/a.py", 3, "needle")]
assert "unreadable path" in (result.warning or "")
assert "/srv/locked" in (result.warning or "")
+ if other:
+ assert other.strip() in (result.warning or "")
class CountEnvironment(RecordingEnvironment):
@@ -361,9 +364,14 @@ def test_grep_content_search_non_permission_error_still_fails_closed():
@pytest.mark.parametrize("order", ["discovery", "modified"])
-def test_find_exit1_with_payload_degrades_to_partial(order):
[email protected]("diagnostic,expected", [
+ ("", ["unreadable", "partial"]),
+ ("find: '/narrow/gone': No such file or directory\n",
+ ["Some paths could not be read: find: '/narrow/gone': No such file or directory"]),
+])
+def test_find_exit1_with_payload_degrades_to_partial(order, diagnostic, expected):
output = "/narrow/partial.py\n" if order == "discovery" else "10 /narrow/partial.py\n"
- env = GrepOnlyEnvironment(rg_output=output, rg_code=1)
+ env = GrepOnlyEnvironment(rg_output=diagnostic + output, rg_code=1)
result = ShellFileOperations(env).search(
"*.py", path="/narrow", target="files", order=order)
@@ -371,9 +379,10 @@ def test_find_exit1_with_payload_degrades_to_partial(order):
assert result.error is None
assert result.files == ["/narrow/partial.py"]
assert result.total_count == 1
- assert result.warning is not None
- assert "unreadable" in (result.warning or "")
- assert "partial" in (result.warning or "")
+ for text in expected:
+ assert text in (result.warning or "")
+ if diagnostic:
+ assert "permission" not in (result.warning or "").lower()
@pytest.mark.parametrize("order", ["discovery", "modified"])
diff --git a/tests/tools/test_search_unreadable_bound.py b/tests/tools/test_search_unreadable_bound.py
index 3c8a53adfe5..f97980243d4 100644
--- a/tests/tools/test_search_unreadable_bound.py
+++ b/tests/tools/test_search_unreadable_bound.py
@@ -11,6 +11,7 @@ import shutil
import pytest
+from tools import file_operations_search
from tools.environments.local import LocalEnvironment
from tools.file_operations import ShellFileOperations
@@ -41,17 +42,30 @@ def locked_tree(tmp_path):
("content", {"output_mode": "files_only"}),
("content", {}),
])
-def test_many_unreadable_dirs_do_not_eat_the_result_budget(locked_tree, target, kwargs):
+def test_many_unreadable_dirs_do_not_eat_the_result_budget(locked_tree, target, kwargs, monkeypatch):
if not shutil.which("rg"):
pytest.skip("needs ripgrep")
ops = ShellFileOperations(LocalEnvironment(cwd=str(locked_tree)), cwd=str(locked_tree))
pattern = "*.log" if target == "files" else "needle"
- result = ops.search(pattern, path=str(locked_tree), target=target, limit=3, **kwargs)
- assert result.error is None
- found = result.files or [m.path for m in result.matches]
+
+ # Bounded run: rg is stopped at the limit, so how many skips it reported first
+ # depends on traversal order; only the budget and the warning's presence are fixed.
+ bounded = ops.search(pattern, path=str(locked_tree), target=target, limit=3, **kwargs)
+ assert bounded.error is None
+ found = bounded.files or [m.path for m in bounded.matches]
assert len(found) == 3
assert all(p.endswith(".log") for p in found)
- assert "Skipped 60 unreadable path(s)" in (result.warning or "")
+ assert "unreadable path(s) with restricted permissions" in (bounded.warning or "")
+
+ # Full traversal under a one-line stderr cap: every skip is still counted, the
+ # ones past the cap as "+N more".
+ monkeypatch.setattr(file_operations_search, "_STDERR_RETAIN_BYTES", 1)
+ full = ops.search(pattern, path=str(locked_tree), target=target, limit=50, **kwargs)
+ assert full.error is None
+ found = full.files or [m.path for m in full.matches]
+ assert sorted(os.path.basename(p) for p in found) == [f"f{i}.log" for i in range(5)]
+ assert "Skipped 60 unreadable path(s)" in (full.warning or "")
+ assert "(+59 more)" in (full.warning or "")
def test_find_fallback_names_the_unreadable_dir(locked_tree, monkeypatch):
diff --git a/tools/file_operations_search.py b/tools/file_operations_search.py
index 009a505f10e..6be12a7750f 100644
--- a/tools/file_operations_search.py
+++ b/tools/file_operations_search.py
@@ -172,18 +172,35 @@ _PERMISSION_DENIED_RE = re.compile(
_FIND_QUOTES_RE = re.compile(r"^['\u2018]|['\u2019]$")
+# The native rg runner keeps at most this much stderr (a huge tree of unreadable dirs
+# would otherwise be held in memory whole) and appends one tally line for the rest.
+_STDERR_RETAIN_BYTES = 64 * 1024
+_OMITTED_DIAGNOSTICS_RE = re.compile(r"^rg: \.\.\. (\d+) more diagnostic line\(s\) omitted$")
+
+
+def _omitted_diagnostics_line(count: int) -> str:
+ return f"rg: ... {count} more diagnostic line(s) omitted"
+
+
+def _omitted_diagnostic_count(diagnostics: str) -> int:
+ return sum(int(m.group(1)) for line in diagnostics.splitlines()
+ if (m := _OMITTED_DIAGNOSTICS_RE.match(line.strip())))
+
def _is_permission_only_error(diagnostics: str) -> bool:
"""True when every diagnostic line reports an unreadable path (rg or grep).
That is a partial-success condition — the tool still emits results for the
- readable remainder — not a broken invocation like a bad glob or regex."""
- lines = [line.strip() for line in diagnostics.splitlines() if line.strip()]
+ readable remainder — not a broken invocation like a bad glob or regex. The
+ native runner's omitted-lines tally is judged by the lines it kept."""
+ lines = [line.strip() for line in diagnostics.splitlines()
+ if line.strip() and not _OMITTED_DIAGNOSTICS_RE.match(line.strip())]
return bool(lines) and all(_PERMISSION_DENIED_RE.match(line) for line in lines)
def _extract_unreadable_paths(diagnostics: str) -> list[str]:
"""Paths reported unreadable by rg or grep, deduplicated in first-seen order."""
paths: list[str] = []
+ seen: set[str] = set()
for line in diagnostics.splitlines():
match = _PERMISSION_DENIED_RE.match(line.strip())
if not match:
@@ -191,20 +208,22 @@ def _extract_unreadable_paths(diagnostics: str) -> list[str]:
path = match.group("path").strip()
if match.group("tool") == "find":
path = _FIND_QUOTES_RE.sub("", path)
- if path and path not in paths:
+ if path and path not in seen:
+ seen.add(path)
paths.append(path)
return paths
-def _compact_unreadable_paths(paths: list[str], keep: int = 3) -> str:
+def _compact_unreadable_paths(paths: list[str], keep: int = 3, omitted: int = 0) -> str:
shown = ", ".join(paths[:keep])
- extra = len(paths) - keep
+ extra = max(len(paths) - keep, 0) + omitted
return shown + (f" (+{extra} more)" if extra > 0 else "")
-def _skipped_unreadable_warning(paths: list[str]) -> str:
- return ("Skipped " + str(len(paths)) + " unreadable path(s) with restricted "
- "permissions: " + _compact_unreadable_paths(paths)
+def _skipped_unreadable_warning(paths: list[str], omitted: int = 0) -> str:
+ """``omitted`` counts diagnostics the native runner dropped past its retention cap."""
+ return ("Skipped " + str(len(paths) + omitted) + " unreadable path(s) with restricted "
+ "permissions: " + _compact_unreadable_paths(paths, omitted=omitted)
+ ". Results may be incomplete; fix permissions or run as a user that "
"can read them to include them.")
@@ -214,7 +233,7 @@ def _prepend_skipped_paths_warning(diagnostics: str, existing: Optional[str]) ->
paths = _extract_unreadable_paths(diagnostics)
if not paths:
return existing
- warning = _skipped_unreadable_warning(paths)
+ warning = _skipped_unreadable_warning(paths, _omitted_diagnostic_count(diagnostics))
return warning if not existing else f"{warning} {existing}"
@@ -268,10 +287,29 @@ def _find_exit_verdict(exit_code: int, raw_files: list[str], diagnostics: list[s
return SearchResult(error=_FIND_MODIFIED_ORDER_ERROR), None
detail = f": {diagnostics[0]}" if diagnostics else "."
return SearchResult(error=f"File search failed while running bounded find traversal{detail}"), None
+ # A non-permission diagnostic (a dir removed mid-walk) must not be reported as a permission
+ # problem: name it, plus any permission skips, so the user chases the real cause.
+ other = next((d.strip() for d in diagnostics if d.strip() and not _PERMISSION_DENIED_RE.match(d.strip())), None)
+ if other:
+ return None, f"Some paths could not be read: {other}. Results may be incomplete." + (
+ f" {warning}" if warning else "")
return None, warning or ("find traversal skipped one or more unreadable paths (restricted "
"permissions); results are partial.")
+def _drain_stderr_capped(pipe, chunks: list[bytes], omitted: list[int]) -> None:
+ """Read ``pipe`` to EOF (rg must never block on a full stderr pipe) but keep only the
+ first ``_STDERR_RETAIN_BYTES``; the first line is always kept so a lone huge diagnostic
+ still names the failure. ``omitted[0]`` counts the dropped lines."""
+ kept = 0
+ for raw in pipe or ():
+ if chunks and kept + len(raw) > _STDERR_RETAIN_BYTES:
+ omitted[0] += 1
+ continue
+ chunks.append(raw)
+ kept += len(raw)
+
+
def _parse_search_context_line(line: str) -> tuple[str, int, str] | None:
"""Parse a ``path-line-content`` context line using the RIGHTMOST numeric
separator (filenames may contain ``-<digits>-`` segments):
@@ -346,9 +384,15 @@ def _parse_search_output(result, output_mode: str, limit: int, offset: int,
return SearchResult(total_count=0, warning=warning)
return SearchResult(error=f"Search failed: {error_msg}", total_count=0)
skipped_warning = _prepend_skipped_paths_warning(diagnostics, None)
- tool_errors = [ln.strip() for ln in diagnostics.splitlines() if ln.lstrip().startswith(("rg: ", "grep: "))]
- if not skipped_warning and result.exit_code == 2 and tool_errors:
- skipped_warning = f"The search reported errors for some paths; results may be incomplete ({tool_errors[0]})."
+ # A non-permission error beside the skips (a path removed mid-search) is named too, not
+ # folded into the permission warning.
+ other_errors = [ln.strip() for ln in diagnostics.splitlines()
+ if ln.lstrip().startswith(("rg: ", "grep: "))
+ and not _PERMISSION_DENIED_RE.match(ln.strip())
+ and not _OMITTED_DIAGNOSTICS_RE.match(ln.strip())]
+ if result.exit_code == 2 and other_errors:
+ error_warning = f"The search reported errors for some paths; results may be incomplete ({other_errors[0]})."
+ skipped_warning = f"{error_warning} {skipped_warning}" if skipped_warning else error_warning
if skipped_warning:
warning = skipped_warning if not warning else f"{skipped_warning} {warning}"
lines = [ln for ln in payload.strip().split('\n') if ln]
@@ -531,11 +575,12 @@ class SearchMixin:
drainer = threading.Thread(target=_drain, daemon=True)
drainer.start()
err_chunks: list[bytes] = []
+ err_omitted = [0]
err_drainer = None
if merge_stderr:
err_pipe = proc.stderr
err_drainer = threading.Thread( # health: allow HX012 -- pipe drainer, reads no profile-scoped state
- target=lambda: err_chunks.extend(err_pipe or ()), daemon=True)
+ target=_drain_stderr_capped, args=(err_pipe, err_chunks, err_omitted), daemon=True)
err_drainer.start()
deadline = time.monotonic() + timeout
exit_code: Optional[int] = None
@@ -559,7 +604,10 @@ class SearchMixin:
err_drainer.join()
if proc.stderr is not None:
proc.stderr.close()
- stdout = b"".join(err_chunks).decode("utf-8", errors="replace") + stdout
+ err_text = b"".join(err_chunks).decode("utf-8", errors="replace")
+ if err_omitted[0]:
+ err_text = err_text.rstrip("\n") + "\n" + _omitted_diagnostics_line(err_omitted[0]) + "\n"
+ stdout = err_text + stdout
if exit_code == 124:
return ExecuteResult(stdout=stdout + f"\n[Command timed out after {timeout}s]", exit_code=124)
if exit_code == 130:
@@ -1125,7 +1173,7 @@ class SearchMixin:
skipped = _extract_unreadable_paths(diagnostics) \
if _is_permission_only_error(diagnostics) else []
permission_only = result.exit_code == 2 and bool(skipped)
- warning = _skipped_unreadable_warning(skipped) if skipped else None
+ warning = _skipped_unreadable_warning(skipped, _omitted_diagnostic_count(diagnostics)) if skipped else None
if scoped_common:
all_files = [
f if posixpath.isabs(f) else posixpath.normpath(posixpath.join(scoped_common, f))