Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(health): count Python pathlib-named calls as file IO only on a path receiver (#4764)

Raghav Chamadiyaabc33ec19d1b
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
index 788c5222a..7f4ee59b5 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py
@@ -78,6 +78,7 @@ from .python_nodes import (
_text,
_value_refs,
)
+from .python_paths import PythonPathReceivers
from .python_sources import PythonLoopSources
if TYPE_CHECKING:
@@ -101,7 +102,9 @@ def _db_sink(method: str, is_attribute: bool, evidence: bool) -> str | None:
return "db" if method in PY_DB_UNAMBIGUOUS or evidence else None
-class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):
+class PythonPerfDialect(
+ PythonBatchForms, PythonLoopSources, PythonPathReceivers, IterableSources
+):
language = "python"
lock_acquire_functions = frozenset({"acquire", "__enter__"})
spin_loop_header = re.compile(r"while\s+True\s*:")
@@ -155,8 +158,9 @@ class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):
return _db_sink(method, is_attribute, has_db_import or root_kind == "db")
if method == "Popen" or method in _PY_ROOTED_PROCESS_VERBS.get(root, ()):
return "subprocess"
- # A verb under its literal module root (bare ``open`` too), or a pathlib member call.
- if method in _PY_ROOTED_FILE_VERBS.get(root, ()) or (is_attribute and method in PY_FS_METHODS):
+ # A verb under its literal module root (bare ``open`` too). Pathlib member
+ # calls need their receiver, so ``call_sink_kind`` decides those.
+ if method in _PY_ROOTED_FILE_VERBS.get(root, ()):
return "filesystem"
# An HTTP verb only as a member call on an imported client: a bare ``get(...)`` is not one.
if method == "urlopen" or (
@@ -178,6 +182,8 @@ class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):
self._reads_a_registry(call, io_names) or _runs_schema_statement(call, self.reaching_value)
):
return None
+ if kind is None and self.callee_method_name(call) in PY_FS_METHODS and self.on_path(call):
+ return "filesystem"
orm_get = self._orm_get(call) if kind is None else None
if orm_get is not None:
# ``get`` alone is ``dict.get`` / ``requests.get``; the model-class shape
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py
index 81efd5c2b..d9a3f9bc2 100644
--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py
@@ -45,9 +45,9 @@ PY_UNBOUNDED_READ_BOUND_METHODS: frozenset[str] = frozenset(
{"limit", "range", "single", "maybe_single", "first", "count", "head", "aggregate"}
)
-# Matched on the method name alone: only ``pathlib.Path`` spells these, and the
-# receiver is usually a local (``src_path.read_text()``) that ``io_names`` cannot
-# classify. Metadata calls (``exists`` / ``stat`` / ``glob``) are cheap and left out.
+# Pathlib round trips. Streams and terminals spell them too (``IOStream.read_bytes``),
+# so a call counts only on a receiver shown to be a path (``python_paths.py``).
+# Metadata calls (``exists`` / ``stat`` / ``glob``) are cheap and left out.
PY_FS_METHODS: frozenset[str] = frozenset(
{"read_text", "read_bytes", "write_text", "write_bytes"}
)
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_paths.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_paths.py
new file mode 100644
index 000000000..0f634df75
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_paths.py
@@ -0,0 +1,145 @@
+"""Whether the receiver of a Python ``read_text`` / ``write_bytes`` call is a path.
+
+Tornado's ``IOStream.read_bytes`` and a terminal's ``write_text`` spell the same
+methods as ``pathlib.Path``, so the method name alone is no filesystem evidence.
+A receiver counts as a path when the code shows it: a ``Path(...)`` call, a ``/``
+join, a name ending in ``path`` / ``file``, a parameter typed ``Path``, a local
+bound to one of those, or a loop variable over ``glob`` / ``iterdir`` or over a
+collection of paths. Any other receiver is no sink: an unknown local loses a
+real pathlib loop, which is cheaper than flagging every stream read.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+from .iterables import IterableSources
+from .python_lexicon import _PY_PASS_THROUGH, _PY_SCOPES
+from .python_nodes import _field_text, _text
+
+if TYPE_CHECKING:
+ from collections.abc import Callable
+
+ from tree_sitter import Node
+
+# ``path``, ``fpath``, ``src_path``, ``template_file``, ``configFile``; not ``profile``.
+_PATH_NAME_RE = re.compile(r"(?i:(?:^|_)(?:f|file)?(?:path|file))$|[a-z](?:Path|File)$")
+_PATHS_NAME_RE = re.compile(r"(?i:(?:^|_)(?:f|file)?(?:paths|files))$|[a-z](?:Paths|Files)$")
+# ``Path``, ``PurePosixPath``, ``zipfile.Path``, ``anyio.Path``, ``UPath``.
+_PATH_CTOR_RE = re.compile(r"Path$")
+_PATH_METHODS = frozenset({"joinpath", "with_name", "with_stem", "with_suffix"})
+_PATH_LISTINGS = frozenset({"glob", "rglob", "iterdir"})
+_COLLECTIONS = frozenset({"list", "set", "tuple"})
+_COMPREHENSIONS = frozenset({"list_comprehension", "set_comprehension", "generator_expression"})
+# Names followed through a binding or a loop header before giving up.
+_HOPS = 3
+
+
+class PythonPathReceivers(IterableSources):
+ """Mixed into :class:`~.python.PythonPerfDialect`."""
+
+ def on_path(self, call: Node) -> bool:
+ """*call* is a member call whose receiver the code shows is a path."""
+ fn = call.child_by_field_name("function")
+ recv = fn.child_by_field_name("object") if fn is not None and fn.type == "attribute" else None
+ return recv is not None and self._is_path(recv, _HOPS)
+
+ def _is_path(self, node: Node, hops: int) -> bool:
+ while node.type == "parenthesized_expression" and node.named_child_count == 1:
+ node = node.named_children[0]
+ if node.type == "binary_operator":
+ return _field_text(node, "operator") == "/"
+ if node.type == "call":
+ method = self.callee_method_name(node) or ""
+ return bool(_PATH_CTOR_RE.search(method)) or method in _PATH_METHODS
+ if node.type == "attribute":
+ return bool(_PATH_NAME_RE.search(_field_text(node, "attribute")))
+ if node.type != "identifier":
+ return False
+ if _PATH_NAME_RE.search(_text(node)):
+ return True
+ return hops > 0 and self._name_holds(node, hops - 1, self._is_path)
+
+ def _yields_paths(self, node: Node, hops: int) -> bool:
+ """Every item *node* yields is a path."""
+ if node.type == "call":
+ return self._call_yields_paths(node, hops)
+ if node.type in _COLLECTIONS:
+ items = node.named_children
+ return bool(items) and all(self._is_path(item, hops) for item in items)
+ if node.type in _COMPREHENSIONS:
+ body = node.child_by_field_name("body")
+ return body is not None and self._is_path(body, hops)
+ if node.type != "identifier":
+ return False
+ if _PATHS_NAME_RE.search(_text(node)):
+ return True
+ return hops > 0 and self._name_holds(node, hops - 1, self._yields_paths)
+
+ def _call_yields_paths(self, call: Node, hops: int) -> bool:
+ """``d.glob(...)``, or ``sorted(xs)`` over items that are paths."""
+ method = self.callee_method_name(call) or ""
+ if method in _PATH_LISTINGS:
+ return True
+ walked = _walked_arg(call) if method in _PY_PASS_THROUGH else None
+ return walked is not None and self._yields_paths(walked, hops)
+
+ def _name_holds(self, name: Node, hops: int, check: Callable[[Node, int], bool]) -> bool:
+ """The binding, loop header or ``Path`` annotation that gives *name* its value."""
+ text = name.text or b""
+ value = self.reaching_value(name, text)
+ if value is not None:
+ return check(value, hops)
+ header = _loop_header(name, text)
+ if header is not None:
+ return self._yields_paths(header, hops)
+ return _typed_path_param(name, text)
+
+
+def _walked_arg(call: Node) -> Node | None:
+ """The first argument, or the generator that is the whole argument list."""
+ args = call.child_by_field_name("arguments")
+ if args is None or args.type in _COMPREHENSIONS:
+ return args
+ return args.named_children[0] if args.named_child_count else None
+
+
+def _loop_header(name: Node, text: bytes) -> Node | None:
+ """The iterable of the innermost loop around *name* that binds it: a ``for``
+ statement it sits in, or a ``for`` clause of a comprehension it sits in."""
+ node = name.parent
+ while node is not None and node.type not in _PY_SCOPES:
+ if node.type == "for_statement":
+ clauses = [node]
+ elif node.type in _COMPREHENSIONS:
+ clauses = [c for c in node.named_children if c.type == "for_in_clause"]
+ else:
+ clauses = []
+ for clause in clauses:
+ left = clause.child_by_field_name("left")
+ if left is not None and left.type == "identifier" and left.text == text:
+ return clause.child_by_field_name("right")
+ node = node.parent
+ return None
+
+
+def _typed_path_param(name: Node, text: bytes) -> bool:
+ """*name* is a parameter of its function annotated with a ``Path`` type."""
+ fn = name.parent
+ while fn is not None and fn.type != "function_definition":
+ fn = fn.parent
+ params = fn.child_by_field_name("parameters") if fn is not None else None
+ typed = ("typed_parameter", "typed_default_parameter")
+ for param in params.named_children if params is not None else ():
+ if param.type in typed and _param_name(param) == text:
+ return "Path" in _field_text(param, "type")
+ return False
+
+
+def _param_name(param: Node) -> bytes | None:
+ """A typed parameter's name (``typed_parameter`` has no ``name`` field)."""
+ ident = param.child_by_field_name("name") or next(
+ (c for c in param.children if c.type == "identifier"), None
+ )
+ return ident.text if ident is not None else None
diff --git a/tests/unit/health/test_perf_dialects.py b/tests/unit/health/test_perf_dialects.py
index 91710d029..8e504e28a 100644
--- a/tests/unit/health/test_perf_dialects.py
+++ b/tests/unit/health/test_perf_dialects.py
@@ -798,18 +798,61 @@ def test_python_pathlib_round_trips_are_filesystem_sinks():
Python's filesystem lexicon recognised only the bare ``open(...)`` builtin,
so the idiom the whole ecosystem actually uses was invisible: a loop full of
``Path.read_text()`` scored a perfect 10/10 on the performance dimension.
- Matched on the method name alone because only a ``pathlib.Path`` spells
- these — the receiver is usually a plain local (``src_path.read_text()``),
- so there is no import to classify.
"""
for method in ("read_text", "read_bytes", "write_text", "write_bytes"):
- src = f"def f(ps):\n for p in ps:\n p.{method}()\n"
+ src = f"def f(d):\n for p in d.glob('*'):\n p.{method}()\n"
assert ("io_in_loop", "filesystem") in _hits("python", src), method
# The constructor-receiver form resolves the same way.
ctor = "from pathlib import Path\ndef f(ps):\n for p in ps:\n Path(p).read_text()\n"
assert ("io_in_loop", "filesystem") in _hits("python", ctor)
+ "src",
+ [
+ "def f(d, names):\n for n in names:\n (d / n).read_text()\n",
+ "def f(d, names):\n for n in names:\n t = d / n\n t.write_text('x')\n",
+ "def f(xs):\n for x in xs:\n x.config_file.read_bytes()\n",
+ "from pathlib import Path\ndef f(xs: list[Path]):\n for x in xs:\n x.read_text()\n",
+ "def f(d):\n srcs = [d / 'a.py', d / 'b.py']\n for s in srcs:\n s.write_text('x')\n",
+ "def f(d, ns):\n for s in sorted(d / n for n in ns):\n s.read_bytes()\n",
+ "def f(d):\n for p in (q for q in d.glob('*') if q.is_file()):\n p.read_text()\n",
+ "import pathlib\ndef f(ns):\n for n in ns:\n pathlib.Path(n).read_text()\n",
+ "def f(xs):\n for x in xs:\n x.srcPath.read_text()\n",
+ ],
+)
+def test_python_path_receivers_stay_filesystem_sinks(src):
+ assert ("io_in_loop", "filesystem") in _hits("python", src)
+
+
+def test_python_awaited_path_read_is_a_serial_filesystem_await():
+ src = (
+ "import anyio\nasync def f(ns):\n for n in ns:\n"
+ " await anyio.Path(n).read_text()\n"
+ )
+ assert ("serial_await_in_loop", "filesystem") in _hits("python", src)
+
+
+ "src",
+ [
+ # tornado ``IOStream.read_bytes``: a socket read, not a file.
+ "class C:\n async def f(self, n):\n while n:\n"
+ " await self.stream.read_bytes(n)\n",
+ # rich's terminal writer.
+ "def f(term, parts):\n for t in parts:\n term.write_text(t)\n",
+ # An unknown receiver is no sink: streams and terminals spell these names too.
+ "def f(ps):\n for p in ps:\n p.read_text()\n",
+ # A name that only ends in the letters ``file``.
+ "def f(profile, ns):\n for n in ns:\n profile.read_bytes(n)\n",
+ # A binding that never settles on a path.
+ "def f(ns):\n a = a\n for n in ns:\n a.read_text()\n",
+ ],
+)
+def test_python_non_path_receivers_are_not_filesystem_sinks(src):
+ assert not any(detail == "filesystem" for _, detail in _hits("python", src))
+
+
def test_python_metadata_syscalls_are_not_filesystem_sinks():
"""``exists`` / ``stat`` / ``glob`` in a loop are cheap and usually correct.
@@ -931,7 +974,7 @@ def test_python_point_reads_do_not_become_hot_path_candidates():
assert [f.bare_sink_kind for f in fc.perf_fn_facts] == ["filesystem"]
# Still a per-iteration finding when it IS in a loop.
assert ("io_in_loop", "filesystem") in _hits(
- "python", "def f(ps):\n for p in ps:\n p.write_text('x')\n"
+ "python", "def f(paths):\n for p in paths:\n p.write_text('x')\n"
)