Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(health): count Python pathlib-named calls as file IO only on a path receiver (#4764)
Raghav Chamadiyaabc33ec19d1b
diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.pyindex 788c5222a..7f4ee59b5 100644--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python.py@@ -78,6 +78,7 @@ from .python_nodes import (_text,_value_refs,)+from .python_paths import PythonPathReceiversfrom .python_sources import PythonLoopSourcesif TYPE_CHECKING:@@ -101,7 +102,9 @@ def _db_sink(method: str, is_attribute: bool, evidence: bool) -> str | None:return "db" if method in PY_DB_UNAMBIGUOUS or evidence else None-class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):+class PythonPerfDialect(+ PythonBatchForms, PythonLoopSources, PythonPathReceivers, IterableSources+):language = "python"lock_acquire_functions = frozenset({"acquire", "__enter__"})spin_loop_header = re.compile(r"while\s+True\s*:")@@ -155,8 +158,9 @@ class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):return _db_sink(method, is_attribute, has_db_import or root_kind == "db")if method == "Popen" or method in _PY_ROOTED_PROCESS_VERBS.get(root, ()):return "subprocess"- # A verb under its literal module root (bare ``open`` too), or a pathlib member call.- if method in _PY_ROOTED_FILE_VERBS.get(root, ()) or (is_attribute and method in PY_FS_METHODS):+ # A verb under its literal module root (bare ``open`` too). Pathlib member+ # calls need their receiver, so ``call_sink_kind`` decides those.+ if method in _PY_ROOTED_FILE_VERBS.get(root, ()):return "filesystem"# An HTTP verb only as a member call on an imported client: a bare ``get(...)`` is not one.if method == "urlopen" or (@@ -178,6 +182,8 @@ class PythonPerfDialect(PythonBatchForms, PythonLoopSources, IterableSources):self._reads_a_registry(call, io_names) or _runs_schema_statement(call, self.reaching_value)):return None+ if kind is None and self.callee_method_name(call) in PY_FS_METHODS and self.on_path(call):+ return "filesystem"orm_get = self._orm_get(call) if kind is None else Noneif orm_get is not None:# ``get`` alone is ``dict.get`` / ``requests.get``; the model-class shapediff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.pyindex 81efd5c2b..d9a3f9bc2 100644--- a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_lexicon.py@@ -45,9 +45,9 @@ PY_UNBOUNDED_READ_BOUND_METHODS: frozenset[str] = frozenset({"limit", "range", "single", "maybe_single", "first", "count", "head", "aggregate"})-# Matched on the method name alone: only ``pathlib.Path`` spells these, and the-# receiver is usually a local (``src_path.read_text()``) that ``io_names`` cannot-# classify. Metadata calls (``exists`` / ``stat`` / ``glob``) are cheap and left out.+# Pathlib round trips. Streams and terminals spell them too (``IOStream.read_bytes``),+# so a call counts only on a receiver shown to be a path (``python_paths.py``).+# Metadata calls (``exists`` / ``stat`` / ``glob``) are cheap and left out.PY_FS_METHODS: frozenset[str] = frozenset({"read_text", "read_bytes", "write_text", "write_bytes"})diff --git a/packages/core/src/repowise/core/analysis/health/perf/dialects/python_paths.py b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_paths.pynew file mode 100644index 000000000..0f634df75--- /dev/null+++ b/packages/core/src/repowise/core/analysis/health/perf/dialects/python_paths.py@@ -0,0 +1,145 @@+"""Whether the receiver of a Python ``read_text`` / ``write_bytes`` call is a path.++Tornado's ``IOStream.read_bytes`` and a terminal's ``write_text`` spell the same+methods as ``pathlib.Path``, so the method name alone is no filesystem evidence.+A receiver counts as a path when the code shows it: a ``Path(...)`` call, a ``/``+join, a name ending in ``path`` / ``file``, a parameter typed ``Path``, a local+bound to one of those, or a loop variable over ``glob`` / ``iterdir`` or over a+collection of paths. Any other receiver is no sink: an unknown local loses a+real pathlib loop, which is cheaper than flagging every stream read.+"""++from __future__ import annotations++import re+from typing import TYPE_CHECKING++from .iterables import IterableSources+from .python_lexicon import _PY_PASS_THROUGH, _PY_SCOPES+from .python_nodes import _field_text, _text++if TYPE_CHECKING:+ from collections.abc import Callable++ from tree_sitter import Node++# ``path``, ``fpath``, ``src_path``, ``template_file``, ``configFile``; not ``profile``.+_PATH_NAME_RE = re.compile(r"(?i:(?:^|_)(?:f|file)?(?:path|file))$|[a-z](?:Path|File)$")+_PATHS_NAME_RE = re.compile(r"(?i:(?:^|_)(?:f|file)?(?:paths|files))$|[a-z](?:Paths|Files)$")+# ``Path``, ``PurePosixPath``, ``zipfile.Path``, ``anyio.Path``, ``UPath``.+_PATH_CTOR_RE = re.compile(r"Path$")+_PATH_METHODS = frozenset({"joinpath", "with_name", "with_stem", "with_suffix"})+_PATH_LISTINGS = frozenset({"glob", "rglob", "iterdir"})+_COLLECTIONS = frozenset({"list", "set", "tuple"})+_COMPREHENSIONS = frozenset({"list_comprehension", "set_comprehension", "generator_expression"})+# Names followed through a binding or a loop header before giving up.+_HOPS = 3+++class PythonPathReceivers(IterableSources):+ """Mixed into :class:`~.python.PythonPerfDialect`."""++ def on_path(self, call: Node) -> bool:+ """*call* is a member call whose receiver the code shows is a path."""+ fn = call.child_by_field_name("function")+ recv = fn.child_by_field_name("object") if fn is not None and fn.type == "attribute" else None+ return recv is not None and self._is_path(recv, _HOPS)++ def _is_path(self, node: Node, hops: int) -> bool:+ while node.type == "parenthesized_expression" and node.named_child_count == 1:+ node = node.named_children[0]+ if node.type == "binary_operator":+ return _field_text(node, "operator") == "/"+ if node.type == "call":+ method = self.callee_method_name(node) or ""+ return bool(_PATH_CTOR_RE.search(method)) or method in _PATH_METHODS+ if node.type == "attribute":+ return bool(_PATH_NAME_RE.search(_field_text(node, "attribute")))+ if node.type != "identifier":+ return False+ if _PATH_NAME_RE.search(_text(node)):+ return True+ return hops > 0 and self._name_holds(node, hops - 1, self._is_path)++ def _yields_paths(self, node: Node, hops: int) -> bool:+ """Every item *node* yields is a path."""+ if node.type == "call":+ return self._call_yields_paths(node, hops)+ if node.type in _COLLECTIONS:+ items = node.named_children+ return bool(items) and all(self._is_path(item, hops) for item in items)+ if node.type in _COMPREHENSIONS:+ body = node.child_by_field_name("body")+ return body is not None and self._is_path(body, hops)+ if node.type != "identifier":+ return False+ if _PATHS_NAME_RE.search(_text(node)):+ return True+ return hops > 0 and self._name_holds(node, hops - 1, self._yields_paths)++ def _call_yields_paths(self, call: Node, hops: int) -> bool:+ """``d.glob(...)``, or ``sorted(xs)`` over items that are paths."""+ method = self.callee_method_name(call) or ""+ if method in _PATH_LISTINGS:+ return True+ walked = _walked_arg(call) if method in _PY_PASS_THROUGH else None+ return walked is not None and self._yields_paths(walked, hops)++ def _name_holds(self, name: Node, hops: int, check: Callable[[Node, int], bool]) -> bool:+ """The binding, loop header or ``Path`` annotation that gives *name* its value."""+ text = name.text or b""+ value = self.reaching_value(name, text)+ if value is not None:+ return check(value, hops)+ header = _loop_header(name, text)+ if header is not None:+ return self._yields_paths(header, hops)+ return _typed_path_param(name, text)+++def _walked_arg(call: Node) -> Node | None:+ """The first argument, or the generator that is the whole argument list."""+ args = call.child_by_field_name("arguments")+ if args is None or args.type in _COMPREHENSIONS:+ return args+ return args.named_children[0] if args.named_child_count else None+++def _loop_header(name: Node, text: bytes) -> Node | None:+ """The iterable of the innermost loop around *name* that binds it: a ``for``+ statement it sits in, or a ``for`` clause of a comprehension it sits in."""+ node = name.parent+ while node is not None and node.type not in _PY_SCOPES:+ if node.type == "for_statement":+ clauses = [node]+ elif node.type in _COMPREHENSIONS:+ clauses = [c for c in node.named_children if c.type == "for_in_clause"]+ else:+ clauses = []+ for clause in clauses:+ left = clause.child_by_field_name("left")+ if left is not None and left.type == "identifier" and left.text == text:+ return clause.child_by_field_name("right")+ node = node.parent+ return None+++def _typed_path_param(name: Node, text: bytes) -> bool:+ """*name* is a parameter of its function annotated with a ``Path`` type."""+ fn = name.parent+ while fn is not None and fn.type != "function_definition":+ fn = fn.parent+ params = fn.child_by_field_name("parameters") if fn is not None else None+ typed = ("typed_parameter", "typed_default_parameter")+ for param in params.named_children if params is not None else ():+ if param.type in typed and _param_name(param) == text:+ return "Path" in _field_text(param, "type")+ return False+++def _param_name(param: Node) -> bytes | None:+ """A typed parameter's name (``typed_parameter`` has no ``name`` field)."""+ ident = param.child_by_field_name("name") or next(+ (c for c in param.children if c.type == "identifier"), None+ )+ return ident.text if ident is not None else Nonediff --git a/tests/unit/health/test_perf_dialects.py b/tests/unit/health/test_perf_dialects.pyindex 91710d029..8e504e28a 100644--- a/tests/unit/health/test_perf_dialects.py+++ b/tests/unit/health/test_perf_dialects.py@@ -798,18 +798,61 @@ def test_python_pathlib_round_trips_are_filesystem_sinks():Python's filesystem lexicon recognised only the bare ``open(...)`` builtin,so the idiom the whole ecosystem actually uses was invisible: a loop full of``Path.read_text()`` scored a perfect 10/10 on the performance dimension.- Matched on the method name alone because only a ``pathlib.Path`` spells- these — the receiver is usually a plain local (``src_path.read_text()``),- so there is no import to classify."""for method in ("read_text", "read_bytes", "write_text", "write_bytes"):- src = f"def f(ps):\n for p in ps:\n p.{method}()\n"+ src = f"def f(d):\n for p in d.glob('*'):\n p.{method}()\n"assert ("io_in_loop", "filesystem") in _hits("python", src), method# The constructor-receiver form resolves the same way.ctor = "from pathlib import Path\ndef f(ps):\n for p in ps:\n Path(p).read_text()\n"assert ("io_in_loop", "filesystem") in _hits("python", ctor)+ "src",+ [+ "def f(d, names):\n for n in names:\n (d / n).read_text()\n",+ "def f(d, names):\n for n in names:\n t = d / n\n t.write_text('x')\n",+ "def f(xs):\n for x in xs:\n x.config_file.read_bytes()\n",+ "from pathlib import Path\ndef f(xs: list[Path]):\n for x in xs:\n x.read_text()\n",+ "def f(d):\n srcs = [d / 'a.py', d / 'b.py']\n for s in srcs:\n s.write_text('x')\n",+ "def f(d, ns):\n for s in sorted(d / n for n in ns):\n s.read_bytes()\n",+ "def f(d):\n for p in (q for q in d.glob('*') if q.is_file()):\n p.read_text()\n",+ "import pathlib\ndef f(ns):\n for n in ns:\n pathlib.Path(n).read_text()\n",+ "def f(xs):\n for x in xs:\n x.srcPath.read_text()\n",+ ],+)+def test_python_path_receivers_stay_filesystem_sinks(src):+ assert ("io_in_loop", "filesystem") in _hits("python", src)+++def test_python_awaited_path_read_is_a_serial_filesystem_await():+ src = (+ "import anyio\nasync def f(ns):\n for n in ns:\n"+ " await anyio.Path(n).read_text()\n"+ )+ assert ("serial_await_in_loop", "filesystem") in _hits("python", src)+++ "src",+ [+ # tornado ``IOStream.read_bytes``: a socket read, not a file.+ "class C:\n async def f(self, n):\n while n:\n"+ " await self.stream.read_bytes(n)\n",+ # rich's terminal writer.+ "def f(term, parts):\n for t in parts:\n term.write_text(t)\n",+ # An unknown receiver is no sink: streams and terminals spell these names too.+ "def f(ps):\n for p in ps:\n p.read_text()\n",+ # A name that only ends in the letters ``file``.+ "def f(profile, ns):\n for n in ns:\n profile.read_bytes(n)\n",+ # A binding that never settles on a path.+ "def f(ns):\n a = a\n for n in ns:\n a.read_text()\n",+ ],+)+def test_python_non_path_receivers_are_not_filesystem_sinks(src):+ assert not any(detail == "filesystem" for _, detail in _hits("python", src))++def test_python_metadata_syscalls_are_not_filesystem_sinks():"""``exists`` / ``stat`` / ``glob`` in a loop are cheap and usually correct.@@ -931,7 +974,7 @@ def test_python_point_reads_do_not_become_hot_path_candidates():assert [f.bare_sink_kind for f in fc.perf_fn_facts] == ["filesystem"]# Still a per-iteration finding when it IS in a loop.assert ("io_in_loop", "filesystem") in _hits(- "python", "def f(ps):\n for p in ps:\n p.write_text('x')\n"+ "python", "def f(paths):\n for p in paths:\n p.write_text('x')\n")