Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(core): keep SyntaxWarnings from analysed Python source off the console (#3890)

Raghav Chamadiyad6adf9faa280
diff --git a/packages/core/src/repowise/core/analysis/conftest_routes.py b/packages/core/src/repowise/core/analysis/conftest_routes.py
index 6f4c8e26b..bd3ddf37c 100644
--- a/packages/core/src/repowise/core/analysis/conftest_routes.py
+++ b/packages/core/src/repowise/core/analysis/conftest_routes.py
@@ -70,6 +70,8 @@ from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
+from repowise.core.pyparse import parse_python, symtable_python
+
from ..ingestion.framework_edges.pytest_edges import (
CONFTEST_HINT,
FIXTURE_HINT,
@@ -496,8 +498,8 @@ def local_names(source: str) -> frozenset[str]:
import-time side effect that patches the standard library.
"""
try:
- tree = ast.parse(source)
- table = symtable.symtable(source, "<module>", "exec")
+ tree = parse_python(source)
+ table = symtable_python(source)
except (SyntaxError, ValueError):
return frozenset()
counts: dict[str, int] = {}
@@ -636,7 +638,7 @@ class ConftestFacts:
@classmethod
def parse(cls, source: str, path: str) -> ConftestFacts:
try:
- tree = ast.parse(source)
+ tree = parse_python(source)
except (SyntaxError, ValueError):
return cls(path, error=f"{path} does not parse")
facts = cls(path, here=module_parts(path)[:-1], parents=_parents(tree))
diff --git a/packages/core/src/repowise/core/analysis/dead_code/entry_shape.py b/packages/core/src/repowise/core/analysis/dead_code/entry_shape.py
index 0d9fbad01..ff8c956a2 100644
--- a/packages/core/src/repowise/core/analysis/dead_code/entry_shape.py
+++ b/packages/core/src/repowise/core/analysis/dead_code/entry_shape.py
@@ -31,6 +31,8 @@ from collections.abc import Mapping
from pathlib import PurePosixPath
from typing import Any
+from repowise.core.pyparse import parse_python
+
from ...ingestion.extractors.visibility import _TS_BLOCK_COMMENT, _TS_CJS_EXPORTS_RE
from ...ingestion.languages.registry import REGISTRY
from .models import DeadCodeFindingData, DeadCodeKind
@@ -87,7 +89,7 @@ def _parse_python(path: str, blob: bytes) -> ast.Module | None:
if REGISTRY.from_extension(PurePosixPath(path).suffix) != "python" or path.endswith(".pyi"):
return None
try:
- return ast.parse(blob)
+ return parse_python(blob)
except (SyntaxError, ValueError, RecursionError):
return None
diff --git a/packages/core/src/repowise/core/analysis/public_api.py b/packages/core/src/repowise/core/analysis/public_api.py
index 26ad2af20..ef4302d20 100644
--- a/packages/core/src/repowise/core/analysis/public_api.py
+++ b/packages/core/src/repowise/core/analysis/public_api.py
@@ -25,6 +25,7 @@ from repowise.core.ingestion.languages.python_modules import (
dotted_module_for,
)
from repowise.core.ingestion.package_roots import module_for, package_roots_from_paths
+from repowise.core.pyparse import parse_python
from repowise.core.workspace.code_api import dunder_all
# A Rust ``impl`` block is not a name anyone imports.
@@ -151,7 +152,7 @@ def _lazy_map_from(
"""Return ``(public_name, target_module, target_name)`` lazy exports."""
try:
source = Path(source_path or path).read_text(encoding="utf-8", errors="replace")
- tree = ast.parse(source)
+ tree = parse_python(source)
except (OSError, SyntaxError, ValueError):
return []
current = dotted_module_for(path, paths)
diff --git a/packages/core/src/repowise/core/analysis/security_scan.py b/packages/core/src/repowise/core/analysis/security_scan.py
index 3c91e2c14..894a31722 100644
--- a/packages/core/src/repowise/core/analysis/security_scan.py
+++ b/packages/core/src/repowise/core/analysis/security_scan.py
@@ -31,6 +31,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from repowise.core.analysis.secret_allowlist import secret_path_allowlist
from repowise.core.code_origin import code_origin
+from repowise.core.pyparse import parse_python
from repowise.core.support_paths import DOC_EXTENSIONS
from repowise.core.test_paths import is_test_related_path
@@ -615,7 +616,7 @@ def _http_bindings(tree: ast.AST) -> set[str]:
def _tls_verify_off_lines(source: str) -> set[int] | None:
"""Lines where ``verify=False`` reaches an HTTP client; None when *source* does not parse."""
try:
- tree = ast.parse(source)
+ tree = parse_python(source)
except SyntaxError:
return None
bound = _http_bindings(tree)
@@ -1134,7 +1135,7 @@ def _call_findings(file_path: str, source: str, masked: Callable[[], str]) -> li
return []
if is_python:
try:
- tree = ast.parse(source)
+ tree = parse_python(source)
except SyntaxError:
pass
else:
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py
index 8b2683fd0..a208f3a33 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/django.py
@@ -5,6 +5,8 @@ import re
from collections.abc import Callable
from pathlib import Path, PurePosixPath
+from repowise.core.pyparse import parse_python
+
from .base import DynamicEdge, DynamicHintExtractor
#: Reads a module's syntax tree and returns the class names Django uses there.
@@ -90,7 +92,7 @@ def _used_names(path: Path, used_names: UsedNames | None) -> tuple[str, ...]:
if used_names is None:
return ()
try:
- tree = ast.parse(path.read_text(encoding="utf-8", errors="ignore"), filename=str(path))
+ tree = parse_python(path.read_text(encoding="utf-8", errors="ignore"), filename=str(path))
except (OSError, SyntaxError, ValueError):
return ()
return tuple(sorted(used_names(tree)))
@@ -209,7 +211,7 @@ def edge_strings(path: str, blob: bytes) -> list | None:
if p.name != "settings.py" and not (p.parent.name == "settings" and p.suffix == ".py"):
return []
try:
- return settings_entries(ast.parse(blob.decode("utf-8", "ignore")))
+ return settings_entries(parse_python(blob.decode("utf-8", "ignore")))
except Exception: # as in _scan_settings, which skips any file that will not parse
return None
@@ -244,7 +246,7 @@ class DjangoDynamicHints(DynamicHintExtractor):
for settings_file in settings_files:
try:
source = settings_file.read_text(encoding="utf-8", errors="ignore")
- tree = ast.parse(source, filename=str(settings_file))
+ tree = parse_python(source, filename=str(settings_file))
except Exception:
continue
diff --git a/packages/core/src/repowise/core/ingestion/dynamic_hints/python_package_listing.py b/packages/core/src/repowise/core/ingestion/dynamic_hints/python_package_listing.py
index 43f32f321..05a5e9a6e 100644
--- a/packages/core/src/repowise/core/ingestion/dynamic_hints/python_package_listing.py
+++ b/packages/core/src/repowise/core/ingestion/dynamic_hints/python_package_listing.py
@@ -37,6 +37,8 @@ import ast
import re
from collections.abc import Callable
+from repowise.core.pyparse import parse_python
+
# Cheap gate before parsing: a listing of ``__path__`` somewhere in the text.
OWN_LISTING_RE = re.compile(rb"(?:iter_modules|walk_packages)\(\s*(?:path\s*=\s*)?__path__\b")
@@ -293,7 +295,7 @@ def _listing_loops(tree: ast.AST):
def _parse(blob: bytes) -> ast.AST | None:
try:
- return ast.parse(blob)
+ return parse_python(blob)
except (SyntaxError, ValueError):
return None
diff --git a/packages/core/src/repowise/core/ingestion/extractors/visibility.py b/packages/core/src/repowise/core/ingestion/extractors/visibility.py
index c25d10c38..8cbfd8f3b 100644
--- a/packages/core/src/repowise/core/ingestion/extractors/visibility.py
+++ b/packages/core/src/repowise/core/ingestion/extractors/visibility.py
@@ -19,6 +19,8 @@ import re
from collections.abc import Callable
from typing import TYPE_CHECKING
+from repowise.core.pyparse import parse_python
+
from .helpers import node_text
if TYPE_CHECKING:
@@ -97,7 +99,7 @@ def py_module_all_names(src: str) -> frozenset[str] | None:
if "__all__" not in src:
return None
try:
- tree = ast.parse(src)
+ tree = parse_python(src)
except (SyntaxError, ValueError):
return None
diff --git a/packages/core/src/repowise/core/ingestion/framework_edges/test_path_strings.py b/packages/core/src/repowise/core/ingestion/framework_edges/test_path_strings.py
index f6dfa2b17..933add05e 100644
--- a/packages/core/src/repowise/core/ingestion/framework_edges/test_path_strings.py
+++ b/packages/core/src/repowise/core/ingestion/framework_edges/test_path_strings.py
@@ -24,6 +24,8 @@ import posixpath
import re
from typing import TYPE_CHECKING, Any
+from repowise.core.pyparse import parse_python
+
from ...test_paths import is_test_path
from ..resolvers import ResolverContext
from ..source_text import source_bytes
@@ -187,7 +189,7 @@ def _bind_function_args(scope: _Scope, args: ast.arguments) -> None:
def python_ancestor_targets(test_path: str, text: bytes, path_set: set[str]) -> list[str]:
"""Indexed code files *test_path* names via Path(__file__) ancestor anchors or variables."""
try:
- tree = ast.parse(text)
+ tree = parse_python(text)
except Exception:
return []
diff --git a/packages/core/src/repowise/core/ingestion/python_local_refs.py b/packages/core/src/repowise/core/ingestion/python_local_refs.py
index 06288a089..f7a92c78e 100644
--- a/packages/core/src/repowise/core/ingestion/python_local_refs.py
+++ b/packages/core/src/repowise/core/ingestion/python_local_refs.py
@@ -48,6 +48,8 @@ import ast
from collections.abc import Iterator
from collections.abc import Set as AbstractSet
+from repowise.core.pyparse import parse_python
+
__all__ = ["extract_python_local_refs", "local_refs_from_tree", "parse_python_module"]
@@ -88,7 +90,7 @@ def extract_python_local_refs(
def parse_python_module(source: str) -> ast.Module | None:
"""*source* as an ``ast`` module, or ``None`` when Python 3 cannot parse it."""
try:
- return ast.parse(source)
+ return parse_python(source)
except (SyntaxError, ValueError):
return None
@@ -185,7 +187,7 @@ def _forward_ref_nodes(annotation: ast.expr) -> Iterator[tuple[ast.AST, int]]:
if not (isinstance(sub, ast.Constant) and isinstance(sub.value, str)):
continue
try:
- inner = ast.parse(sub.value, mode="eval")
+ inner = parse_python(sub.value, mode="eval")
except (SyntaxError, ValueError):
continue
for ref in ast.walk(inner):
diff --git a/packages/core/src/repowise/core/pyparse.py b/packages/core/src/repowise/core/pyparse.py
new file mode 100644
index 000000000..203de021b
--- /dev/null
+++ b/packages/core/src/repowise/core/pyparse.py
@@ -0,0 +1,28 @@
+"""Parse repo-owned Python source without leaking its warnings to the console.
+
+An invalid escape sequence is a SyntaxWarning from Python 3.12 and a
+DeprecationWarning before it, so both are silenced.
+"""
+
+from __future__ import annotations
+
+import ast
+import symtable
+import warnings
+from typing import Any
+
+
+def parse_python(source: str | bytes, **kwargs: Any) -> ast.Module:
+ """``ast.parse`` that stays quiet about warnings in the analysed repo's own code."""
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", SyntaxWarning)
+ warnings.simplefilter("ignore", DeprecationWarning)
+ return ast.parse(source, **kwargs)
+
+
+def symtable_python(source: str) -> symtable.SymbolTable:
+ """Module symbol table of *source*, quiet like ``parse_python``."""
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", SyntaxWarning)
+ warnings.simplefilter("ignore", DeprecationWarning)
+ return symtable.symtable(source, "<module>", "exec")
diff --git a/packages/core/src/repowise/core/workspace/code_api.py b/packages/core/src/repowise/core/workspace/code_api.py
index 5c990db20..bd69f7015 100644
--- a/packages/core/src/repowise/core/workspace/code_api.py
+++ b/packages/core/src/repowise/core/workspace/code_api.py
@@ -38,6 +38,8 @@ import logging
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any
+from repowise.core.pyparse import parse_python
+
if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path
@@ -209,7 +211,7 @@ def dunder_all(path: Path) -> frozenset[str] | None:
import ast
try:
- tree = ast.parse(path.read_text(encoding="utf-8", errors="replace"))
+ tree = parse_python(path.read_text(encoding="utf-8", errors="replace"))
except (OSError, SyntaxError, ValueError):
return None
names: set[str] | None = None
diff --git a/tests/unit/core/test_pyparse.py b/tests/unit/core/test_pyparse.py
new file mode 100644
index 000000000..77eaa3cf1
--- /dev/null
+++ b/tests/unit/core/test_pyparse.py
@@ -0,0 +1,23 @@
+"""Analysed repo source with an invalid escape must not leak SyntaxWarnings."""
+
+from __future__ import annotations
+
+import warnings
+
+from repowise.core.ingestion.framework_edges.test_path_strings import python_ancestor_targets
+from repowise.core.pyparse import parse_python, symtable_python
+
+SOURCE = 'import re\nPATTERN = re.compile("\\s+")\n'
+
+
+def test_helpers_stay_quiet() -> None:
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ parse_python(SOURCE)
+ symtable_python(SOURCE)
+
+
+def test_test_path_scan_stays_quiet() -> None:
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ python_ancestor_targets("tests/test_x.py", SOURCE.encode(), {"tests/test_x.py"})