Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(ingestion): stop C++ attribute and namespace macros from hiding declarations (#4814)
Swati Ahujaa9101000c7f5
diff --git a/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py b/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.pyindex a48f2f665..794510efa 100644--- a/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py+++ b/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py@@ -4,6 +4,8 @@ from __future__ import annotationsimport re+from ..extractors.synthetic_symbols.cpp_macros import _FAST_REJECT_TOKENS as _SYNTHESIZED_MACROS+_PASCAL_USES_IN_CLAUSE_RE = re.compile(rb"\bin\b[ \t]*'(?:[^'\r\n]|'')*'")@@ -165,6 +167,50 @@ _CPP_TRAILING_ANNOTATION_RE = re.compile(re.MULTILINE,)+# An attribute macro before a constructor (``FMT_CONSTEXPR Foo(int x) : x_(x)+# {}``). The grammar reads the macro as a return type, a typed function may not+# have a member-initializer list, and recovery then leaves the class open to+# the end of the file, so every later function loses its symbol. Group 1 is+# the macro run; the ``: init`` after the parameter list, checked in code, is+# what makes the declaration a constructor and not ``TYPE var(args);``.+_CPP_CTOR_MACRO_RE = re.compile(+ rb"^[ \t]*(?:(?:constexpr|explicit|inline)[ \t]+)*"+ rb"((?:[A-Z][A-Z0-9_]+[ \t]+)+)"+ rb"(?:(?:constexpr|explicit|inline)[ \t]+)*[A-Za-z_]\w*[ \t]*\(",+ re.MULTILINE,+)+_CPP_CTOR_INIT_RE = re.compile(rb"\s*(?:noexcept\b\s*(?:\([^()]*\))?\s*)?:(?!:)")++# A macro alone on its line (``FMT_BEGIN_NAMESPACE``, ``FMT_EXPORT``, ``Q_OBJECT``,+# ``FMT_PRAGMA_CLANG(diagnostic push)``) expands to a namespace opener, an+# attribute or nothing. With no ``;`` the grammar reads it as the type of the+# declaration after it, which renames or swallows that declaration. Group 1 is+# a bare name at any indent; group 2 is a call-shaped one at column 0 that a+# declaration follows, so an indented ``LOG(x)`` statement, a ``TEST(a, b)``+# opening a body and a last enumerator before ``}`` all stay. Neither is+# blanked before a typeless function head (``BOOL\nWINAPI\nDllMain(...)``),+# where the macro is the return type, nor is a registration macro whose+# text the synthetic-symbol pass reads (``NAPI_MODULE(name, Init)``). A head+# has a lowercase letter, so another macro line (``FMT_PRAGMA(x)``) is none.+# A call-shaped macro the file defines itself stays too: it may wrap a+# ``push_macro``/``pop_macro`` pragma the macro-state pass reads.+_CPP_TYPELESS_HEAD = (+ rb"[ \t]*\r?\n(?:[\r\n \t]*[A-Z][A-Z0-9_]+[ \t]*\r?\n)*"+ rb"[\r\n \t]*~?(?=[\w:]*[a-z])[A-Za-z_][\w:]*[ \t]*\("+)+_CPP_MACRO_LINE_RE = re.compile(+ rb"^(?<!\\\n)(?<!\\\r\n)(?:[ \t]*([A-Z][A-Z0-9_]+)(?=[ \t]*\r?\n[\r\n \t]*[A-Za-z_#/])"+ rb"|(?!" + b"|".join(re.escape(t.encode()) for t in _SYNTHESIZED_MACROS) + rb")"+ rb"(([A-Z][A-Z0-9_]+)\((?:[^()\r\n]|\([^()\r\n]*\))*\))(?=[ \t]*\r?\n[\r\n \t]*[A-Za-z_]))"+ rb"(?!" + _CPP_TYPELESS_HEAD + rb")",+ re.MULTILINE,+)+_CPP_FUNCTION_MACRO_DEFINE_RE = re.compile(rb"^[ \t]*#[ \t]*define[ \t]+(\w+)\(", re.MULTILINE)+# The same macros written ahead of a template on its line (``FMT_EXPORT template <...>``).+_CPP_TEMPLATE_PREFIX_MACRO_RE = re.compile(+ rb"^[ \t]*((?:[A-Z][A-Z0-9_]+[ \t]+)+)(?=template\b)", re.MULTILINE+)+# Words that put a following ``RELEASE(p)`` in statement position, where it# is a real call and must stay._CPP_STATEMENT_WORDS = frozenset(@@ -204,6 +250,38 @@ def _open_paren_of(source: bytes, close: int) -> int:return -1+def _close_paren_of(source: bytes, open_: int) -> int:+ """Index of the ``)`` matching the ``(`` at *open_*, or -1."""+ depth = 0+ for i in range(open_, min(open_ + _CPP_PAREN_SCAN_LIMIT, len(source))):+ if source[i] == ord("("):+ depth += 1+ elif source[i] == ord(")"):+ depth -= 1+ if depth == 0:+ return i+ return -1+++def _ctor_macro_spans(source: bytes) -> list[tuple[int, int]]:+ """Macro runs before a constructor that has a member-initializer list."""+ spans = []+ for m in _CPP_CTOR_MACRO_RE.finditer(source):+ close = _close_paren_of(source, m.end() - 1)+ if close >= 0 and _CPP_CTOR_INIT_RE.match(source, close + 1):+ spans.append(m.span(1))+ return spans+++def _macro_line_spans(source: bytes) -> list[tuple[int, int]]:+ """Macro-only lines, less call-shaped ones naming a macro this file defines."""+ matches = list(_CPP_MACRO_LINE_RE.finditer(source))+ if not matches:+ return []+ local = set(_CPP_FUNCTION_MACRO_DEFINE_RE.findall(source))+ return [m.span(1) if m.group(1) else m.span(2) for m in matches if m.group(3) not in local]++def _ends_declarator(source: bytes, end: int) -> bool:"""True when the text before *end* is a declarator, not a statement head.@@ -245,14 +323,19 @@ def cpp_class_head_macros(source: bytes) -> frozenset[str]:def prepare_cpp_source(source: bytes) -> bytes:- """Blank class-head export macros and trailing thread-safety annotations.-- Both are macros the grammar cannot see expanded, and both make it build- the wrong tree: a missing class and a lost first method, or a hidden field- and a method named after the annotation. Spans are blanked in place, so- offsets and line numbers outside them are unchanged.+ """Blank class-head export macros, constructor attribute macros,+ macro-only lines and trailing thread-safety annotations.++ All are macros the grammar cannot see expanded, and all make it build the+ wrong tree: a missing class and a lost first method, a class left open+ over the rest of the file, a declaration read as typed by the macro before+ it, or a hidden field and a method named after the annotation. Spans are+ blanked in place, so offsets and line numbers outside them are unchanged."""spans = [m.span(1) for m in _CPP_CLASS_HEAD_MACRO_RE.finditer(source)]+ spans += _ctor_macro_spans(source)+ spans += _macro_line_spans(source)+ spans += [m.span(1) for m in _CPP_TEMPLATE_PREFIX_MACRO_RE.finditer(source)]spans += [m.span()for m in _CPP_TRAILING_ANNOTATION_RE.finditer(source)diff --git a/tests/unit/ingestion/parser/test_cpp_source_prep.py b/tests/unit/ingestion/parser/test_cpp_source_prep.pyindex 3f75e819e..df965c8ef 100644--- a/tests/unit/ingestion/parser/test_cpp_source_prep.py+++ b/tests/unit/ingestion/parser/test_cpp_source_prep.py@@ -1,10 +1,10 @@-"""C++ source sanitizer: class-head export macros and thread-safety annotations."""+"""C++ source sanitizer: macros the grammar misreads and thread-safety annotations."""from __future__ import annotationsimport pytest-from repowise.core.ingestion.lang_helpers.source_prep import prepare_cpp_source+from repowise.core.ingestion.lang_helpers.source_prep import _close_paren_of, prepare_cpp_sourcefrom repowise.core.ingestion.parser import ASTParserfrom tests.unit.ingestion.parser._helpers import _make_file_info@@ -30,6 +30,50 @@ ANNOTATED_CLASS = b"""class DBImpl {};"""+MACRO_CONSTRUCTOR = b"""struct size_padding {+ unsigned size;++ FMT_CONSTEXPR size_padding(int num_digits, unsigned prefix)+ : size(prefix + num_digits) {+ if (size > 0) size = 1;+ }+};++template <typename T> auto write_int(T out, int value) -> T {+ return out;+}++class buffer {+ public:+ FMT_CONSTEXPR20 buffer(buffer&& other) noexcept : size_(other.size_) {}+ auto size() const -> int { return size_; }++ private:+ int size_;+};+"""++MACRO_LINES = b"""FMT_BEGIN_NAMESPACE+FMT_PRAGMA_CLANG(diagnostic push)++FMT_EXPORT+template <typename Char> class basic_string_view {+ public:+ auto size() const -> int { return size_; }++ private:+ int size_;+};++FMT_EXPORT+enum class align { none, left };++namespace detail {+template <typename T> auto to_unsigned(T value) -> unsigned { return value; }+} // namespace detail+FMT_END_NAMESPACE+"""+def _symbols(parser: ASTParser, source: bytes) -> set[tuple[str | None, str, str]]:result = parser.parse_file(_make_file_info("cpp_pkg/db.h", "cpp"), source)@@ -49,6 +93,17 @@ class TestCppSourcePrep:assert "EXCLUSIVE_LOCKS_REQUIRED" not in namesassert "GUARDED_BY" not in names+ def test_macro_constructor_with_initializers_closes_its_class(self, parser: ASTParser) -> None:+ symbols = _symbols(parser, MACRO_CONSTRUCTOR)+ assert (None, "write_int", "function") in symbols+ assert ("buffer", "size", "method") in symbols++ def test_macro_lines_do_not_type_the_next_declaration(self, parser: ASTParser) -> None:+ symbols = _symbols(parser, MACRO_LINES)+ assert ("basic_string_view", "size", "method") in symbols+ assert any(name == "to_unsigned" for _, name, _ in symbols)+ assert not any(name.startswith("FMT_") for _, name, _ in symbols)+@pytest.mark.parametrize(("source", "macro"),[@@ -60,6 +115,17 @@ class TestCppSourcePrep:(b" void F() const REQUIRES(a, f(b));", b"REQUIRES(a, f(b))"),(b" void F() NO_THREAD_SAFETY_ANALYSIS {", b"NO_THREAD_SAFETY_ANALYSIS"),(b" Foo* p ABSL_PT_GUARDED_BY(mu) = nullptr;", b"ABSL_PT_GUARDED_BY(mu)"),+ (b" FMT_CONSTEXPR Pad(int n) : n_(n) {}", b"FMT_CONSTEXPR "),+ (b" constexpr FMT_INLINE Args(const S& s)\n : d_(s) {}", b"FMT_INLINE "),+ (b" A_API B_INLINE explicit Buf(Buf&& o) noexcept\n : base(o) {", b"A_API B_INLINE "),+ (b"FMT_BEGIN_NAMESPACE\nnamespace detail {", b"FMT_BEGIN_NAMESPACE"),+ (b" Q_OBJECT\n public:", b"Q_OBJECT"),+ (b"FMT_EXPORT\r\ntemplate <typename T> struct F;", b"FMT_EXPORT"),+ (b"FMT_EXPORT template <typename T> class S {", b"FMT_EXPORT "),+ (+ b'FMT_PRAGMA_CLANG(diagnostic error "-Wx")\n\nenum class E {};',+ b'FMT_PRAGMA_CLANG(diagnostic error "-Wx")',+ ),],)def test_blanks_macro_spans_in_place(self, source: bytes, macro: bytes) -> None:@@ -84,6 +150,29 @@ class TestCppSourcePrep:b" struct TM tm{0};",b" int NO_THREAD_SAFETY_ANALYSIS_X;",b"#define F(x) RELEASE(x)",+ # A macro-typed declaration without an initializer list stays.+ b" FMT_CONSTEXPR Pad(int n) {}",+ b" HANDLE h(open(path));",+ b" FOO_T x(1); y = c ? a : b;",+ b" MACRO_T Make(int n) const;",+ b" Base::Base(int n) : n_(n) {}",+ b" Q_INVOKABLE T f(int n) :: x;",+ # A macro line that opens a body, ends the file, continues an+ # expression, or sits in a statement is left alone.+ b"TEST(Suite, Name)\n{",+ b" Py_BEGIN_ALLOW_THREADS\n {",+ b"NAPI_MODULE(addon, Init)\n}",+ b"NAPI_MODULE(addon, Init)\n",+ b" LOG_STATE(x)\n run();",+ b"REGISTER_OP(Foo)\n .Input(x);",+ b"#define A \\\n B_MACRO \\\n c",+ b" int x = FLAG_A |\n FLAG_B;",+ b"enum E {\n RED,\n BLUE\n};",+ b"BOOL\nWINAPI\nDllMain(HINSTANCE h)\n{",+ b"HRESULT\r\nFoo::Open(int x) {",+ b"NAPI_MODULE(addon, Init)\nstatic int later;",+ b"#undef A\\\nPI\nstruct API Fwd;",+ b'#define POP() _Pragma("pop_macro(\\"API\\")")\nPOP()\nstruct API Fwd;',# A ``)`` with no ``(`` in reach is not a declarator.b" x) GUARDED_BY(mu_);",b" void F(" + b"a," * 2100 + b"b) REQUIRES(mu);",@@ -98,3 +187,25 @@ class TestCppSourcePrep:assert len(out) == len(source)assert out.splitlines() != source.splitlines()assert out.count(b"\n") == source.count(b"\n")+++class TestCloseParenOf:+ def test_returns_the_matching_close(self) -> None:+ src = b"Pad(int n) : n_(n) {}"+ assert _close_paren_of(src, src.index(b"(")) == src.index(b")")++ def test_nested_parens_return_the_outer_close(self) -> None:+ src = b"Pad(f(g(x))) : n_(1) {}"+ assert _close_paren_of(src, src.index(b"(")) == src.index(b"))) ") + 2++ def test_unterminated_paren_returns_minus_one(self) -> None:+ src = b"FMT_CONSTEXPR Pad(int n"+ assert _close_paren_of(src, src.index(b"(")) == -1++ def test_close_beyond_the_scan_limit_returns_minus_one(self) -> None:+ src = b"Pad(" + b"a," * 2100 + b"b)"+ assert _close_paren_of(src, 3) == -1++ def test_unterminated_ctor_macro_run_blanks_nothing(self) -> None:+ src = b"FMT_CONSTEXPR Pad(int n"+ assert prepare_cpp_source(src) is src