Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(ingestion): stop C++ attribute and namespace macros from hiding declarations (#4814)

Swati Ahujaa9101000c7f5
diff --git a/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py b/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py
index a48f2f665..794510efa 100644
--- a/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py
+++ b/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py
@@ -4,6 +4,8 @@ from __future__ import annotations
import re
+from ..extractors.synthetic_symbols.cpp_macros import _FAST_REJECT_TOKENS as _SYNTHESIZED_MACROS
+
_PASCAL_USES_IN_CLAUSE_RE = re.compile(rb"\bin\b[ \t]*'(?:[^'\r\n]|'')*'")
@@ -165,6 +167,50 @@ _CPP_TRAILING_ANNOTATION_RE = re.compile(
re.MULTILINE,
)
+# An attribute macro before a constructor (``FMT_CONSTEXPR Foo(int x) : x_(x)
+# {}``). The grammar reads the macro as a return type, a typed function may not
+# have a member-initializer list, and recovery then leaves the class open to
+# the end of the file, so every later function loses its symbol. Group 1 is
+# the macro run; the ``: init`` after the parameter list, checked in code, is
+# what makes the declaration a constructor and not ``TYPE var(args);``.
+_CPP_CTOR_MACRO_RE = re.compile(
+ rb"^[ \t]*(?:(?:constexpr|explicit|inline)[ \t]+)*"
+ rb"((?:[A-Z][A-Z0-9_]+[ \t]+)+)"
+ rb"(?:(?:constexpr|explicit|inline)[ \t]+)*[A-Za-z_]\w*[ \t]*\(",
+ re.MULTILINE,
+)
+_CPP_CTOR_INIT_RE = re.compile(rb"\s*(?:noexcept\b\s*(?:\([^()]*\))?\s*)?:(?!:)")
+
+# A macro alone on its line (``FMT_BEGIN_NAMESPACE``, ``FMT_EXPORT``, ``Q_OBJECT``,
+# ``FMT_PRAGMA_CLANG(diagnostic push)``) expands to a namespace opener, an
+# attribute or nothing. With no ``;`` the grammar reads it as the type of the
+# declaration after it, which renames or swallows that declaration. Group 1 is
+# a bare name at any indent; group 2 is a call-shaped one at column 0 that a
+# declaration follows, so an indented ``LOG(x)`` statement, a ``TEST(a, b)``
+# opening a body and a last enumerator before ``}`` all stay. Neither is
+# blanked before a typeless function head (``BOOL\nWINAPI\nDllMain(...)``),
+# where the macro is the return type, nor is a registration macro whose
+# text the synthetic-symbol pass reads (``NAPI_MODULE(name, Init)``). A head
+# has a lowercase letter, so another macro line (``FMT_PRAGMA(x)``) is none.
+# A call-shaped macro the file defines itself stays too: it may wrap a
+# ``push_macro``/``pop_macro`` pragma the macro-state pass reads.
+_CPP_TYPELESS_HEAD = (
+ rb"[ \t]*\r?\n(?:[\r\n \t]*[A-Z][A-Z0-9_]+[ \t]*\r?\n)*"
+ rb"[\r\n \t]*~?(?=[\w:]*[a-z])[A-Za-z_][\w:]*[ \t]*\("
+)
+_CPP_MACRO_LINE_RE = re.compile(
+ rb"^(?<!\\\n)(?<!\\\r\n)(?:[ \t]*([A-Z][A-Z0-9_]+)(?=[ \t]*\r?\n[\r\n \t]*[A-Za-z_#/])"
+ rb"|(?!" + b"|".join(re.escape(t.encode()) for t in _SYNTHESIZED_MACROS) + rb")"
+ rb"(([A-Z][A-Z0-9_]+)\((?:[^()\r\n]|\([^()\r\n]*\))*\))(?=[ \t]*\r?\n[\r\n \t]*[A-Za-z_]))"
+ rb"(?!" + _CPP_TYPELESS_HEAD + rb")",
+ re.MULTILINE,
+)
+_CPP_FUNCTION_MACRO_DEFINE_RE = re.compile(rb"^[ \t]*#[ \t]*define[ \t]+(\w+)\(", re.MULTILINE)
+# The same macros written ahead of a template on its line (``FMT_EXPORT template <...>``).
+_CPP_TEMPLATE_PREFIX_MACRO_RE = re.compile(
+ rb"^[ \t]*((?:[A-Z][A-Z0-9_]+[ \t]+)+)(?=template\b)", re.MULTILINE
+)
+
# Words that put a following ``RELEASE(p)`` in statement position, where it
# is a real call and must stay.
_CPP_STATEMENT_WORDS = frozenset(
@@ -204,6 +250,38 @@ def _open_paren_of(source: bytes, close: int) -> int:
return -1
+def _close_paren_of(source: bytes, open_: int) -> int:
+ """Index of the ``)`` matching the ``(`` at *open_*, or -1."""
+ depth = 0
+ for i in range(open_, min(open_ + _CPP_PAREN_SCAN_LIMIT, len(source))):
+ if source[i] == ord("("):
+ depth += 1
+ elif source[i] == ord(")"):
+ depth -= 1
+ if depth == 0:
+ return i
+ return -1
+
+
+def _ctor_macro_spans(source: bytes) -> list[tuple[int, int]]:
+ """Macro runs before a constructor that has a member-initializer list."""
+ spans = []
+ for m in _CPP_CTOR_MACRO_RE.finditer(source):
+ close = _close_paren_of(source, m.end() - 1)
+ if close >= 0 and _CPP_CTOR_INIT_RE.match(source, close + 1):
+ spans.append(m.span(1))
+ return spans
+
+
+def _macro_line_spans(source: bytes) -> list[tuple[int, int]]:
+ """Macro-only lines, less call-shaped ones naming a macro this file defines."""
+ matches = list(_CPP_MACRO_LINE_RE.finditer(source))
+ if not matches:
+ return []
+ local = set(_CPP_FUNCTION_MACRO_DEFINE_RE.findall(source))
+ return [m.span(1) if m.group(1) else m.span(2) for m in matches if m.group(3) not in local]
+
+
def _ends_declarator(source: bytes, end: int) -> bool:
"""True when the text before *end* is a declarator, not a statement head.
@@ -245,14 +323,19 @@ def cpp_class_head_macros(source: bytes) -> frozenset[str]:
def prepare_cpp_source(source: bytes) -> bytes:
- """Blank class-head export macros and trailing thread-safety annotations.
-
- Both are macros the grammar cannot see expanded, and both make it build
- the wrong tree: a missing class and a lost first method, or a hidden field
- and a method named after the annotation. Spans are blanked in place, so
- offsets and line numbers outside them are unchanged.
+ """Blank class-head export macros, constructor attribute macros,
+ macro-only lines and trailing thread-safety annotations.
+
+ All are macros the grammar cannot see expanded, and all make it build the
+ wrong tree: a missing class and a lost first method, a class left open
+ over the rest of the file, a declaration read as typed by the macro before
+ it, or a hidden field and a method named after the annotation. Spans are
+ blanked in place, so offsets and line numbers outside them are unchanged.
"""
spans = [m.span(1) for m in _CPP_CLASS_HEAD_MACRO_RE.finditer(source)]
+ spans += _ctor_macro_spans(source)
+ spans += _macro_line_spans(source)
+ spans += [m.span(1) for m in _CPP_TEMPLATE_PREFIX_MACRO_RE.finditer(source)]
spans += [
m.span()
for m in _CPP_TRAILING_ANNOTATION_RE.finditer(source)
diff --git a/tests/unit/ingestion/parser/test_cpp_source_prep.py b/tests/unit/ingestion/parser/test_cpp_source_prep.py
index 3f75e819e..df965c8ef 100644
--- a/tests/unit/ingestion/parser/test_cpp_source_prep.py
+++ b/tests/unit/ingestion/parser/test_cpp_source_prep.py
@@ -1,10 +1,10 @@
-"""C++ source sanitizer: class-head export macros and thread-safety annotations."""
+"""C++ source sanitizer: macros the grammar misreads and thread-safety annotations."""
from __future__ import annotations
import pytest
-from repowise.core.ingestion.lang_helpers.source_prep import prepare_cpp_source
+from repowise.core.ingestion.lang_helpers.source_prep import _close_paren_of, prepare_cpp_source
from repowise.core.ingestion.parser import ASTParser
from tests.unit.ingestion.parser._helpers import _make_file_info
@@ -30,6 +30,50 @@ ANNOTATED_CLASS = b"""class DBImpl {
};
"""
+MACRO_CONSTRUCTOR = b"""struct size_padding {
+ unsigned size;
+
+ FMT_CONSTEXPR size_padding(int num_digits, unsigned prefix)
+ : size(prefix + num_digits) {
+ if (size > 0) size = 1;
+ }
+};
+
+template <typename T> auto write_int(T out, int value) -> T {
+ return out;
+}
+
+class buffer {
+ public:
+ FMT_CONSTEXPR20 buffer(buffer&& other) noexcept : size_(other.size_) {}
+ auto size() const -> int { return size_; }
+
+ private:
+ int size_;
+};
+"""
+
+MACRO_LINES = b"""FMT_BEGIN_NAMESPACE
+FMT_PRAGMA_CLANG(diagnostic push)
+
+FMT_EXPORT
+template <typename Char> class basic_string_view {
+ public:
+ auto size() const -> int { return size_; }
+
+ private:
+ int size_;
+};
+
+FMT_EXPORT
+enum class align { none, left };
+
+namespace detail {
+template <typename T> auto to_unsigned(T value) -> unsigned { return value; }
+} // namespace detail
+FMT_END_NAMESPACE
+"""
+
def _symbols(parser: ASTParser, source: bytes) -> set[tuple[str | None, str, str]]:
result = parser.parse_file(_make_file_info("cpp_pkg/db.h", "cpp"), source)
@@ -49,6 +93,17 @@ class TestCppSourcePrep:
assert "EXCLUSIVE_LOCKS_REQUIRED" not in names
assert "GUARDED_BY" not in names
+ def test_macro_constructor_with_initializers_closes_its_class(self, parser: ASTParser) -> None:
+ symbols = _symbols(parser, MACRO_CONSTRUCTOR)
+ assert (None, "write_int", "function") in symbols
+ assert ("buffer", "size", "method") in symbols
+
+ def test_macro_lines_do_not_type_the_next_declaration(self, parser: ASTParser) -> None:
+ symbols = _symbols(parser, MACRO_LINES)
+ assert ("basic_string_view", "size", "method") in symbols
+ assert any(name == "to_unsigned" for _, name, _ in symbols)
+ assert not any(name.startswith("FMT_") for _, name, _ in symbols)
+
@pytest.mark.parametrize(
("source", "macro"),
[
@@ -60,6 +115,17 @@ class TestCppSourcePrep:
(b" void F() const REQUIRES(a, f(b));", b"REQUIRES(a, f(b))"),
(b" void F() NO_THREAD_SAFETY_ANALYSIS {", b"NO_THREAD_SAFETY_ANALYSIS"),
(b" Foo* p ABSL_PT_GUARDED_BY(mu) = nullptr;", b"ABSL_PT_GUARDED_BY(mu)"),
+ (b" FMT_CONSTEXPR Pad(int n) : n_(n) {}", b"FMT_CONSTEXPR "),
+ (b" constexpr FMT_INLINE Args(const S& s)\n : d_(s) {}", b"FMT_INLINE "),
+ (b" A_API B_INLINE explicit Buf(Buf&& o) noexcept\n : base(o) {", b"A_API B_INLINE "),
+ (b"FMT_BEGIN_NAMESPACE\nnamespace detail {", b"FMT_BEGIN_NAMESPACE"),
+ (b" Q_OBJECT\n public:", b"Q_OBJECT"),
+ (b"FMT_EXPORT\r\ntemplate <typename T> struct F;", b"FMT_EXPORT"),
+ (b"FMT_EXPORT template <typename T> class S {", b"FMT_EXPORT "),
+ (
+ b'FMT_PRAGMA_CLANG(diagnostic error "-Wx")\n\nenum class E {};',
+ b'FMT_PRAGMA_CLANG(diagnostic error "-Wx")',
+ ),
],
)
def test_blanks_macro_spans_in_place(self, source: bytes, macro: bytes) -> None:
@@ -84,6 +150,29 @@ class TestCppSourcePrep:
b" struct TM tm{0};",
b" int NO_THREAD_SAFETY_ANALYSIS_X;",
b"#define F(x) RELEASE(x)",
+ # A macro-typed declaration without an initializer list stays.
+ b" FMT_CONSTEXPR Pad(int n) {}",
+ b" HANDLE h(open(path));",
+ b" FOO_T x(1); y = c ? a : b;",
+ b" MACRO_T Make(int n) const;",
+ b" Base::Base(int n) : n_(n) {}",
+ b" Q_INVOKABLE T f(int n) :: x;",
+ # A macro line that opens a body, ends the file, continues an
+ # expression, or sits in a statement is left alone.
+ b"TEST(Suite, Name)\n{",
+ b" Py_BEGIN_ALLOW_THREADS\n {",
+ b"NAPI_MODULE(addon, Init)\n}",
+ b"NAPI_MODULE(addon, Init)\n",
+ b" LOG_STATE(x)\n run();",
+ b"REGISTER_OP(Foo)\n .Input(x);",
+ b"#define A \\\n B_MACRO \\\n c",
+ b" int x = FLAG_A |\n FLAG_B;",
+ b"enum E {\n RED,\n BLUE\n};",
+ b"BOOL\nWINAPI\nDllMain(HINSTANCE h)\n{",
+ b"HRESULT\r\nFoo::Open(int x) {",
+ b"NAPI_MODULE(addon, Init)\nstatic int later;",
+ b"#undef A\\\nPI\nstruct API Fwd;",
+ b'#define POP() _Pragma("pop_macro(\\"API\\")")\nPOP()\nstruct API Fwd;',
# A ``)`` with no ``(`` in reach is not a declarator.
b" x) GUARDED_BY(mu_);",
b" void F(" + b"a," * 2100 + b"b) REQUIRES(mu);",
@@ -98,3 +187,25 @@ class TestCppSourcePrep:
assert len(out) == len(source)
assert out.splitlines() != source.splitlines()
assert out.count(b"\n") == source.count(b"\n")
+
+
+class TestCloseParenOf:
+ def test_returns_the_matching_close(self) -> None:
+ src = b"Pad(int n) : n_(n) {}"
+ assert _close_paren_of(src, src.index(b"(")) == src.index(b")")
+
+ def test_nested_parens_return_the_outer_close(self) -> None:
+ src = b"Pad(f(g(x))) : n_(1) {}"
+ assert _close_paren_of(src, src.index(b"(")) == src.index(b"))) ") + 2
+
+ def test_unterminated_paren_returns_minus_one(self) -> None:
+ src = b"FMT_CONSTEXPR Pad(int n"
+ assert _close_paren_of(src, src.index(b"(")) == -1
+
+ def test_close_beyond_the_scan_limit_returns_minus_one(self) -> None:
+ src = b"Pad(" + b"a," * 2100 + b"b)"
+ assert _close_paren_of(src, 3) == -1
+
+ def test_unterminated_ctor_macro_run_blanks_nothing(self) -> None:
+ src = b"FMT_CONSTEXPR Pad(int n"
+ assert prepare_cpp_source(src) is src