Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(ingestion): parse C++ classes behind export macros and fields behind thread-safety annotations (#4785)
Raghav Chamadiya36d040c3185c
diff --git a/packages/core/src/repowise/core/ingestion/cpp_export_macros.py b/packages/core/src/repowise/core/ingestion/cpp_export_macros.pyindex bdca57a4e..9f00fe126 100644--- a/packages/core/src/repowise/core/ingestion/cpp_export_macros.py+++ b/packages/core/src/repowise/core/ingestion/cpp_export_macros.py@@ -183,9 +183,16 @@ class CppExportTypes:self.parents[range_node.id] = name-def collect_cpp_export_types(matches: list[dict], src: str) -> CppExportTypes:- """Recover every macro-decorated type the query matched in one C++ file."""+def collect_cpp_export_types(+ matches: list[dict], src: str, blanked_macros: frozenset[str] = frozenset()+) -> CppExportTypes:+ """Recover every macro-decorated type the query matched in one C++ file.++ *blanked_macros* are class-head macros the source sanitizer already+ removed; their ``#define`` lines are suppressed like a body-form match's.+ """found = CppExportTypes(namespace_ids=_misread_namespace_ids(matches))+ found.macro_names.update(blanked_macros)cpp_export_matches = [capture_dictfor capture_dict in matchesdiff --git a/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py b/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.pyindex 10816f00d..a48f2f665 100644--- a/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py+++ b/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py@@ -1,4 +1,4 @@-"""Byte-preserving source sanitizers for grammar gaps in Pascal, Objective-C and Apex."""+"""Byte-preserving source sanitizers for grammar gaps in Pascal, Objective-C, Apex and C++."""from __future__ import annotations@@ -29,11 +29,15 @@ def _blank_matches(source: bytes, pattern: re.Pattern[bytes]) -> bytes:and never a newline, so no pattern can consume a line break it did notmatch."""- if not pattern.search(source):+ return _blank_spans(source, [match.span() for match in pattern.finditer(source)])+++def _blank_spans(source: bytes, spans: list[tuple[int, int]]) -> bytes:+ """Overwrite each ``(start, end)`` span with spaces, keeping the length."""+ if not spans:return sourceout = bytearray(source)- for match in pattern.finditer(source):- start, end = match.span()+ for start, end in spans:out[start:end] = b" " * (end - start)return bytes(out)@@ -126,3 +130,132 @@ _APEX_NAMESPACE_TOKEN_RE = re.compile(rb"%%%[A-Z_]+%%%")def prepare_apex_source(source: bytes) -> bytes:"""Blank CumulusCI namespace tokens in place, preserving offsets."""return _blank_matches(source, _APEX_NAMESPACE_TOKEN_RE)+++# ``class LEVELDB_EXPORT DB {``: tree-sitter-cpp reads the ALL-CAPS export+# macro as the class name and ``DB`` as a declarator, so the class node is+# lost and its first method is dropped. Group 1 is the macro run; it must be+# followed by the real name (template arguments allowed) and a body or base+# clause, so ``class DB {``, ``class DB final {`` and a bodyless forward+# declaration, which already parses, never match. Call-shaped forms like+# ``CAPABILITY("mutex")`` qualify too. The body brace must follow whitespace,+# so a brace-initialized variable (``struct TM tm{0};``) keeps its type.+_CPP_CLASS_HEAD_MACRO_RE = re.compile(+ rb"\b(?:class|struct)[ \t]+((?:[A-Z][A-Z0-9_]*(?:\([^()\r\n]*\))?[ \t]+)+)"+ rb"(?!final\b)[A-Za-z_]\w*(?:<[^;{}]*>)?(?:[ \t]+final)?(?=\s+\{|\s*:(?!:))"+)++# Clang thread-safety annotations written after a declarator+# (``int x_ GUARDED_BY(mu_);``, ``void F() EXCLUSIVE_LOCKS_REQUIRED(mu_);``).+# The grammar reads them as a second declarator, which hides the field and+# renames the method after the macro. Enumerated by name because the plain+# ``IDENT(...)`` shape is also an ordinary call.+_CPP_TRAILING_ANNOTATION_RE = re.compile(+ rb"(?<!\w)(?:ABSL_)?(?:"+ rb"(?:PT_GUARDED_BY|GUARDED_BY|ACQUIRED_AFTER|ACQUIRED_BEFORE"+ rb"|REQUIRES|REQUIRES_SHARED|EXCLUSIVE_LOCKS_REQUIRED|SHARED_LOCKS_REQUIRED"+ rb"|LOCKS_EXCLUDED|EXCLUDES|ACQUIRE|ACQUIRE_SHARED|RELEASE|RELEASE_SHARED"+ rb"|RELEASE_GENERIC|TRY_ACQUIRE|TRY_ACQUIRE_SHARED|EXCLUSIVE_LOCK_FUNCTION"+ rb"|SHARED_LOCK_FUNCTION|UNLOCK_FUNCTION|EXCLUSIVE_TRYLOCK_FUNCTION"+ rb"|SHARED_TRYLOCK_FUNCTION|ASSERT_CAPABILITY|ASSERT_SHARED_CAPABILITY"+ rb"|ASSERT_EXCLUSIVE_LOCK|ASSERT_SHARED_LOCK|RETURN_CAPABILITY|LOCK_RETURNED)"+ rb"\((?:[^()\r\n]|\([^()\r\n]*\))*\)"+ rb"|NO_THREAD_SAFETY_ANALYSIS\b)"+ rb"(?=[ \t]*(?:[;{=,:\w]|//|/\*|\r?$))",+ re.MULTILINE,+)++# Words that put a following ``RELEASE(p)`` in statement position, where it+# is a real call and must stay.+_CPP_STATEMENT_WORDS = frozenset(+ {b"return", b"else", b"do", b"case", b"throw", b"co_return", b"co_yield", b"goto"}+ | {b"if", b"while", b"for", b"switch", b"catch", b"sizeof", b"new", b"delete"}+)+# How far back a ``)`` looks for its ``(``: longer than any parameter list,+# short enough that a stray paren in a comment cannot scan the whole file.+_CPP_PAREN_SCAN_LIMIT = 4096+++def _skip_space_back(source: bytes, end: int) -> int:+ while end > 0 and source[end - 1] in b" \t\r\n":+ end -= 1+ return end+++def _word_before(source: bytes, end: int) -> bytes:+ """Return the identifier that ends at *end*, skipping whitespace before it."""+ end = _skip_space_back(source, end)+ start = end+ while start > 0 and (chr(source[start - 1]).isalnum() or source[start - 1] == ord("_")):+ start -= 1+ return source[start:end]+++def _open_paren_of(source: bytes, close: int) -> int:+ """Index of the ``(`` matching the ``)`` at *close*, or -1."""+ depth = 0+ for i in range(close, max(close - _CPP_PAREN_SCAN_LIMIT, -1), -1):+ if source[i] == ord(")"):+ depth += 1+ elif source[i] == ord("("):+ depth -= 1+ if depth == 0:+ return i+ return -1+++def _ends_declarator(source: bytes, end: int) -> bool:+ """True when the text before *end* is a declarator, not a statement head.++ ``void F() REQUIRES(mu)`` and ``int x_ GUARDED_BY(mu)`` qualify;+ ``if (p) RELEASE(p);`` and ``return RELEASE(p);`` do not, and neither+ does a macro at the start of a statement or on a preprocessor line. The+ annotation may sit on the line after its declarator.+ """+ line_start = source.rfind(b"\n", 0, end) + 1+ if source[line_start:end].lstrip().startswith(b"#"):+ return False+ end = _skip_space_back(source, end)+ if end == 0:+ return False+ prev = source[end - 1 : end]+ if prev == b")":+ end = _open_paren_of(source, end - 1)+ if end < 0:+ return False+ elif not (prev.isalnum() or prev in b"_]"):+ return False+ return _word_before(source, end) not in _CPP_STATEMENT_WORDS+++_CALL_ARGS_RE = re.compile(rb"\([^()]*\)")+++def cpp_class_head_macros(source: bytes) -> frozenset[str]:+ """Names of the class-head macros :func:`prepare_cpp_source` blanks.++ The parser reads the blanked bytes, so it takes these from the original+ source to keep hiding each macro's own ``#define`` like other export macros.+ """+ return frozenset(+ name.decode("ascii")+ for m in _CPP_CLASS_HEAD_MACRO_RE.finditer(source)+ for name in _CALL_ARGS_RE.sub(b" ", m.group(1)).split()+ )+++def prepare_cpp_source(source: bytes) -> bytes:+ """Blank class-head export macros and trailing thread-safety annotations.++ Both are macros the grammar cannot see expanded, and both make it build+ the wrong tree: a missing class and a lost first method, or a hidden field+ and a method named after the annotation. Spans are blanked in place, so+ offsets and line numbers outside them are unchanged.+ """+ spans = [m.span(1) for m in _CPP_CLASS_HEAD_MACRO_RE.finditer(source)]+ spans += [+ m.span()+ for m in _CPP_TRAILING_ANNOTATION_RE.finditer(source)+ if _ends_declarator(source, m.start())+ ]+ return _blank_spans(source, spans)diff --git a/packages/core/src/repowise/core/ingestion/parser/__init__.py b/packages/core/src/repowise/core/ingestion/parser/__init__.pyindex 6a935985e..e1e54785c 100644--- a/packages/core/src/repowise/core/ingestion/parser/__init__.py+++ b/packages/core/src/repowise/core/ingestion/parser/__init__.py@@ -219,7 +219,7 @@ class ASTParser:matches = _run_query(query, root) if query is not None else []symbols = _finalize_symbols(- _extract_symbols(matches, config, file_info, src), root, src, file_info+ _extract_symbols(matches, config, file_info, src, original_source), root, src, file_info)_restore_kotlin_annotations(symbols, recovered_annotations)imports = _extract_imports(matches, config, file_info, src)diff --git a/packages/core/src/repowise/core/ingestion/parser/symbols.py b/packages/core/src/repowise/core/ingestion/parser/symbols.pyindex d41caea89..9185397a8 100644--- a/packages/core/src/repowise/core/ingestion/parser/symbols.py+++ b/packages/core/src/repowise/core/ingestion/parser/symbols.py@@ -23,6 +23,7 @@ from ..extractors.cobol import cobol_condition_parentfrom ..extractors.synthetic_symbols import extract_synthetic_symbolsfrom ..extractors.synthetic_symbols.csharp_extension_blocks import drop_extension_block_symbolsfrom ..extractors.visibility import py_module_all_names, ts_deferred_export_names+from ..lang_helpers.source_prep import cpp_class_head_macrosfrom ..language_configs import LanguageConfigfrom ..models import FileInfo, Symbolfrom ..parser_helpers import (@@ -220,6 +221,7 @@ def _extract_symbols(config: LanguageConfig,file_info: FileInfo,src: str,+ original_source: bytes = b"",) -> list[Symbol]:language = file_info.languagesymbols: list[Symbol] = []@@ -244,7 +246,9 @@ def _extract_symbols(if language == "python":py_all_names = py_module_all_names(src)cpp_exports = (- collect_cpp_export_types(matches, src) if language == "cpp" else CppExportTypes()+ collect_cpp_export_types(matches, src, cpp_class_head_macros(original_source))+ if language == "cpp"+ else CppExportTypes())file = _SymbolPass(config, file_info, src, cpp_exports, ts_deferred_exports, py_all_names)diff --git a/packages/core/src/repowise/core/ingestion/sfc_source.py b/packages/core/src/repowise/core/ingestion/sfc_source.pyindex da813651e..c0a9b949a 100644--- a/packages/core/src/repowise/core/ingestion/sfc_source.py+++ b/packages/core/src/repowise/core/ingestion/sfc_source.py@@ -900,4 +900,8 @@ def prepare_source(language: str, source: bytes, *, path: str | None = None) ->from .cobol import prepare_cobol_sourcereturn prepare_cobol_source(source)+ if language == "cpp" and source:+ from .lang_helpers.source_prep import prepare_cpp_source++ return prepare_cpp_source(source)return sourcediff --git a/tests/unit/health/test_complexity_walker.py b/tests/unit/health/test_complexity_walker.pyindex 6632412de..deb3567a3 100644--- a/tests/unit/health/test_complexity_walker.py+++ b/tests/unit/health/test_complexity_walker.py@@ -678,6 +678,26 @@ EXPORT_API const struct S *fn5(int a) { if (a) { return 0; } return 0; }assert expected <= {r.name for r in results}+def test_cpp_export_macro_class_and_annotated_fields_are_measured(tmp_path):+ # ``class API Big {`` used to lose the class and its first method, and+ # ``int x_ GUARDED_BY(mu_);`` hid the field from cohesion.+ source = b"""class LIB_API Big {+ public:+ void a() { x_ = 1; }+ void b() { y_ = 2; }+ private:+ int x_ GUARDED_BY(mu_);+ int y_ GUARDED_BY(mu_);+};+"""+ _require_language("cpp")+ p = tmp_path / "big.h"+ p.write_bytes(source)+ classes = {c.name: c for c in walk_file(str(p), "cpp", source).classes}+ assert classes["Big"].method_count == 2+ assert classes["Big"].field_count == 2++def _names_by_line(tmp_path, language: str, source: bytes) -> dict[int, str]:p = tmp_path / ("unit.cpp" if language == "cpp" else "unit.c")p.write_bytes(source)diff --git a/tests/unit/ingestion/parser/test_cpp_source_prep.py b/tests/unit/ingestion/parser/test_cpp_source_prep.pynew file mode 100644index 000000000..3f75e819e--- /dev/null+++ b/tests/unit/ingestion/parser/test_cpp_source_prep.py@@ -0,0 +1,100 @@+"""C++ source sanitizer: class-head export macros and thread-safety annotations."""++from __future__ import annotations++import pytest++from repowise.core.ingestion.lang_helpers.source_prep import prepare_cpp_source+from repowise.core.ingestion.parser import ASTParser+from tests.unit.ingestion.parser._helpers import _make_file_info++EXPORTED_CLASS = b"""namespace leveldb {+class LEVELDB_EXPORT DB {+ public:+ static Status Open(const Options& options, DB** dbptr);+ DB() = default;+ virtual ~DB();+};+} // namespace leveldb+"""++ANNOTATED_CLASS = b"""class DBImpl {+ public:+ void CompactMemTable() EXCLUSIVE_LOCKS_REQUIRED(mutex_);+ Status Recover(VersionEdit* edit)+ EXCLUSIVE_LOCKS_REQUIRED(mutex_);+ private:+ port::Mutex mutex_;+ MemTable* imm_ GUARDED_BY(mutex_);+ bool has_imm_ GUARDED_BY(mutex_) = false;+};+"""+++def _symbols(parser: ASTParser, source: bytes) -> set[tuple[str | None, str, str]]:+ result = parser.parse_file(_make_file_info("cpp_pkg/db.h", "cpp"), source)+ return {(s.parent_name, s.name, s.kind) for s in result.symbols}+++class TestCppSourcePrep:+ def test_export_macro_class_keeps_its_name_and_first_method(self, parser: ASTParser) -> None:+ symbols = _symbols(parser, EXPORTED_CLASS)+ assert (None, "DB", "class") in symbols+ assert ("DB", "Open", "method") in symbols+ assert not any(name == "LEVELDB_EXPORT" for _, name, _ in symbols)++ def test_annotated_methods_keep_their_names(self, parser: ASTParser) -> None:+ names = {name for _, name, _ in _symbols(parser, ANNOTATED_CLASS)}+ assert {"CompactMemTable", "Recover"} <= names+ assert "EXCLUSIVE_LOCKS_REQUIRED" not in names+ assert "GUARDED_BY" not in names++ @pytest.mark.parametrize(+ ("source", "macro"),+ [+ (b"class FOO_API Bar {", b"FOO_API "),+ (b"struct A B Bar final : X {", b"A B "),+ (b'class CAPABILITY("mutex") Mutex {', b'CAPABILITY("mutex") '),+ (b"class GTEST_API_ M<std::string>\n : public B {", b"GTEST_API_ "),+ (b" int x_ GUARDED_BY(mu_);", b"GUARDED_BY(mu_)"),+ (b" void F() const REQUIRES(a, f(b));", b"REQUIRES(a, f(b))"),+ (b" void F() NO_THREAD_SAFETY_ANALYSIS {", b"NO_THREAD_SAFETY_ANALYSIS"),+ (b" Foo* p ABSL_PT_GUARDED_BY(mu) = nullptr;", b"ABSL_PT_GUARDED_BY(mu)"),+ ],+ )+ def test_blanks_macro_spans_in_place(self, source: bytes, macro: bytes) -> None:+ assert prepare_cpp_source(source) == source.replace(macro, b" " * len(macro))++ @pytest.mark.parametrize(+ "source",+ [+ b"class DB {",+ b"class DB final {",+ b"class DB : public Base {",+ b"struct [[nodiscard]] Foo {",+ b"class alignas(8) Foo {",+ b"class LEVELDB_EXPORT Cache;",+ b"std::vector<A::B> x;",+ b" if (p) RELEASE(p);",+ b" return RELEASE(p);",+ b" RELEASE(p);",+ b"#define GUARDED_BY(x) __attribute__((guarded_by(x)))",+ b"#ifndef NO_THREAD_SAFETY_ANALYSIS",+ b" int x_ MY_GUARD(mu_);",+ b" struct TM tm{0};",+ b" int NO_THREAD_SAFETY_ANALYSIS_X;",+ b"#define F(x) RELEASE(x)",+ # A ``)`` with no ``(`` in reach is not a declarator.+ b" x) GUARDED_BY(mu_);",+ b" void F(" + b"a," * 2100 + b"b) REQUIRES(mu);",+ ],+ )+ def test_leaves_real_code_alone(self, source: bytes) -> None:+ assert prepare_cpp_source(source) is source++ def test_offsets_and_lines_are_unchanged(self) -> None:+ for source in (EXPORTED_CLASS, ANNOTATED_CLASS):+ out = prepare_cpp_source(source)+ assert len(out) == len(source)+ assert out.splitlines() != source.splitlines()+ assert out.count(b"\n") == source.count(b"\n")diff --git a/tests/unit/ingestion/test_cpp_inclass_declarations.py b/tests/unit/ingestion/test_cpp_inclass_declarations.pyindex 4e26b99fb..b4209ca16 100644--- a/tests/unit/ingestion/test_cpp_inclass_declarations.py+++ b/tests/unit/ingestion/test_cpp_inclass_declarations.py@@ -161,12 +161,13 @@ class TestExportMacroClassBodies:The body becomes a ``compound_statement`` and its members statement-context``declaration`` nodes, so none of the ``field_declaration`` patterns reachthem, and ``= 0`` wraps the declarator in an ``init_declarator`` the plain- forward-declaration pattern cannot match either.+ forward-declaration pattern cannot match either. An ALL-CAPS macro is+ blanked before parsing, so these cases use a lower-case one."""def test_a_pure_virtual_member_lands(self) -> None:assert _methods(- "class LEVELDB_EXPORT Iterator {\n"+ "class leveldb_export Iterator {\n"" public:\n"" virtual void Seek(const Slice& t) = 0;\n"" void RegisterCleanup(int a);\n"@@ -180,7 +181,7 @@ class TestExportMacroClassBodies:constructor with a member-init list does too."""source = (- "class LEVELDB_EXPORT R {\n"+ "class leveldb_export R {\n"" public:\n"" size_t Index(size_t i) const { return i + 1; }\n"" virtual void Real() = 0;\n"@@ -195,7 +196,7 @@ class TestExportMacroClassBodies:def test_a_deleted_function_is_not_a_method_declaration(self) -> None:got = _methods(- "class LEVELDB_EXPORT S {\n"+ "class leveldb_export S {\n"" public:\n"" void gone(int n) = delete;\n"" virtual void kept() = 0;\n"