Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(ingestion): parse C++ classes behind export macros and fields behind thread-safety annotations (#4785)

Raghav Chamadiya36d040c3185c
diff --git a/packages/core/src/repowise/core/ingestion/cpp_export_macros.py b/packages/core/src/repowise/core/ingestion/cpp_export_macros.py
index bdca57a4e..9f00fe126 100644
--- a/packages/core/src/repowise/core/ingestion/cpp_export_macros.py
+++ b/packages/core/src/repowise/core/ingestion/cpp_export_macros.py
@@ -183,9 +183,16 @@ class CppExportTypes:
self.parents[range_node.id] = name
-def collect_cpp_export_types(matches: list[dict], src: str) -> CppExportTypes:
- """Recover every macro-decorated type the query matched in one C++ file."""
+def collect_cpp_export_types(
+ matches: list[dict], src: str, blanked_macros: frozenset[str] = frozenset()
+) -> CppExportTypes:
+ """Recover every macro-decorated type the query matched in one C++ file.
+
+ *blanked_macros* are class-head macros the source sanitizer already
+ removed; their ``#define`` lines are suppressed like a body-form match's.
+ """
found = CppExportTypes(namespace_ids=_misread_namespace_ids(matches))
+ found.macro_names.update(blanked_macros)
cpp_export_matches = [
capture_dict
for capture_dict in matches
diff --git a/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py b/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py
index 10816f00d..a48f2f665 100644
--- a/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py
+++ b/packages/core/src/repowise/core/ingestion/lang_helpers/source_prep.py
@@ -1,4 +1,4 @@
-"""Byte-preserving source sanitizers for grammar gaps in Pascal, Objective-C and Apex."""
+"""Byte-preserving source sanitizers for grammar gaps in Pascal, Objective-C, Apex and C++."""
from __future__ import annotations
@@ -29,11 +29,15 @@ def _blank_matches(source: bytes, pattern: re.Pattern[bytes]) -> bytes:
and never a newline, so no pattern can consume a line break it did not
match.
"""
- if not pattern.search(source):
+ return _blank_spans(source, [match.span() for match in pattern.finditer(source)])
+
+
+def _blank_spans(source: bytes, spans: list[tuple[int, int]]) -> bytes:
+ """Overwrite each ``(start, end)`` span with spaces, keeping the length."""
+ if not spans:
return source
out = bytearray(source)
- for match in pattern.finditer(source):
- start, end = match.span()
+ for start, end in spans:
out[start:end] = b" " * (end - start)
return bytes(out)
@@ -126,3 +130,132 @@ _APEX_NAMESPACE_TOKEN_RE = re.compile(rb"%%%[A-Z_]+%%%")
def prepare_apex_source(source: bytes) -> bytes:
"""Blank CumulusCI namespace tokens in place, preserving offsets."""
return _blank_matches(source, _APEX_NAMESPACE_TOKEN_RE)
+
+
+# ``class LEVELDB_EXPORT DB {``: tree-sitter-cpp reads the ALL-CAPS export
+# macro as the class name and ``DB`` as a declarator, so the class node is
+# lost and its first method is dropped. Group 1 is the macro run; it must be
+# followed by the real name (template arguments allowed) and a body or base
+# clause, so ``class DB {``, ``class DB final {`` and a bodyless forward
+# declaration, which already parses, never match. Call-shaped forms like
+# ``CAPABILITY("mutex")`` qualify too. The body brace must follow whitespace,
+# so a brace-initialized variable (``struct TM tm{0};``) keeps its type.
+_CPP_CLASS_HEAD_MACRO_RE = re.compile(
+ rb"\b(?:class|struct)[ \t]+((?:[A-Z][A-Z0-9_]*(?:\([^()\r\n]*\))?[ \t]+)+)"
+ rb"(?!final\b)[A-Za-z_]\w*(?:<[^;{}]*>)?(?:[ \t]+final)?(?=\s+\{|\s*:(?!:))"
+)
+
+# Clang thread-safety annotations written after a declarator
+# (``int x_ GUARDED_BY(mu_);``, ``void F() EXCLUSIVE_LOCKS_REQUIRED(mu_);``).
+# The grammar reads them as a second declarator, which hides the field and
+# renames the method after the macro. Enumerated by name because the plain
+# ``IDENT(...)`` shape is also an ordinary call.
+_CPP_TRAILING_ANNOTATION_RE = re.compile(
+ rb"(?<!\w)(?:ABSL_)?(?:"
+ rb"(?:PT_GUARDED_BY|GUARDED_BY|ACQUIRED_AFTER|ACQUIRED_BEFORE"
+ rb"|REQUIRES|REQUIRES_SHARED|EXCLUSIVE_LOCKS_REQUIRED|SHARED_LOCKS_REQUIRED"
+ rb"|LOCKS_EXCLUDED|EXCLUDES|ACQUIRE|ACQUIRE_SHARED|RELEASE|RELEASE_SHARED"
+ rb"|RELEASE_GENERIC|TRY_ACQUIRE|TRY_ACQUIRE_SHARED|EXCLUSIVE_LOCK_FUNCTION"
+ rb"|SHARED_LOCK_FUNCTION|UNLOCK_FUNCTION|EXCLUSIVE_TRYLOCK_FUNCTION"
+ rb"|SHARED_TRYLOCK_FUNCTION|ASSERT_CAPABILITY|ASSERT_SHARED_CAPABILITY"
+ rb"|ASSERT_EXCLUSIVE_LOCK|ASSERT_SHARED_LOCK|RETURN_CAPABILITY|LOCK_RETURNED)"
+ rb"\((?:[^()\r\n]|\([^()\r\n]*\))*\)"
+ rb"|NO_THREAD_SAFETY_ANALYSIS\b)"
+ rb"(?=[ \t]*(?:[;{=,:\w]|//|/\*|\r?$))",
+ re.MULTILINE,
+)
+
+# Words that put a following ``RELEASE(p)`` in statement position, where it
+# is a real call and must stay.
+_CPP_STATEMENT_WORDS = frozenset(
+ {b"return", b"else", b"do", b"case", b"throw", b"co_return", b"co_yield", b"goto"}
+ | {b"if", b"while", b"for", b"switch", b"catch", b"sizeof", b"new", b"delete"}
+)
+# How far back a ``)`` looks for its ``(``: longer than any parameter list,
+# short enough that a stray paren in a comment cannot scan the whole file.
+_CPP_PAREN_SCAN_LIMIT = 4096
+
+
+def _skip_space_back(source: bytes, end: int) -> int:
+ while end > 0 and source[end - 1] in b" \t\r\n":
+ end -= 1
+ return end
+
+
+def _word_before(source: bytes, end: int) -> bytes:
+ """Return the identifier that ends at *end*, skipping whitespace before it."""
+ end = _skip_space_back(source, end)
+ start = end
+ while start > 0 and (chr(source[start - 1]).isalnum() or source[start - 1] == ord("_")):
+ start -= 1
+ return source[start:end]
+
+
+def _open_paren_of(source: bytes, close: int) -> int:
+ """Index of the ``(`` matching the ``)`` at *close*, or -1."""
+ depth = 0
+ for i in range(close, max(close - _CPP_PAREN_SCAN_LIMIT, -1), -1):
+ if source[i] == ord(")"):
+ depth += 1
+ elif source[i] == ord("("):
+ depth -= 1
+ if depth == 0:
+ return i
+ return -1
+
+
+def _ends_declarator(source: bytes, end: int) -> bool:
+ """True when the text before *end* is a declarator, not a statement head.
+
+ ``void F() REQUIRES(mu)`` and ``int x_ GUARDED_BY(mu)`` qualify;
+ ``if (p) RELEASE(p);`` and ``return RELEASE(p);`` do not, and neither
+ does a macro at the start of a statement or on a preprocessor line. The
+ annotation may sit on the line after its declarator.
+ """
+ line_start = source.rfind(b"\n", 0, end) + 1
+ if source[line_start:end].lstrip().startswith(b"#"):
+ return False
+ end = _skip_space_back(source, end)
+ if end == 0:
+ return False
+ prev = source[end - 1 : end]
+ if prev == b")":
+ end = _open_paren_of(source, end - 1)
+ if end < 0:
+ return False
+ elif not (prev.isalnum() or prev in b"_]"):
+ return False
+ return _word_before(source, end) not in _CPP_STATEMENT_WORDS
+
+
+_CALL_ARGS_RE = re.compile(rb"\([^()]*\)")
+
+
+def cpp_class_head_macros(source: bytes) -> frozenset[str]:
+ """Names of the class-head macros :func:`prepare_cpp_source` blanks.
+
+ The parser reads the blanked bytes, so it takes these from the original
+ source to keep hiding each macro's own ``#define`` like other export macros.
+ """
+ return frozenset(
+ name.decode("ascii")
+ for m in _CPP_CLASS_HEAD_MACRO_RE.finditer(source)
+ for name in _CALL_ARGS_RE.sub(b" ", m.group(1)).split()
+ )
+
+
+def prepare_cpp_source(source: bytes) -> bytes:
+ """Blank class-head export macros and trailing thread-safety annotations.
+
+ Both are macros the grammar cannot see expanded, and both make it build
+ the wrong tree: a missing class and a lost first method, or a hidden field
+ and a method named after the annotation. Spans are blanked in place, so
+ offsets and line numbers outside them are unchanged.
+ """
+ spans = [m.span(1) for m in _CPP_CLASS_HEAD_MACRO_RE.finditer(source)]
+ spans += [
+ m.span()
+ for m in _CPP_TRAILING_ANNOTATION_RE.finditer(source)
+ if _ends_declarator(source, m.start())
+ ]
+ return _blank_spans(source, spans)
diff --git a/packages/core/src/repowise/core/ingestion/parser/__init__.py b/packages/core/src/repowise/core/ingestion/parser/__init__.py
index 6a935985e..e1e54785c 100644
--- a/packages/core/src/repowise/core/ingestion/parser/__init__.py
+++ b/packages/core/src/repowise/core/ingestion/parser/__init__.py
@@ -219,7 +219,7 @@ class ASTParser:
matches = _run_query(query, root) if query is not None else []
symbols = _finalize_symbols(
- _extract_symbols(matches, config, file_info, src), root, src, file_info
+ _extract_symbols(matches, config, file_info, src, original_source), root, src, file_info
)
_restore_kotlin_annotations(symbols, recovered_annotations)
imports = _extract_imports(matches, config, file_info, src)
diff --git a/packages/core/src/repowise/core/ingestion/parser/symbols.py b/packages/core/src/repowise/core/ingestion/parser/symbols.py
index d41caea89..9185397a8 100644
--- a/packages/core/src/repowise/core/ingestion/parser/symbols.py
+++ b/packages/core/src/repowise/core/ingestion/parser/symbols.py
@@ -23,6 +23,7 @@ from ..extractors.cobol import cobol_condition_parent
from ..extractors.synthetic_symbols import extract_synthetic_symbols
from ..extractors.synthetic_symbols.csharp_extension_blocks import drop_extension_block_symbols
from ..extractors.visibility import py_module_all_names, ts_deferred_export_names
+from ..lang_helpers.source_prep import cpp_class_head_macros
from ..language_configs import LanguageConfig
from ..models import FileInfo, Symbol
from ..parser_helpers import (
@@ -220,6 +221,7 @@ def _extract_symbols(
config: LanguageConfig,
file_info: FileInfo,
src: str,
+ original_source: bytes = b"",
) -> list[Symbol]:
language = file_info.language
symbols: list[Symbol] = []
@@ -244,7 +246,9 @@ def _extract_symbols(
if language == "python":
py_all_names = py_module_all_names(src)
cpp_exports = (
- collect_cpp_export_types(matches, src) if language == "cpp" else CppExportTypes()
+ collect_cpp_export_types(matches, src, cpp_class_head_macros(original_source))
+ if language == "cpp"
+ else CppExportTypes()
)
file = _SymbolPass(config, file_info, src, cpp_exports, ts_deferred_exports, py_all_names)
diff --git a/packages/core/src/repowise/core/ingestion/sfc_source.py b/packages/core/src/repowise/core/ingestion/sfc_source.py
index da813651e..c0a9b949a 100644
--- a/packages/core/src/repowise/core/ingestion/sfc_source.py
+++ b/packages/core/src/repowise/core/ingestion/sfc_source.py
@@ -900,4 +900,8 @@ def prepare_source(language: str, source: bytes, *, path: str | None = None) ->
from .cobol import prepare_cobol_source
return prepare_cobol_source(source)
+ if language == "cpp" and source:
+ from .lang_helpers.source_prep import prepare_cpp_source
+
+ return prepare_cpp_source(source)
return source
diff --git a/tests/unit/health/test_complexity_walker.py b/tests/unit/health/test_complexity_walker.py
index 6632412de..deb3567a3 100644
--- a/tests/unit/health/test_complexity_walker.py
+++ b/tests/unit/health/test_complexity_walker.py
@@ -678,6 +678,26 @@ EXPORT_API const struct S *fn5(int a) { if (a) { return 0; } return 0; }
assert expected <= {r.name for r in results}
+def test_cpp_export_macro_class_and_annotated_fields_are_measured(tmp_path):
+ # ``class API Big {`` used to lose the class and its first method, and
+ # ``int x_ GUARDED_BY(mu_);`` hid the field from cohesion.
+ source = b"""class LIB_API Big {
+ public:
+ void a() { x_ = 1; }
+ void b() { y_ = 2; }
+ private:
+ int x_ GUARDED_BY(mu_);
+ int y_ GUARDED_BY(mu_);
+};
+"""
+ _require_language("cpp")
+ p = tmp_path / "big.h"
+ p.write_bytes(source)
+ classes = {c.name: c for c in walk_file(str(p), "cpp", source).classes}
+ assert classes["Big"].method_count == 2
+ assert classes["Big"].field_count == 2
+
+
def _names_by_line(tmp_path, language: str, source: bytes) -> dict[int, str]:
p = tmp_path / ("unit.cpp" if language == "cpp" else "unit.c")
p.write_bytes(source)
diff --git a/tests/unit/ingestion/parser/test_cpp_source_prep.py b/tests/unit/ingestion/parser/test_cpp_source_prep.py
new file mode 100644
index 000000000..3f75e819e
--- /dev/null
+++ b/tests/unit/ingestion/parser/test_cpp_source_prep.py
@@ -0,0 +1,100 @@
+"""C++ source sanitizer: class-head export macros and thread-safety annotations."""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.ingestion.lang_helpers.source_prep import prepare_cpp_source
+from repowise.core.ingestion.parser import ASTParser
+from tests.unit.ingestion.parser._helpers import _make_file_info
+
+EXPORTED_CLASS = b"""namespace leveldb {
+class LEVELDB_EXPORT DB {
+ public:
+ static Status Open(const Options& options, DB** dbptr);
+ DB() = default;
+ virtual ~DB();
+};
+} // namespace leveldb
+"""
+
+ANNOTATED_CLASS = b"""class DBImpl {
+ public:
+ void CompactMemTable() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ Status Recover(VersionEdit* edit)
+ EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ private:
+ port::Mutex mutex_;
+ MemTable* imm_ GUARDED_BY(mutex_);
+ bool has_imm_ GUARDED_BY(mutex_) = false;
+};
+"""
+
+
+def _symbols(parser: ASTParser, source: bytes) -> set[tuple[str | None, str, str]]:
+ result = parser.parse_file(_make_file_info("cpp_pkg/db.h", "cpp"), source)
+ return {(s.parent_name, s.name, s.kind) for s in result.symbols}
+
+
+class TestCppSourcePrep:
+ def test_export_macro_class_keeps_its_name_and_first_method(self, parser: ASTParser) -> None:
+ symbols = _symbols(parser, EXPORTED_CLASS)
+ assert (None, "DB", "class") in symbols
+ assert ("DB", "Open", "method") in symbols
+ assert not any(name == "LEVELDB_EXPORT" for _, name, _ in symbols)
+
+ def test_annotated_methods_keep_their_names(self, parser: ASTParser) -> None:
+ names = {name for _, name, _ in _symbols(parser, ANNOTATED_CLASS)}
+ assert {"CompactMemTable", "Recover"} <= names
+ assert "EXCLUSIVE_LOCKS_REQUIRED" not in names
+ assert "GUARDED_BY" not in names
+
+ @pytest.mark.parametrize(
+ ("source", "macro"),
+ [
+ (b"class FOO_API Bar {", b"FOO_API "),
+ (b"struct A B Bar final : X {", b"A B "),
+ (b'class CAPABILITY("mutex") Mutex {', b'CAPABILITY("mutex") '),
+ (b"class GTEST_API_ M<std::string>\n : public B {", b"GTEST_API_ "),
+ (b" int x_ GUARDED_BY(mu_);", b"GUARDED_BY(mu_)"),
+ (b" void F() const REQUIRES(a, f(b));", b"REQUIRES(a, f(b))"),
+ (b" void F() NO_THREAD_SAFETY_ANALYSIS {", b"NO_THREAD_SAFETY_ANALYSIS"),
+ (b" Foo* p ABSL_PT_GUARDED_BY(mu) = nullptr;", b"ABSL_PT_GUARDED_BY(mu)"),
+ ],
+ )
+ def test_blanks_macro_spans_in_place(self, source: bytes, macro: bytes) -> None:
+ assert prepare_cpp_source(source) == source.replace(macro, b" " * len(macro))
+
+ @pytest.mark.parametrize(
+ "source",
+ [
+ b"class DB {",
+ b"class DB final {",
+ b"class DB : public Base {",
+ b"struct [[nodiscard]] Foo {",
+ b"class alignas(8) Foo {",
+ b"class LEVELDB_EXPORT Cache;",
+ b"std::vector<A::B> x;",
+ b" if (p) RELEASE(p);",
+ b" return RELEASE(p);",
+ b" RELEASE(p);",
+ b"#define GUARDED_BY(x) __attribute__((guarded_by(x)))",
+ b"#ifndef NO_THREAD_SAFETY_ANALYSIS",
+ b" int x_ MY_GUARD(mu_);",
+ b" struct TM tm{0};",
+ b" int NO_THREAD_SAFETY_ANALYSIS_X;",
+ b"#define F(x) RELEASE(x)",
+ # A ``)`` with no ``(`` in reach is not a declarator.
+ b" x) GUARDED_BY(mu_);",
+ b" void F(" + b"a," * 2100 + b"b) REQUIRES(mu);",
+ ],
+ )
+ def test_leaves_real_code_alone(self, source: bytes) -> None:
+ assert prepare_cpp_source(source) is source
+
+ def test_offsets_and_lines_are_unchanged(self) -> None:
+ for source in (EXPORTED_CLASS, ANNOTATED_CLASS):
+ out = prepare_cpp_source(source)
+ assert len(out) == len(source)
+ assert out.splitlines() != source.splitlines()
+ assert out.count(b"\n") == source.count(b"\n")
diff --git a/tests/unit/ingestion/test_cpp_inclass_declarations.py b/tests/unit/ingestion/test_cpp_inclass_declarations.py
index 4e26b99fb..b4209ca16 100644
--- a/tests/unit/ingestion/test_cpp_inclass_declarations.py
+++ b/tests/unit/ingestion/test_cpp_inclass_declarations.py
@@ -161,12 +161,13 @@ class TestExportMacroClassBodies:
The body becomes a ``compound_statement`` and its members statement-context
``declaration`` nodes, so none of the ``field_declaration`` patterns reach
them, and ``= 0`` wraps the declarator in an ``init_declarator`` the plain
- forward-declaration pattern cannot match either.
+ forward-declaration pattern cannot match either. An ALL-CAPS macro is
+ blanked before parsing, so these cases use a lower-case one.
"""
def test_a_pure_virtual_member_lands(self) -> None:
assert _methods(
- "class LEVELDB_EXPORT Iterator {\n"
+ "class leveldb_export Iterator {\n"
" public:\n"
" virtual void Seek(const Slice& t) = 0;\n"
" void RegisterCleanup(int a);\n"
@@ -180,7 +181,7 @@ class TestExportMacroClassBodies:
constructor with a member-init list does too.
"""
source = (
- "class LEVELDB_EXPORT R {\n"
+ "class leveldb_export R {\n"
" public:\n"
" size_t Index(size_t i) const { return i + 1; }\n"
" virtual void Real() = 0;\n"
@@ -195,7 +196,7 @@ class TestExportMacroClassBodies:
def test_a_deleted_function_is_not_a_method_declaration(self) -> None:
got = _methods(
- "class LEVELDB_EXPORT S {\n"
+ "class leveldb_export S {\n"
" public:\n"
" void gone(int n) = delete;\n"
" virtual void kept() = 0;\n"