Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(ingestion): stop indexing PL/I .inc members as C++ (#4553)

arnavsx3f2bde4f781e6
diff --git a/packages/core/src/repowise/core/ingestion/languages/specs/cpp.py b/packages/core/src/repowise/core/ingestion/languages/specs/cpp.py
index d8688e228..342817d4c 100644
--- a/packages/core/src/repowise/core/ingestion/languages/specs/cpp.py
+++ b/packages/core/src/repowise/core/ingestion/languages/specs/cpp.py
@@ -18,9 +18,9 @@ from .c import TEST_SUPPORT_STEM_SUFFIXES
#: no symbols, no edges, no error. Reassigned here on the corpus count: 24 of
#: the 25 code-bearing ``.inc`` files are C++ and none are Pascal, on a corpus
#: that does contain Pascal. The one PHP ``.inc`` (a Drush plugin) is the known
-#: cost; ``.inc`` is shared by C++, PHP and Pascal in the wild, and a
-#: content- or sibling-aware router is the real answer if a second language's
-#: usage ever shows up in the corpus.
+#: cost of extension-only routing. ``.inc`` is also shared with PL/I, PHP and
+#: Pascal: traverser's content check now keeps this C++ default only when the
+#: file's head contains C/C++ evidence, leaving other members unindexed.
INCLUDE_FRAGMENT_EXTENSIONS: frozenset[str] = frozenset({".inl", ".ipp", ".tpp", ".inc"})
SPEC = LanguageSpec(
diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.py
index dd487dee9..bc9aca6cc 100644
--- a/packages/core/src/repowise/core/ingestion/traverser.py
+++ b/packages/core/src/repowise/core/ingestion/traverser.py
@@ -1126,6 +1126,63 @@ def _objc_header_language(abs_path: Path, source: bytes | None = None) -> Langua
return "objectivec" if _OBJC_HEADER_DECLARATION_RE.search(head) else None
+# Unlike .inl/.ipp/.tpp, .inc is shared with PL/I, Pascal and PHP. Keep the
+# extension's C++ default only when the head contains a C/C++ construct.
+_CPP_INC_SNIFF_BYTES = 16384
+_CPP_INC_SIGNAL_RE = re.compile(
+ rb"^[ \t]*#[ \t]*(?:include|define|undef|if|ifdef|ifndef|elif|else|endif|pragma|error|line)\b"
+ rb"|\bnamespace\s+[A-Za-z_]|\btemplate\s*<"
+ rb"|\b[A-Za-z_]\w*::[A-Za-z_]"
+ rb"|\b(?:class|struct|union|enum)\s+[A-Za-z_]"
+ rb"|\b(?:void|bool|char|short|int|long|float|double|auto)(?:\s+|\s*[*&]+\s*)[A-Za-z_]",
+ re.MULTILINE,
+)
+_C_COMMENT_OR_STRING_RE = re.compile(
+ rb"//[^\n]*|/\*.*?(?:\*/|\Z)|\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*'",
+ re.DOTALL,
+)
+
+
+def _inc_declarations(head: bytes) -> bytes:
+ """Blank nested brace blocks and Pascal comments, retaining declarations.
+
+ A brace alone cannot distinguish C++ code from a Pascal comment. C++
+ declarations outside a body still supply evidence; text inside Pascal
+ comments must not supply it, even when comments nest.
+ """
+ blanked = bytearray(head)
+ closers: list[bytes] = []
+ start = 0
+ for m in re.finditer(rb"\{|\}|\(\*|\*\)", head):
+ token = m.group()
+ if token in (b"{", b"(*"):
+ if not closers:
+ start = m.start()
+ closers.append(b"}" if token == b"{" else b"*)")
+ elif closers and token == closers[-1]:
+ closers.pop()
+ if not closers:
+ blanked[start : m.end()] = re.sub(rb"[^\n]", b" ", head[start : m.end()])
+ if closers:
+ blanked[start:] = re.sub(rb"[^\n]", b" ", head[start:])
+ return bytes(blanked)
+
+
+def _cpp_inc_language(abs_path: Path, source: bytes | None = None) -> LanguageTag | None:
+ """``cpp`` for an .inc with C++ evidence outside comments and strings, else None.
+
+ Includes in an unsupported language must stay unindexed, rather than
+ becoming C++ files whose missing importers look like dead code.
+ """
+ head = _head(abs_path, source, _CPP_INC_SNIFF_BYTES)
+ if head is None:
+ return None
+ head = _C_COMMENT_OR_STRING_RE.sub(lambda m: re.sub(rb"[^\n]", b" ", m.group(0)), head)
+ if b"<?" in head:
+ return None # PHP's opening tags override any C++-like declarations.
+ return "cpp" if _CPP_INC_SIGNAL_RE.search(_inc_declarations(head)) else None
+
+
# DCLGEN writes the table declaration, then the ``01`` record, at the top of a
# member; a 750-column table (the DB2 maximum) declares in about 62 KB.
_DCLGEN_SNIFF_BYTES = 131072
@@ -1152,6 +1209,8 @@ def _content_language(
so the working tree is not read in their place.
"""
if language == "cpp":
+ if abs_path.suffix.lower() == ".inc":
+ return _cpp_inc_language(abs_path, source)
return _objc_header_language(abs_path, source) or language
return language or _dclgen_language(abs_path, source)
diff --git a/tests/unit/ingestion/test_traverser.py b/tests/unit/ingestion/test_traverser.py
index d93e0c47b..0c513f2e6 100644
--- a/tests/unit/ingestion/test_traverser.py
+++ b/tests/unit/ingestion/test_traverser.py
@@ -13,6 +13,7 @@ from repowise.core.ingestion import traverser as traverser_mod
from repowise.core.ingestion.traverser import (
FileTraverser,
_compile_gitignore,
+ _content_language,
_detect_language,
_is_generated,
_scan_package_dir,
@@ -182,6 +183,80 @@ class TestLanguageDetection:
assert _detect_language(f) == "python"
+class TestIncLanguageDetection:
+ @pytest.mark.parametrize("suffix", [".inc", ".INC"])
+ @pytest.mark.parametrize(
+ "content",
+ [
+ '#include "impl.h"\n',
+ "# define REGISTER(T) register_type<T>()\n",
+ "#if defined(USE_FAST_PATH)\n#endif\n",
+ "namespace library\n",
+ "template <typename T> T identity(T);\n",
+ "constexpr int answer() { return 42; }\n",
+ "int answer() { return 42; }\n",
+ "int counts[] = { 1, 2 };\n",
+ 'std::string name() { return "example"; }\n',
+ 'const char *tag = "<?php";\n',
+ "/* <?php */\n// <?php\nstruct Value { int count; };\n",
+ "/* License banner */\n// implementation\nint answer() { return 42; }\n",
+ ],
+ )
+ def test_cpp_fragments_stay_cpp(self, tmp_path: Path, suffix: str, content: str) -> None:
+ f = tmp_path / f"impl{suffix}"
+ f.write_text(content)
+ assert _detect_language(f) == "cpp"
+
+ @pytest.mark.parametrize(
+ "content",
+ [
+ "/* BankDemo-style PL/I include member */\n"
+ " 05 DD_AREAS,\n 10 DD_ENV CHAR(4);\n",
+ "DCL 1 RECORD,\n 5 NAME CHAR(20);\n",
+ "%INCLUDE CBANKVAC;\n",
+ "/* #include <header.h>\nnamespace example { template <class T> } */\n"
+ "// #define EXAMPLE\n05 DD_AREAS,\n",
+ "DCL TEXT CHAR(80) INIT('namespace example { template <class T> }');\n",
+ 'DCL TEXT CHAR(80) INIT("/* not a comment */ namespace example {}");\n',
+ "/* An unfinished comment containing { and #define\n",
+ "{ field layout }\nvar Count: Integer;\n",
+ "{$I types.inc}\n",
+ "{ namespace example { template <class T> } }\nvar Count: Integer;\n",
+ "(* #include <example.h>\nnamespace example { } *)\nvar Count: Integer;\n",
+ "{ An unfinished comment with namespace example\n",
+ "<?php\nfunction countItems() { return 1; }\n",
+ "<?php\nnamespace App\\Models;\n",
+ "<?= renderItems(); ?>\n",
+ "<? function countItems() { return 1; } ?>\n",
+ # Braces without a C++ declaration can also be a Pascal comment.
+ "{ 1, 2 },\n",
+ "",
+ ],
+ )
+ def test_members_without_cpp_evidence_are_unknown(self, tmp_path: Path, content: str) -> None:
+ f = tmp_path / "member.inc"
+ f.write_text(content)
+ assert _detect_language(f) == "unknown"
+
+ def test_source_bytes_take_precedence_over_the_working_tree(self, tmp_path: Path) -> None:
+ f = tmp_path / "member.inc"
+ f.write_text("namespace library {}\n")
+ assert _content_language(f, "cpp", b"05 DD_AREAS,\n10 DD_ENV CHAR(4);\n") is None
+ f.write_text("05 DD_AREAS,\n10 DD_ENV CHAR(4);\n")
+ assert _content_language(f, "cpp", b"namespace library {}\n") == "cpp"
+
+ def test_unreadable_member_is_not_assumed_to_be_cpp(self, tmp_path: Path) -> None:
+ assert _content_language(tmp_path / "missing.inc", "cpp") is None
+
+ def test_other_cpp_fragment_extensions_do_not_need_a_content_signal(
+ self, tmp_path: Path
+ ) -> None:
+ for suffix in (".inl", ".ipp", ".tpp"):
+ f = tmp_path / f"table{suffix}"
+ f.write_text("REGISTER_TYPE(Widget)\n")
+ assert _detect_language(f) == "cpp"
+
+
# ---------------------------------------------------------------------------
# File traversal
# ---------------------------------------------------------------------------
@@ -226,6 +301,15 @@ class TestFileTraverser:
assert found == {"DCLTRTYP.dcl": "cobol"}
assert traverser.stats.skipped_unknown_language == 1
+ def test_only_cpp_inc_members_are_indexed(self, tmp_path: Path) -> None:
+ (tmp_path / "CDATED.inc").write_text(
+ "/* PL/I include */\n05 DD_AREAS,\n10 DD_ENV CHAR(4);\n"
+ )
+ (tmp_path / "impl.inc").write_text("inline int answer() { return 42; }\n")
+ traverser = FileTraverser(tmp_path)
+ assert {f.path: f.language for f in traverser.traverse()} == {"impl.inc": "cpp"}
+ assert traverser.stats.skipped_unknown_language == 1
+
def test_skips_unity_generated_dirs(self, tmp_path: Path) -> None:
(tmp_path / "ProjectSettings").mkdir()
(tmp_path / "ProjectSettings" / "ProjectVersion.txt").write_text(