Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(cpp): one spelling per template specialization in the preprocessed pass
Martin Vogelb87d067637d9
diff --git a/internal/cbm/preprocessor.cpp b/internal/cbm/preprocessor.cppindex 9be6298f3..43ab183d0 100644--- a/internal/cbm/preprocessor.cpp+++ b/internal/cbm/preprocessor.cpp@@ -381,6 +381,56 @@ static bool parse_line_directive(const char *line, size_t len, uint32_t *out_linreturn true;}+// True when `prev` and `tok` touched in the source text: same file and line,+// `prev` ending exactly where `tok` starts (simplecpp's own adjacency test),+// and neither produced by a macro expansion (expansion tokens carry the call+// site's location, so their columns prove nothing).+static bool tokens_source_adjacent(const simplecpp::Token *prev, const simplecpp::Token *tok) {+ if (!prev->macro.empty() || !tok->macro.empty() || !prev->location.sameline(tok->location)) {+ return false;+ }+ const std::string &s = prev->str();+ if (s.find_first_of("\r\n") != std::string::npos) {+ return false;+ }+ return prev->location.col + s.size() == tok->location.col;+}++// The expanded buffer in the source's own spelling. Mirrors+// simplecpp::TokenList::stringify() line for line — the same newlines and+// #line directives, so build_line_map and every line-based remap are unchanged —+// except that tokens which touched in the source stay touching. stringify()+// joins every same-line pair with one space, so the second pass spelled a+// multi-token name differently from the raw pass: the explicit specialization+// `template<> struct bitonic<int64_t, AVX2>` became `bitonic < int64_t , AVX2 >`,+// and every QN built from that text (defs adopted by the #961 rescue, call+// scopes, C-LSP class scopes) minted a second node for the same definition, the+// QN dedup (strcmp) missing it. A gap that held whitespace or a comment is still+// one space, as before.+static std::string stringify_source_spacing(const simplecpp::TokenList &tokens) {+ std::ostringstream ret;+ simplecpp::Location loc;+ loc.line = 1;+ for (const simplecpp::Token *tok = tokens.cfront(); tok; tok = tok->next) {+ if (tok->location.line < loc.line || tok->location.fileIndex != loc.fileIndex) {+ ret << "\n#line " << tok->location.line << " \"" << tokens.file(tok->location)+ << "\"\n";+ loc = tok->location;+ }+ while (tok->location.line > loc.line) {+ ret << '\n';+ loc.line++;+ }+ const simplecpp::Token *prev = tok->previous;+ if (prev && prev->location.sameline(tok->location) && !tokens_source_adjacent(prev, tok)) {+ ret << ' ';+ }+ ret << tok->str();+ loc.adjust(tok->str());+ }+ return ret.str();+}+static bool build_line_map(const std::string &expanded, const std::string &main_file,uint32_t *original_line_by_expanded_line,uint8_t *belongs_to_main_file) {@@ -476,7 +526,7 @@ CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_lesimplecpp::preprocess(output, rawtokens, files, filedata, dui);- std::string result = output.stringify();+ std::string result = stringify_source_spacing(output);// Clean up loaded file datasimplecpp::cleanup(filedata);diff --git a/tests/test_c_lsp.c b/tests/test_c_lsp.cindex 0dc3abb80..a4e46c7bc 100644--- a/tests/test_c_lsp.c+++ b/tests/test_c_lsp.c@@ -16278,21 +16278,23 @@ TEST(clsp_c_global_initializer_does_not_inherit_previous_function_caller) {}static int assert_cpp_family_preprocessed_collision_isolated(CBMLanguage language) {- /* simplecpp removes the 41-byte comment payload and normalizes tokens. As- * verified by the non-vacuity assertions below, raw alpha.render and the- * macro-hidden expanded bravo.render both land at numeric span 182:200,- * even though those offsets refer to different source texts. */+ /* The preprocessed buffer drops the 21-byte comment payload and the line+ * indentation, keeps source spacing between source tokens and one space+ * between macro-expansion tokens. As verified by the non-vacuity assertions+ * below, raw alpha.render and the macro-hidden expanded bravo.render both+ * land at numeric span 162:180, even though those offsets refer to+ * different source texts. */static const char source[] ="struct Alpha { int render(){return 1;} };\n""struct Bravo { int render(){return 2;} };\n""int occurrence_probe(){\n"" Alpha alpha; Bravo bravo;\n"- " /*xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/ alpha . render ( );\n"+ " /*xxxxxxxxxxxxxxxxxxxxx*/ alpha . render ( );\n""#define HIDDEN_RENDER(x) x.render()\n"" return HIDDEN_RENDER(bravo);\n""}\n";- const uint32_t collision_start = 182;- const uint32_t collision_end = 200;+ const uint32_t collision_start = 162;+ const uint32_t collision_end = 180;ASSERT_EQ((uint32_t)(strstr(source, "alpha . render ( )") - source), collision_start);CBMFileResult *result = extract_c_family(source, language);@@ -16354,19 +16356,20 @@ TEST(clsp_cuda_preprocessed_coordinate_collision_isolated) {}TEST(clsp_c_preprocessed_coordinate_collision_isolated) {- /* The raw fp call and macro-hidden expanded fp call both occupy 206:212,- * but data flow resolves them to different targets. */+ /* The raw fp call and macro-hidden expanded fp call both occupy 180:186+ * (comment payload sized against the preprocessed buffer's spacing), but+ * data flow resolves them to different targets. */static const char source[] ="static int alpha_target(void){return 1;}\n""static int bravo_target(void){return 2;}\n""int occurrence_probe(void){\n"" int (*fp)(void) = alpha_target;\n"- " /*xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/ fp ( );\n"+ " /*xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/ fp ( );\n""#define HIDDEN_FP_CALL() (fp = bravo_target, fp())\n"" return HIDDEN_FP_CALL();\n""}\n";- const uint32_t collision_start = 206;- const uint32_t collision_end = 212;+ const uint32_t collision_start = 180;+ const uint32_t collision_end = 186;const char *raw_probe = strstr(source, "occurrence_probe");ASSERT_NOT_NULL(raw_probe);ASSERT_EQ((uint32_t)(strstr(raw_probe, "fp ( )") - source), collision_start);diff --git a/tests/test_extraction.c b/tests/test_extraction.cindex f66b72e16..7382e147e 100644--- a/tests/test_extraction.c+++ b/tests/test_extraction.c@@ -7424,6 +7424,71 @@ TEST(extract_c_clean_file_no_recovery_duplicates_issue961) {PASS();}+/* True when `qn` carries the token-joined spelling of the `box<int, 2>`+ * template-id that the preprocessed buffer used to produce. */+static bool qn_has_token_spaced_box(const char *qn) {+ return qn && (strstr(qn, "box <") || strstr(qn, "< int") || strstr(qn, "int ,"));+}++/* dotnet/runtime vxsort shape (src/coreclr/gc/vxsort/smallsort/+ * bitonic_sort.*.generated.h): methods of an explicit specialization+ * `template<> struct bitonic<int64_t, AVX2>` carry an attribute macro defined in+ * another header (INLINE / NOINLINE), so the raw parse has ERROR regions on the+ * method lines and the #961 rescue re-extracts them from the preprocessed+ * buffer. That buffer used to join every token with one space, spelling the+ * scope `box < int , 2 >` where the raw pass spells `box<int, 2>`; the QN dedup+ * (strcmp) missed, so every method got a second node and the CALLS split across+ * both. One definition, one spelling, on every path. */+TEST(extract_cpp_template_specialization_preprocessed_qn_single_spelling) {+ CBMFileResult *r = extract("#ifndef BOX_H\n"+ "#define BOX_H\n"+ "namespace ns {\n"+ "template <typename T, int M> struct box {\n"+ " static void sort(T *p);\n"+ "};\n"+ "template<> struct box<int, 2> {\n"+ " static INLINE void first(int *p) { p[0] = 0; }\n"+ " static INLINE void second(int *p) { first(p); }\n"+ "};\n"+ "}\n"+ "#endif\n",+ CBM_LANG_CPP, "t", "box.h");+ ASSERT_NOT_NULL(r);+ int first_count = 0;+ int second_count = 0;+ const char *first_qn = NULL;+ for (int i = 0; i < r->defs.count; i++) {+ const CBMDefinition *d = &r->defs.items[i];+ if (qn_has_token_spaced_box(d->qualified_name)) {+ fprintf(stderr, " [tmplqn] FAIL token-spaced def QN: %s\n", d->qualified_name);+ }+ ASSERT_FALSE(qn_has_token_spaced_box(d->qualified_name));+ if (d->name && strcmp(d->name, "first") == 0) {+ first_count++;+ first_qn = d->qualified_name;+ }+ if (d->name && strcmp(d->name, "second") == 0) {+ second_count++;+ }+ }+ ASSERT_EQ(first_count, 1);+ ASSERT_EQ(second_count, 1);+ ASSERT_NOT_NULL(first_qn);+ ASSERT_NOT_NULL(strstr(first_qn, "ns.box<int, 2>.first"));+ /* Call scopes and LSP-resolved endpoints name the same single spelling, so+ * no CALLS edge lands on (or starts from) a phantom second node. */+ for (int i = 0; i < r->calls.count; i++) {+ ASSERT_FALSE(qn_has_token_spaced_box(r->calls.items[i].enclosing_func_qn));+ }+ for (int i = 0; i < r->resolved_calls.count; i++) {+ const CBMResolvedCall *rc = &r->resolved_calls.items[i];+ ASSERT_FALSE(qn_has_token_spaced_box(rc->caller_qn));+ ASSERT_FALSE(qn_has_token_spaced_box(rc->callee_qn));+ }+ cbm_free_result(r);+ PASS();+}+/* #1989: build-system export macros (UE "<MOD>_API", CMake generate_export_header* "<lib>_EXPORT") are empty on the real compile line but opaque to tree-sitter,* so `class MOD_API Foo` misparses. The preprocessed second pass predefines the@@ -12359,6 +12424,7 @@ SUITE(extraction) {RUN_TEST(extract_cpp_preproc_macro_generated_callable_skipped_issue949);RUN_TEST(extract_c_ifdef_split_brace_after_include_remapped_issue949);RUN_TEST(extract_c_clean_file_no_recovery_duplicates_issue961);+ RUN_TEST(extract_cpp_template_specialization_preprocessed_qn_single_spelling);RUN_TEST(extract_cpp_export_macro_class_recovery_issue1989);RUN_TEST(extract_cpp_export_macro_struct_recovery_issue1989);RUN_TEST(extract_cpp_export_macro_enum_recovery_issue1989);