Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
Merge main into fix/issue-2078 for the release batch
Martin Vogelea2e36aa7497
diff --cc tests/test_parse_coverage.cindex 1cc2f8e98,5dbc38e4a..def6dffa9--- a/tests/test_parse_coverage.c+++ b/tests/test_parse_coverage.c@@@ -1638,247 -1615,25 +1638,265 @@@ TEST(cs_malformed_conditional_remains_pPASS();}+/* ── #2078: every file is parsed as if it ended with a newline ───────────────+ *+ * Several grammars need a line terminator that a file's last line may simply+ * not have. tree-sitter-markdown is the sharpest case: an opening code fence,+ * an ATX heading or a bare list marker as the final bytes of a file leaves a+ * WIDTH-BEARING ERROR (so the #1610 zero-width rule cannot excuse it), and a+ * file that is only "```" came back as a whole-file error. Appending one "\n"+ * made every one of these clean -- the reporter proved it on 21 real files.+ *+ * cbm now feeds the parser one virtual "\n" past EOF when the last byte is not+ * already a newline, then clamps the tree back to the real length, so no+ * consumer sees a byte, a line or a range that is not in the file. */+TEST(markdown_unterminated_last_line_is_clean_issue2078) {+ const char *cases[] = {+ "text\n```", /* opening fence at EOF (upstream PR #262, closed unmerged) */+ "```", /* the whole file is one opening fence */+ "# Top", /* ATX heading */+ "- ", /* bare list marker */+ };+ int flagged = 0;+ for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {+ CBMFileResult *r = do_extract(cases[i], CBM_LANG_MARKDOWN, "doc.md");+ ASSERT_NOT_NULL(r);+ if (r->parse_incomplete || r->parse_unusable) {+ fprintf(stderr, " case %zu flagged: partial=%d unusable=%d ranges=%s\n", i,+ r->parse_incomplete, r->parse_unusable,+ r->error_ranges ? r->error_ranges : "(none)");+ flagged++;+ }+ cbm_free_result(r);+ }+ if (flagged) {+ FAIL("a markdown file must not be flagged only because its last line is unterminated");+ }+ PASS();+}++/* Controls: the same bytes WITH the newline were already clean and stay so. */+TEST(markdown_terminated_last_line_controls_unchanged_issue2078) {+ const char *cases[] = {"text\n```\n", "```\n", "# Top\n", "- \n"};+ for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {+ CBMFileResult *r = do_extract(cases[i], CBM_LANG_MARKDOWN, "doc.md");+ ASSERT_NOT_NULL(r);+ ASSERT_FALSE(r->parse_incomplete);+ ASSERT_FALSE(r->parse_unusable);+ cbm_free_result(r);+ }+ PASS();+}++/* Everything extraction reports, as one string, so two results compare+ * byte-for-byte: defs (every text field and range), calls (exact byte spans),+ * imports and the coverage verdict. */+typedef struct {+ char *buf;+ size_t cap;+ size_t len;+} dump_buf_t;++static void dump_str(dump_buf_t *b, const char *tag, const char *s) {+ int n = snprintf(b->buf + b->len, b->cap - b->len, " %s=%s", tag, s ? s : "~");+ if (n > 0 && (size_t)n < b->cap - b->len) {+ b->len += (size_t)n;+ }+}++static void dump_num(dump_buf_t *b, const char *tag, long v) {+ int n = snprintf(b->buf + b->len, b->cap - b->len, " %s=%ld", tag, v);+ if (n > 0 && (size_t)n < b->cap - b->len) {+ b->len += (size_t)n;+ }+}++/* The Module def spans the whole file, and its end line follows the line+ * convention for a terminated last line (#1967), so it is compared on its own+ * rather than through the dump. */+static char *dump_extraction(const CBMFileResult *r) {+ dump_buf_t b = {(char *)calloc(1u << 16, 1), 1u << 16, 0};+ if (!b.buf) {+ return NULL;+ }+ for (int i = 0; i < r->defs.count; i++) {+ const CBMDefinition *d = &r->defs.items[i];+ if (d->label && strcmp(d->label, "Module") == 0) {+ continue;+ }+ dump_str(&b, "\nD", d->qualified_name);+ dump_str(&b, "name", d->name);+ dump_str(&b, "label", d->label);+ dump_num(&b, "start", d->start_line);+ dump_num(&b, "end", d->end_line);+ dump_num(&b, "lines", d->lines);+ dump_num(&b, "cx", d->complexity);+ dump_num(&b, "params", d->param_count);+ dump_str(&b, "sig", d->signature);+ dump_str(&b, "ret", d->return_type);+ dump_str(&b, "doc", d->docstring);+ dump_str(&b, "prof", d->structural_profile);+ dump_str(&b, "tok", d->body_tokens);+ }+ for (int i = 0; i < r->calls.count; i++) {+ const CBMCall *c = &r->calls.items[i];+ dump_str(&b, "\nC", c->callee_name);+ dump_str(&b, "in", c->enclosing_func_qn);+ dump_num(&b, "line", c->start_line);+ dump_num(&b, "from", (long)c->site_start_byte);+ dump_num(&b, "to", (long)c->site_end_byte);+ dump_num(&b, "args", c->arg_count);+ }+ for (int i = 0; i < r->imports.count; i++) {+ dump_str(&b, "\nI", r->imports.items[i].local_name);+ dump_str(&b, "path", r->imports.items[i].module_path);+ }+ dump_str(&b, "\nP", r->error_ranges);+ dump_num(&b, "partial", r->parse_incomplete);+ dump_num(&b, "unusable", r->parse_unusable);+ dump_str(&b, "", "\n");+ return b.buf;+}++static uint32_t module_end_line(const CBMFileResult *r) {+ for (int i = 0; i < r->defs.count; i++) {+ if (r->defs.items[i].label && strcmp(r->defs.items[i].label, "Module") == 0) {+ return r->defs.items[i].end_line;+ }+ }+ return 0;+}++static char *extract_dump(const char *src, CBMLanguage lang, const char *path,+ uint32_t *module_end) {+ CBMFileResult *r = do_extract(src, lang, path);+ if (!r) {+ return NULL;+ }+ char *d = dump_extraction(r);+ if (module_end) {+ *module_end = module_end_line(r);+ }+ cbm_free_result(r);+ return d;+}++/* GUARD: for code grammars a trailing newline is insignificant, so a file+ * without one must extract byte-identically to the same file with one -- every+ * def, range, text field and call byte span. The virtual newline may never+ * show up as phantom content or a phantom line. */+TEST(code_without_final_newline_extracts_identically_issue2078) {+ struct {+ const char *src; /* no trailing newline */+ CBMLanguage lang;+ const char *path;+ uint32_t lines;+ } cases[] = {+ {"#include <stdio.h>\n\nstatic int beta(int x) {\n return x + 1;\n}\n\n"+ "void alpha(void) { printf(\"a\"); beta(2); }",+ CBM_LANG_C, "a.c", 7},+ {"import os\n\n\nclass K:\n def m(self, a):\n return os.path.join(a)\n\n\n"+ "def f(x):\n \"\"\"doc\"\"\"\n return K().m(x)",+ CBM_LANG_PYTHON, "a.py", 11},+ };+ for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {+ size_t n = strlen(cases[i].src);+ char *terminated = (char *)malloc(n + 2);+ ASSERT_NOT_NULL(terminated);+ memcpy(terminated, cases[i].src, n);+ terminated[n] = '\n';+ terminated[n + 1] = '\0';+ uint32_t bare_module_end = 0;+ char *bare = extract_dump(cases[i].src, cases[i].lang, cases[i].path, &bare_module_end);+ char *term = extract_dump(terminated, cases[i].lang, cases[i].path, NULL);+ free(terminated);+ ASSERT_NOT_NULL(bare);+ ASSERT_NOT_NULL(term);+ bool same = strcmp(bare, term) == 0;+ if (!same) {+ fprintf(stderr, " %s differs:\n--- without final newline%s--- with final newline%s",+ cases[i].path, bare, term);+ }+ free(bare);+ free(term);+ if (!same) {+ FAIL("extraction must not depend on whether the last line is terminated");+ }+ if (bare_module_end != cases[i].lines) {+ fprintf(stderr, " %s: module ends on line %u, file has %u\n", cases[i].path,+ bare_module_end, cases[i].lines);+ FAIL("the virtual newline must not add a line to the module");+ }+ }+ PASS();+}++/* GUARD against phantom content: a heading on an unterminated last line is a+ * one-line file. No def may end past line 1, no text may carry the virtual+ * newline, and the result must equal the terminated file's. */+TEST(virtual_newline_adds_no_phantom_line_or_text_issue2078) {+ CBMFileResult *r = do_extract("# Top", CBM_LANG_MARKDOWN, "top.md");+ ASSERT_NOT_NULL(r);+ bool found = has_def(r, "Top");+ for (int i = 0; i < r->defs.count; i++) {+ const CBMDefinition *d = &r->defs.items[i];+ if (d->end_line > 1 || d->start_line > 1) {+ fprintf(stderr, " def %s spans %u-%u\n", d->name ? d->name : "?", d->start_line,+ d->end_line);+ cbm_free_result(r);+ FAIL("no def may reach past the last real line");+ }+ if ((d->name && strchr(d->name, '\n')) || (d->signature && strchr(d->signature, '\n'))) {+ cbm_free_result(r);+ FAIL("no extracted text may contain the virtual newline");+ }+ }+ cbm_free_result(r);+ if (!found) {+ /* Before #2078 the unterminated heading was silently dropped: no+ * Section, and no parse_partial flag to say so. */+ FAIL("the heading on an unterminated last line must be extracted as a Section");+ }+ PASS();+}++/* GUARD: the empty file and a file that is exactly "\n" get no virtual+ * newline (nothing is unterminated) and stay clean. */+TEST(empty_and_newline_only_files_unchanged_issue2078) {+ const char *srcs[] = {"", "\n"};+ CBMLanguage langs[] = {CBM_LANG_MARKDOWN, CBM_LANG_PYTHON, CBM_LANG_C};+ const char *paths[] = {"e.md", "e.py", "e.c"};+ for (size_t s = 0; s < 2; s++) {+ for (size_t l = 0; l < 3; l++) {+ CBMFileResult *r = do_extract(srcs[s], langs[l], paths[l]);+ ASSERT_NOT_NULL(r);+ ASSERT_FALSE(r->parse_incomplete);+ ASSERT_FALSE(r->parse_unusable);+ cbm_free_result(r);+ }+ }+ PASS();+}++ /* #1967: every line counter over one source buffer answers the same+ * question the same way. A trailing newline ends the last line; it does not+ * open a new one. */+ TEST(source_line_count_one_convention) {+ ASSERT_EQ(cbm_source_line_count("", 0), 1u);+ ASSERT_EQ(cbm_source_line_count("a", 1), 1u);+ ASSERT_EQ(cbm_source_line_count("a\n", 2), 1u);+ ASSERT_EQ(cbm_source_line_count("\n", 1), 1u);+ ASSERT_EQ(cbm_source_line_count("a\nb", 3), 2u);+ ASSERT_EQ(cbm_source_line_count("a\nb\n", 4), 2u);+ ASSERT_EQ(cbm_source_line_count("a\n\n", 3), 2u);+ ASSERT_EQ(cbm_source_line_count("a\r\nb\r\n", 6), 2u);+ /* src_len bounds the count, not a NUL terminator. */+ ASSERT_EQ(cbm_source_line_count("a\nb\nc", 2), 1u);+ PASS();+ }+SUITE(parse_coverage) {+ RUN_TEST(source_line_count_one_convention);RUN_TEST(c_ifdef_split_brace_sets_parse_incomplete);RUN_TEST(c_ifdef_split_brace_neighbors_still_extracted);RUN_TEST(c_error_range_points_at_failed_region);