Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

Merge main into fix/issue-2078 for the release batch

Martin Vogelea2e36aa7497
diff --cc tests/test_parse_coverage.c
index 1cc2f8e98,5dbc38e4a..def6dffa9
--- a/tests/test_parse_coverage.c
+++ b/tests/test_parse_coverage.c
@@@ -1638,247 -1615,25 +1638,265 @@@ TEST(cs_malformed_conditional_remains_p
PASS();
}
+/* ── #2078: every file is parsed as if it ended with a newline ───────────────
+ *
+ * Several grammars need a line terminator that a file's last line may simply
+ * not have. tree-sitter-markdown is the sharpest case: an opening code fence,
+ * an ATX heading or a bare list marker as the final bytes of a file leaves a
+ * WIDTH-BEARING ERROR (so the #1610 zero-width rule cannot excuse it), and a
+ * file that is only "```" came back as a whole-file error. Appending one "\n"
+ * made every one of these clean -- the reporter proved it on 21 real files.
+ *
+ * cbm now feeds the parser one virtual "\n" past EOF when the last byte is not
+ * already a newline, then clamps the tree back to the real length, so no
+ * consumer sees a byte, a line or a range that is not in the file. */
+TEST(markdown_unterminated_last_line_is_clean_issue2078) {
+ const char *cases[] = {
+ "text\n```", /* opening fence at EOF (upstream PR #262, closed unmerged) */
+ "```", /* the whole file is one opening fence */
+ "# Top", /* ATX heading */
+ "- ", /* bare list marker */
+ };
+ int flagged = 0;
+ for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+ CBMFileResult *r = do_extract(cases[i], CBM_LANG_MARKDOWN, "doc.md");
+ ASSERT_NOT_NULL(r);
+ if (r->parse_incomplete || r->parse_unusable) {
+ fprintf(stderr, " case %zu flagged: partial=%d unusable=%d ranges=%s\n", i,
+ r->parse_incomplete, r->parse_unusable,
+ r->error_ranges ? r->error_ranges : "(none)");
+ flagged++;
+ }
+ cbm_free_result(r);
+ }
+ if (flagged) {
+ FAIL("a markdown file must not be flagged only because its last line is unterminated");
+ }
+ PASS();
+}
+
+/* Controls: the same bytes WITH the newline were already clean and stay so. */
+TEST(markdown_terminated_last_line_controls_unchanged_issue2078) {
+ const char *cases[] = {"text\n```\n", "```\n", "# Top\n", "- \n"};
+ for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+ CBMFileResult *r = do_extract(cases[i], CBM_LANG_MARKDOWN, "doc.md");
+ ASSERT_NOT_NULL(r);
+ ASSERT_FALSE(r->parse_incomplete);
+ ASSERT_FALSE(r->parse_unusable);
+ cbm_free_result(r);
+ }
+ PASS();
+}
+
+/* Everything extraction reports, as one string, so two results compare
+ * byte-for-byte: defs (every text field and range), calls (exact byte spans),
+ * imports and the coverage verdict. */
+typedef struct {
+ char *buf;
+ size_t cap;
+ size_t len;
+} dump_buf_t;
+
+static void dump_str(dump_buf_t *b, const char *tag, const char *s) {
+ int n = snprintf(b->buf + b->len, b->cap - b->len, " %s=%s", tag, s ? s : "~");
+ if (n > 0 && (size_t)n < b->cap - b->len) {
+ b->len += (size_t)n;
+ }
+}
+
+static void dump_num(dump_buf_t *b, const char *tag, long v) {
+ int n = snprintf(b->buf + b->len, b->cap - b->len, " %s=%ld", tag, v);
+ if (n > 0 && (size_t)n < b->cap - b->len) {
+ b->len += (size_t)n;
+ }
+}
+
+/* The Module def spans the whole file, and its end line follows the line
+ * convention for a terminated last line (#1967), so it is compared on its own
+ * rather than through the dump. */
+static char *dump_extraction(const CBMFileResult *r) {
+ dump_buf_t b = {(char *)calloc(1u << 16, 1), 1u << 16, 0};
+ if (!b.buf) {
+ return NULL;
+ }
+ for (int i = 0; i < r->defs.count; i++) {
+ const CBMDefinition *d = &r->defs.items[i];
+ if (d->label && strcmp(d->label, "Module") == 0) {
+ continue;
+ }
+ dump_str(&b, "\nD", d->qualified_name);
+ dump_str(&b, "name", d->name);
+ dump_str(&b, "label", d->label);
+ dump_num(&b, "start", d->start_line);
+ dump_num(&b, "end", d->end_line);
+ dump_num(&b, "lines", d->lines);
+ dump_num(&b, "cx", d->complexity);
+ dump_num(&b, "params", d->param_count);
+ dump_str(&b, "sig", d->signature);
+ dump_str(&b, "ret", d->return_type);
+ dump_str(&b, "doc", d->docstring);
+ dump_str(&b, "prof", d->structural_profile);
+ dump_str(&b, "tok", d->body_tokens);
+ }
+ for (int i = 0; i < r->calls.count; i++) {
+ const CBMCall *c = &r->calls.items[i];
+ dump_str(&b, "\nC", c->callee_name);
+ dump_str(&b, "in", c->enclosing_func_qn);
+ dump_num(&b, "line", c->start_line);
+ dump_num(&b, "from", (long)c->site_start_byte);
+ dump_num(&b, "to", (long)c->site_end_byte);
+ dump_num(&b, "args", c->arg_count);
+ }
+ for (int i = 0; i < r->imports.count; i++) {
+ dump_str(&b, "\nI", r->imports.items[i].local_name);
+ dump_str(&b, "path", r->imports.items[i].module_path);
+ }
+ dump_str(&b, "\nP", r->error_ranges);
+ dump_num(&b, "partial", r->parse_incomplete);
+ dump_num(&b, "unusable", r->parse_unusable);
+ dump_str(&b, "", "\n");
+ return b.buf;
+}
+
+static uint32_t module_end_line(const CBMFileResult *r) {
+ for (int i = 0; i < r->defs.count; i++) {
+ if (r->defs.items[i].label && strcmp(r->defs.items[i].label, "Module") == 0) {
+ return r->defs.items[i].end_line;
+ }
+ }
+ return 0;
+}
+
+static char *extract_dump(const char *src, CBMLanguage lang, const char *path,
+ uint32_t *module_end) {
+ CBMFileResult *r = do_extract(src, lang, path);
+ if (!r) {
+ return NULL;
+ }
+ char *d = dump_extraction(r);
+ if (module_end) {
+ *module_end = module_end_line(r);
+ }
+ cbm_free_result(r);
+ return d;
+}
+
+/* GUARD: for code grammars a trailing newline is insignificant, so a file
+ * without one must extract byte-identically to the same file with one -- every
+ * def, range, text field and call byte span. The virtual newline may never
+ * show up as phantom content or a phantom line. */
+TEST(code_without_final_newline_extracts_identically_issue2078) {
+ struct {
+ const char *src; /* no trailing newline */
+ CBMLanguage lang;
+ const char *path;
+ uint32_t lines;
+ } cases[] = {
+ {"#include <stdio.h>\n\nstatic int beta(int x) {\n return x + 1;\n}\n\n"
+ "void alpha(void) { printf(\"a\"); beta(2); }",
+ CBM_LANG_C, "a.c", 7},
+ {"import os\n\n\nclass K:\n def m(self, a):\n return os.path.join(a)\n\n\n"
+ "def f(x):\n \"\"\"doc\"\"\"\n return K().m(x)",
+ CBM_LANG_PYTHON, "a.py", 11},
+ };
+ for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+ size_t n = strlen(cases[i].src);
+ char *terminated = (char *)malloc(n + 2);
+ ASSERT_NOT_NULL(terminated);
+ memcpy(terminated, cases[i].src, n);
+ terminated[n] = '\n';
+ terminated[n + 1] = '\0';
+ uint32_t bare_module_end = 0;
+ char *bare = extract_dump(cases[i].src, cases[i].lang, cases[i].path, &bare_module_end);
+ char *term = extract_dump(terminated, cases[i].lang, cases[i].path, NULL);
+ free(terminated);
+ ASSERT_NOT_NULL(bare);
+ ASSERT_NOT_NULL(term);
+ bool same = strcmp(bare, term) == 0;
+ if (!same) {
+ fprintf(stderr, " %s differs:\n--- without final newline%s--- with final newline%s",
+ cases[i].path, bare, term);
+ }
+ free(bare);
+ free(term);
+ if (!same) {
+ FAIL("extraction must not depend on whether the last line is terminated");
+ }
+ if (bare_module_end != cases[i].lines) {
+ fprintf(stderr, " %s: module ends on line %u, file has %u\n", cases[i].path,
+ bare_module_end, cases[i].lines);
+ FAIL("the virtual newline must not add a line to the module");
+ }
+ }
+ PASS();
+}
+
+/* GUARD against phantom content: a heading on an unterminated last line is a
+ * one-line file. No def may end past line 1, no text may carry the virtual
+ * newline, and the result must equal the terminated file's. */
+TEST(virtual_newline_adds_no_phantom_line_or_text_issue2078) {
+ CBMFileResult *r = do_extract("# Top", CBM_LANG_MARKDOWN, "top.md");
+ ASSERT_NOT_NULL(r);
+ bool found = has_def(r, "Top");
+ for (int i = 0; i < r->defs.count; i++) {
+ const CBMDefinition *d = &r->defs.items[i];
+ if (d->end_line > 1 || d->start_line > 1) {
+ fprintf(stderr, " def %s spans %u-%u\n", d->name ? d->name : "?", d->start_line,
+ d->end_line);
+ cbm_free_result(r);
+ FAIL("no def may reach past the last real line");
+ }
+ if ((d->name && strchr(d->name, '\n')) || (d->signature && strchr(d->signature, '\n'))) {
+ cbm_free_result(r);
+ FAIL("no extracted text may contain the virtual newline");
+ }
+ }
+ cbm_free_result(r);
+ if (!found) {
+ /* Before #2078 the unterminated heading was silently dropped: no
+ * Section, and no parse_partial flag to say so. */
+ FAIL("the heading on an unterminated last line must be extracted as a Section");
+ }
+ PASS();
+}
+
+/* GUARD: the empty file and a file that is exactly "\n" get no virtual
+ * newline (nothing is unterminated) and stay clean. */
+TEST(empty_and_newline_only_files_unchanged_issue2078) {
+ const char *srcs[] = {"", "\n"};
+ CBMLanguage langs[] = {CBM_LANG_MARKDOWN, CBM_LANG_PYTHON, CBM_LANG_C};
+ const char *paths[] = {"e.md", "e.py", "e.c"};
+ for (size_t s = 0; s < 2; s++) {
+ for (size_t l = 0; l < 3; l++) {
+ CBMFileResult *r = do_extract(srcs[s], langs[l], paths[l]);
+ ASSERT_NOT_NULL(r);
+ ASSERT_FALSE(r->parse_incomplete);
+ ASSERT_FALSE(r->parse_unusable);
+ cbm_free_result(r);
+ }
+ }
+ PASS();
+}
+
+ /* #1967: every line counter over one source buffer answers the same
+ * question the same way. A trailing newline ends the last line; it does not
+ * open a new one. */
+ TEST(source_line_count_one_convention) {
+ ASSERT_EQ(cbm_source_line_count("", 0), 1u);
+ ASSERT_EQ(cbm_source_line_count("a", 1), 1u);
+ ASSERT_EQ(cbm_source_line_count("a\n", 2), 1u);
+ ASSERT_EQ(cbm_source_line_count("\n", 1), 1u);
+ ASSERT_EQ(cbm_source_line_count("a\nb", 3), 2u);
+ ASSERT_EQ(cbm_source_line_count("a\nb\n", 4), 2u);
+ ASSERT_EQ(cbm_source_line_count("a\n\n", 3), 2u);
+ ASSERT_EQ(cbm_source_line_count("a\r\nb\r\n", 6), 2u);
+ /* src_len bounds the count, not a NUL terminator. */
+ ASSERT_EQ(cbm_source_line_count("a\nb\nc", 2), 1u);
+ PASS();
+ }
+
SUITE(parse_coverage) {
+ RUN_TEST(source_line_count_one_convention);
RUN_TEST(c_ifdef_split_brace_sets_parse_incomplete);
RUN_TEST(c_ifdef_split_brace_neighbors_still_extracted);
RUN_TEST(c_error_range_points_at_failed_region);