Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
/*
* test_parse_coverage.c — Reproduce-first suite for the best-effort
* parse-coverage signal (#963, Signal A).
*
* ── The gap being reproduced ────────────────────────────────────────────────
* When tree-sitter hits a construct it cannot parse (ERROR/MISSING nodes in
* the tree), extraction silently drops every definition inside the failed
* region — the file looks fully indexed but is not. `ts_node_has_error(root)`
* detects this, yet nothing consumed it: CBMFileResult gained the fields
* parse_incomplete / error_ranges / error_region_count, but the parse site in
* cbm_extract_file_impl never sets them.
*
* Canonical trigger: the preprocessor-blind #ifdef-split-brace pattern in C —
* both branches open `fn(...) {` and share ONE closing brace, so the raw text
* is brace-unbalanced → ERROR node → the guarded function never becomes a
* Function node while neighbors extract fine.
*
* ── The contract these tests enforce ────────────────────────────────────────
* RED (unfixed): parse_incomplete is never set → flagged-file tests fail.
* GREEN (fixed): cbm_extract_file sets parse_incomplete=true iff the tree
* contains ERROR/MISSING nodes, records the 1-based line
* ranges of the TOP-MOST error regions ("start-end,..."),
* bounded by the 256-region cap, and clean files stay
* completely unflagged (no false positives).
*
* BEST-EFFORT framing (must never be weakened the other way): a flag means
* "constructs here were dropped — prefer grep"; the ABSENCE of a flag is NOT
* a completeness guarantee. These tests only pin down the detectable class.
*/
#include "test_framework.h"
#include "cbm.h"
#include "sql_values.h" /* #1735 value-row scanner */
#include "foundation/compat.h" /* cbm_setenv / cbm_unsetenv */
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
/* Convenience extract wrapper (same shape as test_extraction_imports.c). */
static CBMFileResult *do_extract(const char *src, CBMLanguage lang, const char *path) {
return cbm_extract_file(src, (int)strlen(src), lang, "covproj", path, 0, NULL, NULL);
}
/* Return 1 if any extracted definition has the given short name. */
static int has_def(CBMFileResult *r, const char *name) {
for (int i = 0; i < r->defs.count; i++) {
if (r->defs.items[i].name && strcmp(r->defs.items[i].name, name) == 0) {
return 1;
}
}
return 0;
}
/* ── Fixtures ─────────────────────────────────────────────────────────────── */
/* #ifdef-split-brace: both branches open `guarded(...) {`, one shared `}`.
* Preprocessor-blind parse sees unbalanced braces → ERROR region around
* lines 5–11; ok_before/ok_after remain extractable. */
static const char *C_IFDEF_SPLIT = "#include <stdio.h>\n" /* 1 */
"\n" /* 2 */
"void ok_before(void) { printf(\"a\"); }\n" /* 3 */
"\n" /* 4 */
"#ifdef FEATURE_A\n" /* 5 */
"static int guarded(int x) {\n" /* 6 */
"#else\n" /* 7 */
"static int guarded_alt(int x) {\n" /* 8 */
"#endif\n" /* 9 */
" return x + 1;\n" /* 10 */
"}\n" /* 11 */
"\n" /* 12 */
"void ok_after(void) { printf(\"b\"); }\n"; /* 13 */
static const char *C_CLEAN = "#include <stdio.h>\n"
"\n"
"void alpha(void) { printf(\"a\"); }\n"
"\n"
"static int beta(int x) {\n"
" return x + 1;\n"
"}\n";
/* `def broken(:` parses with an ERROR region, but tree-sitter error recovery
* still yields the `broken` function def — a DEFINITELY RECOVERED miss. */
static const char *PY_BROKEN_RECOVERED = "def ok():\n"
" return 1\n"
"\n"
"def broken(:\n"
" pass\n"
"\n"
"def ok2():\n"
" return 2\n";
/* Pure operator garbage between defs: an ERROR region no def walker can
* recover anything from — a genuine, unrecovered miss. */
static const char *PY_GARBAGE = "def ok():\n"
" return 1\n"
"\n"
"%%% ((( garbage ))) %%%\n"
"??? !!!\n"
"\n"
"def ok2():\n"
" return 2\n";
static const char *PY_CLEAN = "def ok():\n"
" return 1\n"
"\n"
"def ok2():\n"
" return 2\n";
/* #1610 fixtures follow. Refinement fixtures live here so they sit beside the
* split-brace fixture they build on. */
/* Same split-brace shape as C_IFDEF_SPLIT, plus real garbage further down.
* Guards against over-suppression: the preprocessor explains the guarded
* region but explains nothing about the garbage, so BOTH must stay flagged
* and they must be reported as two separate ranges, not one big one. */
static const char *C_IFDEF_SPLIT_PLUS_GARBAGE = "#include <stdio.h>\n" /* 1 */
"\n" /* 2 */
"void ok_before(void) { }\n" /* 3 */
"\n" /* 4 */
"#ifdef FEATURE_A\n" /* 5 */
"static int guarded(int x) {\n" /* 6 */
"#else\n" /* 7 */
"static int guarded_alt(int x) {\n" /* 8 */
"#endif\n" /* 9 */
" return x + 1;\n" /* 10 */
"}\n" /* 11 */
"\n" /* 12 */
"%%% ((( &&& ))) %%%\n" /* 13 */
"\n" /* 14 */
"void ok_after(void) { }\n"; /* 15 */
/* Perl formats have a line-oriented body terminated by a lone dot. The
* following named sub pins the important recovery boundary: a grammar must
* both accept the format and resume normal declaration parsing afterwards. */
static const char *PERL_FORMAT_WITH_FOLLOWING_SUB = "package Report;\n"
"format REPORT =\n"
"@<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"
"$headline\n"
".\n"
"sub after_format { return 1; }\n";
/* A grammar refresh must not hide real Perl syntax loss. */
static const char *PERL_MALFORMED = "package Broken;\n"
"sub before_error { return 1; }\n"
"} ] } ]\n"
"sub after_error { return 2; }\n";
/* ── Tests ────────────────────────────────────────────────────────────────── */
TEST(c_ifdef_split_brace_sets_parse_incomplete) {
CBMFileResult *r = do_extract(C_IFDEF_SPLIT, CBM_LANG_C, "split.c");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error); /* parse succeeded — this is the silent-partial class */
ASSERT_TRUE(r->parse_incomplete);
ASSERT_GTE(r->error_region_count, 1);
ASSERT_NOT_NULL(r->error_ranges);
ASSERT_GT((int)strlen(r->error_ranges), 0);
cbm_free_result(r);
PASS();
}
TEST(c_ifdef_split_brace_neighbors_still_extracted) {
/* Documents WHY the flag matters: the file is partially indexed —
* neighbors extract, so nothing else hints at the dropped region. */
CBMFileResult *r = do_extract(C_IFDEF_SPLIT, CBM_LANG_C, "split.c");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(has_def(r, "ok_before"));
ASSERT_TRUE(r->parse_incomplete);
cbm_free_result(r);
PASS();
}
TEST(c_error_range_points_at_failed_region) {
/* The recorded range must overlap the #ifdef construct (lines 5–11) so an
* agent can be pointed at the exact unparsed region. Format is
* "start-end[,start-end...]", 1-based, inclusive. */
CBMFileResult *r = do_extract(C_IFDEF_SPLIT, CBM_LANG_C, "split.c");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_NOT_NULL(r->error_ranges);
unsigned int start = 0;
unsigned int end = 0;
ASSERT_EQ(sscanf(r->error_ranges, "%u-%u", &start, &end), 2);
ASSERT_GTE(start, 1u);
ASSERT_LTE(start, 11u); /* starts at or before the region's last line */
ASSERT_GTE(end, 5u); /* ends at or after the region's first line */
ASSERT_LTE(end, 13u); /* never past EOF */
ASSERT_LTE(start, end);
cbm_free_result(r);
PASS();
}
TEST(c_clean_file_not_flagged) {
/* No false positives: a clean parse must stay completely unflagged. */
CBMFileResult *r = do_extract(C_CLEAN, CBM_LANG_C, "clean.c");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_FALSE(r->parse_incomplete);
ASSERT_EQ(r->error_region_count, 0);
ASSERT_NULL(r->error_ranges);
ASSERT_TRUE(has_def(r, "alpha"));
ASSERT_TRUE(has_def(r, "beta"));
cbm_free_result(r);
PASS();
}
TEST(py_unrecovered_garbage_sets_parse_incomplete) {
CBMFileResult *r = do_extract(PY_GARBAGE, CBM_LANG_PYTHON, "garbage.py");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_GTE(r->error_region_count, 1);
ASSERT_NOT_NULL(r->error_ranges);
ASSERT_TRUE(has_def(r, "ok")); /* partial: clean defs still extracted */
cbm_free_result(r);
PASS();
}
TEST(py_recovered_def_not_flagged) {
/* Recovery subtraction: `def broken(:` produces an ERROR region, but the
* def walker still recovers `broken` covering the whole region — the
* construct IS in the graph, so flagging it would be a false miss. */
CBMFileResult *r = do_extract(PY_BROKEN_RECOVERED, CBM_LANG_PYTHON, "broken.py");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(has_def(r, "broken")); /* the recovery that justifies unflagging */
ASSERT_FALSE(r->parse_incomplete);
ASSERT_EQ(r->error_region_count, 0);
ASSERT_NULL(r->error_ranges);
cbm_free_result(r);
PASS();
}
TEST(py_clean_file_not_flagged) {
CBMFileResult *r = do_extract(PY_CLEAN, CBM_LANG_PYTHON, "clean.py");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->parse_incomplete);
ASSERT_EQ(r->error_region_count, 0);
ASSERT_NULL(r->error_ranges);
cbm_free_result(r);
PASS();
}
/* Read the trailing "+<N>" truncation marker off a range string. Returns N, or
* 0 when the string carries no marker. */
static int ranges_dropped_marker(const char *ranges) {
const char *plus = ranges ? strrchr(ranges, '+') : NULL;
if (!plus || !isdigit((unsigned char)plus[1])) {
return 0;
}
return atoi(plus + 1);
}
TEST(error_region_cap_is_honored) {
/* Pathological input: many separate unrecoverable garbage blocks
* interleaved with valid defs. The collector must stay bounded by its
* 256-region cap (matches CBM_MAX_ERROR_REGIONS in cbm.c) — pathological
* input can't blow up the report, and the flag itself still fires. */
enum { GARBAGE_BLOCKS = 400, LINE_CAP = 256 };
char *src = (char *)malloc(GARBAGE_BLOCKS * 96 + 1);
ASSERT_NOT_NULL(src);
size_t off = 0;
for (int i = 0; i < GARBAGE_BLOCKS; i++) {
off += (size_t)snprintf(
src + off, 96, "def ok%d():\n return %d\n%%%%%% garbage%d ((( %%%%%%\n", i, i, i);
}
CBMFileResult *r = do_extract(src, CBM_LANG_PYTHON, "many_errors.py");
free(src);
ASSERT_NOT_NULL(r);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_GTE(r->error_region_count, 1);
ASSERT_LTE(r->error_region_count, LINE_CAP);
ASSERT_NOT_NULL(r->error_ranges);
cbm_free_result(r);
PASS();
}
/* A clipped range list must say so. 400 garbage blocks overrun the 256-region
* cap, so the report keeps 256 ranges and ends with a "+<N>" marker naming the
* number thrown away. Without the marker the short list reads as a complete
* one, which is the whole defect this guards. */
TEST(error_region_cap_reports_what_it_dropped) {
enum { GARBAGE_BLOCKS = 400, LINE_CAP = 256 };
char *src = (char *)malloc(GARBAGE_BLOCKS * 96 + 1);
ASSERT_NOT_NULL(src);
size_t off = 0;
for (int i = 0; i < GARBAGE_BLOCKS; i++) {
off += (size_t)snprintf(
src + off, 96, "def ok%d():\n return %d\n%%%%%% garbage%d ((( %%%%%%\n", i, i, i);
}
CBMFileResult *r = do_extract(src, CBM_LANG_PYTHON, "cap_marker.py");
free(src);
ASSERT_NOT_NULL(r);
ASSERT_NOT_NULL(r->error_ranges);
/* The cap bound, so the kept list is full and the marker is present. */
ASSERT_EQ(r->error_region_count, LINE_CAP);
int dropped = ranges_dropped_marker(r->error_ranges);
ASSERT_GTE(dropped, 1);
/* Every block produces at most one region, so the total cannot exceed the
* number of blocks — a marker that overcounts would fail here. */
ASSERT_LTE(r->error_region_count + dropped, GARBAGE_BLOCKS);
/* The marker is a SUFFIX: nothing follows it, or a reader stops early and
* silently loses every range after it. */
const char *plus = strrchr(r->error_ranges, '+');
ASSERT_NOT_NULL(plus);
for (const char *c = plus + 1; *c; c++) {
ASSERT_TRUE(isdigit((unsigned char)*c));
}
cbm_free_result(r);
PASS();
}
/* Inverse guard: a file that stays under the cap must carry NO marker, or
* every ordinary report would look clipped. */
TEST(uncapped_ranges_carry_no_marker) {
const char *src = "def ok():\n return 1\n%%% garbage (((\ndef ok2():\n return 2\n";
CBMFileResult *r = do_extract(src, CBM_LANG_PYTHON, "small.py");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_NOT_NULL(r->error_ranges);
ASSERT_EQ(ranges_dropped_marker(r->error_ranges), 0);
ASSERT_NULL(strchr(r->error_ranges, '+'));
cbm_free_result(r);
PASS();
}
/* Trailing recovered functions AFTER the failed #ifdef region must not
* unflag it: recovery evidence must originate INSIDE the region, and the
* unrecovered lines (the first branch's `guarded`) keep it flagged. */
TEST(c_trailing_recovered_defs_keep_flag) {
const char *src = "void ok_before(void) { }\n"
"#ifdef A\n"
"static int guarded(int x) {\n"
"#else\n"
"static int guarded_alt(int x) {\n"
"#endif\n"
" return x + 1;\n"
"}\n"
"void ok_after(void) { }\n"
"static int nested_ok(int y) { return y; }\n";
CBMFileResult *r = do_extract(src, CBM_LANG_C, "probe.c");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(has_def(r, "guarded_alt")); /* partial recovery inside the region */
ASSERT_TRUE(r->parse_incomplete); /* ...but `guarded` is still lost */
ASSERT_GTE(r->error_region_count, 1);
cbm_free_result(r);
PASS();
}
/* ── Suite ────────────────────────────────────────────────────────────────── */
/* ── #1610: a missing FINAL NEWLINE is not a parse failure ────────────────────
*
* A file that does not end with "\n" leaves the grammar's mandatory line
* terminator MISSING. That node is ZERO-WIDTH and sits at EOF: the parser
* consumed no source for it, so by construction nothing was dropped — no
* construct can live in a zero-byte span. Every instruction still parses.
*
* Reported on #1610 for Dockerfile, where a reporter proved with a byte-exact
* matrix that the trigger is independent of BOM, CRLF/LF, exec-form vs
* shell-form and file length — it is purely the absent final newline.
*
* It was never Dockerfile-specific: tcl, fish, gomod and hyprlang flag the same
* way, while ini, fsharp, beancount and others do NOT — only because those
* grammars declare the terminator token hidden rather than visible. Whether a
* user saw a phantom parse_partial came down to a grammar-authoring accident.
*
* The cost was not cosmetic: a phantom flag writes a "<project>::missed" shadow
* row, and until #1609 that row removed the whole project from cross-repo
* linking, as source AND as target. */
TEST(dockerfile_missing_final_newline_not_flagged_issue1610) {
const char *src = "FROM mcr.microsoft.com/dotnet/aspnet:8.0\n"
"ENTRYPOINT [\"dotnet\", \"App.dll\"]"; /* deliberately no \n */
CBMFileResult *r = do_extract(src, CBM_LANG_DOCKERFILE, "Dockerfile");
ASSERT_NOT_NULL(r);
bool flagged = r->parse_incomplete;
cbm_free_result(r);
if (flagged) {
FAIL("a Dockerfile lacking only its final newline must not be parse_partial");
}
PASS();
}
/* The same bytes WITH the newline must stay clean — pins the equivalence the
* reporter's matrix proved, so a future change cannot "fix" one by breaking the
* other. */
TEST(dockerfile_with_final_newline_still_clean_issue1610) {
const char *src = "FROM mcr.microsoft.com/dotnet/aspnet:8.0\n"
"ENTRYPOINT [\"dotnet\", \"App.dll\"]\n";
CBMFileResult *r = do_extract(src, CBM_LANG_DOCKERFILE, "Dockerfile");
ASSERT_NOT_NULL(r);
bool flagged = r->parse_incomplete;
cbm_free_result(r);
if (flagged) {
FAIL("a terminated Dockerfile must not be parse_partial");
}
PASS();
}
/* #1746: on Windows, a Dockerfile whose final instruction is followed by one
* ASCII space and then EOF was reported as parse_partial. */
TEST(dockerfile_trailing_space_at_eof_not_flagged_issue1746) {
const char *src = "FROM scratch\nENTRYPOINT [\"a\"] ";
CBMFileResult *r = do_extract(src, CBM_LANG_DOCKERFILE, "Dockerfile");
ASSERT_NOT_NULL(r);
bool flagged = r->parse_incomplete;
if (flagged) {
fprintf(stderr, " exact issue #1746 fixture flagged: ranges=%s\n",
r->error_ranges ? r->error_ranges : "(none)");
}
cbm_free_result(r);
if (flagged) {
FAIL("trailing horizontal whitespace at Dockerfile EOF must not be parse_partial");
}
PASS();
}
/* The same bytes WITH the LF must stay clean. This is a separate test so the
* control executes even while the exact affected fixture is RED. */
TEST(dockerfile_trailing_space_with_final_newline_clean_issue1746) {
const char *src = "FROM scratch\nENTRYPOINT [\"a\"] \n";
CBMFileResult *r = do_extract(src, CBM_LANG_DOCKERFILE, "Dockerfile");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->parse_incomplete);
cbm_free_result(r);
PASS();
}
/* Removing the trailing space while retaining EOF must also stay clean. */
TEST(dockerfile_without_trailing_space_at_eof_clean_issue1746) {
const char *src = "FROM scratch\nENTRYPOINT [\"a\"]";
CBMFileResult *r = do_extract(src, CBM_LANG_DOCKERFILE, "Dockerfile");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->parse_incomplete);
cbm_free_result(r);
PASS();
}
/* The reporter also observed the same failure when the first line uses CRLF. */
TEST(dockerfile_crlf_trailing_space_at_eof_not_flagged_issue1746) {
const char *src = "FROM scratch\r\nENTRYPOINT [\"a\"] ";
CBMFileResult *r = do_extract(src, CBM_LANG_DOCKERFILE, "Dockerfile");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->parse_incomplete);
cbm_free_result(r);
PASS();
}
/* Language-general, not a Dockerfile patch: these four were each proven to flag
* on a stripped trailing newline. */
TEST(missing_final_newline_not_flagged_across_grammars_issue1610) {
struct {
const char *src;
CBMLanguage lang;
const char *path;
} cases[] = {
{"proc foo {} {}\nproc bar {} {}", CBM_LANG_TCL, "a.tcl"},
{"function foo\n echo hi\nend", CBM_LANG_FISH, "a.fish"},
{"module example.com/m\n\ngo 1.21", CBM_LANG_GOMOD, "go.mod"},
{"general {\n gaps_in = 5\n}", CBM_LANG_HYPRLANG, "hypr.conf"},
};
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
CBMFileResult *r = do_extract(cases[i].src, cases[i].lang, cases[i].path);
ASSERT_NOT_NULL(r);
bool flagged = r->parse_incomplete;
if (flagged) {
fprintf(stderr, " %s flagged: ranges=%s\n", cases[i].path,
r->error_ranges ? r->error_ranges : "(none)");
}
cbm_free_result(r);
if (flagged) {
FAIL("an unterminated final line must not be parse_partial in any grammar");
}
}
PASS();
}
/* GUARD (the reason this suppression is safe rather than convenient): the rule
* is ZERO-WIDTH AT EOF only. A real failure earlier in the file must still be
* reported, and its range must name the broken line — not be swallowed along
* with the terminator. */
TEST(real_error_before_eof_still_flagged_without_final_newline_issue1610) {
/* Built from C_IFDEF_SPLIT, the fixture this suite already proves is
* flagged, with its trailing newline removed. Two conditions now hold at
* once: a genuine width-bearing ERROR mid-file, AND an unterminated last
* line. Suppressing the EOF terminator must not swallow the real one. */
size_t n = strlen(C_IFDEF_SPLIT);
char *unterminated = (char *)malloc(n + 1);
ASSERT_NOT_NULL(unterminated);
memcpy(unterminated, C_IFDEF_SPLIT, n);
unterminated[n - 1] = '\0'; /* drop the final newline */
CBMFileResult *r = do_extract(unterminated, CBM_LANG_C, "split.c");
free(unterminated);
ASSERT_NOT_NULL(r);
bool flagged = r->parse_incomplete;
bool has_ranges = r->error_ranges != NULL;
cbm_free_result(r);
if (!flagged) {
FAIL("a real mid-file parse failure must still be reported when the file also lacks its "
"final newline");
}
if (!has_ranges) {
FAIL("a reported failure must still name its line range");
}
PASS();
}
/* GUARD: a MISSING/ERROR node WITH WIDTH at EOF is a genuine loss and must
* still be flagged. The construct here is broken whether or not the line is
* terminated, so neither the zero-width rule nor the virtual final newline
* (#2078) may excuse it.
*
* This guard used a Makefile whose final recipe line lacks its newline, which
* really did drop the recipe from the tree. Since #2078 the parser sees that
* line terminated and the recipe parses, so there is no loss left to report --
* see makefile_unterminated_recipe_is_parsed_issue2078. */
TEST(width_bearing_error_at_eof_still_flagged_issue1610) {
const char *src = "def ok():\n return 1\nx = (1,"; /* unclosed at EOF */
CBMFileResult *r = do_extract(src, CBM_LANG_PYTHON, "eof.py");
ASSERT_NOT_NULL(r);
bool flagged = r->parse_incomplete;
cbm_free_result(r);
if (!flagged) {
FAIL("a width-bearing parse failure at EOF must still be reported");
}
PASS();
}
/* #2078: the recipe the old #1610 guard lost is now parsed -- the file is not
* flagged, with or without a trailing blank. */
TEST(makefile_unterminated_recipe_is_parsed_issue2078) {
const char *cases[] = {"all:\n\techo hi", "all:\n\techo hi "};
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
CBMFileResult *r = do_extract(cases[i], CBM_LANG_MAKEFILE, "Makefile");
ASSERT_NOT_NULL(r);
bool flagged = r->parse_incomplete;
cbm_free_result(r);
if (flagged) {
FAIL("an unterminated final recipe line must parse like a terminated one");
}
}
PASS();
}
/* #1838: tree-sitter-perl v1.0.0 rejects a valid line-oriented format and
* reports the file as partial. The supported upstream v1.2.1 grammar accepts
* the format and preserves declaration extraction beyond its dot terminator. */
TEST(perl_format_followed_by_named_sub_is_complete_issue1838) {
CBMFileResult *r = do_extract(PERL_FORMAT_WITH_FOLLOWING_SUB, CBM_LANG_PERL, "report.pl");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
bool partial = r->parse_incomplete;
int error_regions = r->error_region_count;
bool has_error_ranges = r->error_ranges != NULL;
bool has_following_sub = has_def(r, "after_format");
if (partial || error_regions != 0 || has_error_ranges || !has_following_sub) {
fprintf(stderr, " Perl format result: partial=%d regions=%d ranges=%s following_sub=%d\n",
partial, error_regions, r->error_ranges ? r->error_ranges : "(none)",
has_following_sub);
cbm_free_result(r);
FAIL("a valid Perl format and its following named sub must parse completely");
}
cbm_free_result(r);
PASS();
}
TEST(perl_malformed_source_remains_partial_issue1838) {
CBMFileResult *r = do_extract(PERL_MALFORMED, CBM_LANG_PERL, "broken.pl");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_GTE(r->error_region_count, 1);
ASSERT_NOT_NULL(r->error_ranges);
cbm_free_result(r);
PASS();
}
/* ── #1746: trailing blanks before EOF are still just an absent newline ───────
*
* #1610 suppressed the zero-width MISSING terminator but tested for it with
* `end == source_len`. Trailing blanks are extras owned by no node, so
* `ENTRYPOINT ["a"] ` + EOF parks it at [29,29) while source_len is 30.
*
* The reporter's byte-exact controls pin the trigger to the PAIR: `] ` + EOF
* flags, `] ` + newline is clean, `]` + EOF is clean. */
TEST(dockerfile_trailing_blank_at_eof_not_flagged_issue1746) {
const char *cases[] = {
"FROM scratch\nENTRYPOINT [\"a\"] ", /* space + EOF — the report */
"FROM scratch\nENTRYPOINT [\"a\"]\t", /* tab + EOF */
"FROM scratch\nENTRYPOINT [\"a\"]\v", /* vertical tab + EOF */
"FROM scratch\nENTRYPOINT [\"a\"]\f", /* form feed + EOF */
"FROM scratch\nENTRYPOINT [\"a\"] \t ", /* run of blanks + EOF */
"FROM scratch\nENTRYPOINT [\"a\"] \r", /* CRLF file truncated to CR */
};
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
CBMFileResult *r = do_extract(cases[i], CBM_LANG_DOCKERFILE, "Dockerfile");
ASSERT_NOT_NULL(r);
bool flagged = r->parse_incomplete;
if (flagged) {
fprintf(stderr, " case %zu flagged: ranges=%s\n", i,
r->error_ranges ? r->error_ranges : "(none)");
}
cbm_free_result(r);
if (flagged) {
FAIL("trailing blanks must not turn an absent final newline into parse_partial");
}
}
PASS();
}
/* GUARD: widening the tail must not swallow a genuine mid-file failure just
* because the file happens to end in blanks. */
TEST(real_error_before_eof_still_flagged_with_trailing_blank_issue1746) {
size_t n = strlen(C_IFDEF_SPLIT);
char *buf = (char *)malloc(n + 1);
ASSERT_NOT_NULL(buf);
memcpy(buf, C_IFDEF_SPLIT, n + 1);
buf[n - 1] = ' '; /* final newline becomes a blank */
CBMFileResult *r = do_extract(buf, CBM_LANG_C, "split.c");
free(buf);
ASSERT_NOT_NULL(r);
bool flagged = r->parse_incomplete;
bool has_ranges = r->error_ranges != NULL;
cbm_free_result(r);
if (!flagged) {
FAIL("a real mid-file parse failure must still be reported when the file ends in blanks");
}
if (!has_ranges) {
FAIL("a reported failure must still name its line range");
}
PASS();
}
/* GUARD: a WIDTH-BEARING loss at EOF stays honest with a blank tail too, and
* only zero-width nodes are excused. (Formerly the unterminated Makefile
* recipe, which #2078 now parses -- see the #1610 guard above.) */
TEST(width_bearing_error_at_eof_still_flagged_with_trailing_blank_issue1746) {
const char *src = "def ok():\n return 1\nx = (1, ";
CBMFileResult *r = do_extract(src, CBM_LANG_PYTHON, "eof.py");
ASSERT_NOT_NULL(r);
bool flagged = r->parse_incomplete;
cbm_free_result(r);
if (!flagged) {
FAIL("a width-bearing parse failure at EOF must still be reported when the file ends in "
"blanks");
}
PASS();
}
/* ── Phase 2: refine raw ranges with the preprocessed tree ──────────────────
*
* The raw parse sees both #ifdef branches at once, so its ERROR node covers
* the whole guarded construct (lines 5-11). The PREPROCESSED parse sees only
* the branch the preprocessor picked, and parses it clean. Every original
* line that shows up clean in that second parse is therefore accounted for,
* and reporting it as unparsed is false.
*
* What is left is the branch the preprocessor threw away — line 6 here. That
* one really is missing from the graph, so it stays flagged. Directive lines
* (#ifdef / #else / #endif) hold no construct, so a range never starts or
* ends on one.
*/
/* Return 1 if the "a-b,c-d" range string covers 1-based `line`. */
static int ranges_cover_line(const char *ranges, unsigned int line) {
const char *p = ranges;
while (p && *p) {
unsigned int s = 0, e = 0;
if (sscanf(p, "%u-%u", &s, &e) == 2 && line >= s && line <= e) {
return 1;
}
p = strchr(p, ',');
if (p) {
p++;
}
}
return 0;
}
/* Total lines covered by every range in the string. */
static unsigned int ranges_total_span(const char *ranges) {
const char *p = ranges;
unsigned int total = 0;
while (p && *p) {
unsigned int s = 0, e = 0;
if (sscanf(p, "%u-%u", &s, &e) == 2 && e >= s) {
total += e - s + 1;
}
p = strchr(p, ',');
if (p) {
p++;
}
}
return total;
}
TEST(c_ifdef_split_range_narrows_to_dropped_branch) {
/* RED before the refinement: the raw range covers the whole 5-11
* construct. GREEN after: only line 6, the branch the preprocessor did
* not pick, is still reported. */
CBMFileResult *r = do_extract(C_IFDEF_SPLIT, CBM_LANG_C, "split.c");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_NOT_NULL(r->error_ranges);
ASSERT_TRUE(ranges_cover_line(r->error_ranges, 6u)); /* dropped branch */
ASSERT_LTE(ranges_total_span(r->error_ranges), 3u); /* was 7 lines */
cbm_free_result(r);
PASS();
}
TEST(c_ifdef_split_range_excludes_lines_the_preprocessor_explained) {
/* Lines 10 and 11 are the shared body and closing brace. They parse
* clean once a branch is chosen, so pointing an agent at them is wrong. */
CBMFileResult *r = do_extract(C_IFDEF_SPLIT, CBM_LANG_C, "split.c");
ASSERT_NOT_NULL(r);
ASSERT_NOT_NULL(r->error_ranges);
ASSERT_FALSE(ranges_cover_line(r->error_ranges, 10u));
ASSERT_FALSE(ranges_cover_line(r->error_ranges, 11u));
ASSERT_FALSE(ranges_cover_line(r->error_ranges, 3u)); /* ok_before */
cbm_free_result(r);
PASS();
}
TEST(c_ifdef_split_range_never_starts_on_a_directive) {
/* Lines 5, 7 and 9 are bare #ifdef / #else / #endif. No construct can
* live on them, so they must not appear in a range. */
CBMFileResult *r = do_extract(C_IFDEF_SPLIT, CBM_LANG_C, "split.c");
ASSERT_NOT_NULL(r);
ASSERT_NOT_NULL(r->error_ranges);
ASSERT_FALSE(ranges_cover_line(r->error_ranges, 5u));
ASSERT_FALSE(ranges_cover_line(r->error_ranges, 7u));
ASSERT_FALSE(ranges_cover_line(r->error_ranges, 9u));
cbm_free_result(r);
PASS();
}
TEST(c_refinement_does_not_suppress_real_garbage) {
/* Anti-over-suppression. The preprocessor cannot explain line 13, so it
* stays flagged even though the guarded region above it narrowed. */
CBMFileResult *r = do_extract(C_IFDEF_SPLIT_PLUS_GARBAGE, CBM_LANG_C, "both.c");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_NOT_NULL(r->error_ranges);
ASSERT_TRUE(ranges_cover_line(r->error_ranges, 13u)); /* the garbage */
ASSERT_FALSE(ranges_cover_line(r->error_ranges, 3u)); /* ok_before */
cbm_free_result(r);
PASS();
}
TEST(c_clean_file_stays_unflagged_after_refinement) {
/* The refinement must never invent a range on a file that parses. */
CBMFileResult *r = do_extract(C_CLEAN, CBM_LANG_C, "clean.c");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->parse_incomplete);
ASSERT_NULL(r->error_ranges);
cbm_free_result(r);
PASS();
}
/* The whole-file class, and the reason the parse_unusable kind exists.
*
* The Phase 2 refinement that narrows a whole-file range using the
* preprocessed parse only runs for C, C++ and CUDA. A Python file whose root
* node is ERROR gets no such help, so it still reports one range covering
* every line — and one range over 80% of a file is not advice worth printing. */
TEST(python_whole_file_error_is_unusable) {
const char *src = ")))\n((( \n]]] [[[\ndef x(:\n";
CBMFileResult *r = do_extract(src, CBM_LANG_PYTHON, "unparseable.py");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_TRUE(r->parse_unusable);
ASSERT_EQ(r->error_region_count, 1);
ASSERT_NOT_NULL(r->error_ranges);
cbm_free_result(r);
PASS();
}
/* Inverse guard, and the one that keeps the kind meaningful: a file with a
* real but LOCAL parse failure must stay parse_partial. If this flipped, every
* flagged file would say "read the source" and the ranges would stop earning
* their keep. */
TEST(local_error_stays_partial_not_unusable) {
const char *src = "def ok():\n return 1\n%%% garbage (((\ndef ok2():\n return 2\n"
"def ok3():\n return 3\ndef ok4():\n return 4\n"
"def ok5():\n return 5\ndef ok6():\n return 6\n";
CBMFileResult *r = do_extract(src, CBM_LANG_PYTHON, "local_error.py");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_FALSE(r->parse_unusable);
cbm_free_result(r);
PASS();
}
/* A clean file is neither. */
TEST(clean_file_is_neither_partial_nor_unusable) {
CBMFileResult *r = do_extract(C_CLEAN, CBM_LANG_C, "clean_kinds.c");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->parse_incomplete);
ASSERT_FALSE(r->parse_unusable);
cbm_free_result(r);
PASS();
}
/* The C file that started this work must NOT land in the unusable class. Its
* whole-file range is exactly what Phase 2 broke up, so if this ever flips
* back to true the refinement has stopped working. */
TEST(c_ifdef_split_is_partial_never_unusable) {
CBMFileResult *r = do_extract(C_IFDEF_SPLIT, CBM_LANG_C, "split_kind.c");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_FALSE(r->parse_unusable);
cbm_free_result(r);
PASS();
}
/* Phase 0 finding 2, pinned so a tree-sitter bump cannot change it quietly.
*
* The C grammar handles `_Thread_local` unevenly, and these are the three
* forms measured on the grammar shipped today:
*
* static _Thread_local int x = 0; parses clean
* static _Thread_local int *p; parses clean
* static _Thread_local char b[8]; fails — flagged as range 1-1
*
* The array line really is missing from the graph, so flagging it is the
* honest answer, not a false positive. This test exists to make a grammar
* bump visible: if a newer grammar fixes the array form, this goes red and
* says so, instead of leaving a wrong note in the plan. (The plan's Phase 0
* also listed the pointer form as failing. It does not fail today.) */
TEST(c_thread_local_grammar_limit_is_pinned_issue963) {
/* The init form used to read as "parses clean", but that was a masking
* effect, not a clean parse: the grammar leaves an ERROR on line 1 and
* salvages a one-line Variable named `int` from it, which the old
* all-or-nothing recovery rule accepted as evidence the line was
* understood. Since 2026-09-16 a one-line salvage is not evidence, so the
* line is flagged — honest, because `x` is what the graph lacks. */
CBMFileResult *ok = do_extract("static _Thread_local int x = 0;\n"
"void f(void) { x = 1; }\n",
CBM_LANG_C, "tls_init.c");
ASSERT_NOT_NULL(ok);
ASSERT_TRUE(ok->parse_incomplete);
ASSERT_NOT_NULL(ok->error_ranges);
ASSERT_STR_EQ("1-1", ok->error_ranges);
ASSERT_TRUE(has_def(ok, "f"));
cbm_free_result(ok);
/* The pointer form: the same masking, the same honest answer now. */
CBMFileResult *ptr = do_extract("static _Thread_local int *p;\n"
"void f(void) { p = 0; }\n",
CBM_LANG_C, "tls_ptr.c");
ASSERT_NOT_NULL(ptr);
ASSERT_TRUE(ptr->parse_incomplete);
ASSERT_NOT_NULL(ptr->error_ranges);
ASSERT_STR_EQ("1-1", ptr->error_ranges);
ASSERT_TRUE(has_def(ptr, "f"));
cbm_free_result(ptr);
CBMFileResult *arr = do_extract("static _Thread_local char b[8];\n"
"void f(void) { b[0] = 0; }\n",
CBM_LANG_C, "tls_arr.c");
ASSERT_NOT_NULL(arr);
ASSERT_TRUE(arr->parse_incomplete);
ASSERT_NOT_NULL(arr->error_ranges);
/* The range names the one broken line, not the whole file. */
ASSERT_STR_EQ("1-1", arr->error_ranges);
/* The clean function below it still reaches the graph. */
ASSERT_TRUE(has_def(arr, "f"));
cbm_free_result(arr);
PASS();
}
/* Two error nodes can sit on ONE line. Line 113 of scripts/setup-windows.ps1
* does exactly that, and the report used to read "113-113,113-113" — the same
* line named twice. A line range says nothing new the second time, so repeated
* or overlapping regions must collapse into one. */
static const char *PS_TWO_ERRORS_ONE_LINE = "Write-Host \"start\"\n" /* 1 */
"wsl.exe -- bash -c $Command 2>&1\n" /* 2 */
"Write-Host \"end\"\n"; /* 3 */
/* An error region that runs to the end of the file stops just after the last
* newline. Tree-sitter calls that position row N, column 0 — a row that holds
* no text. Reading it as a line number named a line past the end of the file:
* scripts/setup-windows.ps1 has 326 lines and the report said "245-327". */
static const char *PS_ERROR_TO_EOF = "} else {\n" /* 1 */
" if ($a) {\n" /* 2 */
" Write-Host x\n" /* 3 */
"}\n"; /* 4 */
TEST(coverage_repeated_error_line_reports_one_range_issue963) {
CBMFileResult *r =
cbm_extract_file(PS_TWO_ERRORS_ONE_LINE, (int)strlen(PS_TWO_ERRORS_ONE_LINE),
CBM_LANG_POWERSHELL, "covproj", "two_errors.ps1", 0, NULL, NULL);
ASSERT_NOT_NULL(r);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_NOT_NULL(r->error_ranges);
/* Line 2 carries two separate error nodes. It must be named once. */
ASSERT_STR_EQ(r->error_ranges, "2-2");
ASSERT_EQ(r->error_region_count, 1);
cbm_free_result(r);
PASS();
}
TEST(coverage_range_never_ends_past_the_last_line_issue963) {
int len = (int)strlen(PS_ERROR_TO_EOF);
CBMFileResult *r = cbm_extract_file(PS_ERROR_TO_EOF, len, CBM_LANG_POWERSHELL, "covproj",
"error_to_eof.ps1", 0, NULL, NULL);
ASSERT_NOT_NULL(r);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_NOT_NULL(r->error_ranges);
/* The file has four lines and ends with a newline. Line 5 does not exist. */
ASSERT_STR_EQ(r->error_ranges, "1-4");
ASSERT_NULL(strstr(r->error_ranges, "5"));
cbm_free_result(r);
PASS();
}
/* ── Residual ranges (2026-09-16) ─────────────────────────────────────────
* The recovery subtraction used to be all-or-nothing: a region stayed flagged
* whole unless every one of its lines was covered by a definition that started
* inside it. On torvalds/linux one ERROR node that swallowed the second half of
* kernel/sched/core.c (lines 5522-11284) survived on the strength of the
* comment and macro lines BETWEEN its 286 extracted functions, and
* check_index_coverage told a reader 68 % of the scheduler was unindexed. The
* ranges must now name only the lines no extracted definition covers. */
/* A brace-unbalanced junk line between two clean functions: the recovery
* walker still extracts both functions, so the reported range must not cover
* either of them, only the junk. */
static const char *C_JUNK_BETWEEN_FUNCTIONS = "int alpha(void) {\n" /* 1 */
" return 1;\n" /* 2 */
"}\n" /* 3 */
"\n" /* 4 */
"} ] junk ( {\n" /* 5 */
"\n" /* 6 */
"int beta(void) {\n" /* 7 */
" return 2;\n" /* 8 */
"}\n" /* 9 */
"\n" /* 10 */
"int gamma(void) {\n" /* 11 */
" return 3;\n" /* 12 */
"}\n"; /* 13 */
static int range_covers_line(const char *ranges, unsigned int line) {
const char *p = ranges;
while (p && *p) {
unsigned int s = 0;
unsigned int e = 0;
if (sscanf(p, "%u-%u", &s, &e) == 2 && s <= line && line <= e) {
return 1;
}
p = strchr(p, ',');
if (p) {
p++;
}
}
return 0;
}
TEST(coverage_range_never_covers_an_extracted_definition) {
CBMFileResult *r = do_extract(C_JUNK_BETWEEN_FUNCTIONS, CBM_LANG_C, "junk.c");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(has_def(r, "alpha"));
ASSERT_TRUE(has_def(r, "beta"));
ASSERT_TRUE(has_def(r, "gamma"));
ASSERT_TRUE(r->parse_incomplete);
ASSERT_NOT_NULL(r->error_ranges);
/* Every extracted definition's own lines are in the graph, so no reported
* range may cover its start line. */
for (int i = 0; i < r->defs.count; i++) {
const CBMDefinition *d = &r->defs.items[i];
if (!d->label || strcmp(d->label, "Module") == 0) {
continue;
}
ASSERT_FALSE(range_covers_line(r->error_ranges, d->start_line));
}
/* And the junk line itself is still reported. */
ASSERT_TRUE(range_covers_line(r->error_ranges, 5u));
cbm_free_result(r);
PASS();
}
/* The lines between recovered definitions are comments and blanks here: a gap
* with no code in it is not a miss, so nothing may be reported for it. The
* junk line stays reported. */
static const char *C_JUNK_WITH_COMMENT_GAPS = "int alpha(void) {\n" /* 1 */
" return 1;\n" /* 2 */
"}\n" /* 3 */
"/* between alpha and the junk */\n" /* 4 */
"} ] junk ( {\n" /* 5 */
"// trailing note\n" /* 6 */
"int beta(void) {\n" /* 7 */
" return 2;\n" /* 8 */
"}\n"; /* 9 */
TEST(coverage_gap_of_only_comments_is_not_a_miss) {
CBMFileResult *r = do_extract(C_JUNK_WITH_COMMENT_GAPS, CBM_LANG_C, "gaps.c");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(has_def(r, "alpha"));
ASSERT_TRUE(has_def(r, "beta"));
ASSERT_TRUE(r->parse_incomplete);
ASSERT_NOT_NULL(r->error_ranges);
ASSERT_TRUE(range_covers_line(r->error_ranges, 5u));
ASSERT_FALSE(range_covers_line(r->error_ranges, 1u));
ASSERT_FALSE(range_covers_line(r->error_ranges, 7u));
ASSERT_FALSE(range_covers_line(r->error_ranges, 9u));
cbm_free_result(r);
PASS();
}
/* ── #1735: SQL data dumps ───────────────────────────────────────────────────
* A mysqldump file is a few CREATE TABLEs followed by megabytes of literal
* INSERT rows. Tree-sitter built a full tree for every row until the parse
* budget ran out, and the whole file — tables included — was skipped as
* "parse timeout". Literal-only rows after the first of each VALUES list are
* now kept out of the parse (sql_values.c). These tests pin that nothing the
* full parse extracted outside those rows is lost (and, for standard '' escapes
* the grammar reads correctly, that nothing changes at all), that a row which
* can reference something is still parsed, and that the parse no longer grows
* with the row count. */
typedef struct {
char *s;
size_t len;
size_t cap;
} cov_buf_t;
static void cov_put(cov_buf_t *b, const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
va_list ap2;
va_copy(ap2, ap);
int n = vsnprintf(NULL, 0, fmt, ap);
va_end(ap);
if (n < 0) {
va_end(ap2);
return;
}
if (b->len + (size_t)n + 1 > b->cap) {
size_t cap = b->cap ? b->cap : 4096;
while (b->len + (size_t)n + 1 > cap) {
cap *= 2;
}
char *grown = realloc(b->s, cap);
if (!grown) {
va_end(ap2);
return;
}
b->s = grown;
b->cap = cap;
}
vsnprintf(b->s + b->len, (size_t)n + 1, fmt, ap2);
va_end(ap2);
b->len += (size_t)n;
}
static const char *const DUMP_WORDS[] = {"Kabul", "Herat", "Amsterdam", "O'Brien",
"Haag", "Rotterdam", "Zuid-Holland", "Utrecht"};
enum { DUMP_WORD_COUNT = sizeof(DUMP_WORDS) / sizeof(DUMP_WORDS[0]) };
static const char *const DUMP_ODD[] = {"NULL", "-12.5", "0x1F", "_binary 'ab'", "DEFAULT",
"TRUE", "1e-3", "X'0A'", "''", "'a''b'"};
enum { DUMP_ODD_COUNT = sizeof(DUMP_ODD) / sizeof(DUMP_ODD[0]) };
/* Deterministic mysqldump-shaped SQL: two tables and a view, then `stmts`
* extended INSERTs of `rows` tuples. `mysql_esc` writes a quote inside a string
* as \' (MySQL) instead of '' (standard). `mixed` puts one tuple holding a
* subquery and one holding a function call in the middle of every INSERT. The
* first two INSERTs separate their tuples with a newline and with a comment, the
* way pretty-printed dumps do. */
static char *sql_dump(int stmts, int rows, bool mysql_esc, bool mixed) {
cov_buf_t b = {NULL, 0, 0};
cov_put(&b, "-- MySQL dump 10.13\n"
"/*!40101 SET NAMES utf8mb4 */;\n"
"CREATE TABLE `city` (\n"
" `ID` int NOT NULL,\n"
" `Name` char(35) NOT NULL DEFAULT '',\n"
" `CountryCode` char(3),\n"
" `Note` text,\n"
" `Population` int\n"
");\n"
"CREATE TABLE country (Code char(3), Name char(52));\n"
"CREATE VIEW big_cities AS SELECT Name FROM city WHERE Population > 1000000;\n"
"LOCK TABLES `city` WRITE;\n");
uint32_t seed = 1735;
int id = 1;
for (int s = 0; s < stmts; s++) {
cov_put(&b, "INSERT INTO `city` VALUES ");
const char *sep = s == 0 ? ",\n" : (s == 1 ? ", /* split */ " : ",");
for (int r = 0; r < rows; r++) {
if (r > 0) {
cov_put(&b, "%s", sep);
}
if (mixed && r == rows / 2) {
cov_put(&b, "((SELECT MAX(Code) FROM country),'x','AAA',NULL,1)");
continue;
}
if (mixed && r == rows / 2 + 1) {
cov_put(&b, "(%d,UPPER('y'),'BBB',NULL,2)", id++);
continue;
}
seed = seed * 1103515245u + 12345u;
const char *w = DUMP_WORDS[(seed >> 8) % DUMP_WORD_COUNT];
cov_put(&b, "(%d,'", id++);
for (const char *p = w; *p; p++) {
cov_put(&b, *p == '\'' ? (mysql_esc ? "\\'" : "''") : "%c", *p);
}
cov_put(&b, "','%c%c%c',%s,%u)", 'A' + (int)(seed % 26), 'A' + (int)((seed >> 5) % 26),
'A' + (int)((seed >> 10) % 26), DUMP_ODD[(seed >> 12) % DUMP_ODD_COUNT],
(seed >> 3) % 9000000u);
}
cov_put(&b, ";\n");
}
cov_put(&b, "UNLOCK TABLES;\n");
return b.s;
}
static void fp_s(cov_buf_t *b, const char *s) {
cov_put(b, "%s|", s ? s : "~");
}
/* Everything the graph is built from except calls and usages, in order (those
* two are compared site by site, sites_agree). */
static char *result_fingerprint(const CBMFileResult *r) {
cov_buf_t b = {NULL, 0, 0};
for (int i = 0; i < r->defs.count; i++) {
const CBMDefinition *d = &r->defs.items[i];
fp_s(&b, d->label);
fp_s(&b, d->name);
fp_s(&b, d->qualified_name);
fp_s(&b, d->signature);
fp_s(&b, d->return_type);
fp_s(&b, d->structural_profile);
fp_s(&b, d->body_tokens);
cov_put(&b, "%u-%u c%d l%d\n", d->start_line, d->end_line, d->complexity, d->lines);
}
for (int i = 0; i < r->imports.count; i++) {
fp_s(&b, r->imports.items[i].local_name);
fp_s(&b, r->imports.items[i].module_path);
cov_put(&b, "imp\n");
}
for (int i = 0; i < r->rw.count; i++) {
fp_s(&b, r->rw.items[i].var_name);
cov_put(&b, "rw%d\n", (int)r->rw.items[i].is_write);
}
for (int i = 0; i < r->type_refs.count; i++) {
fp_s(&b, r->type_refs.items[i].type_name);
cov_put(&b, "tref\n");
}
for (int i = 0; i < r->env_accesses.count; i++) {
fp_s(&b, r->env_accesses.items[i].env_key);
cov_put(&b, "env\n");
}
for (int i = 0; i < r->throws.count; i++) {
fp_s(&b, r->throws.items[i].exception_name);
cov_put(&b, "throw\n");
}
for (int i = 0; i < r->string_refs.count; i++) {
fp_s(&b, r->string_refs.items[i].value);
cov_put(&b, "sref%d\n", (int)r->string_refs.items[i].kind);
}
cov_put(&b, "ta%d it%d rc%d ib%d ch%d\n", r->type_assigns.count, r->impl_traits.count,
r->resolved_calls.count, r->infra_bindings.count, r->channels.count);
return b.s;
}
static int has_usage_named(const CBMFileResult *r, const char *name) {
for (int i = 0; i < r->usages.count; i++) {
if (r->usages.items[i].ref_name && strstr(r->usages.items[i].ref_name, name)) {
return 1;
}
}
return 0;
}
static int has_call_named(const CBMFileResult *r, const char *name) {
for (int i = 0; i < r->calls.count; i++) {
if (r->calls.items[i].callee_name && strstr(r->calls.items[i].callee_name, name)) {
return 1;
}
}
return 0;
}
typedef struct {
const char *name;
uint32_t start;
uint32_t end;
} cov_site_t;
static int collect_sites(const CBMFileResult *r, bool calls, cov_site_t **out) {
int n = calls ? r->calls.count : r->usages.count;
*out = calloc((size_t)(n > 0 ? n : 1), sizeof(cov_site_t));
for (int i = 0; i < n && *out; i++) {
(*out)[i] =
calls ? (cov_site_t){r->calls.items[i].callee_name, r->calls.items[i].site_start_byte,
r->calls.items[i].site_end_byte}
: (cov_site_t){r->usages.items[i].ref_name, r->usages.items[i].site_start_byte,
r->usages.items[i].site_end_byte};
}
return *out ? n : 0;
}
static bool site_in(cov_site_t s, const cov_site_t *arr, int n) {
for (int i = 0; i < n; i++) {
if (arr[i].start == s.start && arr[i].end == s.end && s.name && arr[i].name &&
strcmp(arr[i].name, s.name) == 0) {
return true;
}
}
return false;
}
/* Calls (calls=true) or usages, cut parse against full parse:
* - nothing the full parse found outside the dropped rows may be missing;
* - with `exact`, the cut parse may find nothing the full parse did not.
* The full parse does find things INSIDE dropped rows: the SQL grammar reads
* DEFAULT and a _binary introducer as identifiers, and a MySQL \' escape it
* does not know turns the rest of a string into one. None of those names
* anything. Without `exact` (MySQL escapes) the cut parse may find more: the
* full parse's error recovery around a misread escape swallows neighbouring
* rows, a subquery row among them, and the cut parse no longer misreads them. */
static bool sites_agree(const char *src, const CBMFileResult *cut, const CBMFileResult *full,
bool calls, bool exact) {
cov_site_t *cs = NULL;
cov_site_t *fs = NULL;
int cn = collect_sites(cut, calls, &cs);
int fn = collect_sites(full, calls, &fs);
CBMSqlKeptRanges k = {NULL, 0};
(void)cbm_sql_values_kept_ranges(src, (uint32_t)strlen(src), &k);
bool ok = cs && fs;
for (int i = 0; ok && exact && i < cn; i++) {
if (!site_in(cs[i], fs, fn)) {
fprintf(stderr, " %s only in the cut parse\n", cs[i].name);
ok = false;
}
}
for (int i = 0; ok && i < fn; i++) {
if (site_in(fs[i], cs, cn)) {
continue;
}
for (uint32_t j = 0; j < k.count; j++) {
if (fs[i].start < k.items[j].end_byte && fs[i].end > k.items[j].start_byte) {
fprintf(stderr, " %s lost outside the dropped rows\n", fs[i].name);
ok = false;
}
}
}
cbm_sql_kept_ranges_free(&k);
free(cs);
free(fs);
return ok;
}
static int count_calls_named(const CBMFileResult *r, const char *name) {
int n = 0;
for (int i = 0; i < r->calls.count; i++) {
if (r->calls.items[i].callee_name && strcmp(r->calls.items[i].callee_name, name) == 0) {
n++;
}
}
return n;
}
/* The same source parsed with the row exclusion and without it (test seam). */
static CBMFileResult *extract_sql_full(const char *src, const char *path) {
cbm_setenv("CBM_TEST_SQL_FULL_PARSE_ON", path, 1);
CBMFileResult *r = do_extract(src, CBM_LANG_SQL, path);
cbm_unsetenv("CBM_TEST_SQL_FULL_PARSE_ON");
return r;
}
TEST(sql_dump_literal_rows_leave_the_graph_unchanged_issue1735) {
for (int esc = 0; esc < 2; esc++) {
char *src = sql_dump(4, 60, esc == 1, true);
ASSERT_NOT_NULL(src);
CBMFileResult *cut = do_extract(src, CBM_LANG_SQL, "dump.sql");
CBMFileResult *full = extract_sql_full(src, "dump.sql");
ASSERT_NOT_NULL(cut);
ASSERT_NOT_NULL(full);
ASSERT_FALSE(cut->has_error);
ASSERT_TRUE(has_def(cut, "big_cities"));
char *fp_cut = result_fingerprint(cut);
char *fp_full = result_fingerprint(full);
ASSERT_NOT_NULL(fp_cut);
ASSERT_NOT_NULL(fp_full);
ASSERT_STR_EQ(fp_cut, fp_full);
ASSERT_TRUE(sites_agree(src, cut, full, true, esc == 0));
ASSERT_TRUE(sites_agree(src, cut, full, false, esc == 0));
/* Every INSERT's subquery row is parsed. */
ASSERT_EQ(count_calls_named(cut, "MAX"), 4);
/* ...and the rows really were left out: under a quarter of the file
* is parsed. */
CBMSqlKeptRanges k = {NULL, 0};
ASSERT_TRUE(cbm_sql_values_kept_ranges(src, (uint32_t)strlen(src), &k));
size_t kept = 0;
for (uint32_t j = 0; j < k.count; j++) {
kept += k.items[j].end_byte - k.items[j].start_byte;
}
cbm_sql_kept_ranges_free(&k);
ASSERT_LT(kept * 4, strlen(src));
free(fp_cut);
free(fp_full);
cbm_free_result(cut);
cbm_free_result(full);
free(src);
}
PASS();
}
TEST(sql_dump_tuple_with_subquery_or_call_is_still_parsed_issue1735) {
char *src = sql_dump(1, 20, true, true);
ASSERT_NOT_NULL(src);
CBMFileResult *r = do_extract(src, CBM_LANG_SQL, "dump.sql");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(has_usage_named(r, "country")); /* from the subquery row */
ASSERT_TRUE(has_call_named(r, "UPPER")); /* from the function-call row */
cbm_free_result(r);
free(src);
PASS();
}
TEST(sql_dump_parse_does_not_grow_with_the_row_count_issue1735) {
/* A count, not a clock: with literal rows kept out, doubling them must not
* add a single tree node. */
char *small = sql_dump(1, 500, true, false);
char *big = sql_dump(1, 1000, true, false);
ASSERT_NOT_NULL(small);
ASSERT_NOT_NULL(big);
CBMFileResult *rs = do_extract(small, CBM_LANG_SQL, "small.sql");
CBMFileResult *rb = do_extract(big, CBM_LANG_SQL, "big.sql");
ASSERT_NOT_NULL(rs);
ASSERT_NOT_NULL(rb);
ASSERT_EQ(rs->tree_nodes, rb->tree_nodes);
ASSERT_EQ(rs->defs.count, rb->defs.count);
cbm_free_result(rs);
cbm_free_result(rb);
free(small);
free(big);
PASS();
}
TEST(sql_dump_of_many_megabytes_is_indexed_not_timed_out_issue1735) {
/* ~26 MB with MySQL escapes. The production parse budget is CPU time, so
* extracting under it made the verdict a function of runner speed (an
* ASan arm leg tripped it). The property behind "indexed, not timed out"
* is a count: the parse work is set by the statements, not by the rows.
* So: extract unbudgeted, assert the outcome, and bound the tree by the
* same 330 statements carrying only a handful of rows each (the full
* parse built a tree node for every row token). */
char *src = sql_dump(330, 2000, true, true);
char *few = sql_dump(330, 4, true, true);
ASSERT_NOT_NULL(src);
ASSERT_NOT_NULL(few);
size_t len = strlen(src);
ASSERT_GT(len, (size_t)24 * 1024 * 1024);
CBMFileResult *r =
cbm_extract_file(src, (int)len, CBM_LANG_SQL, "covproj", "world.sql", 0, NULL, NULL);
CBMFileResult *rf =
cbm_extract_file(few, (int)strlen(few), CBM_LANG_SQL, "covproj", "few.sql", 0, NULL, NULL);
free(src);
free(few);
ASSERT_NOT_NULL(r);
ASSERT_NOT_NULL(rf);
bool indexed = !r->has_error && has_def(r, "big_cities") && has_usage_named(r, "country");
uint32_t nodes = r->tree_nodes;
uint32_t few_nodes = rf->tree_nodes;
cbm_free_result(r);
cbm_free_result(rf);
ASSERT_TRUE(indexed);
ASSERT_GT(few_nodes, 0);
ASSERT_LTE(nodes, 2 * few_nodes);
PASS();
}
/* The text the parser sees: every kept range, concatenated. */
static char *kept_text(const char *src, const CBMSqlKeptRanges *k) {
cov_buf_t b = {NULL, 0, 0};
cov_put(&b, "%s", "");
for (uint32_t i = 0; i < k->count; i++) {
cov_put(&b, "%.*s", (int)(k->items[i].end_byte - k->items[i].start_byte),
src + k->items[i].start_byte);
}
return b.s;
}
TEST(sql_values_scanner_excludes_only_literal_rows_issue1735) {
/* {source, what the parser sees} — NULL means nothing is excluded. */
static const char *const cases[][2] = {
{"INSERT INTO t VALUES (1,'a'),(2,'b\\'c'),(3,NULL);", "INSERT INTO t VALUES (1,'a');"},
{"INSERT INTO t VALUES (1),(f(2)),(3);", "INSERT INTO t VALUES (1),(f(2));"},
{"insert into t values (1),(-2.5e3),(0x1F),(_binary 'x'),(DEFAULT),(true),(X'0A'),"
"('it''s'),(.5),();",
"insert into t values (1);"},
{"REPLACE INTO t VALUES (1), /* c */ (2), (3);", "REPLACE INTO t VALUES (1);"},
{"INSERT INTO t VALUES (1),(2) -- c\n,(3);", "INSERT INTO t VALUES (1) -- c\n;"},
{"INSERT INTO t VALUES (1),((SELECT id FROM u)),(x),(\"q\"),(`b`),(a.b);", NULL},
{"INSERT INTO t VALUES (1),('a\nb'),(2);", "INSERT INTO t VALUES (1),('a\nb');"},
{"INSERT INTO t VALUES (1),(2) ON DUPLICATE KEY UPDATE a=VALUES(a);",
"INSERT INTO t VALUES (1) ON DUPLICATE KEY UPDATE a=VALUES(a);"},
{"INSERT INTO t (a, b) VALUES (1, 2), (3, 4);", "INSERT INTO t (a, b) VALUES (1, 2);"},
{"SELECT 'INSERT INTO t VALUES (1),(2)';", NULL},
{"-- INSERT INTO t VALUES (1),(2)\nSELECT 1;", NULL},
{"/* INSERT INTO t VALUES (1),(2) */ SELECT 1;", NULL},
{"SELECT * FROM (VALUES (1),(2)) AS v(x);", NULL},
{"CREATE FUNCTION f() AS $$ SELECT 1; INSERT INTO t VALUES (1),(2); $$;", NULL},
{"INSERT INTO t VALUES (1),(2", NULL},
{"INSERT INTO t VALUES (1),(1abc),(2);", "INSERT INTO t VALUES (1),(1abc);"},
};
for (size_t c = 0; c < sizeof(cases) / sizeof(cases[0]); c++) {
const char *src = cases[c][0];
CBMSqlKeptRanges k = {NULL, 0};
bool cut = cbm_sql_values_kept_ranges(src, (uint32_t)strlen(src), &k);
if (!cases[c][1]) {
if (cut) {
fprintf(stderr, " case %zu excluded rows unexpectedly: %s\n", c, src);
}
ASSERT_FALSE(cut);
continue;
}
ASSERT_TRUE(cut);
char *seen = kept_text(src, &k);
ASSERT_NOT_NULL(seen);
ASSERT_STR_EQ(seen, cases[c][1]);
free(seen);
cbm_sql_kept_ranges_free(&k);
}
PASS();
}
TEST(sql_values_scanner_keeps_positions_of_kept_text_issue1735) {
const char *src = "INSERT INTO t VALUES\n(1),\n(2);\nCREATE TABLE u (id int);\n";
CBMSqlKeptRanges k = {NULL, 0};
ASSERT_TRUE(cbm_sql_values_kept_ranges(src, (uint32_t)strlen(src), &k));
ASSERT_EQ(k.count, 2u);
/* excluded: ",\n(2)" from byte 24 (row 1, col 3) to byte 29 (row 2, col 3) */
ASSERT_EQ(k.items[0].start_byte, 0u);
ASSERT_EQ(k.items[0].end_byte, 24u);
ASSERT_EQ(k.items[0].end_point.row, 1u);
ASSERT_EQ(k.items[0].end_point.column, 3u);
ASSERT_EQ(k.items[1].start_byte, 29u);
ASSERT_EQ(k.items[1].start_point.row, 2u);
ASSERT_EQ(k.items[1].start_point.column, 3u);
ASSERT_EQ(k.items[1].end_byte, (uint32_t)strlen(src));
ASSERT_EQ(k.items[1].end_point.row, 4u);
ASSERT_EQ(k.items[1].end_point.column, 0u);
cbm_sql_kept_ranges_free(&k);
/* The table after the cut keeps its real line. */
CBMFileResult *r = do_extract(src, CBM_LANG_SQL, "pos.sql");
ASSERT_NOT_NULL(r);
bool found = false;
for (int i = 0; i < r->defs.count; i++) {
if (r->defs.items[i].name && strcmp(r->defs.items[i].name, "u") == 0) {
ASSERT_EQ(r->defs.items[i].start_line, 4u);
found = true;
}
}
ASSERT_TRUE(found);
cbm_free_result(r);
PASS();
}
/* ── #1736: a lone '&' in JSX is text, not a parse failure ────────────────
* Upstream tree-sitter-javascript (and the tsx dialect built on it) accepted
* '&' inside a JSX string only before a space/digit or as a complete
* character reference, and never inside JSX text, so `href="...?a=1&b=2"`
* (the reporter's Google Fonts URL) or `<p>Tom &Jerry</p>` produced an ERROR
* node and a parse_partial flag. Both grammars are now self-maintained forks
* (tools/tree-sitter-javascript, tools/tree-sitter-tsx). Each case wraps the
* JSX in a component and puts a second function AFTER it, so a pass also
* proves the definitions around the JSX are extracted. */
static const char *const JSX_AMP_BODIES[] = {
/* the #1736 shape: '&' + letter in a double-quoted attribute */
"<link href=\"https://fonts.x/css2?family=Inter:wght@300;400&family=Syne&display=swap\" />",
"<link href='https://x.com/a?a=1&b=2' />", /* single-quoted attribute */
"<p title=\"a&\">x</p>", /* '&' right before the quote */
"<p>Tom &Jerry</p>", /* '&' + letter in JSX text */
"<p>Tom & Jerry</p>", /* '&' + space in JSX text */
"<th className=\"p-4 font-semibold\">Role & Dept</th>", /* #2481 report reproduction */
"<p>x &1 y</p>", /* '&' + digit in JSX text */
/* controls: references keep parsing; '& ' / '&1' in attributes */
"<p title=\"x&y&z & w &1\">A&B & C</p>",
};
static const struct {
CBMLanguage lang;
const char *path;
} JSX_AMP_LANGS[] = {
{CBM_LANG_JAVASCRIPT, "app/page.js"},
{CBM_LANG_JAVASCRIPT, "app/page.jsx"},
{CBM_LANG_TSX, "app/layout.tsx"},
};
enum { JSX_AMP_SRC_CAP = 512 };
TEST(jsx_lone_ampersand_is_not_parse_partial_issue1736) {
size_t nb = sizeof(JSX_AMP_BODIES) / sizeof(JSX_AMP_BODIES[0]);
size_t nl = sizeof(JSX_AMP_LANGS) / sizeof(JSX_AMP_LANGS[0]);
int failures = 0;
for (size_t l = 0; l < nl; l++) {
for (size_t b = 0; b < nb; b++) {
char src[JSX_AMP_SRC_CAP];
snprintf(src, sizeof(src),
"export default function Page() {\n return (\n %s\n );\n}\n"
"export function After() {\n return 1;\n}\n",
JSX_AMP_BODIES[b]);
CBMFileResult *r = do_extract(src, JSX_AMP_LANGS[l].lang, JSX_AMP_LANGS[l].path);
ASSERT_NOT_NULL(r);
bool bad = r->parse_incomplete || !has_def(r, "Page") || !has_def(r, "After");
if (bad) {
fprintf(stderr, " %s flagged=%d ranges=%s body=%s\n", JSX_AMP_LANGS[l].path,
(int)r->parse_incomplete, r->error_ranges ? r->error_ranges : "(none)",
JSX_AMP_BODIES[b]);
failures++;
}
cbm_free_result(r);
}
}
ASSERT_EQ(failures, 0);
PASS();
}
/* GUARD against an inert test: the same wrapper with genuinely broken JSX is
* still flagged in every one of the three languages, so the green above is the
* grammar accepting '&', not the signal being switched off. */
TEST(jsx_broken_markup_still_parse_partial_issue1736) {
size_t nl = sizeof(JSX_AMP_LANGS) / sizeof(JSX_AMP_LANGS[0]);
for (size_t l = 0; l < nl; l++) {
const char *src = "export default function Page() {\n return (\n"
" <p title=\"a\" =>x</p>\n );\n}\n"
"export function After() {\n return 1;\n}\n";
CBMFileResult *r = do_extract(src, JSX_AMP_LANGS[l].lang, JSX_AMP_LANGS[l].path);
ASSERT_NOT_NULL(r);
bool flagged = r->parse_incomplete;
cbm_free_result(r);
ASSERT_TRUE(flagged);
}
PASS();
}
/* ── #1748: C# 12 collection expressions in conditional branches ─────────────
* The vendored tree-sitter-c-sharp (pin 88366631d598) predated upstream's
* collection-expression support (#402, first released in v0.23.4): `[...]` was
* only reachable as an element_binding_expression, so an empty or a second
* collection literal after `?`/`:` had no valid parse and error recovery ate
* the surrounding statement. Every body below is valid C# 12; the method that
* FOLLOWS it pins that extraction resumes after the conditional. */
#define CS_1748_WRAP(body) \
"class M\n" \
"{\n" \
" async Task Go(bool c, List<int> items)\n" \
" {\n" body "\n" \
" }\n" \
" Task<List<int>> F() => null;\n" \
" void AfterConditional() { }\n" \
"}\n"
TEST(cs_collection_expression_in_conditional_is_complete_issue1748) {
static const char *const bodies[] = {
/* the reported shape: wrapped ternary, empty collection first */
CS_1748_WRAP(" var x = c\n ? []\n : await F();"),
CS_1748_WRAP(" var x = c ? []\n : await F();"),
CS_1748_WRAP(" List<int> x = c\n ? []\n : [items.First()];"),
/* both branches collection expressions, single line */
CS_1748_WRAP(" List<int> x = c ? [] : [1];"),
/* single-line, single collection branch (also failed on the old pin) */
CS_1748_WRAP(" var x = c ? [] : items;"),
CS_1748_WRAP(" var x = c ? [] : await F();"),
CS_1748_WRAP(" return c ? [] : items;"),
CS_1748_WRAP(" if (c) return c ? [] : items;"),
CS_1748_WRAP(" var x = c ? items : [];"),
CS_1748_WRAP(" var x = c switch { true => [], _ => items };"),
CS_1748_WRAP(" int[] a = [1, 2, ..items];"),
/* control: the same wrapped shape without a collection expression */
CS_1748_WRAP(" var x = c\n ? null\n : await F();"),
};
int failures = 0;
for (size_t i = 0; i < sizeof(bodies) / sizeof(bodies[0]); i++) {
CBMFileResult *r = do_extract(bodies[i], CBM_LANG_CSHARP, "M.cs");
ASSERT_NOT_NULL(r);
bool partial = r->parse_incomplete;
bool has_after = has_def(r, "AfterConditional");
if (partial || !has_after) {
fprintf(stderr, " case %zu: partial=%d ranges=%s after_def=%d\n", i, partial,
r->error_ranges ? r->error_ranges : "(none)", has_after);
failures++;
}
cbm_free_result(r);
}
if (failures != 0) {
FAIL("valid C# 12 collection expressions in conditional branches must parse completely");
}
PASS();
}
/* Calls nested in a collection expression (element and spread) must still be
* extracted after the refresh moved them under collection_expression /
* expression_element / spread_element (previously element_binding_expression /
* argument / range_expression). */
TEST(cs_calls_inside_collection_expression_extracted_issue1748) {
const char *src = "class M\n"
"{\n"
" int[] Build(bool c)\n"
" {\n"
" return c ? [] : [Head(), ..Tail()];\n"
" }\n"
" int Head() => 1;\n"
" int[] Tail() => null;\n"
"}\n";
CBMFileResult *r = do_extract(src, CBM_LANG_CSHARP, "M.cs");
ASSERT_NOT_NULL(r);
bool partial = r->parse_incomplete;
bool head = false;
bool tail = false;
for (int i = 0; i < r->calls.count; i++) {
const char *n = r->calls.items[i].callee_name;
if (n && strcmp(n, "Head") == 0) {
head = true;
}
if (n && strcmp(n, "Tail") == 0) {
tail = true;
}
}
cbm_free_result(r);
ASSERT_FALSE(partial);
ASSERT_TRUE(head);
ASSERT_TRUE(tail);
PASS();
}
/* GUARD: the refreshed grammar must not hide genuinely broken C#. */
TEST(cs_malformed_conditional_remains_partial_issue1748) {
CBMFileResult *r =
do_extract(CS_1748_WRAP(" var x = c ? ] : ;"), CBM_LANG_CSHARP, "M.cs");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_NOT_NULL(r->error_ranges);
cbm_free_result(r);
PASS();
}
/* ── #2078: every file is parsed as if it ended with a newline ───────────────
*
* Several grammars need a line terminator that a file's last line may simply
* not have. tree-sitter-markdown is the sharpest case: an opening code fence,
* an ATX heading or a bare list marker as the final bytes of a file leaves a
* WIDTH-BEARING ERROR (so the #1610 zero-width rule cannot excuse it), and a
* file that is only "```" came back as a whole-file error. Appending one "\n"
* made every one of these clean -- the reporter proved it on 21 real files.
*
* cbm now feeds the parser one virtual "\n" past EOF when the last byte is not
* already a newline, then clamps the tree back to the real length, so no
* consumer sees a byte, a line or a range that is not in the file. */
TEST(markdown_unterminated_last_line_is_clean_issue2078) {
const char *cases[] = {
"text\n```", /* opening fence at EOF (upstream PR #262, closed unmerged) */
"```", /* the whole file is one opening fence */
"# Top", /* ATX heading */
"- ", /* bare list marker */
};
int flagged = 0;
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
CBMFileResult *r = do_extract(cases[i], CBM_LANG_MARKDOWN, "doc.md");
ASSERT_NOT_NULL(r);
if (r->parse_incomplete || r->parse_unusable) {
fprintf(stderr, " case %zu flagged: partial=%d unusable=%d ranges=%s\n", i,
r->parse_incomplete, r->parse_unusable,
r->error_ranges ? r->error_ranges : "(none)");
flagged++;
}
cbm_free_result(r);
}
if (flagged) {
FAIL("a markdown file must not be flagged only because its last line is unterminated");
}
PASS();
}
/* Controls: the same bytes WITH the newline were already clean and stay so. */
TEST(markdown_terminated_last_line_controls_unchanged_issue2078) {
const char *cases[] = {"text\n```\n", "```\n", "# Top\n", "- \n"};
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
CBMFileResult *r = do_extract(cases[i], CBM_LANG_MARKDOWN, "doc.md");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->parse_incomplete);
ASSERT_FALSE(r->parse_unusable);
cbm_free_result(r);
}
PASS();
}
/* Everything extraction reports, as one string, so two results compare
* byte-for-byte: defs (every text field and range), calls (exact byte spans),
* imports and the coverage verdict. */
typedef struct {
char *buf;
size_t cap;
size_t len;
} dump_buf_t;
static void dump_str(dump_buf_t *b, const char *tag, const char *s) {
int n = snprintf(b->buf + b->len, b->cap - b->len, " %s=%s", tag, s ? s : "~");
if (n > 0 && (size_t)n < b->cap - b->len) {
b->len += (size_t)n;
}
}
static void dump_num(dump_buf_t *b, const char *tag, long v) {
int n = snprintf(b->buf + b->len, b->cap - b->len, " %s=%ld", tag, v);
if (n > 0 && (size_t)n < b->cap - b->len) {
b->len += (size_t)n;
}
}
/* The Module def spans the whole file, and its end line follows the line
* convention for a terminated last line (#1967), so it is compared on its own
* rather than through the dump. */
static char *dump_extraction(const CBMFileResult *r) {
dump_buf_t b = {(char *)calloc(1u << 16, 1), 1u << 16, 0};
if (!b.buf) {
return NULL;
}
for (int i = 0; i < r->defs.count; i++) {
const CBMDefinition *d = &r->defs.items[i];
if (d->label && strcmp(d->label, "Module") == 0) {
continue;
}
dump_str(&b, "\nD", d->qualified_name);
dump_str(&b, "name", d->name);
dump_str(&b, "label", d->label);
dump_num(&b, "start", d->start_line);
dump_num(&b, "end", d->end_line);
dump_num(&b, "lines", d->lines);
dump_num(&b, "cx", d->complexity);
dump_num(&b, "params", d->param_count);
dump_str(&b, "sig", d->signature);
dump_str(&b, "ret", d->return_type);
dump_str(&b, "doc", d->docstring);
dump_str(&b, "prof", d->structural_profile);
dump_str(&b, "tok", d->body_tokens);
}
for (int i = 0; i < r->calls.count; i++) {
const CBMCall *c = &r->calls.items[i];
dump_str(&b, "\nC", c->callee_name);
dump_str(&b, "in", c->enclosing_func_qn);
dump_num(&b, "line", c->start_line);
dump_num(&b, "from", (long)c->site_start_byte);
dump_num(&b, "to", (long)c->site_end_byte);
dump_num(&b, "args", c->arg_count);
}
for (int i = 0; i < r->imports.count; i++) {
dump_str(&b, "\nI", r->imports.items[i].local_name);
dump_str(&b, "path", r->imports.items[i].module_path);
}
dump_str(&b, "\nP", r->error_ranges);
dump_num(&b, "partial", r->parse_incomplete);
dump_num(&b, "unusable", r->parse_unusable);
dump_str(&b, "", "\n");
return b.buf;
}
static uint32_t module_end_line(const CBMFileResult *r) {
for (int i = 0; i < r->defs.count; i++) {
if (r->defs.items[i].label && strcmp(r->defs.items[i].label, "Module") == 0) {
return r->defs.items[i].end_line;
}
}
return 0;
}
static char *extract_dump(const char *src, CBMLanguage lang, const char *path,
uint32_t *module_end) {
CBMFileResult *r = do_extract(src, lang, path);
if (!r) {
return NULL;
}
char *d = dump_extraction(r);
if (module_end) {
*module_end = module_end_line(r);
}
cbm_free_result(r);
return d;
}
/* GUARD: for code grammars a trailing newline is insignificant, so a file
* without one must extract byte-identically to the same file with one -- every
* def, range, text field and call byte span. The virtual newline may never
* show up as phantom content or a phantom line. */
TEST(code_without_final_newline_extracts_identically_issue2078) {
struct {
const char *src; /* no trailing newline */
CBMLanguage lang;
const char *path;
uint32_t lines;
} cases[] = {
{"#include <stdio.h>\n\nstatic int beta(int x) {\n return x + 1;\n}\n\n"
"void alpha(void) { printf(\"a\"); beta(2); }",
CBM_LANG_C, "a.c", 7},
{"import os\n\n\nclass K:\n def m(self, a):\n return os.path.join(a)\n\n\n"
"def f(x):\n \"\"\"doc\"\"\"\n return K().m(x)",
CBM_LANG_PYTHON, "a.py", 11},
};
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
size_t n = strlen(cases[i].src);
char *terminated = (char *)malloc(n + 2);
ASSERT_NOT_NULL(terminated);
memcpy(terminated, cases[i].src, n);
terminated[n] = '\n';
terminated[n + 1] = '\0';
uint32_t bare_module_end = 0;
char *bare = extract_dump(cases[i].src, cases[i].lang, cases[i].path, &bare_module_end);
char *term = extract_dump(terminated, cases[i].lang, cases[i].path, NULL);
free(terminated);
ASSERT_NOT_NULL(bare);
ASSERT_NOT_NULL(term);
bool same = strcmp(bare, term) == 0;
if (!same) {
fprintf(stderr, " %s differs:\n--- without final newline%s--- with final newline%s",
cases[i].path, bare, term);
}
free(bare);
free(term);
if (!same) {
FAIL("extraction must not depend on whether the last line is terminated");
}
if (bare_module_end != cases[i].lines) {
fprintf(stderr, " %s: module ends on line %u, file has %u\n", cases[i].path,
bare_module_end, cases[i].lines);
FAIL("the virtual newline must not add a line to the module");
}
}
PASS();
}
/* GUARD against phantom content: a heading on an unterminated last line is a
* one-line file. No def may end past line 1, no text may carry the virtual
* newline, and the result must equal the terminated file's. */
TEST(virtual_newline_adds_no_phantom_line_or_text_issue2078) {
CBMFileResult *r = do_extract("# Top", CBM_LANG_MARKDOWN, "top.md");
ASSERT_NOT_NULL(r);
bool found = has_def(r, "Top");
for (int i = 0; i < r->defs.count; i++) {
const CBMDefinition *d = &r->defs.items[i];
if (d->end_line > 1 || d->start_line > 1) {
fprintf(stderr, " def %s spans %u-%u\n", d->name ? d->name : "?", d->start_line,
d->end_line);
cbm_free_result(r);
FAIL("no def may reach past the last real line");
}
if ((d->name && strchr(d->name, '\n')) || (d->signature && strchr(d->signature, '\n'))) {
cbm_free_result(r);
FAIL("no extracted text may contain the virtual newline");
}
}
cbm_free_result(r);
if (!found) {
/* Before #2078 the unterminated heading was silently dropped: no
* Section, and no parse_partial flag to say so. */
FAIL("the heading on an unterminated last line must be extracted as a Section");
}
PASS();
}
/* GUARD: the empty file and a file that is exactly "\n" get no virtual
* newline (nothing is unterminated) and stay clean. */
TEST(empty_and_newline_only_files_unchanged_issue2078) {
const char *srcs[] = {"", "\n"};
CBMLanguage langs[] = {CBM_LANG_MARKDOWN, CBM_LANG_PYTHON, CBM_LANG_C};
const char *paths[] = {"e.md", "e.py", "e.c"};
for (size_t s = 0; s < 2; s++) {
for (size_t l = 0; l < 3; l++) {
CBMFileResult *r = do_extract(srcs[s], langs[l], paths[l]);
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->parse_incomplete);
ASSERT_FALSE(r->parse_unusable);
cbm_free_result(r);
}
}
PASS();
}
/* #1967: every line counter over one source buffer answers the same
* question the same way. A trailing newline ends the last line; it does not
* open a new one. */
TEST(source_line_count_one_convention) {
ASSERT_EQ(cbm_source_line_count("", 0), 1u);
ASSERT_EQ(cbm_source_line_count("a", 1), 1u);
ASSERT_EQ(cbm_source_line_count("a\n", 2), 1u);
ASSERT_EQ(cbm_source_line_count("\n", 1), 1u);
ASSERT_EQ(cbm_source_line_count("a\nb", 3), 2u);
ASSERT_EQ(cbm_source_line_count("a\nb\n", 4), 2u);
ASSERT_EQ(cbm_source_line_count("a\n\n", 3), 2u);
ASSERT_EQ(cbm_source_line_count("a\r\nb\r\n", 6), 2u);
/* src_len bounds the count, not a NUL terminator. */
ASSERT_EQ(cbm_source_line_count("a\nb\nc", 2), 1u);
PASS();
}
/* ── #2211: Pascal/Delphi identifiers with non-ASCII letters ─────────────────
* Delphi accepts non-ASCII letters in identifiers; 1C integrations call its
* OLE API through late-bound Variants with Cyrillic member names. Upstream
* tree-sitter-pascal's identifier token was ASCII-only, so every such name was
* an ERROR node and the unit came out parse_partial. The Cyrillic text is
* written as UTF-8 hex escapes (the source stays ASCII); a `" "` split follows
* an escape run whenever the next character is a hex digit. */
/* The reporter's snippet (#2211), verbatim apart from the escaping:
* v8_Base.<Dokumenty>.<SchetNaOplatuPostavshchika>.<NaytiPoNomeru>(...),
* v8_dst.<Pustaya>(), v8_Base.<Spravochniki>.<Nomenklatura>.<NaytiPoKodu>(...),
* NM.<PoluchitObyekt>().<Zapisat>() -- <...> = the Cyrillic name, transliterated. */
static const char *PAS_CYRILLIC_MEMBERS =
"unit Demo;\n"
"interface\n"
"procedure Demo1C;\n"
"implementation\n"
"procedure Demo1C;\n"
"var\n"
" v8_Base, v8_dst, NM: Variant;\n"
"begin\n"
" v8_dst := "
"v8_Base.\xd0\x94\xd0\xbe\xd0\xba\xd1\x83\xd0\xbc\xd0\xb5\xd0\xbd\xd1\x82\xd1\x8b."
"\xd0\xa1\xd1\x87\xd0\xb5\xd1\x82\xd0\x9d\xd0\xb0\xd0\x9e\xd0\xbf\xd0\xbb\xd0\xb0\xd1\x82\xd1"
"\x83\xd0\x9f\xd0\xbe\xd1\x81\xd1\x82\xd0\xb0\xd0\xb2\xd1\x89\xd0\xb8\xd0\xba\xd0\xb0."
"\xd0\x9d\xd0\xb0\xd0\xb9\xd1\x82\xd0\xb8\xd0\x9f\xd0\xbe\xd0\x9d\xd0\xbe\xd0\xbc\xd0\xb5\xd1"
"\x80\xd1\x83('1', Now);\n"
" if v8_dst.\xd0\x9f\xd1\x83\xd1\x81\xd1\x82\xd0\xb0\xd1\x8f() then Exit;\n"
" NM := "
"v8_Base.\xd0\xa1\xd0\xbf\xd1\x80\xd0\xb0\xd0\xb2\xd0\xbe\xd1\x87\xd0\xbd\xd0\xb8\xd0\xba\xd0"
"\xb8.\xd0\x9d\xd0\xbe\xd0\xbc\xd0\xb5\xd0\xbd\xd0\xba\xd0\xbb\xd0\xb0\xd1\x82\xd1\x83\xd1\x80"
"\xd0\xb0.\xd0\x9d\xd0\xb0\xd0\xb9\xd1\x82\xd0\xb8\xd0\x9f\xd0\xbe\xd0\x9a\xd0\xbe\xd0\xb4\xd1"
"\x83('123', 0);\n"
" if not NM.\xd0\x9f\xd1\x83\xd1\x81\xd1\x82\xd0\xb0\xd1\x8f() then\n"
" "
"NM."
"\xd0\x9f\xd0\xbe\xd0\xbb\xd1\x83\xd1\x87\xd0\xb8\xd1\x82\xd1\x8c\xd0\x9e\xd0\xb1\xd1\x8a\xd0"
"\xb5\xd0\xba\xd1\x82().\xd0\x97\xd0\xb0\xd0\xbf\xd0\xb8\xd1\x81\xd0\xb0\xd1\x82\xd1\x8c();\n"
"end;\n"
"end.\n";
/* A procedure whose own name, parameter and locals are Cyrillic (<ObrabotatDokument>,
* <Imya>, <Schyotchik>2), plus the keyword-boundary cases the wider identifier token must
* not swallow: `begin<Znachenie>` is ONE identifier, `&begin` is an escaped
* identifier, and upper-case BEGIN/END still open and close a block. An ASCII
* neighbour follows, so a keyword lost to the identifier token would surface as
* a missing definition as well as an ERROR. */
#define PAS_CYR_PROC_NAME \
"\xd0\x9e\xd0\xb1\xd1\x80\xd0\xb0\xd0\xb1\xd0\xbe\xd1\x82\xd0\xb0\xd1\x82\xd1\x8c\xd0\x94\xd0" \
"\xbe\xd0\xba\xd1\x83\xd0\xbc\xd0\xb5\xd0\xbd\xd1\x82"
#define PAS_CYR_COUNTER "\xd0\xa1\xd1\x87\xd1\x91\xd1\x82\xd1\x87\xd0\xb8\xd0\xba"
#define PAS_CYR_VALUE "begin\xd0\x97\xd0\xbd\xd0\xb0\xd1\x87\xd0\xb5\xd0\xbd\xd0\xb8\xd0\xb5"
static const char *PAS_CYRILLIC_PROC =
"unit Kw;\n"
"interface\n"
"implementation\n"
"procedure " PAS_CYR_PROC_NAME "(const \xd0\x98\xd0\xbc\xd1\x8f: string);\n"
"var\n"
" " PAS_CYR_VALUE ", &begin: Integer;\n"
" " PAS_CYR_COUNTER "2: Integer;\n"
"begin\n"
" BEGIN " PAS_CYR_COUNTER "2 := 1; END;\n"
" " PAS_CYR_VALUE " := " PAS_CYR_COUNTER "2;\n"
"end;\n"
"procedure AfterIt;\n"
"begin\n"
"end;\n"
"end.\n";
/* ASCII control: the same shapes with ASCII names parsed before the fork and
* must still parse identically. */
static const char *PAS_ASCII_CONTROL = "unit Plain;\n"
"interface\n"
"procedure Run;\n"
"implementation\n"
"procedure Run;\n"
"var\n"
" Base, Doc: Variant;\n"
"begin\n"
" Doc := Base.Documents.Invoice.FindByNumber('1', Now);\n"
" if Doc.IsEmpty() then Exit;\n"
" Doc.GetObject().Write();\n"
"end;\n"
"end.\n";
/* Genuinely broken Pascal (a doubled `:=` and stray parens) must stay flagged:
* the wider identifier token must not make the parser accept garbage. */
static const char *PAS_BROKEN = "unit Broken;\n"
"interface\n"
"implementation\n"
"procedure Bad;\n"
"begin\n"
" x := := 1 )) ;\n"
"end;\n"
"procedure Good;\n"
"begin\n"
"end;\n"
"end.\n";
TEST(pascal_cyrillic_member_calls_not_partial_issue2211) {
CBMFileResult *r = do_extract(PAS_CYRILLIC_MEMBERS, CBM_LANG_PASCAL, "Demo.pas");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->parse_incomplete);
ASSERT_EQ(r->error_region_count, 0);
ASSERT_NULL(r->error_ranges);
ASSERT_TRUE(has_def(r, "Demo1C"));
cbm_free_result(r);
PASS();
}
TEST(pascal_cyrillic_procedure_name_extracted_issue2211) {
CBMFileResult *r = do_extract(PAS_CYRILLIC_PROC, CBM_LANG_PASCAL, "Kw.pas");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->parse_incomplete);
ASSERT_EQ(r->error_region_count, 0);
ASSERT_TRUE(has_def(r, PAS_CYR_PROC_NAME));
ASSERT_TRUE(has_def(r, "AfterIt"));
cbm_free_result(r);
PASS();
}
TEST(pascal_ascii_control_still_clean_issue2211) {
CBMFileResult *r = do_extract(PAS_ASCII_CONTROL, CBM_LANG_PASCAL, "Plain.pas");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->parse_incomplete);
ASSERT_EQ(r->error_region_count, 0);
ASSERT_TRUE(has_def(r, "Run"));
cbm_free_result(r);
PASS();
}
TEST(pascal_broken_source_still_flagged_issue2211) {
CBMFileResult *r = do_extract(PAS_BROKEN, CBM_LANG_PASCAL, "Broken.pas");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(r->parse_incomplete);
ASSERT_GTE(r->error_region_count, 1);
ASSERT_TRUE(has_def(r, "Good"));
cbm_free_result(r);
PASS();
}
SUITE(parse_coverage) {
RUN_TEST(source_line_count_one_convention);
RUN_TEST(c_ifdef_split_brace_sets_parse_incomplete);
RUN_TEST(c_ifdef_split_brace_neighbors_still_extracted);
RUN_TEST(c_error_range_points_at_failed_region);
RUN_TEST(c_clean_file_not_flagged);
RUN_TEST(py_unrecovered_garbage_sets_parse_incomplete);
RUN_TEST(py_recovered_def_not_flagged);
RUN_TEST(py_clean_file_not_flagged);
RUN_TEST(error_region_cap_is_honored);
RUN_TEST(error_region_cap_reports_what_it_dropped);
RUN_TEST(uncapped_ranges_carry_no_marker);
RUN_TEST(python_whole_file_error_is_unusable);
RUN_TEST(local_error_stays_partial_not_unusable);
RUN_TEST(clean_file_is_neither_partial_nor_unusable);
RUN_TEST(c_ifdef_split_is_partial_never_unusable);
RUN_TEST(c_trailing_recovered_defs_keep_flag);
RUN_TEST(dockerfile_missing_final_newline_not_flagged_issue1610);
RUN_TEST(dockerfile_with_final_newline_still_clean_issue1610);
RUN_TEST(dockerfile_trailing_space_at_eof_not_flagged_issue1746);
RUN_TEST(dockerfile_trailing_space_with_final_newline_clean_issue1746);
RUN_TEST(dockerfile_without_trailing_space_at_eof_clean_issue1746);
RUN_TEST(dockerfile_crlf_trailing_space_at_eof_not_flagged_issue1746);
RUN_TEST(missing_final_newline_not_flagged_across_grammars_issue1610);
RUN_TEST(real_error_before_eof_still_flagged_without_final_newline_issue1610);
RUN_TEST(width_bearing_error_at_eof_still_flagged_issue1610);
RUN_TEST(makefile_unterminated_recipe_is_parsed_issue2078);
RUN_TEST(c_ifdef_split_range_narrows_to_dropped_branch);
RUN_TEST(c_ifdef_split_range_excludes_lines_the_preprocessor_explained);
RUN_TEST(c_ifdef_split_range_never_starts_on_a_directive);
RUN_TEST(c_refinement_does_not_suppress_real_garbage);
RUN_TEST(c_clean_file_stays_unflagged_after_refinement);
RUN_TEST(perl_format_followed_by_named_sub_is_complete_issue1838);
RUN_TEST(perl_malformed_source_remains_partial_issue1838);
RUN_TEST(dockerfile_trailing_blank_at_eof_not_flagged_issue1746);
RUN_TEST(real_error_before_eof_still_flagged_with_trailing_blank_issue1746);
RUN_TEST(width_bearing_error_at_eof_still_flagged_with_trailing_blank_issue1746);
RUN_TEST(c_thread_local_grammar_limit_is_pinned_issue963);
RUN_TEST(coverage_repeated_error_line_reports_one_range_issue963);
RUN_TEST(coverage_range_never_ends_past_the_last_line_issue963);
RUN_TEST(coverage_range_never_covers_an_extracted_definition);
RUN_TEST(coverage_gap_of_only_comments_is_not_a_miss);
RUN_TEST(sql_values_scanner_excludes_only_literal_rows_issue1735);
RUN_TEST(sql_values_scanner_keeps_positions_of_kept_text_issue1735);
RUN_TEST(sql_dump_literal_rows_leave_the_graph_unchanged_issue1735);
RUN_TEST(sql_dump_tuple_with_subquery_or_call_is_still_parsed_issue1735);
RUN_TEST(sql_dump_parse_does_not_grow_with_the_row_count_issue1735);
RUN_TEST(sql_dump_of_many_megabytes_is_indexed_not_timed_out_issue1735);
RUN_TEST(jsx_lone_ampersand_is_not_parse_partial_issue1736);
RUN_TEST(jsx_broken_markup_still_parse_partial_issue1736);
RUN_TEST(cs_collection_expression_in_conditional_is_complete_issue1748);
RUN_TEST(cs_calls_inside_collection_expression_extracted_issue1748);
RUN_TEST(cs_malformed_conditional_remains_partial_issue1748);
RUN_TEST(markdown_unterminated_last_line_is_clean_issue2078);
RUN_TEST(markdown_terminated_last_line_controls_unchanged_issue2078);
RUN_TEST(code_without_final_newline_extracts_identically_issue2078);
RUN_TEST(virtual_newline_adds_no_phantom_line_or_text_issue2078);
RUN_TEST(empty_and_newline_only_files_unchanged_issue2078);
RUN_TEST(pascal_cyrillic_member_calls_not_partial_issue2211);
RUN_TEST(pascal_cyrillic_procedure_name_extracted_issue2211);
RUN_TEST(pascal_ascii_control_still_clean_issue2211);
RUN_TEST(pascal_broken_source_still_flagged_issue2211);
}