Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(sql): keep literal INSERT rows out of the SQL parse (#1735)
Martin Vogelc0a06b3d0a90
diff --git a/Makefile.cbm b/Makefile.cbmindex 7a3cd31e1..368fce973 100644--- a/Makefile.cbm+++ b/Makefile.cbm@@ -348,6 +348,7 @@ EXTRACTION_SRCS = \$(CBM_DIR)/extract_channels.c \$(CBM_DIR)/extract_k8s.c \$(CBM_DIR)/extract_dbt.c \+ $(CBM_DIR)/sql_values.c \$(CBM_DIR)/helpers.c \$(CBM_DIR)/result_compact.c \$(CBM_DIR)/result_spill.c \diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.cindex a6fa8fd32..082bca1d6 100644--- a/internal/cbm/cbm.c+++ b/internal/cbm/cbm.c@@ -21,6 +21,7 @@#include "lsp/kotlin_lsp.h"#include "lsp/rust_lsp.h"#include "preprocessor.h"+#include "sql_values.h" // #1735: literal INSERT rows kept out of the SQL parse#include "foundation/compat.h"#include "foundation/compat_fs.h" // cbm_fopen — crash-supervisor per-file marker write#include "foundation/hash_table.h" // CBMHashTable — crash-supervisor quarantine set@@ -2125,6 +2126,21 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguageenum { CBM_EXTRACT_SCRATCH_BLOCK = CBM_SZ_512 * CBM_SZ_1K };enum { CBM_EXTRACT_SCRATCH_KEEP_BYTES = 4 * CBM_SZ_1K * CBM_SZ_1K };+/* The #1735 value-row exclusion is always on. Test builds can turn it off for+ * one file (CBM_TEST_SQL_FULL_PARSE_ON=<rel_path substring>) so a test can+ * compare the excluded parse against the full one on the same source. */+static bool cbm_sql_values_exclusion_on(const char *rel_path) {+#ifdef CBM_ENABLE_TEST_SEAMS+ const char *full_on = getenv("CBM_TEST_SQL_FULL_PARSE_ON");+ if (full_on && full_on[0] && rel_path && strstr(rel_path, full_on)) {+ return false;+ }+#else+ (void)rel_path;+#endif+ return true;+}+static CBMFileResult *extract_file_ex_body(const char *source, int source_len, CBMLanguage language,const char *project, const char *rel_path,int64_t timeout_micros, const char **extra_defines,@@ -2233,7 +2249,19 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C#endif}+ /* #1735: a SQL data dump's literal-only INSERT rows carry no graph content+ * but dominate its parse. Keep them out through included ranges; offsets+ * and positions of everything kept are unchanged. The parser is+ * thread-local and reused, so the ranges are cleared right after. */+ CBMSqlKeptRanges sql_kept = {NULL, 0};+ bool sql_ranged = language == CBM_LANG_SQL && cbm_sql_values_exclusion_on(rel_path) &&+ cbm_sql_values_kept_ranges(source, (uint32_t)source_len, &sql_kept) &&+ ts_parser_set_included_ranges(parser, sql_kept.items, sql_kept.count);TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts);+ if (sql_ranged) {+ (void)ts_parser_set_included_ranges(parser, NULL, 0);+ }+ cbm_sql_kept_ranges_free(&sql_kept);uint64_t t1 = now_ns();#ifdef CBM_ENABLE_TEST_SEAMSt1 += tl_parse_wall_seam_offset_ns; /* the stall seam inflates every wall reading */diff --git a/internal/cbm/sql_values.c b/internal/cbm/sql_values.cnew file mode 100644index 000000000..21d286689--- /dev/null+++ b/internal/cbm/sql_values.c@@ -0,0 +1,474 @@+// sql_values.c — exclude literal-only INSERT value tuples from the SQL parse (#1735).+//+// See sql_values.h for why. The scanner is one forward pass over the file. It+// tracks enough of SQL's lexical structure to never mistake the inside of a+// string, quoted identifier or comment for syntax:+// - '...' strings with both the standard '' escape and MySQL's backslash escape+// - "..." and `...` quoted identifiers, $tag$...$tag$ dollar-quoted bodies+// - -- and # line comments, /* */ block comments+// and it knows only one statement shape: INSERT/REPLACE ... VALUES (..), (..).+//+// Everything it is unsure about stays in the parse. A tuple is excluded only+// when every value in it is a plain literal and the tuple closes cleanly; a+// string holding a raw newline also keeps its tuple, because a dump writes line+// breaks as \n and a raw one means the scan's idea of where strings end may not+// match the grammar's.++#include "sql_values.h"+#include "foundation/mem_core.h"+#include <ctype.h>+#include <stddef.h>+#include <string.h>+#include <strings.h>++typedef struct {+ const char *s;+ uint32_t len;+ uint32_t i;+ uint32_t row;+ uint32_t bol; /* byte offset where the current row starts */+} sqlv_scan_t;++typedef struct {+ TSRange *items;+ uint32_t count;+ uint32_t cap;+ bool failed;+} sqlv_ranges_t;++enum { SQLV_INITIAL_CAP = 64, SQLV_GROWTH = 2 };++static char sqlv_at(const sqlv_scan_t *sc, uint32_t off) {+ return sc->i + off < sc->len ? sc->s[sc->i + off] : '\0';+}++static void sqlv_step(sqlv_scan_t *sc) {+ if (sc->s[sc->i] == '\n') {+ sc->row++;+ sc->bol = sc->i + 1;+ }+ sc->i++;+}++static TSPoint sqlv_point(const sqlv_scan_t *sc) {+ return (TSPoint){sc->row, sc->i - sc->bol};+}++static bool sqlv_word_char(char c) {+ return isalnum((unsigned char)c) || c == '_' || c == '$' || (unsigned char)c >= 0x80;+}++static bool sqlv_word_is(const char *w, uint32_t n, const char *kw) {+ return strlen(kw) == n && strncasecmp(w, kw, n) == 0;+}++/* Consume a '..', "..", or `..` run starting at the opening quote. Doubled+ * quotes escape everywhere; a backslash escapes in strings and "..". Sets+ * *raw_newline when the run crosses a line break. False if it never closes. */+static bool sqlv_skip_quoted(sqlv_scan_t *sc, bool *raw_newline) {+ char q = sc->s[sc->i];+ sqlv_step(sc);+ while (sc->i < sc->len) {+ char c = sc->s[sc->i];+ if (c == '\n') {+ *raw_newline = true;+ }+ if (c == '\\' && q != '`' && sc->i + 1 < sc->len) {+ sqlv_step(sc);+ sqlv_step(sc);+ continue;+ }+ sqlv_step(sc);+ if (c == q) {+ if (sqlv_at(sc, 0) != q) {+ return true;+ }+ sqlv_step(sc); /* doubled quote: an escaped quote character */+ }+ }+ return false;+}++/* $tag$ ... $tag$ (PostgreSQL). Returns false, consuming nothing, when the `$`+ * does not open a dollar quote ($1 parameters, a lone $). */+static bool sqlv_skip_dollar(sqlv_scan_t *sc) {+ uint32_t t = sc->i + 1;+ if (t < sc->len && isdigit((unsigned char)sc->s[t])) {+ return false;+ }+ while (t < sc->len && sc->s[t] != '$' && sqlv_word_char(sc->s[t])) {+ t++;+ }+ if (t >= sc->len || sc->s[t] != '$') {+ return false;+ }+ uint32_t tag_len = t - sc->i + 1; /* the opener, both dollars included */+ const char *tag = sc->s + sc->i;+ for (uint32_t k = 0; k < tag_len; k++) {+ sqlv_step(sc);+ }+ while (sc->i < sc->len) {+ if (sc->s[sc->i] == '$' && sc->len - sc->i >= tag_len &&+ memcmp(sc->s + sc->i, tag, tag_len) == 0) {+ for (uint32_t k = 0; k < tag_len; k++) {+ sqlv_step(sc);+ }+ return true;+ }+ sqlv_step(sc);+ }+ return true;+}++/* Consume one comment starting at the cursor; false if there is none. */+static bool sqlv_skip_comment(sqlv_scan_t *sc) {+ char c = sqlv_at(sc, 0);+ char n = sqlv_at(sc, 1);+ if ((c == '-' && n == '-') || c == '#') {+ while (sc->i < sc->len && sc->s[sc->i] != '\n') {+ sqlv_step(sc);+ }+ return true;+ }+ if (c == '/' && n == '*') {+ sqlv_step(sc);+ sqlv_step(sc);+ while (sc->i < sc->len && !(sc->s[sc->i] == '*' && sqlv_at(sc, 1) == '/')) {+ sqlv_step(sc);+ }+ if (sc->i < sc->len) {+ sqlv_step(sc);+ sqlv_step(sc);+ }+ return true;+ }+ return false;+}++/* Skip whitespace and comments. *ws_only (optional) is cleared if a comment was+ * among them. */+static void sqlv_skip_trivia(sqlv_scan_t *sc, bool *ws_only) {+ while (sc->i < sc->len) {+ if (isspace((unsigned char)sc->s[sc->i])) {+ sqlv_step(sc);+ } else if (sqlv_skip_comment(sc)) {+ if (ws_only) {+ *ws_only = false;+ }+ } else {+ return;+ }+ }+}++/* Consume one quoted or dollar-quoted run if one starts here. */+static bool sqlv_skip_any_quoted(sqlv_scan_t *sc) {+ char c = sc->s[sc->i];+ bool nl = false;+ if (c == '\'' || c == '"' || c == '`') {+ (void)sqlv_skip_quoted(sc, &nl);+ return true;+ }+ return c == '$' && sqlv_skip_dollar(sc);+}++/* From inside an open parenthesis, consume through its matching ')'. */+static bool sqlv_skip_balanced(sqlv_scan_t *sc) {+ int depth = 1;+ while (sc->i < sc->len) {+ if (sqlv_skip_comment(sc) || sqlv_skip_any_quoted(sc)) {+ continue;+ }+ char c = sc->s[sc->i];+ sqlv_step(sc);+ if (c == '(') {+ depth++;+ } else if (c == ')' && --depth == 0) {+ return true;+ }+ }+ return false;+}++static bool sqlv_read_string(sqlv_scan_t *sc) {+ bool nl = false;+ return sqlv_skip_quoted(sc, &nl) && !nl;+}++/* 12, -3.5, 1e-9, .5, 0x1F, 0b101 — followed by a non-word character. */+static bool sqlv_read_number(sqlv_scan_t *sc) {+ char n = sqlv_at(sc, 1);+ if (sqlv_at(sc, 0) == '0' && (n == 'x' || n == 'X') &&+ isxdigit((unsigned char)sqlv_at(sc, 2))) {+ sqlv_step(sc);+ sqlv_step(sc);+ while (isxdigit((unsigned char)sqlv_at(sc, 0))) {+ sqlv_step(sc);+ }+ } else if (sqlv_at(sc, 0) == '0' && (n == 'b' || n == 'B') &&+ (sqlv_at(sc, 2) == '0' || sqlv_at(sc, 2) == '1')) {+ sqlv_step(sc);+ sqlv_step(sc);+ while (sqlv_at(sc, 0) == '0' || sqlv_at(sc, 0) == '1') {+ sqlv_step(sc);+ }+ } else {+ while (isdigit((unsigned char)sqlv_at(sc, 0)) || sqlv_at(sc, 0) == '.') {+ sqlv_step(sc);+ }+ char e = sqlv_at(sc, 0);+ char s1 = sqlv_at(sc, 1);+ if ((e == 'e' || e == 'E') &&+ (isdigit((unsigned char)s1) ||+ ((s1 == '+' || s1 == '-') && isdigit((unsigned char)sqlv_at(sc, 2))))) {+ sqlv_step(sc);+ sqlv_step(sc);+ while (isdigit((unsigned char)sqlv_at(sc, 0))) {+ sqlv_step(sc);+ }+ }+ }+ return !sqlv_word_char(sqlv_at(sc, 0)) && sqlv_at(sc, 0) != '.';+}++/* A word in value position: NULL/TRUE/FALSE/DEFAULT, an X'..' / B'..' / N'..'+ * prefixed string, or a _charset introducer before a string or number. */+static bool sqlv_read_word_literal(sqlv_scan_t *sc) {+ const char *w = sc->s + sc->i;+ uint32_t n = 0;+ while (sqlv_word_char(sqlv_at(sc, 0))) {+ sqlv_step(sc);+ n++;+ }+ if (sqlv_word_is(w, n, "NULL") || sqlv_word_is(w, n, "TRUE") || sqlv_word_is(w, n, "FALSE") ||+ sqlv_word_is(w, n, "DEFAULT")) {+ return true;+ }+ if (n == 1 && strchr("xXbBnN", w[0]) && sqlv_at(sc, 0) == '\'') {+ return sqlv_read_string(sc);+ }+ if (n > 1 && w[0] == '_') {+ sqlv_skip_trivia(sc, NULL);+ if (sqlv_at(sc, 0) == '\'') {+ return sqlv_read_string(sc);+ }+ return isdigit((unsigned char)sqlv_at(sc, 0)) && sqlv_read_number(sc);+ }+ return false;+}++/* Consume one value if it is a plain literal. Never consumes a parenthesis, so+ * on false the caller can still find the tuple's end from the cursor. */+static bool sqlv_read_literal(sqlv_scan_t *sc) {+ char c = sqlv_at(sc, 0);+ char n = sqlv_at(sc, 1);+ if (c == '\'') {+ return sqlv_read_string(sc);+ }+ if ((c == '-' || c == '+') &&+ (isdigit((unsigned char)n) || (n == '.' && isdigit((unsigned char)sqlv_at(sc, 2))))) {+ sqlv_step(sc);+ return sqlv_read_number(sc);+ }+ if (isdigit((unsigned char)c) || (c == '.' && isdigit((unsigned char)n))) {+ return sqlv_read_number(sc);+ }+ if (isalpha((unsigned char)c) || c == '_') {+ return sqlv_read_word_literal(sc);+ }+ return false;+}++/* Consume one tuple starting at its '('. *literal says whether every value in+ * it was a plain literal. Returns false if the tuple never closes. */+static bool sqlv_scan_tuple(sqlv_scan_t *sc, bool *literal) {+ sqlv_step(sc);+ sqlv_skip_trivia(sc, NULL);+ if (sqlv_at(sc, 0) == ')') {+ sqlv_step(sc);+ *literal = true;+ return true;+ }+ while (sqlv_read_literal(sc)) {+ sqlv_skip_trivia(sc, NULL);+ char c = sqlv_at(sc, 0);+ if (c == ')') {+ sqlv_step(sc);+ *literal = true;+ return true;+ }+ if (c != ',') {+ break;+ }+ sqlv_step(sc);+ sqlv_skip_trivia(sc, NULL);+ }+ *literal = false;+ return sqlv_skip_balanced(sc);+}++static void sqlv_push(sqlv_ranges_t *r, TSRange range) {+ if (r->failed) {+ return;+ }+ if (r->count == r->cap) {+ uint32_t cap = r->cap ? r->cap * SQLV_GROWTH : SQLV_INITIAL_CAP;+ TSRange *grown =+ (TSRange *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, r->items, (size_t)cap * sizeof(TSRange));+ if (!grown) {+ r->failed = true;+ return;+ }+ r->items = grown;+ r->cap = cap;+ }+ r->items[r->count++] = range;+}++/* The cursor sits just past VALUES. Keep the first tuple; exclude each later+ * literal-only tuple together with the comma before it. Runs of excluded tuples+ * separated only by whitespace become one exclusion. */+static void sqlv_handle_values(sqlv_scan_t *sc, sqlv_ranges_t *ex) {+ sqlv_skip_trivia(sc, NULL);+ bool literal = false;+ if (sqlv_at(sc, 0) != '(' || !sqlv_scan_tuple(sc, &literal)) {+ return;+ }+ bool pending = false;+ TSRange cur = {0};+ for (;;) {+ bool ws_only = true;+ sqlv_skip_trivia(sc, &ws_only);+ if (sqlv_at(sc, 0) != ',') {+ break;+ }+ uint32_t comma = sc->i;+ TSPoint comma_pt = sqlv_point(sc);+ sqlv_step(sc);+ sqlv_skip_trivia(sc, NULL);+ if (sqlv_at(sc, 0) != '(' || !sqlv_scan_tuple(sc, &literal)) {+ break;+ }+ if (!literal) {+ if (pending) {+ sqlv_push(ex, cur);+ }+ pending = false;+ continue;+ }+ if (!pending || !ws_only) {+ if (pending) {+ sqlv_push(ex, cur);+ }+ cur.start_byte = comma;+ cur.start_point = comma_pt;+ pending = true;+ }+ cur.end_byte = sc->i;+ cur.end_point = sqlv_point(sc);+ }+ if (pending) {+ sqlv_push(ex, cur);+ }+}++typedef struct {+ bool started; /* a token of the current statement has been seen */+ bool insert; /* the statement began with INSERT or REPLACE */+ int depth;+} sqlv_stmt_t;++static void sqlv_on_word(sqlv_scan_t *sc, sqlv_stmt_t *st, sqlv_ranges_t *ex) {+ const char *w = sc->s + sc->i;+ uint32_t n = 0;+ while (sc->i < sc->len && sqlv_word_char(sc->s[sc->i])) {+ sqlv_step(sc);+ n++;+ }+ if (!st->started) {+ st->started = true;+ st->insert = sqlv_word_is(w, n, "INSERT") || sqlv_word_is(w, n, "REPLACE");+ } else if (st->insert && st->depth == 0 &&+ (sqlv_word_is(w, n, "VALUES") || sqlv_word_is(w, n, "VALUE"))) {+ sqlv_handle_values(sc, ex);+ }+}++static void sqlv_scan_file(sqlv_scan_t *sc, sqlv_ranges_t *ex) {+ sqlv_stmt_t st = {false, false, 0};+ while (sc->i < sc->len && !ex->failed) {+ if (sqlv_skip_comment(sc)) {+ continue;+ }+ char c = sc->s[sc->i];+ if (sqlv_skip_any_quoted(sc)) {+ st.started = true;+ continue;+ }+ if (sqlv_word_char(c)) {+ sqlv_on_word(sc, &st, ex);+ continue;+ }+ if (c == ';') {+ st = (sqlv_stmt_t){false, false, 0};+ } else if (c == '(') {+ st.depth++;+ } else if (c == ')' && st.depth > 0) {+ st.depth--;+ }+ if (!isspace((unsigned char)c) && c != ';') {+ st.started = true;+ }+ sqlv_step(sc);+ }+}++bool cbm_sql_values_kept_ranges(const char *src, uint32_t len, CBMSqlKeptRanges *out) {+ out->items = NULL;+ out->count = 0;+ if (!src || len == 0) {+ return false;+ }+ sqlv_scan_t sc = {src, len, 0, 0, 0};+ sqlv_ranges_t ex = {NULL, 0, 0, false};+ sqlv_scan_file(&sc, &ex);+ if (ex.failed || ex.count == 0) {+ cbm_free(CBM_MEM_CLASS_EXTRACT, ex.items);+ return false;+ }+ /* Kept = the complement of the exclusions. The scan ended at EOF, so its+ * cursor point is the end of the file. */+ TSRange *kept =+ (TSRange *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, ((size_t)ex.count + 1) * sizeof(TSRange));+ if (!kept) {+ cbm_free(CBM_MEM_CLASS_EXTRACT, ex.items);+ return false;+ }+ uint32_t n = 0;+ uint32_t from = 0;+ TSPoint from_pt = {0, 0};+ for (uint32_t k = 0; k <= ex.count; k++) {+ uint32_t to = k < ex.count ? ex.items[k].start_byte : len;+ TSPoint to_pt = k < ex.count ? ex.items[k].start_point : sqlv_point(&sc);+ if (to > from) {+ kept[n++] = (TSRange){from_pt, to_pt, from, to};+ }+ if (k < ex.count) {+ from = ex.items[k].end_byte;+ from_pt = ex.items[k].end_point;+ }+ }+ cbm_free(CBM_MEM_CLASS_EXTRACT, ex.items);+ out->items = kept;+ out->count = n;+ return true;+}++void cbm_sql_kept_ranges_free(CBMSqlKeptRanges *ranges) {+ if (!ranges) {+ return;+ }+ cbm_free(CBM_MEM_CLASS_EXTRACT, ranges->items);+ ranges->items = NULL;+ ranges->count = 0;+}diff --git a/internal/cbm/sql_values.h b/internal/cbm/sql_values.hnew file mode 100644index 000000000..5960ec320--- /dev/null+++ b/internal/cbm/sql_values.h@@ -0,0 +1,40 @@+/* sql_values.h — keep a SQL data dump's literal rows out of the parse (#1735).+ *+ * A mysqldump-style file is almost entirely `INSERT ... VALUES (..),(..),...`+ * rows of plain literals. Those rows contribute nothing to the graph (no+ * definition, call or usage lives in a literal), yet tree-sitter builds a full+ * tree for every one of them: ~70 bytes of tree per source byte and seconds of+ * parse per ten megabytes, until the per-file budget gives up and the whole file+ * — its CREATE TABLE statements included — is skipped as "parse timeout".+ *+ * cbm_sql_values_kept_ranges() finds, in one linear quote- and comment-aware+ * scan, every value tuple AFTER THE FIRST of an INSERT/REPLACE ... VALUES list+ * that consists only of literals (strings, numbers, NULL, TRUE, FALSE, DEFAULT),+ * and returns the complement as tree-sitter included ranges. The parser then+ * sees `INSERT INTO t VALUES (first row);` — a complete statement — while every+ * byte offset and line/column of the kept text is unchanged, so every node the+ * extractors read still points at the real source.+ *+ * A syntax rule, not a work cap: the same file always yields the same ranges,+ * whatever its size. A tuple holding anything that could reference code — a+ * subquery, a function call, an identifier, a quoted identifier — is kept. */+#ifndef CBM_SQL_VALUES_H+#define CBM_SQL_VALUES_H++#include "tree_sitter/api.h"+#include <stdbool.h>+#include <stdint.h>++typedef struct {+ TSRange *items; /* ascending, non-overlapping; owned (cbm_sql_kept_ranges_free) */+ uint32_t count;+} CBMSqlKeptRanges;++/* Compute the included ranges for `src`. Returns true and fills `out` only when+ * at least one literal tuple was excluded; returns false (out zeroed) when the+ * whole file must be parsed as-is or on allocation failure. */+bool cbm_sql_values_kept_ranges(const char *src, uint32_t len, CBMSqlKeptRanges *out);++void cbm_sql_kept_ranges_free(CBMSqlKeptRanges *ranges);++#endif /* CBM_SQL_VALUES_H */diff --git a/tests/test_parse_coverage.c b/tests/test_parse_coverage.cindex fd0e64e47..91f99f95d 100644--- a/tests/test_parse_coverage.c+++ b/tests/test_parse_coverage.c@@ -30,6 +30,9 @@#include "test_framework.h"#include "cbm.h"+#include "sql_values.h" /* #1735 value-row scanner */+#include "pipeline/pipeline_internal.h" /* CBM_EXTRACT_BUDGET */+#include <stdarg.h>#include <stdio.h>#include <stdlib.h>#include <string.h>@@ -983,6 +986,445 @@ TEST(coverage_gap_of_only_comments_is_not_a_miss) {PASS();}+/* ── #1735: SQL data dumps ───────────────────────────────────────────────────+ * A mysqldump file is a few CREATE TABLEs followed by megabytes of literal+ * INSERT rows. Tree-sitter built a full tree for every row until the parse+ * budget ran out, and the whole file — tables included — was skipped as+ * "parse timeout". Literal-only rows after the first of each VALUES list are+ * now kept out of the parse (sql_values.c). These tests pin that nothing the+ * full parse extracted outside those rows is lost (and, for standard '' escapes+ * the grammar reads correctly, that nothing changes at all), that a row which+ * can reference something is still parsed, and that the parse no longer grows+ * with the row count. */++typedef struct {+ char *s;+ size_t len;+ size_t cap;+} cov_buf_t;++static void cov_put(cov_buf_t *b, const char *fmt, ...) {+ va_list ap;+ va_start(ap, fmt);+ va_list ap2;+ va_copy(ap2, ap);+ int n = vsnprintf(NULL, 0, fmt, ap);+ va_end(ap);+ if (n < 0) {+ va_end(ap2);+ return;+ }+ if (b->len + (size_t)n + 1 > b->cap) {+ size_t cap = b->cap ? b->cap : 4096;+ while (b->len + (size_t)n + 1 > cap) {+ cap *= 2;+ }+ char *grown = realloc(b->s, cap);+ if (!grown) {+ va_end(ap2);+ return;+ }+ b->s = grown;+ b->cap = cap;+ }+ vsnprintf(b->s + b->len, (size_t)n + 1, fmt, ap2);+ va_end(ap2);+ b->len += (size_t)n;+}++static const char *const DUMP_WORDS[] = {"Kabul", "Herat", "Amsterdam", "O'Brien",+ "Haag", "Rotterdam", "Zuid-Holland", "Utrecht"};+enum { DUMP_WORD_COUNT = sizeof(DUMP_WORDS) / sizeof(DUMP_WORDS[0]) };+static const char *const DUMP_ODD[] = {"NULL", "-12.5", "0x1F", "_binary 'ab'", "DEFAULT",+ "TRUE", "1e-3", "X'0A'", "''", "'a''b'"};+enum { DUMP_ODD_COUNT = sizeof(DUMP_ODD) / sizeof(DUMP_ODD[0]) };++/* Deterministic mysqldump-shaped SQL: two tables and a view, then `stmts`+ * extended INSERTs of `rows` tuples. `mysql_esc` writes a quote inside a string+ * as \' (MySQL) instead of '' (standard). `mixed` puts one tuple holding a+ * subquery and one holding a function call in the middle of every INSERT. The+ * first two INSERTs separate their tuples with a newline and with a comment, the+ * way pretty-printed dumps do. */+static char *sql_dump(int stmts, int rows, bool mysql_esc, bool mixed) {+ cov_buf_t b = {NULL, 0, 0};+ cov_put(&b, "-- MySQL dump 10.13\n"+ "/*!40101 SET NAMES utf8mb4 */;\n"+ "CREATE TABLE `city` (\n"+ " `ID` int NOT NULL,\n"+ " `Name` char(35) NOT NULL DEFAULT '',\n"+ " `CountryCode` char(3),\n"+ " `Note` text,\n"+ " `Population` int\n"+ ");\n"+ "CREATE TABLE country (Code char(3), Name char(52));\n"+ "CREATE VIEW big_cities AS SELECT Name FROM city WHERE Population > 1000000;\n"+ "LOCK TABLES `city` WRITE;\n");+ uint32_t seed = 1735;+ int id = 1;+ for (int s = 0; s < stmts; s++) {+ cov_put(&b, "INSERT INTO `city` VALUES ");+ const char *sep = s == 0 ? ",\n" : (s == 1 ? ", /* split */ " : ",");+ for (int r = 0; r < rows; r++) {+ if (r > 0) {+ cov_put(&b, "%s", sep);+ }+ if (mixed && r == rows / 2) {+ cov_put(&b, "((SELECT MAX(Code) FROM country),'x','AAA',NULL,1)");+ continue;+ }+ if (mixed && r == rows / 2 + 1) {+ cov_put(&b, "(%d,UPPER('y'),'BBB',NULL,2)", id++);+ continue;+ }+ seed = seed * 1103515245u + 12345u;+ const char *w = DUMP_WORDS[(seed >> 8) % DUMP_WORD_COUNT];+ cov_put(&b, "(%d,'", id++);+ for (const char *p = w; *p; p++) {+ cov_put(&b, *p == '\'' ? (mysql_esc ? "\\'" : "''") : "%c", *p);+ }+ cov_put(&b, "','%c%c%c',%s,%u)", 'A' + (int)(seed % 26), 'A' + (int)((seed >> 5) % 26),+ 'A' + (int)((seed >> 10) % 26), DUMP_ODD[(seed >> 12) % DUMP_ODD_COUNT],+ (seed >> 3) % 9000000u);+ }+ cov_put(&b, ";\n");+ }+ cov_put(&b, "UNLOCK TABLES;\n");+ return b.s;+}++static void fp_s(cov_buf_t *b, const char *s) {+ cov_put(b, "%s|", s ? s : "~");+}++/* Everything the graph is built from except calls and usages, in order (those+ * two are compared site by site, sites_agree). */+static char *result_fingerprint(const CBMFileResult *r) {+ cov_buf_t b = {NULL, 0, 0};+ for (int i = 0; i < r->defs.count; i++) {+ const CBMDefinition *d = &r->defs.items[i];+ fp_s(&b, d->label);+ fp_s(&b, d->name);+ fp_s(&b, d->qualified_name);+ fp_s(&b, d->signature);+ fp_s(&b, d->return_type);+ fp_s(&b, d->structural_profile);+ fp_s(&b, d->body_tokens);+ cov_put(&b, "%u-%u c%d l%d\n", d->start_line, d->end_line, d->complexity, d->lines);+ }+ for (int i = 0; i < r->imports.count; i++) {+ fp_s(&b, r->imports.items[i].local_name);+ fp_s(&b, r->imports.items[i].module_path);+ cov_put(&b, "imp\n");+ }+ for (int i = 0; i < r->rw.count; i++) {+ fp_s(&b, r->rw.items[i].var_name);+ cov_put(&b, "rw%d\n", (int)r->rw.items[i].is_write);+ }+ for (int i = 0; i < r->type_refs.count; i++) {+ fp_s(&b, r->type_refs.items[i].type_name);+ cov_put(&b, "tref\n");+ }+ for (int i = 0; i < r->env_accesses.count; i++) {+ fp_s(&b, r->env_accesses.items[i].env_key);+ cov_put(&b, "env\n");+ }+ for (int i = 0; i < r->throws.count; i++) {+ fp_s(&b, r->throws.items[i].exception_name);+ cov_put(&b, "throw\n");+ }+ for (int i = 0; i < r->string_refs.count; i++) {+ fp_s(&b, r->string_refs.items[i].value);+ cov_put(&b, "sref%d\n", (int)r->string_refs.items[i].kind);+ }+ cov_put(&b, "ta%d it%d rc%d ib%d ch%d\n", r->type_assigns.count, r->impl_traits.count,+ r->resolved_calls.count, r->infra_bindings.count, r->channels.count);+ return b.s;+}++static int has_usage_named(const CBMFileResult *r, const char *name) {+ for (int i = 0; i < r->usages.count; i++) {+ if (r->usages.items[i].ref_name && strstr(r->usages.items[i].ref_name, name)) {+ return 1;+ }+ }+ return 0;+}++static int has_call_named(const CBMFileResult *r, const char *name) {+ for (int i = 0; i < r->calls.count; i++) {+ if (r->calls.items[i].callee_name && strstr(r->calls.items[i].callee_name, name)) {+ return 1;+ }+ }+ return 0;+}++typedef struct {+ const char *name;+ uint32_t start;+ uint32_t end;+} cov_site_t;++static int collect_sites(const CBMFileResult *r, bool calls, cov_site_t **out) {+ int n = calls ? r->calls.count : r->usages.count;+ *out = calloc((size_t)(n > 0 ? n : 1), sizeof(cov_site_t));+ for (int i = 0; i < n && *out; i++) {+ (*out)[i] =+ calls ? (cov_site_t){r->calls.items[i].callee_name, r->calls.items[i].site_start_byte,+ r->calls.items[i].site_end_byte}+ : (cov_site_t){r->usages.items[i].ref_name, r->usages.items[i].site_start_byte,+ r->usages.items[i].site_end_byte};+ }+ return *out ? n : 0;+}++static bool site_in(cov_site_t s, const cov_site_t *arr, int n) {+ for (int i = 0; i < n; i++) {+ if (arr[i].start == s.start && arr[i].end == s.end && s.name && arr[i].name &&+ strcmp(arr[i].name, s.name) == 0) {+ return true;+ }+ }+ return false;+}++/* Calls (calls=true) or usages, cut parse against full parse:+ * - nothing the full parse found outside the dropped rows may be missing;+ * - with `exact`, the cut parse may find nothing the full parse did not.+ * The full parse does find things INSIDE dropped rows: the SQL grammar reads+ * DEFAULT and a _binary introducer as identifiers, and a MySQL \' escape it+ * does not know turns the rest of a string into one. None of those names+ * anything. Without `exact` (MySQL escapes) the cut parse may find more: the+ * full parse's error recovery around a misread escape swallows neighbouring+ * rows, a subquery row among them, and the cut parse no longer misreads them. */+static bool sites_agree(const char *src, const CBMFileResult *cut, const CBMFileResult *full,+ bool calls, bool exact) {+ cov_site_t *cs = NULL;+ cov_site_t *fs = NULL;+ int cn = collect_sites(cut, calls, &cs);+ int fn = collect_sites(full, calls, &fs);+ CBMSqlKeptRanges k = {NULL, 0};+ (void)cbm_sql_values_kept_ranges(src, (uint32_t)strlen(src), &k);+ bool ok = cs && fs;+ for (int i = 0; ok && exact && i < cn; i++) {+ if (!site_in(cs[i], fs, fn)) {+ fprintf(stderr, " %s only in the cut parse\n", cs[i].name);+ ok = false;+ }+ }+ for (int i = 0; ok && i < fn; i++) {+ if (site_in(fs[i], cs, cn)) {+ continue;+ }+ for (uint32_t j = 0; j < k.count; j++) {+ if (fs[i].start < k.items[j].end_byte && fs[i].end > k.items[j].start_byte) {+ fprintf(stderr, " %s lost outside the dropped rows\n", fs[i].name);+ ok = false;+ }+ }+ }+ cbm_sql_kept_ranges_free(&k);+ free(cs);+ free(fs);+ return ok;+}++static int count_calls_named(const CBMFileResult *r, const char *name) {+ int n = 0;+ for (int i = 0; i < r->calls.count; i++) {+ if (r->calls.items[i].callee_name && strcmp(r->calls.items[i].callee_name, name) == 0) {+ n++;+ }+ }+ return n;+}++/* The same source parsed with the row exclusion and without it (test seam). */+static CBMFileResult *extract_sql_full(const char *src, const char *path) {+ setenv("CBM_TEST_SQL_FULL_PARSE_ON", path, 1);+ CBMFileResult *r = do_extract(src, CBM_LANG_SQL, path);+ unsetenv("CBM_TEST_SQL_FULL_PARSE_ON");+ return r;+}++TEST(sql_dump_literal_rows_leave_the_graph_unchanged_issue1735) {+ for (int esc = 0; esc < 2; esc++) {+ char *src = sql_dump(4, 60, esc == 1, true);+ ASSERT_NOT_NULL(src);+ CBMFileResult *cut = do_extract(src, CBM_LANG_SQL, "dump.sql");+ CBMFileResult *full = extract_sql_full(src, "dump.sql");+ ASSERT_NOT_NULL(cut);+ ASSERT_NOT_NULL(full);+ ASSERT_FALSE(cut->has_error);+ ASSERT_TRUE(has_def(cut, "big_cities"));+ char *fp_cut = result_fingerprint(cut);+ char *fp_full = result_fingerprint(full);+ ASSERT_NOT_NULL(fp_cut);+ ASSERT_NOT_NULL(fp_full);+ ASSERT_STR_EQ(fp_cut, fp_full);+ ASSERT_TRUE(sites_agree(src, cut, full, true, esc == 0));+ ASSERT_TRUE(sites_agree(src, cut, full, false, esc == 0));+ /* Every INSERT's subquery row is parsed. */+ ASSERT_EQ(count_calls_named(cut, "MAX"), 4);+ /* ...and the rows really were left out: under a quarter of the file+ * is parsed. */+ CBMSqlKeptRanges k = {NULL, 0};+ ASSERT_TRUE(cbm_sql_values_kept_ranges(src, (uint32_t)strlen(src), &k));+ size_t kept = 0;+ for (uint32_t j = 0; j < k.count; j++) {+ kept += k.items[j].end_byte - k.items[j].start_byte;+ }+ cbm_sql_kept_ranges_free(&k);+ ASSERT_LT(kept * 4, strlen(src));+ free(fp_cut);+ free(fp_full);+ cbm_free_result(cut);+ cbm_free_result(full);+ free(src);+ }+ PASS();+}++TEST(sql_dump_tuple_with_subquery_or_call_is_still_parsed_issue1735) {+ char *src = sql_dump(1, 20, true, true);+ ASSERT_NOT_NULL(src);+ CBMFileResult *r = do_extract(src, CBM_LANG_SQL, "dump.sql");+ ASSERT_NOT_NULL(r);+ ASSERT_TRUE(has_usage_named(r, "country")); /* from the subquery row */+ ASSERT_TRUE(has_call_named(r, "UPPER")); /* from the function-call row */+ cbm_free_result(r);+ free(src);+ PASS();+}++TEST(sql_dump_parse_does_not_grow_with_the_row_count_issue1735) {+ /* A count, not a clock: with literal rows kept out, doubling them must not+ * add a single tree node. */+ char *small = sql_dump(1, 500, true, false);+ char *big = sql_dump(1, 1000, true, false);+ ASSERT_NOT_NULL(small);+ ASSERT_NOT_NULL(big);+ CBMFileResult *rs = do_extract(small, CBM_LANG_SQL, "small.sql");+ CBMFileResult *rb = do_extract(big, CBM_LANG_SQL, "big.sql");+ ASSERT_NOT_NULL(rs);+ ASSERT_NOT_NULL(rb);+ ASSERT_EQ(rs->tree_nodes, rb->tree_nodes);+ ASSERT_EQ(rs->defs.count, rb->defs.count);+ cbm_free_result(rs);+ cbm_free_result(rb);+ free(small);+ free(big);+ PASS();+}++TEST(sql_dump_of_many_megabytes_is_indexed_not_timed_out_issue1735) {+ /* ~26 MB with MySQL escapes, extracted under the production parse budget.+ * The outcome is asserted — the file is indexed and its tables are in the+ * graph — never how long it took. */+ char *src = sql_dump(330, 2000, true, true);+ ASSERT_NOT_NULL(src);+ size_t len = strlen(src);+ ASSERT_GT(len, (size_t)24 * 1024 * 1024);+ CBMFileResult *r = cbm_extract_file(src, (int)len, CBM_LANG_SQL, "covproj", "world.sql",+ CBM_EXTRACT_BUDGET, NULL, NULL);+ ASSERT_NOT_NULL(r);+ if (r->has_error) {+ FAIL(r->error_msg ? r->error_msg : "extraction failed");+ }+ ASSERT_TRUE(has_def(r, "big_cities"));+ ASSERT_TRUE(has_usage_named(r, "country"));+ cbm_free_result(r);+ free(src);+ PASS();+}++/* The text the parser sees: every kept range, concatenated. */+static char *kept_text(const char *src, const CBMSqlKeptRanges *k) {+ cov_buf_t b = {NULL, 0, 0};+ cov_put(&b, "%s", "");+ for (uint32_t i = 0; i < k->count; i++) {+ cov_put(&b, "%.*s", (int)(k->items[i].end_byte - k->items[i].start_byte),+ src + k->items[i].start_byte);+ }+ return b.s;+}++TEST(sql_values_scanner_excludes_only_literal_rows_issue1735) {+ /* {source, what the parser sees} — NULL means nothing is excluded. */+ static const char *const cases[][2] = {+ {"INSERT INTO t VALUES (1,'a'),(2,'b\\'c'),(3,NULL);", "INSERT INTO t VALUES (1,'a');"},+ {"INSERT INTO t VALUES (1),(f(2)),(3);", "INSERT INTO t VALUES (1),(f(2));"},+ {"insert into t values (1),(-2.5e3),(0x1F),(_binary 'x'),(DEFAULT),(true),(X'0A'),"+ "('it''s'),(.5),();",+ "insert into t values (1);"},+ {"REPLACE INTO t VALUES (1), /* c */ (2), (3);", "REPLACE INTO t VALUES (1);"},+ {"INSERT INTO t VALUES (1),(2) -- c\n,(3);", "INSERT INTO t VALUES (1) -- c\n;"},+ {"INSERT INTO t VALUES (1),((SELECT id FROM u)),(x),(\"q\"),(`b`),(a.b);", NULL},+ {"INSERT INTO t VALUES (1),('a\nb'),(2);", "INSERT INTO t VALUES (1),('a\nb');"},+ {"INSERT INTO t VALUES (1),(2) ON DUPLICATE KEY UPDATE a=VALUES(a);",+ "INSERT INTO t VALUES (1) ON DUPLICATE KEY UPDATE a=VALUES(a);"},+ {"INSERT INTO t (a, b) VALUES (1, 2), (3, 4);", "INSERT INTO t (a, b) VALUES (1, 2);"},+ {"SELECT 'INSERT INTO t VALUES (1),(2)';", NULL},+ {"-- INSERT INTO t VALUES (1),(2)\nSELECT 1;", NULL},+ {"/* INSERT INTO t VALUES (1),(2) */ SELECT 1;", NULL},+ {"SELECT * FROM (VALUES (1),(2)) AS v(x);", NULL},+ {"CREATE FUNCTION f() AS $$ SELECT 1; INSERT INTO t VALUES (1),(2); $$;", NULL},+ {"INSERT INTO t VALUES (1),(2", NULL},+ {"INSERT INTO t VALUES (1),(1abc),(2);", "INSERT INTO t VALUES (1),(1abc);"},+ };+ for (size_t c = 0; c < sizeof(cases) / sizeof(cases[0]); c++) {+ const char *src = cases[c][0];+ CBMSqlKeptRanges k = {NULL, 0};+ bool cut = cbm_sql_values_kept_ranges(src, (uint32_t)strlen(src), &k);+ if (!cases[c][1]) {+ if (cut) {+ fprintf(stderr, " case %zu excluded rows unexpectedly: %s\n", c, src);+ }+ ASSERT_FALSE(cut);+ continue;+ }+ ASSERT_TRUE(cut);+ char *seen = kept_text(src, &k);+ ASSERT_NOT_NULL(seen);+ ASSERT_STR_EQ(seen, cases[c][1]);+ free(seen);+ cbm_sql_kept_ranges_free(&k);+ }+ PASS();+}++TEST(sql_values_scanner_keeps_positions_of_kept_text_issue1735) {+ const char *src = "INSERT INTO t VALUES\n(1),\n(2);\nCREATE TABLE u (id int);\n";+ CBMSqlKeptRanges k = {NULL, 0};+ ASSERT_TRUE(cbm_sql_values_kept_ranges(src, (uint32_t)strlen(src), &k));+ ASSERT_EQ(k.count, 2u);+ /* excluded: ",\n(2)" from byte 24 (row 1, col 3) to byte 29 (row 2, col 3) */+ ASSERT_EQ(k.items[0].start_byte, 0u);+ ASSERT_EQ(k.items[0].end_byte, 24u);+ ASSERT_EQ(k.items[0].end_point.row, 1u);+ ASSERT_EQ(k.items[0].end_point.column, 3u);+ ASSERT_EQ(k.items[1].start_byte, 29u);+ ASSERT_EQ(k.items[1].start_point.row, 2u);+ ASSERT_EQ(k.items[1].start_point.column, 3u);+ ASSERT_EQ(k.items[1].end_byte, (uint32_t)strlen(src));+ ASSERT_EQ(k.items[1].end_point.row, 4u);+ ASSERT_EQ(k.items[1].end_point.column, 0u);+ cbm_sql_kept_ranges_free(&k);+ /* The table after the cut keeps its real line. */+ CBMFileResult *r = do_extract(src, CBM_LANG_SQL, "pos.sql");+ ASSERT_NOT_NULL(r);+ bool found = false;+ for (int i = 0; i < r->defs.count; i++) {+ if (r->defs.items[i].name && strcmp(r->defs.items[i].name, "u") == 0) {+ ASSERT_EQ(r->defs.items[i].start_line, 4u);+ found = true;+ }+ }+ ASSERT_TRUE(found);+ cbm_free_result(r);+ PASS();+}+SUITE(parse_coverage) {RUN_TEST(c_ifdef_split_brace_sets_parse_incomplete);RUN_TEST(c_ifdef_split_brace_neighbors_still_extracted);@@ -1023,4 +1465,10 @@ SUITE(parse_coverage) {RUN_TEST(coverage_range_never_ends_past_the_last_line_issue963);RUN_TEST(coverage_range_never_covers_an_extracted_definition);RUN_TEST(coverage_gap_of_only_comments_is_not_a_miss);+ RUN_TEST(sql_values_scanner_excludes_only_literal_rows_issue1735);+ RUN_TEST(sql_values_scanner_keeps_positions_of_kept_text_issue1735);+ RUN_TEST(sql_dump_literal_rows_leave_the_graph_unchanged_issue1735);+ RUN_TEST(sql_dump_tuple_with_subquery_or_call_is_still_parsed_issue1735);+ RUN_TEST(sql_dump_parse_does_not_grow_with_the_row_count_issue1735);+ RUN_TEST(sql_dump_of_many_megabytes_is_indexed_not_timed_out_issue1735);}diff --git a/tests/test_pipeline.c b/tests/test_pipeline.cindex 6ef16a89b..6618a4a5d 100644--- a/tests/test_pipeline.c+++ b/tests/test_pipeline.c@@ -344,6 +344,165 @@ TEST(pipeline_doclinks_edge_lands_in_store) {PASS();}+/* #1735: every node and edge of a stored index, one sorted line each. */+static void graph_listing_append(sqlite3 *db, const char *sql, const char *project, char **buf,+ size_t *len) {+ sqlite3_stmt *stmt = NULL;+ if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {+ return;+ }+ sqlite3_bind_text(stmt, 1, project, -1, SQLITE_TRANSIENT);+ while (sqlite3_step(stmt) == SQLITE_ROW) {+ const char *row = (const char *)sqlite3_column_text(stmt, 0);+ size_t n = row ? strlen(row) : 0;+ char *grown = realloc(*buf, *len + n + 2);+ if (!grown) {+ break;+ }+ *buf = grown;+ memcpy(*buf + *len, row ? row : "", n);+ (*buf)[*len + n] = '\n';+ *len += n + 1;+ (*buf)[*len] = '\0';+ }+ sqlite3_finalize(stmt);+}++static char *graph_listing(const char *db_path, const char *project) {+ sqlite3 *db = NULL;+ if (sqlite3_open_v2(db_path, &db, SQLITE_OPEN_READONLY, NULL) != SQLITE_OK) {+ sqlite3_close(db);+ return NULL;+ }+ char *buf = calloc(1, 1);+ size_t len = 0;+ graph_listing_append(db,+ "SELECT label||'|'||name||'|'||qualified_name||'|'||file_path||'|'||"+ "start_line||'|'||end_line||'|'||properties FROM nodes "+ "WHERE project=?1 ORDER BY 1",+ project, &buf, &len);+ graph_listing_append(db,+ "SELECT e.type||'|'||s.qualified_name||'|'||t.qualified_name||'|'||"+ "e.properties FROM edges e JOIN nodes s ON s.id=e.source_id "+ "JOIN nodes t ON t.id=e.target_id WHERE e.project=?1 ORDER BY 1",+ project, &buf, &len);+ sqlite3_close(db);+ return buf;+}++static char *index_and_list(const char *repo, const char *db_name) {+ char db_path[512];+ snprintf(db_path, sizeof(db_path), "%s/%s", repo, db_name);+ cbm_pipeline_t *p = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL);+ if (!p) {+ return NULL;+ }+ char *listing =+ cbm_pipeline_run(p) == 0 ? graph_listing(db_path, cbm_pipeline_project_name(p)) : NULL;+ cbm_pipeline_free(p);+ return listing;+}++/* True if every line of `sub` is a line of `super`. */+static bool listing_lines_within(const char *sub, const char *super) {+ for (const char *line = sub; *line;) {+ const char *nl = strchr(line, '\n');+ size_t n = nl ? (size_t)(nl - line) : strlen(line);+ bool found = false;+ for (const char *p = super; *p && !found;) {+ const char *pnl = strchr(p, '\n');+ size_t pn = pnl ? (size_t)(pnl - p) : strlen(p);+ found = pn == n && memcmp(p, line, n) == 0;+ p = pnl ? pnl + 1 : p + pn;+ }+ if (!found) {+ fprintf(stderr, " missing from the cut index: %.*s\n", (int)n, line);+ return false;+ }+ line = nl ? nl + 1 : line + n;+ }+ return true;+}++/* A small dump: two tables, a view, three INSERTs of 200 rows, each with one+ * subquery row and one function-call row among literal rows. */+static char *sql_dump_small(bool mysql_esc) {+ size_t cap = 256 * 1024;+ char *src = malloc(cap);+ if (!src) {+ return NULL;+ }+ const char *quoted = mysql_esc ? "(%d,'O\\'Brien',DEFAULT)" : "(%d,'O''Brien',DEFAULT)";+ size_t len = (size_t)snprintf(src, cap,+ "CREATE TABLE city (ID int, Name char(35), Population int);\n"+ "CREATE TABLE country (Code char(3), Name char(52));\n"+ "CREATE VIEW big_cities AS SELECT Name FROM city "+ "WHERE Population > 1000000;\n");+ for (int s = 0; s < 3; s++) {+ len += (size_t)snprintf(src + len, cap - len, "INSERT INTO `city` VALUES ");+ for (int r = 0; r < 200; r++) {+ if (r) {+ src[len++] = ',';+ }+ int id = s * 1000 + r;+ if (r == 100) {+ len += (size_t)snprintf(src + len, cap - len,+ "((SELECT MAX(Code) FROM country),'x',1)");+ } else if (r == 101) {+ len += (size_t)snprintf(src + len, cap - len, "(%d,UPPER('y'),2)", id);+ } else if (r % 3) {+ len += (size_t)snprintf(src + len, cap - len, quoted, id);+ } else {+ len += (size_t)snprintf(src + len, cap - len, "(%d,_binary 'ab',NULL)", id);+ }+ }+ len += (size_t)snprintf(src + len, cap - len, ";\n");+ }+ return src;+}++/* #1735: leaving a dump's literal INSERT rows out of the parse must not change+ * the graph. The same repository is indexed twice — once as shipped and once+ * with the full parse (test seam) — and the stored nodes and edges compared.+ * Standard '' escapes, which the SQL grammar reads correctly: identical.+ * MySQL \' escapes, which it misreads: the full parse's error recovery swallows+ * neighbouring rows, so the cut index must hold everything the full one has+ * (and may hold more — the rows the full parse lost). */+TEST(pipeline_sql_dump_graph_matches_the_full_parse_issue1735) {+ for (int esc = 0; esc < 2; esc++) {+ char tmp[256];+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_sql_dump_XXXXXX");+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));+ char *src = sql_dump_small(esc == 1);+ ASSERT_NOT_NULL(src);+ write_temp_file(tmp, "dump.sql", src);+ free(src);++ char *cut = index_and_list(tmp, "cut.db");+ setenv("CBM_TEST_SQL_FULL_PARSE_ON", "dump.sql", 1);+ char *full = index_and_list(tmp, "full.db");+ unsetenv("CBM_TEST_SQL_FULL_PARSE_ON");+ th_rmtree(tmp);+ ASSERT_NOT_NULL(cut);+ ASSERT_NOT_NULL(full);+ /* Not vacuous: the tables, the view and its lineage are there. */+ ASSERT_NOT_NULL(strstr(cut, "Table|city|"));+ ASSERT_NOT_NULL(strstr(cut, "View|big_cities|"));+ ASSERT_NOT_NULL(strstr(cut, "USAGE|"));+ if (esc == 0 && strcmp(cut, full) != 0) {+ fprintf(stderr, "--- cut ---\n%s--- full ---\n%s", cut, full);+ }+ if (esc == 0) {+ ASSERT_STR_EQ(cut, full);+ } else {+ ASSERT_TRUE(listing_lines_within(full, cut));+ }+ free(cut);+ free(full);+ }+ PASS();+}+/* Spilling must be invisible in the OUTPUT: the same repository indexed with* results parked on disk must produce the same graph as one indexed entirely in* memory. It did not. The namespace map that `use`/`using`/package imports@@ -15110,6 +15269,7 @@ SUITE(pipeline) {/* Integration: structure pass */RUN_TEST(pipeline_grpc_routes_cover_every_service_past_the_old_cap);RUN_TEST(pipeline_doclinks_edge_lands_in_store);+ RUN_TEST(pipeline_sql_dump_graph_matches_the_full_parse_issue1735);RUN_TEST(pipeline_spill_resolves_namespace_imports_like_memory);RUN_TEST(pipeline_structure_nodes);RUN_TEST(pipeline_committed_counts_match_persisted);