Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
doc-links(cs): bounds and checks from security review 2 (S11 to S16, S18)
Martin Vogel8271c21a651e
diff --git a/internal/cbm/doclink_cs.c b/internal/cbm/doclink_cs.cindex 354debc29..2e744069a 100644--- a/internal/cbm/doclink_cs.c+++ b/internal/cbm/doclink_cs.c@@ -727,8 +727,8 @@ typedef struct {int end1; /* ... and where its first branch ended; valid once has_end1 */int n_end1;int first_open; /* first open-brace entry allocated in the current branch */- bool has_end1; /* an #else was seen */- int mark; /* its entry in the brace list */+ bool has_end1; /* an #else was seen */+ int mark; /* its entry in the brace list */} cs_pp_t;enum {@@ -2933,7 +2933,9 @@ char *cbm_doclink_cs_portable_scope(const char *scope) {while (fend < len && p[fend] != '\t') {fend++;}- if (cs_line_field(tag, field)) {+ /* a line number becomes 0; an empty field stays empty, so the+ * copy is never longer than the scope it is made from */+ if (cs_line_field(tag, field) && fend > i) {out[w++] = '0';} else {memcpy(out + w, p + i, fend - i);diff --git a/src/mcp/mcp.c b/src/mcp/mcp.cindex 9584d52d2..3669e89eb 100644--- a/src/mcp/mcp.c+++ b/src/mcp/mcp.c@@ -64,7 +64,8 @@ enum {#include "cypher/cypher.h"#include "discover/discover.h"#include "pipeline/pipeline.h"-#include "callable_sig.h" /* cbm_qn_callable_base_len */+#include "callable_sig.h" /* cbm_qn_callable_base_len */+#include "pipeline/doc_links.h" /* cbm_doclink_reason_name: the reasons the layer writes */#include "pipeline/pass_cross_repo.h"#include "git/git_context.h"#include "cli/cli.h"@@ -6437,6 +6438,18 @@ static bool doc_link_preview_samples(yyjson_mut_doc *doc, const cbm_doc_link_prereturn true;}+/* True for a reason the doc-link layer writes. The table is read back from+ * the database: any other text in it was not written by this layer, and+ * never becomes a key of the report. */+static bool doc_link_reason_known(const char *reason) {+ for (int r = 0; r < CBM_DOCLINK_REASON_COUNT; r++) {+ if (strcmp(reason, cbm_doclink_reason_name(r)) == 0) {+ return true;+ }+ }+ return false;+}+static bool add_doc_links_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store,const char *project, bool with_samples) {int mentions = cbm_store_count_edges_by_type(store, project, "MENTIONS");@@ -6463,6 +6476,7 @@ static bool add_doc_links_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_}bool error = rc != CBM_STORE_OK || mentions < 0 || !present || sample_failed;bool built = false;+ int64_t unrecognized = 0;yyjson_mut_val *dl = yyjson_mut_obj(doc);yyjson_mut_val *unresolved = yyjson_mut_obj(doc);if (!dl || !unresolved) {@@ -6473,12 +6487,22 @@ static bool add_doc_links_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_error = true;continue;}+ if (!doc_link_reason_known(reasons[i].reason)) {+ /* rows this layer did not write: counted under one fixed key */+ unrecognized += reasons[i].count;+ error = true;+ continue;+ }yyjson_mut_val *key = yyjson_mut_strcpy(doc, reasons[i].reason);yyjson_mut_val *number = yyjson_mut_int(doc, reasons[i].count);if (!key || !number || !yyjson_mut_obj_add(unresolved, key, number)) {goto cleanup;}}+ if (unrecognized > 0 &&+ !yyjson_mut_obj_add_int(doc, unresolved, "unrecognized_reason", unrecognized)) {+ goto cleanup;+ }if (!yyjson_mut_obj_add_int(doc, dl, "mentions", mentions < 0 ? 0 : mentions) ||!yyjson_mut_obj_add_val(doc, dl, "unresolved", unresolved) ||!yyjson_mut_obj_add_str(doc, dl, "status", error ? "error" : "ok")) {diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.hindex f44241678..40291d86a 100644--- a/src/pipeline/doc_links.h+++ b/src/pipeline/doc_links.h@@ -216,6 +216,9 @@ extern const cbm_doclink_resolver_t cbm_doclink_cs_resolver;* without a clock. Test builds only. */void cbm_doclink_cs_test_work_reset(void);uint64_t cbm_doclink_cs_test_work(void);+/* The buckets the index build walked to empty its per-file scratch table+ * (reset with the work above). */+uint64_t cbm_doclink_cs_test_scratch_work(void);#endif/* True when `rel_path` is a scope input of some language (scope_input). */diff --git a/src/pipeline/doc_links_cs.c b/src/pipeline/doc_links_cs.cindex 16216d9ce..c957841e7 100644--- a/src/pipeline/doc_links_cs.c+++ b/src/pipeline/doc_links_cs.c@@ -200,6 +200,7 @@ enum {* and every namespace above a declared one. */typedef struct {int parent;+ int depth; /* segments of its full name (the global namespace: 0) */const char *name; /* its own segment */bool declared; /* a file declares it; else it is only above a declared one */bool prod; /* product code declares it, or a namespace under it */@@ -525,6 +526,8 @@ typedef struct {static _Atomic uint64_t cs_work_counter;/* New import index construction is measured separately from lookup work. */static _Atomic uint64_t cs_index_work_counter;+/* Buckets of the node pass's scratch table walked when it is emptied. */+static _Atomic uint64_t cs_scratch_work_counter;static _Atomic bool cs_fail_candidate_alloc;static _Atomic bool cs_candidate_alloc_failed;@@ -540,6 +543,7 @@ bool cbm_doclink_cs_test_candidate_alloc_failed(void) {void cbm_doclink_cs_test_work_reset(void) {atomic_store(&cs_work_counter, 0);atomic_store(&cs_index_work_counter, 0);+ atomic_store(&cs_scratch_work_counter, 0);}uint64_t cbm_doclink_cs_test_work(void) {@@ -550,6 +554,10 @@ uint64_t cbm_doclink_cs_test_index_work(void) {return atomic_load(&cs_index_work_counter);}+uint64_t cbm_doclink_cs_test_scratch_work(void) {+ return atomic_load(&cs_scratch_work_counter);+}+static void cs_work(uint64_t n) {atomic_fetch_add_explicit(&cs_work_counter, n, memory_order_relaxed);}@@ -557,6 +565,10 @@ static void cs_work(uint64_t n) {static void cs_index_work(uint64_t n) {atomic_fetch_add_explicit(&cs_index_work_counter, n, memory_order_relaxed);}++static void cs_scratch_work(uint64_t n) {+ atomic_fetch_add_explicit(&cs_scratch_work_counter, n, memory_order_relaxed);+}#elsestatic void cs_work(uint64_t n) {(void)n;@@ -565,6 +577,10 @@ static void cs_work(uint64_t n) {static void cs_index_work(uint64_t n) {(void)n;}++static void cs_scratch_work(uint64_t n) {+ (void)n;+}#endif/* ── Small helpers ───────────────────────────────────────────────── */@@ -750,16 +766,22 @@ static int ns_make(cs_index_t *ix, int parent, const char *seg, size_t len) {return CS_NONE;}int id = ix->nnss++;- ix->nss[id] = (cs_ns_t){.parent = parent, .name = name};+ ix->nss[id] =+ (cs_ns_t){.parent = parent, .depth = ix->nss[parent].depth + SKIP_ONE, .name = name};cbm_ht_set(ix->ns_by_key, k, (void *)(intptr_t)(id + SKIP_ONE));return id;}/* The namespace the dotted `path` names under `from`, every segment created- * on the way; CS_NONE for a bad name or when memory ran out. */+ * on the way; CS_NONE for a bad name, when memory ran out, and past the+ * scanner's nesting limit: a scope read back from the store is held to the+ * limit its writer keeps (every enclosing namespace is a lookup step). */static int ns_make_path(cs_index_t *ix, int from, const char *path) {int ns = from;for (const char *p = path; ns >= 0 && *p;) {+ if (ix->nss[ns].depth >= CS_MAX_NEST) {+ return CS_NONE;+ }const char *dot = strchr(p, '.');size_t n = dot ? (size_t)(dot - p) : strlen(p);ns = ns_make(ix, ns, p, n);@@ -2029,14 +2051,46 @@ static const cbm_gbuf_node_t *member_node_at(const cbm_gbuf_t *g, const cs_membereturn (m->kind == 'c' ? label_is_callable(n->label) : label_is_value(n->label)) ? n : NULL;}-/* Scratch tables of the node pass: cleared for every file. */+/* Scratch tables of the node pass: emptied for every file. */typedef struct {CBMHashTable *names; /* "<gid>\x1f<name>" -> index + 1 */+ uint32_t sized; /* what the table is sized for: the most keys it held, or+ * its initial capacity -- what emptying it walks */CBMArena keys;int *last; /* per gid: the last type that has it */int cap_last;} cs_node_pass_t;+enum { CS_SCRATCH_MIN = 64, CS_SCRATCH_SLACK = 4 };++/* Empty the scratch table for a file that puts about `need` keys into it.+ * Emptying a table walks every bucket it has, and a table never shrinks: one+ * that a far larger file grew is made anew, so that one large file does not+ * make every later file pay for its size. false when memory ran out. */+static bool node_pass_reset(cs_index_t *ix, cs_node_pass_t *np, int need) {+ uint32_t want = need > CS_SCRATCH_MIN ? (uint32_t)need : CS_SCRATCH_MIN;+ if (np->sized > want * CS_SCRATCH_SLACK) {+ cbm_ht_free(np->names);+ np->names = cbm_ht_create(want);+ np->sized = want;+ if (!np->names) {+ ix->oom = true;+ return false;+ }+ } else {+ cs_scratch_work(np->sized);+ cbm_ht_clear(np->names);+ }+ cbm_arena_rewind(&np->keys); /* the emptied table held the only pointers into it */+ return true;+}++/* Note how many keys the scratch table holds now. */+static void node_pass_held(cs_node_pass_t *np) {+ uint32_t held = cbm_ht_count(np->names);+ np->sized = held > np->sized ? held : np->sized;+}+/* The path group of every type of the file (types with one path -- `Foo` and* `Foo<T>`, and what is nested in them under one name -- share the node* <module>.<path>), and which declaration owns that node: the last one. */@@ -2050,8 +2104,9 @@ static bool assign_gids(cs_index_t *ix, cs_file_t *f, cs_node_pass_t *np) {return false;}}- cbm_ht_clear(np->names);- cbm_arena_rewind(&np->keys); /* the cleared table held the only pointers into it */+ if (!node_pass_reset(ix, np, f->ntypes)) {+ return false;+ }int gids = 0;for (int t = 0; t < f->ntypes; t++) {cs_type_t *ty = &f->types[t];@@ -2098,8 +2153,10 @@ static bool bind_nodes(cs_index_t *ix, cs_file_t *f, const cbm_gbuf_t *g, cs_nod}}/* which member is the last of its (path, name) */- cbm_ht_clear(np->names);- cbm_arena_rewind(&np->keys);+ node_pass_held(np);+ if (!node_pass_reset(ix, np, f->nmembers)) {+ return false;+ }for (int m = 0; m < f->nmembers; m++) {char key[(CS_NAME_BUF) + CBM_SZ_16];snprintf(key, sizeof(key), "%d\x1f%s", f->types[f->members[m].type].gid,@@ -2114,6 +2171,7 @@ static bool bind_nodes(cs_index_t *ix, cs_file_t *f, const cbm_gbuf_t *g, cs_nod}cbm_ht_set(np->names, k, (void *)(intptr_t)(m + SKIP_ONE));}+ node_pass_held(np);int cached = CS_NONE;bool fits = false;for (int m = 0; m < f->nmembers; m++) {@@ -3274,6 +3332,9 @@ static bool parse_seg(const char *s, size_t n, cs_seg_t *out) {if (s[i] == '{' || s[i] == '<') {name_end = i;size_t e = group_end(s, n, i);+ if (e < n) {+ return false; /* text after the type arguments: no name */+ }size_t inner = e > i + PAIR_LEN ? e - i - PAIR_LEN : 0;out->arity = count_top(s + i + SKIP_ONE, inner);out->targs = s + i + SKIP_ONE;@@ -3295,6 +3356,8 @@ static bool parse_seg(const char *s, size_t n, cs_seg_t *out) {}/* Split a dotted path (dots inside type-argument groups do not split). */+/* A path whose brackets do not pair is not read at all: its last segment+ * would be dropped and the reference resolved by the segments before it. */static bool parse_path(const char *s, size_t n, cs_seg_t *segs, int *nsegs) {*nsegs = 0;size_t start = 0;@@ -3304,7 +3367,9 @@ static bool parse_path(const char *s, size_t n, cs_seg_t *segs, int *nsegs) {if (is_open_bracket(c)) {depth++;} else if (is_close_bracket(c)) {- depth--;+ if (--depth < 0) {+ return false;+ }} else if (c == '.' && depth == 0) {if (*nsegs >= CS_MAX_SEGS || !parse_seg(s + start, i - start, &segs[*nsegs])) {return false;@@ -3313,7 +3378,7 @@ static bool parse_path(const char *s, size_t n, cs_seg_t *segs, int *nsegs) {start = i + SKIP_ONE;}}- return *nsegs > 0;+ return depth == 0 && *nsegs > 0;}static const char *const CS_KEYWORD_TYPES[][2] = {@@ -3513,7 +3578,9 @@ static bool parse_params(cs_ref_t *r, const char *s, size_t from, size_t to) {if (is_open_bracket(c)) {depth++;} else if (is_close_bracket(c)) {- depth--;+ if (--depth < 0) {+ return false; /* a bracket that closes nothing */+ }} else if (c == ',' && depth == 0) {if (r->nparams >= CS_MAX_PARAMS) {return false;@@ -3533,7 +3600,9 @@ static bool parse_params(cs_ref_t *r, const char *s, size_t from, size_t to) {}}r->sig[w] = '\0';- return true;+ /* a bracket left open would have swallowed the last parameter: the+ * reference would be matched without it */+ return depth == 0;}/* A written reference into its parts. false for one this code does not@@ -3611,7 +3680,12 @@ static bool parse_cref(const char *raw, cs_ref_t *r) {r->has_params = true;size_t close = group_end(s, n, paren);bool closed = close > paren && s[close - SKIP_ONE] == ')';- if (!parse_params(r, s, paren + SKIP_ONE, closed ? close - SKIP_ONE : n)) {+ /* a parameter list that is not closed, or text after it, is not+ * matched by what can be read of it */+ for (size_t k = close; closed && k < n; k++) {+ closed = isspace((unsigned char)s[k]) != 0;+ }+ if (!closed || !parse_params(r, s, paren + SKIP_ONE, close - SKIP_ONE)) {return false;}}@@ -5664,7 +5738,7 @@ static const char *build_files(cs_index_t *ix, const cbm_doclink_build_in_t *in)/* The graph node of every declaration. false when memory ran out. */static bool build_nodes(cs_index_t *ix, const cbm_gbuf_t *g) {- cs_node_pass_t np = {.names = cbm_ht_create(CBM_SZ_256)};+ cs_node_pass_t np = {.names = cbm_ht_create(CBM_SZ_256), .sized = CBM_SZ_256};cbm_arena_init(&np.keys);bool ok = np.names != NULL;for (int i = 0; ok && i < ix->nfiles; i++) {diff --git a/src/pipeline/doc_links_msbuild.c b/src/pipeline/doc_links_msbuild.cindex b665a9f47..ec3375709 100644--- a/src/pipeline/doc_links_msbuild.c+++ b/src/pipeline/doc_links_msbuild.c@@ -299,9 +299,12 @@ bool cbm_msb_add(cbm_msb_t *m, const char *rel_path, const char *scope) {}line = nl ? nl + SKIP_ONE : NULL;}- if (bad_group || import_group) {+ if (bad_group || import_group || !f->readable) {/* A malformed group is unknown, never an empty closed scope. Keep- * this project conservative without rejecting other project files. */+ * this project conservative without rejecting other project files.+ * So is a file the scan did not read (malformed, or larger than a+ * project file): what it holds is unknown, not absent -- the+ * projects that evaluate it have an open scope. */f->readable = false;f->recs[0] = (msb_rec_t){.tag = 'Y'};f->nrecs = 1;@@ -2495,8 +2498,8 @@ static bool st_enter(msb_eval_t *ev, int file, bool poison) {}ev->frames[ev->nframes - 1].owner = c;st_write(ev, (uint32_t)file + 1, domain, NULL);- if (!poison)- ev->unevaluable += !ev->m->files[file].readable;+ /* a file that was not read is counted (and opens the scope) by its 'Y'+ * record, as the walk reaches it */return !ev->oom;}static void st_finish(msb_eval_t *ev, st_component_t *c) {@@ -2628,8 +2631,8 @@ static void msb_import(msb_eval_t *ev, int file, const msb_rec_t *r) {if (target >= 0 && !seen_has(ev, ev->m->files[target].rel_path) && msb_push(ev, target)) {msb_work(SKIP_ONE);cbm_ht_set(ev->seen, ev->m->files[target].rel_path, (void *)&MSB_SET_PRESENT);- /* a file that was not read (malformed, or too large) says nothing */- ev->unevaluable += !ev->m->files[target].readable;+ /* a file that was not read (malformed, or too large) is counted and+ * opens the scope by its 'Y' record (cbm_msb_add) */}}@@ -2660,7 +2663,6 @@ static void msb_pass1(msb_eval_t *ev, int file) {return;msb_work(SKIP_ONE);cbm_ht_set(ev->seen, rel, (void *)&MSB_SET_PRESENT);- ev->unevaluable += !ev->m->files[file].readable;}while (ev->nframes > base && !ev->oom) {/* an import pushes a frame, which may move the array: the frame isdiff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.cindex 40c2cd902..03ccf415b 100644--- a/src/pipeline/pipeline_incremental.c+++ b/src/pipeline/pipeline_incremental.c@@ -1592,6 +1592,30 @@ static void row_dep_visitor(const char *key, void *value, void *userdata) {/* Files whose unresolved doc-link rows mention a removed name as an* identifier token. Keys are borrowed from `rows`. */+/* A byte of an identifier as the resolvers read one: a letter, a digit, '_',+ * and every byte of a multi-byte character (doc_links_cs.c ident_ok). */+static bool doc_row_ident_byte(unsigned char c) {+ return isalnum(c) || c == '_' || c >= CBM_SZ_128;+}++/* True when the identifier raw[0, n) is one of the removed names -- of any+ * length. When memory runs out the answer is yes: re-resolving a file that+ * did not need it costs time, keeping a stale row costs correctness. */+static bool doc_row_token_removed(const CBMHashTable *removed, const char *s, size_t n) {+ char small[CBM_SZ_512];+ char *tok = n < sizeof(small) ? small : (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, n + SKIP_ONE);+ if (!tok) {+ return true;+ }+ memcpy(tok, s, n);+ tok[n] = '\0';+ bool hit = cbm_ht_get(removed, tok) != NULL;+ if (tok != small) {+ cbm_free(CBM_MEM_CLASS_OTHER, tok);+ }+ return hit;+}+static void doc_row_name_dependents(const cbm_doc_link_row_t *rows, int n,const CBMHashTable *removed, CBMHashTable *out_paths) {if (!removed || cbm_ht_count(removed) == 0) {@@ -1604,22 +1628,17 @@ static void doc_row_name_dependents(const cbm_doc_link_row_t *rows, int n,continue;}for (const char *p = raw; *p;) {- while (*p && !(isalnum((unsigned char)*p) || *p == '_')) {+ while (*p && !doc_row_ident_byte((unsigned char)*p)) {p++;}const char *s = p;- while (*p && (isalnum((unsigned char)*p) || *p == '_')) {+ while (*p && doc_row_ident_byte((unsigned char)*p)) {p++;}- char tok[CBM_SZ_512];size_t tl = (size_t)(p - s);- if (tl > 0 && tl < sizeof(tok)) {- memcpy(tok, s, tl);- tok[tl] = '\0';- if (cbm_ht_get(removed, tok)) {- cbm_ht_set(out_paths, rel, (void *)rel);- break;- }+ if (tl > 0 && doc_row_token_removed(removed, s, tl)) {+ cbm_ht_set(out_paths, rel, (void *)rel);+ break;}}}diff --git a/tests/test_doc_mentions.c b/tests/test_doc_mentions.cindex 25ddd9783..e8f8272d0 100644--- a/tests/test_doc_mentions.c+++ b/tests/test_doc_mentions.c@@ -4584,7 +4584,9 @@ static int dm_preview_bounds_checks(const char *db, const char *project) {yyjson_val *sample = yyjson_arr_get(samples, 0);yyjson_val *metadata = yyjson_obj_get(report, "samples_preview");const char *note = yyjson_get_str(yyjson_obj_get(report, "samples_preview_note"));- bounded = dm_preview_status_is(env, "ok") && dm_preview_json_consistent(env) &&+ /* the row's reason is no reason this layer writes: the+ * status is error (S12); its sample is still shown bounded */+ bounded = dm_preview_status_is(env, "error") && dm_preview_json_consistent(env) &&yyjson_arr_size(samples) == 1 && yyjson_obj_size(sample) == 5 &&yyjson_arr_size(metadata) == 4 && note &&strstr(note, "doc_link_unresolved") && bounded;@@ -4606,7 +4608,7 @@ static int dm_preview_bounds_checks(const char *db, const char *project) {const char *metadata = table ? strstr(table, "\n samples_preview:") : NULL;size_t bytes = table ? (metadata ? (size_t)(metadata - table) : strlen(table)) : 0;bounded = table && metadata && bytes <= 5000 &&- dm_preview_compact_status(env, "ok") &&+ dm_preview_compact_status(env, "error") &&strstr(metadata, "\n samples_preview_note:") &&strstr(table, "(cols: rel_path line syntax raw reason)") && bounded;for (int field = 0; field < 4; field++) {@@ -6236,7 +6238,8 @@ TEST(doc_mentions_msbuild_gate) {ASSERT_EQ(steps, strlen("<Settings>"));/* a file that was not read is counted where it is evaluated, as the- * project file itself or as an import; it opens nobody's scope */+ * project file itself or as an import; what it holds is unknown, so the+ * scope of the project that evaluates it is open (S16) */const dm_project_file_t files[] = {{"eng/Huge.props", big},{"eng/Broken.props", "<Project><PropertyGroup>"},@@ -6250,7 +6253,7 @@ TEST(doc_mentions_msbuild_gate) {ASSERT_TRUE(dm_msb_eval(files, 3, "App.csproj", &res));ASSERT_TRUE(dm_has_using(&res, 'n', "Still.Here"));ASSERT_EQ(res.unevaluable, 2);- ASSERT_FALSE(res.open);+ ASSERT_TRUE(res.open);cbm_msb_result_free(&res);ASSERT_TRUE(dm_msb_eval(files, 2, "eng/Broken.props", &res));ASSERT_EQ(res.count, 0);@@ -9102,6 +9105,331 @@ TEST(doc_mentions_cs_damaged_stored_scope) {PASS();}+/* ── security review 2 ───────────────────────────────────────────── */++/* The portable copy of a scope writes 0 for every line number and is never+ * longer than the scope it is made from: an empty line field stays empty+ * (writing a 0 for it made the copy one byte longer per such field, past the+ * end of its buffer). */+TEST(doc_mentions_cs_portable_scope_bound) {+ static const char blob[] = "cs1\n"+ "R\t1\t0\t\t\tN\n"+ "T\t1\t\t\tc\t-\tC\t\t\n"+ "M\t\tc\t0\t0\tGo\t\t\n"+ "X\t\t\n"+ "X\t\t\n"+ "X\t\t\n";+ char *portable = cbm_doclink_cs_portable_scope(blob);+ ASSERT_NOT_NULL(portable);+ size_t n = strlen(portable);+ bool same = strcmp(portable, blob) == 0;+ cbm_free(CBM_MEM_CLASS_OTHER, portable);+ ASSERT_LTE(n, strlen(blob));+ ASSERT_TRUE(same);+ PASS();+}++/* A reference whose brackets do not pair, or that goes on after its+ * parameter list, is unparseable: it is never resolved by the part that+ * can be read (the segments before the open bracket, the parameters before+ * it). Decoys: the same references written whole bind. */+TEST(doc_mentions_cs_unbalanced_brackets) {+ static const dm_source_t files[] = {+ {"src/Outer.cs",+ "namespace Acme\n"+ "{\n"+ " public class Outer\n"+ " {\n"+ " public class Inner<T> { }\n"+ " public void M(int a) { }\n"+ " public void M(int a, System.Collections.Generic.List<string> b) { }\n"+ " }\n"+ "\n"+ " /// <summary><see cref=\"Outer.Inner{T\"/> <see cref=\"Outer.Inner}\"/>\n"+ " /// <see cref=\"Outer.M(int\"/> <see cref=\"Outer.M(int, List{string)\"/>\n"+ " /// <see cref=\"Outer.M(int))\"/> <see cref=\"Outer.M(int)x\"/>\n"+ " /// <see cref=\"Outer.Inner{T}x\"/></summary>\n"+ " public class Broken { }\n"+ "\n"+ " /// <summary><see cref=\"Outer.Inner{T}\"/> <see cref=\"Outer.M(int)\"/></summary>\n"+ " public class Whole { }\n"+ "}\n"},+ };+ static const dm_want_t wants[] = {+ {"src/Outer.cs", "Outer.Inner{T", "Outer.Broken", NULL, "unparseable", "Outer.Outer", NULL},+ {"src/Outer.cs", "Outer.Inner}", "Outer.Broken", NULL, "unparseable", "Outer.Outer", NULL},+ {"src/Outer.cs", "Outer.M(int", "Outer.Broken", NULL, "unparseable", "Outer.Outer.M", NULL},+ {"src/Outer.cs", "Outer.M(int, List{string)", "Outer.Broken", NULL, "unparseable",+ "Outer.Outer.M", NULL},+ {"src/Outer.cs", "Outer.M(int))", "Outer.Broken", NULL, "unparseable", "Outer.Outer.M",+ NULL},+ {"src/Outer.cs", "Outer.M(int)x", "Outer.Broken", NULL, "unparseable", "Outer.Outer.M",+ NULL},+ {"src/Outer.cs", "Outer.Inner{T}x", "Outer.Broken", NULL, "unparseable",+ "Outer.Outer.Inner", NULL},+ {"src/Outer.cs", "Outer.Inner{T}", "Outer.Whole", "Outer.Outer.Inner", NULL, NULL, NULL},+ {"src/Outer.cs", "Outer.M(int)", "Outer.Whole", "Outer.Outer.M", NULL, NULL, NULL},+ };+ ASSERT_EQ(dm_check_repo("brackets", files, DM_COUNT(files), wants, DM_COUNT(wants)), 0);+ PASS();+}++enum { DM_DEEP_SEGMENTS = 70 }; /* past the scanner's 64 namespace segments */++/* A scope read back from the store is held to the nesting its writer keeps:+ * a region whose namespace has more segments than the scanner ever writes+ * fails the build of the run (bad_scope, the error row), as any other stored+ * scope this code did not write. The next run rebuilds everything. */+TEST(doc_mentions_cs_stored_scope_nesting) {+ char tmp[256];+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_dm_deep_XXXXXX");+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));+ char repo[400];+ snprintf(repo, sizeof(repo), "%s/repo", tmp);+ th_write_file(TH_PATH(repo, "src/A.cs"), "namespace N\n{\n public class Target { }\n}\n");+ th_write_file(TH_PATH(repo, "src/B.cs"), "namespace N\n"+ "{\n"+ " /// <summary><see cref=\"Target\"/></summary>\n"+ " public class Uses { }\n"+ "}\n");+ char db[512];+ snprintf(db, sizeof(db), "%s/deep.db", tmp);+ ASSERT_EQ(dm_index(repo, db, NULL), 0);+ char props[512];+ int n = 0;+ dm_edge(db, "B.Uses", "A.Target", props, sizeof(props), &n);+ ASSERT_EQ(n, 1);+ /* A's region record names a namespace DM_DEEP_SEGMENTS segments deep */+ char sql[1024];+ size_t w = (size_t)snprintf(sql, sizeof(sql),+ "UPDATE lsp_surface SET defs_json = replace(defs_json, "+ "'\\tN\\n', '\\tN");+ for (int i = 1; i < DM_DEEP_SEGMENTS; i++) {+ w += (size_t)snprintf(sql + w, sizeof(sql) - w, ".a");+ }+ snprintf(sql + w, sizeof(sql) - w,+ "\\n') WHERE rel_path = 'src/A.cs' AND instr(defs_json, '\\tN\\n') > 0");+ ASSERT_EQ(dm_exec(db, sql), 1);+ th_write_file(TH_PATH(repo, "src/B.cs"),+ "namespace N\n"+ "{\n"+ " /// <summary>Again <see cref=\"Target\"/></summary>\n"+ " public class Uses { }\n"+ "}\n");+ cbm_pipeline_incremental_test_reset_faults();+ ASSERT_EQ(dm_index(repo, db, NULL), 0);+ int errors = dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE reason = 'error'");+ /* the next run rebuilds everything, and the edge is back */+ ASSERT_EQ(dm_index(repo, db, NULL), 0);+ cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route();+ dm_edge(db, "B.Uses", "A.Target", props, sizeof(props), &n);+ dm_unlink_db(db);+ th_rmtree(tmp);+ ASSERT_EQ(errors, 1);+ ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_FORCED_FULL);+ ASSERT_EQ(n, 1);+ PASS();+}++/* One row whose reason this layer never writes, then index_status in both+ * forms: no key carries that text, the row is counted under one fixed key,+ * and the status is error. 0 when all of that holds. */+static int dm_unknown_reason_checks(const char *db, const char *project) {+ if (dm_exec(db, "INSERT INTO doc_link_unresolved(project, rel_path, line, syntax, raw, reason) "+ "SELECT project, 'src/Injected.cs', 1, 'see', 'X', 'injected_reason_text' "+ "FROM doc_link_unresolved LIMIT 1") != 1) {+ return 1;+ }+ int bad = 0;+ char *text = dm_index_status(project, false);+ const char *block = text ? strstr(text, "doc_links:\n") : NULL;+ if (!block || strstr(text, "injected_reason_text") ||+ !strstr(block, "\n unrecognized_reason: 1\n") || !strstr(block, "\n status: error\n")) {+ printf(" index_status text: %s\n", block ? block : "(no doc_links block)");+ bad = 1;+ }+ free(text);+ yyjson_doc *env = dm_preview_status(project, true);+ yyjson_val *unresolved = yyjson_obj_get(dm_preview_report(env), "unresolved");+ if (!unresolved || yyjson_obj_get(unresolved, "injected_reason_text") ||+ yyjson_get_int(yyjson_obj_get(unresolved, "unrecognized_reason")) != 1 ||+ !dm_preview_status_is(env, "error")) {+ printf(" index_status json: unknown reason shown, or status not error\n");+ bad = 1;+ }+ yyjson_doc_free(env);+ return bad;+}++TEST(doc_mentions_index_status_unknown_reason) {+ ASSERT_EQ(dm_preview_fixture(dm_unknown_reason_checks), 0);+ PASS();+}++enum { DM_HUGE_PROJECT = 1100000 }; /* past CSX_MAX_PROJECT_BYTES */++/* A project file the scan does not read -- larger than a project file is,+ * as the project itself or as a file it imports -- holds what nobody knows,+ * not nothing: a simple name found nowhere in such a project is external.+ * Decoy: a project whose files were read keeps a closed scope (missing). */+TEST(doc_mentions_msbuild_unread_project_opens) {+ char *filler = malloc(DM_HUGE_PROJECT + 1);+ ASSERT_NOT_NULL(filler);+ memset(filler, 'x', DM_HUGE_PROJECT);+ filler[DM_HUGE_PROJECT] = '\0';+ char *huge_project =+ dm_repeated("<Project Sdk=\"Microsoft.NET.Sdk\"><!-- ", filler, 1, " --></Project>\n");+ char *huge_props = dm_repeated("<Project><!-- ", filler, 1, " --></Project>\n");+ free(filler);+ ASSERT_NOT_NULL(huge_project);+ ASSERT_NOT_NULL(huge_props);+ const dm_source_t files[] = {+ {"a/A.csproj", huge_project},+ {"a/UsesA.cs", "namespace App\n"+ "{\n"+ " /// <summary><see cref=\"NowhereA\"/></summary>\n"+ " public class UsesA { }\n"+ "}\n"},+ {"b/Directory.Build.props", huge_props},+ {"b/B.csproj", DM_EMPTY_PROJECT},+ {"b/UsesB.cs", "namespace App\n"+ "{\n"+ " /// <summary><see cref=\"NowhereB\"/></summary>\n"+ " public class UsesB { }\n"+ "}\n"},+ {"c/C.csproj", DM_EMPTY_PROJECT},+ {"c/UsesC.cs", "namespace App\n"+ "{\n"+ " /// <summary><see cref=\"NowhereC\"/></summary>\n"+ " public class UsesC { }\n"+ "}\n"},+ };+ static const dm_want_t wants[] = {+ {"a/UsesA.cs", "NowhereA", "UsesA.UsesA", NULL, "external", NULL, NULL},+ {"b/UsesB.cs", "NowhereB", "UsesB.UsesB", NULL, "external", NULL, NULL},+ {"c/UsesC.cs", "NowhereC", "UsesC.UsesC", NULL, "missing", NULL, NULL},+ };+ int bad = dm_check_repo("unread", files, DM_COUNT(files), wants, DM_COUNT(wants));+ free(huge_project);+ free(huge_props);+ ASSERT_EQ(bad, 0);+ PASS();+}++enum { DM_BIG_MEMBERS = 20000, DM_SMALL_FILES = 200 };++/* What the resolver looks at to build its index over one file of+ * DM_BIG_MEMBERS fields followed (in path order) by DM_SMALL_FILES files of+ * one type and one field each. 0 when the repository cannot be indexed. */+static uint64_t dm_scratch_work(void) {+ char tmp[256];+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_dm_scratch_XXXXXX");+ if (!cbm_mkdtemp(tmp)) {+ return 0;+ }+ size_t cap = (size_t)DM_BIG_MEMBERS * 32 + 64;+ char *src = malloc(cap);+ if (!src) {+ th_rmtree(tmp);+ return 0;+ }+ size_t w = (size_t)snprintf(src, cap, "namespace Big\n{\n public class Wide\n {\n");+ for (int i = 0; i < DM_BIG_MEMBERS; i++) {+ w += (size_t)snprintf(src + w, cap - w, " public int f%d;\n", i);+ }+ snprintf(src + w, cap - w, " }\n}\n");+ th_write_file(TH_PATH(tmp, "a/Big.cs"), src);+ free(src);+ for (int i = 0; i < DM_SMALL_FILES; i++) {+ char rel[64];+ char text[256];+ snprintf(rel, sizeof(rel), "b/S%03d.cs", i);+ snprintf(text, sizeof(text),+ "namespace Small\n{\n%s public class C%03d { public int x; }\n}\n",+ i == 0 ? " /// <summary><see cref=\"Nowhere\"/></summary>\n" : "", i);+ th_write_file(TH_PATH(tmp, rel), text);+ }+ char db[512];+ snprintf(db, sizeof(db), "%s/scratch.db", tmp);+ cbm_doclink_cs_test_work_reset();+ uint64_t work = dm_index(tmp, db, NULL) == 0 ? cbm_doclink_cs_test_scratch_work() : 0;+ int rows = dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved");+ dm_unlink_db(db);+ th_rmtree(tmp);+ return rows == 1 ? work : 0;+}++/* The per-file scratch table of the index build is not emptied at the size+ * one large file grew it to: after a file of DM_BIG_MEMBERS names, every one+ * of DM_SMALL_FILES small files costs about its own size. The work stays+ * within a small multiple of all the names the files declare (emptying the+ * large table for every small file cost DM_SMALL_FILES times its size). */+TEST(doc_mentions_cs_scratch_table_work) {+ uint64_t names = (uint64_t)DM_BIG_MEMBERS + (uint64_t)DM_SMALL_FILES * 2;+ uint64_t work = dm_scratch_work();+ printf(" scratch table: %llu steps for %llu names\n", (unsigned long long)work,+ (unsigned long long)names);+ ASSERT_TRUE(work > 0);+ ASSERT_LTE(work, names * 4);+ PASS();+}++/* An incremental repair finds the files whose unresolved rows name what a+ * changed file no longer declares by the identifiers the resolver reads --+ * a name with a non-ASCII letter included. The parts of `Hub` in two shared+ * trees both declare the member `Grüße` (the reference is ambiguous); one of+ * them stops declaring it, and the repaired index equals a full one (the+ * reference binds). */+TEST(doc_mentions_incremental_non_ascii_name) {+ char tmp[256];+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_dm_utf8name_XXXXXX");+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));+ char repo[400];+ char db[512];+ char full_db[512];+ snprintf(repo, sizeof(repo), "%s/repo", tmp);+ snprintf(db, sizeof(db), "%s/inc.db", tmp);+ snprintf(full_db, sizeof(full_db), "%s/full.db", tmp);+ th_write_file(TH_PATH(repo, "t1/X.cs"),+ "namespace N\n{\n public partial class Hub { public void Gr\xC3\xBC\xC3\x9F"+ "e() { } }\n}\n");+ th_write_file(TH_PATH(repo, "t2/Y.cs"),+ "namespace N\n{\n public partial class Hub { public void Gr\xC3\xBC\xC3\x9F"+ "e() { } public void Keep() { } }\n}\n");+ th_write_file(TH_PATH(repo, "p/P.csproj"), DM_EMPTY_PROJECT);+ th_write_file(TH_PATH(repo, "p/Uses.cs"),+ "namespace App\n"+ "{\n"+ " /// <summary><see cref=\"N.Hub.Gr\xC3\xBC\xC3\x9F"+ "e\"/></summary>\n"+ " public class Uses { }\n"+ "}\n");+ ASSERT_EQ(dm_index(repo, db, NULL), 0);+ char reason[64];+ dm_row(db, "p/Uses.cs",+ "N.Hub.Gr\xC3\xBC\xC3\x9F"+ "e",+ reason, sizeof(reason), NULL, 0);+ bool ambiguous = strcmp(reason, "ambiguous") == 0;+ th_write_file(TH_PATH(repo, "t2/Y.cs"),+ "namespace N\n{\n public partial class Hub { public void Keep() { } }\n}\n");+ int step = dm_step(repo, db, full_db, "a name with a non-ASCII letter removed",+ CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);+ char props[512];+ int n = 0;+ dm_edge(db, "Uses.Uses",+ "X.Hub.Gr\xC3\xBC\xC3\x9F"+ "e",+ props, sizeof(props), &n);+ dm_unlink_db(db);+ dm_unlink_db(full_db);+ th_rmtree(tmp);+ ASSERT_TRUE(ambiguous);+ ASSERT_EQ(step, 0);+ ASSERT_EQ(n, 1);+ PASS();+}+SUITE(doc_mentions) {RUN_TEST(doc_mentions_extract_cs_tokens);RUN_TEST(doc_mentions_cs_scope_blob);@@ -9203,4 +9531,11 @@ SUITE(doc_mentions) {RUN_TEST(doc_mentions_msbuild_many_project_files);RUN_TEST(doc_mentions_cs_scan_nesting_limits);RUN_TEST(doc_mentions_cs_damaged_stored_scope);+ RUN_TEST(doc_mentions_cs_portable_scope_bound);+ RUN_TEST(doc_mentions_cs_unbalanced_brackets);+ RUN_TEST(doc_mentions_cs_stored_scope_nesting);+ RUN_TEST(doc_mentions_index_status_unknown_reason);+ RUN_TEST(doc_mentions_msbuild_unread_project_opens);+ RUN_TEST(doc_mentions_cs_scratch_table_work);+ RUN_TEST(doc_mentions_incremental_non_ascii_name);}