Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

doc-links(cs): bounds and checks from security review 2 (S11 to S16, S18)

Martin Vogel8271c21a651e
diff --git a/internal/cbm/doclink_cs.c b/internal/cbm/doclink_cs.c
index 354debc29..2e744069a 100644
--- a/internal/cbm/doclink_cs.c
+++ b/internal/cbm/doclink_cs.c
@@ -727,8 +727,8 @@ typedef struct {
int end1; /* ... and where its first branch ended; valid once has_end1 */
int n_end1;
int first_open; /* first open-brace entry allocated in the current branch */
- bool has_end1; /* an #else was seen */
- int mark; /* its entry in the brace list */
+ bool has_end1; /* an #else was seen */
+ int mark; /* its entry in the brace list */
} cs_pp_t;
enum {
@@ -2933,7 +2933,9 @@ char *cbm_doclink_cs_portable_scope(const char *scope) {
while (fend < len && p[fend] != '\t') {
fend++;
}
- if (cs_line_field(tag, field)) {
+ /* a line number becomes 0; an empty field stays empty, so the
+ * copy is never longer than the scope it is made from */
+ if (cs_line_field(tag, field) && fend > i) {
out[w++] = '0';
} else {
memcpy(out + w, p + i, fend - i);
diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c
index 9584d52d2..3669e89eb 100644
--- a/src/mcp/mcp.c
+++ b/src/mcp/mcp.c
@@ -64,7 +64,8 @@ enum {
#include "cypher/cypher.h"
#include "discover/discover.h"
#include "pipeline/pipeline.h"
-#include "callable_sig.h" /* cbm_qn_callable_base_len */
+#include "callable_sig.h" /* cbm_qn_callable_base_len */
+#include "pipeline/doc_links.h" /* cbm_doclink_reason_name: the reasons the layer writes */
#include "pipeline/pass_cross_repo.h"
#include "git/git_context.h"
#include "cli/cli.h"
@@ -6437,6 +6438,18 @@ static bool doc_link_preview_samples(yyjson_mut_doc *doc, const cbm_doc_link_pre
return true;
}
+/* True for a reason the doc-link layer writes. The table is read back from
+ * the database: any other text in it was not written by this layer, and
+ * never becomes a key of the report. */
+static bool doc_link_reason_known(const char *reason) {
+ for (int r = 0; r < CBM_DOCLINK_REASON_COUNT; r++) {
+ if (strcmp(reason, cbm_doclink_reason_name(r)) == 0) {
+ return true;
+ }
+ }
+ return false;
+}
+
static bool add_doc_links_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store,
const char *project, bool with_samples) {
int mentions = cbm_store_count_edges_by_type(store, project, "MENTIONS");
@@ -6463,6 +6476,7 @@ static bool add_doc_links_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_
}
bool error = rc != CBM_STORE_OK || mentions < 0 || !present || sample_failed;
bool built = false;
+ int64_t unrecognized = 0;
yyjson_mut_val *dl = yyjson_mut_obj(doc);
yyjson_mut_val *unresolved = yyjson_mut_obj(doc);
if (!dl || !unresolved) {
@@ -6473,12 +6487,22 @@ static bool add_doc_links_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_
error = true;
continue;
}
+ if (!doc_link_reason_known(reasons[i].reason)) {
+ /* rows this layer did not write: counted under one fixed key */
+ unrecognized += reasons[i].count;
+ error = true;
+ continue;
+ }
yyjson_mut_val *key = yyjson_mut_strcpy(doc, reasons[i].reason);
yyjson_mut_val *number = yyjson_mut_int(doc, reasons[i].count);
if (!key || !number || !yyjson_mut_obj_add(unresolved, key, number)) {
goto cleanup;
}
}
+ if (unrecognized > 0 &&
+ !yyjson_mut_obj_add_int(doc, unresolved, "unrecognized_reason", unrecognized)) {
+ goto cleanup;
+ }
if (!yyjson_mut_obj_add_int(doc, dl, "mentions", mentions < 0 ? 0 : mentions) ||
!yyjson_mut_obj_add_val(doc, dl, "unresolved", unresolved) ||
!yyjson_mut_obj_add_str(doc, dl, "status", error ? "error" : "ok")) {
diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h
index f44241678..40291d86a 100644
--- a/src/pipeline/doc_links.h
+++ b/src/pipeline/doc_links.h
@@ -216,6 +216,9 @@ extern const cbm_doclink_resolver_t cbm_doclink_cs_resolver;
* without a clock. Test builds only. */
void cbm_doclink_cs_test_work_reset(void);
uint64_t cbm_doclink_cs_test_work(void);
+/* The buckets the index build walked to empty its per-file scratch table
+ * (reset with the work above). */
+uint64_t cbm_doclink_cs_test_scratch_work(void);
#endif
/* True when `rel_path` is a scope input of some language (scope_input). */
diff --git a/src/pipeline/doc_links_cs.c b/src/pipeline/doc_links_cs.c
index 16216d9ce..c957841e7 100644
--- a/src/pipeline/doc_links_cs.c
+++ b/src/pipeline/doc_links_cs.c
@@ -200,6 +200,7 @@ enum {
* and every namespace above a declared one. */
typedef struct {
int parent;
+ int depth; /* segments of its full name (the global namespace: 0) */
const char *name; /* its own segment */
bool declared; /* a file declares it; else it is only above a declared one */
bool prod; /* product code declares it, or a namespace under it */
@@ -525,6 +526,8 @@ typedef struct {
static _Atomic uint64_t cs_work_counter;
/* New import index construction is measured separately from lookup work. */
static _Atomic uint64_t cs_index_work_counter;
+/* Buckets of the node pass's scratch table walked when it is emptied. */
+static _Atomic uint64_t cs_scratch_work_counter;
static _Atomic bool cs_fail_candidate_alloc;
static _Atomic bool cs_candidate_alloc_failed;
@@ -540,6 +543,7 @@ bool cbm_doclink_cs_test_candidate_alloc_failed(void) {
void cbm_doclink_cs_test_work_reset(void) {
atomic_store(&cs_work_counter, 0);
atomic_store(&cs_index_work_counter, 0);
+ atomic_store(&cs_scratch_work_counter, 0);
}
uint64_t cbm_doclink_cs_test_work(void) {
@@ -550,6 +554,10 @@ uint64_t cbm_doclink_cs_test_index_work(void) {
return atomic_load(&cs_index_work_counter);
}
+uint64_t cbm_doclink_cs_test_scratch_work(void) {
+ return atomic_load(&cs_scratch_work_counter);
+}
+
static void cs_work(uint64_t n) {
atomic_fetch_add_explicit(&cs_work_counter, n, memory_order_relaxed);
}
@@ -557,6 +565,10 @@ static void cs_work(uint64_t n) {
static void cs_index_work(uint64_t n) {
atomic_fetch_add_explicit(&cs_index_work_counter, n, memory_order_relaxed);
}
+
+static void cs_scratch_work(uint64_t n) {
+ atomic_fetch_add_explicit(&cs_scratch_work_counter, n, memory_order_relaxed);
+}
#else
static void cs_work(uint64_t n) {
(void)n;
@@ -565,6 +577,10 @@ static void cs_work(uint64_t n) {
static void cs_index_work(uint64_t n) {
(void)n;
}
+
+static void cs_scratch_work(uint64_t n) {
+ (void)n;
+}
#endif
/* ── Small helpers ───────────────────────────────────────────────── */
@@ -750,16 +766,22 @@ static int ns_make(cs_index_t *ix, int parent, const char *seg, size_t len) {
return CS_NONE;
}
int id = ix->nnss++;
- ix->nss[id] = (cs_ns_t){.parent = parent, .name = name};
+ ix->nss[id] =
+ (cs_ns_t){.parent = parent, .depth = ix->nss[parent].depth + SKIP_ONE, .name = name};
cbm_ht_set(ix->ns_by_key, k, (void *)(intptr_t)(id + SKIP_ONE));
return id;
}
/* The namespace the dotted `path` names under `from`, every segment created
- * on the way; CS_NONE for a bad name or when memory ran out. */
+ * on the way; CS_NONE for a bad name, when memory ran out, and past the
+ * scanner's nesting limit: a scope read back from the store is held to the
+ * limit its writer keeps (every enclosing namespace is a lookup step). */
static int ns_make_path(cs_index_t *ix, int from, const char *path) {
int ns = from;
for (const char *p = path; ns >= 0 && *p;) {
+ if (ix->nss[ns].depth >= CS_MAX_NEST) {
+ return CS_NONE;
+ }
const char *dot = strchr(p, '.');
size_t n = dot ? (size_t)(dot - p) : strlen(p);
ns = ns_make(ix, ns, p, n);
@@ -2029,14 +2051,46 @@ static const cbm_gbuf_node_t *member_node_at(const cbm_gbuf_t *g, const cs_membe
return (m->kind == 'c' ? label_is_callable(n->label) : label_is_value(n->label)) ? n : NULL;
}
-/* Scratch tables of the node pass: cleared for every file. */
+/* Scratch tables of the node pass: emptied for every file. */
typedef struct {
CBMHashTable *names; /* "<gid>\x1f<name>" -> index + 1 */
+ uint32_t sized; /* what the table is sized for: the most keys it held, or
+ * its initial capacity -- what emptying it walks */
CBMArena keys;
int *last; /* per gid: the last type that has it */
int cap_last;
} cs_node_pass_t;
+enum { CS_SCRATCH_MIN = 64, CS_SCRATCH_SLACK = 4 };
+
+/* Empty the scratch table for a file that puts about `need` keys into it.
+ * Emptying a table walks every bucket it has, and a table never shrinks: one
+ * that a far larger file grew is made anew, so that one large file does not
+ * make every later file pay for its size. false when memory ran out. */
+static bool node_pass_reset(cs_index_t *ix, cs_node_pass_t *np, int need) {
+ uint32_t want = need > CS_SCRATCH_MIN ? (uint32_t)need : CS_SCRATCH_MIN;
+ if (np->sized > want * CS_SCRATCH_SLACK) {
+ cbm_ht_free(np->names);
+ np->names = cbm_ht_create(want);
+ np->sized = want;
+ if (!np->names) {
+ ix->oom = true;
+ return false;
+ }
+ } else {
+ cs_scratch_work(np->sized);
+ cbm_ht_clear(np->names);
+ }
+ cbm_arena_rewind(&np->keys); /* the emptied table held the only pointers into it */
+ return true;
+}
+
+/* Note how many keys the scratch table holds now. */
+static void node_pass_held(cs_node_pass_t *np) {
+ uint32_t held = cbm_ht_count(np->names);
+ np->sized = held > np->sized ? held : np->sized;
+}
+
/* The path group of every type of the file (types with one path -- `Foo` and
* `Foo<T>`, and what is nested in them under one name -- share the node
* <module>.<path>), and which declaration owns that node: the last one. */
@@ -2050,8 +2104,9 @@ static bool assign_gids(cs_index_t *ix, cs_file_t *f, cs_node_pass_t *np) {
return false;
}
}
- cbm_ht_clear(np->names);
- cbm_arena_rewind(&np->keys); /* the cleared table held the only pointers into it */
+ if (!node_pass_reset(ix, np, f->ntypes)) {
+ return false;
+ }
int gids = 0;
for (int t = 0; t < f->ntypes; t++) {
cs_type_t *ty = &f->types[t];
@@ -2098,8 +2153,10 @@ static bool bind_nodes(cs_index_t *ix, cs_file_t *f, const cbm_gbuf_t *g, cs_nod
}
}
/* which member is the last of its (path, name) */
- cbm_ht_clear(np->names);
- cbm_arena_rewind(&np->keys);
+ node_pass_held(np);
+ if (!node_pass_reset(ix, np, f->nmembers)) {
+ return false;
+ }
for (int m = 0; m < f->nmembers; m++) {
char key[(CS_NAME_BUF) + CBM_SZ_16];
snprintf(key, sizeof(key), "%d\x1f%s", f->types[f->members[m].type].gid,
@@ -2114,6 +2171,7 @@ static bool bind_nodes(cs_index_t *ix, cs_file_t *f, const cbm_gbuf_t *g, cs_nod
}
cbm_ht_set(np->names, k, (void *)(intptr_t)(m + SKIP_ONE));
}
+ node_pass_held(np);
int cached = CS_NONE;
bool fits = false;
for (int m = 0; m < f->nmembers; m++) {
@@ -3274,6 +3332,9 @@ static bool parse_seg(const char *s, size_t n, cs_seg_t *out) {
if (s[i] == '{' || s[i] == '<') {
name_end = i;
size_t e = group_end(s, n, i);
+ if (e < n) {
+ return false; /* text after the type arguments: no name */
+ }
size_t inner = e > i + PAIR_LEN ? e - i - PAIR_LEN : 0;
out->arity = count_top(s + i + SKIP_ONE, inner);
out->targs = s + i + SKIP_ONE;
@@ -3295,6 +3356,8 @@ static bool parse_seg(const char *s, size_t n, cs_seg_t *out) {
}
/* Split a dotted path (dots inside type-argument groups do not split). */
+/* A path whose brackets do not pair is not read at all: its last segment
+ * would be dropped and the reference resolved by the segments before it. */
static bool parse_path(const char *s, size_t n, cs_seg_t *segs, int *nsegs) {
*nsegs = 0;
size_t start = 0;
@@ -3304,7 +3367,9 @@ static bool parse_path(const char *s, size_t n, cs_seg_t *segs, int *nsegs) {
if (is_open_bracket(c)) {
depth++;
} else if (is_close_bracket(c)) {
- depth--;
+ if (--depth < 0) {
+ return false;
+ }
} else if (c == '.' && depth == 0) {
if (*nsegs >= CS_MAX_SEGS || !parse_seg(s + start, i - start, &segs[*nsegs])) {
return false;
@@ -3313,7 +3378,7 @@ static bool parse_path(const char *s, size_t n, cs_seg_t *segs, int *nsegs) {
start = i + SKIP_ONE;
}
}
- return *nsegs > 0;
+ return depth == 0 && *nsegs > 0;
}
static const char *const CS_KEYWORD_TYPES[][2] = {
@@ -3513,7 +3578,9 @@ static bool parse_params(cs_ref_t *r, const char *s, size_t from, size_t to) {
if (is_open_bracket(c)) {
depth++;
} else if (is_close_bracket(c)) {
- depth--;
+ if (--depth < 0) {
+ return false; /* a bracket that closes nothing */
+ }
} else if (c == ',' && depth == 0) {
if (r->nparams >= CS_MAX_PARAMS) {
return false;
@@ -3533,7 +3600,9 @@ static bool parse_params(cs_ref_t *r, const char *s, size_t from, size_t to) {
}
}
r->sig[w] = '\0';
- return true;
+ /* a bracket left open would have swallowed the last parameter: the
+ * reference would be matched without it */
+ return depth == 0;
}
/* A written reference into its parts. false for one this code does not
@@ -3611,7 +3680,12 @@ static bool parse_cref(const char *raw, cs_ref_t *r) {
r->has_params = true;
size_t close = group_end(s, n, paren);
bool closed = close > paren && s[close - SKIP_ONE] == ')';
- if (!parse_params(r, s, paren + SKIP_ONE, closed ? close - SKIP_ONE : n)) {
+ /* a parameter list that is not closed, or text after it, is not
+ * matched by what can be read of it */
+ for (size_t k = close; closed && k < n; k++) {
+ closed = isspace((unsigned char)s[k]) != 0;
+ }
+ if (!closed || !parse_params(r, s, paren + SKIP_ONE, close - SKIP_ONE)) {
return false;
}
}
@@ -5664,7 +5738,7 @@ static const char *build_files(cs_index_t *ix, const cbm_doclink_build_in_t *in)
/* The graph node of every declaration. false when memory ran out. */
static bool build_nodes(cs_index_t *ix, const cbm_gbuf_t *g) {
- cs_node_pass_t np = {.names = cbm_ht_create(CBM_SZ_256)};
+ cs_node_pass_t np = {.names = cbm_ht_create(CBM_SZ_256), .sized = CBM_SZ_256};
cbm_arena_init(&np.keys);
bool ok = np.names != NULL;
for (int i = 0; ok && i < ix->nfiles; i++) {
diff --git a/src/pipeline/doc_links_msbuild.c b/src/pipeline/doc_links_msbuild.c
index b665a9f47..ec3375709 100644
--- a/src/pipeline/doc_links_msbuild.c
+++ b/src/pipeline/doc_links_msbuild.c
@@ -299,9 +299,12 @@ bool cbm_msb_add(cbm_msb_t *m, const char *rel_path, const char *scope) {
}
line = nl ? nl + SKIP_ONE : NULL;
}
- if (bad_group || import_group) {
+ if (bad_group || import_group || !f->readable) {
/* A malformed group is unknown, never an empty closed scope. Keep
- * this project conservative without rejecting other project files. */
+ * this project conservative without rejecting other project files.
+ * So is a file the scan did not read (malformed, or larger than a
+ * project file): what it holds is unknown, not absent -- the
+ * projects that evaluate it have an open scope. */
f->readable = false;
f->recs[0] = (msb_rec_t){.tag = 'Y'};
f->nrecs = 1;
@@ -2495,8 +2498,8 @@ static bool st_enter(msb_eval_t *ev, int file, bool poison) {
}
ev->frames[ev->nframes - 1].owner = c;
st_write(ev, (uint32_t)file + 1, domain, NULL);
- if (!poison)
- ev->unevaluable += !ev->m->files[file].readable;
+ /* a file that was not read is counted (and opens the scope) by its 'Y'
+ * record, as the walk reaches it */
return !ev->oom;
}
static void st_finish(msb_eval_t *ev, st_component_t *c) {
@@ -2628,8 +2631,8 @@ static void msb_import(msb_eval_t *ev, int file, const msb_rec_t *r) {
if (target >= 0 && !seen_has(ev, ev->m->files[target].rel_path) && msb_push(ev, target)) {
msb_work(SKIP_ONE);
cbm_ht_set(ev->seen, ev->m->files[target].rel_path, (void *)&MSB_SET_PRESENT);
- /* a file that was not read (malformed, or too large) says nothing */
- ev->unevaluable += !ev->m->files[target].readable;
+ /* a file that was not read (malformed, or too large) is counted and
+ * opens the scope by its 'Y' record (cbm_msb_add) */
}
}
@@ -2660,7 +2663,6 @@ static void msb_pass1(msb_eval_t *ev, int file) {
return;
msb_work(SKIP_ONE);
cbm_ht_set(ev->seen, rel, (void *)&MSB_SET_PRESENT);
- ev->unevaluable += !ev->m->files[file].readable;
}
while (ev->nframes > base && !ev->oom) {
/* an import pushes a frame, which may move the array: the frame is
diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c
index 40c2cd902..03ccf415b 100644
--- a/src/pipeline/pipeline_incremental.c
+++ b/src/pipeline/pipeline_incremental.c
@@ -1592,6 +1592,30 @@ static void row_dep_visitor(const char *key, void *value, void *userdata) {
/* Files whose unresolved doc-link rows mention a removed name as an
* identifier token. Keys are borrowed from `rows`. */
+/* A byte of an identifier as the resolvers read one: a letter, a digit, '_',
+ * and every byte of a multi-byte character (doc_links_cs.c ident_ok). */
+static bool doc_row_ident_byte(unsigned char c) {
+ return isalnum(c) || c == '_' || c >= CBM_SZ_128;
+}
+
+/* True when the identifier raw[0, n) is one of the removed names -- of any
+ * length. When memory runs out the answer is yes: re-resolving a file that
+ * did not need it costs time, keeping a stale row costs correctness. */
+static bool doc_row_token_removed(const CBMHashTable *removed, const char *s, size_t n) {
+ char small[CBM_SZ_512];
+ char *tok = n < sizeof(small) ? small : (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, n + SKIP_ONE);
+ if (!tok) {
+ return true;
+ }
+ memcpy(tok, s, n);
+ tok[n] = '\0';
+ bool hit = cbm_ht_get(removed, tok) != NULL;
+ if (tok != small) {
+ cbm_free(CBM_MEM_CLASS_OTHER, tok);
+ }
+ return hit;
+}
+
static void doc_row_name_dependents(const cbm_doc_link_row_t *rows, int n,
const CBMHashTable *removed, CBMHashTable *out_paths) {
if (!removed || cbm_ht_count(removed) == 0) {
@@ -1604,22 +1628,17 @@ static void doc_row_name_dependents(const cbm_doc_link_row_t *rows, int n,
continue;
}
for (const char *p = raw; *p;) {
- while (*p && !(isalnum((unsigned char)*p) || *p == '_')) {
+ while (*p && !doc_row_ident_byte((unsigned char)*p)) {
p++;
}
const char *s = p;
- while (*p && (isalnum((unsigned char)*p) || *p == '_')) {
+ while (*p && doc_row_ident_byte((unsigned char)*p)) {
p++;
}
- char tok[CBM_SZ_512];
size_t tl = (size_t)(p - s);
- if (tl > 0 && tl < sizeof(tok)) {
- memcpy(tok, s, tl);
- tok[tl] = '\0';
- if (cbm_ht_get(removed, tok)) {
- cbm_ht_set(out_paths, rel, (void *)rel);
- break;
- }
+ if (tl > 0 && doc_row_token_removed(removed, s, tl)) {
+ cbm_ht_set(out_paths, rel, (void *)rel);
+ break;
}
}
}
diff --git a/tests/test_doc_mentions.c b/tests/test_doc_mentions.c
index 25ddd9783..e8f8272d0 100644
--- a/tests/test_doc_mentions.c
+++ b/tests/test_doc_mentions.c
@@ -4584,7 +4584,9 @@ static int dm_preview_bounds_checks(const char *db, const char *project) {
yyjson_val *sample = yyjson_arr_get(samples, 0);
yyjson_val *metadata = yyjson_obj_get(report, "samples_preview");
const char *note = yyjson_get_str(yyjson_obj_get(report, "samples_preview_note"));
- bounded = dm_preview_status_is(env, "ok") && dm_preview_json_consistent(env) &&
+ /* the row's reason is no reason this layer writes: the
+ * status is error (S12); its sample is still shown bounded */
+ bounded = dm_preview_status_is(env, "error") && dm_preview_json_consistent(env) &&
yyjson_arr_size(samples) == 1 && yyjson_obj_size(sample) == 5 &&
yyjson_arr_size(metadata) == 4 && note &&
strstr(note, "doc_link_unresolved") && bounded;
@@ -4606,7 +4608,7 @@ static int dm_preview_bounds_checks(const char *db, const char *project) {
const char *metadata = table ? strstr(table, "\n samples_preview:") : NULL;
size_t bytes = table ? (metadata ? (size_t)(metadata - table) : strlen(table)) : 0;
bounded = table && metadata && bytes <= 5000 &&
- dm_preview_compact_status(env, "ok") &&
+ dm_preview_compact_status(env, "error") &&
strstr(metadata, "\n samples_preview_note:") &&
strstr(table, "(cols: rel_path line syntax raw reason)") && bounded;
for (int field = 0; field < 4; field++) {
@@ -6236,7 +6238,8 @@ TEST(doc_mentions_msbuild_gate) {
ASSERT_EQ(steps, strlen("<Settings>"));
/* a file that was not read is counted where it is evaluated, as the
- * project file itself or as an import; it opens nobody's scope */
+ * project file itself or as an import; what it holds is unknown, so the
+ * scope of the project that evaluates it is open (S16) */
const dm_project_file_t files[] = {
{"eng/Huge.props", big},
{"eng/Broken.props", "<Project><PropertyGroup>"},
@@ -6250,7 +6253,7 @@ TEST(doc_mentions_msbuild_gate) {
ASSERT_TRUE(dm_msb_eval(files, 3, "App.csproj", &res));
ASSERT_TRUE(dm_has_using(&res, 'n', "Still.Here"));
ASSERT_EQ(res.unevaluable, 2);
- ASSERT_FALSE(res.open);
+ ASSERT_TRUE(res.open);
cbm_msb_result_free(&res);
ASSERT_TRUE(dm_msb_eval(files, 2, "eng/Broken.props", &res));
ASSERT_EQ(res.count, 0);
@@ -9102,6 +9105,331 @@ TEST(doc_mentions_cs_damaged_stored_scope) {
PASS();
}
+/* ── security review 2 ───────────────────────────────────────────── */
+
+/* The portable copy of a scope writes 0 for every line number and is never
+ * longer than the scope it is made from: an empty line field stays empty
+ * (writing a 0 for it made the copy one byte longer per such field, past the
+ * end of its buffer). */
+TEST(doc_mentions_cs_portable_scope_bound) {
+ static const char blob[] = "cs1\n"
+ "R\t1\t0\t\t\tN\n"
+ "T\t1\t\t\tc\t-\tC\t\t\n"
+ "M\t\tc\t0\t0\tGo\t\t\n"
+ "X\t\t\n"
+ "X\t\t\n"
+ "X\t\t\n";
+ char *portable = cbm_doclink_cs_portable_scope(blob);
+ ASSERT_NOT_NULL(portable);
+ size_t n = strlen(portable);
+ bool same = strcmp(portable, blob) == 0;
+ cbm_free(CBM_MEM_CLASS_OTHER, portable);
+ ASSERT_LTE(n, strlen(blob));
+ ASSERT_TRUE(same);
+ PASS();
+}
+
+/* A reference whose brackets do not pair, or that goes on after its
+ * parameter list, is unparseable: it is never resolved by the part that
+ * can be read (the segments before the open bracket, the parameters before
+ * it). Decoys: the same references written whole bind. */
+TEST(doc_mentions_cs_unbalanced_brackets) {
+ static const dm_source_t files[] = {
+ {"src/Outer.cs",
+ "namespace Acme\n"
+ "{\n"
+ " public class Outer\n"
+ " {\n"
+ " public class Inner<T> { }\n"
+ " public void M(int a) { }\n"
+ " public void M(int a, System.Collections.Generic.List<string> b) { }\n"
+ " }\n"
+ "\n"
+ " /// <summary><see cref=\"Outer.Inner{T\"/> <see cref=\"Outer.Inner}\"/>\n"
+ " /// <see cref=\"Outer.M(int\"/> <see cref=\"Outer.M(int, List{string)\"/>\n"
+ " /// <see cref=\"Outer.M(int))\"/> <see cref=\"Outer.M(int)x\"/>\n"
+ " /// <see cref=\"Outer.Inner{T}x\"/></summary>\n"
+ " public class Broken { }\n"
+ "\n"
+ " /// <summary><see cref=\"Outer.Inner{T}\"/> <see cref=\"Outer.M(int)\"/></summary>\n"
+ " public class Whole { }\n"
+ "}\n"},
+ };
+ static const dm_want_t wants[] = {
+ {"src/Outer.cs", "Outer.Inner{T", "Outer.Broken", NULL, "unparseable", "Outer.Outer", NULL},
+ {"src/Outer.cs", "Outer.Inner}", "Outer.Broken", NULL, "unparseable", "Outer.Outer", NULL},
+ {"src/Outer.cs", "Outer.M(int", "Outer.Broken", NULL, "unparseable", "Outer.Outer.M", NULL},
+ {"src/Outer.cs", "Outer.M(int, List{string)", "Outer.Broken", NULL, "unparseable",
+ "Outer.Outer.M", NULL},
+ {"src/Outer.cs", "Outer.M(int))", "Outer.Broken", NULL, "unparseable", "Outer.Outer.M",
+ NULL},
+ {"src/Outer.cs", "Outer.M(int)x", "Outer.Broken", NULL, "unparseable", "Outer.Outer.M",
+ NULL},
+ {"src/Outer.cs", "Outer.Inner{T}x", "Outer.Broken", NULL, "unparseable",
+ "Outer.Outer.Inner", NULL},
+ {"src/Outer.cs", "Outer.Inner{T}", "Outer.Whole", "Outer.Outer.Inner", NULL, NULL, NULL},
+ {"src/Outer.cs", "Outer.M(int)", "Outer.Whole", "Outer.Outer.M", NULL, NULL, NULL},
+ };
+ ASSERT_EQ(dm_check_repo("brackets", files, DM_COUNT(files), wants, DM_COUNT(wants)), 0);
+ PASS();
+}
+
+enum { DM_DEEP_SEGMENTS = 70 }; /* past the scanner's 64 namespace segments */
+
+/* A scope read back from the store is held to the nesting its writer keeps:
+ * a region whose namespace has more segments than the scanner ever writes
+ * fails the build of the run (bad_scope, the error row), as any other stored
+ * scope this code did not write. The next run rebuilds everything. */
+TEST(doc_mentions_cs_stored_scope_nesting) {
+ char tmp[256];
+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_dm_deep_XXXXXX");
+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
+ char repo[400];
+ snprintf(repo, sizeof(repo), "%s/repo", tmp);
+ th_write_file(TH_PATH(repo, "src/A.cs"), "namespace N\n{\n public class Target { }\n}\n");
+ th_write_file(TH_PATH(repo, "src/B.cs"), "namespace N\n"
+ "{\n"
+ " /// <summary><see cref=\"Target\"/></summary>\n"
+ " public class Uses { }\n"
+ "}\n");
+ char db[512];
+ snprintf(db, sizeof(db), "%s/deep.db", tmp);
+ ASSERT_EQ(dm_index(repo, db, NULL), 0);
+ char props[512];
+ int n = 0;
+ dm_edge(db, "B.Uses", "A.Target", props, sizeof(props), &n);
+ ASSERT_EQ(n, 1);
+ /* A's region record names a namespace DM_DEEP_SEGMENTS segments deep */
+ char sql[1024];
+ size_t w = (size_t)snprintf(sql, sizeof(sql),
+ "UPDATE lsp_surface SET defs_json = replace(defs_json, "
+ "'\\tN\\n', '\\tN");
+ for (int i = 1; i < DM_DEEP_SEGMENTS; i++) {
+ w += (size_t)snprintf(sql + w, sizeof(sql) - w, ".a");
+ }
+ snprintf(sql + w, sizeof(sql) - w,
+ "\\n') WHERE rel_path = 'src/A.cs' AND instr(defs_json, '\\tN\\n') > 0");
+ ASSERT_EQ(dm_exec(db, sql), 1);
+ th_write_file(TH_PATH(repo, "src/B.cs"),
+ "namespace N\n"
+ "{\n"
+ " /// <summary>Again <see cref=\"Target\"/></summary>\n"
+ " public class Uses { }\n"
+ "}\n");
+ cbm_pipeline_incremental_test_reset_faults();
+ ASSERT_EQ(dm_index(repo, db, NULL), 0);
+ int errors = dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE reason = 'error'");
+ /* the next run rebuilds everything, and the edge is back */
+ ASSERT_EQ(dm_index(repo, db, NULL), 0);
+ cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route();
+ dm_edge(db, "B.Uses", "A.Target", props, sizeof(props), &n);
+ dm_unlink_db(db);
+ th_rmtree(tmp);
+ ASSERT_EQ(errors, 1);
+ ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_FORCED_FULL);
+ ASSERT_EQ(n, 1);
+ PASS();
+}
+
+/* One row whose reason this layer never writes, then index_status in both
+ * forms: no key carries that text, the row is counted under one fixed key,
+ * and the status is error. 0 when all of that holds. */
+static int dm_unknown_reason_checks(const char *db, const char *project) {
+ if (dm_exec(db, "INSERT INTO doc_link_unresolved(project, rel_path, line, syntax, raw, reason) "
+ "SELECT project, 'src/Injected.cs', 1, 'see', 'X', 'injected_reason_text' "
+ "FROM doc_link_unresolved LIMIT 1") != 1) {
+ return 1;
+ }
+ int bad = 0;
+ char *text = dm_index_status(project, false);
+ const char *block = text ? strstr(text, "doc_links:\n") : NULL;
+ if (!block || strstr(text, "injected_reason_text") ||
+ !strstr(block, "\n unrecognized_reason: 1\n") || !strstr(block, "\n status: error\n")) {
+ printf(" index_status text: %s\n", block ? block : "(no doc_links block)");
+ bad = 1;
+ }
+ free(text);
+ yyjson_doc *env = dm_preview_status(project, true);
+ yyjson_val *unresolved = yyjson_obj_get(dm_preview_report(env), "unresolved");
+ if (!unresolved || yyjson_obj_get(unresolved, "injected_reason_text") ||
+ yyjson_get_int(yyjson_obj_get(unresolved, "unrecognized_reason")) != 1 ||
+ !dm_preview_status_is(env, "error")) {
+ printf(" index_status json: unknown reason shown, or status not error\n");
+ bad = 1;
+ }
+ yyjson_doc_free(env);
+ return bad;
+}
+
+TEST(doc_mentions_index_status_unknown_reason) {
+ ASSERT_EQ(dm_preview_fixture(dm_unknown_reason_checks), 0);
+ PASS();
+}
+
+enum { DM_HUGE_PROJECT = 1100000 }; /* past CSX_MAX_PROJECT_BYTES */
+
+/* A project file the scan does not read -- larger than a project file is,
+ * as the project itself or as a file it imports -- holds what nobody knows,
+ * not nothing: a simple name found nowhere in such a project is external.
+ * Decoy: a project whose files were read keeps a closed scope (missing). */
+TEST(doc_mentions_msbuild_unread_project_opens) {
+ char *filler = malloc(DM_HUGE_PROJECT + 1);
+ ASSERT_NOT_NULL(filler);
+ memset(filler, 'x', DM_HUGE_PROJECT);
+ filler[DM_HUGE_PROJECT] = '\0';
+ char *huge_project =
+ dm_repeated("<Project Sdk=\"Microsoft.NET.Sdk\"><!-- ", filler, 1, " --></Project>\n");
+ char *huge_props = dm_repeated("<Project><!-- ", filler, 1, " --></Project>\n");
+ free(filler);
+ ASSERT_NOT_NULL(huge_project);
+ ASSERT_NOT_NULL(huge_props);
+ const dm_source_t files[] = {
+ {"a/A.csproj", huge_project},
+ {"a/UsesA.cs", "namespace App\n"
+ "{\n"
+ " /// <summary><see cref=\"NowhereA\"/></summary>\n"
+ " public class UsesA { }\n"
+ "}\n"},
+ {"b/Directory.Build.props", huge_props},
+ {"b/B.csproj", DM_EMPTY_PROJECT},
+ {"b/UsesB.cs", "namespace App\n"
+ "{\n"
+ " /// <summary><see cref=\"NowhereB\"/></summary>\n"
+ " public class UsesB { }\n"
+ "}\n"},
+ {"c/C.csproj", DM_EMPTY_PROJECT},
+ {"c/UsesC.cs", "namespace App\n"
+ "{\n"
+ " /// <summary><see cref=\"NowhereC\"/></summary>\n"
+ " public class UsesC { }\n"
+ "}\n"},
+ };
+ static const dm_want_t wants[] = {
+ {"a/UsesA.cs", "NowhereA", "UsesA.UsesA", NULL, "external", NULL, NULL},
+ {"b/UsesB.cs", "NowhereB", "UsesB.UsesB", NULL, "external", NULL, NULL},
+ {"c/UsesC.cs", "NowhereC", "UsesC.UsesC", NULL, "missing", NULL, NULL},
+ };
+ int bad = dm_check_repo("unread", files, DM_COUNT(files), wants, DM_COUNT(wants));
+ free(huge_project);
+ free(huge_props);
+ ASSERT_EQ(bad, 0);
+ PASS();
+}
+
+enum { DM_BIG_MEMBERS = 20000, DM_SMALL_FILES = 200 };
+
+/* What the resolver looks at to build its index over one file of
+ * DM_BIG_MEMBERS fields followed (in path order) by DM_SMALL_FILES files of
+ * one type and one field each. 0 when the repository cannot be indexed. */
+static uint64_t dm_scratch_work(void) {
+ char tmp[256];
+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_dm_scratch_XXXXXX");
+ if (!cbm_mkdtemp(tmp)) {
+ return 0;
+ }
+ size_t cap = (size_t)DM_BIG_MEMBERS * 32 + 64;
+ char *src = malloc(cap);
+ if (!src) {
+ th_rmtree(tmp);
+ return 0;
+ }
+ size_t w = (size_t)snprintf(src, cap, "namespace Big\n{\n public class Wide\n {\n");
+ for (int i = 0; i < DM_BIG_MEMBERS; i++) {
+ w += (size_t)snprintf(src + w, cap - w, " public int f%d;\n", i);
+ }
+ snprintf(src + w, cap - w, " }\n}\n");
+ th_write_file(TH_PATH(tmp, "a/Big.cs"), src);
+ free(src);
+ for (int i = 0; i < DM_SMALL_FILES; i++) {
+ char rel[64];
+ char text[256];
+ snprintf(rel, sizeof(rel), "b/S%03d.cs", i);
+ snprintf(text, sizeof(text),
+ "namespace Small\n{\n%s public class C%03d { public int x; }\n}\n",
+ i == 0 ? " /// <summary><see cref=\"Nowhere\"/></summary>\n" : "", i);
+ th_write_file(TH_PATH(tmp, rel), text);
+ }
+ char db[512];
+ snprintf(db, sizeof(db), "%s/scratch.db", tmp);
+ cbm_doclink_cs_test_work_reset();
+ uint64_t work = dm_index(tmp, db, NULL) == 0 ? cbm_doclink_cs_test_scratch_work() : 0;
+ int rows = dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved");
+ dm_unlink_db(db);
+ th_rmtree(tmp);
+ return rows == 1 ? work : 0;
+}
+
+/* The per-file scratch table of the index build is not emptied at the size
+ * one large file grew it to: after a file of DM_BIG_MEMBERS names, every one
+ * of DM_SMALL_FILES small files costs about its own size. The work stays
+ * within a small multiple of all the names the files declare (emptying the
+ * large table for every small file cost DM_SMALL_FILES times its size). */
+TEST(doc_mentions_cs_scratch_table_work) {
+ uint64_t names = (uint64_t)DM_BIG_MEMBERS + (uint64_t)DM_SMALL_FILES * 2;
+ uint64_t work = dm_scratch_work();
+ printf(" scratch table: %llu steps for %llu names\n", (unsigned long long)work,
+ (unsigned long long)names);
+ ASSERT_TRUE(work > 0);
+ ASSERT_LTE(work, names * 4);
+ PASS();
+}
+
+/* An incremental repair finds the files whose unresolved rows name what a
+ * changed file no longer declares by the identifiers the resolver reads --
+ * a name with a non-ASCII letter included. The parts of `Hub` in two shared
+ * trees both declare the member `Grüße` (the reference is ambiguous); one of
+ * them stops declaring it, and the repaired index equals a full one (the
+ * reference binds). */
+TEST(doc_mentions_incremental_non_ascii_name) {
+ char tmp[256];
+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_dm_utf8name_XXXXXX");
+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
+ char repo[400];
+ char db[512];
+ char full_db[512];
+ snprintf(repo, sizeof(repo), "%s/repo", tmp);
+ snprintf(db, sizeof(db), "%s/inc.db", tmp);
+ snprintf(full_db, sizeof(full_db), "%s/full.db", tmp);
+ th_write_file(TH_PATH(repo, "t1/X.cs"),
+ "namespace N\n{\n public partial class Hub { public void Gr\xC3\xBC\xC3\x9F"
+ "e() { } }\n}\n");
+ th_write_file(TH_PATH(repo, "t2/Y.cs"),
+ "namespace N\n{\n public partial class Hub { public void Gr\xC3\xBC\xC3\x9F"
+ "e() { } public void Keep() { } }\n}\n");
+ th_write_file(TH_PATH(repo, "p/P.csproj"), DM_EMPTY_PROJECT);
+ th_write_file(TH_PATH(repo, "p/Uses.cs"),
+ "namespace App\n"
+ "{\n"
+ " /// <summary><see cref=\"N.Hub.Gr\xC3\xBC\xC3\x9F"
+ "e\"/></summary>\n"
+ " public class Uses { }\n"
+ "}\n");
+ ASSERT_EQ(dm_index(repo, db, NULL), 0);
+ char reason[64];
+ dm_row(db, "p/Uses.cs",
+ "N.Hub.Gr\xC3\xBC\xC3\x9F"
+ "e",
+ reason, sizeof(reason), NULL, 0);
+ bool ambiguous = strcmp(reason, "ambiguous") == 0;
+ th_write_file(TH_PATH(repo, "t2/Y.cs"),
+ "namespace N\n{\n public partial class Hub { public void Keep() { } }\n}\n");
+ int step = dm_step(repo, db, full_db, "a name with a non-ASCII letter removed",
+ CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
+ char props[512];
+ int n = 0;
+ dm_edge(db, "Uses.Uses",
+ "X.Hub.Gr\xC3\xBC\xC3\x9F"
+ "e",
+ props, sizeof(props), &n);
+ dm_unlink_db(db);
+ dm_unlink_db(full_db);
+ th_rmtree(tmp);
+ ASSERT_TRUE(ambiguous);
+ ASSERT_EQ(step, 0);
+ ASSERT_EQ(n, 1);
+ PASS();
+}
+
SUITE(doc_mentions) {
RUN_TEST(doc_mentions_extract_cs_tokens);
RUN_TEST(doc_mentions_cs_scope_blob);
@@ -9203,4 +9531,11 @@ SUITE(doc_mentions) {
RUN_TEST(doc_mentions_msbuild_many_project_files);
RUN_TEST(doc_mentions_cs_scan_nesting_limits);
RUN_TEST(doc_mentions_cs_damaged_stored_scope);
+ RUN_TEST(doc_mentions_cs_portable_scope_bound);
+ RUN_TEST(doc_mentions_cs_unbalanced_brackets);
+ RUN_TEST(doc_mentions_cs_stored_scope_nesting);
+ RUN_TEST(doc_mentions_index_status_unknown_reason);
+ RUN_TEST(doc_mentions_msbuild_unread_project_opens);
+ RUN_TEST(doc_mentions_cs_scratch_table_work);
+ RUN_TEST(doc_mentions_incremental_non_ascii_name);
}