Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
doc-links(cs): a scope this run wrote that does not parse costs only its file; blobs are UTF-8 (S8, S10)
Martin Vogel429a40fcb372
diff --git a/internal/cbm/doclink_cs.c b/internal/cbm/doclink_cs.cindex 14c8a7b03..5996b0003 100644--- a/internal/cbm/doclink_cs.c+++ b/internal/cbm/doclink_cs.c@@ -122,6 +122,69 @@ static void sb_putu(cs_sb_t *sb, uint32_t v) {}}+/* ── Well-formed text ─────────────────────────────────────────────+ *+ * A scope blob is stored as a JSON string, and a JSON writer refuses a+ * string that is not UTF-8: every byte the scans write is well-formed UTF-8.+ * A name or a text of a C# source that is not is not kept (a name: the+ * declaration is not placed; a text: "?"); project-file text gets U+FFFD for+ * every byte that is no part of a well-formed sequence. Both are a function+ * of the file's bytes alone. */++static const char CS_REPLACEMENT[] = "\xEF\xBF\xBD"; /* U+FFFD */++enum {+ CS_UTF8_TAIL_MASK = 0xC0,+ CS_UTF8_TAIL = 0x80,+};++/* Length of the well-formed UTF-8 sequence at s[0, n), 1 to 4; 0 when there+ * is none: a stray continuation byte, an overlong form, a surrogate, a code+ * point past U+10FFFF, a sequence cut short. */+static size_t cs_utf8_len(const unsigned char *s, size_t n) {+ if (n == 0) {+ return 0;+ }+ unsigned char c = s[0];+ if (c < 0x80) {+ return SKIP_ONE;+ }+ size_t len = 0;+ if (c >= 0xC2 && c <= 0xDF) {+ len = PAIR_LEN;+ } else if (c >= 0xE0 && c <= 0xEF) {+ len = 3;+ } else if (c >= 0xF0 && c <= 0xF4) {+ len = 4;+ }+ if (len == 0 || n < len) {+ return 0;+ }+ unsigned char c1 = s[1];+ if ((c == 0xE0 && c1 < 0xA0) || (c == 0xED && c1 > 0x9F) || (c == 0xF0 && c1 < 0x90) ||+ (c == 0xF4 && c1 > 0x8F)) {+ return 0; /* overlong, surrogate, or past U+10FFFF */+ }+ for (size_t k = SKIP_ONE; k < len; k++) {+ if ((s[k] & CS_UTF8_TAIL_MASK) != CS_UTF8_TAIL) {+ return 0;+ }+ }+ return len;+}++/* True when s[0, n) is well-formed UTF-8. */+static bool cs_utf8_ok(const char *s, size_t n) {+ for (size_t i = 0; i < n;) {+ size_t l = cs_utf8_len((const unsigned char *)s + i, n - i);+ if (l == 0) {+ return false;+ }+ i += l;+ }+ return true;+}+/* ── Doc-comment references ──────────────────────────────────────── */static int cs_tag_syntax(const char *name, size_t len) {@@ -939,9 +1002,10 @@ static TSNode cs_type_params(TSNode decl) {/* Node text without whitespace and without verbatim '@' markers, appended to* sb. `global::` prefixes are dropped. Text that is too long to be a type- * name, or that holds a scope separator or non-whitespace control byte, is- * written as "?" (an unresolvable name: the declaring type then counts as- * having an open hierarchy). The whole field is replaced before any copy. */+ * name, that holds a scope separator or non-whitespace control byte, or that+ * is not well-formed UTF-8, is written as "?" (an unresolvable name: the+ * declaring type then counts as having an open hierarchy). The whole field is+ * replaced before any copy. */static void cs_put_text_nows(cs_scan_t *s, TSNode n) {const char *src = s->ctx->source;uint32_t a = ts_node_start_byte(n);@@ -955,7 +1019,7 @@ static void cs_put_text_nows(cs_scan_t *s, TSNode n) {ok = c != '|' && c != ';' && c != '{' && c != '}' &&!((c < 0x20 && !isspace(c)) || c == 0x7f);}- if (!ok) {+ if (!ok || !cs_utf8_ok(src + a, b - a)) {sb_putc(&s->sb, '?');return;}@@ -978,8 +1042,9 @@ static void *cs_tmp_alloc(cs_scan_t *s, size_t n) {}/* Copy of source bytes [a, b) without whitespace and '@'. NULL unless it is a- * (dotted) identifier of sane length: an error-recovered parse can hand back a- * "name" spanning arbitrary code, which must not become a declaration. */+ * (dotted) identifier of sane length and well-formed UTF-8: an error-recovered+ * parse can hand back a "name" spanning arbitrary code, which must not become+ * a declaration, and a name that is not UTF-8 cannot be written. */static char *cs_ident_dup(cs_scan_t *s, uint32_t a, uint32_t b) {const char *src = s->ctx->source;if (b <= a || b - a > CS_NAME_MAX) {@@ -1001,7 +1066,7 @@ static char *cs_ident_dup(cs_scan_t *s, uint32_t a, uint32_t b) {out[w++] = (char)c;}out[w] = '\0';- return w > 0 ? out : NULL;+ return w > 0 && cs_utf8_ok(out, w) ? out : NULL;}static char *cs_name_dup(cs_scan_t *s, TSNode n) {@@ -1073,6 +1138,9 @@ static void cs_put_sig(cs_scan_t *s, TSNode params) {uint32_t a = ts_node_start_byte(ty);uint32_t b = ts_node_end_byte(ty);cbm_doclink_cs_norm_type(src + a, (size_t)(b - a), norm, sizeof(norm));+ if (!cs_utf8_ok(norm, strlen(norm))) {+ snprintf(norm, sizeof(norm), "?"); /* a type nothing is known about */+ }}if (!first) {sb_putc(&s->sb, '|');@@ -1829,7 +1897,7 @@ static char *cs_namespace_name_dup(cs_scan_t *s, uint32_t a, uint32_t b) {i = cs_skip_space(src, i, b);if (i == b) {out[w] = '\0';- return out;+ return cs_utf8_ok(out, w) ? out : NULL; /* a name that cannot be written */}if (src[i] != '.') {return NULL;@@ -1937,7 +2005,7 @@ static const char *cs_text_tparams(cs_scan_t *s, uint32_t *pos, uint32_t stop) {if (c == '>') {out[w] = '\0';*pos = k + SKIP_ONE;- return out;+ return cs_utf8_ok(out, w) ? out : NULL; /* names that cannot be written */}} else if (cs_word_char((unsigned char)c)) {uint32_t e = cs_ident_end(src, k, limit, false);@@ -3301,9 +3369,12 @@ static void csx_put_char(cs_sb_t *sb, unsigned char c) {}}-/* A code point as UTF-8. */+/* A code point as UTF-8. A surrogate (a numeric reference may name one) is+ * no character: it is written as U+FFFD. */static void csx_put_codepoint(cs_sb_t *sb, uint32_t cp) {- if (cp < 0x80) {+ if (cp >= 0xD800 && cp <= 0xDFFF) {+ sb_puts(sb, CS_REPLACEMENT);+ } else if (cp < 0x80) {csx_put_char(sb, (unsigned char)cp);} else if (cp < 0x800) {sb_putc(sb, (char)(0xC0 | (cp >> 6)));@@ -3356,6 +3427,19 @@ static uint32_t csx_entity(const csx_t *x, uint32_t i, uint32_t end, uint32_t *creturn 0;}+/* The bytes src[i, end) of a project file that start at i into sb: a+ * well-formed UTF-8 sequence as it stands, a byte that starts none as+ * U+FFFD. Returns the index past what was taken. */+static uint32_t csx_put_utf8(cs_sb_t *sb, const char *src, uint32_t i, uint32_t end) {+ size_t len = cs_utf8_len((const unsigned char *)src + i, (size_t)(end - i));+ if (len == 0) {+ sb_puts(sb, CS_REPLACEMENT);+ return i + SKIP_ONE;+ }+ sb_putn(sb, src + i, len);+ return i + (uint32_t)len;+}+/* The text of `v`, entities decoded (unless `raw`), escaped. */static void csx_put_text(cs_sb_t *sb, const csx_t *x, csx_span_t v, bool raw) {for (uint32_t i = v.s; i < v.e;) {@@ -3364,6 +3448,8 @@ static void csx_put_text(cs_sb_t *sb, const csx_t *x, csx_span_t v, bool raw) {if (past) {csx_put_codepoint(sb, cp);i = past;+ } else if ((unsigned char)x->src[i] >= 0x80) {+ i = csx_put_utf8(sb, x->src, i, v.e);} else {csx_put_char(sb, (unsigned char)x->src[i]);i++;@@ -3371,6 +3457,14 @@ static void csx_put_text(cs_sb_t *sb, const csx_t *x, csx_span_t v, bool raw) {}}+/* An element's name into sb, each byte that starts no well-formed UTF-8+ * sequence as U+FFFD (the tokenizer's names hold no separator). */+static void csx_put_name(cs_sb_t *sb, const csx_t *x, csx_span_t name) {+ for (uint32_t i = name.s; i < name.e;) {+ i = csx_put_utf8(sb, x->src, i, name.e);+ }+}+/* A field: a tab, then nothing for an absent attribute, else '=' and its text. */static void csx_put_field(cs_sb_t *sb, const csx_t *x, csx_span_t v) {sb_putc(sb, '\t');@@ -3435,7 +3529,7 @@ static void csx_put_property(csx_scan_t *p) {sb_putc(&p->out, 'V');csx_put_field(&p->out, &p->x, t->attr[CSX_A_CONDITION]);sb_putc(&p->out, '\t');- sb_putn(&p->out, p->x.src + t->name.s, t->name.e - t->name.s);+ csx_put_name(&p->out, &p->x, t->name);sb_putc(&p->out, '\t');if (p->prop_complex) {sb_putc(&p->out, '?');@@ -3452,7 +3546,7 @@ static void csx_put_property(csx_scan_t *p) {static void csx_choose_child(csx_scan_t *p, const csx_tok_t *t) {if (p->choose_props >= 0 && p->depth == p->choose_props) {sb_puts(&p->out, "K\t");- sb_putn(&p->out, p->x.src + t->name.s, t->name.e - t->name.s);+ csx_put_name(&p->out, &p->x, t->name);sb_putc(&p->out, '\n');} else if (csx_is(&p->x, t->name, "PropertyGroup") && p->choose_props < 0 &&t->kind == CSX_OPEN) {diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.hindex 69fed4bd1..9a6452a77 100644--- a/src/pipeline/doc_links.h+++ b/src/pipeline/doc_links.h@@ -222,6 +222,11 @@ uint64_t cbm_doclink_cs_test_scratch_work(void);/* Test seam (doc_links.c): the nth MENTIONS edge insert from now on fails as* if memory ran out (0: none). */void cbm_doclinks_test_fail_edge_insert_after(int nth);+/* Test seams (doc_links_cs.c): spoil the scope of the file at `rel_path`+ * (every record, then one the reader refuses; NULL or "": none), and ask the+ * reader whether it takes a scope blob. */+void cbm_doclink_cs_test_spoil_scope(const char *rel_path);+bool cbm_doclink_cs_test_scope_parses(const char *scope);#endif/* True when `rel_path` is a scope input of some language (scope_input). */diff --git a/src/pipeline/doc_links_cs.c b/src/pipeline/doc_links_cs.cindex c957841e7..92ab111e6 100644--- a/src/pipeline/doc_links_cs.c+++ b/src/pipeline/doc_links_cs.c@@ -335,6 +335,9 @@ typedef struct {int *members_by_start;cs_tparam_t *tparams; /* by (owner, name) */int ntparams;+ /* Its scope was written in this run and this reader refuses it: the file+ * declares nothing here, and its own references are graph gaps. */+ bool rejected;} cs_file_t;typedef struct {@@ -468,8 +471,33 @@ typedef enum {/* Counted while references are resolved (by every worker at once). */typedef struct {_Atomic uint64_t ambiguous[CS_WHY_COUNT];+ _Atomic uint64_t rejected; /* references of files whose fresh scope was refused */} cs_stats_t;+/* What parsing one scope set in the tables all files share: so that a scope+ * written in this run that this reader refuses can be taken back whole, and+ * costs only its own file. Reset for every file. */+typedef struct {+ int ns;+ bool declared;+ bool prod;+} cs_ns_was_t;++typedef struct {+ CBMHashTable *ht;+ const char *key;+} cs_mark_was_t;++typedef struct {+ int nnss; /* namespaces before the file */+ cs_ns_was_t *flags;+ int nflags;+ int cap_flags;+ cs_mark_was_t *marks;+ int nmarks;+ int cap_marks;+} cs_undo_t;+typedef struct {CBMArena arena;bool oom; /* an index allocation failed */@@ -517,6 +545,8 @@ typedef struct {int nshared; /* shared trees */cs_stats_t *stats;bool bind_inherited; /* CS_BIND_INHERITED */+ cs_undo_t undo; /* of the file being parsed */+ int rejected; /* files whose scope, written in this run, was refused */} cs_index_t;#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS@@ -627,11 +657,56 @@ static bool ht_mark(cs_index_t *ix, CBMHashTable *ht, const char *key) {return true;}+/* Grow an undo array to hold one more entry. false when memory ran out. */+static bool undo_room(cs_index_t *ix, void **arr, int n, int *cap, size_t size) {+ if (n < *cap) {+ return true;+ }+ int ncap = *cap ? *cap * PAIR_LEN : CBM_SZ_16;+ void *grown = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)ncap * size);+ if (!grown) {+ ix->oom = true;+ return false;+ }+ if (n > 0) {+ memcpy(grown, *arr, (size_t)n * size);+ }+ cbm_free(CBM_MEM_CLASS_OTHER, *arr);+ *arr = grown;+ *cap = ncap;+ return true;+}++/* Remember the flags of namespace `ns` before the file's scope sets them. */+static bool undo_note_ns(cs_index_t *ix, int ns) {+ cs_undo_t *u = &ix->undo;+ if (ns >= u->nnss) {+ return true; /* made by this file: taken back as a whole */+ }+ if (!undo_room(ix, (void **)&u->flags, u->nflags, &u->cap_flags, sizeof(cs_ns_was_t))) {+ return false;+ }+ u->flags[u->nflags++] =+ (cs_ns_was_t){.ns = ns, .declared = ix->nss[ns].declared, .prod = ix->nss[ns].prod};+ return true;+}+/* A type name declared in `f` at a place no namespace or outer type could be* established for. Product code never binds test-only declarations, so a name* only test files quarantine blocks references from test files only. */static bool quarantine_name(cs_index_t *ix, const cs_file_t *f, const char *name) {- return ht_mark(ix, f->is_test ? ix->quarantine_test : ix->quarantine, name);+ CBMHashTable *ht = f->is_test ? ix->quarantine_test : ix->quarantine;+ if (cbm_ht_get(ht, name)) {+ return true;+ }+ char *k = ix_strdup(ix, name);+ cs_undo_t *u = &ix->undo;+ if (!k || !undo_room(ix, (void **)&u->marks, u->nmarks, &u->cap_marks, sizeof(cs_mark_was_t))) {+ return false;+ }+ cbm_ht_set(ht, k, (void *)k);+ u->marks[u->nmarks++] = (cs_mark_was_t){.ht = ht, .key = k};+ return true;}static bool cs_is_test_path(const char *rel) {@@ -937,13 +1012,16 @@ static bool parse_region(cs_index_t *ix, cs_file_t *f, char **fld, int n, int *n}(*next_region)++;int ns = ns_make_path(ix, f->regions[parent].ns, fld[5]);- if (ns < 0) {+ if (ns < 0 || !undo_note_ns(ix, ns)) {return false;}ix->nss[ns].declared = true;/* product code declares it, and with it every namespace above: one that* is marked has its upper ones marked already */for (int up = ns; !f->is_test && up > 0 && !ix->nss[up].prod; up = ix->nss[up].parent) {+ if (!undo_note_ns(ix, up)) {+ return false;+ }ix->nss[up].prod = true;}f->regions[id] = (cs_region_t){.parent = parent,@@ -5621,6 +5699,18 @@ static void cs_destroy(void *index) {if (ix->stats) {log_ambiguous(ix->stats);}+ if (ix->rejected > 0) {+ /* scopes written in this run that this reader refused: each cost+ * only its own file, whose references are graph gaps */+ char files[CBM_SZ_32];+ char refs[CBM_SZ_32];+ snprintf(files, sizeof(files), "%d", ix->rejected);+ snprintf(refs, sizeof(refs), "%llu",+ (unsigned long long)(ix->stats ? atomic_load(&ix->stats->rejected) : 0));+ cbm_log_warn("doc_links.cs.rejected_scopes", "files", files, "references", refs);+ }+ cbm_free(CBM_MEM_CLASS_OTHER, ix->undo.flags);+ cbm_free(CBM_MEM_CLASS_OTHER, ix->undo.marks);cbm_free(CBM_MEM_CLASS_OTHER, ix->stats);cbm_ht_free(ix->project_above);cbm_ht_free(ix->ns_by_key);@@ -5688,10 +5778,96 @@ static bool build_tables(cs_index_t *ix, const cbm_doclink_build_in_t *in) {return collect_projects(ix, in);}-/* Every file of the index with what its scope declares. Returns the path of- * a file whose scope is not one this code wrote (the index is not built over- * such a scope: nothing may be resolved around a declaration that is not- * known), "" when memory ran out, NULL when all is well. */+/* Start the undo record of one file's scope. */+static void undo_begin(cs_index_t *ix) {+ ix->undo.nnss = ix->nnss;+ ix->undo.nflags = 0;+ ix->undo.nmarks = 0;+}++/* Take back what the file's scope set in the shared tables: the names it+ * quarantined, the flags it set on namespaces that were there before it, and+ * the namespaces it made. */+static void undo_scope(cs_index_t *ix) {+ cs_undo_t *u = &ix->undo;+ for (int i = u->nmarks - SKIP_ONE; i >= 0; i--) {+ cbm_ht_delete(u->marks[i].ht, u->marks[i].key);+ }+ for (int i = u->nflags - SKIP_ONE; i >= 0; i--) {+ ix->nss[u->flags[i].ns].declared = u->flags[i].declared;+ ix->nss[u->flags[i].ns].prod = u->flags[i].prod;+ }+ for (int id = ix->nnss - SKIP_ONE; id >= u->nnss; id--) {+ char key[CS_NAME_BUF + CBM_SZ_16];+ const cs_ns_t *n = &ix->nss[id];+ if (ns_key(key, sizeof(key), n->parent, n->name, strlen(n->name))) {+ cbm_ht_delete(ix->ns_by_key, key);+ }+ }+ ix->nnss = u->nnss;+ u->nflags = 0;+ u->nmarks = 0;+}++#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS+/* Test seam: the scope of the file at this path is spoiled -- every record+ * it has, then one this reader refuses -- as if its writer and this reader+ * disagreed after the file's declarations were read. "" or NULL: none. */+static char cs_test_spoiled_path[CBM_SZ_512];++void cbm_doclink_cs_test_spoil_scope(const char *rel_path) {+ snprintf(cs_test_spoiled_path, sizeof(cs_test_spoiled_path), "%s", rel_path ? rel_path : "");+}++static const char *cs_test_spoiled(cs_index_t *ix, const char *rel_path, const char *scope) {+ if (!scope || !cs_test_spoiled_path[0] || strcmp(rel_path, cs_test_spoiled_path) != 0) {+ return scope;+ }+ const char *spoiled = cbm_arena_sprintf(&ix->arena, "%sZ\tspoiled\n", scope);+ return spoiled ? spoiled : scope;+}++/* Test seam: true when this reader takes the scope blob `scope` (its record+ * checks, on an index of its own). A test holds every blob the scanner+ * writes against it. */+bool cbm_doclink_cs_test_scope_parses(const char *scope) {+ cs_index_t *ix = (cs_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*ix));+ if (!ix) {+ return false;+ }+ cbm_arena_init(&ix->arena);+ ix->ns_by_key = cbm_ht_create(CBM_SZ_64);+ ix->quarantine = cbm_ht_create(CBM_SZ_64);+ ix->quarantine_test = cbm_ht_create(CBM_SZ_64);+ ix->nss = (cs_ns_t *)ix_zalloc(ix, CBM_SZ_256 * sizeof(cs_ns_t));+ bool ok = ix->ns_by_key && ix->quarantine && ix->quarantine_test && ix->nss;+ if (ok) {+ ix->nscap = CBM_SZ_256;+ ix->nnss = SKIP_ONE;+ ix->nss[0] = (cs_ns_t){.parent = CS_NONE, .name = ""};+ cs_file_t f = {.rel_path = "Scope.cs"};+ undo_begin(ix);+ ok = parse_scope(ix, &f, scope) && !ix->oom;+ }+ cbm_ht_free(ix->ns_by_key);+ cbm_ht_free(ix->quarantine);+ cbm_ht_free(ix->quarantine_test);+ cbm_free(CBM_MEM_CLASS_OTHER, ix->undo.flags);+ cbm_free(CBM_MEM_CLASS_OTHER, ix->undo.marks);+ cbm_arena_destroy(&ix->arena);+ cbm_free(CBM_MEM_CLASS_OTHER, ix);+ return ok;+}+#endif++/* Every file of the index with what its scope declares. A scope read back+ * from the store that this reader refuses is not one this code wrote: the+ * index is not built over it (nothing may be resolved around a declaration+ * that is not known), and its path is returned. A scope written in this run+ * that the reader refuses is the scanner's and this reader's disagreement,+ * which costs only its own file: what its parse set is taken back, the file+ * declares nothing, and its references are graph gaps (counted, logged).+ * Returns "" when memory ran out, NULL when all is well. */static const char *build_files(cs_index_t *ix, const cbm_doclink_build_in_t *in) {CBMHashTable *dir_unit = cbm_ht_create(CBM_SZ_1K);const char *bad = dir_unit ? NULL : "";@@ -5704,9 +5880,23 @@ static const char *build_files(cs_index_t *ix, const cbm_doclink_build_in_t *in)f->is_test = cs_is_test_path(src->rel_path);/* a project file declares nothing: its blob went to the evaluator */const char *scope = cbm_msb_is_project_scope(src->scope) ? NULL : src->scope;+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS+ scope = cs_test_spoiled(ix, src->rel_path, scope);+#endif+ undo_begin(ix);if (scope && !parse_scope(ix, f, scope)) {- bad = ix->oom ? "" : src->rel_path;- break;+ bool fresh = src->run_file >= 0 && src->run_file < in->run_file_count;+ if (ix->oom || !fresh) {+ bad = ix->oom ? "" : src->rel_path;+ break;+ }+ undo_scope(ix);+ *f = (cs_file_t){.rel_path = f->rel_path,+ .module_qn = f->module_qn,+ .is_test = f->is_test,+ .rejected = true};+ ix->rejected++;+ scope = NULL;}if (!scope) {/* no scope: an empty file (only its own region) */@@ -5887,6 +6077,12 @@ static void cs_resolve(const void *index, int run_file, const CBMDocLink *link,* declaration. Both are declared-but-unplaced, i.e. graph gaps. */int file = ix->run_to_file[run_file];const cs_file_t *f = &ix->files[file];+ if (f->rejected) {+ /* its scope was refused: nothing is known around the reference */+ out->reason = CBM_DOCLINK_REASON_GRAPH_GAP;+ atomic_fetch_add_explicit(&ix->stats->rejected, 1, memory_order_relaxed);+ return;+ }bool file_doc = (link->flags & CBM_DOCLINK_FLAG_FILE) != 0;if ((!file_doc && line_unplaced(f, link->def_line)) || names_quarantined(ix, f, &r)) {out->reason = CBM_DOCLINK_REASON_GRAPH_GAP;diff --git a/tests/test_doc_mentions.c b/tests/test_doc_mentions.cindex 0969fdc6b..bcad1ed4f 100644--- a/tests/test_doc_mentions.c+++ b/tests/test_doc_mentions.c@@ -814,10 +814,16 @@ TEST(doc_mentions_cs_control_publication) {PASS();}-/* Temporary observation: extract real source bytes, then exercise the public- * surface codec with complete definitions and with each input isolated. */+/* S10: extract real source bytes, then run the surface writer of the run+ * with the file's complete definitions, without its scope, and with only+ * its scope. Every run must succeed, twice alike, and the stored `dl` must+ * be the portable scope. `expect`: 1 the inserted bytes stay in the scope+ * (well-formed UTF-8), 0 they do not (malformed), -1 not asked (an entity). */+static bool dm_scope_utf8_ok(const char *scope);+static bool dm_utf8_probe_surface(CBMFileResult *bad, CBMFileResult *healthy, CBMLanguage language,- const char *path, const char *label, const char *bytes) {+ const char *path, const char *label, const char *bytes,+ int expect) {CBMFileResult *cache[] = {bad, healthy};cbm_file_info_t files[] = {{.rel_path = (char *)path, .language = language},{.rel_path = "Healthy.cs", .language = CBM_LANG_CSHARP}};@@ -865,20 +871,33 @@ static bool dm_utf8_probe_surface(CBMFileResult *bad, CBMFileResult *healthy, CBint scope_rc =cbm_lsp_surface_build_rows(NULL, "p", cache, files, 2, NULL, NULL, &rows, &count);bool retained = saved_scope && strstr(saved_scope, bytes) != NULL;- printf("doc_utf8_probe case=%s scope=%d retained=%d defs=%d full_rc=%d full_count=%d "- "nodl_rc=%d nodl_count=%d scope_rc=%d scope_count=%d repeat=%d dl_equal=%d\n",- label, saved_scope != NULL, retained, def_count, full_rc, full_count, nodl_rc,- nodl_count, scope_rc, count, repeat, dl_equal);+ bool written = full_rc == 0 && full_count == 2 && again_rc == 0 && dl_equal && nodl_rc == 0 &&+ nodl_count == 2 && scope_rc == 0 && count == 2;+ bool kept = expect < 0 || retained == (expect == 1);+ bool utf8 = saved_scope && dm_scope_utf8_ok(saved_scope);+ bool correct = saved_scope != NULL && def_count > 0 && repeat && written && kept && utf8;+ if (!correct) {+ printf("doc_utf8_probe case=%s scope=%d retained=%d utf8=%d defs=%d full_rc=%d "+ "full_count=%d nodl_rc=%d nodl_count=%d scope_rc=%d scope_count=%d repeat=%d "+ "dl_equal=%d\n",+ label, saved_scope != NULL, retained, utf8, def_count, full_rc, full_count, nodl_rc,+ nodl_count, scope_rc, count, repeat, dl_equal);+ }cbm_store_free_lsp_surfaces(rows, count);cbm_free(CBM_MEM_CLASS_OTHER, portable);free(defs);free(modules[0]);free(modules[1]);cbm_arena_destroy(&arena);- return saved_scope != NULL && def_count > 0 && repeat;+ return correct;}-TEST(doc_mentions_cs_utf8_observation) {+/* S10: a malformed byte sequence in any place a scope keeps text -- a name+ * or a text of a C# source, an element, a value or an attribute of a project+ * file, a numeric reference to a surrogate -- leaves a well-formed scope, and+ * the surface writer of the run succeeds. Well-formed sequences stay as they+ * are; malformed ones do not reach the scope. */+TEST(doc_mentions_cs_utf8_surfaces) {static const struct {const char *label, *bytes;} sequences[] = {@@ -927,7 +946,6 @@ TEST(doc_mentions_cs_utf8_observation) {int healthy_count = 0;int healthy_rc = cbm_lsp_surface_build_rows(NULL, "p", &healthy, &healthy_file, 1, NULL, NULL,&healthy_rows, &healthy_count);- printf("doc_utf8_probe_healthy rc=%d count=%d\n", healthy_rc, healthy_count);correct = correct && healthy_rc == 0 && healthy_count == 1;cbm_store_free_lsp_surfaces(healthy_rows, healthy_count);for (size_t p = 0; p < sizeof(positions) / sizeof(positions[0]); p++) {@@ -947,8 +965,9 @@ TEST(doc_mentions_cs_utf8_observation) {CBMFileResult *bad = cbm_extract_file(source, (int)(a + b + c + d),positions[p].language, "p", path, 0, NULL, NULL);snprintf(label, sizeof(label), "%s/%s", positions[p].label, sequences[s].label);+ bool well_formed = s < 4; /* ascii, valid2, valid3, valid4 */bool setup = bad && dm_utf8_probe_surface(bad, healthy, positions[p].language, path,- label, sequences[s].bytes);+ label, sequences[s].bytes, well_formed);correct = setup && correct;cbm_free_result(bad);}@@ -968,7 +987,7 @@ TEST(doc_mentions_cs_utf8_observation) {snprintf(label, sizeof(label), "entity_%s/%zu", position == 0 ? "property" : "using",e);bool setup = bad && dm_utf8_probe_surface(bad, healthy, CBM_LANG_XML, "App.csproj",- label, entities[e]);+ label, entities[e], -1);correct = setup && correct;cbm_free_result(bad);}@@ -9527,6 +9546,214 @@ static int dm_alloc_failure_errors(int point) {return errors;}+/* True when s[0, n) is well-formed UTF-8 (no surrogate, nothing past+ * U+10FFFF, no overlong form, nothing cut short). */+static bool dm_utf8_ok(const char *s, size_t n) {+ const unsigned char *u = (const unsigned char *)s;+ for (size_t i = 0; i < n;) {+ unsigned char c = u[i];+ size_t len = c < 0x80 ? 1+ : (c >= 0xC2 && c <= 0xDF) ? 2+ : (c >= 0xE0 && c <= 0xEF) ? 3+ : (c >= 0xF0 && c <= 0xF4) ? 4+ : 0;+ if (len == 0 || i + len > n) {+ return false;+ }+ if (len > 1) {+ unsigned char c1 = u[i + 1];+ if ((c == 0xE0 && c1 < 0xA0) || (c == 0xED && c1 > 0x9F) || (c == 0xF0 && c1 < 0x90) ||+ (c == 0xF4 && c1 > 0x8F)) {+ return false;+ }+ for (size_t k = 1; k < len; k++) {+ if ((u[i + k] & 0xC0) != 0x80) {+ return false;+ }+ }+ }+ i += len;+ }+ return true;+}++static bool dm_scope_utf8_ok(const char *scope) {+ return dm_utf8_ok(scope, strlen(scope));+}++/* S8, S9, S10: every scope blob the scanners write passes the reader's own+ * record checks, is a C string of the length that was built, and is+ * well-formed UTF-8 -- for namespace names with empty segments, and for+ * control bytes and malformed UTF-8 in every place a C# scope keeps text.+ * A project file's blob is well-formed UTF-8 for malformed bytes and for+ * numeric references to surrogates. */+TEST(doc_mentions_cs_scanner_output_reads_back) {+ static const struct {+ const char *label, *bytes;+ size_t n;+ } bytes[] = {+ {"nul", "\x00", 1},+ {"soh", "\x01", 1},+ {"us", "\x1F", 1},+ {"del", "\x7F", 1},+ {"continuation", "\x80", 1},+ {"overlong", "\xC0\xAF", 2},+ {"surrogate", "\xED\xA0\x80", 3},+ {"above", "\xF4\x90\x80\x80", 4},+ {"truncated", "\xE2\x82", 2},+ {"valid", "\xC3\xA9", 2},+ };+ static const struct {+ const char *label, *before, *after;+ } places[] = {+ {"using_target", "using Acme.", "Name;\nclass Local {}\n"},+ {"alias_target", "using Alias = Acme.", "Name;\nclass Local {}\n"},+ {"namespace_name", "namespace N", "Name { class Local {} }\n"},+ {"type_name", "class N", "Name {}\n"},+ {"method_name", "class Local { void N", "Name() {} }\n"},+ {"parameter_type", "class Local { void M(N", "Name arg) {} }\n"},+ {"type_parameter", "class Local<N", "Name> {}\n"},+ {"method_type_parameter", "class Local { void M<N", "Name>() {} }\n"},+ {"field_name", "class Local { int N", "Name; }\n"},+ {"base_type", "class Local : Acme.N", "Name {}\n"},+ {"broken_file", "class Local { void M( { N", "Name } }\n"},+ };+ bool ok = true;+ for (size_t p = 0; p < sizeof(places) / sizeof(places[0]); p++) {+ for (size_t b = 0; b < sizeof(bytes) / sizeof(bytes[0]); b++) {+ char source[512];+ size_t a = strlen(places[p].before);+ size_t c = strlen(places[p].after);+ memcpy(source, places[p].before, a);+ memcpy(source + a, bytes[b].bytes, bytes[b].n);+ memcpy(source + a + bytes[b].n, places[p].after, c);+ size_t len = a + bytes[b].n + c;+ source[len] = '\0';+ cbm_doclink_cs_test_cost_reset();+ CBMFileResult *r =+ cbm_extract_file(source, (int)len, CBM_LANG_CSHARP, "p", "Bad.cs", 0, NULL, NULL);+ const char *scope = r ? r->doc_scope : NULL;+ size_t built = (size_t)cbm_doclink_cs_test_scope_bytes();+ bool good = scope && strlen(scope) == built && dm_utf8_ok(scope, strlen(scope)) &&+ cbm_doclink_cs_test_scope_parses(scope);+ if (!good) {+ printf(" %s/%s: scope=%d length=%zu built=%zu utf8=%d reads=%d\n", places[p].label,+ bytes[b].label, scope != NULL, scope ? strlen(scope) : (size_t)0, built,+ scope ? dm_utf8_ok(scope, strlen(scope)) : 0,+ scope ? cbm_doclink_cs_test_scope_parses(scope) : 0);+ ok = false;+ }+ cbm_free_result(r);+ }+ }+ for (size_t i = 0; i < sizeof(dm_namespace_cases) / sizeof(dm_namespace_cases[0]); i++) {+ for (int file_scoped = 0; file_scoped < 2; file_scoped++) {+ char source[2048];+ if (!dm_namespace_source(source, sizeof(source), i, file_scoped != 0)) {+ ok = false;+ continue;+ }+ CBMFileResult *r = dm_extract(source, CBM_LANG_CSHARP, "Bad.cs");+ const char *scope = r ? r->doc_scope : NULL;+ if (!scope || !cbm_doclink_cs_test_scope_parses(scope)) {+ printf(" namespace %s/%s: the reader refuses the scope\n",+ dm_namespace_cases[i].id, file_scoped ? "file" : "block");+ ok = false;+ }+ cbm_free_result(r);+ }+ }+ static const char *xml[] = {+ "<Project><PropertyGroup><P>N\x80Name</P></PropertyGroup></Project>",+ "<Project><PropertyGroup><P\xC3\x28>Value</P\xC3\x28></PropertyGroup></Project>",+ "<Project><PropertyGroup><P>N�Name�</P></PropertyGroup></Project>",+ "<Project><ItemGroup><Using Include=\"Acme.N\xED\xA0\x80Name\" /></ItemGroup></Project>",+ "<Project><PropertyGroup Condition=\"'\xF4\x90\x80\x80'=='x'\"><P>V</P></PropertyGroup>"+ "</Project>",+ "<Project><Choose><When Condition=\"true\"><PropertyGroup><Q\x80>V</Q\x80>"+ "</PropertyGroup></When></Choose></Project>",+ };+ for (size_t i = 0; i < sizeof(xml) / sizeof(xml[0]); i++) {+ CBMFileResult *r = cbm_extract_file(xml[i], (int)strlen(xml[i]), CBM_LANG_XML, "p",+ "App.csproj", 0, NULL, NULL);+ const char *scope = r ? r->doc_scope : NULL;+ if (!scope || !dm_utf8_ok(scope, strlen(scope))) {+ printf(" project file %zu: blob=%d utf8=%d\n", i, scope != NULL,+ scope ? dm_utf8_ok(scope, strlen(scope)) : 0);+ ok = false;+ }+ cbm_free_result(r);+ }+ ASSERT_TRUE(ok);+ PASS();+}++/* S8: a scope written in this run that the reader refuses costs only its own+ * file. The status stays ok and the other file's edges are there; the file's+ * own reference is a graph gap; and nothing its parse set before the refusal+ * stays -- a namespace it declares does not stand beside a type of that name+ * (Shadow, Good.Lurker), and a name it quarantined does not block another+ * file's declaration (Hidden2). */+TEST(doc_mentions_cs_rejected_scope_contained) {+ static const dm_source_t files[] = {+ {"Bad.cs", "namespace Shadow\n"+ "{\n"+ " public class Inner { }\n"+ "}\n"+ "namespace Good\n"+ "{\n"+ " namespace Lurker { public class Deep { } }\n"+ "\n"+ " /// <summary><see cref=\"Target\"/></summary>\n"+ " public class FromBad { }\n"+ "}\n"+ "namespace .Broken\n"+ "{\n"+ " public class Hidden2 { }\n"+ "}\n"},+ {"Healthy.cs", "public class Shadow { }\n"+ "namespace Good\n"+ "{\n"+ " public class Target { }\n"+ " public class Lurker { }\n"+ " public class Hidden2 { }\n"+ "\n"+ " /// <summary><see cref=\"Target\"/> <see cref=\"Shadow\"/>\n"+ " /// <see cref=\"Lurker\"/> <see cref=\"Hidden2\"/></summary>\n"+ " public class Uses { }\n"+ "}\n"},+ };+ static const dm_want_t wants[] = {+ {"Healthy.cs", "Target", "Healthy.Uses", "Healthy.Target", NULL, NULL, NULL},+ {"Healthy.cs", "Shadow", "Healthy.Uses", "Healthy.Shadow", NULL, NULL, NULL},+ {"Healthy.cs", "Lurker", "Healthy.Uses", "Healthy.Lurker", NULL, NULL, NULL},+ {"Healthy.cs", "Hidden2", "Healthy.Uses", "Healthy.Hidden2", NULL, NULL, NULL},+ {"Bad.cs", "Target", "Bad.FromBad", NULL, "graph_gap", "Healthy.Target", NULL},+ };+ char tmp[256];+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_dm_rejected_XXXXXX");+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));+ for (int i = 0; i < DM_COUNT(files); i++) {+ th_write_file(TH_PATH(tmp, files[i].path), files[i].text);+ }+ char db[512];+ snprintf(db, sizeof(db), "%s/rejected.db", tmp);+ cbm_doclink_cs_test_spoil_scope("Bad.cs");+ int bad = dm_index(tmp, db, NULL) == 0 ? 0 : -1;+ cbm_doclink_cs_test_spoil_scope(NULL);+ int errors = bad == 0 ? dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved "+ "WHERE reason = 'error'")+ : -1;+ for (int i = 0; bad >= 0 && i < DM_COUNT(wants); i++) {+ bad += dm_want_failed(db, &wants[i]);+ }+ dm_unlink_db(db);+ th_rmtree(tmp);+ ASSERT_EQ(errors, 0);+ ASSERT_EQ(bad, 0);+ PASS();+}+TEST(doc_mentions_alloc_failure_status) {static const struct {int point;@@ -9557,7 +9784,7 @@ SUITE(doc_mentions) {RUN_TEST(doc_mentions_cs_namespace_boundaries);RUN_TEST(doc_mentions_cs_control_scopes);RUN_TEST(doc_mentions_cs_control_publication);- RUN_TEST(doc_mentions_cs_utf8_observation);+ RUN_TEST(doc_mentions_cs_utf8_surfaces);RUN_TEST(doc_mentions_cs_namespace_publication);RUN_TEST(doc_mentions_cs_scope_parse_errors);RUN_TEST(doc_mentions_cs_norm_type);@@ -9659,4 +9886,6 @@ SUITE(doc_mentions) {RUN_TEST(doc_mentions_cs_scratch_table_work);RUN_TEST(doc_mentions_incremental_non_ascii_name);RUN_TEST(doc_mentions_alloc_failure_status);+ RUN_TEST(doc_mentions_cs_scanner_output_reads_back);+ RUN_TEST(doc_mentions_cs_rejected_scope_contained);}