Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

doc-links(cs): a scope this run wrote that does not parse costs only its file; blobs are UTF-8 (S8, S10)

Martin Vogel429a40fcb372
diff --git a/internal/cbm/doclink_cs.c b/internal/cbm/doclink_cs.c
index 14c8a7b03..5996b0003 100644
--- a/internal/cbm/doclink_cs.c
+++ b/internal/cbm/doclink_cs.c
@@ -122,6 +122,69 @@ static void sb_putu(cs_sb_t *sb, uint32_t v) {
}
}
+/* ── Well-formed text ─────────────────────────────────────────────
+ *
+ * A scope blob is stored as a JSON string, and a JSON writer refuses a
+ * string that is not UTF-8: every byte the scans write is well-formed UTF-8.
+ * A name or a text of a C# source that is not is not kept (a name: the
+ * declaration is not placed; a text: "?"); project-file text gets U+FFFD for
+ * every byte that is no part of a well-formed sequence. Both are a function
+ * of the file's bytes alone. */
+
+static const char CS_REPLACEMENT[] = "\xEF\xBF\xBD"; /* U+FFFD */
+
+enum {
+ CS_UTF8_TAIL_MASK = 0xC0,
+ CS_UTF8_TAIL = 0x80,
+};
+
+/* Length of the well-formed UTF-8 sequence at s[0, n), 1 to 4; 0 when there
+ * is none: a stray continuation byte, an overlong form, a surrogate, a code
+ * point past U+10FFFF, a sequence cut short. */
+static size_t cs_utf8_len(const unsigned char *s, size_t n) {
+ if (n == 0) {
+ return 0;
+ }
+ unsigned char c = s[0];
+ if (c < 0x80) {
+ return SKIP_ONE;
+ }
+ size_t len = 0;
+ if (c >= 0xC2 && c <= 0xDF) {
+ len = PAIR_LEN;
+ } else if (c >= 0xE0 && c <= 0xEF) {
+ len = 3;
+ } else if (c >= 0xF0 && c <= 0xF4) {
+ len = 4;
+ }
+ if (len == 0 || n < len) {
+ return 0;
+ }
+ unsigned char c1 = s[1];
+ if ((c == 0xE0 && c1 < 0xA0) || (c == 0xED && c1 > 0x9F) || (c == 0xF0 && c1 < 0x90) ||
+ (c == 0xF4 && c1 > 0x8F)) {
+ return 0; /* overlong, surrogate, or past U+10FFFF */
+ }
+ for (size_t k = SKIP_ONE; k < len; k++) {
+ if ((s[k] & CS_UTF8_TAIL_MASK) != CS_UTF8_TAIL) {
+ return 0;
+ }
+ }
+ return len;
+}
+
+/* True when s[0, n) is well-formed UTF-8. */
+static bool cs_utf8_ok(const char *s, size_t n) {
+ for (size_t i = 0; i < n;) {
+ size_t l = cs_utf8_len((const unsigned char *)s + i, n - i);
+ if (l == 0) {
+ return false;
+ }
+ i += l;
+ }
+ return true;
+}
+
/* ── Doc-comment references ──────────────────────────────────────── */
static int cs_tag_syntax(const char *name, size_t len) {
@@ -939,9 +1002,10 @@ static TSNode cs_type_params(TSNode decl) {
/* Node text without whitespace and without verbatim '@' markers, appended to
* sb. `global::` prefixes are dropped. Text that is too long to be a type
- * name, or that holds a scope separator or non-whitespace control byte, is
- * written as "?" (an unresolvable name: the declaring type then counts as
- * having an open hierarchy). The whole field is replaced before any copy. */
+ * name, that holds a scope separator or non-whitespace control byte, or that
+ * is not well-formed UTF-8, is written as "?" (an unresolvable name: the
+ * declaring type then counts as having an open hierarchy). The whole field is
+ * replaced before any copy. */
static void cs_put_text_nows(cs_scan_t *s, TSNode n) {
const char *src = s->ctx->source;
uint32_t a = ts_node_start_byte(n);
@@ -955,7 +1019,7 @@ static void cs_put_text_nows(cs_scan_t *s, TSNode n) {
ok = c != '|' && c != ';' && c != '{' && c != '}' &&
!((c < 0x20 && !isspace(c)) || c == 0x7f);
}
- if (!ok) {
+ if (!ok || !cs_utf8_ok(src + a, b - a)) {
sb_putc(&s->sb, '?');
return;
}
@@ -978,8 +1042,9 @@ static void *cs_tmp_alloc(cs_scan_t *s, size_t n) {
}
/* Copy of source bytes [a, b) without whitespace and '@'. NULL unless it is a
- * (dotted) identifier of sane length: an error-recovered parse can hand back a
- * "name" spanning arbitrary code, which must not become a declaration. */
+ * (dotted) identifier of sane length and well-formed UTF-8: an error-recovered
+ * parse can hand back a "name" spanning arbitrary code, which must not become
+ * a declaration, and a name that is not UTF-8 cannot be written. */
static char *cs_ident_dup(cs_scan_t *s, uint32_t a, uint32_t b) {
const char *src = s->ctx->source;
if (b <= a || b - a > CS_NAME_MAX) {
@@ -1001,7 +1066,7 @@ static char *cs_ident_dup(cs_scan_t *s, uint32_t a, uint32_t b) {
out[w++] = (char)c;
}
out[w] = '\0';
- return w > 0 ? out : NULL;
+ return w > 0 && cs_utf8_ok(out, w) ? out : NULL;
}
static char *cs_name_dup(cs_scan_t *s, TSNode n) {
@@ -1073,6 +1138,9 @@ static void cs_put_sig(cs_scan_t *s, TSNode params) {
uint32_t a = ts_node_start_byte(ty);
uint32_t b = ts_node_end_byte(ty);
cbm_doclink_cs_norm_type(src + a, (size_t)(b - a), norm, sizeof(norm));
+ if (!cs_utf8_ok(norm, strlen(norm))) {
+ snprintf(norm, sizeof(norm), "?"); /* a type nothing is known about */
+ }
}
if (!first) {
sb_putc(&s->sb, '|');
@@ -1829,7 +1897,7 @@ static char *cs_namespace_name_dup(cs_scan_t *s, uint32_t a, uint32_t b) {
i = cs_skip_space(src, i, b);
if (i == b) {
out[w] = '\0';
- return out;
+ return cs_utf8_ok(out, w) ? out : NULL; /* a name that cannot be written */
}
if (src[i] != '.') {
return NULL;
@@ -1937,7 +2005,7 @@ static const char *cs_text_tparams(cs_scan_t *s, uint32_t *pos, uint32_t stop) {
if (c == '>') {
out[w] = '\0';
*pos = k + SKIP_ONE;
- return out;
+ return cs_utf8_ok(out, w) ? out : NULL; /* names that cannot be written */
}
} else if (cs_word_char((unsigned char)c)) {
uint32_t e = cs_ident_end(src, k, limit, false);
@@ -3301,9 +3369,12 @@ static void csx_put_char(cs_sb_t *sb, unsigned char c) {
}
}
-/* A code point as UTF-8. */
+/* A code point as UTF-8. A surrogate (a numeric reference may name one) is
+ * no character: it is written as U+FFFD. */
static void csx_put_codepoint(cs_sb_t *sb, uint32_t cp) {
- if (cp < 0x80) {
+ if (cp >= 0xD800 && cp <= 0xDFFF) {
+ sb_puts(sb, CS_REPLACEMENT);
+ } else if (cp < 0x80) {
csx_put_char(sb, (unsigned char)cp);
} else if (cp < 0x800) {
sb_putc(sb, (char)(0xC0 | (cp >> 6)));
@@ -3356,6 +3427,19 @@ static uint32_t csx_entity(const csx_t *x, uint32_t i, uint32_t end, uint32_t *c
return 0;
}
+/* The bytes src[i, end) of a project file that start at i into sb: a
+ * well-formed UTF-8 sequence as it stands, a byte that starts none as
+ * U+FFFD. Returns the index past what was taken. */
+static uint32_t csx_put_utf8(cs_sb_t *sb, const char *src, uint32_t i, uint32_t end) {
+ size_t len = cs_utf8_len((const unsigned char *)src + i, (size_t)(end - i));
+ if (len == 0) {
+ sb_puts(sb, CS_REPLACEMENT);
+ return i + SKIP_ONE;
+ }
+ sb_putn(sb, src + i, len);
+ return i + (uint32_t)len;
+}
+
/* The text of `v`, entities decoded (unless `raw`), escaped. */
static void csx_put_text(cs_sb_t *sb, const csx_t *x, csx_span_t v, bool raw) {
for (uint32_t i = v.s; i < v.e;) {
@@ -3364,6 +3448,8 @@ static void csx_put_text(cs_sb_t *sb, const csx_t *x, csx_span_t v, bool raw) {
if (past) {
csx_put_codepoint(sb, cp);
i = past;
+ } else if ((unsigned char)x->src[i] >= 0x80) {
+ i = csx_put_utf8(sb, x->src, i, v.e);
} else {
csx_put_char(sb, (unsigned char)x->src[i]);
i++;
@@ -3371,6 +3457,14 @@ static void csx_put_text(cs_sb_t *sb, const csx_t *x, csx_span_t v, bool raw) {
}
}
+/* An element's name into sb, each byte that starts no well-formed UTF-8
+ * sequence as U+FFFD (the tokenizer's names hold no separator). */
+static void csx_put_name(cs_sb_t *sb, const csx_t *x, csx_span_t name) {
+ for (uint32_t i = name.s; i < name.e;) {
+ i = csx_put_utf8(sb, x->src, i, name.e);
+ }
+}
+
/* A field: a tab, then nothing for an absent attribute, else '=' and its text. */
static void csx_put_field(cs_sb_t *sb, const csx_t *x, csx_span_t v) {
sb_putc(sb, '\t');
@@ -3435,7 +3529,7 @@ static void csx_put_property(csx_scan_t *p) {
sb_putc(&p->out, 'V');
csx_put_field(&p->out, &p->x, t->attr[CSX_A_CONDITION]);
sb_putc(&p->out, '\t');
- sb_putn(&p->out, p->x.src + t->name.s, t->name.e - t->name.s);
+ csx_put_name(&p->out, &p->x, t->name);
sb_putc(&p->out, '\t');
if (p->prop_complex) {
sb_putc(&p->out, '?');
@@ -3452,7 +3546,7 @@ static void csx_put_property(csx_scan_t *p) {
static void csx_choose_child(csx_scan_t *p, const csx_tok_t *t) {
if (p->choose_props >= 0 && p->depth == p->choose_props) {
sb_puts(&p->out, "K\t");
- sb_putn(&p->out, p->x.src + t->name.s, t->name.e - t->name.s);
+ csx_put_name(&p->out, &p->x, t->name);
sb_putc(&p->out, '\n');
} else if (csx_is(&p->x, t->name, "PropertyGroup") && p->choose_props < 0 &&
t->kind == CSX_OPEN) {
diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h
index 69fed4bd1..9a6452a77 100644
--- a/src/pipeline/doc_links.h
+++ b/src/pipeline/doc_links.h
@@ -222,6 +222,11 @@ uint64_t cbm_doclink_cs_test_scratch_work(void);
/* Test seam (doc_links.c): the nth MENTIONS edge insert from now on fails as
* if memory ran out (0: none). */
void cbm_doclinks_test_fail_edge_insert_after(int nth);
+/* Test seams (doc_links_cs.c): spoil the scope of the file at `rel_path`
+ * (every record, then one the reader refuses; NULL or "": none), and ask the
+ * reader whether it takes a scope blob. */
+void cbm_doclink_cs_test_spoil_scope(const char *rel_path);
+bool cbm_doclink_cs_test_scope_parses(const char *scope);
#endif
/* True when `rel_path` is a scope input of some language (scope_input). */
diff --git a/src/pipeline/doc_links_cs.c b/src/pipeline/doc_links_cs.c
index c957841e7..92ab111e6 100644
--- a/src/pipeline/doc_links_cs.c
+++ b/src/pipeline/doc_links_cs.c
@@ -335,6 +335,9 @@ typedef struct {
int *members_by_start;
cs_tparam_t *tparams; /* by (owner, name) */
int ntparams;
+ /* Its scope was written in this run and this reader refuses it: the file
+ * declares nothing here, and its own references are graph gaps. */
+ bool rejected;
} cs_file_t;
typedef struct {
@@ -468,8 +471,33 @@ typedef enum {
/* Counted while references are resolved (by every worker at once). */
typedef struct {
_Atomic uint64_t ambiguous[CS_WHY_COUNT];
+ _Atomic uint64_t rejected; /* references of files whose fresh scope was refused */
} cs_stats_t;
+/* What parsing one scope set in the tables all files share: so that a scope
+ * written in this run that this reader refuses can be taken back whole, and
+ * costs only its own file. Reset for every file. */
+typedef struct {
+ int ns;
+ bool declared;
+ bool prod;
+} cs_ns_was_t;
+
+typedef struct {
+ CBMHashTable *ht;
+ const char *key;
+} cs_mark_was_t;
+
+typedef struct {
+ int nnss; /* namespaces before the file */
+ cs_ns_was_t *flags;
+ int nflags;
+ int cap_flags;
+ cs_mark_was_t *marks;
+ int nmarks;
+ int cap_marks;
+} cs_undo_t;
+
typedef struct {
CBMArena arena;
bool oom; /* an index allocation failed */
@@ -517,6 +545,8 @@ typedef struct {
int nshared; /* shared trees */
cs_stats_t *stats;
bool bind_inherited; /* CS_BIND_INHERITED */
+ cs_undo_t undo; /* of the file being parsed */
+ int rejected; /* files whose scope, written in this run, was refused */
} cs_index_t;
#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
@@ -627,11 +657,56 @@ static bool ht_mark(cs_index_t *ix, CBMHashTable *ht, const char *key) {
return true;
}
+/* Grow an undo array to hold one more entry. false when memory ran out. */
+static bool undo_room(cs_index_t *ix, void **arr, int n, int *cap, size_t size) {
+ if (n < *cap) {
+ return true;
+ }
+ int ncap = *cap ? *cap * PAIR_LEN : CBM_SZ_16;
+ void *grown = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)ncap * size);
+ if (!grown) {
+ ix->oom = true;
+ return false;
+ }
+ if (n > 0) {
+ memcpy(grown, *arr, (size_t)n * size);
+ }
+ cbm_free(CBM_MEM_CLASS_OTHER, *arr);
+ *arr = grown;
+ *cap = ncap;
+ return true;
+}
+
+/* Remember the flags of namespace `ns` before the file's scope sets them. */
+static bool undo_note_ns(cs_index_t *ix, int ns) {
+ cs_undo_t *u = &ix->undo;
+ if (ns >= u->nnss) {
+ return true; /* made by this file: taken back as a whole */
+ }
+ if (!undo_room(ix, (void **)&u->flags, u->nflags, &u->cap_flags, sizeof(cs_ns_was_t))) {
+ return false;
+ }
+ u->flags[u->nflags++] =
+ (cs_ns_was_t){.ns = ns, .declared = ix->nss[ns].declared, .prod = ix->nss[ns].prod};
+ return true;
+}
+
/* A type name declared in `f` at a place no namespace or outer type could be
* established for. Product code never binds test-only declarations, so a name
* only test files quarantine blocks references from test files only. */
static bool quarantine_name(cs_index_t *ix, const cs_file_t *f, const char *name) {
- return ht_mark(ix, f->is_test ? ix->quarantine_test : ix->quarantine, name);
+ CBMHashTable *ht = f->is_test ? ix->quarantine_test : ix->quarantine;
+ if (cbm_ht_get(ht, name)) {
+ return true;
+ }
+ char *k = ix_strdup(ix, name);
+ cs_undo_t *u = &ix->undo;
+ if (!k || !undo_room(ix, (void **)&u->marks, u->nmarks, &u->cap_marks, sizeof(cs_mark_was_t))) {
+ return false;
+ }
+ cbm_ht_set(ht, k, (void *)k);
+ u->marks[u->nmarks++] = (cs_mark_was_t){.ht = ht, .key = k};
+ return true;
}
static bool cs_is_test_path(const char *rel) {
@@ -937,13 +1012,16 @@ static bool parse_region(cs_index_t *ix, cs_file_t *f, char **fld, int n, int *n
}
(*next_region)++;
int ns = ns_make_path(ix, f->regions[parent].ns, fld[5]);
- if (ns < 0) {
+ if (ns < 0 || !undo_note_ns(ix, ns)) {
return false;
}
ix->nss[ns].declared = true;
/* product code declares it, and with it every namespace above: one that
* is marked has its upper ones marked already */
for (int up = ns; !f->is_test && up > 0 && !ix->nss[up].prod; up = ix->nss[up].parent) {
+ if (!undo_note_ns(ix, up)) {
+ return false;
+ }
ix->nss[up].prod = true;
}
f->regions[id] = (cs_region_t){.parent = parent,
@@ -5621,6 +5699,18 @@ static void cs_destroy(void *index) {
if (ix->stats) {
log_ambiguous(ix->stats);
}
+ if (ix->rejected > 0) {
+ /* scopes written in this run that this reader refused: each cost
+ * only its own file, whose references are graph gaps */
+ char files[CBM_SZ_32];
+ char refs[CBM_SZ_32];
+ snprintf(files, sizeof(files), "%d", ix->rejected);
+ snprintf(refs, sizeof(refs), "%llu",
+ (unsigned long long)(ix->stats ? atomic_load(&ix->stats->rejected) : 0));
+ cbm_log_warn("doc_links.cs.rejected_scopes", "files", files, "references", refs);
+ }
+ cbm_free(CBM_MEM_CLASS_OTHER, ix->undo.flags);
+ cbm_free(CBM_MEM_CLASS_OTHER, ix->undo.marks);
cbm_free(CBM_MEM_CLASS_OTHER, ix->stats);
cbm_ht_free(ix->project_above);
cbm_ht_free(ix->ns_by_key);
@@ -5688,10 +5778,96 @@ static bool build_tables(cs_index_t *ix, const cbm_doclink_build_in_t *in) {
return collect_projects(ix, in);
}
-/* Every file of the index with what its scope declares. Returns the path of
- * a file whose scope is not one this code wrote (the index is not built over
- * such a scope: nothing may be resolved around a declaration that is not
- * known), "" when memory ran out, NULL when all is well. */
+/* Start the undo record of one file's scope. */
+static void undo_begin(cs_index_t *ix) {
+ ix->undo.nnss = ix->nnss;
+ ix->undo.nflags = 0;
+ ix->undo.nmarks = 0;
+}
+
+/* Take back what the file's scope set in the shared tables: the names it
+ * quarantined, the flags it set on namespaces that were there before it, and
+ * the namespaces it made. */
+static void undo_scope(cs_index_t *ix) {
+ cs_undo_t *u = &ix->undo;
+ for (int i = u->nmarks - SKIP_ONE; i >= 0; i--) {
+ cbm_ht_delete(u->marks[i].ht, u->marks[i].key);
+ }
+ for (int i = u->nflags - SKIP_ONE; i >= 0; i--) {
+ ix->nss[u->flags[i].ns].declared = u->flags[i].declared;
+ ix->nss[u->flags[i].ns].prod = u->flags[i].prod;
+ }
+ for (int id = ix->nnss - SKIP_ONE; id >= u->nnss; id--) {
+ char key[CS_NAME_BUF + CBM_SZ_16];
+ const cs_ns_t *n = &ix->nss[id];
+ if (ns_key(key, sizeof(key), n->parent, n->name, strlen(n->name))) {
+ cbm_ht_delete(ix->ns_by_key, key);
+ }
+ }
+ ix->nnss = u->nnss;
+ u->nflags = 0;
+ u->nmarks = 0;
+}
+
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+/* Test seam: the scope of the file at this path is spoiled -- every record
+ * it has, then one this reader refuses -- as if its writer and this reader
+ * disagreed after the file's declarations were read. "" or NULL: none. */
+static char cs_test_spoiled_path[CBM_SZ_512];
+
+void cbm_doclink_cs_test_spoil_scope(const char *rel_path) {
+ snprintf(cs_test_spoiled_path, sizeof(cs_test_spoiled_path), "%s", rel_path ? rel_path : "");
+}
+
+static const char *cs_test_spoiled(cs_index_t *ix, const char *rel_path, const char *scope) {
+ if (!scope || !cs_test_spoiled_path[0] || strcmp(rel_path, cs_test_spoiled_path) != 0) {
+ return scope;
+ }
+ const char *spoiled = cbm_arena_sprintf(&ix->arena, "%sZ\tspoiled\n", scope);
+ return spoiled ? spoiled : scope;
+}
+
+/* Test seam: true when this reader takes the scope blob `scope` (its record
+ * checks, on an index of its own). A test holds every blob the scanner
+ * writes against it. */
+bool cbm_doclink_cs_test_scope_parses(const char *scope) {
+ cs_index_t *ix = (cs_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*ix));
+ if (!ix) {
+ return false;
+ }
+ cbm_arena_init(&ix->arena);
+ ix->ns_by_key = cbm_ht_create(CBM_SZ_64);
+ ix->quarantine = cbm_ht_create(CBM_SZ_64);
+ ix->quarantine_test = cbm_ht_create(CBM_SZ_64);
+ ix->nss = (cs_ns_t *)ix_zalloc(ix, CBM_SZ_256 * sizeof(cs_ns_t));
+ bool ok = ix->ns_by_key && ix->quarantine && ix->quarantine_test && ix->nss;
+ if (ok) {
+ ix->nscap = CBM_SZ_256;
+ ix->nnss = SKIP_ONE;
+ ix->nss[0] = (cs_ns_t){.parent = CS_NONE, .name = ""};
+ cs_file_t f = {.rel_path = "Scope.cs"};
+ undo_begin(ix);
+ ok = parse_scope(ix, &f, scope) && !ix->oom;
+ }
+ cbm_ht_free(ix->ns_by_key);
+ cbm_ht_free(ix->quarantine);
+ cbm_ht_free(ix->quarantine_test);
+ cbm_free(CBM_MEM_CLASS_OTHER, ix->undo.flags);
+ cbm_free(CBM_MEM_CLASS_OTHER, ix->undo.marks);
+ cbm_arena_destroy(&ix->arena);
+ cbm_free(CBM_MEM_CLASS_OTHER, ix);
+ return ok;
+}
+#endif
+
+/* Every file of the index with what its scope declares. A scope read back
+ * from the store that this reader refuses is not one this code wrote: the
+ * index is not built over it (nothing may be resolved around a declaration
+ * that is not known), and its path is returned. A scope written in this run
+ * that the reader refuses is the scanner's and this reader's disagreement,
+ * which costs only its own file: what its parse set is taken back, the file
+ * declares nothing, and its references are graph gaps (counted, logged).
+ * Returns "" when memory ran out, NULL when all is well. */
static const char *build_files(cs_index_t *ix, const cbm_doclink_build_in_t *in) {
CBMHashTable *dir_unit = cbm_ht_create(CBM_SZ_1K);
const char *bad = dir_unit ? NULL : "";
@@ -5704,9 +5880,23 @@ static const char *build_files(cs_index_t *ix, const cbm_doclink_build_in_t *in)
f->is_test = cs_is_test_path(src->rel_path);
/* a project file declares nothing: its blob went to the evaluator */
const char *scope = cbm_msb_is_project_scope(src->scope) ? NULL : src->scope;
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+ scope = cs_test_spoiled(ix, src->rel_path, scope);
+#endif
+ undo_begin(ix);
if (scope && !parse_scope(ix, f, scope)) {
- bad = ix->oom ? "" : src->rel_path;
- break;
+ bool fresh = src->run_file >= 0 && src->run_file < in->run_file_count;
+ if (ix->oom || !fresh) {
+ bad = ix->oom ? "" : src->rel_path;
+ break;
+ }
+ undo_scope(ix);
+ *f = (cs_file_t){.rel_path = f->rel_path,
+ .module_qn = f->module_qn,
+ .is_test = f->is_test,
+ .rejected = true};
+ ix->rejected++;
+ scope = NULL;
}
if (!scope) {
/* no scope: an empty file (only its own region) */
@@ -5887,6 +6077,12 @@ static void cs_resolve(const void *index, int run_file, const CBMDocLink *link,
* declaration. Both are declared-but-unplaced, i.e. graph gaps. */
int file = ix->run_to_file[run_file];
const cs_file_t *f = &ix->files[file];
+ if (f->rejected) {
+ /* its scope was refused: nothing is known around the reference */
+ out->reason = CBM_DOCLINK_REASON_GRAPH_GAP;
+ atomic_fetch_add_explicit(&ix->stats->rejected, 1, memory_order_relaxed);
+ return;
+ }
bool file_doc = (link->flags & CBM_DOCLINK_FLAG_FILE) != 0;
if ((!file_doc && line_unplaced(f, link->def_line)) || names_quarantined(ix, f, &r)) {
out->reason = CBM_DOCLINK_REASON_GRAPH_GAP;
diff --git a/tests/test_doc_mentions.c b/tests/test_doc_mentions.c
index 0969fdc6b..bcad1ed4f 100644
--- a/tests/test_doc_mentions.c
+++ b/tests/test_doc_mentions.c
@@ -814,10 +814,16 @@ TEST(doc_mentions_cs_control_publication) {
PASS();
}
-/* Temporary observation: extract real source bytes, then exercise the public
- * surface codec with complete definitions and with each input isolated. */
+/* S10: extract real source bytes, then run the surface writer of the run
+ * with the file's complete definitions, without its scope, and with only
+ * its scope. Every run must succeed, twice alike, and the stored `dl` must
+ * be the portable scope. `expect`: 1 the inserted bytes stay in the scope
+ * (well-formed UTF-8), 0 they do not (malformed), -1 not asked (an entity). */
+static bool dm_scope_utf8_ok(const char *scope);
+
static bool dm_utf8_probe_surface(CBMFileResult *bad, CBMFileResult *healthy, CBMLanguage language,
- const char *path, const char *label, const char *bytes) {
+ const char *path, const char *label, const char *bytes,
+ int expect) {
CBMFileResult *cache[] = {bad, healthy};
cbm_file_info_t files[] = {{.rel_path = (char *)path, .language = language},
{.rel_path = "Healthy.cs", .language = CBM_LANG_CSHARP}};
@@ -865,20 +871,33 @@ static bool dm_utf8_probe_surface(CBMFileResult *bad, CBMFileResult *healthy, CB
int scope_rc =
cbm_lsp_surface_build_rows(NULL, "p", cache, files, 2, NULL, NULL, &rows, &count);
bool retained = saved_scope && strstr(saved_scope, bytes) != NULL;
- printf("doc_utf8_probe case=%s scope=%d retained=%d defs=%d full_rc=%d full_count=%d "
- "nodl_rc=%d nodl_count=%d scope_rc=%d scope_count=%d repeat=%d dl_equal=%d\n",
- label, saved_scope != NULL, retained, def_count, full_rc, full_count, nodl_rc,
- nodl_count, scope_rc, count, repeat, dl_equal);
+ bool written = full_rc == 0 && full_count == 2 && again_rc == 0 && dl_equal && nodl_rc == 0 &&
+ nodl_count == 2 && scope_rc == 0 && count == 2;
+ bool kept = expect < 0 || retained == (expect == 1);
+ bool utf8 = saved_scope && dm_scope_utf8_ok(saved_scope);
+ bool correct = saved_scope != NULL && def_count > 0 && repeat && written && kept && utf8;
+ if (!correct) {
+ printf("doc_utf8_probe case=%s scope=%d retained=%d utf8=%d defs=%d full_rc=%d "
+ "full_count=%d nodl_rc=%d nodl_count=%d scope_rc=%d scope_count=%d repeat=%d "
+ "dl_equal=%d\n",
+ label, saved_scope != NULL, retained, utf8, def_count, full_rc, full_count, nodl_rc,
+ nodl_count, scope_rc, count, repeat, dl_equal);
+ }
cbm_store_free_lsp_surfaces(rows, count);
cbm_free(CBM_MEM_CLASS_OTHER, portable);
free(defs);
free(modules[0]);
free(modules[1]);
cbm_arena_destroy(&arena);
- return saved_scope != NULL && def_count > 0 && repeat;
+ return correct;
}
-TEST(doc_mentions_cs_utf8_observation) {
+/* S10: a malformed byte sequence in any place a scope keeps text -- a name
+ * or a text of a C# source, an element, a value or an attribute of a project
+ * file, a numeric reference to a surrogate -- leaves a well-formed scope, and
+ * the surface writer of the run succeeds. Well-formed sequences stay as they
+ * are; malformed ones do not reach the scope. */
+TEST(doc_mentions_cs_utf8_surfaces) {
static const struct {
const char *label, *bytes;
} sequences[] = {
@@ -927,7 +946,6 @@ TEST(doc_mentions_cs_utf8_observation) {
int healthy_count = 0;
int healthy_rc = cbm_lsp_surface_build_rows(NULL, "p", &healthy, &healthy_file, 1, NULL, NULL,
&healthy_rows, &healthy_count);
- printf("doc_utf8_probe_healthy rc=%d count=%d\n", healthy_rc, healthy_count);
correct = correct && healthy_rc == 0 && healthy_count == 1;
cbm_store_free_lsp_surfaces(healthy_rows, healthy_count);
for (size_t p = 0; p < sizeof(positions) / sizeof(positions[0]); p++) {
@@ -947,8 +965,9 @@ TEST(doc_mentions_cs_utf8_observation) {
CBMFileResult *bad = cbm_extract_file(source, (int)(a + b + c + d),
positions[p].language, "p", path, 0, NULL, NULL);
snprintf(label, sizeof(label), "%s/%s", positions[p].label, sequences[s].label);
+ bool well_formed = s < 4; /* ascii, valid2, valid3, valid4 */
bool setup = bad && dm_utf8_probe_surface(bad, healthy, positions[p].language, path,
- label, sequences[s].bytes);
+ label, sequences[s].bytes, well_formed);
correct = setup && correct;
cbm_free_result(bad);
}
@@ -968,7 +987,7 @@ TEST(doc_mentions_cs_utf8_observation) {
snprintf(label, sizeof(label), "entity_%s/%zu", position == 0 ? "property" : "using",
e);
bool setup = bad && dm_utf8_probe_surface(bad, healthy, CBM_LANG_XML, "App.csproj",
- label, entities[e]);
+ label, entities[e], -1);
correct = setup && correct;
cbm_free_result(bad);
}
@@ -9527,6 +9546,214 @@ static int dm_alloc_failure_errors(int point) {
return errors;
}
+/* True when s[0, n) is well-formed UTF-8 (no surrogate, nothing past
+ * U+10FFFF, no overlong form, nothing cut short). */
+static bool dm_utf8_ok(const char *s, size_t n) {
+ const unsigned char *u = (const unsigned char *)s;
+ for (size_t i = 0; i < n;) {
+ unsigned char c = u[i];
+ size_t len = c < 0x80 ? 1
+ : (c >= 0xC2 && c <= 0xDF) ? 2
+ : (c >= 0xE0 && c <= 0xEF) ? 3
+ : (c >= 0xF0 && c <= 0xF4) ? 4
+ : 0;
+ if (len == 0 || i + len > n) {
+ return false;
+ }
+ if (len > 1) {
+ unsigned char c1 = u[i + 1];
+ if ((c == 0xE0 && c1 < 0xA0) || (c == 0xED && c1 > 0x9F) || (c == 0xF0 && c1 < 0x90) ||
+ (c == 0xF4 && c1 > 0x8F)) {
+ return false;
+ }
+ for (size_t k = 1; k < len; k++) {
+ if ((u[i + k] & 0xC0) != 0x80) {
+ return false;
+ }
+ }
+ }
+ i += len;
+ }
+ return true;
+}
+
+static bool dm_scope_utf8_ok(const char *scope) {
+ return dm_utf8_ok(scope, strlen(scope));
+}
+
+/* S8, S9, S10: every scope blob the scanners write passes the reader's own
+ * record checks, is a C string of the length that was built, and is
+ * well-formed UTF-8 -- for namespace names with empty segments, and for
+ * control bytes and malformed UTF-8 in every place a C# scope keeps text.
+ * A project file's blob is well-formed UTF-8 for malformed bytes and for
+ * numeric references to surrogates. */
+TEST(doc_mentions_cs_scanner_output_reads_back) {
+ static const struct {
+ const char *label, *bytes;
+ size_t n;
+ } bytes[] = {
+ {"nul", "\x00", 1},
+ {"soh", "\x01", 1},
+ {"us", "\x1F", 1},
+ {"del", "\x7F", 1},
+ {"continuation", "\x80", 1},
+ {"overlong", "\xC0\xAF", 2},
+ {"surrogate", "\xED\xA0\x80", 3},
+ {"above", "\xF4\x90\x80\x80", 4},
+ {"truncated", "\xE2\x82", 2},
+ {"valid", "\xC3\xA9", 2},
+ };
+ static const struct {
+ const char *label, *before, *after;
+ } places[] = {
+ {"using_target", "using Acme.", "Name;\nclass Local {}\n"},
+ {"alias_target", "using Alias = Acme.", "Name;\nclass Local {}\n"},
+ {"namespace_name", "namespace N", "Name { class Local {} }\n"},
+ {"type_name", "class N", "Name {}\n"},
+ {"method_name", "class Local { void N", "Name() {} }\n"},
+ {"parameter_type", "class Local { void M(N", "Name arg) {} }\n"},
+ {"type_parameter", "class Local<N", "Name> {}\n"},
+ {"method_type_parameter", "class Local { void M<N", "Name>() {} }\n"},
+ {"field_name", "class Local { int N", "Name; }\n"},
+ {"base_type", "class Local : Acme.N", "Name {}\n"},
+ {"broken_file", "class Local { void M( { N", "Name } }\n"},
+ };
+ bool ok = true;
+ for (size_t p = 0; p < sizeof(places) / sizeof(places[0]); p++) {
+ for (size_t b = 0; b < sizeof(bytes) / sizeof(bytes[0]); b++) {
+ char source[512];
+ size_t a = strlen(places[p].before);
+ size_t c = strlen(places[p].after);
+ memcpy(source, places[p].before, a);
+ memcpy(source + a, bytes[b].bytes, bytes[b].n);
+ memcpy(source + a + bytes[b].n, places[p].after, c);
+ size_t len = a + bytes[b].n + c;
+ source[len] = '\0';
+ cbm_doclink_cs_test_cost_reset();
+ CBMFileResult *r =
+ cbm_extract_file(source, (int)len, CBM_LANG_CSHARP, "p", "Bad.cs", 0, NULL, NULL);
+ const char *scope = r ? r->doc_scope : NULL;
+ size_t built = (size_t)cbm_doclink_cs_test_scope_bytes();
+ bool good = scope && strlen(scope) == built && dm_utf8_ok(scope, strlen(scope)) &&
+ cbm_doclink_cs_test_scope_parses(scope);
+ if (!good) {
+ printf(" %s/%s: scope=%d length=%zu built=%zu utf8=%d reads=%d\n", places[p].label,
+ bytes[b].label, scope != NULL, scope ? strlen(scope) : (size_t)0, built,
+ scope ? dm_utf8_ok(scope, strlen(scope)) : 0,
+ scope ? cbm_doclink_cs_test_scope_parses(scope) : 0);
+ ok = false;
+ }
+ cbm_free_result(r);
+ }
+ }
+ for (size_t i = 0; i < sizeof(dm_namespace_cases) / sizeof(dm_namespace_cases[0]); i++) {
+ for (int file_scoped = 0; file_scoped < 2; file_scoped++) {
+ char source[2048];
+ if (!dm_namespace_source(source, sizeof(source), i, file_scoped != 0)) {
+ ok = false;
+ continue;
+ }
+ CBMFileResult *r = dm_extract(source, CBM_LANG_CSHARP, "Bad.cs");
+ const char *scope = r ? r->doc_scope : NULL;
+ if (!scope || !cbm_doclink_cs_test_scope_parses(scope)) {
+ printf(" namespace %s/%s: the reader refuses the scope\n",
+ dm_namespace_cases[i].id, file_scoped ? "file" : "block");
+ ok = false;
+ }
+ cbm_free_result(r);
+ }
+ }
+ static const char *xml[] = {
+ "<Project><PropertyGroup><P>N\x80Name</P></PropertyGroup></Project>",
+ "<Project><PropertyGroup><P\xC3\x28>Value</P\xC3\x28></PropertyGroup></Project>",
+ "<Project><PropertyGroup><P>N&#xD800;Name&#55296;</P></PropertyGroup></Project>",
+ "<Project><ItemGroup><Using Include=\"Acme.N\xED\xA0\x80Name\" /></ItemGroup></Project>",
+ "<Project><PropertyGroup Condition=\"'\xF4\x90\x80\x80'=='x'\"><P>V</P></PropertyGroup>"
+ "</Project>",
+ "<Project><Choose><When Condition=\"true\"><PropertyGroup><Q\x80>V</Q\x80>"
+ "</PropertyGroup></When></Choose></Project>",
+ };
+ for (size_t i = 0; i < sizeof(xml) / sizeof(xml[0]); i++) {
+ CBMFileResult *r = cbm_extract_file(xml[i], (int)strlen(xml[i]), CBM_LANG_XML, "p",
+ "App.csproj", 0, NULL, NULL);
+ const char *scope = r ? r->doc_scope : NULL;
+ if (!scope || !dm_utf8_ok(scope, strlen(scope))) {
+ printf(" project file %zu: blob=%d utf8=%d\n", i, scope != NULL,
+ scope ? dm_utf8_ok(scope, strlen(scope)) : 0);
+ ok = false;
+ }
+ cbm_free_result(r);
+ }
+ ASSERT_TRUE(ok);
+ PASS();
+}
+
+/* S8: a scope written in this run that the reader refuses costs only its own
+ * file. The status stays ok and the other file's edges are there; the file's
+ * own reference is a graph gap; and nothing its parse set before the refusal
+ * stays -- a namespace it declares does not stand beside a type of that name
+ * (Shadow, Good.Lurker), and a name it quarantined does not block another
+ * file's declaration (Hidden2). */
+TEST(doc_mentions_cs_rejected_scope_contained) {
+ static const dm_source_t files[] = {
+ {"Bad.cs", "namespace Shadow\n"
+ "{\n"
+ " public class Inner { }\n"
+ "}\n"
+ "namespace Good\n"
+ "{\n"
+ " namespace Lurker { public class Deep { } }\n"
+ "\n"
+ " /// <summary><see cref=\"Target\"/></summary>\n"
+ " public class FromBad { }\n"
+ "}\n"
+ "namespace .Broken\n"
+ "{\n"
+ " public class Hidden2 { }\n"
+ "}\n"},
+ {"Healthy.cs", "public class Shadow { }\n"
+ "namespace Good\n"
+ "{\n"
+ " public class Target { }\n"
+ " public class Lurker { }\n"
+ " public class Hidden2 { }\n"
+ "\n"
+ " /// <summary><see cref=\"Target\"/> <see cref=\"Shadow\"/>\n"
+ " /// <see cref=\"Lurker\"/> <see cref=\"Hidden2\"/></summary>\n"
+ " public class Uses { }\n"
+ "}\n"},
+ };
+ static const dm_want_t wants[] = {
+ {"Healthy.cs", "Target", "Healthy.Uses", "Healthy.Target", NULL, NULL, NULL},
+ {"Healthy.cs", "Shadow", "Healthy.Uses", "Healthy.Shadow", NULL, NULL, NULL},
+ {"Healthy.cs", "Lurker", "Healthy.Uses", "Healthy.Lurker", NULL, NULL, NULL},
+ {"Healthy.cs", "Hidden2", "Healthy.Uses", "Healthy.Hidden2", NULL, NULL, NULL},
+ {"Bad.cs", "Target", "Bad.FromBad", NULL, "graph_gap", "Healthy.Target", NULL},
+ };
+ char tmp[256];
+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_dm_rejected_XXXXXX");
+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
+ for (int i = 0; i < DM_COUNT(files); i++) {
+ th_write_file(TH_PATH(tmp, files[i].path), files[i].text);
+ }
+ char db[512];
+ snprintf(db, sizeof(db), "%s/rejected.db", tmp);
+ cbm_doclink_cs_test_spoil_scope("Bad.cs");
+ int bad = dm_index(tmp, db, NULL) == 0 ? 0 : -1;
+ cbm_doclink_cs_test_spoil_scope(NULL);
+ int errors = bad == 0 ? dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved "
+ "WHERE reason = 'error'")
+ : -1;
+ for (int i = 0; bad >= 0 && i < DM_COUNT(wants); i++) {
+ bad += dm_want_failed(db, &wants[i]);
+ }
+ dm_unlink_db(db);
+ th_rmtree(tmp);
+ ASSERT_EQ(errors, 0);
+ ASSERT_EQ(bad, 0);
+ PASS();
+}
+
TEST(doc_mentions_alloc_failure_status) {
static const struct {
int point;
@@ -9557,7 +9784,7 @@ SUITE(doc_mentions) {
RUN_TEST(doc_mentions_cs_namespace_boundaries);
RUN_TEST(doc_mentions_cs_control_scopes);
RUN_TEST(doc_mentions_cs_control_publication);
- RUN_TEST(doc_mentions_cs_utf8_observation);
+ RUN_TEST(doc_mentions_cs_utf8_surfaces);
RUN_TEST(doc_mentions_cs_namespace_publication);
RUN_TEST(doc_mentions_cs_scope_parse_errors);
RUN_TEST(doc_mentions_cs_norm_type);
@@ -9659,4 +9886,6 @@ SUITE(doc_mentions) {
RUN_TEST(doc_mentions_cs_scratch_table_work);
RUN_TEST(doc_mentions_incremental_non_ascii_name);
RUN_TEST(doc_mentions_alloc_failure_status);
+ RUN_TEST(doc_mentions_cs_scanner_output_reads_back);
+ RUN_TEST(doc_mentions_cs_rejected_scope_contained);
}