Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
feat(semantic): doc candidates by format and position; whole files and home folders
Martin Vogel75588a2b6915
diff --git a/src/mcp/mcp.c b/src/mcp/mcp.cindex 3321eb1f6..441b3d0e3 100644--- a/src/mcp/mcp.c+++ b/src/mcp/mcp.c@@ -12859,68 +12859,154 @@ static char *snippet_budget_floor(bool json_format, int max_output_tokens) {return json;}-/* Doc -> code candidates (doc_link_candidates): on a Section, the functions it- * is possibly about; on a function or method, the sections that possibly- * describe it -- highest p first. Candidates with a judged probability for the- * agent to verify, never links. A row whose other end no longer resolves (a- * delta update renamed or removed it) is skipped. */-enum { MCP_DOC_CANDIDATES_MAX = 5 };-static void add_doc_candidates(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store,- const cbm_node_t *node) {- if (!store || !node->label || !node->qualified_name || !node->project) {- return;- }- bool section = strcmp(node->label, "Section") == 0;- if (!section && strcmp(node->label, "Function") != 0 && strcmp(node->label, "Method") != 0) {- return;- }- cbm_doc_candidate_t *rows = NULL;- int n = 0;- if (cbm_store_doc_candidates_get(store, node->project, section ? node->qualified_name : NULL,- section ? NULL : node->qualified_name, MCP_DOC_CANDIDATES_MAX,- &rows, &n) != CBM_STORE_OK) {- return;- }- yyjson_mut_val *arr = yyjson_mut_arr(doc);+/* Doc -> code candidates (doc_link_candidates): on a Section, the functions,+ * whole files and the home folder it is possibly about; on a function or+ * method, the sections that possibly describe it; on a file (get_file_outline),+ * the sections that possibly describe the file or a folder holding it --+ * highest p first. Candidates with a judged probability for the agent to+ * verify, never links. A row whose other end no longer resolves (a delta+ * update renamed or removed it) is skipped. */+enum { MCP_DOC_CANDIDATES_MAX = 5, MCP_DOC_CANDIDATES_SECTION_MAX = 10 };+static const char MCP_DOC_CANDIDATES_NOTE[] =+ "doc<->code candidates by shared terms or the doc's folder, not links: p is the judged "+ "probability the section is about the target; verify";++/* The evidence fields of a candidate row as shown: kind, position, and (on+ * the section side) the shared terms and name tokens. */+static void doc_candidate_evidence(yyjson_mut_doc *doc, yyjson_mut_val *o, const char *evidence,+ bool counts) {+ yyjson_doc *ev = evidence ? yyjson_read(evidence, strlen(evidence), 0) : NULL;+ yyjson_val *root = ev ? yyjson_doc_get_root(ev) : NULL;+ static const char *const STRS[] = {"kind", "position", NULL};+ for (int k = 0; root && STRS[k]; k++) {+ yyjson_val *v = yyjson_obj_get(root, STRS[k]);+ if (v && yyjson_is_str(v)) {+ yyjson_mut_obj_add_strcpy(doc, o, STRS[k], yyjson_get_str(v));+ }+ }+ static const char *const INTS[] = {"shared_terms", "name_tokens", NULL};+ for (int k = 0; counts && root && INTS[k]; k++) {+ yyjson_val *v = yyjson_obj_get(root, INTS[k]);+ if (v && yyjson_is_int(v)) {+ yyjson_mut_obj_add_int(doc, o, INTS[k], (int)yyjson_get_int(v));+ }+ }+ yyjson_doc_free(ev);+}++/* rows -> an array of shown candidates; the other end of each row is its+ * target (by_section) or its section. Returns how many resolved. */+static int doc_candidates_array(yyjson_mut_doc *doc, yyjson_mut_val *arr, cbm_store_t *store,+ const char *project, const cbm_doc_candidate_t *rows, int n,+ bool by_section) {int added = 0;for (int i = 0; i < n; i++) {cbm_node_t other = {0};- if (cbm_store_find_node_by_qn(store, node->project,- section ? rows[i].target_qn : rows[i].section_qn,+ if (cbm_store_find_node_by_qn(store, project,+ by_section ? rows[i].target_qn : rows[i].section_qn,&other) != CBM_STORE_OK) {continue;}yyjson_mut_val *o = yyjson_mut_obj(doc);- yyjson_mut_obj_add_strcpy(doc, o, section ? "qualified_name" : "section",+ yyjson_mut_obj_add_strcpy(doc, o, by_section ? "qualified_name" : "section",other.qualified_name ? other.qualified_name : "");+ if (by_section) {+ yyjson_mut_obj_add_strcpy(doc, o, "label", other.label ? other.label : "");+ }yyjson_mut_obj_add_strcpy(doc, o, "file_path", other.file_path ? other.file_path : "");yyjson_mut_obj_add_int(doc, o, "start_line", other.start_line);- if (section) {+ if (by_section) {yyjson_mut_obj_add_int(doc, o, "end_line", other.end_line);}yyjson_mut_obj_add_real(doc, o, "p", (double)(int)(rows[i].p * 1000.0 + 0.5) / 1000.0);- int shared = 0;- int name_tokens = 0;- if (section && rows[i].evidence &&- sscanf(rows[i].evidence, "{\"shared_terms\":%d,\"name_tokens\":%d}", &shared,- &name_tokens) == 2) {- yyjson_mut_obj_add_int(doc, o, "shared_terms", shared);- yyjson_mut_obj_add_int(doc, o, "name_tokens", name_tokens);- }+ doc_candidate_evidence(doc, o, rows[i].evidence, by_section);yyjson_mut_arr_append(arr, o);added++;free_node_contents(&other);}+ return added;+}++static void add_doc_candidates(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store,+ const cbm_node_t *node) {+ if (!store || !node->label || !node->qualified_name || !node->project) {+ return;+ }+ bool section = strcmp(node->label, "Section") == 0;+ if (!section && strcmp(node->label, "Function") != 0 && strcmp(node->label, "Method") != 0) {+ return;+ }+ cbm_doc_candidate_t *rows = NULL;+ int n = 0;+ if (cbm_store_doc_candidates_get(store, node->project, section ? node->qualified_name : NULL,+ section ? NULL : node->qualified_name,+ section ? MCP_DOC_CANDIDATES_SECTION_MAX+ : MCP_DOC_CANDIDATES_MAX,+ &rows, &n) != CBM_STORE_OK) {+ return;+ }+ yyjson_mut_val *arr = yyjson_mut_arr(doc);+ int added = doc_candidates_array(doc, arr, store, node->project, rows, n, section);cbm_store_doc_candidates_free(rows, n);if (added > 0) {yyjson_mut_obj_add_val(doc, root, section ? "possibly_about" : "possibly_described_in",arr);- yyjson_mut_obj_add_str(doc, root, "candidates_note",- "doc<->code candidates by shared terms, not links: p is the "- "judged probability the section is about the function; verify");+ yyjson_mut_obj_add_str(doc, root, "candidates_note", MCP_DOC_CANDIDATES_NOTE);}}+/* get_file_outline: the sections possibly about the file as a whole or about+ * a folder holding it (NULL: none). */+static yyjson_mut_val *file_doc_candidates(yyjson_mut_doc *doc, cbm_store_t *store,+ const char *project, const char *path) {+ cbm_doc_candidate_t *rows = NULL;+ int n = 0;+ if (cbm_store_doc_candidates_for_path(store, project, path, MCP_DOC_CANDIDATES_MAX, &rows,+ &n) != CBM_STORE_OK) {+ return NULL;+ }+ yyjson_mut_val *arr = yyjson_mut_arr(doc);+ int added = doc_candidates_array(doc, arr, store, project, rows, n, false);+ cbm_store_doc_candidates_free(rows, n);+ return added > 0 ? arr : NULL;+}++static const char *mut_str(yyjson_mut_val *o, const char *key) {+ yyjson_mut_val *v = yyjson_mut_obj_get(o, key);+ return v && yyjson_mut_is_str(v) ? yyjson_mut_get_str(v) : "";+}++/* The tree form of file_doc_candidates. */+static void file_doc_candidates_tree(cbm_sb_t *sb, cbm_store_t *store, const char *project,+ const char *path) {+ yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);+ yyjson_mut_val *arr = doc ? file_doc_candidates(doc, store, project, path) : NULL;+ if (arr) {+ static const char *const columns[] = {"section", "file_path", "line", "p", "kind"};+ cbm_tree_table_header(sb, "possibly_described_in", (int)yyjson_mut_arr_size(arr), columns,+ (int)(sizeof(columns) / sizeof(columns[0])));+ size_t i = 0;+ size_t max = 0;+ yyjson_mut_val *o = NULL;+ yyjson_mut_arr_foreach(arr, i, max, o) {+ char line[CBM_SZ_32];+ char p[CBM_SZ_32];+ snprintf(line, sizeof(line), "%d",+ (int)yyjson_mut_get_int(yyjson_mut_obj_get(o, "start_line")));+ snprintf(p, sizeof(p), "%.2f", yyjson_mut_get_real(yyjson_mut_obj_get(o, "p")));+ cbm_tree_row_begin(sb);+ cbm_tree_cell_str(sb, mut_str(o, "section"), true);+ cbm_tree_cell_str(sb, mut_str(o, "file_path"), false);+ cbm_tree_cell_str(sb, line, false);+ cbm_tree_cell_str(sb, p, false);+ cbm_tree_cell_str(sb, mut_str(o, "kind"), false);+ cbm_tree_row_end(sb);+ }+ cbm_tree_scalar_str(sb, "candidates_note", MCP_DOC_CANDIDATES_NOTE);+ }+ yyjson_mut_doc_free(doc);+}+static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node,const char *match_method, bool include_neighbors,cbm_node_t *alternatives, int alt_count, const char *args) {@@ -13404,6 +13490,9 @@ static char *handle_get_file_outline(cbm_mcp_server_t *srv, const char *args) {cbm_tree_scalar_int(&sb, "limit", limit);cbm_tree_scalar_int(&sb, "returned", row_count);cbm_tree_scalar_bool(&sb, "has_more", (int64_t)offset + row_count < total);+ if (offset == 0) {+ file_doc_candidates_tree(&sb, store, project, normalized_path);+ }payload = cbm_sb_finish(&sb);} else {yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);@@ -13439,6 +13528,12 @@ static char *handle_get_file_outline(cbm_mcp_server_t *srv, const char *args) {yyjson_mut_obj_add_int(doc, object, "limit", limit);yyjson_mut_obj_add_int(doc, object, "returned", row_count);yyjson_mut_obj_add_bool(doc, object, "has_more", (int64_t)offset + row_count < total);+ yyjson_mut_val *described =+ offset == 0 ? file_doc_candidates(doc, store, project, normalized_path) : NULL;+ if (described) {+ yyjson_mut_obj_add_val(doc, object, "possibly_described_in", described);+ yyjson_mut_obj_add_str(doc, object, "candidates_note", MCP_DOC_CANDIDATES_NOTE);+ }payload = yy_doc_to_str(doc);yyjson_mut_doc_free(doc);}diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.cindex b0cd4dd3f..d406306e3 100644--- a/src/pipeline/pass_semantic_edges.c+++ b/src/pipeline/pass_semantic_edges.c@@ -1920,9 +1920,20 @@ typedef struct {float name_idf; /* the highest idf among them (0: none) */} doc_hit_t;+typedef struct {+ int file;+ float tfidf;+ int rank;+ int shared;+} doc_file_hit_t;+typedef struct {doc_hit_t hits[SEM_DOC_TOP_K];int n;+ doc_hit_t local[CBM_SEM_DOC_LOCAL_K]; /* inside the home, below the top CBM_SEM_DOC_TOP_K */+ int nlocal;+ doc_file_hit_t files[CBM_SEM_DOC_FILE_K];+ int nfiles;} doc_result_t;static int cmp_doc_cand(const void *pa, const void *pb) {@@ -2062,11 +2073,11 @@ static bool doc_section_gated(const cbm_gbuf_node_t *sec, char *body) {return words >= SEM_DOC_MIN_WORDS && links * 100 < SEM_DOC_LINK_PCT_MAX * words;}-/* A test function, by its file (test directories, test file names) or name. */-static bool doc_is_test_function(const cbm_gbuf_node_t *fn) {+/* A test file: in a test directory, or a test file name. */+static bool doc_is_test_path(const char *path) {static const char *const DIRS[] = {"test/", "tests/", "__tests__/", "testing/", "spec/","specs/", "testdata/", "fixture/", "fixtures/", NULL};- const char *path = fn && fn->file_path ? fn->file_path : "";+ path = path ? path : "";for (int k = 0; DIRS[k]; k++) {for (const char *p = path; (p = strstr(p, DIRS[k])) != NULL; p++) {if (p == path || p[-SKIP_ONE] == '/') {@@ -2076,9 +2087,14 @@ static bool doc_is_test_function(const cbm_gbuf_node_t *fn) {}const char *base = strrchr(path, '/');base = base ? base + SKIP_ONE : path;- if (strncmp(base, "test_", sizeof("test_") - SKIP_ONE) == 0 || strstr(base, "_test.") ||- strstr(base, ".test.") || strstr(base, ".spec.") || strstr(base, "Test.") ||- strstr(base, "Tests.")) {+ return strncmp(base, "test_", sizeof("test_") - SKIP_ONE) == 0 || strstr(base, "_test.") ||+ strstr(base, ".test.") || strstr(base, ".spec.") || strstr(base, "Test.") ||+ strstr(base, "Tests.");+}++/* A test function, by its file or name. */+static bool doc_is_test_function(const cbm_gbuf_node_t *fn) {+ if (doc_is_test_path(fn ? fn->file_path : NULL)) {return true;}const char *name = fn && fn->name ? fn->name : "";@@ -2126,6 +2142,449 @@ static float terms_norm(const float *w, int n) {return sqrtf(s);}+/* ── Phase 6c: whole files and the doc's home folder ── */++/* Whole-file candidates: a file's terms are the sum of its non-test+ * functions' TF-IDF vectors and the TF-IDF vectors of its other definitions'+ * names (a config file's keys are Variable nodes), cut to its+ * SEM_DOC_FILE_TERMS heaviest terms (ties by token index), so a file's vector+ * costs at most what one long function's does. */+enum { SEM_DOC_FILE_TERMS = 64 };+static const char *const DOC_FILE_DEF_LABELS[] = {"Class", "Struct", "Interface", "Enum",+ "Type", "Trait", "Field", "Variable",+ "Constant", "Macro", NULL};++typedef struct {+ int count;+ const cbm_gbuf_node_t **node; /* per file: its File node */+ int *off; /* count + 1: each file's terms in idx / w */+ int *idx;+ float *w;+ float *norm;+ int *post_off; /* nterms + 1: the files holding each term */+ int *post_f;+ CBMHashTable *by_path; /* file path (graph-owned) -> file index + 1 */+} doc_files_t;++typedef struct {+ int idx;+ float w;+} doc_term_t;++static int cmp_term_idx(const void *pa, const void *pb) {+ const doc_term_t *a = pa;+ const doc_term_t *b = pb;+ return (a->idx > b->idx) - (a->idx < b->idx);+}++static int cmp_term_heavy(const void *pa, const void *pb) {+ const doc_term_t *a = pa;+ const doc_term_t *b = pb;+ if (a->w != b->w) {+ return a->w > b->w ? -1 : 1;+ }+ return (a->idx > b->idx) - (a->idx < b->idx);+}++static void doc_files_free(doc_files_t *df) {+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->node);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->off);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->idx);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->w);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->norm);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->post_off);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->post_f);+ if (df->by_path) {+ cbm_ht_free(df->by_path);+ }+ memset(df, 0, sizeof(*df));+}++static int doc_file_of(const doc_files_t *df, const char *path) {+ return path ? (int)(intptr_t)cbm_ht_get(df->by_path, path) - SKIP_ONE : -SKIP_ONE;+}++/* The scratch terms of one file: append n terms, growing the buffer. */+typedef struct {+ doc_term_t *t;+ int n;+ int cap;+} doc_scratch_t;++static bool doc_scratch_add(doc_scratch_t *sc, const int *idx, const float *w, int n) {+ if (sc->n + n > sc->cap) {+ int cap = sc->cap ? sc->cap : CBM_SZ_256;+ while (cap < sc->n + n) {+ cap *= 2;+ }+ doc_term_t *t = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, sc->t, (size_t)cap * sizeof(*t));+ if (!t) {+ return false;+ }+ sc->t = t;+ sc->cap = cap;+ }+ for (int k = 0; k < n; k++) {+ sc->t[sc->n++] = (doc_term_t){.idx = idx[k], .w = w[k]};+ }+ return true;+}++/* Sum the scratch terms per token and keep the heaviest SEM_DOC_FILE_TERMS,+ * written at df->idx/w + off in ascending token order; returns how many. */+static int doc_scratch_reduce(doc_scratch_t *sc, doc_files_t *df, int off) {+ if (sc->n == 0) {+ return 0;+ }+ qsort(sc->t, (size_t)sc->n, sizeof(*sc->t), cmp_term_idx);+ int m = 0;+ for (int k = 0; k < sc->n; k++) {+ if (m > 0 && sc->t[m - SKIP_ONE].idx == sc->t[k].idx) {+ sc->t[m - SKIP_ONE].w += sc->t[k].w;+ } else {+ sc->t[m++] = sc->t[k];+ }+ }+ if (m > SEM_DOC_FILE_TERMS) {+ qsort(sc->t, (size_t)m, sizeof(*sc->t), cmp_term_heavy);+ m = SEM_DOC_FILE_TERMS;+ qsort(sc->t, (size_t)m, sizeof(*sc->t), cmp_term_idx);+ }+ for (int k = 0; k < m; k++) {+ df->idx[off + k] = sc->t[k].idx;+ df->w[off + k] = sc->t[k].w;+ }+ return m;+}++/* The definitions (non-function) of every indexed non-test file, grouped by+ * file: def_off (count + 1) into def_node. */+static bool doc_files_defs(const cbm_gbuf_t *gbuf, const doc_files_t *df, int **def_off,+ const cbm_gbuf_node_t ***def_node) {+ *def_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(int));+ if (!*def_off) {+ return false;+ }+ for (int pass = 0; pass < 2; pass++) {+ int *cursor =+ pass ? cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)df->count * sizeof(int)) : NULL;+ if (pass && !cursor) {+ return false;+ }+ for (int l = 0; DOC_FILE_DEF_LABELS[l]; l++) {+ const cbm_gbuf_node_t **nodes = NULL;+ int n = 0;+ if (cbm_gbuf_find_by_label(gbuf, DOC_FILE_DEF_LABELS[l], &nodes, &n) != 0) {+ continue;+ }+ for (int i = 0; i < n; i++) {+ int f = nodes[i]->name ? doc_file_of(df, nodes[i]->file_path) : -SKIP_ONE;+ if (f >= 0 && !pass) {+ (*def_off)[f + SKIP_ONE]++;+ } else if (f >= 0) {+ (*def_node)[(*def_off)[f] + cursor[f]++] = nodes[i];+ }+ }+ }+ if (!pass) {+ for (int f = 0; f < df->count; f++) {+ (*def_off)[f + SKIP_ONE] += (*def_off)[f];+ }+ *def_node = cbm_alloc(CBM_MEM_CLASS_SEMANTIC,+ ((size_t)(*def_off)[df->count] + SKIP_ONE) * sizeof(**def_node));+ if (!*def_node) {+ return false;+ }+ }+ cbm_free(CBM_MEM_CLASS_SEMANTIC, cursor);+ }+ return true;+}++/* The scratch terms of one definition's name. */+static bool doc_scratch_name(doc_scratch_t *sc, const cbm_sem_corpus_t *corpus, const char *name) {+ char *tok[SEM_DOC_NAME_TOKENS];+ int idx[SEM_DOC_NAME_TOKENS];+ float w[SEM_DOC_NAME_TOKENS];+ int nt = cbm_sem_tokenize(name, tok, SEM_DOC_NAME_TOKENS);+ int nw = nt > 0 ? cbm_sem_tfidf_terms(corpus, tok, NULL, nt, idx, w) : 0;+ for (int t = 0; t < nt; t++) {+ cbm_free(CBM_MEM_CLASS_SEMANTIC, tok[t]);+ }+ return doc_scratch_add(sc, idx, w, nw);+}++/* Each file's term vector, norm, and the term -> files postings. */+static bool doc_files_vectors(const cbm_gbuf_t *gbuf, doc_files_t *df, const cbm_sem_func_t *funcs,+ const int *ffile, int func_count, const cbm_sem_corpus_t *corpus,+ int nterms) {+ int *fn_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(int));+ int *fn_list = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)func_count + SKIP_ONE) * sizeof(int));+ int *def_off = NULL;+ const cbm_gbuf_node_t **def_node = NULL;+ doc_scratch_t sc = {0};+ bool ok = fn_off && fn_list && doc_files_defs(gbuf, df, &def_off, &def_node);+ for (int f = 0; ok && f < func_count; f++) {+ fn_off[ffile[f] + SKIP_ONE] += ffile[f] >= 0;+ }+ for (int i = 0; ok && i < df->count; i++) {+ fn_off[i + SKIP_ONE] += fn_off[i];+ }+ for (int f = 0; ok && f < func_count; f++) {+ if (ffile[f] >= 0) {+ fn_list[fn_off[ffile[f]]++] = f; /* fn_off[i] ends at file i's end: shifted below */+ }+ }+ for (int i = df->count; ok && i > 0; i--) {+ fn_off[i] = fn_off[i - SKIP_ONE];+ }+ if (ok) {+ fn_off[0] = 0;+ }+ for (int i = 0; ok && i < df->count; i++) {+ sc.n = 0;+ for (int k = fn_off[i]; ok && k < fn_off[i + SKIP_ONE]; k++) {+ const cbm_sem_func_t *fn = &funcs[fn_list[k]];+ ok = doc_scratch_add(&sc, fn->tfidf_indices, fn->tfidf_weights, fn->tfidf_len);+ }+ for (int k = def_off[i]; ok && k < def_off[i + SKIP_ONE]; k++) {+ ok = doc_scratch_name(&sc, corpus, def_node[k]->name);+ }+ int m = ok ? doc_scratch_reduce(&sc, df, df->off[i]) : 0;+ df->off[i + SKIP_ONE] = df->off[i] + m;+ df->norm[i] = terms_norm(df->w + df->off[i], m);+ for (int k = 0; k < m; k++) {+ df->post_off[df->idx[df->off[i] + k] + SKIP_ONE]++;+ }+ }+ cbm_free(CBM_MEM_CLASS_SEMANTIC, sc.t);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, fn_off);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, fn_list);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, def_off);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, def_node);+ if (!ok) {+ return false;+ }+ for (int t = 0; t < nterms; t++) {+ df->post_off[t + SKIP_ONE] += df->post_off[t];+ }+ df->post_f =+ cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->post_off[nterms] + SKIP_ONE) * sizeof(int));+ int *cursor = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int));+ ok = df->post_f && cursor;+ for (int i = 0; ok && i < df->count; i++) {+ for (int k = df->off[i]; k < df->off[i + SKIP_ONE]; k++) {+ int t = df->idx[k];+ df->post_f[df->post_off[t] + cursor[t]++] = i;+ }+ }+ cbm_free(CBM_MEM_CLASS_SEMANTIC, cursor);+ return ok;+}++/* The indexed non-test files (their File nodes) with their term vectors;+ * ffile[f] = function f's file (-1: a test function or no File node). */+static bool doc_files_build(const cbm_gbuf_t *gbuf, const cbm_sem_func_t *funcs, int func_count,+ const bool *is_test, const cbm_sem_corpus_t *corpus, int nterms,+ doc_files_t *df, int *ffile) {+ memset(df, 0, sizeof(*df));+ const cbm_gbuf_node_t **files = NULL;+ int nfiles = 0;+ if (cbm_gbuf_find_by_label(gbuf, "File", &files, &nfiles) != 0) {+ nfiles = 0;+ }+ df->by_path = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, (uint32_t)nfiles + SKIP_ONE);+ df->node = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nfiles + SKIP_ONE) * sizeof(*df->node));+ if (!df->by_path || !df->node) {+ return false;+ }+ for (int i = 0; i < nfiles; i++) {+ const char *path = files[i]->file_path;+ if (path && path[0] && !doc_is_test_path(path) && !cbm_ht_has(df->by_path, path)) {+ df->node[df->count] = files[i];+ cbm_ht_set(df->by_path, path, (void *)(intptr_t)(df->count + SKIP_ONE));+ df->count++;+ }+ }+ for (int f = 0; f < func_count; f++) {+ const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[f].node_id);+ ffile[f] = is_test[f] || !fn ? -SKIP_ONE : doc_file_of(df, fn->file_path);+ }+ size_t cap = (size_t)df->count * SEM_DOC_FILE_TERMS + SKIP_ONE;+ df->off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(int));+ df->idx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, cap * sizeof(int));+ df->w = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, cap * sizeof(float));+ df->norm = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(float));+ df->post_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int));+ return df->off && df->idx && df->w && df->norm && df->post_off &&+ doc_files_vectors(gbuf, df, funcs, ffile, func_count, corpus, nterms);+}++/* A doc's home folder: the folder whose code it documents. From the doc's+ * directory, a docs folder (doc, docs, documentation: the deepest one)+ * documents its parent; then climb while the folder holds no indexed code+ * (a non-test file with terms). "" = the repository root: the doc documents+ * the whole project. Returns an allocated string (NULL: out of memory). */+static bool doc_dir_is_docs(const char *name, size_t n) {+ static const char *const DOCS[] = {"doc", "docs", "documentation", NULL};+ for (int k = 0; DOCS[k]; k++) {+ if (strlen(DOCS[k]) == n && strncasecmp(name, DOCS[k], n) == 0) {+ return true;+ }+ }+ return false;+}++static char *doc_home(const char *doc_path, const CBMHashTable *code_dirs) {+ const char *slash = doc_path ? strrchr(doc_path, '/') : NULL;+ size_t len = slash ? (size_t)(slash - doc_path) : 0;+ char *dir = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, len + SKIP_ONE);+ if (!dir) {+ return NULL;+ }+ memcpy(dir, doc_path ? doc_path : "", len);+ dir[len] = '\0';+ size_t cut = len + SKIP_ONE; /* the deepest docs folder's start (none) */+ for (size_t a = 0; a < len;) {+ const char *end = strchr(dir + a, '/');+ size_t b = end ? (size_t)(end - dir) : len;+ if (doc_dir_is_docs(dir + a, b - a)) {+ cut = a;+ }+ a = b + SKIP_ONE;+ }+ if (cut <= len) {+ len = cut > 0 ? cut - SKIP_ONE : 0; /* the docs folder's parent */+ dir[len] = '\0';+ }+ while (len > 0 && !cbm_ht_has(code_dirs, dir)) {+ const char *up = strrchr(dir, '/');+ len = up ? (size_t)(up - dir) : 0;+ dir[len] = '\0';+ }+ return dir;+}++/* Every folder holding indexed code: the directories (and their ancestors)+ * of the files with terms; keys owned by the table's caller (freed by+ * doc_code_dirs_free). */+static void doc_free_key(const char *key, void *value, void *userdata) {+ (void)value;+ (void)userdata;+ cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)key);+}++static void doc_code_dirs_free(CBMHashTable *dirs) {+ if (dirs) {+ cbm_ht_foreach(dirs, doc_free_key, NULL);+ cbm_ht_free(dirs);+ }+}++static CBMHashTable *doc_code_dirs(const doc_files_t *df) {+ CBMHashTable *dirs = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, CBM_SZ_256);+ for (int i = 0; dirs && i < df->count; i++) {+ const char *path = df->node[i]->file_path;+ if (df->off[i + SKIP_ONE] == df->off[i]) {+ continue;+ }+ for (const char *p = strchr(path, '/'); p; p = strchr(p + SKIP_ONE, '/')) {+ size_t n = (size_t)(p - path);+ char *key = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, n + SKIP_ONE);+ if (!key) {+ doc_code_dirs_free(dirs);+ return NULL;+ }+ memcpy(key, path, n);+ key[n] = '\0';+ if (cbm_ht_has(dirs, key)) {+ cbm_free(CBM_MEM_CLASS_SEMANTIC, key);+ } else {+ cbm_ht_set(dirs, key, (void *)(intptr_t)SKIP_ONE);+ }+ }+ }+ return dirs;+}++/* A candidate's position: inside the home (or the home folder itself),+ * outside it, or no position (the home is the repository root). */+static cbm_sem_doc_pos_t doc_position(const char *home, const char *path) {+ if (!home || !home[0]) {+ return CBM_SEM_DOC_POS_GLOBAL;+ }+ size_t n = strlen(home);+ return path && strncmp(path, home, n) == 0 && (path[n] == '/' || path[n] == '\0')+ ? CBM_SEM_DOC_POS_LOCAL+ : CBM_SEM_DOC_POS_OUTSIDE;+}++static const char *doc_position_name(cbm_sem_doc_pos_t pos) {+ return pos == CBM_SEM_DOC_POS_LOCAL ? "local"+ : pos == CBM_SEM_DOC_POS_OUTSIDE ? "outside"+ : "global";+}++/* Per section: its doc's home, whether it is its doc's first section (the+ * one a folder candidate starts from), and the Folder nodes by path. */+typedef struct {+ char **home;+ bool *first;+ CBMHashTable *folders; /* folder path (graph-owned) -> Folder node */+} doc_place_t;++static void doc_place_free(doc_place_t *pl, int nsec) {+ for (int s = 0; pl->home && s < nsec; s++) {+ cbm_free(CBM_MEM_CLASS_SEMANTIC, pl->home[s]);+ }+ cbm_free(CBM_MEM_CLASS_SEMANTIC, pl->home);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, pl->first);+ if (pl->folders) {+ cbm_ht_free(pl->folders);+ }+ memset(pl, 0, sizeof(*pl));+}++static bool doc_place_build(const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t **secs, int nsec,+ const doc_files_t *df, doc_place_t *pl) {+ memset(pl, 0, sizeof(*pl));+ pl->home = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(*pl->home));+ pl->first = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(*pl->first));+ pl->folders = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, CBM_SZ_256);+ CBMHashTable *firsts = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, CBM_SZ_256);+ CBMHashTable *dirs = doc_code_dirs(df);+ bool ok = pl->home && pl->first && pl->folders && firsts && dirs;+ for (int s = 0; ok && s < nsec; s++) {+ pl->home[s] = doc_home(secs[s]->file_path, dirs);+ ok = pl->home[s] != NULL;+ const char *path = secs[s]->file_path;+ if (ok && path) {+ int prev = (int)(intptr_t)cbm_ht_get(firsts, path) - SKIP_ONE;+ if (prev < 0 || secs[s]->start_line < secs[prev]->start_line) {+ cbm_ht_set(firsts, path, (void *)(intptr_t)(s + SKIP_ONE));+ }+ }+ }+ for (int s = 0; ok && s < nsec; s++) {+ const char *path = secs[s]->file_path;+ pl->first[s] = path && (int)(intptr_t)cbm_ht_get(firsts, path) == s + SKIP_ONE;+ }+ const cbm_gbuf_node_t **folders = NULL;+ int nfold = 0;+ if (ok && cbm_gbuf_find_by_label(gbuf, "Folder", &folders, &nfold) == 0) {+ for (int i = 0; i < nfold; i++) {+ if (folders[i]->file_path && folders[i]->file_path[0]) {+ cbm_ht_set(pl->folders, folders[i]->file_path, (void *)folders[i]);+ }+ }+ }+ if (firsts) {+ cbm_ht_free(firsts);+ }+ doc_code_dirs_free(dirs);+ return ok;+}+typedef struct {const cbm_sem_func_t *funcs;int func_count;@@ -2139,6 +2598,10 @@ typedef struct {const bool *is_test;const int *name_off; /* per function: its name tokens' corpus indices */const int *name_idx;+ const char *const *fpath; /* per function: its file */+ const char *const *home; /* per section: its doc's home ("" = the root) */+ const doc_files_t *files;+ int file_key_df;doc_result_t *res;_Atomic int next;} doc_ctx_t;@@ -2169,10 +2632,73 @@ static void doc_name_evidence(const doc_ctx_t *dc, const int *sidx, int ns, doc_}}+/* The section's best CBM_SEM_DOC_FILE_K whole files, retrieved by the+ * section's key terms over the files' postings, scored by cosine. */+static void doc_section_files(doc_ctx_t *dc, int s, const int *sidx, const float *sw, int ns,+ float snorm, int *fstamp, doc_cand_t *fc) {+ const doc_files_t *df = dc->files;+ doc_result_t *r = &dc->res[s];+ int nc = 0;+ for (int k = 0; k < ns && snorm > 0.0F; k++) {+ int t = sidx[k];+ if (df->post_off[t + SKIP_ONE] - df->post_off[t] > dc->file_key_df) {+ continue;+ }+ for (int p = df->post_off[t]; p < df->post_off[t + SKIP_ONE]; p++) {+ int f = df->post_f[p];+ if (fstamp[f] != s) {+ fstamp[f] = s;+ fc[nc++] = (doc_cand_t){.func = f};+ }+ }+ }+ for (int c = 0; c < nc; c++) {+ int f = fc[c].func;+ int len = df->off[f + SKIP_ONE] - df->off[f];+ float dot = doc_terms_dot(sidx, sw, ns, df->idx + df->off[f], df->w + df->off[f], len,+ &fc[c].shared);+ float denom = snorm * df->norm[f];+ fc[c].score = denom > 0.0F ? dot / denom : 0.0F;+ }+ if (nc > 1) {+ qsort(fc, (size_t)nc, sizeof(fc[0]), cmp_doc_cand);+ }+ for (int c = 0; c < nc && c < CBM_SEM_DOC_FILE_K; c++) {+ r->files[r->nfiles++] = (doc_file_hit_t){+ .file = fc[c].func, .tfidf = fc[c].score, .rank = c + SKIP_ONE, .shared = fc[c].shared};+ }+}++/* Up to CBM_SEM_DOC_LOCAL_K functions inside the section's home that rank+ * below the top CBM_SEM_DOC_TOP_K, at CBM_SEM_DOC_MIN_SCORE or more (cands+ * sorted best first). */+static void doc_section_local(doc_ctx_t *dc, int s, const doc_cand_t *cands, int nc,+ const int *sidx, int ns) {+ doc_result_t *r = &dc->res[s];+ const char *home = dc->home[s];+ for (int c = CBM_SEM_DOC_TOP_K; home && home[0] && c < nc && r->nlocal < CBM_SEM_DOC_LOCAL_K;+ c++) {+ if (cands[c].score < CBM_SEM_DOC_MIN_SCORE) {+ break;+ }+ if (doc_position(home, dc->fpath[cands[c].func]) != CBM_SEM_DOC_POS_LOCAL) {+ continue;+ }+ doc_hit_t *hit = &r->local[r->nlocal++];+ *hit = (doc_hit_t){.func = cands[c].func,+ .tfidf = cands[c].score,+ .rank = c + SKIP_ONE,+ .shared = cands[c].shared};+ doc_name_evidence(dc, sidx, ns, hit);+ }+}+static void doc_section_one(doc_ctx_t *dc, int s, int *stamp, doc_cand_t *cands, int *sidx,- float *sw, char *body) {+ float *sw, char *body, int *fstamp, doc_cand_t *fc) {doc_result_t *r = &dc->res[s];r->n = 0;+ r->nlocal = 0;+ r->nfiles = 0;if (!doc_section_gated(dc->secs[s], body)) {return;}@@ -2215,100 +2741,209 @@ static void doc_section_one(doc_ctx_t *dc, int s, int *stamp, doc_cand_t *cands,.shared = cands[c].shared};doc_name_evidence(dc, sidx, ns, hit);}+ doc_section_local(dc, s, cands, nc, sidx, ns);+ doc_section_files(dc, s, sidx, sw, ns, snorm, fstamp, fc);}static void doc_section_worker(int worker_id, void *ctx_ptr) {(void)worker_id;doc_ctx_t *dc = ctx_ptr;+ int nf = dc->files->count;int *stamp = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)dc->func_count * sizeof(int));doc_cand_t *cands =cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)dc->func_count * sizeof(doc_cand_t));+ int *fstamp = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nf + SKIP_ONE) * sizeof(int));+ doc_cand_t *fc = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nf + SKIP_ONE) * sizeof(*fc));int *sidx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)SEM_DOC_MAX_TOKENS * sizeof(int));float *sw = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)SEM_DOC_MAX_TOKENS * sizeof(float));char *body = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, SEM_DOC_TEXT_MAX);- if (stamp && cands && sidx && sw && body) {+ if (stamp && cands && fstamp && fc && sidx && sw && body) {for (int f = 0; f < dc->func_count; f++) {stamp[f] = -1;}+ for (int f = 0; f < nf; f++) {+ fstamp[f] = -1;+ }while (true) {int s = atomic_fetch_add_explicit(&dc->next, SKIP_ONE, memory_order_relaxed);if (s >= dc->nsec) {break;}- doc_section_one(dc, s, stamp, cands, sidx, sw, body);+ doc_section_one(dc, s, stamp, cands, sidx, sw, body, fstamp, fc);}}cbm_free(CBM_MEM_CLASS_SEMANTIC, stamp);cbm_free(CBM_MEM_CLASS_SEMANTIC, cands);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, fstamp);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, fc);cbm_free(CBM_MEM_CLASS_SEMANTIC, sidx);cbm_free(CBM_MEM_CLASS_SEMANTIC, sw);cbm_free(CBM_MEM_CLASS_SEMANTIC, body);}-static void doc_section_lines(FILE *out, const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sec,- const cbm_sem_func_t *funcs, const doc_result_t *r) {- for (int h = 0; h < r->n; h++) {- const doc_hit_t *hit = &r->hits[h];- const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[hit->func].node_id);- (void)fprintf(out, "%s\t%s\t%.6f\t%d\t%d\t%s:%d-%d\t%s:%d-%d\t%d\t%.6f\n",- sec->qualified_name ? sec->qualified_name : "?",- fn && fn->qualified_name ? fn->qualified_name : "?", (double)hit->tfidf,- hit->rank, hit->shared, sec->file_path ? sec->file_path : "?",- sec->start_line, sec->end_line, fn && fn->file_path ? fn->file_path : "?",- fn ? fn->start_line : 0, fn ? fn->end_line : 0, hit->name_shared,- (double)hit->name_idf);+static const char *const DOC_KIND_NAME[CBM_SEM_DOC_KIND_COUNT] = {"function", "local", "file",+ "folder"};++/* The doc's home Folder node, for the doc's first section (NULL: another+ * section, or the home is the repository root). */+static const cbm_gbuf_node_t *doc_home_folder(const doc_place_t *pl, int s) {+ return pl->first[s] && pl->home[s][0] ? cbm_ht_get(pl->folders, pl->home[s]) : NULL;+}++static cbm_sem_doc_pos_t doc_target_position(const char *home, const cbm_gbuf_node_t *target,+ cbm_sem_doc_kind_t kind) {+ return kind == CBM_SEM_DOC_KIND_FOLDER ? CBM_SEM_DOC_POS_LOCAL+ : doc_position(home, target->file_path);+}++/* One CBM_SEM_DOC_SIGNALS line: section and target qualified names, tfidf,+ * rank, shared terms, both locations, the name evidence, then the kind, the+ * position and the doc's home ("." = the repository root). */+static void doc_signal_line(FILE *out, const cbm_gbuf_node_t *sec, const cbm_gbuf_node_t *t,+ const doc_hit_t *hit, cbm_sem_doc_kind_t kind, const char *home) {+ (void)fprintf(out, "%s\t%s\t%.6f\t%d\t%d\t%s:%d-%d\t%s:%d-%d\t%d\t%.6f\t%s\t%s\t%s\n",+ sec->qualified_name ? sec->qualified_name : "?",+ t && t->qualified_name ? t->qualified_name : "?", (double)hit->tfidf, hit->rank,+ hit->shared, sec->file_path ? sec->file_path : "?", sec->start_line,+ sec->end_line, t && t->file_path ? t->file_path : "?", t ? t->start_line : 0,+ t ? t->end_line : 0, hit->name_shared, (double)hit->name_idf, DOC_KIND_NAME[kind],+ t ? doc_position_name(doc_target_position(home, t, kind)) : "?",+ home[0] ? home : ".");+}++static void doc_section_lines(FILE *out, const cbm_gbuf_t *gbuf, const doc_ctx_t *dc,+ const doc_place_t *pl, int s) {+ const cbm_gbuf_node_t *sec = dc->secs[s];+ const doc_result_t *r = &dc->res[s];+ for (int h = 0; h < r->n + r->nlocal; h++) {+ const doc_hit_t *hit = h < r->n ? &r->hits[h] : &r->local[h - r->n];+ const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, dc->funcs[hit->func].node_id);+ doc_signal_line(out, sec, fn, hit,+ h < r->n ? CBM_SEM_DOC_KIND_FUNCTION : CBM_SEM_DOC_KIND_LOCAL, pl->home[s]);+ }+ for (int h = 0; h < r->nfiles; h++) {+ const doc_file_hit_t *fh = &r->files[h];+ doc_hit_t hit = {.tfidf = fh->tfidf, .rank = fh->rank, .shared = fh->shared};+ doc_signal_line(out, sec, dc->files->node[fh->file], &hit, CBM_SEM_DOC_KIND_FILE,+ pl->home[s]);+ }+ const cbm_gbuf_node_t *folder = doc_home_folder(pl, s);+ if (folder) {+ doc_hit_t hit = {.rank = SKIP_ONE};+ doc_signal_line(out, sec, folder, &hit, CBM_SEM_DOC_KIND_FOLDER, pl->home[s]);}}-/* The stored candidates of every section (rank <= CBM_SEM_DOC_TOP_K, tfidf >=- * CBM_SEM_DOC_MIN_SCORE), in CBM_MEM_CLASS_STORE for cbm_store_doc_candidates_free; p at two- * decimals, as on SEMANTICALLY_RELATED edges. */-static cbm_doc_candidate_t *doc_candidate_rows(const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t **secs,- int nsec, const cbm_sem_func_t *funcs,- const doc_result_t *res, int *out_count) {+/* Does the section already link the target exactly (a MENTIONS edge)? A+ * candidate for that pair would only repeat a fact the graph holds. */+static bool doc_pair_linked(const cbm_gbuf_t *gbuf, int64_t sec_id, int64_t fn_id) {+ const cbm_gbuf_edge_t **edges = NULL;int n = 0;- for (int s = 0; s < nsec; s++) {- for (int h = 0; h < res[s].n; h++) {- n += res[s].hits[h].rank <= CBM_SEM_DOC_TOP_K &&- res[s].hits[h].tfidf >= CBM_SEM_DOC_MIN_SCORE;+ if (cbm_gbuf_find_edges_by_source_type(gbuf, sec_id, "MENTIONS", &edges, &n) != 0) {+ return false;+ }+ for (int i = 0; i < n; i++) {+ if (edges[i]->target_id == fn_id) {+ return true;+ }+ }+ return false;+}++typedef struct {+ cbm_doc_candidate_t *rows;+ int n;+} doc_rows_t;++/* Append the candidate (section -> target) when its judged p is above 0 and+ * the graph does not link the pair exactly; false: out of memory (the row at+ * rw->n may hold partial strings). */+static bool doc_row_add(doc_rows_t *rw, const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sec,+ const cbm_gbuf_node_t *target, const doc_hit_t *hit,+ cbm_sem_doc_kind_t kind, const char *home) {+ if (!target || !target->qualified_name || !sec->qualified_name) {+ return true;+ }+ cbm_sem_doc_pos_t pos = doc_target_position(home, target, kind);+ cbm_sem_doc_format_t fmt = cbm_sem_doc_format(sec->file_path);+ float p = kind == CBM_SEM_DOC_KIND_FOLDER+ ? cbm_sem_doc_folder_p(fmt, sec->file_path)+ : cbm_sem_doc_calibrated_p(fmt, kind, pos, hit->tfidf);+ if (p <= 0.0F || doc_pair_linked(gbuf, sec->id, target->id)) {+ return true; /* not stored for this band, kind or format; or linked exactly */+ }+ char ev[CBM_SZ_128];+ snprintf(ev, sizeof(ev),+ "{\"shared_terms\":%d,\"name_tokens\":%d,\"kind\":\"%s\",\"position\":\"%s\"}",+ hit->shared, hit->name_shared, DOC_KIND_NAME[kind], doc_position_name(pos));+ rw->rows[rw->n] = (cbm_doc_candidate_t){+ .section_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, sec->qualified_name),+ .target_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, target->qualified_name),+ .rank = hit->rank,+ .score = hit->tfidf,+ .p = cbm_sem_p_2dp(p),+ .evidence = cbm_mem_strdup(CBM_MEM_CLASS_STORE, ev)};+ const cbm_doc_candidate_t *row = &rw->rows[rw->n];+ if (!row->section_qn || !row->target_qn || !row->evidence) {+ return false;+ }+ rw->n++;+ return true;+}++static bool doc_section_rows(doc_rows_t *rw, const cbm_gbuf_t *gbuf, const doc_ctx_t *dc,+ const doc_place_t *pl, int s) {+ const cbm_gbuf_node_t *sec = dc->secs[s];+ const doc_result_t *r = &dc->res[s];+ bool ok = true;+ for (int h = 0; ok && h < r->n + r->nlocal; h++) {+ const doc_hit_t *hit = h < r->n ? &r->hits[h] : &r->local[h - r->n];+ if (h < r->n && (hit->rank > CBM_SEM_DOC_TOP_K || hit->tfidf < CBM_SEM_DOC_MIN_SCORE)) {+ continue;}+ ok =+ doc_row_add(rw, gbuf, sec, cbm_gbuf_find_by_id(gbuf, dc->funcs[hit->func].node_id), hit,+ h < r->n ? CBM_SEM_DOC_KIND_FUNCTION : CBM_SEM_DOC_KIND_LOCAL, pl->home[s]);+ }+ for (int h = 0; ok && h < r->nfiles; h++) {+ const doc_file_hit_t *fh = &r->files[h];+ doc_hit_t hit = {.tfidf = fh->tfidf, .rank = fh->rank, .shared = fh->shared};+ ok = doc_row_add(rw, gbuf, sec, dc->files->node[fh->file], &hit, CBM_SEM_DOC_KIND_FILE,+ pl->home[s]);+ }+ doc_hit_t first = {.rank = SKIP_ONE};+ return ok && doc_row_add(rw, gbuf, sec, doc_home_folder(pl, s), &first, CBM_SEM_DOC_KIND_FOLDER,+ pl->home[s]);+}++/* The stored candidates of every section -- its best CBM_SEM_DOC_TOP_K+ * functions at CBM_SEM_DOC_MIN_SCORE or more, its local functions below them,+ * its best files, and (for a doc's first section) the doc's home folder --+ * each with its judged p, in CBM_MEM_CLASS_STORE for+ * cbm_store_doc_candidates_free; p at two decimals, as on+ * SEMANTICALLY_RELATED edges. */+static cbm_doc_candidate_t *doc_candidate_rows(const cbm_gbuf_t *gbuf, const doc_ctx_t *dc,+ const doc_place_t *pl, int *out_count) {+ int n = 0;+ for (int s = 0; s < dc->nsec; s++) {+ const doc_result_t *r = &dc->res[s];+ n += (r->n < CBM_SEM_DOC_TOP_K ? r->n : CBM_SEM_DOC_TOP_K) + r->nlocal + r->nfiles ++ SKIP_ONE;}*out_count = 0;- cbm_doc_candidate_t *rows =- n > 0 ? cbm_calloc(CBM_MEM_CLASS_STORE, (size_t)n * sizeof(*rows)) : NULL;- if (!rows) {+ doc_rows_t rw = {.rows = n > 0 ? cbm_calloc(CBM_MEM_CLASS_STORE, (size_t)n * sizeof(*rw.rows))+ : NULL};+ if (!rw.rows) {return NULL;}- int k = 0;- for (int s = 0; s < nsec; s++) {- for (int h = 0; h < res[s].n; h++) {- const doc_hit_t *hit = &res[s].hits[h];- if (hit->rank > CBM_SEM_DOC_TOP_K || hit->tfidf < CBM_SEM_DOC_MIN_SCORE) {- continue;- }- const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[hit->func].node_id);- if (!fn || !fn->qualified_name || !secs[s]->qualified_name) {- continue;- }- char ev[CBM_SZ_64];- snprintf(ev, sizeof(ev), "{\"shared_terms\":%d,\"name_tokens\":%d}", hit->shared,- hit->name_shared);- rows[k] = (cbm_doc_candidate_t){- .section_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, secs[s]->qualified_name),- .target_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, fn->qualified_name),- .rank = hit->rank,- .score = hit->tfidf,- .p = (double)(int)(cbm_sem_doc_calibrated_p(hit->tfidf) * 100.0F + 0.5F) / 100.0,- .evidence = cbm_mem_strdup(CBM_MEM_CLASS_STORE, ev)};- if (!rows[k].section_qn || !rows[k].target_qn || !rows[k].evidence) {- cbm_store_doc_candidates_free(rows, k + SKIP_ONE);- return NULL;- }- k++;+ for (int s = 0; s < dc->nsec; s++) {+ if (!doc_section_rows(&rw, gbuf, dc, pl, s)) {+ cbm_store_doc_candidates_free(rw.rows, rw.n + SKIP_ONE);+ return NULL;}}- *out_count = k;- return rows;+ *out_count = rw.n;+ return rw.rows;}static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *funcs,@@ -2332,10 +2967,15 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *int *name_idx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC,((size_t)func_count * SEM_DOC_NAME_TOKENS + SKIP_ONE) * sizeof(int));doc_result_t *res = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(doc_result_t));+ const char **fn_paths =+ cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(*fn_paths));+ int *ffile = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(int));+ doc_files_t files = {0};+ doc_place_t place = {0};int *post_f = NULL;int *cursor = NULL;FILE *out = NULL;- if (secs && post_off && fnorm && is_test && name_off && name_idx && res) {+ if (secs && post_off && fnorm && is_test && name_off && name_idx && res && fn_paths && ffile) {for (int f = 0; f < func_count; f++) {for (int k = 0; k < funcs[f].tfidf_len; k++) {post_off[funcs[f].tfidf_indices[k] + SKIP_ONE]++;@@ -2343,6 +2983,7 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *fnorm[f] = terms_norm(funcs[f].tfidf_weights, funcs[f].tfidf_len);const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[f].node_id);is_test[f] = doc_is_test_function(fn);+ fn_paths[f] = fn && fn->file_path ? fn->file_path : "";char *tok[SEM_DOC_NAME_TOKENS];int nt = fn && fn->name ? cbm_sem_tokenize(fn->name, tok, SEM_DOC_NAME_TOKENS) : 0;int base = name_off[f];@@ -2374,12 +3015,22 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *post_f[post_off[t] + cursor[t]++] = f;}}+ memcpy(secs, found, (size_t)nsec * sizeof(*secs));+ qsort(secs, (size_t)nsec, sizeof(*secs), cmp_section_node);+ }+ bool placed =+ post_f && cursor &&+ doc_files_build(gbuf, funcs, func_count, is_test, corpus, nterms, &files, ffile) &&+ doc_place_build(gbuf, secs, nsec, &files, &place);+ if (placed) {int key_df = func_count / SEM_DOC_KEY_DF_DIV;if (key_df < SEM_DOC_KEY_DF_MIN) {key_df = SEM_DOC_KEY_DF_MIN;}- memcpy(secs, found, (size_t)nsec * sizeof(*secs));- qsort(secs, (size_t)nsec, sizeof(*secs), cmp_section_node);+ int file_key_df = files.count / SEM_DOC_KEY_DF_DIV;+ if (file_key_df < SEM_DOC_KEY_DF_MIN) {+ file_key_df = SEM_DOC_KEY_DF_MIN;+ }doc_ctx_t dc = {.funcs = funcs,.func_count = func_count,.corpus = corpus,@@ -2392,21 +3043,25 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *.is_test = is_test,.name_off = name_off,.name_idx = name_idx,+ .fpath = fn_paths,+ .home = (const char *const *)place.home,+ .files = &files,+ .file_key_df = file_key_df,.res = res};atomic_init(&dc.next, 0);cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false};cbm_parallel_for(worker_count, doc_section_worker, &dc, opts);int row_count = 0;- cbm_doc_candidate_t *rows = doc_candidate_rows(gbuf, secs, nsec, funcs, res, &row_count);+ cbm_doc_candidate_t *rows = doc_candidate_rows(gbuf, &dc, &place, &row_count);cbm_pipeline_set_doc_candidates(ctx->pipeline, rows, row_count);- cbm_log_info("pass.semantic.doc_candidates", "sections", itoa_log(nsec), "rows",- itoa_log(row_count));+ cbm_log_info("pass.semantic.doc_candidates", "sections", itoa_log(nsec), "files",+ itoa_log(files.count), "rows", itoa_log(row_count));char path[CBM_SZ_1K];out = cbm_safe_getenv("CBM_SEM_DOC_SIGNALS", path, sizeof(path), NULL)? cbm_fopen(path, "w"): NULL;for (int s = 0; out && s < nsec; s++) {- doc_section_lines(out, gbuf, secs[s], funcs, &res[s]);+ doc_section_lines(out, gbuf, &dc, &place, s);}/* CBM_SEM_DOC_FUNCS=<path>: every function with its location and the* test flag, for measuring candidates the TF-IDF generator does not@@ -2438,6 +3093,10 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *cbm_free(CBM_MEM_CLASS_SEMANTIC, name_off);cbm_free(CBM_MEM_CLASS_SEMANTIC, name_idx);cbm_free(CBM_MEM_CLASS_SEMANTIC, res);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, fn_paths);+ cbm_free(CBM_MEM_CLASS_SEMANTIC, ffile);+ doc_place_free(&place, nsec);+ doc_files_free(&files);}int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) {diff --git a/src/semantic/semantic.c b/src/semantic/semantic.cindex bbdd57b56..af738151e 100644--- a/src/semantic/semantic.c+++ b/src/semantic/semantic.c@@ -1866,16 +1866,111 @@ float cbm_sem_calibrated_p(float score) {* candidates -- a section's best five non-test functions -- from five held-out* repositories, three reading rounds; the top three measured bands pooled to* stay monotone). Below the first cut (judged 0.06) nothing is stored. */-static const float SEM_DOC_P_CUTS[] = {CBM_SEM_DOC_MIN_SCORE, 0.30F, 0.40F};-static const float SEM_DOC_P_BANDS[] = {0.06F, 0.36F, 0.42F, 0.548F};+enum { SEM_DOC_P_NBANDS = 4 };+static const float SEM_DOC_P_CUTS[SEM_DOC_P_NBANDS - 1] = {CBM_SEM_DOC_MIN_SCORE, 0.30F, 0.40F};+/* Per format, p below the first cut and in each band; 0 = not stored (a+ * band judged below 0.20). Blind-judged, three reading rounds, pooled where+ * a higher band judged lower:+ * Markdown by kind and position since SEM7: SEM_DOC_P_MD_* below (its+ * first curve, 2026-10-07: 274 candidates of five repositories,+ * 0.06 / 0.36 / 0.42 / 0.55, did not separate positions).+ * reST 2026-10-08, 120 candidates, three Sphinx projects; 0.20-0.30+ * judged 0.125, so reST starts at 0.30.+ * AsciiDoc 2026-10-08, 114 candidates, one repository (the only one of+ * its search that qualified).+ * PDF no qualifying repository: not measured, nothing stored. */+static const float SEM_DOC_P_BANDS[CBM_SEM_DOC_FMT_COUNT][SEM_DOC_P_NBANDS] = {+ [CBM_SEM_DOC_FMT_OTHER] = {0.0F, 0.0F, 0.0F, 0.0F},+ [CBM_SEM_DOC_FMT_MARKDOWN] = {0.0F, 0.0F, 0.0F, 0.0F}, /* SEM_DOC_P_MD_* */+ [CBM_SEM_DOC_FMT_RST] = {0.0F, 0.0F, 0.35F, 0.625F},+ [CBM_SEM_DOC_FMT_ADOC] = {0.0F, 0.225F, 0.25F, 0.50F},+ [CBM_SEM_DOC_FMT_PDF] = {0.0F, 0.0F, 0.0F, 0.0F},+};++cbm_sem_doc_format_t cbm_sem_doc_format(const char *path) {+ static const struct {+ const char *ext;+ cbm_sem_doc_format_t fmt;+ } exts[] = {{".md", CBM_SEM_DOC_FMT_MARKDOWN}, {".mdx", CBM_SEM_DOC_FMT_MARKDOWN},+ {".markdown", CBM_SEM_DOC_FMT_MARKDOWN}, {".rst", CBM_SEM_DOC_FMT_RST},+ {".rest", CBM_SEM_DOC_FMT_RST}, {".txt", CBM_SEM_DOC_FMT_RST},+ {".adoc", CBM_SEM_DOC_FMT_ADOC}, {".asciidoc", CBM_SEM_DOC_FMT_ADOC},+ {".asc", CBM_SEM_DOC_FMT_ADOC}, {".pdf", CBM_SEM_DOC_FMT_PDF}};+ const char *dot = path ? strrchr(path, '.') : NULL;+ if (!dot || strchr(dot, '/')) {+ return CBM_SEM_DOC_FMT_OTHER;+ }+ for (size_t i = 0; i < sizeof(exts) / sizeof(exts[0]); i++) {+ if (strcasecmp(dot, exts[i].ext) == 0) {+ return exts[i].fmt;+ }+ }+ return CBM_SEM_DOC_FMT_OTHER;+}++double cbm_sem_p_2dp(float p) {+ /* 1e-4 on the x100 scale: a float holds 0.325 as 0.32499998..., 1.2e-6+ * below the half; judged values have at most three decimals, so the+ * nearest one that must round down (x.xx4) is 0.1 away */+ return (double)(long)((double)p * 100.0 + 0.5 + 1e-4) / 100.0;+}++/* Markdown by kind and position (SEM7, 2026-10-08: 490 blind-judged+ * candidates from four monorepos with per-component docs, three reading+ * rounds, pooled where a higher band judged lower). A doc's position decides+ * most: functions outside the doc's home were right 6 of 90 (none stored);+ * inside it 0.18 / 0.33 / 0.53. A doc of the whole project: SEM7's and+ * sample 4's judged items pooled (the user's choice; 158 items, nine+ * repositories): 0.17 / 0.22 / 0.51. Local extras (home functions below the+ * top 5) 19 of 60 = 0.32 at every band; whole files 0.12 below 0.40, then+ * 0.30. Below 0.20 nothing. */+static const float SEM_DOC_P_MD_FUNCTION[CBM_SEM_DOC_POS_COUNT][SEM_DOC_P_NBANDS] = {+ [CBM_SEM_DOC_POS_GLOBAL] = {0.0F, 0.0F, 0.217F, 0.509F},+ [CBM_SEM_DOC_POS_LOCAL] = {0.0F, 0.0F, 0.325F, 0.525F},+ [CBM_SEM_DOC_POS_OUTSIDE] = {0.0F, 0.0F, 0.0F, 0.0F},+};+static const float SEM_DOC_P_MD_LOCAL[SEM_DOC_P_NBANDS] = {0.0F, 0.317F, 0.317F, 0.317F};+static const float SEM_DOC_P_MD_FILE[SEM_DOC_P_NBANDS] = {0.0F, 0.0F, 0.0F, 0.30F};-float cbm_sem_doc_calibrated_p(float score) {+float cbm_sem_doc_calibrated_p(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind,+ cbm_sem_doc_pos_t pos, float score) {+ if ((int)fmt < 0 || fmt >= CBM_SEM_DOC_FMT_COUNT || (int)pos < 0 ||+ pos >= CBM_SEM_DOC_POS_COUNT) {+ return 0.0F;+ }size_t band = 0;- while (band < sizeof(SEM_DOC_P_CUTS) / sizeof(SEM_DOC_P_CUTS[0]) &&- score >= SEM_DOC_P_CUTS[band]) {+ while (band < SEM_DOC_P_NBANDS - 1 && score >= SEM_DOC_P_CUTS[band]) {band++;}- return SEM_DOC_P_BANDS[band];+ if (fmt != CBM_SEM_DOC_FMT_MARKDOWN) {+ /* measured for function candidates only, without position (SEM6) */+ return kind == CBM_SEM_DOC_KIND_FUNCTION ? SEM_DOC_P_BANDS[fmt][band] : 0.0F;+ }+ switch (kind) {+ case CBM_SEM_DOC_KIND_FUNCTION:+ return SEM_DOC_P_MD_FUNCTION[pos][band];+ case CBM_SEM_DOC_KIND_LOCAL:+ return pos == CBM_SEM_DOC_POS_LOCAL ? SEM_DOC_P_MD_LOCAL[band] : 0.0F;+ case CBM_SEM_DOC_KIND_FILE:+ return SEM_DOC_P_MD_FILE[band];+ default:+ return 0.0F; /* a folder candidate: cbm_sem_doc_folder_p */+ }+}++/* A doc's home folder as its candidate (SEM7): a README or index doc was+ * right 26 of 30 (0.87); any other doc 5 of 30 (not stored). Markdown only. */+float cbm_sem_doc_folder_p(cbm_sem_doc_format_t fmt, const char *doc_path) {+ if (fmt != CBM_SEM_DOC_FMT_MARKDOWN || !doc_path) {+ return 0.0F;+ }+ const char *base = strrchr(doc_path, '/');+ base = base ? base + 1 : doc_path;+ const char *dot = strrchr(base, '.');+ size_t n = dot ? (size_t)(dot - base) : strlen(base);+ bool readme = (n == sizeof("readme") - 1 && strncasecmp(base, "readme", n) == 0) ||+ (n == sizeof("index") - 1 && strncasecmp(base, "index", n) == 0);+ return readme ? 0.867F : 0.0F;}float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,diff --git a/src/semantic/semantic.h b/src/semantic/semantic.hindex e641e2f29..66f4cc0a1 100644--- a/src/semantic/semantic.h+++ b/src/semantic/semantic.h@@ -270,7 +270,58 @@ float cbm_sem_calibrated_p(float score);* about the function. */#define CBM_SEM_DOC_TOP_K 5#define CBM_SEM_DOC_MIN_SCORE 0.20F-float cbm_sem_doc_calibrated_p(float score);++/* A section's document format, by its file's extension: candidates are+ * judged, and stored, per format. */+typedef enum {+ CBM_SEM_DOC_FMT_OTHER = 0,+ CBM_SEM_DOC_FMT_MARKDOWN,+ CBM_SEM_DOC_FMT_RST, /* .rst, and .txt: a .txt file only has sections as Sphinx source */+ CBM_SEM_DOC_FMT_ADOC,+ CBM_SEM_DOC_FMT_PDF,+ CBM_SEM_DOC_FMT_COUNT,+} cbm_sem_doc_format_t;+cbm_sem_doc_format_t cbm_sem_doc_format(const char *path);++/* Where a candidate sits relative to the doc's home folder (the folder whose+ * code the doc documents, see pass_semantic_edges.c): the home is the+ * repository root (the doc documents the whole project), or the candidate is+ * inside or outside the home. */+typedef enum {+ CBM_SEM_DOC_POS_GLOBAL = 0,+ CBM_SEM_DOC_POS_LOCAL,+ CBM_SEM_DOC_POS_OUTSIDE,+ CBM_SEM_DOC_POS_COUNT,+} cbm_sem_doc_pos_t;++/* What a candidate points at: one of a section's best CBM_SEM_DOC_TOP_K+ * functions; a function inside the home below that top K (at most+ * CBM_SEM_DOC_LOCAL_K per section); a whole file; the doc's home folder. */+typedef enum {+ CBM_SEM_DOC_KIND_FUNCTION = 0,+ CBM_SEM_DOC_KIND_LOCAL,+ CBM_SEM_DOC_KIND_FILE,+ CBM_SEM_DOC_KIND_FOLDER,+ CBM_SEM_DOC_KIND_COUNT,+} cbm_sem_doc_kind_t;+#define CBM_SEM_DOC_LOCAL_K 2+#define CBM_SEM_DOC_FILE_K 3++/* The probability that a section of format fmt is about a candidate of this+ * kind and position with this tfidf, from the judged curves; 0 when that band+ * is not stored (judged below 0.20) or has no judged curve. A folder+ * candidate has no score: cbm_sem_doc_folder_p. */+float cbm_sem_doc_calibrated_p(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind,+ cbm_sem_doc_pos_t pos, float score);++/* The probability that the doc at doc_path documents its home folder, by+ * the doc's kind (a README or index file, or any other doc); 0 = not+ * stored. */+float cbm_sem_doc_folder_p(cbm_sem_doc_format_t fmt, const char *doc_path);++/* p as stored and shown: two decimals, half up (0.225 -> 0.23, 0.325 ->+ * 0.33; the epsilon absorbs the float representation of a judged x.xx5). */+double cbm_sem_p_2dp(float p);/* Module proximity multiplier based on file paths. */float cbm_sem_proximity(const char *path_a, const char *path_b);diff --git a/src/store/store.c b/src/store/store.cindex 7a64fd17b..b974e5e39 100644--- a/src/store/store.c+++ b/src/store/store.c@@ -10808,14 +10808,9 @@ int cbm_store_doc_candidates_replace(cbm_store_t *s, const char *project,return exec_sql(s, "COMMIT;");}-int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn,- const char *target_qn, int limit, cbm_doc_candidate_t **out,- int *count) {- if (!s || !s->db || !project || !out || !count || (!section_qn == !target_qn) || limit <= 0) {- return CBM_STORE_ERR;- }- *out = NULL;- *count = 0;+/* Does the database hold the candidates table? CBM_STORE_NOT_FOUND: no (a+ * database written by an older build). */+static int doc_candidates_table(cbm_store_t *s) {sqlite3_stmt *probe = NULL;if (sqlite3_prepare_v2(s->db,"SELECT 1 FROM sqlite_master WHERE type='table' AND "@@ -10833,19 +10828,15 @@ int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const charstore_set_error_sqlite(s, "doc_candidates probe step");return CBM_STORE_ERR;}- const char *sql = section_qn ? "SELECT section_qn, target_qn, rank, score, p, evidence FROM "- "doc_link_candidates WHERE project=?1 AND section_qn=?2 "- "ORDER BY p DESC, score DESC, target_qn LIMIT ?3;"- : "SELECT section_qn, target_qn, rank, score, p, evidence FROM "- "doc_link_candidates WHERE project=?1 AND target_qn=?2 "- "ORDER BY p DESC, score DESC, section_qn LIMIT ?3;";- sqlite3_stmt *stmt = NULL;- if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {- store_set_error_sqlite(s, "doc_candidates get");- return CBM_STORE_ERR;- }+ return CBM_STORE_OK;+}++/* Run a prepared candidates query (project ?1, key ?2, limit ?3; columns+ * section_qn, target_qn, rank, score, p, evidence) into *out; finalizes it. */+static int doc_candidates_read(sqlite3_stmt *stmt, const char *project, const char *key, int limit,+ cbm_doc_candidate_t **out, int *count) {bind_text(stmt, SKIP_ONE, project);- bind_text(stmt, ST_COL_2, section_qn ? section_qn : target_qn);+ bind_text(stmt, ST_COL_2, key);sqlite3_bind_int(stmt, ST_COL_3, limit);cbm_doc_candidate_t *rows = cbm_calloc(CBM_MEM_CLASS_STORE, (size_t)limit * sizeof(*rows));if (!rows) {@@ -10871,6 +10862,53 @@ int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const charreturn CBM_STORE_OK;}+static int doc_candidates_query(cbm_store_t *s, const char *sql, const char *project,+ const char *key, int limit, cbm_doc_candidate_t **out, int *count) {+ *out = NULL;+ *count = 0;+ int rc = doc_candidates_table(s);+ if (rc != CBM_STORE_OK) {+ return rc;+ }+ sqlite3_stmt *stmt = NULL;+ if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {+ store_set_error_sqlite(s, "doc_candidates get");+ return CBM_STORE_ERR;+ }+ return doc_candidates_read(stmt, project, key, limit, out, count);+}++int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn,+ const char *target_qn, int limit, cbm_doc_candidate_t **out,+ int *count) {+ if (!s || !s->db || !project || !out || !count || (!section_qn == !target_qn) || limit <= 0) {+ return CBM_STORE_ERR;+ }+ const char *sql = section_qn ? "SELECT section_qn, target_qn, rank, score, p, evidence FROM "+ "doc_link_candidates WHERE project=?1 AND section_qn=?2 "+ "ORDER BY p DESC, score DESC, target_qn LIMIT ?3;"+ : "SELECT section_qn, target_qn, rank, score, p, evidence FROM "+ "doc_link_candidates WHERE project=?1 AND target_qn=?2 "+ "ORDER BY p DESC, score DESC, section_qn LIMIT ?3;";+ return doc_candidates_query(s, sql, project, section_qn ? section_qn : target_qn, limit, out,+ count);+}++int cbm_store_doc_candidates_for_path(cbm_store_t *s, const char *project, const char *file_path,+ int limit, cbm_doc_candidate_t **out, int *count) {+ if (!s || !s->db || !project || !file_path || !file_path[0] || !out || !count || limit <= 0) {+ return CBM_STORE_ERR;+ }+ static const char SQL[] = "SELECT c.section_qn, c.target_qn, c.rank, c.score, c.p, c.evidence "+ "FROM doc_link_candidates c JOIN nodes n "+ "ON n.project = c.project AND n.qualified_name = c.target_qn "+ "WHERE c.project=?1 AND ((n.label='File' AND n.file_path=?2) OR "+ "(n.label='Folder' AND n.file_path <> '' AND "+ "substr(?2, 1, length(n.file_path) + 1) = n.file_path || '/')) "+ "ORDER BY c.p DESC, c.score DESC, c.section_qn LIMIT ?3;";+ return doc_candidates_query(s, SQL, project, file_path, limit, out, count);+}+void cbm_store_doc_candidates_free(cbm_doc_candidate_t *rows, int count) {for (int i = 0; rows && i < count; i++) {cbm_free(CBM_MEM_CLASS_STORE, (void *)rows[i].section_qn);diff --git a/src/store/store.h b/src/store/store.hindex e7fb3142b..2a8d64048 100644--- a/src/store/store.h+++ b/src/store/store.h@@ -1110,6 +1110,12 @@ int cbm_store_doc_candidates_replace(cbm_store_t *s, const char *project,int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn,const char *target_qn, int limit, cbm_doc_candidate_t **out,int *count);++/* The candidates that point at a file as a whole: at its File node, or at a+ * Folder node that holds it (a doc's home folder), highest p first, then+ * score, at most limit. Allocation and CBM_STORE_NOT_FOUND as above. */+int cbm_store_doc_candidates_for_path(cbm_store_t *s, const char *project, const char *file_path,+ int limit, cbm_doc_candidate_t **out, int *count);void cbm_store_doc_candidates_free(cbm_doc_candidate_t *rows, int count);/* ADR section parsing/rendering (pure functions, no store needed) */diff --git a/tests/test_mcp.c b/tests/test_mcp.cindex 6ae9a0814..abd19c124 100644--- a/tests/test_mcp.c+++ b/tests/test_mcp.c@@ -2546,6 +2546,98 @@ TEST(tool_get_file_outline_returns_bounded_filtered_columnar_rows_issue469) {PASS();}+/* get_file_outline shows the sections possibly about the file as a whole or+ * about a folder holding it (highest p first, kind shown), in both formats,+ * on the first page only; a candidate for a function in the file is not one+ * of them. */+TEST(tool_get_file_outline_shows_doc_candidates_of_file_and_folders) {+ cbm_mcp_server_t *srv = setup_mcp_with_data();+ cbm_store_t *store = cbm_mcp_server_store(srv);+ ASSERT_NOT_NULL(store);+ ASSERT_EQ(cbm_store_upsert_project(store, "doc-outline", "/tmp/doc-outline"), CBM_STORE_OK);+ cbm_mcp_server_set_project(srv, "doc-outline");+ const struct {+ const char *label, *qn, *path;+ int start;+ } nodes[] = {{"File", "doc-outline.pkg.mail.parse.__file__", "pkg/mail/parse.py", 0},+ {"Function", "doc-outline.pkg.mail.parse.parse_mail", "pkg/mail/parse.py", 3},+ {"Folder", "doc-outline.pkg.mail", "pkg/mail", 0},+ {"Section", "doc-outline.pkg.mail.README.Mail", "pkg/mail/README.md", 1},+ {"Section", "doc-outline.docs.parsing.Parsing", "docs/parsing.md", 5}};+ for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) {+ cbm_node_t node = {.project = "doc-outline",+ .label = nodes[i].label,+ .name = nodes[i].qn,+ .qualified_name = nodes[i].qn,+ .file_path = nodes[i].path,+ .start_line = nodes[i].start,+ .end_line = nodes[i].start + 2};+ ASSERT_GT(cbm_store_upsert_node(store, &node), 0);+ }+ const cbm_doc_candidate_t rows[] = {+ {.section_qn = "doc-outline.docs.parsing.Parsing",+ .target_qn = "doc-outline.pkg.mail.parse.__file__",+ .rank = 1,+ .score = 0.41,+ .p = 0.5,+ .evidence = "{\"shared_terms\":4,\"name_tokens\":0,\"kind\":\"file\","+ "\"position\":\"global\"}"},+ {.section_qn = "doc-outline.pkg.mail.README.Mail",+ .target_qn = "doc-outline.pkg.mail",+ .rank = 1,+ .p = 0.8,+ .evidence = "{\"shared_terms\":0,\"name_tokens\":0,\"kind\":\"folder\","+ "\"position\":\"local\"}"},+ {.section_qn = "doc-outline.docs.parsing.Parsing",+ .target_qn = "doc-outline.pkg.mail.parse.parse_mail",+ .rank = 1,+ .score = 0.5,+ .p = 0.9,+ .evidence = "{\"shared_terms\":5,\"name_tokens\":1,\"kind\":\"function\","+ "\"position\":\"global\"}"},+ };+ ASSERT_EQ(cbm_store_doc_candidates_replace(store, "doc-outline", rows, 3), CBM_STORE_OK);++ char *raw =+ cbm_mcp_handle_tool(srv, "get_file_outline",+ "{\"project\":\"doc-outline\",\"file_path\":\"pkg/mail/parse.py\","+ "\"format\":\"json\"}");+ char *text = extract_text_content(raw);+ free(raw);+ ASSERT_NOT_NULL(text);+ yyjson_doc *doc = yyjson_read(text, strlen(text), 0);+ ASSERT_NOT_NULL(doc);+ yyjson_val *in = yyjson_obj_get(yyjson_doc_get_root(doc), "possibly_described_in");+ ASSERT_EQ((int)yyjson_arr_size(in), 2);+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "section")),+ "doc-outline.pkg.mail.README.Mail");+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "kind")), "folder");+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 1), "kind")), "file");+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 1), "file_path")),+ "docs/parsing.md");+ ASSERT_NOT_NULL(yyjson_obj_get(yyjson_doc_get_root(doc), "candidates_note"));+ yyjson_doc_free(doc);+ free(text);++ raw = cbm_mcp_handle_tool(srv, "get_file_outline",+ "{\"project\":\"doc-outline\",\"file_path\":\"pkg/mail/parse.py\"}");+ ASSERT_NOT_NULL(raw);+ ASSERT_NOT_NULL(strstr(raw, "possibly_described_in"));+ ASSERT_NOT_NULL(strstr(raw, "doc-outline.pkg.mail.README.Mail"));+ ASSERT_NOT_NULL(strstr(raw, "folder"));+ free(raw);++ raw = cbm_mcp_handle_tool(srv, "get_file_outline",+ "{\"project\":\"doc-outline\",\"file_path\":\"pkg/mail/parse.py\","+ "\"offset\":1}");+ ASSERT_NOT_NULL(raw);+ ASSERT_NULL(strstr(raw, "possibly_described_in"));+ free(raw);++ cbm_mcp_server_free(srv);+ PASS();+}+TEST(tool_get_file_outline_validates_json_path_limit_and_cancel_issue469) {cbm_mcp_server_t *srv = setup_mcp_with_data();cbm_store_t *store = cbm_mcp_server_store(srv);@@ -7618,9 +7710,10 @@ TEST(tool_trace_call_path_qn_fallback_frees_name_miss) {cbm_store_free_nodes(by_name, by_name_count);char *resp = cbm_mcp_server_handle(- srv, "{\"jsonrpc\":\"2.0\",\"id\":63,\"method\":\"tools/call\","- "\"params\":{\"name\":\"trace_call_path\",\"arguments\":{\"function_name\":"- "\"qnfb-proj.src.qnfb_entry\",\"project\":\"qnfb-proj\",\"direction\":\"outbound\"}}}");+ srv,+ "{\"jsonrpc\":\"2.0\",\"id\":63,\"method\":\"tools/call\","+ "\"params\":{\"name\":\"trace_call_path\",\"arguments\":{\"function_name\":"+ "\"qnfb-proj.src.qnfb_entry\",\"project\":\"qnfb-proj\",\"direction\":\"outbound\"}}}");ASSERT_NOT_NULL(resp);char *inner = extract_text_content(resp);ASSERT_NOT_NULL(inner);@@ -18537,7 +18630,8 @@ TEST(snippet_doc_candidates_both_directions) {.rank = 1,.score = 0.45,.p = 0.548,- .evidence = "{\"shared_terms\":5,\"name_tokens\":2}"},+ .evidence = "{\"shared_terms\":5,\"name_tokens\":2,\"kind\":\"function\","+ "\"position\":\"local\"}"},{.section_qn = "test-project.README.Orders",.target_qn = "test-project.gone.Missing",.rank = 3,@@ -18560,6 +18654,9 @@ TEST(snippet_doc_candidates_both_directions) {ASSERT_FLOAT_EQ(yyjson_get_num(yyjson_obj_get(first, "p")), 0.548, 1e-9);ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(first, "shared_terms")), 5);ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(first, "start_line")), 7);+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "label")), "Function");+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "kind")), "function");+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "position")), "local");ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(about, 1), "qualified_name")),"test-project.cmd.server.main.HandleRequest");ASSERT_NOT_NULL(yyjson_obj_get(yyjson_doc_get_root(doc), "candidates_note"));@@ -23784,6 +23881,7 @@ SUITE(mcp) {RUN_TEST(tool_compare_graphs_validation_and_scan_cap_are_atomic_issue525);RUN_TEST(tool_compare_graphs_cancel_and_readonly_handles_release_issue525);RUN_TEST(tool_get_file_outline_returns_bounded_filtered_columnar_rows_issue469);+ RUN_TEST(tool_get_file_outline_shows_doc_candidates_of_file_and_folders);RUN_TEST(tool_get_file_outline_validates_json_path_limit_and_cancel_issue469);RUN_TEST(tool_search_graph_basic);RUN_TEST(tool_search_graph_empty_hint_describes_combined_filters_issue1366);diff --git a/tests/test_pipeline.c b/tests/test_pipeline.cindex c11c2842c..ddf7bc5a5 100644--- a/tests/test_pipeline.c+++ b/tests/test_pipeline.c@@ -18948,14 +18948,44 @@ TEST(pipeline_semantic_edges_carry_p) {PASS();}+/* The p a stored candidate row must carry: the curve of the row's kind and+ * position (read from its evidence) at its score, two decimals; -1 for an+ * unknown kind or position. *is_function: a function candidate (top 5). */+static double doc_row_want(const char *evidence, cbm_sem_doc_format_t fmt, float score,+ bool *is_function) {+ static const struct {+ const char *name;+ cbm_sem_doc_kind_t kind;+ } KINDS[] = {{"\"kind\":\"function\"", CBM_SEM_DOC_KIND_FUNCTION},+ {"\"kind\":\"local\"", CBM_SEM_DOC_KIND_LOCAL},+ {"\"kind\":\"file\"", CBM_SEM_DOC_KIND_FILE}};+ static const struct {+ const char *name;+ cbm_sem_doc_pos_t pos;+ } POSITIONS[] = {{"\"position\":\"global\"", CBM_SEM_DOC_POS_GLOBAL},+ {"\"position\":\"local\"", CBM_SEM_DOC_POS_LOCAL},+ {"\"position\":\"outside\"", CBM_SEM_DOC_POS_OUTSIDE}};+ int k = -1;+ int p = -1;+ for (int i = 0; evidence && i < 3; i++) {+ k = k < 0 && strstr(evidence, KINDS[i].name) ? i : k;+ p = p < 0 && strstr(evidence, POSITIONS[i].name) ? i : p;+ }+ *is_function = k == 0;+ return k < 0 || p < 0 ? -1.0+ : cbm_sem_p_2dp(cbm_sem_doc_calibrated_p(fmt, KINDS[k].kind,+ POSITIONS[p].pos, score));+}+/* Count a section's stored doc -> code candidates (section QN containing- * heading) and check each row's shape: rank within the top- * CBM_SEM_DOC_TOP_K, score at or above the floor, p the doc curve's band at- * two decimals, never a test function. Returns -1 on a bad row. */+ * heading) and check each row's shape: a function within the top+ * CBM_SEM_DOC_TOP_K (top: the best of them), score at or above the floor, p+ * the curve of its kind and position at two decimals, never a test+ * function. Returns -1 on a bad row. */static int doc_candidate_rows_for(cbm_store_t *s, const char *heading, char *top, size_t top_sz) {sqlite3_stmt *st = NULL;if (sqlite3_prepare_v2(cbm_store_get_db(s),- "SELECT target_qn, rank, score, p FROM doc_link_candidates "+ "SELECT target_qn, rank, score, p, evidence FROM doc_link_candidates ""WHERE instr(section_qn, ?1) > 0 ORDER BY rank",-1, &st, NULL) != SQLITE_OK) {return -1;@@ -18963,16 +18993,20 @@ static int doc_candidate_rows_for(cbm_store_t *s, const char *heading, char *topsqlite3_bind_text(st, 1, heading, -1, SQLITE_TRANSIENT);int n = 0;bool ok = true;+ bool have_top = false;while (ok && sqlite3_step(st) == SQLITE_ROW) {const char *target = (const char *)sqlite3_column_text(st, 0);int rank = sqlite3_column_int(st, 1);double score = sqlite3_column_double(st, 2);double p = sqlite3_column_double(st, 3);- double want = (double)(int)(cbm_sem_doc_calibrated_p((float)score) * 100.0F + 0.5F) / 100.0;- ok = target && rank >= 1 && rank <= CBM_SEM_DOC_TOP_K &&+ bool is_function = false;+ double want = doc_row_want((const char *)sqlite3_column_text(st, 4),+ CBM_SEM_DOC_FMT_MARKDOWN, (float)score, &is_function);+ ok = target && rank >= 1 && (!is_function || rank <= CBM_SEM_DOC_TOP_K) &&score >= (double)CBM_SEM_DOC_MIN_SCORE && p == want && !strstr(target, ".Test");- if (ok && n == 0 && top) {+ if (ok && is_function && !have_top && top) {snprintf(top, top_sz, "%s", target);+ have_top = true;}n++;}@@ -19061,6 +19095,306 @@ TEST(pipeline_doc_candidates_published_and_kept_by_delta) {PASS();}+/* One Python family per subject: parse / load / validate / write, each with+ * a docstring, so the functions' corpus has the subject words to match. */+static void write_py_family(const char *base, const char *subject) {+ char path[512];+ snprintf(path, sizeof(path), "%s/%s.py", base, subject);+ char body[4096];+ snprintf(body, sizeof(body),+ "def parse_%s_config(path):\n"+ " \"\"\"Parse the %s config file and return the parsed %s settings.\"\"\"\n"+ " return load_%s_config(path)\n\n"+ "def load_%s_config(path):\n"+ " \"\"\"Load the %s config from disk and validate every %s field.\"\"\"\n"+ " cfg = {}\n validate_%s_config(cfg)\n return cfg\n\n"+ "def validate_%s_config(cfg):\n"+ " \"\"\"Check the %s config for missing fields.\"\"\"\n"+ " return cfg is not None\n\n"+ "def write_%s_config(path, cfg):\n"+ " \"\"\"Write the %s config back to disk after validating it.\"\"\"\n"+ " validate_%s_config(cfg)\n",+ subject, subject, subject, subject, subject, subject, subject, subject, subject,+ subject, subject, subject, subject);+ th_write_file(path, body);+}++/* The candidate count, and the bad rows, of one section (its QN contains+ * heading and its file ends in ext): p must be the format's curve value at+ * two decimals and the tfidf at or above the format's first stored band. */+static int doc_rows_of(cbm_store_t *s, const char *heading, const char *ext,+ cbm_sem_doc_format_t fmt, const char *forbidden_target, int *bad) {+ sqlite3_stmt *st = NULL;+ *bad = 0;+ if (sqlite3_prepare_v2(+ cbm_store_get_db(s),+ "SELECT c.target_qn, c.score, c.p, c.evidence FROM doc_link_candidates c "+ "JOIN nodes n ON n.qualified_name = c.section_qn "+ "WHERE instr(c.section_qn, ?1) > 0 AND n.file_path LIKE ?2",+ -1, &st, NULL) != SQLITE_OK) {+ return -1;+ }+ char like[32];+ snprintf(like, sizeof(like), "%%%s", ext);+ sqlite3_bind_text(st, 1, heading, -1, SQLITE_TRANSIENT);+ sqlite3_bind_text(st, 2, like, -1, SQLITE_TRANSIENT);+ int n = 0;+ while (sqlite3_step(st) == SQLITE_ROW) {+ const char *target = (const char *)sqlite3_column_text(st, 0);+ float score = (float)sqlite3_column_double(st, 1);+ double p = sqlite3_column_double(st, 2);+ bool is_function = false;+ double want =+ doc_row_want((const char *)sqlite3_column_text(st, 3), fmt, score, &is_function);+ *bad += want <= 0.0 || p != want ||+ (forbidden_target && target && strstr(target, forbidden_target));+ n++;+ }+ sqlite3_finalize(st);+ return n;+}++/* Candidates never repeat an exact link: a section whose code span names a+ * function (a MENTIONS edge) does not also list that function as a+ * candidate. And each format carries its own judged curve: reST stores only+ * from tfidf 0.30, with reST's p. */+TEST(pipeline_doc_candidates_skip_exact_links_and_use_format_curves) {+ char tmp[256];+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_docfmt_XXXXXX");+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));+ static const char *const subjects[] = {"mail", "cache", "queue", "user", "server", "client"};+ for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) {+ write_py_family(tmp, subjects[i]);+ }+ char docs[512];+ snprintf(docs, sizeof(docs), "%s/docs", tmp);+ ASSERT_TRUE(cbm_mkdir_p(docs, 0755));+ write_temp_file(tmp, "docs/mail.md",+ "# Mail settings\n\n"+ "The mail config file is parsed by `mail.parse_mail_config`, which loads "+ "the mail config from disk and validates every mail field before the "+ "parsed mail settings are returned to the caller for use.\n");+ write_temp_file(tmp, "docs/mail.rst",+ "Mail loading\n============\n\n"+ "Load the mail config from disk and validate every mail field: missing "+ "mail fields fail the validation, and the loaded mail config is written "+ "back to disk after validating it.\n");+ char db[512];+ snprintf(db, sizeof(db), "%s/docfmt.db", tmp);+ cbm_pipeline_incremental_test_reset_faults();+ cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL);+ ASSERT_NOT_NULL(p);+ ASSERT_EQ(cbm_pipeline_run(p), 0);+ cbm_pipeline_free(p);++ cbm_store_t *s = cbm_store_open_path(db);+ ASSERT_NOT_NULL(s);+ /* the exact link exists ... */+ sqlite3_stmt *st = NULL;+ ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s),+ "SELECT count(*) FROM edges e JOIN nodes a ON a.id = e.source_id "+ "JOIN nodes b ON b.id = e.target_id WHERE e.type = 'MENTIONS' "+ "AND a.label = 'Section' AND a.file_path = 'docs/mail.md' "+ "AND b.name = 'parse_mail_config'",+ -1, &st, NULL),+ SQLITE_OK);+ ASSERT_EQ(sqlite3_step(st), SQLITE_ROW);+ int linked = sqlite3_column_int(st, 0);+ sqlite3_finalize(st);+ /* ... and is not repeated as a candidate, while the rest of the section's+ * candidates are stored on the Markdown curve */+ int md_bad = 0;+ int rst_bad = 0;+ int md = doc_rows_of(s, "Mail-settings", ".md", CBM_SEM_DOC_FMT_MARKDOWN, ".parse_mail_config",+ &md_bad);+ int rst = doc_rows_of(s, "Mail-loading", ".rst", CBM_SEM_DOC_FMT_RST, NULL, &rst_bad);+ cbm_store_close(s);+ th_rmtree(tmp);+ cbm_pipeline_incremental_test_reset_faults();+ ASSERT_EQ(linked, 1);+ ASSERT_GT(md, 0);+ ASSERT_EQ(md_bad, 0);+ ASSERT_GT(rst, 0);+ ASSERT_EQ(rst_bad, 0);+ PASS();+}++/* Lines of a CBM_SEM_DOC_SIGNALS dump whose section file, kind, position,+ * home and target file equal the given values (NULL: any). */+static int doc_signal_count(const char *dump, const char *sec_file, const char *kind,+ const char *pos, const char *home, const char *target_file) {+ FILE *f = cbm_fopen(dump, "r");+ if (!f) {+ return -1;+ }+ char line[4096];+ int n = 0;+ while (fgets(line, sizeof(line), f)) {+ line[strcspn(line, "\r\n")] = '\0';+ char *col[12];+ int nc = 0;+ for (char *p = line; nc < 12; nc++) {+ col[nc] = p;+ char *tab = strchr(p, '\t');+ if (!tab) {+ nc++;+ break;+ }+ *tab = '\0';+ p = tab + 1;+ }+ if (nc != 12) {+ continue;+ }+ char *colon = strrchr(col[5], ':');+ if (colon) {+ *colon = '\0';+ }+ colon = strrchr(col[6], ':');+ if (colon) {+ *colon = '\0';+ }+ n += (!sec_file || strcmp(col[5], sec_file) == 0) && (!kind || strcmp(col[9], kind) == 0) &&+ (!pos || strcmp(col[10], pos) == 0) && (!home || strcmp(col[11], home) == 0) &&+ (!target_file || strcmp(col[6], target_file) == 0);+ }+ fclose(f);+ return n;+}++/* Document position: a doc's home is its folder (a docs folder documents its+ * parent; a folder without code climbs to one with code; the root = the+ * whole project); a candidate inside the home is local, elsewhere outside;+ * a doc with a home below the root gets its home folder as a candidate;+ * whole files are candidates; local functions below the top 5 are added. */+TEST(pipeline_doc_candidates_home_position_and_kinds) {+ char tmp[256];+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_docpos_XXXXXX");+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));+ char dir[512];+ static const char *const pkgs[] = {"pkg/mail", "pkg/mail/sub", "pkg/cache", "pkg/queue",+ "pkg/mail/docs", "docs", "guides"};+ for (size_t i = 0; i < sizeof(pkgs) / sizeof(pkgs[0]); i++) {+ snprintf(dir, sizeof(dir), "%s/%s", tmp, pkgs[i]);+ ASSERT_TRUE(cbm_mkdir_p(dir, 0755));+ }+ snprintf(dir, sizeof(dir), "%s/pkg/mail", tmp);+ write_py_family(dir, "mail");+ snprintf(dir, sizeof(dir), "%s/pkg/mail/sub", tmp);+ write_py_family(dir, "mail");+ snprintf(dir, sizeof(dir), "%s/pkg/cache", tmp);+ write_py_family(dir, "cache");+ snprintf(dir, sizeof(dir), "%s/pkg/queue", tmp);+ write_py_family(dir, "queue");+ static const char body[] =+ "The mail config file is parsed, loaded from disk and validated: every mail "+ "field is checked, missing mail fields fail the validation, and the parsed mail "+ "settings are written back to disk after validating the mail config.\n";+ static const char *const doc_files[] = {"pkg/mail/README.md", "pkg/mail/docs/guide.md",+ "docs/overview.md", "guides/notes.md"};+ for (size_t i = 0; i < sizeof(doc_files) / sizeof(doc_files[0]); i++) {+ char text[1024];+ snprintf(text, sizeof(text), "# Mail config %d\n\n%s", (int)i, body);+ write_temp_file(tmp, doc_files[i], text);+ }+ char db[512];+ char dump[512];+ snprintf(db, sizeof(db), "%s/docpos.db", tmp);+ snprintf(dump, sizeof(dump), "%s/signals.tsv", tmp);+ cbm_setenv("CBM_SEM_DOC_SIGNALS", dump, 1);+ cbm_pipeline_incremental_test_reset_faults();+ cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL);+ ASSERT_NOT_NULL(p);+ int rc = cbm_pipeline_run(p);+ cbm_pipeline_free(p);+ cbm_unsetenv("CBM_SEM_DOC_SIGNALS");+ ASSERT_EQ(rc, 0);++ const char *readme = "pkg/mail/README.md";+ const char *guide = "pkg/mail/docs/guide.md";+ /* homes: the package; its docs folder documents the package; top-level+ * docs and a folder without code document the whole project */+ int readme_lines = doc_signal_count(dump, readme, NULL, NULL, NULL, NULL);+ int readme_home = doc_signal_count(dump, readme, NULL, NULL, "pkg/mail", NULL);+ int guide_lines = doc_signal_count(dump, guide, NULL, NULL, NULL, NULL);+ int guide_home = doc_signal_count(dump, guide, NULL, NULL, "pkg/mail", NULL);+ int overview_lines = doc_signal_count(dump, "docs/overview.md", NULL, NULL, NULL, NULL);+ int overview_root = doc_signal_count(dump, "docs/overview.md", NULL, "global", ".", NULL);+ int notes_lines = doc_signal_count(dump, "guides/notes.md", NULL, NULL, NULL, NULL);+ int notes_root = doc_signal_count(dump, "guides/notes.md", NULL, "global", ".", NULL);+ /* positions */+ int local_fn =+ doc_signal_count(dump, readme, "function", "local", "pkg/mail", "pkg/mail/mail.py");+ int local_sub =+ doc_signal_count(dump, readme, NULL, "local", "pkg/mail", "pkg/mail/sub/mail.py");+ int outside_fn = doc_signal_count(dump, readme, "function", "outside", NULL, NULL);+ int wrong_local = doc_signal_count(dump, readme, NULL, "local", NULL, "pkg/cache/cache.py");+ /* kinds */+ int folders = doc_signal_count(dump, NULL, "folder", "local", NULL, "pkg/mail");+ int folder_readme = doc_signal_count(dump, readme, "folder", "local", "pkg/mail", "pkg/mail");+ int root_folders = doc_signal_count(dump, "docs/overview.md", "folder", NULL, NULL, NULL) ++ doc_signal_count(dump, "guides/notes.md", "folder", NULL, NULL, NULL);+ int files = doc_signal_count(dump, readme, "file", "local", NULL, NULL);+ int extras = doc_signal_count(dump, readme, "local", "local", "pkg/mail", NULL);+ /* stored: the README's home folder (a README: p 0.87), not the guide's+ * (another doc), no function outside the home, every row on its curve */+ cbm_store_t *s = cbm_store_open_path(db);+ ASSERT_NOT_NULL(s);+ sqlite3_stmt *st = NULL;+ ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s),+ "SELECT c.score, c.p, c.evidence, n.file_path FROM "+ "doc_link_candidates c JOIN nodes n ON n.qualified_name = "+ "c.section_qn",+ -1, &st, NULL),+ SQLITE_OK);+ int stored_folders = 0;+ int stored_outside = 0;+ int off_curve = 0;+ double folder_p = 0.0;+ while (sqlite3_step(st) == SQLITE_ROW) {+ const char *ev = (const char *)sqlite3_column_text(st, 2);+ const char *path = (const char *)sqlite3_column_text(st, 3);+ double p = sqlite3_column_double(st, 1);+ if (ev && strstr(ev, "\"kind\":\"folder\"")) {+ stored_folders++;+ folder_p = p;+ off_curve += strcmp(path, readme) != 0;+ continue;+ }+ stored_outside +=+ ev && strstr(ev, "\"position\":\"outside\"") && strstr(ev, "\"kind\":\"function\"");+ bool is_function = false;+ off_curve += p != doc_row_want(ev, CBM_SEM_DOC_FMT_MARKDOWN,+ (float)sqlite3_column_double(st, 0), &is_function);+ }+ sqlite3_finalize(st);+ cbm_store_close(s);+ th_rmtree(tmp);+ cbm_pipeline_incremental_test_reset_faults();+ ASSERT_GT(readme_lines, 0);+ ASSERT_EQ(readme_home, readme_lines);+ ASSERT_GT(guide_lines, 0);+ ASSERT_EQ(guide_home, guide_lines);+ ASSERT_GT(overview_lines, 0);+ ASSERT_EQ(overview_root, overview_lines);+ ASSERT_GT(notes_lines, 0);+ ASSERT_EQ(notes_root, notes_lines);+ ASSERT_GT(local_fn, 0);+ ASSERT_GT(local_sub, 0);+ ASSERT_GT(outside_fn, 0);+ ASSERT_EQ(wrong_local, 0);+ ASSERT_EQ(folders, 2); /* README and the docs guide; one per doc, never per section */+ ASSERT_EQ(folder_readme, 1);+ ASSERT_EQ(root_folders, 0);+ ASSERT_GT(files, 0);+ ASSERT_GT(extras, 0);+ ASSERT_EQ(stored_folders, 1);+ ASSERT_FLOAT_EQ(folder_p, 0.87, 1e-9);+ ASSERT_EQ(stored_outside, 0);+ ASSERT_EQ(off_curve, 0);+ PASS();+}+TEST(pipeline_semantic_edges_no_functions) {char tmp[256];snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX");@@ -19976,6 +20310,8 @@ SUITE(pipeline) {RUN_TEST(pipeline_semantic_pair_signals_never_change_the_graph);RUN_TEST(pipeline_semantic_edges_carry_p);RUN_TEST(pipeline_doc_candidates_published_and_kept_by_delta);+ RUN_TEST(pipeline_doc_candidates_skip_exact_links_and_use_format_curves);+ RUN_TEST(pipeline_doc_candidates_home_position_and_kinds);RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153);RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153);RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153);diff --git a/tests/test_semantic.c b/tests/test_semantic.cindex 006cad45c..f1ebd09f7 100644--- a/tests/test_semantic.c+++ b/tests/test_semantic.c@@ -513,25 +513,135 @@ TEST(sem_calibrated_p_bands) {PASS();}-/* Doc -> code candidates: the judged bands (sample 4), monotone, with nothing- * stored below CBM_SEM_DOC_MIN_SCORE (that band judged 0.06). */+static float docp(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind, cbm_sem_doc_pos_t pos,+ float score) {+ return cbm_sem_doc_calibrated_p(fmt, kind, pos, score);+}++/* Doc -> code candidates, Markdown (sample 7): p by kind and position --+ * functions outside the doc's home never stored, inside it from 0.30, for a+ * doc of the whole project (samples 4 and 7 pooled) from 0.30 too; local+ * extras 0.32 inside the home+ * only; whole files from 0.40; nothing below CBM_SEM_DOC_MIN_SCORE; every+ * curve monotone. A format without a judged curve stores nothing. */TEST(sem_doc_calibrated_p_bands) {- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.10F), 0.06F, 0.0);- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.199F), 0.06F, 0.0);- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(CBM_SEM_DOC_MIN_SCORE), 0.36F, 0.0);- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.299F), 0.36F, 0.0);- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.30F), 0.42F, 0.0);- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.40F), 0.548F, 0.0);- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(1.00F), 0.548F, 0.0);- float last = 0.0F;+ const cbm_sem_doc_format_t md = CBM_SEM_DOC_FMT_MARKDOWN;+ const cbm_sem_doc_kind_t fn = CBM_SEM_DOC_KIND_FUNCTION;+ const cbm_sem_doc_pos_t global = CBM_SEM_DOC_POS_GLOBAL;+ const cbm_sem_doc_pos_t local = CBM_SEM_DOC_POS_LOCAL;+ const cbm_sem_doc_pos_t outside = CBM_SEM_DOC_POS_OUTSIDE;+ ASSERT_FLOAT_EQ(docp(md, fn, local, 0.199F), 0.0F, 0.0);+ ASSERT_FLOAT_EQ(docp(md, fn, local, CBM_SEM_DOC_MIN_SCORE), 0.0F, 0.0);+ ASSERT_FLOAT_EQ(docp(md, fn, local, 0.30F), 0.325F, 0.0);+ ASSERT_FLOAT_EQ(docp(md, fn, local, 0.40F), 0.525F, 0.0);+ ASSERT_FLOAT_EQ(docp(md, fn, global, 0.299F), 0.0F, 0.0);+ ASSERT_FLOAT_EQ(docp(md, fn, global, 0.30F), 0.217F, 0.0);+ ASSERT_FLOAT_EQ(docp(md, fn, global, 0.40F), 0.509F, 0.0);+ ASSERT_FLOAT_EQ(docp(md, fn, outside, 0.90F), 0.0F, 0.0);+ ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.199F), 0.0F, 0.0);+ ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.20F), 0.317F, 0.0);+ ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.90F), 0.317F, 0.0);+ ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FILE, outside, 0.399F), 0.0F, 0.0);+ ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FILE, outside, 0.40F), 0.30F, 0.0);+ ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FOLDER, local, 0.90F), 0.0F, 0.0);+ static const cbm_sem_doc_kind_t KINDS[] = {CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_KIND_LOCAL,+ CBM_SEM_DOC_KIND_FILE};+ for (size_t k = 0; k < sizeof(KINDS) / sizeof(KINDS[0]); k++) {+ for (int pos = 0; pos < CBM_SEM_DOC_POS_COUNT; pos++) {+ float last = 0.0F;+ for (int milli = 0; milli <= 1000; milli++) {+ float s = (float)milli / 1000.0F;+ float p = docp(md, KINDS[k], (cbm_sem_doc_pos_t)pos, s);+ ASSERT_TRUE(p >= last);+ last = p;+ ASSERT_TRUE(pos != CBM_SEM_DOC_POS_OUTSIDE || KINDS[k] == CBM_SEM_DOC_KIND_FILE ||+ p == 0.0F);+ ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_OTHER, KINDS[k], (cbm_sem_doc_pos_t)pos, s),+ 0.0F, 0.0);+ ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_PDF, KINDS[k], (cbm_sem_doc_pos_t)pos, s),+ 0.0F, 0.0);+ }+ }+ }+ ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_COUNT, fn, global, 0.9F), 0.0F, 0.0);+ /* a home folder: README and index docs only, Markdown only */+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/README.md"), 0.867F, 0.0);+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/docs/index.mdx"), 0.867F, 0.0);+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/readme"), 0.867F, 0.0);+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/guide.md"), 0.0F, 0.0);+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/README-old.md"), 0.0F, 0.0);+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(CBM_SEM_DOC_FMT_RST, "pkg/mail/index.rst"), 0.0F, 0.0);+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, NULL), 0.0F, 0.0);+ /* reST (sample 6): 0.20-0.30 judged 0.125, not stored */+ const cbm_sem_doc_format_t rst = CBM_SEM_DOC_FMT_RST;+ ASSERT_FLOAT_EQ(+ cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.25F),+ 0.0F, 0.0);+ ASSERT_FLOAT_EQ(+ cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.30F),+ 0.35F, 0.0);+ ASSERT_FLOAT_EQ(+ cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.40F),+ 0.625F, 0.0);+ /* AsciiDoc (sample 6): every band at or above the floor is stored */+ const cbm_sem_doc_format_t adoc = CBM_SEM_DOC_FMT_ADOC;+ ASSERT_FLOAT_EQ(+ cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.199F),+ 0.0F, 0.0);+ ASSERT_FLOAT_EQ(+ cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.20F),+ 0.225F, 0.0);+ ASSERT_FLOAT_EQ(+ cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.35F),+ 0.25F, 0.0);+ ASSERT_FLOAT_EQ(+ cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.90F),+ 0.50F, 0.0);for (int milli = 0; milli <= 1000; milli++) {- float p = cbm_sem_doc_calibrated_p((float)milli / 1000.0F);- ASSERT_TRUE(p >= last);- last = p;+ float s = (float)milli / 1000.0F;+ ASSERT_TRUE(+ cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, s) >=+ cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL,+ s - 0.001F));+ ASSERT_TRUE(+ cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, s) >=+ cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL,+ s - 0.001F));}PASS();}+/* p is stored and shown at two decimals, half up, whatever the float+ * representation of the judged value (0.225F is 0.22499999...). */+TEST(sem_p_two_decimals_half_up) {+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.225F), 0.23, 1e-12);+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.325F), 0.33, 1e-12); /* float 0.32499998... */+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.525F), 0.53, 1e-12); /* float 0.52499997... */+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.324F), 0.32, 1e-12);+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.317F), 0.32, 1e-12);+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.867F), 0.87, 1e-12);+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.625F), 0.63, 1e-12);+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.548F), 0.55, 1e-12);+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.36F), 0.36, 1e-12);+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.0F), 0.0, 1e-12);+ PASS();+}++/* A section's format comes from its file's extension, case-insensitively;+ * a .txt section is reStructuredText (plain text has no sections). */+TEST(sem_doc_format_by_extension) {+ ASSERT_EQ(cbm_sem_doc_format("docs/guide.md"), CBM_SEM_DOC_FMT_MARKDOWN);+ ASSERT_EQ(cbm_sem_doc_format("README.MARKDOWN"), CBM_SEM_DOC_FMT_MARKDOWN);+ ASSERT_EQ(cbm_sem_doc_format("docs/index.rst"), CBM_SEM_DOC_FMT_RST);+ ASSERT_EQ(cbm_sem_doc_format("docs/topics/db.txt"), CBM_SEM_DOC_FMT_RST);+ ASSERT_EQ(cbm_sem_doc_format("manual/intro.adoc"), CBM_SEM_DOC_FMT_ADOC);+ ASSERT_EQ(cbm_sem_doc_format("paper.pdf"), CBM_SEM_DOC_FMT_PDF);+ ASSERT_EQ(cbm_sem_doc_format("src/main.go"), CBM_SEM_DOC_FMT_OTHER);+ ASSERT_EQ(cbm_sem_doc_format("v1.2/NOTES"), CBM_SEM_DOC_FMT_OTHER);+ ASSERT_EQ(cbm_sem_doc_format(NULL), CBM_SEM_DOC_FMT_OTHER);+ PASS();+}+/* A properties string holding a quote is read whole, and escapes are not* glued to the next word (until 2026-10 the read stopped at the first quote* and kept "\n" raw, so "\nreturn" became the token "nreturn"). */@@ -731,6 +841,8 @@ SUITE(semantic) {RUN_TEST(sem_admit_best_first_keeps_the_best);RUN_TEST(sem_calibrated_p_bands);RUN_TEST(sem_doc_calibrated_p_bands);+ RUN_TEST(sem_doc_format_by_extension);+ RUN_TEST(sem_p_two_decimals_half_up);RUN_TEST(sem_json_str_honours_escapes);RUN_TEST(sem_corpus_add_null_doc);RUN_TEST(sem_corpus_free_null);diff --git a/tests/test_store_arch.c b/tests/test_store_arch.cindex 6c6037770..23ea5c935 100644--- a/tests/test_store_arch.c+++ b/tests/test_store_arch.c@@ -275,7 +275,8 @@ TEST(arch_path_scoping) {ASSERT_TRUE(whole_pkg_nodes > scoped_pkg_nodes);ASSERT_EQ(scoped_pkg_nodes, 1);- ASSERT_TRUE(cbm_store_count_nodes(s, "pscope") > cbm_store_count_nodes_scoped(s, "pscope", "apps/foo"));+ ASSERT_TRUE(cbm_store_count_nodes(s, "pscope") >+ cbm_store_count_nodes_scoped(s, "pscope", "apps/foo"));cbm_architecture_info_t scoped_slash;memset(&scoped_slash, 0, sizeof(scoped_slash));@@ -703,6 +704,63 @@ TEST(doc_candidates_replace_and_get) {PASS();}+/* Whole-file candidates of a path: the rows whose target is the path's File+ * node or a Folder holding it -- never a sibling folder sharing a name prefix+ * (pkg/mailer, pk), a function in the file, or another project's rows. */+TEST(doc_candidates_for_path_file_and_folders) {+ cbm_store_t *s = cbm_store_open_memory();+ ASSERT_NOT_NULL(s);+ cbm_doc_candidate_t *got = NULL;+ int n = -1;+ ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "pkg/mail/parse.py", 5, &got, &n),+ CBM_STORE_NOT_FOUND);+ cbm_store_upsert_project(s, "p", "/tmp/p");+ const struct {+ const char *label, *qn, *path;+ } nodes[] = {{"File", "p.pkg.mail.parse.__file__", "pkg/mail/parse.py"},+ {"Function", "p.pkg.mail.parse.parse_mail", "pkg/mail/parse.py"},+ {"Folder", "p.pkg.mail", "pkg/mail"},+ {"Folder", "p.pkg", "pkg"},+ {"Folder", "p.pkg.mailer", "pkg/mailer"},+ {"Folder", "p.pk", "pk"}};+ for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) {+ cbm_node_t node = {.project = "p",+ .label = nodes[i].label,+ .name = nodes[i].qn,+ .qualified_name = nodes[i].qn,+ .file_path = nodes[i].path};+ ASSERT_TRUE(cbm_store_upsert_node(s, &node) > 0);+ }+ const cbm_doc_candidate_t rows[] = {+ {.section_qn = "p.docs.a", .target_qn = "p.pkg.mail.parse.__file__", .p = 0.5},+ {.section_qn = "p.docs.b", .target_qn = "p.pkg.mail", .p = 0.7},+ {.section_qn = "p.docs.c", .target_qn = "p.pkg", .p = 0.3},+ {.section_qn = "p.docs.d", .target_qn = "p.pkg.mailer", .p = 0.9},+ {.section_qn = "p.docs.e", .target_qn = "p.pk", .p = 0.9},+ {.section_qn = "p.docs.f", .target_qn = "p.pkg.mail.parse.parse_mail", .p = 0.95},+ };+ ASSERT_EQ(cbm_store_doc_candidates_replace(s, "p", rows, 6), CBM_STORE_OK);+ const cbm_doc_candidate_t other = {+ .section_qn = "q.docs.a", .target_qn = "p.pkg.mail", .p = 0.99};+ ASSERT_EQ(cbm_store_doc_candidates_replace(s, "q", &other, 1), CBM_STORE_OK);++ ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "pkg/mail/parse.py", 5, &got, &n),+ CBM_STORE_OK);+ ASSERT_EQ(n, 3);+ ASSERT_STR_EQ(got[0].section_qn, "p.docs.b"); /* highest p first */+ ASSERT_STR_EQ(got[1].section_qn, "p.docs.a");+ ASSERT_STR_EQ(got[2].section_qn, "p.docs.c");+ cbm_store_doc_candidates_free(got, n);++ ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "pkg/mail/parse.py", 1, &got, &n),+ CBM_STORE_OK);+ ASSERT_EQ(n, 1);+ cbm_store_doc_candidates_free(got, n);+ ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "", 5, &got, &n), CBM_STORE_ERR);+ cbm_store_close(s);+ PASS();+}+TEST(adr_parse_sections_basic) {cbm_adr_sections_t sec = cbm_adr_parse_sections("## PURPOSE\nFoo\n\n## STACK\nBar");ASSERT_EQ(sec.count, 2);@@ -1051,11 +1109,11 @@ static int adr_check_splice(const char *in, const char *name, const char *body,return 0;}-#define CHECK_SPLICE(in, name, body, expect) \- do { \- if (adr_check_splice((in), (name), (body), (expect)) != 0) { \- return 1; \- } \+#define CHECK_SPLICE(in, name, body, expect) \+ do { \+ if (adr_check_splice((in), (name), (body), (expect)) != 0) { \+ return 1; \+ } \} while (0)/* THE acceptance property. Every document below is a case where rebuilding@@ -1170,9 +1228,9 @@ TEST(adr_splice_mixed_line_endings) {TEST(adr_splice_matches_headings_across_line_endings) {adr_name_collect_t c;memset(&c, 0, sizeof(c));- ASSERT_EQ(cbm_adr_scan_headings("## PURPOSE\r\nFoo\r\n\r\n## STACK\r\nBar",- adr_collect_names, &c),- CBM_STORE_OK);+ ASSERT_EQ(+ cbm_adr_scan_headings("## PURPOSE\r\nFoo\r\n\r\n## STACK\r\nBar", adr_collect_names, &c),+ CBM_STORE_OK);ASSERT_STR_EQ(c.buf, "[PURPOSE][STACK]");/* A fenced block with CRLF still hides its heading, and still closes. */@@ -1778,6 +1836,7 @@ SUITE(store_arch) {RUN_TEST(adr_delete);RUN_TEST(adr_delete_not_found);RUN_TEST(doc_candidates_replace_and_get);+ RUN_TEST(doc_candidates_for_path_file_and_folders);RUN_TEST(adr_parse_sections_basic);RUN_TEST(adr_parse_sections_all_six);RUN_TEST(adr_parse_sections_non_canonical);