Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

feat(semantic): doc candidates by format and position; whole files and home folders

Martin Vogel75588a2b6915
diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c
index 3321eb1f6..441b3d0e3 100644
--- a/src/mcp/mcp.c
+++ b/src/mcp/mcp.c
@@ -12859,68 +12859,154 @@ static char *snippet_budget_floor(bool json_format, int max_output_tokens) {
return json;
}
-/* Doc -> code candidates (doc_link_candidates): on a Section, the functions it
- * is possibly about; on a function or method, the sections that possibly
- * describe it -- highest p first. Candidates with a judged probability for the
- * agent to verify, never links. A row whose other end no longer resolves (a
- * delta update renamed or removed it) is skipped. */
-enum { MCP_DOC_CANDIDATES_MAX = 5 };
-static void add_doc_candidates(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store,
- const cbm_node_t *node) {
- if (!store || !node->label || !node->qualified_name || !node->project) {
- return;
- }
- bool section = strcmp(node->label, "Section") == 0;
- if (!section && strcmp(node->label, "Function") != 0 && strcmp(node->label, "Method") != 0) {
- return;
- }
- cbm_doc_candidate_t *rows = NULL;
- int n = 0;
- if (cbm_store_doc_candidates_get(store, node->project, section ? node->qualified_name : NULL,
- section ? NULL : node->qualified_name, MCP_DOC_CANDIDATES_MAX,
- &rows, &n) != CBM_STORE_OK) {
- return;
- }
- yyjson_mut_val *arr = yyjson_mut_arr(doc);
+/* Doc -> code candidates (doc_link_candidates): on a Section, the functions,
+ * whole files and the home folder it is possibly about; on a function or
+ * method, the sections that possibly describe it; on a file (get_file_outline),
+ * the sections that possibly describe the file or a folder holding it --
+ * highest p first. Candidates with a judged probability for the agent to
+ * verify, never links. A row whose other end no longer resolves (a delta
+ * update renamed or removed it) is skipped. */
+enum { MCP_DOC_CANDIDATES_MAX = 5, MCP_DOC_CANDIDATES_SECTION_MAX = 10 };
+static const char MCP_DOC_CANDIDATES_NOTE[] =
+ "doc<->code candidates by shared terms or the doc's folder, not links: p is the judged "
+ "probability the section is about the target; verify";
+
+/* The evidence fields of a candidate row as shown: kind, position, and (on
+ * the section side) the shared terms and name tokens. */
+static void doc_candidate_evidence(yyjson_mut_doc *doc, yyjson_mut_val *o, const char *evidence,
+ bool counts) {
+ yyjson_doc *ev = evidence ? yyjson_read(evidence, strlen(evidence), 0) : NULL;
+ yyjson_val *root = ev ? yyjson_doc_get_root(ev) : NULL;
+ static const char *const STRS[] = {"kind", "position", NULL};
+ for (int k = 0; root && STRS[k]; k++) {
+ yyjson_val *v = yyjson_obj_get(root, STRS[k]);
+ if (v && yyjson_is_str(v)) {
+ yyjson_mut_obj_add_strcpy(doc, o, STRS[k], yyjson_get_str(v));
+ }
+ }
+ static const char *const INTS[] = {"shared_terms", "name_tokens", NULL};
+ for (int k = 0; counts && root && INTS[k]; k++) {
+ yyjson_val *v = yyjson_obj_get(root, INTS[k]);
+ if (v && yyjson_is_int(v)) {
+ yyjson_mut_obj_add_int(doc, o, INTS[k], (int)yyjson_get_int(v));
+ }
+ }
+ yyjson_doc_free(ev);
+}
+
+/* rows -> an array of shown candidates; the other end of each row is its
+ * target (by_section) or its section. Returns how many resolved. */
+static int doc_candidates_array(yyjson_mut_doc *doc, yyjson_mut_val *arr, cbm_store_t *store,
+ const char *project, const cbm_doc_candidate_t *rows, int n,
+ bool by_section) {
int added = 0;
for (int i = 0; i < n; i++) {
cbm_node_t other = {0};
- if (cbm_store_find_node_by_qn(store, node->project,
- section ? rows[i].target_qn : rows[i].section_qn,
+ if (cbm_store_find_node_by_qn(store, project,
+ by_section ? rows[i].target_qn : rows[i].section_qn,
&other) != CBM_STORE_OK) {
continue;
}
yyjson_mut_val *o = yyjson_mut_obj(doc);
- yyjson_mut_obj_add_strcpy(doc, o, section ? "qualified_name" : "section",
+ yyjson_mut_obj_add_strcpy(doc, o, by_section ? "qualified_name" : "section",
other.qualified_name ? other.qualified_name : "");
+ if (by_section) {
+ yyjson_mut_obj_add_strcpy(doc, o, "label", other.label ? other.label : "");
+ }
yyjson_mut_obj_add_strcpy(doc, o, "file_path", other.file_path ? other.file_path : "");
yyjson_mut_obj_add_int(doc, o, "start_line", other.start_line);
- if (section) {
+ if (by_section) {
yyjson_mut_obj_add_int(doc, o, "end_line", other.end_line);
}
yyjson_mut_obj_add_real(doc, o, "p", (double)(int)(rows[i].p * 1000.0 + 0.5) / 1000.0);
- int shared = 0;
- int name_tokens = 0;
- if (section && rows[i].evidence &&
- sscanf(rows[i].evidence, "{\"shared_terms\":%d,\"name_tokens\":%d}", &shared,
- &name_tokens) == 2) {
- yyjson_mut_obj_add_int(doc, o, "shared_terms", shared);
- yyjson_mut_obj_add_int(doc, o, "name_tokens", name_tokens);
- }
+ doc_candidate_evidence(doc, o, rows[i].evidence, by_section);
yyjson_mut_arr_append(arr, o);
added++;
free_node_contents(&other);
}
+ return added;
+}
+
+static void add_doc_candidates(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store,
+ const cbm_node_t *node) {
+ if (!store || !node->label || !node->qualified_name || !node->project) {
+ return;
+ }
+ bool section = strcmp(node->label, "Section") == 0;
+ if (!section && strcmp(node->label, "Function") != 0 && strcmp(node->label, "Method") != 0) {
+ return;
+ }
+ cbm_doc_candidate_t *rows = NULL;
+ int n = 0;
+ if (cbm_store_doc_candidates_get(store, node->project, section ? node->qualified_name : NULL,
+ section ? NULL : node->qualified_name,
+ section ? MCP_DOC_CANDIDATES_SECTION_MAX
+ : MCP_DOC_CANDIDATES_MAX,
+ &rows, &n) != CBM_STORE_OK) {
+ return;
+ }
+ yyjson_mut_val *arr = yyjson_mut_arr(doc);
+ int added = doc_candidates_array(doc, arr, store, node->project, rows, n, section);
cbm_store_doc_candidates_free(rows, n);
if (added > 0) {
yyjson_mut_obj_add_val(doc, root, section ? "possibly_about" : "possibly_described_in",
arr);
- yyjson_mut_obj_add_str(doc, root, "candidates_note",
- "doc<->code candidates by shared terms, not links: p is the "
- "judged probability the section is about the function; verify");
+ yyjson_mut_obj_add_str(doc, root, "candidates_note", MCP_DOC_CANDIDATES_NOTE);
}
}
+/* get_file_outline: the sections possibly about the file as a whole or about
+ * a folder holding it (NULL: none). */
+static yyjson_mut_val *file_doc_candidates(yyjson_mut_doc *doc, cbm_store_t *store,
+ const char *project, const char *path) {
+ cbm_doc_candidate_t *rows = NULL;
+ int n = 0;
+ if (cbm_store_doc_candidates_for_path(store, project, path, MCP_DOC_CANDIDATES_MAX, &rows,
+ &n) != CBM_STORE_OK) {
+ return NULL;
+ }
+ yyjson_mut_val *arr = yyjson_mut_arr(doc);
+ int added = doc_candidates_array(doc, arr, store, project, rows, n, false);
+ cbm_store_doc_candidates_free(rows, n);
+ return added > 0 ? arr : NULL;
+}
+
+static const char *mut_str(yyjson_mut_val *o, const char *key) {
+ yyjson_mut_val *v = yyjson_mut_obj_get(o, key);
+ return v && yyjson_mut_is_str(v) ? yyjson_mut_get_str(v) : "";
+}
+
+/* The tree form of file_doc_candidates. */
+static void file_doc_candidates_tree(cbm_sb_t *sb, cbm_store_t *store, const char *project,
+ const char *path) {
+ yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
+ yyjson_mut_val *arr = doc ? file_doc_candidates(doc, store, project, path) : NULL;
+ if (arr) {
+ static const char *const columns[] = {"section", "file_path", "line", "p", "kind"};
+ cbm_tree_table_header(sb, "possibly_described_in", (int)yyjson_mut_arr_size(arr), columns,
+ (int)(sizeof(columns) / sizeof(columns[0])));
+ size_t i = 0;
+ size_t max = 0;
+ yyjson_mut_val *o = NULL;
+ yyjson_mut_arr_foreach(arr, i, max, o) {
+ char line[CBM_SZ_32];
+ char p[CBM_SZ_32];
+ snprintf(line, sizeof(line), "%d",
+ (int)yyjson_mut_get_int(yyjson_mut_obj_get(o, "start_line")));
+ snprintf(p, sizeof(p), "%.2f", yyjson_mut_get_real(yyjson_mut_obj_get(o, "p")));
+ cbm_tree_row_begin(sb);
+ cbm_tree_cell_str(sb, mut_str(o, "section"), true);
+ cbm_tree_cell_str(sb, mut_str(o, "file_path"), false);
+ cbm_tree_cell_str(sb, line, false);
+ cbm_tree_cell_str(sb, p, false);
+ cbm_tree_cell_str(sb, mut_str(o, "kind"), false);
+ cbm_tree_row_end(sb);
+ }
+ cbm_tree_scalar_str(sb, "candidates_note", MCP_DOC_CANDIDATES_NOTE);
+ }
+ yyjson_mut_doc_free(doc);
+}
+
static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node,
const char *match_method, bool include_neighbors,
cbm_node_t *alternatives, int alt_count, const char *args) {
@@ -13404,6 +13490,9 @@ static char *handle_get_file_outline(cbm_mcp_server_t *srv, const char *args) {
cbm_tree_scalar_int(&sb, "limit", limit);
cbm_tree_scalar_int(&sb, "returned", row_count);
cbm_tree_scalar_bool(&sb, "has_more", (int64_t)offset + row_count < total);
+ if (offset == 0) {
+ file_doc_candidates_tree(&sb, store, project, normalized_path);
+ }
payload = cbm_sb_finish(&sb);
} else {
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
@@ -13439,6 +13528,12 @@ static char *handle_get_file_outline(cbm_mcp_server_t *srv, const char *args) {
yyjson_mut_obj_add_int(doc, object, "limit", limit);
yyjson_mut_obj_add_int(doc, object, "returned", row_count);
yyjson_mut_obj_add_bool(doc, object, "has_more", (int64_t)offset + row_count < total);
+ yyjson_mut_val *described =
+ offset == 0 ? file_doc_candidates(doc, store, project, normalized_path) : NULL;
+ if (described) {
+ yyjson_mut_obj_add_val(doc, object, "possibly_described_in", described);
+ yyjson_mut_obj_add_str(doc, object, "candidates_note", MCP_DOC_CANDIDATES_NOTE);
+ }
payload = yy_doc_to_str(doc);
yyjson_mut_doc_free(doc);
}
diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c
index b0cd4dd3f..d406306e3 100644
--- a/src/pipeline/pass_semantic_edges.c
+++ b/src/pipeline/pass_semantic_edges.c
@@ -1920,9 +1920,20 @@ typedef struct {
float name_idf; /* the highest idf among them (0: none) */
} doc_hit_t;
+typedef struct {
+ int file;
+ float tfidf;
+ int rank;
+ int shared;
+} doc_file_hit_t;
+
typedef struct {
doc_hit_t hits[SEM_DOC_TOP_K];
int n;
+ doc_hit_t local[CBM_SEM_DOC_LOCAL_K]; /* inside the home, below the top CBM_SEM_DOC_TOP_K */
+ int nlocal;
+ doc_file_hit_t files[CBM_SEM_DOC_FILE_K];
+ int nfiles;
} doc_result_t;
static int cmp_doc_cand(const void *pa, const void *pb) {
@@ -2062,11 +2073,11 @@ static bool doc_section_gated(const cbm_gbuf_node_t *sec, char *body) {
return words >= SEM_DOC_MIN_WORDS && links * 100 < SEM_DOC_LINK_PCT_MAX * words;
}
-/* A test function, by its file (test directories, test file names) or name. */
-static bool doc_is_test_function(const cbm_gbuf_node_t *fn) {
+/* A test file: in a test directory, or a test file name. */
+static bool doc_is_test_path(const char *path) {
static const char *const DIRS[] = {"test/", "tests/", "__tests__/", "testing/", "spec/",
"specs/", "testdata/", "fixture/", "fixtures/", NULL};
- const char *path = fn && fn->file_path ? fn->file_path : "";
+ path = path ? path : "";
for (int k = 0; DIRS[k]; k++) {
for (const char *p = path; (p = strstr(p, DIRS[k])) != NULL; p++) {
if (p == path || p[-SKIP_ONE] == '/') {
@@ -2076,9 +2087,14 @@ static bool doc_is_test_function(const cbm_gbuf_node_t *fn) {
}
const char *base = strrchr(path, '/');
base = base ? base + SKIP_ONE : path;
- if (strncmp(base, "test_", sizeof("test_") - SKIP_ONE) == 0 || strstr(base, "_test.") ||
- strstr(base, ".test.") || strstr(base, ".spec.") || strstr(base, "Test.") ||
- strstr(base, "Tests.")) {
+ return strncmp(base, "test_", sizeof("test_") - SKIP_ONE) == 0 || strstr(base, "_test.") ||
+ strstr(base, ".test.") || strstr(base, ".spec.") || strstr(base, "Test.") ||
+ strstr(base, "Tests.");
+}
+
+/* A test function, by its file or name. */
+static bool doc_is_test_function(const cbm_gbuf_node_t *fn) {
+ if (doc_is_test_path(fn ? fn->file_path : NULL)) {
return true;
}
const char *name = fn && fn->name ? fn->name : "";
@@ -2126,6 +2142,449 @@ static float terms_norm(const float *w, int n) {
return sqrtf(s);
}
+/* ── Phase 6c: whole files and the doc's home folder ── */
+
+/* Whole-file candidates: a file's terms are the sum of its non-test
+ * functions' TF-IDF vectors and the TF-IDF vectors of its other definitions'
+ * names (a config file's keys are Variable nodes), cut to its
+ * SEM_DOC_FILE_TERMS heaviest terms (ties by token index), so a file's vector
+ * costs at most what one long function's does. */
+enum { SEM_DOC_FILE_TERMS = 64 };
+static const char *const DOC_FILE_DEF_LABELS[] = {"Class", "Struct", "Interface", "Enum",
+ "Type", "Trait", "Field", "Variable",
+ "Constant", "Macro", NULL};
+
+typedef struct {
+ int count;
+ const cbm_gbuf_node_t **node; /* per file: its File node */
+ int *off; /* count + 1: each file's terms in idx / w */
+ int *idx;
+ float *w;
+ float *norm;
+ int *post_off; /* nterms + 1: the files holding each term */
+ int *post_f;
+ CBMHashTable *by_path; /* file path (graph-owned) -> file index + 1 */
+} doc_files_t;
+
+typedef struct {
+ int idx;
+ float w;
+} doc_term_t;
+
+static int cmp_term_idx(const void *pa, const void *pb) {
+ const doc_term_t *a = pa;
+ const doc_term_t *b = pb;
+ return (a->idx > b->idx) - (a->idx < b->idx);
+}
+
+static int cmp_term_heavy(const void *pa, const void *pb) {
+ const doc_term_t *a = pa;
+ const doc_term_t *b = pb;
+ if (a->w != b->w) {
+ return a->w > b->w ? -1 : 1;
+ }
+ return (a->idx > b->idx) - (a->idx < b->idx);
+}
+
+static void doc_files_free(doc_files_t *df) {
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->node);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->off);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->idx);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->w);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->norm);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->post_off);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, df->post_f);
+ if (df->by_path) {
+ cbm_ht_free(df->by_path);
+ }
+ memset(df, 0, sizeof(*df));
+}
+
+static int doc_file_of(const doc_files_t *df, const char *path) {
+ return path ? (int)(intptr_t)cbm_ht_get(df->by_path, path) - SKIP_ONE : -SKIP_ONE;
+}
+
+/* The scratch terms of one file: append n terms, growing the buffer. */
+typedef struct {
+ doc_term_t *t;
+ int n;
+ int cap;
+} doc_scratch_t;
+
+static bool doc_scratch_add(doc_scratch_t *sc, const int *idx, const float *w, int n) {
+ if (sc->n + n > sc->cap) {
+ int cap = sc->cap ? sc->cap : CBM_SZ_256;
+ while (cap < sc->n + n) {
+ cap *= 2;
+ }
+ doc_term_t *t = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, sc->t, (size_t)cap * sizeof(*t));
+ if (!t) {
+ return false;
+ }
+ sc->t = t;
+ sc->cap = cap;
+ }
+ for (int k = 0; k < n; k++) {
+ sc->t[sc->n++] = (doc_term_t){.idx = idx[k], .w = w[k]};
+ }
+ return true;
+}
+
+/* Sum the scratch terms per token and keep the heaviest SEM_DOC_FILE_TERMS,
+ * written at df->idx/w + off in ascending token order; returns how many. */
+static int doc_scratch_reduce(doc_scratch_t *sc, doc_files_t *df, int off) {
+ if (sc->n == 0) {
+ return 0;
+ }
+ qsort(sc->t, (size_t)sc->n, sizeof(*sc->t), cmp_term_idx);
+ int m = 0;
+ for (int k = 0; k < sc->n; k++) {
+ if (m > 0 && sc->t[m - SKIP_ONE].idx == sc->t[k].idx) {
+ sc->t[m - SKIP_ONE].w += sc->t[k].w;
+ } else {
+ sc->t[m++] = sc->t[k];
+ }
+ }
+ if (m > SEM_DOC_FILE_TERMS) {
+ qsort(sc->t, (size_t)m, sizeof(*sc->t), cmp_term_heavy);
+ m = SEM_DOC_FILE_TERMS;
+ qsort(sc->t, (size_t)m, sizeof(*sc->t), cmp_term_idx);
+ }
+ for (int k = 0; k < m; k++) {
+ df->idx[off + k] = sc->t[k].idx;
+ df->w[off + k] = sc->t[k].w;
+ }
+ return m;
+}
+
+/* The definitions (non-function) of every indexed non-test file, grouped by
+ * file: def_off (count + 1) into def_node. */
+static bool doc_files_defs(const cbm_gbuf_t *gbuf, const doc_files_t *df, int **def_off,
+ const cbm_gbuf_node_t ***def_node) {
+ *def_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(int));
+ if (!*def_off) {
+ return false;
+ }
+ for (int pass = 0; pass < 2; pass++) {
+ int *cursor =
+ pass ? cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)df->count * sizeof(int)) : NULL;
+ if (pass && !cursor) {
+ return false;
+ }
+ for (int l = 0; DOC_FILE_DEF_LABELS[l]; l++) {
+ const cbm_gbuf_node_t **nodes = NULL;
+ int n = 0;
+ if (cbm_gbuf_find_by_label(gbuf, DOC_FILE_DEF_LABELS[l], &nodes, &n) != 0) {
+ continue;
+ }
+ for (int i = 0; i < n; i++) {
+ int f = nodes[i]->name ? doc_file_of(df, nodes[i]->file_path) : -SKIP_ONE;
+ if (f >= 0 && !pass) {
+ (*def_off)[f + SKIP_ONE]++;
+ } else if (f >= 0) {
+ (*def_node)[(*def_off)[f] + cursor[f]++] = nodes[i];
+ }
+ }
+ }
+ if (!pass) {
+ for (int f = 0; f < df->count; f++) {
+ (*def_off)[f + SKIP_ONE] += (*def_off)[f];
+ }
+ *def_node = cbm_alloc(CBM_MEM_CLASS_SEMANTIC,
+ ((size_t)(*def_off)[df->count] + SKIP_ONE) * sizeof(**def_node));
+ if (!*def_node) {
+ return false;
+ }
+ }
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, cursor);
+ }
+ return true;
+}
+
+/* The scratch terms of one definition's name. */
+static bool doc_scratch_name(doc_scratch_t *sc, const cbm_sem_corpus_t *corpus, const char *name) {
+ char *tok[SEM_DOC_NAME_TOKENS];
+ int idx[SEM_DOC_NAME_TOKENS];
+ float w[SEM_DOC_NAME_TOKENS];
+ int nt = cbm_sem_tokenize(name, tok, SEM_DOC_NAME_TOKENS);
+ int nw = nt > 0 ? cbm_sem_tfidf_terms(corpus, tok, NULL, nt, idx, w) : 0;
+ for (int t = 0; t < nt; t++) {
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, tok[t]);
+ }
+ return doc_scratch_add(sc, idx, w, nw);
+}
+
+/* Each file's term vector, norm, and the term -> files postings. */
+static bool doc_files_vectors(const cbm_gbuf_t *gbuf, doc_files_t *df, const cbm_sem_func_t *funcs,
+ const int *ffile, int func_count, const cbm_sem_corpus_t *corpus,
+ int nterms) {
+ int *fn_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(int));
+ int *fn_list = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)func_count + SKIP_ONE) * sizeof(int));
+ int *def_off = NULL;
+ const cbm_gbuf_node_t **def_node = NULL;
+ doc_scratch_t sc = {0};
+ bool ok = fn_off && fn_list && doc_files_defs(gbuf, df, &def_off, &def_node);
+ for (int f = 0; ok && f < func_count; f++) {
+ fn_off[ffile[f] + SKIP_ONE] += ffile[f] >= 0;
+ }
+ for (int i = 0; ok && i < df->count; i++) {
+ fn_off[i + SKIP_ONE] += fn_off[i];
+ }
+ for (int f = 0; ok && f < func_count; f++) {
+ if (ffile[f] >= 0) {
+ fn_list[fn_off[ffile[f]]++] = f; /* fn_off[i] ends at file i's end: shifted below */
+ }
+ }
+ for (int i = df->count; ok && i > 0; i--) {
+ fn_off[i] = fn_off[i - SKIP_ONE];
+ }
+ if (ok) {
+ fn_off[0] = 0;
+ }
+ for (int i = 0; ok && i < df->count; i++) {
+ sc.n = 0;
+ for (int k = fn_off[i]; ok && k < fn_off[i + SKIP_ONE]; k++) {
+ const cbm_sem_func_t *fn = &funcs[fn_list[k]];
+ ok = doc_scratch_add(&sc, fn->tfidf_indices, fn->tfidf_weights, fn->tfidf_len);
+ }
+ for (int k = def_off[i]; ok && k < def_off[i + SKIP_ONE]; k++) {
+ ok = doc_scratch_name(&sc, corpus, def_node[k]->name);
+ }
+ int m = ok ? doc_scratch_reduce(&sc, df, df->off[i]) : 0;
+ df->off[i + SKIP_ONE] = df->off[i] + m;
+ df->norm[i] = terms_norm(df->w + df->off[i], m);
+ for (int k = 0; k < m; k++) {
+ df->post_off[df->idx[df->off[i] + k] + SKIP_ONE]++;
+ }
+ }
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, sc.t);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, fn_off);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, fn_list);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, def_off);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, def_node);
+ if (!ok) {
+ return false;
+ }
+ for (int t = 0; t < nterms; t++) {
+ df->post_off[t + SKIP_ONE] += df->post_off[t];
+ }
+ df->post_f =
+ cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->post_off[nterms] + SKIP_ONE) * sizeof(int));
+ int *cursor = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int));
+ ok = df->post_f && cursor;
+ for (int i = 0; ok && i < df->count; i++) {
+ for (int k = df->off[i]; k < df->off[i + SKIP_ONE]; k++) {
+ int t = df->idx[k];
+ df->post_f[df->post_off[t] + cursor[t]++] = i;
+ }
+ }
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, cursor);
+ return ok;
+}
+
+/* The indexed non-test files (their File nodes) with their term vectors;
+ * ffile[f] = function f's file (-1: a test function or no File node). */
+static bool doc_files_build(const cbm_gbuf_t *gbuf, const cbm_sem_func_t *funcs, int func_count,
+ const bool *is_test, const cbm_sem_corpus_t *corpus, int nterms,
+ doc_files_t *df, int *ffile) {
+ memset(df, 0, sizeof(*df));
+ const cbm_gbuf_node_t **files = NULL;
+ int nfiles = 0;
+ if (cbm_gbuf_find_by_label(gbuf, "File", &files, &nfiles) != 0) {
+ nfiles = 0;
+ }
+ df->by_path = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, (uint32_t)nfiles + SKIP_ONE);
+ df->node = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nfiles + SKIP_ONE) * sizeof(*df->node));
+ if (!df->by_path || !df->node) {
+ return false;
+ }
+ for (int i = 0; i < nfiles; i++) {
+ const char *path = files[i]->file_path;
+ if (path && path[0] && !doc_is_test_path(path) && !cbm_ht_has(df->by_path, path)) {
+ df->node[df->count] = files[i];
+ cbm_ht_set(df->by_path, path, (void *)(intptr_t)(df->count + SKIP_ONE));
+ df->count++;
+ }
+ }
+ for (int f = 0; f < func_count; f++) {
+ const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[f].node_id);
+ ffile[f] = is_test[f] || !fn ? -SKIP_ONE : doc_file_of(df, fn->file_path);
+ }
+ size_t cap = (size_t)df->count * SEM_DOC_FILE_TERMS + SKIP_ONE;
+ df->off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(int));
+ df->idx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, cap * sizeof(int));
+ df->w = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, cap * sizeof(float));
+ df->norm = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)df->count + SKIP_ONE) * sizeof(float));
+ df->post_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int));
+ return df->off && df->idx && df->w && df->norm && df->post_off &&
+ doc_files_vectors(gbuf, df, funcs, ffile, func_count, corpus, nterms);
+}
+
+/* A doc's home folder: the folder whose code it documents. From the doc's
+ * directory, a docs folder (doc, docs, documentation: the deepest one)
+ * documents its parent; then climb while the folder holds no indexed code
+ * (a non-test file with terms). "" = the repository root: the doc documents
+ * the whole project. Returns an allocated string (NULL: out of memory). */
+static bool doc_dir_is_docs(const char *name, size_t n) {
+ static const char *const DOCS[] = {"doc", "docs", "documentation", NULL};
+ for (int k = 0; DOCS[k]; k++) {
+ if (strlen(DOCS[k]) == n && strncasecmp(name, DOCS[k], n) == 0) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static char *doc_home(const char *doc_path, const CBMHashTable *code_dirs) {
+ const char *slash = doc_path ? strrchr(doc_path, '/') : NULL;
+ size_t len = slash ? (size_t)(slash - doc_path) : 0;
+ char *dir = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, len + SKIP_ONE);
+ if (!dir) {
+ return NULL;
+ }
+ memcpy(dir, doc_path ? doc_path : "", len);
+ dir[len] = '\0';
+ size_t cut = len + SKIP_ONE; /* the deepest docs folder's start (none) */
+ for (size_t a = 0; a < len;) {
+ const char *end = strchr(dir + a, '/');
+ size_t b = end ? (size_t)(end - dir) : len;
+ if (doc_dir_is_docs(dir + a, b - a)) {
+ cut = a;
+ }
+ a = b + SKIP_ONE;
+ }
+ if (cut <= len) {
+ len = cut > 0 ? cut - SKIP_ONE : 0; /* the docs folder's parent */
+ dir[len] = '\0';
+ }
+ while (len > 0 && !cbm_ht_has(code_dirs, dir)) {
+ const char *up = strrchr(dir, '/');
+ len = up ? (size_t)(up - dir) : 0;
+ dir[len] = '\0';
+ }
+ return dir;
+}
+
+/* Every folder holding indexed code: the directories (and their ancestors)
+ * of the files with terms; keys owned by the table's caller (freed by
+ * doc_code_dirs_free). */
+static void doc_free_key(const char *key, void *value, void *userdata) {
+ (void)value;
+ (void)userdata;
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)key);
+}
+
+static void doc_code_dirs_free(CBMHashTable *dirs) {
+ if (dirs) {
+ cbm_ht_foreach(dirs, doc_free_key, NULL);
+ cbm_ht_free(dirs);
+ }
+}
+
+static CBMHashTable *doc_code_dirs(const doc_files_t *df) {
+ CBMHashTable *dirs = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, CBM_SZ_256);
+ for (int i = 0; dirs && i < df->count; i++) {
+ const char *path = df->node[i]->file_path;
+ if (df->off[i + SKIP_ONE] == df->off[i]) {
+ continue;
+ }
+ for (const char *p = strchr(path, '/'); p; p = strchr(p + SKIP_ONE, '/')) {
+ size_t n = (size_t)(p - path);
+ char *key = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, n + SKIP_ONE);
+ if (!key) {
+ doc_code_dirs_free(dirs);
+ return NULL;
+ }
+ memcpy(key, path, n);
+ key[n] = '\0';
+ if (cbm_ht_has(dirs, key)) {
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, key);
+ } else {
+ cbm_ht_set(dirs, key, (void *)(intptr_t)SKIP_ONE);
+ }
+ }
+ }
+ return dirs;
+}
+
+/* A candidate's position: inside the home (or the home folder itself),
+ * outside it, or no position (the home is the repository root). */
+static cbm_sem_doc_pos_t doc_position(const char *home, const char *path) {
+ if (!home || !home[0]) {
+ return CBM_SEM_DOC_POS_GLOBAL;
+ }
+ size_t n = strlen(home);
+ return path && strncmp(path, home, n) == 0 && (path[n] == '/' || path[n] == '\0')
+ ? CBM_SEM_DOC_POS_LOCAL
+ : CBM_SEM_DOC_POS_OUTSIDE;
+}
+
+static const char *doc_position_name(cbm_sem_doc_pos_t pos) {
+ return pos == CBM_SEM_DOC_POS_LOCAL ? "local"
+ : pos == CBM_SEM_DOC_POS_OUTSIDE ? "outside"
+ : "global";
+}
+
+/* Per section: its doc's home, whether it is its doc's first section (the
+ * one a folder candidate starts from), and the Folder nodes by path. */
+typedef struct {
+ char **home;
+ bool *first;
+ CBMHashTable *folders; /* folder path (graph-owned) -> Folder node */
+} doc_place_t;
+
+static void doc_place_free(doc_place_t *pl, int nsec) {
+ for (int s = 0; pl->home && s < nsec; s++) {
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, pl->home[s]);
+ }
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, pl->home);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, pl->first);
+ if (pl->folders) {
+ cbm_ht_free(pl->folders);
+ }
+ memset(pl, 0, sizeof(*pl));
+}
+
+static bool doc_place_build(const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t **secs, int nsec,
+ const doc_files_t *df, doc_place_t *pl) {
+ memset(pl, 0, sizeof(*pl));
+ pl->home = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(*pl->home));
+ pl->first = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(*pl->first));
+ pl->folders = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, CBM_SZ_256);
+ CBMHashTable *firsts = cbm_ht_create_in(CBM_MEM_CLASS_SEMANTIC, CBM_SZ_256);
+ CBMHashTable *dirs = doc_code_dirs(df);
+ bool ok = pl->home && pl->first && pl->folders && firsts && dirs;
+ for (int s = 0; ok && s < nsec; s++) {
+ pl->home[s] = doc_home(secs[s]->file_path, dirs);
+ ok = pl->home[s] != NULL;
+ const char *path = secs[s]->file_path;
+ if (ok && path) {
+ int prev = (int)(intptr_t)cbm_ht_get(firsts, path) - SKIP_ONE;
+ if (prev < 0 || secs[s]->start_line < secs[prev]->start_line) {
+ cbm_ht_set(firsts, path, (void *)(intptr_t)(s + SKIP_ONE));
+ }
+ }
+ }
+ for (int s = 0; ok && s < nsec; s++) {
+ const char *path = secs[s]->file_path;
+ pl->first[s] = path && (int)(intptr_t)cbm_ht_get(firsts, path) == s + SKIP_ONE;
+ }
+ const cbm_gbuf_node_t **folders = NULL;
+ int nfold = 0;
+ if (ok && cbm_gbuf_find_by_label(gbuf, "Folder", &folders, &nfold) == 0) {
+ for (int i = 0; i < nfold; i++) {
+ if (folders[i]->file_path && folders[i]->file_path[0]) {
+ cbm_ht_set(pl->folders, folders[i]->file_path, (void *)folders[i]);
+ }
+ }
+ }
+ if (firsts) {
+ cbm_ht_free(firsts);
+ }
+ doc_code_dirs_free(dirs);
+ return ok;
+}
+
typedef struct {
const cbm_sem_func_t *funcs;
int func_count;
@@ -2139,6 +2598,10 @@ typedef struct {
const bool *is_test;
const int *name_off; /* per function: its name tokens' corpus indices */
const int *name_idx;
+ const char *const *fpath; /* per function: its file */
+ const char *const *home; /* per section: its doc's home ("" = the root) */
+ const doc_files_t *files;
+ int file_key_df;
doc_result_t *res;
_Atomic int next;
} doc_ctx_t;
@@ -2169,10 +2632,73 @@ static void doc_name_evidence(const doc_ctx_t *dc, const int *sidx, int ns, doc_
}
}
+/* The section's best CBM_SEM_DOC_FILE_K whole files, retrieved by the
+ * section's key terms over the files' postings, scored by cosine. */
+static void doc_section_files(doc_ctx_t *dc, int s, const int *sidx, const float *sw, int ns,
+ float snorm, int *fstamp, doc_cand_t *fc) {
+ const doc_files_t *df = dc->files;
+ doc_result_t *r = &dc->res[s];
+ int nc = 0;
+ for (int k = 0; k < ns && snorm > 0.0F; k++) {
+ int t = sidx[k];
+ if (df->post_off[t + SKIP_ONE] - df->post_off[t] > dc->file_key_df) {
+ continue;
+ }
+ for (int p = df->post_off[t]; p < df->post_off[t + SKIP_ONE]; p++) {
+ int f = df->post_f[p];
+ if (fstamp[f] != s) {
+ fstamp[f] = s;
+ fc[nc++] = (doc_cand_t){.func = f};
+ }
+ }
+ }
+ for (int c = 0; c < nc; c++) {
+ int f = fc[c].func;
+ int len = df->off[f + SKIP_ONE] - df->off[f];
+ float dot = doc_terms_dot(sidx, sw, ns, df->idx + df->off[f], df->w + df->off[f], len,
+ &fc[c].shared);
+ float denom = snorm * df->norm[f];
+ fc[c].score = denom > 0.0F ? dot / denom : 0.0F;
+ }
+ if (nc > 1) {
+ qsort(fc, (size_t)nc, sizeof(fc[0]), cmp_doc_cand);
+ }
+ for (int c = 0; c < nc && c < CBM_SEM_DOC_FILE_K; c++) {
+ r->files[r->nfiles++] = (doc_file_hit_t){
+ .file = fc[c].func, .tfidf = fc[c].score, .rank = c + SKIP_ONE, .shared = fc[c].shared};
+ }
+}
+
+/* Up to CBM_SEM_DOC_LOCAL_K functions inside the section's home that rank
+ * below the top CBM_SEM_DOC_TOP_K, at CBM_SEM_DOC_MIN_SCORE or more (cands
+ * sorted best first). */
+static void doc_section_local(doc_ctx_t *dc, int s, const doc_cand_t *cands, int nc,
+ const int *sidx, int ns) {
+ doc_result_t *r = &dc->res[s];
+ const char *home = dc->home[s];
+ for (int c = CBM_SEM_DOC_TOP_K; home && home[0] && c < nc && r->nlocal < CBM_SEM_DOC_LOCAL_K;
+ c++) {
+ if (cands[c].score < CBM_SEM_DOC_MIN_SCORE) {
+ break;
+ }
+ if (doc_position(home, dc->fpath[cands[c].func]) != CBM_SEM_DOC_POS_LOCAL) {
+ continue;
+ }
+ doc_hit_t *hit = &r->local[r->nlocal++];
+ *hit = (doc_hit_t){.func = cands[c].func,
+ .tfidf = cands[c].score,
+ .rank = c + SKIP_ONE,
+ .shared = cands[c].shared};
+ doc_name_evidence(dc, sidx, ns, hit);
+ }
+}
+
static void doc_section_one(doc_ctx_t *dc, int s, int *stamp, doc_cand_t *cands, int *sidx,
- float *sw, char *body) {
+ float *sw, char *body, int *fstamp, doc_cand_t *fc) {
doc_result_t *r = &dc->res[s];
r->n = 0;
+ r->nlocal = 0;
+ r->nfiles = 0;
if (!doc_section_gated(dc->secs[s], body)) {
return;
}
@@ -2215,100 +2741,209 @@ static void doc_section_one(doc_ctx_t *dc, int s, int *stamp, doc_cand_t *cands,
.shared = cands[c].shared};
doc_name_evidence(dc, sidx, ns, hit);
}
+ doc_section_local(dc, s, cands, nc, sidx, ns);
+ doc_section_files(dc, s, sidx, sw, ns, snorm, fstamp, fc);
}
static void doc_section_worker(int worker_id, void *ctx_ptr) {
(void)worker_id;
doc_ctx_t *dc = ctx_ptr;
+ int nf = dc->files->count;
int *stamp = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)dc->func_count * sizeof(int));
doc_cand_t *cands =
cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)dc->func_count * sizeof(doc_cand_t));
+ int *fstamp = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nf + SKIP_ONE) * sizeof(int));
+ doc_cand_t *fc = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nf + SKIP_ONE) * sizeof(*fc));
int *sidx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)SEM_DOC_MAX_TOKENS * sizeof(int));
float *sw = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)SEM_DOC_MAX_TOKENS * sizeof(float));
char *body = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, SEM_DOC_TEXT_MAX);
- if (stamp && cands && sidx && sw && body) {
+ if (stamp && cands && fstamp && fc && sidx && sw && body) {
for (int f = 0; f < dc->func_count; f++) {
stamp[f] = -1;
}
+ for (int f = 0; f < nf; f++) {
+ fstamp[f] = -1;
+ }
while (true) {
int s = atomic_fetch_add_explicit(&dc->next, SKIP_ONE, memory_order_relaxed);
if (s >= dc->nsec) {
break;
}
- doc_section_one(dc, s, stamp, cands, sidx, sw, body);
+ doc_section_one(dc, s, stamp, cands, sidx, sw, body, fstamp, fc);
}
}
cbm_free(CBM_MEM_CLASS_SEMANTIC, stamp);
cbm_free(CBM_MEM_CLASS_SEMANTIC, cands);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, fstamp);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, fc);
cbm_free(CBM_MEM_CLASS_SEMANTIC, sidx);
cbm_free(CBM_MEM_CLASS_SEMANTIC, sw);
cbm_free(CBM_MEM_CLASS_SEMANTIC, body);
}
-static void doc_section_lines(FILE *out, const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sec,
- const cbm_sem_func_t *funcs, const doc_result_t *r) {
- for (int h = 0; h < r->n; h++) {
- const doc_hit_t *hit = &r->hits[h];
- const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[hit->func].node_id);
- (void)fprintf(out, "%s\t%s\t%.6f\t%d\t%d\t%s:%d-%d\t%s:%d-%d\t%d\t%.6f\n",
- sec->qualified_name ? sec->qualified_name : "?",
- fn && fn->qualified_name ? fn->qualified_name : "?", (double)hit->tfidf,
- hit->rank, hit->shared, sec->file_path ? sec->file_path : "?",
- sec->start_line, sec->end_line, fn && fn->file_path ? fn->file_path : "?",
- fn ? fn->start_line : 0, fn ? fn->end_line : 0, hit->name_shared,
- (double)hit->name_idf);
+static const char *const DOC_KIND_NAME[CBM_SEM_DOC_KIND_COUNT] = {"function", "local", "file",
+ "folder"};
+
+/* The doc's home Folder node, for the doc's first section (NULL: another
+ * section, or the home is the repository root). */
+static const cbm_gbuf_node_t *doc_home_folder(const doc_place_t *pl, int s) {
+ return pl->first[s] && pl->home[s][0] ? cbm_ht_get(pl->folders, pl->home[s]) : NULL;
+}
+
+static cbm_sem_doc_pos_t doc_target_position(const char *home, const cbm_gbuf_node_t *target,
+ cbm_sem_doc_kind_t kind) {
+ return kind == CBM_SEM_DOC_KIND_FOLDER ? CBM_SEM_DOC_POS_LOCAL
+ : doc_position(home, target->file_path);
+}
+
+/* One CBM_SEM_DOC_SIGNALS line: section and target qualified names, tfidf,
+ * rank, shared terms, both locations, the name evidence, then the kind, the
+ * position and the doc's home ("." = the repository root). */
+static void doc_signal_line(FILE *out, const cbm_gbuf_node_t *sec, const cbm_gbuf_node_t *t,
+ const doc_hit_t *hit, cbm_sem_doc_kind_t kind, const char *home) {
+ (void)fprintf(out, "%s\t%s\t%.6f\t%d\t%d\t%s:%d-%d\t%s:%d-%d\t%d\t%.6f\t%s\t%s\t%s\n",
+ sec->qualified_name ? sec->qualified_name : "?",
+ t && t->qualified_name ? t->qualified_name : "?", (double)hit->tfidf, hit->rank,
+ hit->shared, sec->file_path ? sec->file_path : "?", sec->start_line,
+ sec->end_line, t && t->file_path ? t->file_path : "?", t ? t->start_line : 0,
+ t ? t->end_line : 0, hit->name_shared, (double)hit->name_idf, DOC_KIND_NAME[kind],
+ t ? doc_position_name(doc_target_position(home, t, kind)) : "?",
+ home[0] ? home : ".");
+}
+
+static void doc_section_lines(FILE *out, const cbm_gbuf_t *gbuf, const doc_ctx_t *dc,
+ const doc_place_t *pl, int s) {
+ const cbm_gbuf_node_t *sec = dc->secs[s];
+ const doc_result_t *r = &dc->res[s];
+ for (int h = 0; h < r->n + r->nlocal; h++) {
+ const doc_hit_t *hit = h < r->n ? &r->hits[h] : &r->local[h - r->n];
+ const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, dc->funcs[hit->func].node_id);
+ doc_signal_line(out, sec, fn, hit,
+ h < r->n ? CBM_SEM_DOC_KIND_FUNCTION : CBM_SEM_DOC_KIND_LOCAL, pl->home[s]);
+ }
+ for (int h = 0; h < r->nfiles; h++) {
+ const doc_file_hit_t *fh = &r->files[h];
+ doc_hit_t hit = {.tfidf = fh->tfidf, .rank = fh->rank, .shared = fh->shared};
+ doc_signal_line(out, sec, dc->files->node[fh->file], &hit, CBM_SEM_DOC_KIND_FILE,
+ pl->home[s]);
+ }
+ const cbm_gbuf_node_t *folder = doc_home_folder(pl, s);
+ if (folder) {
+ doc_hit_t hit = {.rank = SKIP_ONE};
+ doc_signal_line(out, sec, folder, &hit, CBM_SEM_DOC_KIND_FOLDER, pl->home[s]);
}
}
-/* The stored candidates of every section (rank <= CBM_SEM_DOC_TOP_K, tfidf >=
- * CBM_SEM_DOC_MIN_SCORE), in CBM_MEM_CLASS_STORE for cbm_store_doc_candidates_free; p at two
- * decimals, as on SEMANTICALLY_RELATED edges. */
-static cbm_doc_candidate_t *doc_candidate_rows(const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t **secs,
- int nsec, const cbm_sem_func_t *funcs,
- const doc_result_t *res, int *out_count) {
+/* Does the section already link the target exactly (a MENTIONS edge)? A
+ * candidate for that pair would only repeat a fact the graph holds. */
+static bool doc_pair_linked(const cbm_gbuf_t *gbuf, int64_t sec_id, int64_t fn_id) {
+ const cbm_gbuf_edge_t **edges = NULL;
int n = 0;
- for (int s = 0; s < nsec; s++) {
- for (int h = 0; h < res[s].n; h++) {
- n += res[s].hits[h].rank <= CBM_SEM_DOC_TOP_K &&
- res[s].hits[h].tfidf >= CBM_SEM_DOC_MIN_SCORE;
+ if (cbm_gbuf_find_edges_by_source_type(gbuf, sec_id, "MENTIONS", &edges, &n) != 0) {
+ return false;
+ }
+ for (int i = 0; i < n; i++) {
+ if (edges[i]->target_id == fn_id) {
+ return true;
+ }
+ }
+ return false;
+}
+
+typedef struct {
+ cbm_doc_candidate_t *rows;
+ int n;
+} doc_rows_t;
+
+/* Append the candidate (section -> target) when its judged p is above 0 and
+ * the graph does not link the pair exactly; false: out of memory (the row at
+ * rw->n may hold partial strings). */
+static bool doc_row_add(doc_rows_t *rw, const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sec,
+ const cbm_gbuf_node_t *target, const doc_hit_t *hit,
+ cbm_sem_doc_kind_t kind, const char *home) {
+ if (!target || !target->qualified_name || !sec->qualified_name) {
+ return true;
+ }
+ cbm_sem_doc_pos_t pos = doc_target_position(home, target, kind);
+ cbm_sem_doc_format_t fmt = cbm_sem_doc_format(sec->file_path);
+ float p = kind == CBM_SEM_DOC_KIND_FOLDER
+ ? cbm_sem_doc_folder_p(fmt, sec->file_path)
+ : cbm_sem_doc_calibrated_p(fmt, kind, pos, hit->tfidf);
+ if (p <= 0.0F || doc_pair_linked(gbuf, sec->id, target->id)) {
+ return true; /* not stored for this band, kind or format; or linked exactly */
+ }
+ char ev[CBM_SZ_128];
+ snprintf(ev, sizeof(ev),
+ "{\"shared_terms\":%d,\"name_tokens\":%d,\"kind\":\"%s\",\"position\":\"%s\"}",
+ hit->shared, hit->name_shared, DOC_KIND_NAME[kind], doc_position_name(pos));
+ rw->rows[rw->n] = (cbm_doc_candidate_t){
+ .section_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, sec->qualified_name),
+ .target_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, target->qualified_name),
+ .rank = hit->rank,
+ .score = hit->tfidf,
+ .p = cbm_sem_p_2dp(p),
+ .evidence = cbm_mem_strdup(CBM_MEM_CLASS_STORE, ev)};
+ const cbm_doc_candidate_t *row = &rw->rows[rw->n];
+ if (!row->section_qn || !row->target_qn || !row->evidence) {
+ return false;
+ }
+ rw->n++;
+ return true;
+}
+
+static bool doc_section_rows(doc_rows_t *rw, const cbm_gbuf_t *gbuf, const doc_ctx_t *dc,
+ const doc_place_t *pl, int s) {
+ const cbm_gbuf_node_t *sec = dc->secs[s];
+ const doc_result_t *r = &dc->res[s];
+ bool ok = true;
+ for (int h = 0; ok && h < r->n + r->nlocal; h++) {
+ const doc_hit_t *hit = h < r->n ? &r->hits[h] : &r->local[h - r->n];
+ if (h < r->n && (hit->rank > CBM_SEM_DOC_TOP_K || hit->tfidf < CBM_SEM_DOC_MIN_SCORE)) {
+ continue;
}
+ ok =
+ doc_row_add(rw, gbuf, sec, cbm_gbuf_find_by_id(gbuf, dc->funcs[hit->func].node_id), hit,
+ h < r->n ? CBM_SEM_DOC_KIND_FUNCTION : CBM_SEM_DOC_KIND_LOCAL, pl->home[s]);
+ }
+ for (int h = 0; ok && h < r->nfiles; h++) {
+ const doc_file_hit_t *fh = &r->files[h];
+ doc_hit_t hit = {.tfidf = fh->tfidf, .rank = fh->rank, .shared = fh->shared};
+ ok = doc_row_add(rw, gbuf, sec, dc->files->node[fh->file], &hit, CBM_SEM_DOC_KIND_FILE,
+ pl->home[s]);
+ }
+ doc_hit_t first = {.rank = SKIP_ONE};
+ return ok && doc_row_add(rw, gbuf, sec, doc_home_folder(pl, s), &first, CBM_SEM_DOC_KIND_FOLDER,
+ pl->home[s]);
+}
+
+/* The stored candidates of every section -- its best CBM_SEM_DOC_TOP_K
+ * functions at CBM_SEM_DOC_MIN_SCORE or more, its local functions below them,
+ * its best files, and (for a doc's first section) the doc's home folder --
+ * each with its judged p, in CBM_MEM_CLASS_STORE for
+ * cbm_store_doc_candidates_free; p at two decimals, as on
+ * SEMANTICALLY_RELATED edges. */
+static cbm_doc_candidate_t *doc_candidate_rows(const cbm_gbuf_t *gbuf, const doc_ctx_t *dc,
+ const doc_place_t *pl, int *out_count) {
+ int n = 0;
+ for (int s = 0; s < dc->nsec; s++) {
+ const doc_result_t *r = &dc->res[s];
+ n += (r->n < CBM_SEM_DOC_TOP_K ? r->n : CBM_SEM_DOC_TOP_K) + r->nlocal + r->nfiles +
+ SKIP_ONE;
}
*out_count = 0;
- cbm_doc_candidate_t *rows =
- n > 0 ? cbm_calloc(CBM_MEM_CLASS_STORE, (size_t)n * sizeof(*rows)) : NULL;
- if (!rows) {
+ doc_rows_t rw = {.rows = n > 0 ? cbm_calloc(CBM_MEM_CLASS_STORE, (size_t)n * sizeof(*rw.rows))
+ : NULL};
+ if (!rw.rows) {
return NULL;
}
- int k = 0;
- for (int s = 0; s < nsec; s++) {
- for (int h = 0; h < res[s].n; h++) {
- const doc_hit_t *hit = &res[s].hits[h];
- if (hit->rank > CBM_SEM_DOC_TOP_K || hit->tfidf < CBM_SEM_DOC_MIN_SCORE) {
- continue;
- }
- const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[hit->func].node_id);
- if (!fn || !fn->qualified_name || !secs[s]->qualified_name) {
- continue;
- }
- char ev[CBM_SZ_64];
- snprintf(ev, sizeof(ev), "{\"shared_terms\":%d,\"name_tokens\":%d}", hit->shared,
- hit->name_shared);
- rows[k] = (cbm_doc_candidate_t){
- .section_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, secs[s]->qualified_name),
- .target_qn = cbm_mem_strdup(CBM_MEM_CLASS_STORE, fn->qualified_name),
- .rank = hit->rank,
- .score = hit->tfidf,
- .p = (double)(int)(cbm_sem_doc_calibrated_p(hit->tfidf) * 100.0F + 0.5F) / 100.0,
- .evidence = cbm_mem_strdup(CBM_MEM_CLASS_STORE, ev)};
- if (!rows[k].section_qn || !rows[k].target_qn || !rows[k].evidence) {
- cbm_store_doc_candidates_free(rows, k + SKIP_ONE);
- return NULL;
- }
- k++;
+ for (int s = 0; s < dc->nsec; s++) {
+ if (!doc_section_rows(&rw, gbuf, dc, pl, s)) {
+ cbm_store_doc_candidates_free(rw.rows, rw.n + SKIP_ONE);
+ return NULL;
}
}
- *out_count = k;
- return rows;
+ *out_count = rw.n;
+ return rw.rows;
}
static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *funcs,
@@ -2332,10 +2967,15 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *
int *name_idx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC,
((size_t)func_count * SEM_DOC_NAME_TOKENS + SKIP_ONE) * sizeof(int));
doc_result_t *res = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(doc_result_t));
+ const char **fn_paths =
+ cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(*fn_paths));
+ int *ffile = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(int));
+ doc_files_t files = {0};
+ doc_place_t place = {0};
int *post_f = NULL;
int *cursor = NULL;
FILE *out = NULL;
- if (secs && post_off && fnorm && is_test && name_off && name_idx && res) {
+ if (secs && post_off && fnorm && is_test && name_off && name_idx && res && fn_paths && ffile) {
for (int f = 0; f < func_count; f++) {
for (int k = 0; k < funcs[f].tfidf_len; k++) {
post_off[funcs[f].tfidf_indices[k] + SKIP_ONE]++;
@@ -2343,6 +2983,7 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *
fnorm[f] = terms_norm(funcs[f].tfidf_weights, funcs[f].tfidf_len);
const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[f].node_id);
is_test[f] = doc_is_test_function(fn);
+ fn_paths[f] = fn && fn->file_path ? fn->file_path : "";
char *tok[SEM_DOC_NAME_TOKENS];
int nt = fn && fn->name ? cbm_sem_tokenize(fn->name, tok, SEM_DOC_NAME_TOKENS) : 0;
int base = name_off[f];
@@ -2374,12 +3015,22 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *
post_f[post_off[t] + cursor[t]++] = f;
}
}
+ memcpy(secs, found, (size_t)nsec * sizeof(*secs));
+ qsort(secs, (size_t)nsec, sizeof(*secs), cmp_section_node);
+ }
+ bool placed =
+ post_f && cursor &&
+ doc_files_build(gbuf, funcs, func_count, is_test, corpus, nterms, &files, ffile) &&
+ doc_place_build(gbuf, secs, nsec, &files, &place);
+ if (placed) {
int key_df = func_count / SEM_DOC_KEY_DF_DIV;
if (key_df < SEM_DOC_KEY_DF_MIN) {
key_df = SEM_DOC_KEY_DF_MIN;
}
- memcpy(secs, found, (size_t)nsec * sizeof(*secs));
- qsort(secs, (size_t)nsec, sizeof(*secs), cmp_section_node);
+ int file_key_df = files.count / SEM_DOC_KEY_DF_DIV;
+ if (file_key_df < SEM_DOC_KEY_DF_MIN) {
+ file_key_df = SEM_DOC_KEY_DF_MIN;
+ }
doc_ctx_t dc = {.funcs = funcs,
.func_count = func_count,
.corpus = corpus,
@@ -2392,21 +3043,25 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *
.is_test = is_test,
.name_off = name_off,
.name_idx = name_idx,
+ .fpath = fn_paths,
+ .home = (const char *const *)place.home,
+ .files = &files,
+ .file_key_df = file_key_df,
.res = res};
atomic_init(&dc.next, 0);
cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false};
cbm_parallel_for(worker_count, doc_section_worker, &dc, opts);
int row_count = 0;
- cbm_doc_candidate_t *rows = doc_candidate_rows(gbuf, secs, nsec, funcs, res, &row_count);
+ cbm_doc_candidate_t *rows = doc_candidate_rows(gbuf, &dc, &place, &row_count);
cbm_pipeline_set_doc_candidates(ctx->pipeline, rows, row_count);
- cbm_log_info("pass.semantic.doc_candidates", "sections", itoa_log(nsec), "rows",
- itoa_log(row_count));
+ cbm_log_info("pass.semantic.doc_candidates", "sections", itoa_log(nsec), "files",
+ itoa_log(files.count), "rows", itoa_log(row_count));
char path[CBM_SZ_1K];
out = cbm_safe_getenv("CBM_SEM_DOC_SIGNALS", path, sizeof(path), NULL)
? cbm_fopen(path, "w")
: NULL;
for (int s = 0; out && s < nsec; s++) {
- doc_section_lines(out, gbuf, secs[s], funcs, &res[s]);
+ doc_section_lines(out, gbuf, &dc, &place, s);
}
/* CBM_SEM_DOC_FUNCS=<path>: every function with its location and the
* test flag, for measuring candidates the TF-IDF generator does not
@@ -2438,6 +3093,10 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *
cbm_free(CBM_MEM_CLASS_SEMANTIC, name_off);
cbm_free(CBM_MEM_CLASS_SEMANTIC, name_idx);
cbm_free(CBM_MEM_CLASS_SEMANTIC, res);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, fn_paths);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, ffile);
+ doc_place_free(&place, nsec);
+ doc_files_free(&files);
}
int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) {
diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c
index bbdd57b56..af738151e 100644
--- a/src/semantic/semantic.c
+++ b/src/semantic/semantic.c
@@ -1866,16 +1866,111 @@ float cbm_sem_calibrated_p(float score) {
* candidates -- a section's best five non-test functions -- from five held-out
* repositories, three reading rounds; the top three measured bands pooled to
* stay monotone). Below the first cut (judged 0.06) nothing is stored. */
-static const float SEM_DOC_P_CUTS[] = {CBM_SEM_DOC_MIN_SCORE, 0.30F, 0.40F};
-static const float SEM_DOC_P_BANDS[] = {0.06F, 0.36F, 0.42F, 0.548F};
+enum { SEM_DOC_P_NBANDS = 4 };
+static const float SEM_DOC_P_CUTS[SEM_DOC_P_NBANDS - 1] = {CBM_SEM_DOC_MIN_SCORE, 0.30F, 0.40F};
+/* Per format, p below the first cut and in each band; 0 = not stored (a
+ * band judged below 0.20). Blind-judged, three reading rounds, pooled where
+ * a higher band judged lower:
+ * Markdown by kind and position since SEM7: SEM_DOC_P_MD_* below (its
+ * first curve, 2026-10-07: 274 candidates of five repositories,
+ * 0.06 / 0.36 / 0.42 / 0.55, did not separate positions).
+ * reST 2026-10-08, 120 candidates, three Sphinx projects; 0.20-0.30
+ * judged 0.125, so reST starts at 0.30.
+ * AsciiDoc 2026-10-08, 114 candidates, one repository (the only one of
+ * its search that qualified).
+ * PDF no qualifying repository: not measured, nothing stored. */
+static const float SEM_DOC_P_BANDS[CBM_SEM_DOC_FMT_COUNT][SEM_DOC_P_NBANDS] = {
+ [CBM_SEM_DOC_FMT_OTHER] = {0.0F, 0.0F, 0.0F, 0.0F},
+ [CBM_SEM_DOC_FMT_MARKDOWN] = {0.0F, 0.0F, 0.0F, 0.0F}, /* SEM_DOC_P_MD_* */
+ [CBM_SEM_DOC_FMT_RST] = {0.0F, 0.0F, 0.35F, 0.625F},
+ [CBM_SEM_DOC_FMT_ADOC] = {0.0F, 0.225F, 0.25F, 0.50F},
+ [CBM_SEM_DOC_FMT_PDF] = {0.0F, 0.0F, 0.0F, 0.0F},
+};
+
+cbm_sem_doc_format_t cbm_sem_doc_format(const char *path) {
+ static const struct {
+ const char *ext;
+ cbm_sem_doc_format_t fmt;
+ } exts[] = {{".md", CBM_SEM_DOC_FMT_MARKDOWN}, {".mdx", CBM_SEM_DOC_FMT_MARKDOWN},
+ {".markdown", CBM_SEM_DOC_FMT_MARKDOWN}, {".rst", CBM_SEM_DOC_FMT_RST},
+ {".rest", CBM_SEM_DOC_FMT_RST}, {".txt", CBM_SEM_DOC_FMT_RST},
+ {".adoc", CBM_SEM_DOC_FMT_ADOC}, {".asciidoc", CBM_SEM_DOC_FMT_ADOC},
+ {".asc", CBM_SEM_DOC_FMT_ADOC}, {".pdf", CBM_SEM_DOC_FMT_PDF}};
+ const char *dot = path ? strrchr(path, '.') : NULL;
+ if (!dot || strchr(dot, '/')) {
+ return CBM_SEM_DOC_FMT_OTHER;
+ }
+ for (size_t i = 0; i < sizeof(exts) / sizeof(exts[0]); i++) {
+ if (strcasecmp(dot, exts[i].ext) == 0) {
+ return exts[i].fmt;
+ }
+ }
+ return CBM_SEM_DOC_FMT_OTHER;
+}
+
+double cbm_sem_p_2dp(float p) {
+ /* 1e-4 on the x100 scale: a float holds 0.325 as 0.32499998..., 1.2e-6
+ * below the half; judged values have at most three decimals, so the
+ * nearest one that must round down (x.xx4) is 0.1 away */
+ return (double)(long)((double)p * 100.0 + 0.5 + 1e-4) / 100.0;
+}
+
+/* Markdown by kind and position (SEM7, 2026-10-08: 490 blind-judged
+ * candidates from four monorepos with per-component docs, three reading
+ * rounds, pooled where a higher band judged lower). A doc's position decides
+ * most: functions outside the doc's home were right 6 of 90 (none stored);
+ * inside it 0.18 / 0.33 / 0.53. A doc of the whole project: SEM7's and
+ * sample 4's judged items pooled (the user's choice; 158 items, nine
+ * repositories): 0.17 / 0.22 / 0.51. Local extras (home functions below the
+ * top 5) 19 of 60 = 0.32 at every band; whole files 0.12 below 0.40, then
+ * 0.30. Below 0.20 nothing. */
+static const float SEM_DOC_P_MD_FUNCTION[CBM_SEM_DOC_POS_COUNT][SEM_DOC_P_NBANDS] = {
+ [CBM_SEM_DOC_POS_GLOBAL] = {0.0F, 0.0F, 0.217F, 0.509F},
+ [CBM_SEM_DOC_POS_LOCAL] = {0.0F, 0.0F, 0.325F, 0.525F},
+ [CBM_SEM_DOC_POS_OUTSIDE] = {0.0F, 0.0F, 0.0F, 0.0F},
+};
+static const float SEM_DOC_P_MD_LOCAL[SEM_DOC_P_NBANDS] = {0.0F, 0.317F, 0.317F, 0.317F};
+static const float SEM_DOC_P_MD_FILE[SEM_DOC_P_NBANDS] = {0.0F, 0.0F, 0.0F, 0.30F};
-float cbm_sem_doc_calibrated_p(float score) {
+float cbm_sem_doc_calibrated_p(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind,
+ cbm_sem_doc_pos_t pos, float score) {
+ if ((int)fmt < 0 || fmt >= CBM_SEM_DOC_FMT_COUNT || (int)pos < 0 ||
+ pos >= CBM_SEM_DOC_POS_COUNT) {
+ return 0.0F;
+ }
size_t band = 0;
- while (band < sizeof(SEM_DOC_P_CUTS) / sizeof(SEM_DOC_P_CUTS[0]) &&
- score >= SEM_DOC_P_CUTS[band]) {
+ while (band < SEM_DOC_P_NBANDS - 1 && score >= SEM_DOC_P_CUTS[band]) {
band++;
}
- return SEM_DOC_P_BANDS[band];
+ if (fmt != CBM_SEM_DOC_FMT_MARKDOWN) {
+ /* measured for function candidates only, without position (SEM6) */
+ return kind == CBM_SEM_DOC_KIND_FUNCTION ? SEM_DOC_P_BANDS[fmt][band] : 0.0F;
+ }
+ switch (kind) {
+ case CBM_SEM_DOC_KIND_FUNCTION:
+ return SEM_DOC_P_MD_FUNCTION[pos][band];
+ case CBM_SEM_DOC_KIND_LOCAL:
+ return pos == CBM_SEM_DOC_POS_LOCAL ? SEM_DOC_P_MD_LOCAL[band] : 0.0F;
+ case CBM_SEM_DOC_KIND_FILE:
+ return SEM_DOC_P_MD_FILE[band];
+ default:
+ return 0.0F; /* a folder candidate: cbm_sem_doc_folder_p */
+ }
+}
+
+/* A doc's home folder as its candidate (SEM7): a README or index doc was
+ * right 26 of 30 (0.87); any other doc 5 of 30 (not stored). Markdown only. */
+float cbm_sem_doc_folder_p(cbm_sem_doc_format_t fmt, const char *doc_path) {
+ if (fmt != CBM_SEM_DOC_FMT_MARKDOWN || !doc_path) {
+ return 0.0F;
+ }
+ const char *base = strrchr(doc_path, '/');
+ base = base ? base + 1 : doc_path;
+ const char *dot = strrchr(base, '.');
+ size_t n = dot ? (size_t)(dot - base) : strlen(base);
+ bool readme = (n == sizeof("readme") - 1 && strncasecmp(base, "readme", n) == 0) ||
+ (n == sizeof("index") - 1 && strncasecmp(base, "index", n) == 0);
+ return readme ? 0.867F : 0.0F;
}
float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h
index e641e2f29..66f4cc0a1 100644
--- a/src/semantic/semantic.h
+++ b/src/semantic/semantic.h
@@ -270,7 +270,58 @@ float cbm_sem_calibrated_p(float score);
* about the function. */
#define CBM_SEM_DOC_TOP_K 5
#define CBM_SEM_DOC_MIN_SCORE 0.20F
-float cbm_sem_doc_calibrated_p(float score);
+
+/* A section's document format, by its file's extension: candidates are
+ * judged, and stored, per format. */
+typedef enum {
+ CBM_SEM_DOC_FMT_OTHER = 0,
+ CBM_SEM_DOC_FMT_MARKDOWN,
+ CBM_SEM_DOC_FMT_RST, /* .rst, and .txt: a .txt file only has sections as Sphinx source */
+ CBM_SEM_DOC_FMT_ADOC,
+ CBM_SEM_DOC_FMT_PDF,
+ CBM_SEM_DOC_FMT_COUNT,
+} cbm_sem_doc_format_t;
+cbm_sem_doc_format_t cbm_sem_doc_format(const char *path);
+
+/* Where a candidate sits relative to the doc's home folder (the folder whose
+ * code the doc documents, see pass_semantic_edges.c): the home is the
+ * repository root (the doc documents the whole project), or the candidate is
+ * inside or outside the home. */
+typedef enum {
+ CBM_SEM_DOC_POS_GLOBAL = 0,
+ CBM_SEM_DOC_POS_LOCAL,
+ CBM_SEM_DOC_POS_OUTSIDE,
+ CBM_SEM_DOC_POS_COUNT,
+} cbm_sem_doc_pos_t;
+
+/* What a candidate points at: one of a section's best CBM_SEM_DOC_TOP_K
+ * functions; a function inside the home below that top K (at most
+ * CBM_SEM_DOC_LOCAL_K per section); a whole file; the doc's home folder. */
+typedef enum {
+ CBM_SEM_DOC_KIND_FUNCTION = 0,
+ CBM_SEM_DOC_KIND_LOCAL,
+ CBM_SEM_DOC_KIND_FILE,
+ CBM_SEM_DOC_KIND_FOLDER,
+ CBM_SEM_DOC_KIND_COUNT,
+} cbm_sem_doc_kind_t;
+#define CBM_SEM_DOC_LOCAL_K 2
+#define CBM_SEM_DOC_FILE_K 3
+
+/* The probability that a section of format fmt is about a candidate of this
+ * kind and position with this tfidf, from the judged curves; 0 when that band
+ * is not stored (judged below 0.20) or has no judged curve. A folder
+ * candidate has no score: cbm_sem_doc_folder_p. */
+float cbm_sem_doc_calibrated_p(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind,
+ cbm_sem_doc_pos_t pos, float score);
+
+/* The probability that the doc at doc_path documents its home folder, by
+ * the doc's kind (a README or index file, or any other doc); 0 = not
+ * stored. */
+float cbm_sem_doc_folder_p(cbm_sem_doc_format_t fmt, const char *doc_path);
+
+/* p as stored and shown: two decimals, half up (0.225 -> 0.23, 0.325 ->
+ * 0.33; the epsilon absorbs the float representation of a judged x.xx5). */
+double cbm_sem_p_2dp(float p);
/* Module proximity multiplier based on file paths. */
float cbm_sem_proximity(const char *path_a, const char *path_b);
diff --git a/src/store/store.c b/src/store/store.c
index 7a64fd17b..b974e5e39 100644
--- a/src/store/store.c
+++ b/src/store/store.c
@@ -10808,14 +10808,9 @@ int cbm_store_doc_candidates_replace(cbm_store_t *s, const char *project,
return exec_sql(s, "COMMIT;");
}
-int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn,
- const char *target_qn, int limit, cbm_doc_candidate_t **out,
- int *count) {
- if (!s || !s->db || !project || !out || !count || (!section_qn == !target_qn) || limit <= 0) {
- return CBM_STORE_ERR;
- }
- *out = NULL;
- *count = 0;
+/* Does the database hold the candidates table? CBM_STORE_NOT_FOUND: no (a
+ * database written by an older build). */
+static int doc_candidates_table(cbm_store_t *s) {
sqlite3_stmt *probe = NULL;
if (sqlite3_prepare_v2(s->db,
"SELECT 1 FROM sqlite_master WHERE type='table' AND "
@@ -10833,19 +10828,15 @@ int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char
store_set_error_sqlite(s, "doc_candidates probe step");
return CBM_STORE_ERR;
}
- const char *sql = section_qn ? "SELECT section_qn, target_qn, rank, score, p, evidence FROM "
- "doc_link_candidates WHERE project=?1 AND section_qn=?2 "
- "ORDER BY p DESC, score DESC, target_qn LIMIT ?3;"
- : "SELECT section_qn, target_qn, rank, score, p, evidence FROM "
- "doc_link_candidates WHERE project=?1 AND target_qn=?2 "
- "ORDER BY p DESC, score DESC, section_qn LIMIT ?3;";
- sqlite3_stmt *stmt = NULL;
- if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
- store_set_error_sqlite(s, "doc_candidates get");
- return CBM_STORE_ERR;
- }
+ return CBM_STORE_OK;
+}
+
+/* Run a prepared candidates query (project ?1, key ?2, limit ?3; columns
+ * section_qn, target_qn, rank, score, p, evidence) into *out; finalizes it. */
+static int doc_candidates_read(sqlite3_stmt *stmt, const char *project, const char *key, int limit,
+ cbm_doc_candidate_t **out, int *count) {
bind_text(stmt, SKIP_ONE, project);
- bind_text(stmt, ST_COL_2, section_qn ? section_qn : target_qn);
+ bind_text(stmt, ST_COL_2, key);
sqlite3_bind_int(stmt, ST_COL_3, limit);
cbm_doc_candidate_t *rows = cbm_calloc(CBM_MEM_CLASS_STORE, (size_t)limit * sizeof(*rows));
if (!rows) {
@@ -10871,6 +10862,53 @@ int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char
return CBM_STORE_OK;
}
+static int doc_candidates_query(cbm_store_t *s, const char *sql, const char *project,
+ const char *key, int limit, cbm_doc_candidate_t **out, int *count) {
+ *out = NULL;
+ *count = 0;
+ int rc = doc_candidates_table(s);
+ if (rc != CBM_STORE_OK) {
+ return rc;
+ }
+ sqlite3_stmt *stmt = NULL;
+ if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
+ store_set_error_sqlite(s, "doc_candidates get");
+ return CBM_STORE_ERR;
+ }
+ return doc_candidates_read(stmt, project, key, limit, out, count);
+}
+
+int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn,
+ const char *target_qn, int limit, cbm_doc_candidate_t **out,
+ int *count) {
+ if (!s || !s->db || !project || !out || !count || (!section_qn == !target_qn) || limit <= 0) {
+ return CBM_STORE_ERR;
+ }
+ const char *sql = section_qn ? "SELECT section_qn, target_qn, rank, score, p, evidence FROM "
+ "doc_link_candidates WHERE project=?1 AND section_qn=?2 "
+ "ORDER BY p DESC, score DESC, target_qn LIMIT ?3;"
+ : "SELECT section_qn, target_qn, rank, score, p, evidence FROM "
+ "doc_link_candidates WHERE project=?1 AND target_qn=?2 "
+ "ORDER BY p DESC, score DESC, section_qn LIMIT ?3;";
+ return doc_candidates_query(s, sql, project, section_qn ? section_qn : target_qn, limit, out,
+ count);
+}
+
+int cbm_store_doc_candidates_for_path(cbm_store_t *s, const char *project, const char *file_path,
+ int limit, cbm_doc_candidate_t **out, int *count) {
+ if (!s || !s->db || !project || !file_path || !file_path[0] || !out || !count || limit <= 0) {
+ return CBM_STORE_ERR;
+ }
+ static const char SQL[] = "SELECT c.section_qn, c.target_qn, c.rank, c.score, c.p, c.evidence "
+ "FROM doc_link_candidates c JOIN nodes n "
+ "ON n.project = c.project AND n.qualified_name = c.target_qn "
+ "WHERE c.project=?1 AND ((n.label='File' AND n.file_path=?2) OR "
+ "(n.label='Folder' AND n.file_path <> '' AND "
+ "substr(?2, 1, length(n.file_path) + 1) = n.file_path || '/')) "
+ "ORDER BY c.p DESC, c.score DESC, c.section_qn LIMIT ?3;";
+ return doc_candidates_query(s, SQL, project, file_path, limit, out, count);
+}
+
void cbm_store_doc_candidates_free(cbm_doc_candidate_t *rows, int count) {
for (int i = 0; rows && i < count; i++) {
cbm_free(CBM_MEM_CLASS_STORE, (void *)rows[i].section_qn);
diff --git a/src/store/store.h b/src/store/store.h
index e7fb3142b..2a8d64048 100644
--- a/src/store/store.h
+++ b/src/store/store.h
@@ -1110,6 +1110,12 @@ int cbm_store_doc_candidates_replace(cbm_store_t *s, const char *project,
int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn,
const char *target_qn, int limit, cbm_doc_candidate_t **out,
int *count);
+
+/* The candidates that point at a file as a whole: at its File node, or at a
+ * Folder node that holds it (a doc's home folder), highest p first, then
+ * score, at most limit. Allocation and CBM_STORE_NOT_FOUND as above. */
+int cbm_store_doc_candidates_for_path(cbm_store_t *s, const char *project, const char *file_path,
+ int limit, cbm_doc_candidate_t **out, int *count);
void cbm_store_doc_candidates_free(cbm_doc_candidate_t *rows, int count);
/* ADR section parsing/rendering (pure functions, no store needed) */
diff --git a/tests/test_mcp.c b/tests/test_mcp.c
index 6ae9a0814..abd19c124 100644
--- a/tests/test_mcp.c
+++ b/tests/test_mcp.c
@@ -2546,6 +2546,98 @@ TEST(tool_get_file_outline_returns_bounded_filtered_columnar_rows_issue469) {
PASS();
}
+/* get_file_outline shows the sections possibly about the file as a whole or
+ * about a folder holding it (highest p first, kind shown), in both formats,
+ * on the first page only; a candidate for a function in the file is not one
+ * of them. */
+TEST(tool_get_file_outline_shows_doc_candidates_of_file_and_folders) {
+ cbm_mcp_server_t *srv = setup_mcp_with_data();
+ cbm_store_t *store = cbm_mcp_server_store(srv);
+ ASSERT_NOT_NULL(store);
+ ASSERT_EQ(cbm_store_upsert_project(store, "doc-outline", "/tmp/doc-outline"), CBM_STORE_OK);
+ cbm_mcp_server_set_project(srv, "doc-outline");
+ const struct {
+ const char *label, *qn, *path;
+ int start;
+ } nodes[] = {{"File", "doc-outline.pkg.mail.parse.__file__", "pkg/mail/parse.py", 0},
+ {"Function", "doc-outline.pkg.mail.parse.parse_mail", "pkg/mail/parse.py", 3},
+ {"Folder", "doc-outline.pkg.mail", "pkg/mail", 0},
+ {"Section", "doc-outline.pkg.mail.README.Mail", "pkg/mail/README.md", 1},
+ {"Section", "doc-outline.docs.parsing.Parsing", "docs/parsing.md", 5}};
+ for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) {
+ cbm_node_t node = {.project = "doc-outline",
+ .label = nodes[i].label,
+ .name = nodes[i].qn,
+ .qualified_name = nodes[i].qn,
+ .file_path = nodes[i].path,
+ .start_line = nodes[i].start,
+ .end_line = nodes[i].start + 2};
+ ASSERT_GT(cbm_store_upsert_node(store, &node), 0);
+ }
+ const cbm_doc_candidate_t rows[] = {
+ {.section_qn = "doc-outline.docs.parsing.Parsing",
+ .target_qn = "doc-outline.pkg.mail.parse.__file__",
+ .rank = 1,
+ .score = 0.41,
+ .p = 0.5,
+ .evidence = "{\"shared_terms\":4,\"name_tokens\":0,\"kind\":\"file\","
+ "\"position\":\"global\"}"},
+ {.section_qn = "doc-outline.pkg.mail.README.Mail",
+ .target_qn = "doc-outline.pkg.mail",
+ .rank = 1,
+ .p = 0.8,
+ .evidence = "{\"shared_terms\":0,\"name_tokens\":0,\"kind\":\"folder\","
+ "\"position\":\"local\"}"},
+ {.section_qn = "doc-outline.docs.parsing.Parsing",
+ .target_qn = "doc-outline.pkg.mail.parse.parse_mail",
+ .rank = 1,
+ .score = 0.5,
+ .p = 0.9,
+ .evidence = "{\"shared_terms\":5,\"name_tokens\":1,\"kind\":\"function\","
+ "\"position\":\"global\"}"},
+ };
+ ASSERT_EQ(cbm_store_doc_candidates_replace(store, "doc-outline", rows, 3), CBM_STORE_OK);
+
+ char *raw =
+ cbm_mcp_handle_tool(srv, "get_file_outline",
+ "{\"project\":\"doc-outline\",\"file_path\":\"pkg/mail/parse.py\","
+ "\"format\":\"json\"}");
+ char *text = extract_text_content(raw);
+ free(raw);
+ ASSERT_NOT_NULL(text);
+ yyjson_doc *doc = yyjson_read(text, strlen(text), 0);
+ ASSERT_NOT_NULL(doc);
+ yyjson_val *in = yyjson_obj_get(yyjson_doc_get_root(doc), "possibly_described_in");
+ ASSERT_EQ((int)yyjson_arr_size(in), 2);
+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "section")),
+ "doc-outline.pkg.mail.README.Mail");
+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "kind")), "folder");
+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 1), "kind")), "file");
+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 1), "file_path")),
+ "docs/parsing.md");
+ ASSERT_NOT_NULL(yyjson_obj_get(yyjson_doc_get_root(doc), "candidates_note"));
+ yyjson_doc_free(doc);
+ free(text);
+
+ raw = cbm_mcp_handle_tool(srv, "get_file_outline",
+ "{\"project\":\"doc-outline\",\"file_path\":\"pkg/mail/parse.py\"}");
+ ASSERT_NOT_NULL(raw);
+ ASSERT_NOT_NULL(strstr(raw, "possibly_described_in"));
+ ASSERT_NOT_NULL(strstr(raw, "doc-outline.pkg.mail.README.Mail"));
+ ASSERT_NOT_NULL(strstr(raw, "folder"));
+ free(raw);
+
+ raw = cbm_mcp_handle_tool(srv, "get_file_outline",
+ "{\"project\":\"doc-outline\",\"file_path\":\"pkg/mail/parse.py\","
+ "\"offset\":1}");
+ ASSERT_NOT_NULL(raw);
+ ASSERT_NULL(strstr(raw, "possibly_described_in"));
+ free(raw);
+
+ cbm_mcp_server_free(srv);
+ PASS();
+}
+
TEST(tool_get_file_outline_validates_json_path_limit_and_cancel_issue469) {
cbm_mcp_server_t *srv = setup_mcp_with_data();
cbm_store_t *store = cbm_mcp_server_store(srv);
@@ -7618,9 +7710,10 @@ TEST(tool_trace_call_path_qn_fallback_frees_name_miss) {
cbm_store_free_nodes(by_name, by_name_count);
char *resp = cbm_mcp_server_handle(
- srv, "{\"jsonrpc\":\"2.0\",\"id\":63,\"method\":\"tools/call\","
- "\"params\":{\"name\":\"trace_call_path\",\"arguments\":{\"function_name\":"
- "\"qnfb-proj.src.qnfb_entry\",\"project\":\"qnfb-proj\",\"direction\":\"outbound\"}}}");
+ srv,
+ "{\"jsonrpc\":\"2.0\",\"id\":63,\"method\":\"tools/call\","
+ "\"params\":{\"name\":\"trace_call_path\",\"arguments\":{\"function_name\":"
+ "\"qnfb-proj.src.qnfb_entry\",\"project\":\"qnfb-proj\",\"direction\":\"outbound\"}}}");
ASSERT_NOT_NULL(resp);
char *inner = extract_text_content(resp);
ASSERT_NOT_NULL(inner);
@@ -18537,7 +18630,8 @@ TEST(snippet_doc_candidates_both_directions) {
.rank = 1,
.score = 0.45,
.p = 0.548,
- .evidence = "{\"shared_terms\":5,\"name_tokens\":2}"},
+ .evidence = "{\"shared_terms\":5,\"name_tokens\":2,\"kind\":\"function\","
+ "\"position\":\"local\"}"},
{.section_qn = "test-project.README.Orders",
.target_qn = "test-project.gone.Missing",
.rank = 3,
@@ -18560,6 +18654,9 @@ TEST(snippet_doc_candidates_both_directions) {
ASSERT_FLOAT_EQ(yyjson_get_num(yyjson_obj_get(first, "p")), 0.548, 1e-9);
ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(first, "shared_terms")), 5);
ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(first, "start_line")), 7);
+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "label")), "Function");
+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "kind")), "function");
+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "position")), "local");
ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(about, 1), "qualified_name")),
"test-project.cmd.server.main.HandleRequest");
ASSERT_NOT_NULL(yyjson_obj_get(yyjson_doc_get_root(doc), "candidates_note"));
@@ -23784,6 +23881,7 @@ SUITE(mcp) {
RUN_TEST(tool_compare_graphs_validation_and_scan_cap_are_atomic_issue525);
RUN_TEST(tool_compare_graphs_cancel_and_readonly_handles_release_issue525);
RUN_TEST(tool_get_file_outline_returns_bounded_filtered_columnar_rows_issue469);
+ RUN_TEST(tool_get_file_outline_shows_doc_candidates_of_file_and_folders);
RUN_TEST(tool_get_file_outline_validates_json_path_limit_and_cancel_issue469);
RUN_TEST(tool_search_graph_basic);
RUN_TEST(tool_search_graph_empty_hint_describes_combined_filters_issue1366);
diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c
index c11c2842c..ddf7bc5a5 100644
--- a/tests/test_pipeline.c
+++ b/tests/test_pipeline.c
@@ -18948,14 +18948,44 @@ TEST(pipeline_semantic_edges_carry_p) {
PASS();
}
+/* The p a stored candidate row must carry: the curve of the row's kind and
+ * position (read from its evidence) at its score, two decimals; -1 for an
+ * unknown kind or position. *is_function: a function candidate (top 5). */
+static double doc_row_want(const char *evidence, cbm_sem_doc_format_t fmt, float score,
+ bool *is_function) {
+ static const struct {
+ const char *name;
+ cbm_sem_doc_kind_t kind;
+ } KINDS[] = {{"\"kind\":\"function\"", CBM_SEM_DOC_KIND_FUNCTION},
+ {"\"kind\":\"local\"", CBM_SEM_DOC_KIND_LOCAL},
+ {"\"kind\":\"file\"", CBM_SEM_DOC_KIND_FILE}};
+ static const struct {
+ const char *name;
+ cbm_sem_doc_pos_t pos;
+ } POSITIONS[] = {{"\"position\":\"global\"", CBM_SEM_DOC_POS_GLOBAL},
+ {"\"position\":\"local\"", CBM_SEM_DOC_POS_LOCAL},
+ {"\"position\":\"outside\"", CBM_SEM_DOC_POS_OUTSIDE}};
+ int k = -1;
+ int p = -1;
+ for (int i = 0; evidence && i < 3; i++) {
+ k = k < 0 && strstr(evidence, KINDS[i].name) ? i : k;
+ p = p < 0 && strstr(evidence, POSITIONS[i].name) ? i : p;
+ }
+ *is_function = k == 0;
+ return k < 0 || p < 0 ? -1.0
+ : cbm_sem_p_2dp(cbm_sem_doc_calibrated_p(fmt, KINDS[k].kind,
+ POSITIONS[p].pos, score));
+}
+
/* Count a section's stored doc -> code candidates (section QN containing
- * heading) and check each row's shape: rank within the top
- * CBM_SEM_DOC_TOP_K, score at or above the floor, p the doc curve's band at
- * two decimals, never a test function. Returns -1 on a bad row. */
+ * heading) and check each row's shape: a function within the top
+ * CBM_SEM_DOC_TOP_K (top: the best of them), score at or above the floor, p
+ * the curve of its kind and position at two decimals, never a test
+ * function. Returns -1 on a bad row. */
static int doc_candidate_rows_for(cbm_store_t *s, const char *heading, char *top, size_t top_sz) {
sqlite3_stmt *st = NULL;
if (sqlite3_prepare_v2(cbm_store_get_db(s),
- "SELECT target_qn, rank, score, p FROM doc_link_candidates "
+ "SELECT target_qn, rank, score, p, evidence FROM doc_link_candidates "
"WHERE instr(section_qn, ?1) > 0 ORDER BY rank",
-1, &st, NULL) != SQLITE_OK) {
return -1;
@@ -18963,16 +18993,20 @@ static int doc_candidate_rows_for(cbm_store_t *s, const char *heading, char *top
sqlite3_bind_text(st, 1, heading, -1, SQLITE_TRANSIENT);
int n = 0;
bool ok = true;
+ bool have_top = false;
while (ok && sqlite3_step(st) == SQLITE_ROW) {
const char *target = (const char *)sqlite3_column_text(st, 0);
int rank = sqlite3_column_int(st, 1);
double score = sqlite3_column_double(st, 2);
double p = sqlite3_column_double(st, 3);
- double want = (double)(int)(cbm_sem_doc_calibrated_p((float)score) * 100.0F + 0.5F) / 100.0;
- ok = target && rank >= 1 && rank <= CBM_SEM_DOC_TOP_K &&
+ bool is_function = false;
+ double want = doc_row_want((const char *)sqlite3_column_text(st, 4),
+ CBM_SEM_DOC_FMT_MARKDOWN, (float)score, &is_function);
+ ok = target && rank >= 1 && (!is_function || rank <= CBM_SEM_DOC_TOP_K) &&
score >= (double)CBM_SEM_DOC_MIN_SCORE && p == want && !strstr(target, ".Test");
- if (ok && n == 0 && top) {
+ if (ok && is_function && !have_top && top) {
snprintf(top, top_sz, "%s", target);
+ have_top = true;
}
n++;
}
@@ -19061,6 +19095,306 @@ TEST(pipeline_doc_candidates_published_and_kept_by_delta) {
PASS();
}
+/* One Python family per subject: parse / load / validate / write, each with
+ * a docstring, so the functions' corpus has the subject words to match. */
+static void write_py_family(const char *base, const char *subject) {
+ char path[512];
+ snprintf(path, sizeof(path), "%s/%s.py", base, subject);
+ char body[4096];
+ snprintf(body, sizeof(body),
+ "def parse_%s_config(path):\n"
+ " \"\"\"Parse the %s config file and return the parsed %s settings.\"\"\"\n"
+ " return load_%s_config(path)\n\n"
+ "def load_%s_config(path):\n"
+ " \"\"\"Load the %s config from disk and validate every %s field.\"\"\"\n"
+ " cfg = {}\n validate_%s_config(cfg)\n return cfg\n\n"
+ "def validate_%s_config(cfg):\n"
+ " \"\"\"Check the %s config for missing fields.\"\"\"\n"
+ " return cfg is not None\n\n"
+ "def write_%s_config(path, cfg):\n"
+ " \"\"\"Write the %s config back to disk after validating it.\"\"\"\n"
+ " validate_%s_config(cfg)\n",
+ subject, subject, subject, subject, subject, subject, subject, subject, subject,
+ subject, subject, subject, subject);
+ th_write_file(path, body);
+}
+
+/* The candidate count, and the bad rows, of one section (its QN contains
+ * heading and its file ends in ext): p must be the format's curve value at
+ * two decimals and the tfidf at or above the format's first stored band. */
+static int doc_rows_of(cbm_store_t *s, const char *heading, const char *ext,
+ cbm_sem_doc_format_t fmt, const char *forbidden_target, int *bad) {
+ sqlite3_stmt *st = NULL;
+ *bad = 0;
+ if (sqlite3_prepare_v2(
+ cbm_store_get_db(s),
+ "SELECT c.target_qn, c.score, c.p, c.evidence FROM doc_link_candidates c "
+ "JOIN nodes n ON n.qualified_name = c.section_qn "
+ "WHERE instr(c.section_qn, ?1) > 0 AND n.file_path LIKE ?2",
+ -1, &st, NULL) != SQLITE_OK) {
+ return -1;
+ }
+ char like[32];
+ snprintf(like, sizeof(like), "%%%s", ext);
+ sqlite3_bind_text(st, 1, heading, -1, SQLITE_TRANSIENT);
+ sqlite3_bind_text(st, 2, like, -1, SQLITE_TRANSIENT);
+ int n = 0;
+ while (sqlite3_step(st) == SQLITE_ROW) {
+ const char *target = (const char *)sqlite3_column_text(st, 0);
+ float score = (float)sqlite3_column_double(st, 1);
+ double p = sqlite3_column_double(st, 2);
+ bool is_function = false;
+ double want =
+ doc_row_want((const char *)sqlite3_column_text(st, 3), fmt, score, &is_function);
+ *bad += want <= 0.0 || p != want ||
+ (forbidden_target && target && strstr(target, forbidden_target));
+ n++;
+ }
+ sqlite3_finalize(st);
+ return n;
+}
+
+/* Candidates never repeat an exact link: a section whose code span names a
+ * function (a MENTIONS edge) does not also list that function as a
+ * candidate. And each format carries its own judged curve: reST stores only
+ * from tfidf 0.30, with reST's p. */
+TEST(pipeline_doc_candidates_skip_exact_links_and_use_format_curves) {
+ char tmp[256];
+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_docfmt_XXXXXX");
+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
+ static const char *const subjects[] = {"mail", "cache", "queue", "user", "server", "client"};
+ for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) {
+ write_py_family(tmp, subjects[i]);
+ }
+ char docs[512];
+ snprintf(docs, sizeof(docs), "%s/docs", tmp);
+ ASSERT_TRUE(cbm_mkdir_p(docs, 0755));
+ write_temp_file(tmp, "docs/mail.md",
+ "# Mail settings\n\n"
+ "The mail config file is parsed by `mail.parse_mail_config`, which loads "
+ "the mail config from disk and validates every mail field before the "
+ "parsed mail settings are returned to the caller for use.\n");
+ write_temp_file(tmp, "docs/mail.rst",
+ "Mail loading\n============\n\n"
+ "Load the mail config from disk and validate every mail field: missing "
+ "mail fields fail the validation, and the loaded mail config is written "
+ "back to disk after validating it.\n");
+ char db[512];
+ snprintf(db, sizeof(db), "%s/docfmt.db", tmp);
+ cbm_pipeline_incremental_test_reset_faults();
+ cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL);
+ ASSERT_NOT_NULL(p);
+ ASSERT_EQ(cbm_pipeline_run(p), 0);
+ cbm_pipeline_free(p);
+
+ cbm_store_t *s = cbm_store_open_path(db);
+ ASSERT_NOT_NULL(s);
+ /* the exact link exists ... */
+ sqlite3_stmt *st = NULL;
+ ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s),
+ "SELECT count(*) FROM edges e JOIN nodes a ON a.id = e.source_id "
+ "JOIN nodes b ON b.id = e.target_id WHERE e.type = 'MENTIONS' "
+ "AND a.label = 'Section' AND a.file_path = 'docs/mail.md' "
+ "AND b.name = 'parse_mail_config'",
+ -1, &st, NULL),
+ SQLITE_OK);
+ ASSERT_EQ(sqlite3_step(st), SQLITE_ROW);
+ int linked = sqlite3_column_int(st, 0);
+ sqlite3_finalize(st);
+ /* ... and is not repeated as a candidate, while the rest of the section's
+ * candidates are stored on the Markdown curve */
+ int md_bad = 0;
+ int rst_bad = 0;
+ int md = doc_rows_of(s, "Mail-settings", ".md", CBM_SEM_DOC_FMT_MARKDOWN, ".parse_mail_config",
+ &md_bad);
+ int rst = doc_rows_of(s, "Mail-loading", ".rst", CBM_SEM_DOC_FMT_RST, NULL, &rst_bad);
+ cbm_store_close(s);
+ th_rmtree(tmp);
+ cbm_pipeline_incremental_test_reset_faults();
+ ASSERT_EQ(linked, 1);
+ ASSERT_GT(md, 0);
+ ASSERT_EQ(md_bad, 0);
+ ASSERT_GT(rst, 0);
+ ASSERT_EQ(rst_bad, 0);
+ PASS();
+}
+
+/* Lines of a CBM_SEM_DOC_SIGNALS dump whose section file, kind, position,
+ * home and target file equal the given values (NULL: any). */
+static int doc_signal_count(const char *dump, const char *sec_file, const char *kind,
+ const char *pos, const char *home, const char *target_file) {
+ FILE *f = cbm_fopen(dump, "r");
+ if (!f) {
+ return -1;
+ }
+ char line[4096];
+ int n = 0;
+ while (fgets(line, sizeof(line), f)) {
+ line[strcspn(line, "\r\n")] = '\0';
+ char *col[12];
+ int nc = 0;
+ for (char *p = line; nc < 12; nc++) {
+ col[nc] = p;
+ char *tab = strchr(p, '\t');
+ if (!tab) {
+ nc++;
+ break;
+ }
+ *tab = '\0';
+ p = tab + 1;
+ }
+ if (nc != 12) {
+ continue;
+ }
+ char *colon = strrchr(col[5], ':');
+ if (colon) {
+ *colon = '\0';
+ }
+ colon = strrchr(col[6], ':');
+ if (colon) {
+ *colon = '\0';
+ }
+ n += (!sec_file || strcmp(col[5], sec_file) == 0) && (!kind || strcmp(col[9], kind) == 0) &&
+ (!pos || strcmp(col[10], pos) == 0) && (!home || strcmp(col[11], home) == 0) &&
+ (!target_file || strcmp(col[6], target_file) == 0);
+ }
+ fclose(f);
+ return n;
+}
+
+/* Document position: a doc's home is its folder (a docs folder documents its
+ * parent; a folder without code climbs to one with code; the root = the
+ * whole project); a candidate inside the home is local, elsewhere outside;
+ * a doc with a home below the root gets its home folder as a candidate;
+ * whole files are candidates; local functions below the top 5 are added. */
+TEST(pipeline_doc_candidates_home_position_and_kinds) {
+ char tmp[256];
+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_docpos_XXXXXX");
+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
+ char dir[512];
+ static const char *const pkgs[] = {"pkg/mail", "pkg/mail/sub", "pkg/cache", "pkg/queue",
+ "pkg/mail/docs", "docs", "guides"};
+ for (size_t i = 0; i < sizeof(pkgs) / sizeof(pkgs[0]); i++) {
+ snprintf(dir, sizeof(dir), "%s/%s", tmp, pkgs[i]);
+ ASSERT_TRUE(cbm_mkdir_p(dir, 0755));
+ }
+ snprintf(dir, sizeof(dir), "%s/pkg/mail", tmp);
+ write_py_family(dir, "mail");
+ snprintf(dir, sizeof(dir), "%s/pkg/mail/sub", tmp);
+ write_py_family(dir, "mail");
+ snprintf(dir, sizeof(dir), "%s/pkg/cache", tmp);
+ write_py_family(dir, "cache");
+ snprintf(dir, sizeof(dir), "%s/pkg/queue", tmp);
+ write_py_family(dir, "queue");
+ static const char body[] =
+ "The mail config file is parsed, loaded from disk and validated: every mail "
+ "field is checked, missing mail fields fail the validation, and the parsed mail "
+ "settings are written back to disk after validating the mail config.\n";
+ static const char *const doc_files[] = {"pkg/mail/README.md", "pkg/mail/docs/guide.md",
+ "docs/overview.md", "guides/notes.md"};
+ for (size_t i = 0; i < sizeof(doc_files) / sizeof(doc_files[0]); i++) {
+ char text[1024];
+ snprintf(text, sizeof(text), "# Mail config %d\n\n%s", (int)i, body);
+ write_temp_file(tmp, doc_files[i], text);
+ }
+ char db[512];
+ char dump[512];
+ snprintf(db, sizeof(db), "%s/docpos.db", tmp);
+ snprintf(dump, sizeof(dump), "%s/signals.tsv", tmp);
+ cbm_setenv("CBM_SEM_DOC_SIGNALS", dump, 1);
+ cbm_pipeline_incremental_test_reset_faults();
+ cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL);
+ ASSERT_NOT_NULL(p);
+ int rc = cbm_pipeline_run(p);
+ cbm_pipeline_free(p);
+ cbm_unsetenv("CBM_SEM_DOC_SIGNALS");
+ ASSERT_EQ(rc, 0);
+
+ const char *readme = "pkg/mail/README.md";
+ const char *guide = "pkg/mail/docs/guide.md";
+ /* homes: the package; its docs folder documents the package; top-level
+ * docs and a folder without code document the whole project */
+ int readme_lines = doc_signal_count(dump, readme, NULL, NULL, NULL, NULL);
+ int readme_home = doc_signal_count(dump, readme, NULL, NULL, "pkg/mail", NULL);
+ int guide_lines = doc_signal_count(dump, guide, NULL, NULL, NULL, NULL);
+ int guide_home = doc_signal_count(dump, guide, NULL, NULL, "pkg/mail", NULL);
+ int overview_lines = doc_signal_count(dump, "docs/overview.md", NULL, NULL, NULL, NULL);
+ int overview_root = doc_signal_count(dump, "docs/overview.md", NULL, "global", ".", NULL);
+ int notes_lines = doc_signal_count(dump, "guides/notes.md", NULL, NULL, NULL, NULL);
+ int notes_root = doc_signal_count(dump, "guides/notes.md", NULL, "global", ".", NULL);
+ /* positions */
+ int local_fn =
+ doc_signal_count(dump, readme, "function", "local", "pkg/mail", "pkg/mail/mail.py");
+ int local_sub =
+ doc_signal_count(dump, readme, NULL, "local", "pkg/mail", "pkg/mail/sub/mail.py");
+ int outside_fn = doc_signal_count(dump, readme, "function", "outside", NULL, NULL);
+ int wrong_local = doc_signal_count(dump, readme, NULL, "local", NULL, "pkg/cache/cache.py");
+ /* kinds */
+ int folders = doc_signal_count(dump, NULL, "folder", "local", NULL, "pkg/mail");
+ int folder_readme = doc_signal_count(dump, readme, "folder", "local", "pkg/mail", "pkg/mail");
+ int root_folders = doc_signal_count(dump, "docs/overview.md", "folder", NULL, NULL, NULL) +
+ doc_signal_count(dump, "guides/notes.md", "folder", NULL, NULL, NULL);
+ int files = doc_signal_count(dump, readme, "file", "local", NULL, NULL);
+ int extras = doc_signal_count(dump, readme, "local", "local", "pkg/mail", NULL);
+ /* stored: the README's home folder (a README: p 0.87), not the guide's
+ * (another doc), no function outside the home, every row on its curve */
+ cbm_store_t *s = cbm_store_open_path(db);
+ ASSERT_NOT_NULL(s);
+ sqlite3_stmt *st = NULL;
+ ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s),
+ "SELECT c.score, c.p, c.evidence, n.file_path FROM "
+ "doc_link_candidates c JOIN nodes n ON n.qualified_name = "
+ "c.section_qn",
+ -1, &st, NULL),
+ SQLITE_OK);
+ int stored_folders = 0;
+ int stored_outside = 0;
+ int off_curve = 0;
+ double folder_p = 0.0;
+ while (sqlite3_step(st) == SQLITE_ROW) {
+ const char *ev = (const char *)sqlite3_column_text(st, 2);
+ const char *path = (const char *)sqlite3_column_text(st, 3);
+ double p = sqlite3_column_double(st, 1);
+ if (ev && strstr(ev, "\"kind\":\"folder\"")) {
+ stored_folders++;
+ folder_p = p;
+ off_curve += strcmp(path, readme) != 0;
+ continue;
+ }
+ stored_outside +=
+ ev && strstr(ev, "\"position\":\"outside\"") && strstr(ev, "\"kind\":\"function\"");
+ bool is_function = false;
+ off_curve += p != doc_row_want(ev, CBM_SEM_DOC_FMT_MARKDOWN,
+ (float)sqlite3_column_double(st, 0), &is_function);
+ }
+ sqlite3_finalize(st);
+ cbm_store_close(s);
+ th_rmtree(tmp);
+ cbm_pipeline_incremental_test_reset_faults();
+ ASSERT_GT(readme_lines, 0);
+ ASSERT_EQ(readme_home, readme_lines);
+ ASSERT_GT(guide_lines, 0);
+ ASSERT_EQ(guide_home, guide_lines);
+ ASSERT_GT(overview_lines, 0);
+ ASSERT_EQ(overview_root, overview_lines);
+ ASSERT_GT(notes_lines, 0);
+ ASSERT_EQ(notes_root, notes_lines);
+ ASSERT_GT(local_fn, 0);
+ ASSERT_GT(local_sub, 0);
+ ASSERT_GT(outside_fn, 0);
+ ASSERT_EQ(wrong_local, 0);
+ ASSERT_EQ(folders, 2); /* README and the docs guide; one per doc, never per section */
+ ASSERT_EQ(folder_readme, 1);
+ ASSERT_EQ(root_folders, 0);
+ ASSERT_GT(files, 0);
+ ASSERT_GT(extras, 0);
+ ASSERT_EQ(stored_folders, 1);
+ ASSERT_FLOAT_EQ(folder_p, 0.87, 1e-9);
+ ASSERT_EQ(stored_outside, 0);
+ ASSERT_EQ(off_curve, 0);
+ PASS();
+}
+
TEST(pipeline_semantic_edges_no_functions) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX");
@@ -19976,6 +20310,8 @@ SUITE(pipeline) {
RUN_TEST(pipeline_semantic_pair_signals_never_change_the_graph);
RUN_TEST(pipeline_semantic_edges_carry_p);
RUN_TEST(pipeline_doc_candidates_published_and_kept_by_delta);
+ RUN_TEST(pipeline_doc_candidates_skip_exact_links_and_use_format_curves);
+ RUN_TEST(pipeline_doc_candidates_home_position_and_kinds);
RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153);
RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153);
RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153);
diff --git a/tests/test_semantic.c b/tests/test_semantic.c
index 006cad45c..f1ebd09f7 100644
--- a/tests/test_semantic.c
+++ b/tests/test_semantic.c
@@ -513,25 +513,135 @@ TEST(sem_calibrated_p_bands) {
PASS();
}
-/* Doc -> code candidates: the judged bands (sample 4), monotone, with nothing
- * stored below CBM_SEM_DOC_MIN_SCORE (that band judged 0.06). */
+static float docp(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind, cbm_sem_doc_pos_t pos,
+ float score) {
+ return cbm_sem_doc_calibrated_p(fmt, kind, pos, score);
+}
+
+/* Doc -> code candidates, Markdown (sample 7): p by kind and position --
+ * functions outside the doc's home never stored, inside it from 0.30, for a
+ * doc of the whole project (samples 4 and 7 pooled) from 0.30 too; local
+ * extras 0.32 inside the home
+ * only; whole files from 0.40; nothing below CBM_SEM_DOC_MIN_SCORE; every
+ * curve monotone. A format without a judged curve stores nothing. */
TEST(sem_doc_calibrated_p_bands) {
- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.10F), 0.06F, 0.0);
- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.199F), 0.06F, 0.0);
- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(CBM_SEM_DOC_MIN_SCORE), 0.36F, 0.0);
- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.299F), 0.36F, 0.0);
- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.30F), 0.42F, 0.0);
- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.40F), 0.548F, 0.0);
- ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(1.00F), 0.548F, 0.0);
- float last = 0.0F;
+ const cbm_sem_doc_format_t md = CBM_SEM_DOC_FMT_MARKDOWN;
+ const cbm_sem_doc_kind_t fn = CBM_SEM_DOC_KIND_FUNCTION;
+ const cbm_sem_doc_pos_t global = CBM_SEM_DOC_POS_GLOBAL;
+ const cbm_sem_doc_pos_t local = CBM_SEM_DOC_POS_LOCAL;
+ const cbm_sem_doc_pos_t outside = CBM_SEM_DOC_POS_OUTSIDE;
+ ASSERT_FLOAT_EQ(docp(md, fn, local, 0.199F), 0.0F, 0.0);
+ ASSERT_FLOAT_EQ(docp(md, fn, local, CBM_SEM_DOC_MIN_SCORE), 0.0F, 0.0);
+ ASSERT_FLOAT_EQ(docp(md, fn, local, 0.30F), 0.325F, 0.0);
+ ASSERT_FLOAT_EQ(docp(md, fn, local, 0.40F), 0.525F, 0.0);
+ ASSERT_FLOAT_EQ(docp(md, fn, global, 0.299F), 0.0F, 0.0);
+ ASSERT_FLOAT_EQ(docp(md, fn, global, 0.30F), 0.217F, 0.0);
+ ASSERT_FLOAT_EQ(docp(md, fn, global, 0.40F), 0.509F, 0.0);
+ ASSERT_FLOAT_EQ(docp(md, fn, outside, 0.90F), 0.0F, 0.0);
+ ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.199F), 0.0F, 0.0);
+ ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.20F), 0.317F, 0.0);
+ ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.90F), 0.317F, 0.0);
+ ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FILE, outside, 0.399F), 0.0F, 0.0);
+ ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FILE, outside, 0.40F), 0.30F, 0.0);
+ ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FOLDER, local, 0.90F), 0.0F, 0.0);
+ static const cbm_sem_doc_kind_t KINDS[] = {CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_KIND_LOCAL,
+ CBM_SEM_DOC_KIND_FILE};
+ for (size_t k = 0; k < sizeof(KINDS) / sizeof(KINDS[0]); k++) {
+ for (int pos = 0; pos < CBM_SEM_DOC_POS_COUNT; pos++) {
+ float last = 0.0F;
+ for (int milli = 0; milli <= 1000; milli++) {
+ float s = (float)milli / 1000.0F;
+ float p = docp(md, KINDS[k], (cbm_sem_doc_pos_t)pos, s);
+ ASSERT_TRUE(p >= last);
+ last = p;
+ ASSERT_TRUE(pos != CBM_SEM_DOC_POS_OUTSIDE || KINDS[k] == CBM_SEM_DOC_KIND_FILE ||
+ p == 0.0F);
+ ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_OTHER, KINDS[k], (cbm_sem_doc_pos_t)pos, s),
+ 0.0F, 0.0);
+ ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_PDF, KINDS[k], (cbm_sem_doc_pos_t)pos, s),
+ 0.0F, 0.0);
+ }
+ }
+ }
+ ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_COUNT, fn, global, 0.9F), 0.0F, 0.0);
+ /* a home folder: README and index docs only, Markdown only */
+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/README.md"), 0.867F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/docs/index.mdx"), 0.867F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/readme"), 0.867F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/guide.md"), 0.0F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/README-old.md"), 0.0F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(CBM_SEM_DOC_FMT_RST, "pkg/mail/index.rst"), 0.0F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, NULL), 0.0F, 0.0);
+ /* reST (sample 6): 0.20-0.30 judged 0.125, not stored */
+ const cbm_sem_doc_format_t rst = CBM_SEM_DOC_FMT_RST;
+ ASSERT_FLOAT_EQ(
+ cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.25F),
+ 0.0F, 0.0);
+ ASSERT_FLOAT_EQ(
+ cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.30F),
+ 0.35F, 0.0);
+ ASSERT_FLOAT_EQ(
+ cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.40F),
+ 0.625F, 0.0);
+ /* AsciiDoc (sample 6): every band at or above the floor is stored */
+ const cbm_sem_doc_format_t adoc = CBM_SEM_DOC_FMT_ADOC;
+ ASSERT_FLOAT_EQ(
+ cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.199F),
+ 0.0F, 0.0);
+ ASSERT_FLOAT_EQ(
+ cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.20F),
+ 0.225F, 0.0);
+ ASSERT_FLOAT_EQ(
+ cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.35F),
+ 0.25F, 0.0);
+ ASSERT_FLOAT_EQ(
+ cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.90F),
+ 0.50F, 0.0);
for (int milli = 0; milli <= 1000; milli++) {
- float p = cbm_sem_doc_calibrated_p((float)milli / 1000.0F);
- ASSERT_TRUE(p >= last);
- last = p;
+ float s = (float)milli / 1000.0F;
+ ASSERT_TRUE(
+ cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, s) >=
+ cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL,
+ s - 0.001F));
+ ASSERT_TRUE(
+ cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, s) >=
+ cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL,
+ s - 0.001F));
}
PASS();
}
+/* p is stored and shown at two decimals, half up, whatever the float
+ * representation of the judged value (0.225F is 0.22499999...). */
+TEST(sem_p_two_decimals_half_up) {
+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.225F), 0.23, 1e-12);
+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.325F), 0.33, 1e-12); /* float 0.32499998... */
+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.525F), 0.53, 1e-12); /* float 0.52499997... */
+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.324F), 0.32, 1e-12);
+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.317F), 0.32, 1e-12);
+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.867F), 0.87, 1e-12);
+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.625F), 0.63, 1e-12);
+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.548F), 0.55, 1e-12);
+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.36F), 0.36, 1e-12);
+ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.0F), 0.0, 1e-12);
+ PASS();
+}
+
+/* A section's format comes from its file's extension, case-insensitively;
+ * a .txt section is reStructuredText (plain text has no sections). */
+TEST(sem_doc_format_by_extension) {
+ ASSERT_EQ(cbm_sem_doc_format("docs/guide.md"), CBM_SEM_DOC_FMT_MARKDOWN);
+ ASSERT_EQ(cbm_sem_doc_format("README.MARKDOWN"), CBM_SEM_DOC_FMT_MARKDOWN);
+ ASSERT_EQ(cbm_sem_doc_format("docs/index.rst"), CBM_SEM_DOC_FMT_RST);
+ ASSERT_EQ(cbm_sem_doc_format("docs/topics/db.txt"), CBM_SEM_DOC_FMT_RST);
+ ASSERT_EQ(cbm_sem_doc_format("manual/intro.adoc"), CBM_SEM_DOC_FMT_ADOC);
+ ASSERT_EQ(cbm_sem_doc_format("paper.pdf"), CBM_SEM_DOC_FMT_PDF);
+ ASSERT_EQ(cbm_sem_doc_format("src/main.go"), CBM_SEM_DOC_FMT_OTHER);
+ ASSERT_EQ(cbm_sem_doc_format("v1.2/NOTES"), CBM_SEM_DOC_FMT_OTHER);
+ ASSERT_EQ(cbm_sem_doc_format(NULL), CBM_SEM_DOC_FMT_OTHER);
+ PASS();
+}
+
/* A properties string holding a quote is read whole, and escapes are not
* glued to the next word (until 2026-10 the read stopped at the first quote
* and kept "\n" raw, so "\nreturn" became the token "nreturn"). */
@@ -731,6 +841,8 @@ SUITE(semantic) {
RUN_TEST(sem_admit_best_first_keeps_the_best);
RUN_TEST(sem_calibrated_p_bands);
RUN_TEST(sem_doc_calibrated_p_bands);
+ RUN_TEST(sem_doc_format_by_extension);
+ RUN_TEST(sem_p_two_decimals_half_up);
RUN_TEST(sem_json_str_honours_escapes);
RUN_TEST(sem_corpus_add_null_doc);
RUN_TEST(sem_corpus_free_null);
diff --git a/tests/test_store_arch.c b/tests/test_store_arch.c
index 6c6037770..23ea5c935 100644
--- a/tests/test_store_arch.c
+++ b/tests/test_store_arch.c
@@ -275,7 +275,8 @@ TEST(arch_path_scoping) {
ASSERT_TRUE(whole_pkg_nodes > scoped_pkg_nodes);
ASSERT_EQ(scoped_pkg_nodes, 1);
- ASSERT_TRUE(cbm_store_count_nodes(s, "pscope") > cbm_store_count_nodes_scoped(s, "pscope", "apps/foo"));
+ ASSERT_TRUE(cbm_store_count_nodes(s, "pscope") >
+ cbm_store_count_nodes_scoped(s, "pscope", "apps/foo"));
cbm_architecture_info_t scoped_slash;
memset(&scoped_slash, 0, sizeof(scoped_slash));
@@ -703,6 +704,63 @@ TEST(doc_candidates_replace_and_get) {
PASS();
}
+/* Whole-file candidates of a path: the rows whose target is the path's File
+ * node or a Folder holding it -- never a sibling folder sharing a name prefix
+ * (pkg/mailer, pk), a function in the file, or another project's rows. */
+TEST(doc_candidates_for_path_file_and_folders) {
+ cbm_store_t *s = cbm_store_open_memory();
+ ASSERT_NOT_NULL(s);
+ cbm_doc_candidate_t *got = NULL;
+ int n = -1;
+ ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "pkg/mail/parse.py", 5, &got, &n),
+ CBM_STORE_NOT_FOUND);
+ cbm_store_upsert_project(s, "p", "/tmp/p");
+ const struct {
+ const char *label, *qn, *path;
+ } nodes[] = {{"File", "p.pkg.mail.parse.__file__", "pkg/mail/parse.py"},
+ {"Function", "p.pkg.mail.parse.parse_mail", "pkg/mail/parse.py"},
+ {"Folder", "p.pkg.mail", "pkg/mail"},
+ {"Folder", "p.pkg", "pkg"},
+ {"Folder", "p.pkg.mailer", "pkg/mailer"},
+ {"Folder", "p.pk", "pk"}};
+ for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) {
+ cbm_node_t node = {.project = "p",
+ .label = nodes[i].label,
+ .name = nodes[i].qn,
+ .qualified_name = nodes[i].qn,
+ .file_path = nodes[i].path};
+ ASSERT_TRUE(cbm_store_upsert_node(s, &node) > 0);
+ }
+ const cbm_doc_candidate_t rows[] = {
+ {.section_qn = "p.docs.a", .target_qn = "p.pkg.mail.parse.__file__", .p = 0.5},
+ {.section_qn = "p.docs.b", .target_qn = "p.pkg.mail", .p = 0.7},
+ {.section_qn = "p.docs.c", .target_qn = "p.pkg", .p = 0.3},
+ {.section_qn = "p.docs.d", .target_qn = "p.pkg.mailer", .p = 0.9},
+ {.section_qn = "p.docs.e", .target_qn = "p.pk", .p = 0.9},
+ {.section_qn = "p.docs.f", .target_qn = "p.pkg.mail.parse.parse_mail", .p = 0.95},
+ };
+ ASSERT_EQ(cbm_store_doc_candidates_replace(s, "p", rows, 6), CBM_STORE_OK);
+ const cbm_doc_candidate_t other = {
+ .section_qn = "q.docs.a", .target_qn = "p.pkg.mail", .p = 0.99};
+ ASSERT_EQ(cbm_store_doc_candidates_replace(s, "q", &other, 1), CBM_STORE_OK);
+
+ ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "pkg/mail/parse.py", 5, &got, &n),
+ CBM_STORE_OK);
+ ASSERT_EQ(n, 3);
+ ASSERT_STR_EQ(got[0].section_qn, "p.docs.b"); /* highest p first */
+ ASSERT_STR_EQ(got[1].section_qn, "p.docs.a");
+ ASSERT_STR_EQ(got[2].section_qn, "p.docs.c");
+ cbm_store_doc_candidates_free(got, n);
+
+ ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "pkg/mail/parse.py", 1, &got, &n),
+ CBM_STORE_OK);
+ ASSERT_EQ(n, 1);
+ cbm_store_doc_candidates_free(got, n);
+ ASSERT_EQ(cbm_store_doc_candidates_for_path(s, "p", "", 5, &got, &n), CBM_STORE_ERR);
+ cbm_store_close(s);
+ PASS();
+}
+
TEST(adr_parse_sections_basic) {
cbm_adr_sections_t sec = cbm_adr_parse_sections("## PURPOSE\nFoo\n\n## STACK\nBar");
ASSERT_EQ(sec.count, 2);
@@ -1051,11 +1109,11 @@ static int adr_check_splice(const char *in, const char *name, const char *body,
return 0;
}
-#define CHECK_SPLICE(in, name, body, expect) \
- do { \
- if (adr_check_splice((in), (name), (body), (expect)) != 0) { \
- return 1; \
- } \
+#define CHECK_SPLICE(in, name, body, expect) \
+ do { \
+ if (adr_check_splice((in), (name), (body), (expect)) != 0) { \
+ return 1; \
+ } \
} while (0)
/* THE acceptance property. Every document below is a case where rebuilding
@@ -1170,9 +1228,9 @@ TEST(adr_splice_mixed_line_endings) {
TEST(adr_splice_matches_headings_across_line_endings) {
adr_name_collect_t c;
memset(&c, 0, sizeof(c));
- ASSERT_EQ(cbm_adr_scan_headings("## PURPOSE\r\nFoo\r\n\r\n## STACK\r\nBar",
- adr_collect_names, &c),
- CBM_STORE_OK);
+ ASSERT_EQ(
+ cbm_adr_scan_headings("## PURPOSE\r\nFoo\r\n\r\n## STACK\r\nBar", adr_collect_names, &c),
+ CBM_STORE_OK);
ASSERT_STR_EQ(c.buf, "[PURPOSE][STACK]");
/* A fenced block with CRLF still hides its heading, and still closes. */
@@ -1778,6 +1836,7 @@ SUITE(store_arch) {
RUN_TEST(adr_delete);
RUN_TEST(adr_delete_not_found);
RUN_TEST(doc_candidates_replace_and_get);
+ RUN_TEST(doc_candidates_for_path_file_and_folders);
RUN_TEST(adr_parse_sections_basic);
RUN_TEST(adr_parse_sections_all_six);
RUN_TEST(adr_parse_sections_non_canonical);