Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

feat(semantic): doc section -> code candidates with a judged p

Martin Vogel842535e7aef7
diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c
index bd4196eea..3321eb1f6 100644
--- a/src/mcp/mcp.c
+++ b/src/mcp/mcp.c
@@ -12859,6 +12859,68 @@ static char *snippet_budget_floor(bool json_format, int max_output_tokens) {
return json;
}
+/* Doc -> code candidates (doc_link_candidates): on a Section, the functions it
+ * is possibly about; on a function or method, the sections that possibly
+ * describe it -- highest p first. Candidates with a judged probability for the
+ * agent to verify, never links. A row whose other end no longer resolves (a
+ * delta update renamed or removed it) is skipped. */
+enum { MCP_DOC_CANDIDATES_MAX = 5 };
+static void add_doc_candidates(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_store_t *store,
+ const cbm_node_t *node) {
+ if (!store || !node->label || !node->qualified_name || !node->project) {
+ return;
+ }
+ bool section = strcmp(node->label, "Section") == 0;
+ if (!section && strcmp(node->label, "Function") != 0 && strcmp(node->label, "Method") != 0) {
+ return;
+ }
+ cbm_doc_candidate_t *rows = NULL;
+ int n = 0;
+ if (cbm_store_doc_candidates_get(store, node->project, section ? node->qualified_name : NULL,
+ section ? NULL : node->qualified_name, MCP_DOC_CANDIDATES_MAX,
+ &rows, &n) != CBM_STORE_OK) {
+ return;
+ }
+ yyjson_mut_val *arr = yyjson_mut_arr(doc);
+ int added = 0;
+ for (int i = 0; i < n; i++) {
+ cbm_node_t other = {0};
+ if (cbm_store_find_node_by_qn(store, node->project,
+ section ? rows[i].target_qn : rows[i].section_qn,
+ &other) != CBM_STORE_OK) {
+ continue;
+ }
+ yyjson_mut_val *o = yyjson_mut_obj(doc);
+ yyjson_mut_obj_add_strcpy(doc, o, section ? "qualified_name" : "section",
+ other.qualified_name ? other.qualified_name : "");
+ yyjson_mut_obj_add_strcpy(doc, o, "file_path", other.file_path ? other.file_path : "");
+ yyjson_mut_obj_add_int(doc, o, "start_line", other.start_line);
+ if (section) {
+ yyjson_mut_obj_add_int(doc, o, "end_line", other.end_line);
+ }
+ yyjson_mut_obj_add_real(doc, o, "p", (double)(int)(rows[i].p * 1000.0 + 0.5) / 1000.0);
+ int shared = 0;
+ int name_tokens = 0;
+ if (section && rows[i].evidence &&
+ sscanf(rows[i].evidence, "{\"shared_terms\":%d,\"name_tokens\":%d}", &shared,
+ &name_tokens) == 2) {
+ yyjson_mut_obj_add_int(doc, o, "shared_terms", shared);
+ yyjson_mut_obj_add_int(doc, o, "name_tokens", name_tokens);
+ }
+ yyjson_mut_arr_append(arr, o);
+ added++;
+ free_node_contents(&other);
+ }
+ cbm_store_doc_candidates_free(rows, n);
+ if (added > 0) {
+ yyjson_mut_obj_add_val(doc, root, section ? "possibly_about" : "possibly_described_in",
+ arr);
+ yyjson_mut_obj_add_str(doc, root, "candidates_note",
+ "doc<->code candidates by shared terms, not links: p is the "
+ "judged probability the section is about the function; verify");
+ }
+}
+
static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node,
const char *match_method, bool include_neighbors,
cbm_node_t *alternatives, int alt_count, const char *args) {
@@ -13075,6 +13137,8 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node,
yyjson_mut_obj_add_val(doc, root_obj, "alternatives", arr);
}
+ add_doc_candidates(doc, root_obj, store, node);
+
bool bounded_default = max_output_tokens > 0;
size_t snippet_byte_budget =
bounded_default ? (size_t)max_output_tokens * (size_t)MCP_OUTPUT_BYTES_PER_TOKEN_ESTIMATE
diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c
index da131d3a9..c5caf3a83 100644
--- a/src/pipeline/pass_semantic_edges.c
+++ b/src/pipeline/pass_semantic_edges.c
@@ -30,6 +30,8 @@
#include "foundation/platform.h"
#include "foundation/profile.h"
+#include <ctype.h>
+#include <math.h>
#include <stdatomic.h>
#include <stddef.h>
#include <stdio.h>
@@ -1870,6 +1872,573 @@ static cbm_sem_corpus_t *run_token_phases_batched(cbm_gbuf_t *gbuf, cbm_sem_func
/* ── Pass entry point ────────────────────────────────────────────── */
+/* ── Phase 6c: doc sections -> functions (candidate generator) ───── */
+
+/* Doc -> code candidates: for every Section node that can be about code, the
+ * functions whose TF-IDF terms best match the section's heading + body, over
+ * the FUNCTIONS' corpus (sections never change a function-pair score).
+ * Gate: no candidates for sections whose heading names a release (v1.2), a
+ * changelog, licence, contributing, links, contents, install or support
+ * kind of section, whose body has fewer than SEM_DOC_MIN_WORDS words, or
+ * whose body is mostly links.
+ * Targets: never a test function (by path or name); candidates come from
+ * the section's key terms (held by at most one in SEM_DOC_KEY_DF_DIV
+ * functions, or SEM_DOC_KEY_DF_MIN) and are scored on every shared term.
+ * Stored (doc_link_candidates, never graph edges): a section's best
+ * CBM_SEM_DOC_TOP_K with tfidf >= CBM_SEM_DOC_MIN_SCORE, each with its judged
+ * probability (cbm_sem_doc_calibrated_p) -- candidates for an agent to
+ * verify, not links.
+ * CBM_SEM_DOC_SIGNALS=<path> also dumps every section's best SEM_DOC_TOP_K:
+ * section and function qualified names, tfidf, rank, shared terms, both
+ * locations, and the name evidence (the function's name tokens present in the
+ * section, and the highest idf among them) -- the population the judged
+ * calibration samples are drawn from. */
+enum {
+ SEM_DOC_TOP_K = 10,
+ SEM_DOC_KEY_DF_MIN = 50,
+ SEM_DOC_KEY_DF_DIV = 20,
+ SEM_DOC_MAX_TOKENS = 1024,
+ SEM_DOC_TEXT_MAX = 8192,
+ SEM_DOC_NAME_TOKENS = 16,
+ SEM_DOC_MIN_WORDS = 20,
+ SEM_DOC_LINK_PCT_MAX = 20, /* links per 100 body words */
+ SEM_DOC_WORD_MIN_LEN = 3,
+};
+
+typedef struct {
+ float score;
+ int func;
+ int shared;
+} doc_cand_t;
+
+typedef struct {
+ int func;
+ float tfidf;
+ int rank;
+ int shared;
+ int name_shared; /* the function's NAME tokens present in the section */
+ float name_idf; /* the highest idf among them (0: none) */
+} doc_hit_t;
+
+typedef struct {
+ doc_hit_t hits[SEM_DOC_TOP_K];
+ int n;
+} doc_result_t;
+
+static int cmp_doc_cand(const void *pa, const void *pb) {
+ const doc_cand_t *a = pa;
+ const doc_cand_t *b = pb;
+ if (a->score != b->score) {
+ return a->score > b->score ? -1 : 1;
+ }
+ return (a->func > b->func) - (a->func < b->func);
+}
+
+static int cmp_section_node(const void *pa, const void *pb) {
+ const cbm_gbuf_node_t *a = *(const cbm_gbuf_node_t *const *)pa;
+ const cbm_gbuf_node_t *b = *(const cbm_gbuf_node_t *const *)pb;
+ int c = strcmp(a->qualified_name ? a->qualified_name : "",
+ b->qualified_name ? b->qualified_name : "");
+ if (c) {
+ return c;
+ }
+ return (a->start_line > b->start_line) - (a->start_line < b->start_line);
+}
+
+/* Heading kinds that are about the project, not about code: a release
+ * number, or one of these words (a trailing '*' matches any ending). */
+static const char *const DOC_SKIP_HEADING[] = {"changelog",
+ "change log",
+ "release note*",
+ "what's new",
+ "license",
+ "licence",
+ "contribut*",
+ "acknowledg*",
+ "credits",
+ "sponsor*",
+ "author*",
+ "code of conduct",
+ "security policy",
+ "links",
+ "references",
+ "resources",
+ "further reading",
+ "table of contents",
+ "contents",
+ "toc",
+ "installation",
+ "install",
+ "getting started",
+ "support",
+ "community",
+ "faq",
+ "roadmap",
+ "todo",
+ NULL};
+
+static bool doc_word_char(char c) {
+ return isalnum((unsigned char)c) || c == '\'';
+}
+
+/* Does lowercase heading lc contain word (or word* as a prefix) at a word start? */
+static bool doc_heading_has(const char *lc, const char *word) {
+ size_t wl = strlen(word);
+ bool prefix = wl > 0 && word[wl - SKIP_ONE] == '*';
+ size_t ml = prefix ? wl - SKIP_ONE : wl;
+ for (const char *p = lc; *p; p++) {
+ if (strncmp(p, word, ml) != 0) {
+ continue;
+ }
+ if (p > lc && doc_word_char(p[-SKIP_ONE])) {
+ continue;
+ }
+ if (!prefix && doc_word_char(p[ml])) {
+ continue;
+ }
+ return true;
+ }
+ return false;
+}
+
+/* A release-number heading: "v1.2", "1.2.3", "[0.12.15]", "Gin v1.11.0". */
+static bool doc_heading_is_release(const char *lc) {
+ for (const char *p = lc; *p; p++) {
+ if (p > lc && doc_word_char(p[-SKIP_ONE])) {
+ continue;
+ }
+ const char *q = *p == 'v' ? p + SKIP_ONE : p;
+ if (!isdigit((unsigned char)*q)) {
+ continue;
+ }
+ while (isdigit((unsigned char)*q)) {
+ q++;
+ }
+ if (*q == '.' && isdigit((unsigned char)q[SKIP_ONE])) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/* Can this section be about code? (heading kind, body length, link share) */
+static bool doc_section_gated(const cbm_gbuf_node_t *sec, char *body) {
+ char lc[CBM_SZ_256];
+ size_t n = 0;
+ for (const char *p = sec->name ? sec->name : ""; *p && n + SKIP_ONE < sizeof(lc); p++) {
+ lc[n++] = (char)tolower((unsigned char)*p);
+ }
+ lc[n] = '\0';
+ if (doc_heading_is_release(lc)) {
+ return false;
+ }
+ for (int k = 0; DOC_SKIP_HEADING[k]; k++) {
+ if (doc_heading_has(lc, DOC_SKIP_HEADING[k])) {
+ return false;
+ }
+ }
+ if (!json_str_value(sec->properties_json, "docstring", body, SEM_DOC_TEXT_MAX)) {
+ return false;
+ }
+ int words = 0;
+ int links = 0;
+ int run = 0;
+ for (const char *p = body;; p++) {
+ if (isalpha((unsigned char)*p)) {
+ run++;
+ continue;
+ }
+ words += run >= SEM_DOC_WORD_MIN_LEN ? SKIP_ONE : 0;
+ run = 0;
+ if (!*p) {
+ break;
+ }
+ if (strncmp(p, "http://", sizeof("http://") - SKIP_ONE) == 0 ||
+ strncmp(p, "https://", sizeof("https://") - SKIP_ONE) == 0 ||
+ strncmp(p, "](", sizeof("](") - SKIP_ONE) == 0) {
+ links++;
+ }
+ }
+ return words >= SEM_DOC_MIN_WORDS && links * 100 < SEM_DOC_LINK_PCT_MAX * words;
+}
+
+/* A test function, by its file (test directories, test file names) or name. */
+static bool doc_is_test_function(const cbm_gbuf_node_t *fn) {
+ static const char *const DIRS[] = {"test/", "tests/", "__tests__/", "testing/", "spec/",
+ "specs/", "testdata/", "fixture/", "fixtures/", NULL};
+ const char *path = fn && fn->file_path ? fn->file_path : "";
+ for (int k = 0; DIRS[k]; k++) {
+ for (const char *p = path; (p = strstr(p, DIRS[k])) != NULL; p++) {
+ if (p == path || p[-SKIP_ONE] == '/') {
+ return true;
+ }
+ }
+ }
+ const char *base = strrchr(path, '/');
+ base = base ? base + SKIP_ONE : path;
+ if (strncmp(base, "test_", sizeof("test_") - SKIP_ONE) == 0 || strstr(base, "_test.") ||
+ strstr(base, ".test.") || strstr(base, ".spec.") || strstr(base, "Test.") ||
+ strstr(base, "Tests.")) {
+ return true;
+ }
+ const char *name = fn && fn->name ? fn->name : "";
+ return strncmp(name, "test_", sizeof("test_") - SKIP_ONE) == 0 ||
+ (strncmp(name, "Test", sizeof("Test") - SKIP_ONE) == 0 &&
+ (isupper((unsigned char)name[4]) || name[4] == '_'));
+}
+
+/* The section's tokens (heading + body); the caller frees each token. */
+static int doc_section_tokens(const cbm_gbuf_node_t *n, const char *body, char **tokens) {
+ int count = cbm_sem_tokenize(n->name, tokens, SEM_DOC_MAX_TOKENS);
+ if (body && count < SEM_DOC_MAX_TOKENS) {
+ count += cbm_sem_tokenize(body, tokens + count, SEM_DOC_MAX_TOKENS - count);
+ }
+ return count;
+}
+
+/* Dot product and shared-term count of two ascending term lists. */
+static float doc_terms_dot(const int *ia, const float *wa, int na, const int *ib, const float *wb,
+ int nb, int *shared) {
+ float dot = 0.0F;
+ int a = 0;
+ int b = 0;
+ *shared = 0;
+ while (a < na && b < nb) {
+ if (ia[a] == ib[b]) {
+ dot += wa[a] * wb[b];
+ (*shared)++;
+ a++;
+ b++;
+ } else if (ia[a] < ib[b]) {
+ a++;
+ } else {
+ b++;
+ }
+ }
+ return dot;
+}
+
+static float terms_norm(const float *w, int n) {
+ float s = 0.0F;
+ for (int k = 0; k < n; k++) {
+ s += w[k] * w[k];
+ }
+ return sqrtf(s);
+}
+
+typedef struct {
+ const cbm_sem_func_t *funcs;
+ int func_count;
+ const cbm_sem_corpus_t *corpus;
+ const cbm_gbuf_node_t **secs;
+ int nsec;
+ const int *post_off;
+ const int *post_f;
+ int key_df;
+ const float *fnorm;
+ const bool *is_test;
+ const int *name_off; /* per function: its name tokens' corpus indices */
+ const int *name_idx;
+ doc_result_t *res;
+ _Atomic int next;
+} doc_ctx_t;
+
+/* Name-field evidence of one hit: the function's name tokens that the
+ * section holds (sidx ascending), and the highest idf among them. */
+static void doc_name_evidence(const doc_ctx_t *dc, const int *sidx, int ns, doc_hit_t *hit) {
+ hit->name_shared = 0;
+ hit->name_idf = 0.0F;
+ for (int k = dc->name_off[hit->func]; k < dc->name_off[hit->func + SKIP_ONE]; k++) {
+ int t = dc->name_idx[k];
+ int lo = 0;
+ int hi = ns - SKIP_ONE;
+ while (lo <= hi) {
+ int mid = (lo + hi) / 2;
+ if (sidx[mid] == t) {
+ float idf = cbm_sem_corpus_idf_at(dc->corpus, t);
+ hit->name_shared++;
+ hit->name_idf = idf > hit->name_idf ? idf : hit->name_idf;
+ break;
+ }
+ if (sidx[mid] < t) {
+ lo = mid + SKIP_ONE;
+ } else {
+ hi = mid - SKIP_ONE;
+ }
+ }
+ }
+}
+
+static void doc_section_one(doc_ctx_t *dc, int s, int *stamp, doc_cand_t *cands, int *sidx,
+ float *sw, char *body) {
+ doc_result_t *r = &dc->res[s];
+ r->n = 0;
+ if (!doc_section_gated(dc->secs[s], body)) {
+ return;
+ }
+ char *tokens[SEM_DOC_MAX_TOKENS];
+ int count = doc_section_tokens(dc->secs[s], body, tokens);
+ int ns = cbm_sem_tfidf_terms(dc->corpus, tokens, NULL, count, sidx, sw);
+ for (int t = 0; t < count; t++) {
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, tokens[t]);
+ }
+ float snorm = terms_norm(sw, ns);
+ int nc = 0;
+ for (int k = 0; k < ns && snorm > 0.0F; k++) {
+ int t = sidx[k];
+ if (dc->post_off[t + SKIP_ONE] - dc->post_off[t] > dc->key_df) {
+ continue;
+ }
+ for (int p = dc->post_off[t]; p < dc->post_off[t + SKIP_ONE]; p++) {
+ int f = dc->post_f[p];
+ if (stamp[f] != s && !dc->is_test[f]) {
+ stamp[f] = s;
+ cands[nc++] = (doc_cand_t){.func = f};
+ }
+ }
+ }
+ for (int c = 0; c < nc; c++) {
+ const cbm_sem_func_t *fn = &dc->funcs[cands[c].func];
+ float dot = doc_terms_dot(sidx, sw, ns, fn->tfidf_indices, fn->tfidf_weights, fn->tfidf_len,
+ &cands[c].shared);
+ float denom = snorm * dc->fnorm[cands[c].func];
+ cands[c].score = denom > 0.0F ? dot / denom : 0.0F;
+ }
+ if (nc > 1) {
+ qsort(cands, (size_t)nc, sizeof(cands[0]), cmp_doc_cand);
+ }
+ for (int c = 0; c < nc && c < SEM_DOC_TOP_K; c++) {
+ doc_hit_t *hit = &r->hits[r->n++];
+ *hit = (doc_hit_t){.func = cands[c].func,
+ .tfidf = cands[c].score,
+ .rank = c + SKIP_ONE,
+ .shared = cands[c].shared};
+ doc_name_evidence(dc, sidx, ns, hit);
+ }
+}
+
+static void doc_section_worker(int worker_id, void *ctx_ptr) {
+ (void)worker_id;
+ doc_ctx_t *dc = ctx_ptr;
+ int *stamp = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)dc->func_count * sizeof(int));
+ doc_cand_t *cands =
+ cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)dc->func_count * sizeof(doc_cand_t));
+ int *sidx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)SEM_DOC_MAX_TOKENS * sizeof(int));
+ float *sw = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)SEM_DOC_MAX_TOKENS * sizeof(float));
+ char *body = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, SEM_DOC_TEXT_MAX);
+ if (stamp && cands && sidx && sw && body) {
+ for (int f = 0; f < dc->func_count; f++) {
+ stamp[f] = -1;
+ }
+ while (true) {
+ int s = atomic_fetch_add_explicit(&dc->next, SKIP_ONE, memory_order_relaxed);
+ if (s >= dc->nsec) {
+ break;
+ }
+ doc_section_one(dc, s, stamp, cands, sidx, sw, body);
+ }
+ }
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, stamp);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, cands);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, sidx);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, sw);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, body);
+}
+
+static void doc_section_lines(FILE *out, const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sec,
+ const cbm_sem_func_t *funcs, const doc_result_t *r) {
+ for (int h = 0; h < r->n; h++) {
+ const doc_hit_t *hit = &r->hits[h];
+ const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[hit->func].node_id);
+ (void)fprintf(out, "%s\t%s\t%.6f\t%d\t%d\t%s:%d-%d\t%s:%d-%d\t%d\t%.6f\n",
+ sec->qualified_name ? sec->qualified_name : "?",
+ fn && fn->qualified_name ? fn->qualified_name : "?", (double)hit->tfidf,
+ hit->rank, hit->shared, sec->file_path ? sec->file_path : "?",
+ sec->start_line, sec->end_line, fn && fn->file_path ? fn->file_path : "?",
+ fn ? fn->start_line : 0, fn ? fn->end_line : 0, hit->name_shared,
+ (double)hit->name_idf);
+ }
+}
+
+/* The stored candidates of every section (rank <= CBM_SEM_DOC_TOP_K, tfidf >=
+ * CBM_SEM_DOC_MIN_SCORE), malloc'd for cbm_store_doc_candidates_free; p at two
+ * decimals, as on SEMANTICALLY_RELATED edges. */
+static cbm_doc_candidate_t *doc_candidate_rows(const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t **secs,
+ int nsec, const cbm_sem_func_t *funcs,
+ const doc_result_t *res, int *out_count) {
+ int n = 0;
+ for (int s = 0; s < nsec; s++) {
+ for (int h = 0; h < res[s].n; h++) {
+ n += res[s].hits[h].rank <= CBM_SEM_DOC_TOP_K &&
+ res[s].hits[h].tfidf >= CBM_SEM_DOC_MIN_SCORE;
+ }
+ }
+ *out_count = 0;
+ cbm_doc_candidate_t *rows = n > 0 ? calloc((size_t)n, sizeof(*rows)) : NULL;
+ if (!rows) {
+ return NULL;
+ }
+ int k = 0;
+ for (int s = 0; s < nsec; s++) {
+ for (int h = 0; h < res[s].n; h++) {
+ const doc_hit_t *hit = &res[s].hits[h];
+ if (hit->rank > CBM_SEM_DOC_TOP_K || hit->tfidf < CBM_SEM_DOC_MIN_SCORE) {
+ continue;
+ }
+ const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[hit->func].node_id);
+ if (!fn || !fn->qualified_name || !secs[s]->qualified_name) {
+ continue;
+ }
+ char ev[CBM_SZ_64];
+ snprintf(ev, sizeof(ev), "{\"shared_terms\":%d,\"name_tokens\":%d}", hit->shared,
+ hit->name_shared);
+ rows[k] = (cbm_doc_candidate_t){
+ .section_qn = strdup(secs[s]->qualified_name),
+ .target_qn = strdup(fn->qualified_name),
+ .rank = hit->rank,
+ .score = hit->tfidf,
+ .p = (double)(int)(cbm_sem_doc_calibrated_p(hit->tfidf) * 100.0F + 0.5F) / 100.0,
+ .evidence = strdup(ev)};
+ if (!rows[k].section_qn || !rows[k].target_qn || !rows[k].evidence) {
+ cbm_store_doc_candidates_free(rows, k + SKIP_ONE);
+ return NULL;
+ }
+ k++;
+ }
+ }
+ *out_count = k;
+ return rows;
+}
+
+static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *funcs,
+ int func_count, const cbm_sem_corpus_t *corpus, int worker_count) {
+ const cbm_gbuf_t *gbuf = ctx->gbuf;
+ if (!corpus || func_count <= 0) {
+ return;
+ }
+ const cbm_gbuf_node_t **found = NULL;
+ int nsec = 0;
+ if (cbm_gbuf_find_by_label(gbuf, "Section", &found, &nsec) != 0 || nsec == 0) {
+ return;
+ }
+ int nterms = cbm_sem_corpus_token_count(corpus);
+ const cbm_gbuf_node_t **secs = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(*secs));
+ int *post_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int));
+ float *fnorm = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(float));
+ bool *is_test = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(bool));
+ int *name_off =
+ cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)func_count + SKIP_ONE) * sizeof(int));
+ int *name_idx = cbm_alloc(CBM_MEM_CLASS_SEMANTIC,
+ ((size_t)func_count * SEM_DOC_NAME_TOKENS + SKIP_ONE) * sizeof(int));
+ doc_result_t *res = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)nsec * sizeof(doc_result_t));
+ int *post_f = NULL;
+ int *cursor = NULL;
+ FILE *out = NULL;
+ if (secs && post_off && fnorm && is_test && name_off && name_idx && res) {
+ for (int f = 0; f < func_count; f++) {
+ for (int k = 0; k < funcs[f].tfidf_len; k++) {
+ post_off[funcs[f].tfidf_indices[k] + SKIP_ONE]++;
+ }
+ fnorm[f] = terms_norm(funcs[f].tfidf_weights, funcs[f].tfidf_len);
+ const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[f].node_id);
+ is_test[f] = doc_is_test_function(fn);
+ char *tok[SEM_DOC_NAME_TOKENS];
+ int nt = fn && fn->name ? cbm_sem_tokenize(fn->name, tok, SEM_DOC_NAME_TOKENS) : 0;
+ int base = name_off[f];
+ int m = 0;
+ for (int t = 0; t < nt; t++) {
+ int idx = cbm_sem_corpus_token_index(corpus, tok[t]);
+ bool dup = false;
+ for (int u = 0; u < m; u++) {
+ dup = dup || name_idx[base + u] == idx;
+ }
+ if (idx >= 0 && !dup) {
+ name_idx[base + m++] = idx;
+ }
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, tok[t]);
+ }
+ name_off[f + SKIP_ONE] = base + m;
+ }
+ for (int t = 0; t < nterms; t++) {
+ post_off[t + SKIP_ONE] += post_off[t];
+ }
+ post_f =
+ cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)post_off[nterms] + SKIP_ONE) * sizeof(int));
+ cursor = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)nterms + SKIP_ONE) * sizeof(int));
+ }
+ if (post_f && cursor) {
+ for (int f = 0; f < func_count; f++) {
+ for (int k = 0; k < funcs[f].tfidf_len; k++) {
+ int t = funcs[f].tfidf_indices[k];
+ post_f[post_off[t] + cursor[t]++] = f;
+ }
+ }
+ int key_df = func_count / SEM_DOC_KEY_DF_DIV;
+ if (key_df < SEM_DOC_KEY_DF_MIN) {
+ key_df = SEM_DOC_KEY_DF_MIN;
+ }
+ memcpy(secs, found, (size_t)nsec * sizeof(*secs));
+ qsort(secs, (size_t)nsec, sizeof(*secs), cmp_section_node);
+ doc_ctx_t dc = {.funcs = funcs,
+ .func_count = func_count,
+ .corpus = corpus,
+ .secs = secs,
+ .nsec = nsec,
+ .post_off = post_off,
+ .post_f = post_f,
+ .key_df = key_df,
+ .fnorm = fnorm,
+ .is_test = is_test,
+ .name_off = name_off,
+ .name_idx = name_idx,
+ .res = res};
+ atomic_init(&dc.next, 0);
+ cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false};
+ cbm_parallel_for(worker_count, doc_section_worker, &dc, opts);
+ int row_count = 0;
+ cbm_doc_candidate_t *rows = doc_candidate_rows(gbuf, secs, nsec, funcs, res, &row_count);
+ cbm_pipeline_set_doc_candidates(ctx->pipeline, rows, row_count);
+ cbm_log_info("pass.semantic.doc_candidates", "sections", itoa_log(nsec), "rows",
+ itoa_log(row_count));
+ char path[CBM_SZ_1K];
+ out = cbm_safe_getenv("CBM_SEM_DOC_SIGNALS", path, sizeof(path), NULL)
+ ? cbm_fopen(path, "w")
+ : NULL;
+ for (int s = 0; out && s < nsec; s++) {
+ doc_section_lines(out, gbuf, secs[s], funcs, &res[s]);
+ }
+ /* CBM_SEM_DOC_FUNCS=<path>: every function with its location and the
+ * test flag, for measuring candidates the TF-IDF generator does not
+ * produce (judged samples only). */
+ char fpath[CBM_SZ_1K];
+ FILE *fout = cbm_safe_getenv("CBM_SEM_DOC_FUNCS", fpath, sizeof(fpath), NULL)
+ ? cbm_fopen(fpath, "w")
+ : NULL;
+ for (int f = 0; fout && f < func_count; f++) {
+ const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_id(gbuf, funcs[f].node_id);
+ (void)fprintf(fout, "%s\t%s:%d-%d\t%d\n",
+ fn && fn->qualified_name ? fn->qualified_name : "?",
+ fn && fn->file_path ? fn->file_path : "?", fn ? fn->start_line : 0,
+ fn ? fn->end_line : 0, is_test[f] ? 1 : 0);
+ }
+ if (fout) {
+ (void)fclose(fout);
+ }
+ }
+ if (out) {
+ (void)fclose(out);
+ }
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, cursor);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, post_f);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, secs);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, post_off);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, fnorm);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, is_test);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, name_off);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, name_idx);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, res);
+}
+
int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) {
/* Controlled by pipeline mode (moderate/full), not env var */
cbm_log_info("pass.start", "pass", "semantic_edges");
@@ -1958,6 +2527,9 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) {
sem_mem_mark("6_score");
+ /* Phase 6c: doc sections -> function candidates (doc_link_candidates). */
+ phase6c_doc_sections(ctx, funcs, func_count, corpus, worker_count);
+
/* Phase 7: Cleanup */
CBM_PROF_START(t_phase7);
free_lsh_buckets(band_buckets);
diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c
index 38e1cb4b2..436a85de4 100644
--- a/src/pipeline/pipeline.c
+++ b/src/pipeline/pipeline.c
@@ -273,6 +273,11 @@ struct cbm_pipeline {
int doc_link_row_count;
bool doc_links_failed;
bool doc_links_ran;
+ /* Doc -> code candidates from pass_semantic_edges, published with the
+ * generation (heap rows, cbm_store_doc_candidates_free in
+ * cbm_pipeline_free). NULL when the pass found none or did not run. */
+ cbm_doc_candidate_t *doc_candidates;
+ int doc_candidate_count;
/* Deterministic test-only seam at the final publication boundary. Kept
* per pipeline so concurrent test/process activity cannot cross-trigger. */
@@ -915,6 +920,23 @@ void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *row
p->surface_row_count = count;
}
+void cbm_pipeline_set_doc_candidates(cbm_pipeline_t *p, cbm_doc_candidate_t *rows, int count) {
+ if (!p) {
+ cbm_store_doc_candidates_free(rows, count);
+ return;
+ }
+ cbm_store_doc_candidates_free(p->doc_candidates, p->doc_candidate_count);
+ p->doc_candidates = rows;
+ p->doc_candidate_count = rows ? count : 0;
+}
+
+const cbm_doc_candidate_t *cbm_pipeline_doc_candidates(const cbm_pipeline_t *p, int *count) {
+ if (count) {
+ *count = p ? p->doc_candidate_count : 0;
+ }
+ return p ? p->doc_candidates : NULL;
+}
+
static void pipeline_release_test_config(cbm_pipeline_t *p) {
if (!p) {
return;
@@ -995,6 +1017,9 @@ void cbm_pipeline_free(cbm_pipeline_t *p) {
p->surface_rows = NULL;
p->surface_row_count = 0;
pipeline_reset_doc_links(p);
+ cbm_store_doc_candidates_free(p->doc_candidates, p->doc_candidate_count);
+ p->doc_candidates = NULL;
+ p->doc_candidate_count = 0;
cbm_git_context_free(&p->git_ctx);
/* gbuf, store, registry freed during/after run */
/* Defensive owner cleanup; normal runs release after publication. */
@@ -3075,6 +3100,14 @@ static int publish_staged_impl(char *stage_path, const cbm_pipeline_generation_t
ok = cbm_store_adr_store(store, generation->project, generation->adr_content) ==
CBM_STORE_OK;
}
+ /* Doc -> code candidates belong to the generation too: replaced inside
+ * the staging store (the table is created here), so a reader never sees
+ * candidates of another graph. */
+ if (ok && !generation->doc_candidates_in_place) {
+ ok =
+ cbm_store_doc_candidates_replace(store, generation->project, generation->doc_candidates,
+ generation->doc_candidate_count) == CBM_STORE_OK;
+ }
if (ok) {
ok = cbm_store_set_format_version(store, CBM_INDEX_FORMAT_VERSION) == CBM_STORE_OK;
@@ -3366,6 +3399,8 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas
.doc_link_row_count = p->doc_link_row_count,
/* every full route runs a doc-link phase; none having run is a fault */
.doc_links_failed = p->doc_links_failed || !p->doc_links_ran,
+ .doc_candidates = p->doc_candidates,
+ .doc_candidate_count = p->doc_candidate_count,
};
free(db_dir);
diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c
index bc3beb7e6..330d36d89 100644
--- a/src/pipeline/pipeline_incremental.c
+++ b/src/pipeline/pipeline_incremental.c
@@ -2082,7 +2082,8 @@ static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *p
const cbm_coverage_row_t *cov, int cov_count,
const cbm_coverage_meta_t *meta_template,
const cbm_lsp_surface_row_t *surface_rows, int surface_row_count,
- legacy_doc_rows_t doc) {
+ legacy_doc_rows_t doc, const cbm_doc_candidate_t *doc_candidates,
+ int doc_candidate_count) {
struct timespec t;
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
cbm_pipeline_generation_t generation = {
@@ -2101,6 +2102,8 @@ static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *p
.doc_link_rows = doc.rows,
.doc_link_row_count = doc.count,
.doc_links_failed = doc.failed,
+ .doc_candidates = doc_candidates,
+ .doc_candidate_count = doc_candidate_count,
};
int rc = cbm_pipeline_publish_generation(&generation);
cbm_log_info("incremental.dump", "rc", itoa_buf(rc), "elapsed_ms",
@@ -3169,6 +3172,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
.doc_link_rows = doc_rows,
.doc_link_row_count = doc_row_count,
.doc_links_failed = doc_failed,
+ .doc_candidates_in_place = true, /* the clone keeps the previous rows */
};
cbm_store_close(staging);
staging = NULL;
@@ -3796,9 +3800,13 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
* would satisfy a future closure plan with yesterday's surface; an
* empty table just routes the next incremental to a full rebuild. */
legacy_doc_rows_t doc_out = {.rows = doc_rows, .count = doc_row_count, .failed = doc_failed};
- int persist_rc =
- dump_and_persist(existing, db_path, project, cbm_pipeline_cancelled_ptr(p), manifest,
- manifest_count, saved_adr, cov, cov_n, &coverage_meta, NULL, 0, doc_out);
+ /* Doc candidates: the post-passes ran the semantic pass over the whole
+ * rehydrated project, so this run's rows are complete. */
+ int doc_cand_n = 0;
+ const cbm_doc_candidate_t *doc_cands = cbm_pipeline_doc_candidates(p, &doc_cand_n);
+ int persist_rc = dump_and_persist(existing, db_path, project, cbm_pipeline_cancelled_ptr(p),
+ manifest, manifest_count, saved_adr, cov, cov_n,
+ &coverage_meta, NULL, 0, doc_out, doc_cands, doc_cand_n);
cbm_doclinks_free_rows(doc_rows, doc_row_count);
cbm_pipeline_free_semantic_manifest(manifest, manifest_count);
free(saved_adr);
diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h
index b00dcd699..6bcbb6361 100644
--- a/src/pipeline/pipeline_internal.h
+++ b/src/pipeline/pipeline_internal.h
@@ -1310,6 +1310,16 @@ typedef struct {
const cbm_doc_link_row_t *doc_link_rows;
int doc_link_row_count;
bool doc_links_failed;
+ /* Doc -> code candidates of this generation (pass_semantic_edges; may be
+ * NULL: none found, or a mode without the semantic pass), written into
+ * the staging store with the graph. */
+ const cbm_doc_candidate_t *doc_candidates;
+ int doc_candidate_count;
+ /* True when the staging store already holds this generation's
+ * candidates: the delta clone keeps the previous generation's rows (its
+ * proxy buffer has no section text or function tokens to recompute
+ * them); a row whose node is gone resolves to nothing when read. */
+ bool doc_candidates_in_place;
} cbm_pipeline_generation_t;
/* Serialize and fully populate a sibling staging database, then atomically
@@ -1382,6 +1392,12 @@ void cbm_pipeline_set_doc_link_rows(cbm_pipeline_t *p, cbm_doc_link_row_t *rows,
void cbm_pipeline_take_doc_link_rows(cbm_pipeline_t *p, cbm_doc_link_row_t **rows, int *count,
bool *failed, bool *ran);
+/* Hand the run's doc -> code candidates (pass_semantic_edges) to the
+ * pipeline. Takes ownership (rows and strings malloc'd, released with
+ * cbm_store_doc_candidates_free); a NULL p frees them. Passing NULL/0 clears. */
+void cbm_pipeline_set_doc_candidates(cbm_pipeline_t *p, cbm_doc_candidate_t *rows, int count);
+const cbm_doc_candidate_t *cbm_pipeline_doc_candidates(const cbm_pipeline_t *p, int *count);
+
/* Pipeline accessors for incremental use */
const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p);
const cbm_test_declarations_t *cbm_pipeline_test_declarations(const cbm_pipeline_t *p);
diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c
index 907b03a30..bbdd57b56 100644
--- a/src/semantic/semantic.c
+++ b/src/semantic/semantic.c
@@ -1861,6 +1861,23 @@ float cbm_sem_calibrated_p(float score) {
return SEM_P_BANDS[band];
}
+/* Doc -> code candidates by their tfidf: each band's p the judged share of
+ * sections that are about the function (2026-10-07: 274 blind-judged
+ * candidates -- a section's best five non-test functions -- from five held-out
+ * repositories, three reading rounds; the top three measured bands pooled to
+ * stay monotone). Below the first cut (judged 0.06) nothing is stored. */
+static const float SEM_DOC_P_CUTS[] = {CBM_SEM_DOC_MIN_SCORE, 0.30F, 0.40F};
+static const float SEM_DOC_P_BANDS[] = {0.06F, 0.36F, 0.42F, 0.548F};
+
+float cbm_sem_doc_calibrated_p(float score) {
+ size_t band = 0;
+ while (band < sizeof(SEM_DOC_P_CUTS) / sizeof(SEM_DOC_P_CUTS[0]) &&
+ score >= SEM_DOC_P_CUTS[band]) {
+ band++;
+ }
+ return SEM_DOC_P_BANDS[band];
+}
+
float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
const cbm_sem_config_t *cfg) {
if (!a || !b || !cfg) {
diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h
index 143909ef6..e641e2f29 100644
--- a/src/semantic/semantic.h
+++ b/src/semantic/semantic.h
@@ -264,6 +264,14 @@ const char *cbm_sem_json_str(const char *json, const char *key, char *buf, int b
* held-out samples (the edge property "p"). */
float cbm_sem_calibrated_p(float score);
+/* Doc -> code candidates: a section's best CBM_SEM_DOC_TOP_K non-test
+ * functions by TF-IDF, stored when their tfidf is at least
+ * CBM_SEM_DOC_MIN_SCORE, each with the judged probability that the section is
+ * about the function. */
+#define CBM_SEM_DOC_TOP_K 5
+#define CBM_SEM_DOC_MIN_SCORE 0.20F
+float cbm_sem_doc_calibrated_p(float score);
+
/* Module proximity multiplier based on file paths. */
float cbm_sem_proximity(const char *path_a, const char *path_b);
diff --git a/src/store/store.c b/src/store/store.c
index 4f76915d5..aad3f79e6 100644
--- a/src/store/store.c
+++ b/src/store/store.c
@@ -10746,6 +10746,137 @@ int cbm_adr_validate_section_keys(const char **keys, int count, char *errbuf, in
return CBM_STORE_OK;
}
+/* ── Doc -> code candidates ─────────────────────────────────────── */
+
+static const char DOC_CANDIDATES_DDL[] =
+ "CREATE TABLE IF NOT EXISTS doc_link_candidates ("
+ " project TEXT NOT NULL,"
+ " section_qn TEXT NOT NULL,"
+ " target_qn TEXT NOT NULL,"
+ " rank INTEGER NOT NULL,"
+ " score REAL NOT NULL,"
+ " p REAL NOT NULL,"
+ " evidence TEXT NOT NULL DEFAULT ''"
+ ");"
+ "CREATE INDEX IF NOT EXISTS idx_doc_link_candidates_section "
+ "ON doc_link_candidates(project, section_qn);"
+ "CREATE INDEX IF NOT EXISTS idx_doc_link_candidates_target "
+ "ON doc_link_candidates(project, target_qn);";
+
+int cbm_store_doc_candidates_replace(cbm_store_t *s, const char *project,
+ const cbm_doc_candidate_t *rows, int count) {
+ if (!s || !s->db || !project || count < 0 || (count > 0 && !rows)) {
+ return CBM_STORE_ERR;
+ }
+ if (exec_sql(s, DOC_CANDIDATES_DDL) != CBM_STORE_OK || exec_sql(s, "BEGIN;") != CBM_STORE_OK) {
+ return CBM_STORE_ERR;
+ }
+ sqlite3_stmt *del = NULL;
+ sqlite3_stmt *ins = NULL;
+ bool ok = sqlite3_prepare_v2(s->db, "DELETE FROM doc_link_candidates WHERE project = ?1;",
+ CBM_NOT_FOUND, &del, NULL) == SQLITE_OK &&
+ sqlite3_prepare_v2(s->db,
+ "INSERT INTO doc_link_candidates "
+ "(project, section_qn, target_qn, rank, score, p, evidence) "
+ "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);",
+ CBM_NOT_FOUND, &ins, NULL) == SQLITE_OK;
+ if (ok) {
+ bind_text(del, SKIP_ONE, project);
+ ok = sqlite3_step(del) == SQLITE_DONE;
+ }
+ for (int i = 0; ok && i < count; i++) {
+ if (!rows[i].section_qn || !rows[i].target_qn) {
+ continue;
+ }
+ bind_text(ins, SKIP_ONE, project);
+ bind_text(ins, ST_COL_2, rows[i].section_qn);
+ bind_text(ins, ST_COL_3, rows[i].target_qn);
+ sqlite3_bind_int(ins, ST_COL_4, rows[i].rank);
+ sqlite3_bind_double(ins, ST_COL_5, rows[i].score);
+ sqlite3_bind_double(ins, ST_COL_6, rows[i].p);
+ bind_text(ins, CBM_SZ_7, rows[i].evidence ? rows[i].evidence : "");
+ ok = sqlite3_step(ins) == SQLITE_DONE;
+ sqlite3_reset(ins);
+ }
+ sqlite3_finalize(del);
+ sqlite3_finalize(ins);
+ if (!ok) {
+ store_set_error_sqlite(s, "doc_candidates replace");
+ (void)exec_sql(s, "ROLLBACK;");
+ return CBM_STORE_ERR;
+ }
+ return exec_sql(s, "COMMIT;");
+}
+
+int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn,
+ const char *target_qn, int limit, cbm_doc_candidate_t **out,
+ int *count) {
+ if (!s || !s->db || !project || !out || !count || (!section_qn == !target_qn) || limit <= 0) {
+ return CBM_STORE_ERR;
+ }
+ *out = NULL;
+ *count = 0;
+ sqlite3_stmt *probe = NULL;
+ if (sqlite3_prepare_v2(s->db,
+ "SELECT 1 FROM sqlite_master WHERE type='table' AND "
+ "name='doc_link_candidates' LIMIT 1;",
+ CBM_NOT_FOUND, &probe, NULL) != SQLITE_OK) {
+ store_set_error_sqlite(s, "doc_candidates probe");
+ return CBM_STORE_ERR;
+ }
+ int rc = sqlite3_step(probe);
+ sqlite3_finalize(probe);
+ if (rc == SQLITE_DONE) {
+ return CBM_STORE_NOT_FOUND;
+ }
+ if (rc != SQLITE_ROW) {
+ store_set_error_sqlite(s, "doc_candidates probe step");
+ return CBM_STORE_ERR;
+ }
+ const char *sql = section_qn ? "SELECT section_qn, target_qn, rank, score, p, evidence FROM "
+ "doc_link_candidates WHERE project=?1 AND section_qn=?2 "
+ "ORDER BY p DESC, score DESC, target_qn LIMIT ?3;"
+ : "SELECT section_qn, target_qn, rank, score, p, evidence FROM "
+ "doc_link_candidates WHERE project=?1 AND target_qn=?2 "
+ "ORDER BY p DESC, score DESC, section_qn LIMIT ?3;";
+ sqlite3_stmt *stmt = NULL;
+ if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
+ store_set_error_sqlite(s, "doc_candidates get");
+ return CBM_STORE_ERR;
+ }
+ bind_text(stmt, SKIP_ONE, project);
+ bind_text(stmt, ST_COL_2, section_qn ? section_qn : target_qn);
+ sqlite3_bind_int(stmt, ST_COL_3, limit);
+ cbm_doc_candidate_t *rows = calloc((size_t)limit, sizeof(*rows));
+ if (!rows) {
+ sqlite3_finalize(stmt);
+ return CBM_STORE_ERR;
+ }
+ int n = 0;
+ while (n < limit && sqlite3_step(stmt) == SQLITE_ROW) {
+ rows[n].section_qn = heap_strdup((const char *)sqlite3_column_text(stmt, 0));
+ rows[n].target_qn = heap_strdup((const char *)sqlite3_column_text(stmt, SKIP_ONE));
+ rows[n].rank = sqlite3_column_int(stmt, ST_COL_2);
+ rows[n].score = sqlite3_column_double(stmt, ST_COL_3);
+ rows[n].p = sqlite3_column_double(stmt, ST_COL_4);
+ rows[n].evidence = heap_strdup((const char *)sqlite3_column_text(stmt, ST_COL_5));
+ n++;
+ }
+ sqlite3_finalize(stmt);
+ *out = rows;
+ *count = n;
+ return CBM_STORE_OK;
+}
+
+void cbm_store_doc_candidates_free(cbm_doc_candidate_t *rows, int count) {
+ for (int i = 0; rows && i < count; i++) {
+ free((void *)rows[i].section_qn);
+ free((void *)rows[i].target_qn);
+ free((void *)rows[i].evidence);
+ }
+ free(rows);
+}
+
void cbm_adr_sections_free(cbm_adr_sections_t *s) {
if (!s) {
return;
diff --git a/src/store/store.h b/src/store/store.h
index 8dc67ee48..881ba3dc9 100644
--- a/src/store/store.h
+++ b/src/store/store.h
@@ -1083,6 +1083,34 @@ int cbm_store_adr_update_sections(cbm_store_t *s, const char *project, const cha
const char **values, int count, cbm_adr_t *out);
void cbm_store_adr_free(cbm_adr_t *adr);
+/* ── Doc -> code candidates (for the agent to verify) ───────────── */
+
+/* One documentation section -> function candidate, with the calibrated
+ * probability that the section is about the function. Candidates, not links:
+ * never graph edges; a short ranked list per section for the agent to judge. */
+typedef struct {
+ const char *section_qn;
+ const char *target_qn;
+ int rank;
+ double score;
+ double p;
+ const char *evidence; /* JSON: {"terms":n,"name_tokens":m} */
+} cbm_doc_candidate_t;
+
+/* Replace the project's candidates. The table is created here, at publish
+ * (not in the schema): a database written by an older build has none, and
+ * readers treat that as "no candidates". */
+int cbm_store_doc_candidates_replace(cbm_store_t *s, const char *project,
+ const cbm_doc_candidate_t *rows, int count);
+
+/* The candidates of one section (section_qn) or for one function (target_qn;
+ * pass the other NULL), highest p first, then score, at most limit. *out is
+ * allocated (cbm_store_doc_candidates_free). CBM_STORE_NOT_FOUND: no table. */
+int cbm_store_doc_candidates_get(cbm_store_t *s, const char *project, const char *section_qn,
+ const char *target_qn, int limit, cbm_doc_candidate_t **out,
+ int *count);
+void cbm_store_doc_candidates_free(cbm_doc_candidate_t *rows, int count);
+
/* ADR section parsing/rendering (pure functions, no store needed) */
enum { PROPS_MAX = 16 };
diff --git a/tests/test_mcp.c b/tests/test_mcp.c
index 5d107a8b3..6ae9a0814 100644
--- a/tests/test_mcp.c
+++ b/tests/test_mcp.c
@@ -18508,6 +18508,89 @@ TEST(snippet_exact_qn) {
PASS();
}
+/* Doc -> code candidates on get_code_snippet: a Section lists the functions
+ * it is possibly about (highest p first, evidence included), a function the
+ * sections that possibly describe it; a row whose other end no longer exists
+ * is skipped, and a node without rows gets no candidate keys at all. */
+TEST(snippet_doc_candidates_both_directions) {
+ char tmp[256];
+ cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp));
+ ASSERT_NOT_NULL(srv);
+ cbm_store_t *st = cbm_mcp_server_store(srv);
+ cbm_node_t sec = {.project = "test-project",
+ .label = "Section",
+ .name = "Orders",
+ .qualified_name = "test-project.README.Orders",
+ .file_path = "README.md",
+ .start_line = 1,
+ .end_line = 4};
+ ASSERT_TRUE(cbm_store_upsert_node(st, &sec) > 0);
+ const cbm_doc_candidate_t rows[] = {
+ {.section_qn = "test-project.README.Orders",
+ .target_qn = "test-project.cmd.server.main.HandleRequest",
+ .rank = 2,
+ .score = 0.25,
+ .p = 0.36,
+ .evidence = "{\"shared_terms\":2,\"name_tokens\":0}"},
+ {.section_qn = "test-project.README.Orders",
+ .target_qn = "test-project.cmd.server.main.ProcessOrder",
+ .rank = 1,
+ .score = 0.45,
+ .p = 0.548,
+ .evidence = "{\"shared_terms\":5,\"name_tokens\":2}"},
+ {.section_qn = "test-project.README.Orders",
+ .target_qn = "test-project.gone.Missing",
+ .rank = 3,
+ .score = 0.31,
+ .p = 0.42,
+ .evidence = "{\"shared_terms\":1,\"name_tokens\":0}"},
+ };
+ ASSERT_EQ(cbm_store_doc_candidates_replace(st, "test-project", rows, 3), CBM_STORE_OK);
+
+ char *resp = call_snippet(srv, "{\"qualified_name\":\"test-project.README.Orders\","
+ "\"project\":\"test-project\"}");
+ ASSERT_NOT_NULL(resp);
+ yyjson_doc *doc = yyjson_read(resp, strlen(resp), 0);
+ ASSERT_NOT_NULL(doc);
+ yyjson_val *about = yyjson_obj_get(yyjson_doc_get_root(doc), "possibly_about");
+ ASSERT_EQ((int)yyjson_arr_size(about), 2);
+ yyjson_val *first = yyjson_arr_get(about, 0);
+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(first, "qualified_name")),
+ "test-project.cmd.server.main.ProcessOrder");
+ ASSERT_FLOAT_EQ(yyjson_get_num(yyjson_obj_get(first, "p")), 0.548, 1e-9);
+ ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(first, "shared_terms")), 5);
+ ASSERT_EQ((int)yyjson_get_int(yyjson_obj_get(first, "start_line")), 7);
+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(about, 1), "qualified_name")),
+ "test-project.cmd.server.main.HandleRequest");
+ ASSERT_NOT_NULL(yyjson_obj_get(yyjson_doc_get_root(doc), "candidates_note"));
+ yyjson_doc_free(doc);
+ free(resp);
+
+ resp = call_snippet(srv, "{\"qualified_name\":\"test-project.cmd.server.main.ProcessOrder\","
+ "\"project\":\"test-project\"}");
+ ASSERT_NOT_NULL(resp);
+ doc = yyjson_read(resp, strlen(resp), 0);
+ ASSERT_NOT_NULL(doc);
+ yyjson_val *in = yyjson_obj_get(yyjson_doc_get_root(doc), "possibly_described_in");
+ ASSERT_EQ((int)yyjson_arr_size(in), 1);
+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "section")),
+ "test-project.README.Orders");
+ ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_arr_get(in, 0), "file_path")), "README.md");
+ yyjson_doc_free(doc);
+ free(resp);
+
+ resp = call_snippet(srv, "{\"qualified_name\":\"test-project.cmd.server.Run\","
+ "\"project\":\"test-project\"}");
+ ASSERT_NOT_NULL(resp);
+ ASSERT_NULL(strstr(resp, "possibly_"));
+ ASSERT_NULL(strstr(resp, "candidates_note"));
+ free(resp);
+
+ cbm_mcp_server_free(srv);
+ cleanup_snippet_dir(tmp);
+ PASS();
+}
+
/* ── TestSnippet_QNSuffix ─────────────────────────────────────── */
TEST(snippet_qn_suffix) {
@@ -23952,6 +24035,7 @@ SUITE(mcp) {
/* Snippet resolution (port of snippet_test.go) */
RUN_TEST(snippet_exact_qn);
+ RUN_TEST(snippet_doc_candidates_both_directions);
RUN_TEST(snippet_qn_suffix);
RUN_TEST(snippet_unique_short_name);
RUN_TEST(snippet_c_macro_namespace_c1);
diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c
index 206e370a7..d724f0213 100644
--- a/tests/test_pipeline.c
+++ b/tests/test_pipeline.c
@@ -18948,6 +18948,117 @@ TEST(pipeline_semantic_edges_carry_p) {
PASS();
}
+/* Count a section's stored doc -> code candidates (section QN containing
+ * heading) and check each row's shape: rank within the top
+ * CBM_SEM_DOC_TOP_K, score at or above the floor, p the doc curve's band at
+ * two decimals, never a test function. Returns -1 on a bad row. */
+static int doc_candidate_rows_for(cbm_store_t *s, const char *heading, char *top, size_t top_sz) {
+ sqlite3_stmt *st = NULL;
+ if (sqlite3_prepare_v2(cbm_store_get_db(s),
+ "SELECT target_qn, rank, score, p FROM doc_link_candidates "
+ "WHERE instr(section_qn, ?1) > 0 ORDER BY rank",
+ -1, &st, NULL) != SQLITE_OK) {
+ return -1;
+ }
+ sqlite3_bind_text(st, 1, heading, -1, SQLITE_TRANSIENT);
+ int n = 0;
+ bool ok = true;
+ while (ok && sqlite3_step(st) == SQLITE_ROW) {
+ const char *target = (const char *)sqlite3_column_text(st, 0);
+ int rank = sqlite3_column_int(st, 1);
+ double score = sqlite3_column_double(st, 2);
+ double p = sqlite3_column_double(st, 3);
+ double want = (double)(int)(cbm_sem_doc_calibrated_p((float)score) * 100.0F + 0.5F) / 100.0;
+ ok = target && rank >= 1 && rank <= CBM_SEM_DOC_TOP_K &&
+ score >= (double)CBM_SEM_DOC_MIN_SCORE && p == want && !strstr(target, ".Test");
+ if (ok && n == 0 && top) {
+ snprintf(top, top_sz, "%s", target);
+ }
+ n++;
+ }
+ sqlite3_finalize(st);
+ return ok ? n : -1;
+}
+
+/* Doc -> code candidates end to end: a full index publishes a section's
+ * calibrated candidates (the best match first, a test function never, a
+ * changelog section none), and a delta-route reindex keeps them -- its proxy
+ * buffer cannot recompute them, so the clone's rows must survive. */
+TEST(pipeline_doc_candidates_published_and_kept_by_delta) {
+ char tmp[256];
+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_doccand_XXXXXX");
+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
+ static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"};
+ for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) {
+ char file[64];
+ snprintf(file, sizeof(file), "%s_config.go", subjects[i]);
+ write_sem_family(tmp, file, subjects[i]);
+ }
+ /* A test whose comment is the section's own text: the best match of all,
+ * and still never a candidate. */
+ write_temp_file(tmp, "mail_config_test.go",
+ "package main\n\n"
+ "// TestParseMailConfig: the mail config file is parsed by ParseMailConfig,\n"
+ "// which loads the mail config from disk and validates the mail config\n"
+ "// fields before anything reads them. Missing fields fail the validation,\n"
+ "// and the parsed mail config can be written back to disk afterwards.\n"
+ "func TestParseMailConfig(t *testing.T) {\n"
+ "\tParseMailConfig(\"mail.conf\")\n}\n");
+ write_temp_file(tmp, "README.md",
+ "# Mail configuration\n\n"
+ "The mail config file is parsed by ParseMailConfig, which loads the mail "
+ "config from disk and validates the mail config fields before anything "
+ "reads them. Missing fields fail the validation, and the parsed mail "
+ "config can be written back to disk afterwards.\n\n"
+ "## Changelog\n\n"
+ "The mail config loader now validates missing fields and writes the mail "
+ "config back to disk after parsing, so a parsed mail config file always "
+ "carries every field it needs.\n");
+ char db[512];
+ snprintf(db, sizeof(db), "%s/doccand.db", tmp);
+
+ cbm_pipeline_incremental_test_reset_faults();
+ cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL);
+ ASSERT_NOT_NULL(p);
+ ASSERT_EQ(cbm_pipeline_run(p), 0);
+ cbm_pipeline_free(p);
+
+ cbm_store_t *s = cbm_store_open_path(db);
+ ASSERT_NOT_NULL(s);
+ char top[512] = "";
+ int mail = doc_candidate_rows_for(s, "Mail-configuration", top, sizeof(top));
+ int changelog = doc_candidate_rows_for(s, "Changelog", NULL, 0);
+ cbm_store_close(s);
+ ASSERT_GT(mail, 0);
+ ASSERT_NOT_NULL(strstr(top, "Mail"));
+ ASSERT_EQ(changelog, 0);
+
+ /* Delta route: an edit that keeps every definition name (closure repair
+ * declines added names); the candidates are kept as they were. */
+ char edited[512];
+ snprintf(edited, sizeof(edited), "%s/user_config.go", tmp);
+ FILE *ef = cbm_fopen(edited, "a");
+ ASSERT_NOT_NULL(ef);
+ (void)fputs("\n// edited after the first index\n", ef);
+ (void)fclose(ef);
+ cbm_pipeline_incremental_test_reset_faults();
+ p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL);
+ ASSERT_NOT_NULL(p);
+ ASSERT_EQ(cbm_pipeline_run(p), 0);
+ ASSERT_EQ(cbm_pipeline_incremental_test_last_route(), CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
+ cbm_pipeline_free(p);
+ s = cbm_store_open_path(db);
+ ASSERT_NOT_NULL(s);
+ char top_after[512] = "";
+ int mail_after = doc_candidate_rows_for(s, "Mail-configuration", top_after, sizeof(top_after));
+ cbm_store_close(s);
+ ASSERT_EQ(mail_after, mail);
+ ASSERT_STR_EQ(top_after, top);
+ th_rmtree(tmp);
+ cbm_pipeline_incremental_test_reset_faults();
+ PASS();
+}
+
TEST(pipeline_semantic_edges_no_functions) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX");
@@ -19862,6 +19973,7 @@ SUITE(pipeline) {
RUN_TEST(pipeline_semantic_batched_matches_unbatched);
RUN_TEST(pipeline_semantic_pair_signals_never_change_the_graph);
RUN_TEST(pipeline_semantic_edges_carry_p);
+ RUN_TEST(pipeline_doc_candidates_published_and_kept_by_delta);
RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153);
RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153);
RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153);
diff --git a/tests/test_semantic.c b/tests/test_semantic.c
index 98dd0b0ec..006cad45c 100644
--- a/tests/test_semantic.c
+++ b/tests/test_semantic.c
@@ -513,6 +513,25 @@ TEST(sem_calibrated_p_bands) {
PASS();
}
+/* Doc -> code candidates: the judged bands (sample 4), monotone, with nothing
+ * stored below CBM_SEM_DOC_MIN_SCORE (that band judged 0.06). */
+TEST(sem_doc_calibrated_p_bands) {
+ ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.10F), 0.06F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.199F), 0.06F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(CBM_SEM_DOC_MIN_SCORE), 0.36F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.299F), 0.36F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.30F), 0.42F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(0.40F), 0.548F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_doc_calibrated_p(1.00F), 0.548F, 0.0);
+ float last = 0.0F;
+ for (int milli = 0; milli <= 1000; milli++) {
+ float p = cbm_sem_doc_calibrated_p((float)milli / 1000.0F);
+ ASSERT_TRUE(p >= last);
+ last = p;
+ }
+ PASS();
+}
+
/* A properties string holding a quote is read whole, and escapes are not
* glued to the next word (until 2026-10 the read stopped at the first quote
* and kept "\n" raw, so "\nreturn" became the token "nreturn"). */
@@ -711,6 +730,7 @@ SUITE(semantic) {
RUN_TEST(sem_combine_weighs_signals);
RUN_TEST(sem_admit_best_first_keeps_the_best);
RUN_TEST(sem_calibrated_p_bands);
+ RUN_TEST(sem_doc_calibrated_p_bands);
RUN_TEST(sem_json_str_honours_escapes);
RUN_TEST(sem_corpus_add_null_doc);
RUN_TEST(sem_corpus_free_null);
diff --git a/tests/test_store_arch.c b/tests/test_store_arch.c
index 4e9f59068..6c6037770 100644
--- a/tests/test_store_arch.c
+++ b/tests/test_store_arch.c
@@ -621,6 +621,88 @@ TEST(adr_delete_not_found) {
PASS();
}
+/* Doc -> code candidates: a database without the table reads as "none";
+ * a replace swaps exactly one project's rows; reads go highest p first (then
+ * score) in both directions. */
+TEST(doc_candidates_replace_and_get) {
+ cbm_store_t *s = cbm_store_open_memory();
+ ASSERT_NOT_NULL(s);
+ cbm_doc_candidate_t *got = NULL;
+ int n = -1;
+ ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secA", NULL, 5, &got, &n),
+ CBM_STORE_NOT_FOUND);
+
+ const cbm_doc_candidate_t rows[] = {
+ {.section_qn = "p.secA",
+ .target_qn = "p.f2",
+ .rank = 2,
+ .score = 0.25,
+ .p = 0.36,
+ .evidence = "{\"shared_terms\":3,\"name_tokens\":0}"},
+ {.section_qn = "p.secA",
+ .target_qn = "p.f1",
+ .rank = 1,
+ .score = 0.45,
+ .p = 0.548,
+ .evidence = "{\"shared_terms\":6,\"name_tokens\":1}"},
+ {.section_qn = "p.secB",
+ .target_qn = "p.f1",
+ .rank = 1,
+ .score = 0.32,
+ .p = 0.42,
+ .evidence = ""},
+ };
+ ASSERT_EQ(cbm_store_doc_candidates_replace(s, "p", rows, 3), CBM_STORE_OK);
+ const cbm_doc_candidate_t other = {.section_qn = "q.secA",
+ .target_qn = "q.f9",
+ .rank = 1,
+ .score = 0.5,
+ .p = 0.548,
+ .evidence = ""};
+ ASSERT_EQ(cbm_store_doc_candidates_replace(s, "q", &other, 1), CBM_STORE_OK);
+
+ ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secA", NULL, 5, &got, &n), CBM_STORE_OK);
+ ASSERT_EQ(n, 2);
+ ASSERT_STR_EQ(got[0].target_qn, "p.f1");
+ ASSERT_EQ(got[0].rank, 1);
+ ASSERT_STR_EQ(got[0].evidence, "{\"shared_terms\":6,\"name_tokens\":1}");
+ ASSERT_STR_EQ(got[1].target_qn, "p.f2");
+ cbm_store_doc_candidates_free(got, n);
+
+ ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", NULL, "p.f1", 5, &got, &n), CBM_STORE_OK);
+ ASSERT_EQ(n, 2);
+ ASSERT_STR_EQ(got[0].section_qn, "p.secA");
+ ASSERT_STR_EQ(got[1].section_qn, "p.secB");
+ cbm_store_doc_candidates_free(got, n);
+
+ ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", NULL, "p.f1", 1, &got, &n), CBM_STORE_OK);
+ ASSERT_EQ(n, 1);
+ cbm_store_doc_candidates_free(got, n);
+
+ /* a replace drops every earlier row of the project, and only of it */
+ ASSERT_EQ(cbm_store_doc_candidates_replace(s, "p", &rows[2], 1), CBM_STORE_OK);
+ ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secA", NULL, 5, &got, &n), CBM_STORE_OK);
+ ASSERT_EQ(n, 0);
+ cbm_store_doc_candidates_free(got, n);
+ ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secB", NULL, 5, &got, &n), CBM_STORE_OK);
+ ASSERT_EQ(n, 1);
+ cbm_store_doc_candidates_free(got, n);
+ ASSERT_EQ(cbm_store_doc_candidates_get(s, "q", "q.secA", NULL, 5, &got, &n), CBM_STORE_OK);
+ ASSERT_EQ(n, 1);
+ cbm_store_doc_candidates_free(got, n);
+
+ ASSERT_EQ(cbm_store_doc_candidates_replace(s, "p", NULL, 0), CBM_STORE_OK);
+ ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", NULL, "p.f1", 5, &got, &n), CBM_STORE_OK);
+ ASSERT_EQ(n, 0);
+ cbm_store_doc_candidates_free(got, n);
+
+ /* exactly one of section / target */
+ ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", NULL, NULL, 5, &got, &n), CBM_STORE_ERR);
+ ASSERT_EQ(cbm_store_doc_candidates_get(s, "p", "p.secA", "p.f1", 5, &got, &n), CBM_STORE_ERR);
+ cbm_store_close(s);
+ PASS();
+}
+
TEST(adr_parse_sections_basic) {
cbm_adr_sections_t sec = cbm_adr_parse_sections("## PURPOSE\nFoo\n\n## STACK\nBar");
ASSERT_EQ(sec.count, 2);
@@ -1695,6 +1777,7 @@ SUITE(store_arch) {
RUN_TEST(adr_upsert);
RUN_TEST(adr_delete);
RUN_TEST(adr_delete_not_found);
+ RUN_TEST(doc_candidates_replace_and_get);
RUN_TEST(adr_parse_sections_basic);
RUN_TEST(adr_parse_sections_all_six);
RUN_TEST(adr_parse_sections_non_canonical);