Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

feat(semantic): best-first admission; each edge carries a calibrated p

Martin Vogelad25b0b64407
diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c
index a0bcb636d..17e4f55ad 100644
--- a/src/pipeline/pass_semantic_edges.c
+++ b/src/pipeline/pass_semantic_edges.c
@@ -1203,8 +1203,29 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf
}
}
+ /* Admission: best-first by score under the per-function budget
+ * (cbm_sem_admit_best_first); the dumps and the edge inserts below keep
+ * the canonical order. */
+ float *scores = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(float));
+ int *fa = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(int));
+ int *fb = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(int));
+ bool *eligible = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(bool));
+ bool *admitted = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(bool));
+ bool ranked = scores && fa && fb && eligible && admitted;
+ for (int e = 0; ranked && e < n; e++) {
+ scores[e] = pairs[e].score;
+ fa[e] = pairs[e].i;
+ fb[e] = pairs[e].j;
+ eligible[e] = !pairs[e].below;
+ }
+ ranked = ranked && cbm_sem_admit_best_first(scores, fa, fb, eligible, n, max_edges, edge_counts,
+ admitted);
+ if (!ranked) {
+ cbm_log_warn("semantic_edges.admission", "reason", "out_of_memory", "pairs", itoa_log(n));
+ }
+
int total_edges = 0;
- for (int e = 0; e < n; e++) {
+ for (int e = 0; ranked && e < n; e++) {
deferred_edge_t *de = &pairs[e];
if (de->below) {
if (signals) {
@@ -1215,21 +1236,27 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf
if (pair_dump) {
fprintf(pair_dump, "%d %d %.9g\n", de->i, de->j, (double)de->score);
}
- bool admitted = edge_counts[de->i] < max_edges && edge_counts[de->j] < max_edges;
if (signals) {
- pair_signals_line(signals, gbuf, funcs, de, admitted);
+ pair_signals_line(signals, gbuf, funcs, de, admitted[e]);
}
- if (!admitted) {
+ if (!admitted[e]) {
continue;
}
+ /* p from the score as stored, so one shown score carries one p. */
+ char score_text[CBM_SZ_32];
+ snprintf(score_text, sizeof(score_text), "%.3f", (double)de->score);
char props[PROPS_BUF];
- snprintf(props, sizeof(props), "{\"score\":%.3f,\"same_file\":%s}", de->score,
- de->same_file ? "true" : "false");
+ snprintf(props, sizeof(props), "{\"score\":%s,\"same_file\":%s,\"p\":%.2f}", score_text,
+ de->same_file ? "true" : "false",
+ (double)cbm_sem_calibrated_p(strtof(score_text, NULL)));
cbm_gbuf_insert_edge(gbuf, de->source_id, de->target_id, "SEMANTICALLY_RELATED", props);
- edge_counts[de->i]++;
- edge_counts[de->j]++;
total_edges++;
}
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, scores);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, fa);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, fb);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, eligible);
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, admitted);
if (pair_dump) {
(void)fclose(pair_dump);
}
diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c
index c28615d93..d894d941f 100644
--- a/src/semantic/semantic.c
+++ b/src/semantic/semantic.c
@@ -1750,6 +1750,71 @@ float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg) {
return score;
}
+typedef struct {
+ float score;
+ int pos;
+} sem_rank_t;
+
+static int cmp_rank_best_first(const void *pa, const void *pb) {
+ const sem_rank_t *a = pa;
+ const sem_rank_t *b = pb;
+ if (a->score != b->score) {
+ return a->score > b->score ? -1 : 1;
+ }
+ return (a->pos > b->pos) - (a->pos < b->pos);
+}
+
+/* Best-first, not first-come: when a function has more candidate partners
+ * than its budget, it keeps its best ones. Judged on held-out repositories
+ * (2026-10-07): the pairs first-come admitted over budget were 0 of 17
+ * related, the ones best-first admits instead 7 of 20. */
+bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb,
+ const bool *eligible, int n, int max_edges, int *counts,
+ bool *admitted) {
+ if (n <= 0) {
+ return true;
+ }
+ memset(admitted, 0, (size_t)n * sizeof(admitted[0]));
+ sem_rank_t *order = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(sem_rank_t));
+ if (!order) {
+ return false;
+ }
+ int m = 0;
+ for (int k = 0; k < n; k++) {
+ if (eligible[k]) {
+ order[m++] = (sem_rank_t){.score = scores[k], .pos = k};
+ }
+ }
+ qsort(order, (size_t)m, sizeof(order[0]), cmp_rank_best_first);
+ for (int r = 0; r < m; r++) {
+ int k = order[r].pos;
+ if (counts[fa[k]] < max_edges && counts[fb[k]] < max_edges) {
+ counts[fa[k]]++;
+ counts[fb[k]]++;
+ admitted[k] = true;
+ }
+ }
+ cbm_free(CBM_MEM_CLASS_SEMANTIC, order);
+ return true;
+}
+
+/* Score bands at the quartiles of the admitted scores, each band's p the
+ * judged share of related pairs (2026-10-07: 120 blind-judged admitted pairs
+ * from held-out repositories in six languages, three reading rounds; the
+ * curve is monotone as measured). Ships as the edge property "p". The cuts
+ * (measured 0.777752, 0.805782, 0.851543) are at the stored score's three
+ * decimals, so one shown score always carries one p. */
+static const float SEM_P_CUTS[] = {0.778F, 0.806F, 0.852F};
+static const float SEM_P_BANDS[] = {0.556F, 0.640F, 0.731F, 0.857F};
+
+float cbm_sem_calibrated_p(float score) {
+ size_t band = 0;
+ while (band < sizeof(SEM_P_CUTS) / sizeof(SEM_P_CUTS[0]) && score >= SEM_P_CUTS[band]) {
+ band++;
+ }
+ return SEM_P_BANDS[band];
+}
+
float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
const cbm_sem_config_t *cfg) {
if (!a || !b || !cfg) {
diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h
index 34d738796..34f49c0c5 100644
--- a/src/semantic/semantic.h
+++ b/src/semantic/semantic.h
@@ -243,6 +243,21 @@ void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
/* cbm_sem_combined_score from the signal values (the same arithmetic). */
float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg);
+/* Best-first admission under a per-function edge budget. Pair k joins
+ * functions fa[k] and fb[k] with scores[k]; pairs are taken by descending
+ * score, ties in ascending k (the caller's canonical order), and pair k is
+ * admitted when eligible[k] and both functions have fewer than max_edges
+ * admitted pairs. counts holds one zeroed slot per function and is updated.
+ * Returns false (nothing admitted) when the order cannot be allocated. */
+bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb,
+ const bool *eligible, int n, int max_edges, int *counts,
+ bool *admitted);
+
+/* The probability that an admitted SEMANTICALLY_RELATED pair is related, by
+ * its score as stored (three decimals): the judged precision of blind-judged
+ * held-out samples (the edge property "p"). */
+float cbm_sem_calibrated_p(float score);
+
/* Module proximity multiplier based on file paths. */
float cbm_sem_proximity(const char *path_a, const char *path_b);
diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c
index ed7cb7099..206e370a7 100644
--- a/tests/test_pipeline.c
+++ b/tests/test_pipeline.c
@@ -14,6 +14,7 @@
#include "pipeline/pipeline_internal.h"
#include "pipeline/artifact.h"
#include "store/store.h"
+#include "semantic/semantic.h"
#include "git/git_context.h"
#include "foundation/dump_verify.h"
#include "foundation/sha256.h"
@@ -18896,6 +18897,57 @@ TEST(pipeline_semantic_pair_signals_never_change_the_graph) {
PASS();
}
+/* Every SEMANTICALLY_RELATED edge carries "p": the calibrated probability for
+ * its score AS STORED (three decimals), so one shown score has one p. */
+TEST(pipeline_semantic_edges_carry_p) {
+ char tmp[256];
+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_semp_XXXXXX");
+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
+ static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"};
+ for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) {
+ char file[64];
+ snprintf(file, sizeof(file), "%s_config.go", subjects[i]);
+ write_sem_family(tmp, file, subjects[i]);
+ }
+ char db[512];
+ snprintf(db, sizeof(db), "%s/p.db", tmp);
+ cbm_setenv("CBM_SEMANTIC_THRESHOLD", "0.3", 1); /* as above: admits the near-duplicates */
+ cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL);
+ ASSERT_NOT_NULL(p);
+ int rc = cbm_pipeline_run(p);
+ cbm_unsetenv("CBM_SEMANTIC_THRESHOLD");
+ cbm_pipeline_free(p);
+ ASSERT_EQ(rc, 0);
+
+ cbm_store_t *s = cbm_store_open_path(db);
+ ASSERT_NOT_NULL(s);
+ sqlite3_stmt *st = NULL;
+ ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s),
+ "SELECT properties FROM edges WHERE type = 'SEMANTICALLY_RELATED'",
+ -1, &st, NULL),
+ SQLITE_OK);
+ int edges = 0;
+ while (sqlite3_step(st) == SQLITE_ROW) {
+ const char *props = (const char *)sqlite3_column_text(st, 0);
+ ASSERT_NOT_NULL(props);
+ yyjson_doc *doc = yyjson_read(props, strlen(props), 0);
+ ASSERT_NOT_NULL(doc);
+ yyjson_val *score = yyjson_obj_get(yyjson_doc_get_root(doc), "score");
+ yyjson_val *pv = yyjson_obj_get(yyjson_doc_get_root(doc), "p");
+ ASSERT_TRUE(score && yyjson_is_num(score));
+ ASSERT_TRUE(pv && yyjson_is_num(pv));
+ double want = cbm_sem_calibrated_p((float)yyjson_get_num(score));
+ ASSERT_FLOAT_EQ(yyjson_get_num(pv), want, 0.006);
+ yyjson_doc_free(doc);
+ edges++;
+ }
+ sqlite3_finalize(st);
+ cbm_store_close(s);
+ ASSERT_GT(edges, 0);
+ th_rmtree(tmp);
+ PASS();
+}
+
TEST(pipeline_semantic_edges_no_functions) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX");
@@ -19809,6 +19861,7 @@ SUITE(pipeline) {
RUN_TEST(pipeline_semantic_edges_no_functions);
RUN_TEST(pipeline_semantic_batched_matches_unbatched);
RUN_TEST(pipeline_semantic_pair_signals_never_change_the_graph);
+ RUN_TEST(pipeline_semantic_edges_carry_p);
RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153);
RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153);
RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153);
diff --git a/tests/test_semantic.c b/tests/test_semantic.c
index 87bc4dcec..8ae161ed6 100644
--- a/tests/test_semantic.c
+++ b/tests/test_semantic.c
@@ -460,6 +460,61 @@ TEST(sem_combine_weighs_signals) {
PASS();
}
+/* Over budget, a function keeps its BEST partners, not its first ones (a
+ * held-out judged sample: the pairs first-come admitted over budget were 0 of
+ * 17 related). Equal scores keep the canonical order; an ineligible pair is
+ * never admitted and takes no budget. */
+TEST(sem_admit_best_first_keeps_the_best) {
+ /* function 0 has four partners in canonical order, budget 2 */
+ const float scores[] = {0.80F, 0.90F, 0.95F, 0.99F, 0.90F};
+ const int fa[] = {0, 0, 0, 0, 5};
+ const int fb[] = {1, 2, 3, 4, 6};
+ const bool eligible[] = {true, true, true, false, true};
+ int counts[7] = {0};
+ bool admitted[5];
+ ASSERT_TRUE(cbm_sem_admit_best_first(scores, fa, fb, eligible, 5, 2, counts, admitted));
+ ASSERT_FALSE(admitted[0]); /* first-come would have kept 0.80 */
+ ASSERT_TRUE(admitted[1]);
+ ASSERT_TRUE(admitted[2]);
+ ASSERT_FALSE(admitted[3]); /* the best score, but recorded only */
+ ASSERT_TRUE(admitted[4]);
+ ASSERT_EQ(counts[0], 2);
+ ASSERT_EQ(counts[4], 0);
+
+ /* ties: the earlier pair in canonical order wins the last slot */
+ const float tied[] = {0.85F, 0.85F};
+ const int ta[] = {0, 0};
+ const int tb[] = {1, 2};
+ const bool all[] = {true, true};
+ int tcounts[3] = {0};
+ bool tadmitted[2];
+ ASSERT_TRUE(cbm_sem_admit_best_first(tied, ta, tb, all, 2, 1, tcounts, tadmitted));
+ ASSERT_TRUE(tadmitted[0]);
+ ASSERT_FALSE(tadmitted[1]);
+ PASS();
+}
+
+/* p by the stored (three-decimal) score: the judged bands, monotone. */
+TEST(sem_calibrated_p_bands) {
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.750F), 0.556F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.777F), 0.556F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.778F), 0.640F, 0.0);
+ /* the cut sits at the stored precision (0.778), not the raw quartile
+ * 0.777752: a stored "0.778" and a stored "0.777" never share a band */
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.7779F), 0.556F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.805F), 0.640F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.806F), 0.731F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.852F), 0.857F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(1.000F), 0.857F, 0.0);
+ float last = 0.0F;
+ for (int milli = 750; milli <= 1000; milli++) {
+ float p = cbm_sem_calibrated_p((float)milli / 1000.0F);
+ ASSERT_TRUE(p >= last);
+ last = p;
+ }
+ PASS();
+}
+
TEST(sem_corpus_add_null_doc) {
cbm_sem_corpus_t *c = cbm_sem_corpus_new();
ASSERT_NOT_NULL(c);
@@ -638,6 +693,8 @@ SUITE(semantic) {
RUN_TEST(sem_corpus_index_accessors_match_name_lookups);
RUN_TEST(sem_tfidf_terms_compare_vocabulary_not_positions);
RUN_TEST(sem_combine_weighs_signals);
+ RUN_TEST(sem_admit_best_first_keeps_the_best);
+ RUN_TEST(sem_calibrated_p_bands);
RUN_TEST(sem_corpus_add_null_doc);
RUN_TEST(sem_corpus_free_null);
RUN_TEST(sem_get_config_defaults);