Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(semantic): TF-IDF compares vocabulary, not token positions

Martin Vogel968d4a4be9f6
diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c
index 14597289e..a0bcb636d 100644
--- a/src/pipeline/pass_semantic_edges.c
+++ b/src/pipeline/pass_semantic_edges.c
@@ -71,6 +71,7 @@ typedef struct {
int64_t target_id;
float score;
bool same_file;
+ bool below; /* under the threshold: recorded for CBM_SEM_PAIR_SIGNALS only, never admitted */
/* Canonical admission keys (determinism): func index of the discovering
* side, its candidate rank, and the partner's func index. The sequential
* admission pass replays pairs in (i, c) order so which pairs win the
@@ -93,7 +94,7 @@ static void deferred_buf_init(deferred_edge_buf_t *buf) {
}
static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt, float score,
- bool same_file, int i, int j, int c) {
+ bool same_file, bool below, int i, int j, int c) {
if (buf->count >= buf->cap) {
int nc = buf->cap < CBM_SZ_256 ? CBM_SZ_256 : buf->cap * GROW;
deferred_edge_t *grown =
@@ -108,6 +109,7 @@ static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt
.target_id = tgt,
.score = score,
.same_file = same_file,
+ .below = below,
.i = i,
.j = j,
.c = c};
@@ -724,19 +726,10 @@ static void vec_build_worker(int worker_id, void *ctx_ptr) {
token_index[t] = cbm_sem_corpus_token_index(vc->corpus, tokens[t]);
}
- /* TF-IDF weights */
+ /* TF-IDF terms, keyed by corpus token (cbm_sem_tfidf_terms). */
int *indices = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(int));
float *weights = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(float));
- int tfidf_len = 0;
- for (int t = 0; t < tc; t++) {
- float idf = token_index ? cbm_sem_corpus_idf_at(vc->corpus, token_index[t])
- : cbm_sem_corpus_idf(vc->corpus, tokens[t]);
- if (idf > 0.0F) {
- indices[tfidf_len] = t;
- weights[tfidf_len] = idf;
- tfidf_len++;
- }
- }
+ int tfidf_len = cbm_sem_tfidf_terms(vc->corpus, tokens, token_index, tc, indices, weights);
vc->funcs[f].tfidf_indices = indices;
vc->funcs[f].tfidf_weights = weights;
vc->funcs[f].tfidf_len = tfidf_len;
@@ -833,6 +826,7 @@ typedef struct {
uint64_t *signatures;
int *edge_counts; /* budget applied sequentially in phase6b (determinism) */
cbm_sem_config_t cfg;
+ float record_floor; /* kept from here up; under the threshold: signals dump only */
int func_count;
/* LSH buckets (read-only during scoring) */
@@ -927,13 +921,13 @@ static void score_try_emit(score_ctx_t *sc, int i, int j, int c, deferred_edge_b
return;
}
float score = cbm_sem_combined_score(&sc->funcs[i], &sc->funcs[j], &sc->cfg);
- if (score < sc->cfg.threshold) {
+ if (score < sc->record_floor) {
return;
}
bool same_file = sc->funcs[i].file_path && sc->funcs[j].file_path &&
strcmp(sc->funcs[i].file_path, sc->funcs[j].file_path) == 0;
- deferred_buf_push(my_buf, sc->funcs[i].node_id, sc->funcs[j].node_id, score, same_file, i, j,
- c);
+ deferred_buf_push(my_buf, sc->funcs[i].node_id, sc->funcs[j].node_id, score, same_file,
+ score < sc->cfg.threshold, i, j, c);
}
static void score_worker(int worker_id, void *ctx_ptr) {
@@ -1138,8 +1132,31 @@ static int cmp_deferred_edge_canonical(const void *pa, const void *pb) {
* and apply the per-node max_edges budget HERE — single-threaded — so the
* admitted edge set is a pure function of the (canonically sorted) inputs,
* independent of worker count and scheduling. */
+/* CBM_SEM_PAIR_SIGNALS=<path>: one line per recorded pair (from the record
+ * floor up, CBM_SEM_PAIR_SIGNALS_FLOOR, default the threshold), in canonical
+ * order: both qualified names, the score, every signal value and whether the
+ * pair was admitted. The measuring harness for the semantic engine: judged
+ * samples are drawn from it, stratified by score. Off unless set. */
+static void pair_signals_line(FILE *f, const cbm_gbuf_t *gbuf, const cbm_sem_func_t *funcs,
+ const deferred_edge_t *de, bool admitted) {
+ cbm_sem_signals_t s;
+ cbm_sem_signal_values(&funcs[de->i], &funcs[de->j], &s);
+ const cbm_gbuf_node_t *a = cbm_gbuf_find_by_id(gbuf, de->source_id);
+ const cbm_gbuf_node_t *b = cbm_gbuf_find_by_id(gbuf, de->target_id);
+ (void)fprintf(
+ f,
+ "%s\t%s\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.4f\t%d\t%d"
+ "\t%s:%d-%d\t%s:%d-%d\n",
+ a && a->qualified_name ? a->qualified_name : "?",
+ b && b->qualified_name ? b->qualified_name : "?", (double)de->score, (double)s.tfidf,
+ (double)s.ri, (double)s.minhash, (double)s.api, (double)s.type, (double)s.decorator,
+ (double)s.struct_profile, (double)s.proximity, de->same_file ? 1 : 0, admitted ? 1 : 0,
+ a && a->file_path ? a->file_path : "?", a ? a->start_line : 0, a ? a->end_line : 0,
+ b && b->file_path ? b->file_path : "?", b ? b->start_line : 0, b ? b->end_line : 0);
+}
+
static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_bufs, int worker_count,
- int *edge_counts, int max_edges) {
+ int *edge_counts, int max_edges, const cbm_sem_func_t *funcs) {
int total_pairs = 0;
for (int w = 0; w < worker_count; w++) {
total_pairs += worker_bufs[w].count;
@@ -1175,20 +1192,34 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf
* chase the near-threshold SEMANTICALLY_RELATED flicker: find the pair that
* flips, then compare its score between runs. */
FILE *pair_dump = NULL;
+ FILE *signals = NULL;
{
char dump_path[CBM_SZ_1K];
if (cbm_safe_getenv("CBM_SEM_PAIR_DUMP", dump_path, sizeof(dump_path), NULL)) {
pair_dump = cbm_fopen(dump_path, "w");
}
+ if (funcs && cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS", dump_path, sizeof(dump_path), NULL)) {
+ signals = cbm_fopen(dump_path, "w");
+ }
}
int total_edges = 0;
for (int e = 0; e < n; e++) {
deferred_edge_t *de = &pairs[e];
+ if (de->below) {
+ if (signals) {
+ pair_signals_line(signals, gbuf, funcs, de, false);
+ }
+ continue;
+ }
if (pair_dump) {
fprintf(pair_dump, "%d %d %.9g\n", de->i, de->j, (double)de->score);
}
- if (edge_counts[de->i] >= max_edges || edge_counts[de->j] >= max_edges) {
+ bool admitted = edge_counts[de->i] < max_edges && edge_counts[de->j] < max_edges;
+ if (signals) {
+ pair_signals_line(signals, gbuf, funcs, de, admitted);
+ }
+ if (!admitted) {
continue;
}
char props[PROPS_BUF];
@@ -1202,6 +1233,9 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf
if (pair_dump) {
(void)fclose(pair_dump);
}
+ if (signals) {
+ (void)fclose(signals);
+ }
cbm_free(CBM_MEM_CLASS_SEMANTIC, pairs);
return total_edges;
}
@@ -1542,13 +1576,14 @@ static void sem_corpus_dump(const cbm_sem_corpus_t *corpus, const char *suffix)
/* Phase 6a: score candidate pairs in parallel and collect deferred edges. */
static void phase6a_score_candidates(cbm_sem_func_t *funcs, uint64_t *signatures, int *edge_counts,
sem_bucket_t **band_buckets, cbm_sem_config_t cfg,
- deferred_edge_buf_t *worker_bufs, int func_count,
- int worker_count) {
+ float record_floor, deferred_edge_buf_t *worker_bufs,
+ int func_count, int worker_count) {
score_ctx_t sc = {
.funcs = funcs,
.signatures = signatures,
.edge_counts = edge_counts,
.cfg = cfg,
+ .record_floor = record_floor,
.func_count = func_count,
.band_buckets = (void *)band_buckets,
.worker_bufs = worker_bufs,
@@ -1619,14 +1654,30 @@ static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t *
deferred_buf_init(&worker_bufs[w]);
}
+ /* The measuring harness may record pairs below the threshold (never
+ * admitted): CBM_SEM_PAIR_SIGNALS_FLOOR, only with CBM_SEM_PAIR_SIGNALS. */
+ float record_floor = cfg.threshold;
+ {
+ char path[CBM_SZ_1K]; /* the dump path: a buffer too short reads as unset */
+ char buf[CBM_SZ_64];
+ if (cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS", path, sizeof(path), NULL) &&
+ cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS_FLOOR", buf, sizeof(buf), NULL)) {
+ float floor_value = strtof(buf, NULL);
+ if (floor_value > 0.0F && floor_value < cfg.threshold) {
+ record_floor = floor_value;
+ }
+ }
+ }
+
CBM_PROF_START(t_phase6a);
sem_state_dump(funcs, func_count);
- phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, worker_bufs,
- func_count, worker_count);
+ phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, record_floor,
+ worker_bufs, func_count, worker_count);
CBM_PROF_END_N("semantic_edges", "6a_score_parallel", t_phase6a, func_count);
CBM_PROF_START(t_phase6b);
- int total = phase6b_merge_edges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges);
+ int total =
+ phase6b_merge_edges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges, funcs);
CBM_PROF_END_N("semantic_edges", "6b_edge_merge_seq", t_phase6b, total);
cbm_free(CBM_MEM_CLASS_SEMANTIC, worker_bufs);
diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h
index 87b9c160b..b00dcd699 100644
--- a/src/pipeline/pipeline_internal.h
+++ b/src/pipeline/pipeline_internal.h
@@ -1274,8 +1274,13 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr
* sections and PDF pages hold MENTIONS edges into the code, ADR records
* are nodes, and AsciiDoc files, PDFs and a Sphinx project's extra
* source suffixes are indexed. An index built before has none of this
- * for its unchanged documents, so it rebuilds once on upgrade. */
-enum { CBM_SEMANTIC_INDEX_VERSION = 6 };
+ * for its unchanged documents, so it rebuilds once on upgrade.
+ * 7: SEMANTICALLY_RELATED changed: TF-IDF compares vocabulary, string
+ * properties are read with their escapes, admission is best-first and
+ * every edge carries "p"; doc_link_candidates holds doc section -> code
+ * candidates. An index built before has the old pairs and no
+ * candidates, so it rebuilds once on upgrade. */
+enum { CBM_SEMANTIC_INDEX_VERSION = 7 };
typedef struct {
cbm_gbuf_t *gbuf;
diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c
index 2aa1df5a3..c28615d93 100644
--- a/src/semantic/semantic.c
+++ b/src/semantic/semantic.c
@@ -1619,6 +1619,47 @@ static float small_cosine(const float *a, const float *b, int dims) {
return denom < CBM_SEM_DENOM_EPS ? 0.0F : dot / denom;
}
+static int cmp_int_asc(const void *pa, const void *pb) {
+ int a = *(const int *)pa;
+ int b = *(const int *)pb;
+ return (a > b) - (a < b);
+}
+
+/* Terms are keyed by corpus token, so two functions share a term only when
+ * they share the word. (Until 2026-10 the key was the token's POSITION in the
+ * function: any two functions of similar length "shared" their first terms,
+ * and the TF-IDF signal measured length, not vocabulary.) */
+int cbm_sem_tfidf_terms(const cbm_sem_corpus_t *corpus, char *const *tokens, const int *token_index,
+ int count, int *indices, float *weights) {
+ if (!corpus || !indices || !weights || count <= 0) {
+ return 0;
+ }
+ int known = 0;
+ for (int t = 0; t < count; t++) {
+ int idx = token_index ? token_index[t] : cbm_sem_corpus_token_index(corpus, tokens[t]);
+ if (idx >= 0 && cbm_sem_corpus_idf_at(corpus, idx) > 0.0F) {
+ indices[known++] = idx;
+ }
+ }
+ if (known > 1) {
+ qsort(indices, (size_t)known, sizeof(indices[0]), cmp_int_asc);
+ }
+ /* Each run of one index becomes one term; its weight is the idf added once
+ * per occurrence (equal indices have equal idf). */
+ int terms = 0;
+ for (int k = 0; k < known; k++) {
+ float idf = cbm_sem_corpus_idf_at(corpus, indices[k]);
+ if (terms > 0 && indices[terms - SKIP_ONE] == indices[k]) {
+ weights[terms - SKIP_ONE] += idf;
+ } else {
+ indices[terms] = indices[k];
+ weights[terms] = idf;
+ terms++;
+ }
+ }
+ return terms;
+}
+
/* Sparse cosine over two pre-sorted (index, weight) vectors. Returns 0 when
* either side is empty or the magnitude product is below the epsilon guard. */
static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t *b) {
@@ -1651,49 +1692,47 @@ static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t *
return denom > CBM_SEM_DENOM_EPS ? (dot / denom) : 0.0F;
}
-float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
- const cbm_sem_config_t *cfg) {
- if (!a || !b || !cfg) {
- return 0.0F;
+void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
+ cbm_sem_signals_t *out) {
+ memset(out, 0, sizeof(*out));
+ out->proximity = CBM_SEM_UNIT_POS;
+ if (!a || !b) {
+ return;
}
+ if (a->has_minhash && b->has_minhash) {
+ double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash,
+ (const cbm_minhash_t *)b->minhash);
+ out->minhash = (float)j;
+ out->near_copy = j >= CBM_MINHASH_JACCARD_THRESHOLD;
+ }
+ out->tfidf = sparse_tfidf_cosine(a, b);
+ out->ri = cbm_rsq_ip(&a->ri_code, &b->ri_code); /* signal 2: Random Indexing */
+ out->api = cbm_rsq_ip(&a->api_code, &b->api_code); /* signal 4: API signatures */
+ out->type = cbm_rsq_ip(&a->type_code, &b->type_code); /* signal 5: type signatures */
+ out->decorator = cbm_rsq_ip(&a->deco_code, &b->deco_code); /* signal 7: decorators */
+ /* signals 8+9+11: structural profile + data flow + Halstead */
+ out->struct_profile =
+ small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS);
+ out->proximity = cbm_sem_proximity(a->file_path, b->file_path); /* signal 6: multiplier */
+}
+float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg) {
/* Short-circuit: if MinHash Jaccard is already above the SIMILAR_TO threshold,
* the pass_similarity pipeline already emitted a SIMILAR_TO edge for this pair.
* Returning 0 here avoids flooding top-k with cross-service copy-paste boilerplate
* (logging_middleware, shared push/pull handlers) that SIMILAR_TO already covers,
* freeing the edge budget for true semantic leaps (vocabulary-bridged relations). */
- if (a->has_minhash && b->has_minhash) {
- double early_j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash,
- (const cbm_minhash_t *)b->minhash);
- if (early_j >= CBM_MINHASH_JACCARD_THRESHOLD) {
- return 0.0F;
- }
- }
-
- float score = cfg->w_tfidf * sparse_tfidf_cosine(a, b);
-
- /* Signal 2: Random Indexing */
- score += cfg->w_ri * cbm_rsq_ip(&a->ri_code, &b->ri_code);
-
- /* Signal 3: MinHash Jaccard */
- if (a->has_minhash && b->has_minhash) {
- double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash,
- (const cbm_minhash_t *)b->minhash);
- score += cfg->w_minhash * (float)j;
+ if (!s || !cfg || s->near_copy) {
+ return 0.0F;
}
-
- /* Signal 4: API Signatures */
- score += cfg->w_api * cbm_rsq_ip(&a->api_code, &b->api_code);
-
- /* Signal 5: Type Signatures */
- score += cfg->w_type * cbm_rsq_ip(&a->type_code, &b->type_code);
-
- /* Signal 7: Decorator Pattern */
- score += cfg->w_decorator * cbm_rsq_ip(&a->deco_code, &b->deco_code);
-
- /* Signal 8+9+11: Structural profile + data flow + Halstead */
- float sp_score = small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS);
- score += cfg->w_struct_profile * sp_score;
+ /* The terms in the historical order (float addition is not associative). */
+ float score = cfg->w_tfidf * s->tfidf;
+ score += cfg->w_ri * s->ri;
+ score += cfg->w_minhash * s->minhash;
+ score += cfg->w_api * s->api;
+ score += cfg->w_type * s->type;
+ score += cfg->w_decorator * s->decorator;
+ score += cfg->w_struct_profile * s->struct_profile;
/* Signal 6: Module proximity (multiplier, not additive).
* Proximity returns [1.0, 1.10] — a same-file/same-dir boost for ranking.
@@ -1701,17 +1740,26 @@ float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
* cosine-similarity range. Without the clamp, a 0.95 base × 1.10 proximity
* would emit a 1.045 score which violates the semantic of "similarity" and
* breaks downstream consumers that expect a normalized value. */
- score *= cbm_sem_proximity(a->file_path, b->file_path);
+ score *= s->proximity;
if (score > CBM_SEM_UNIT_POS) {
score = CBM_SEM_UNIT_POS;
}
if (score < 0.0F) {
score = 0.0F;
}
-
return score;
}
+float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
+ const cbm_sem_config_t *cfg) {
+ if (!a || !b || !cfg) {
+ return 0.0F;
+ }
+ cbm_sem_signals_t s;
+ cbm_sem_signal_values(a, b, &s);
+ return cbm_sem_combine(&s, cfg);
+}
+
/* ── Graph diffusion ─────────────────────────────────────────────── */
void cbm_sem_diffuse(cbm_sem_vec_t *combined, const cbm_sem_vec_t *neighbors, int neighbor_count,
diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h
index 1f99a54c0..34d738796 100644
--- a/src/semantic/semantic.h
+++ b/src/semantic/semantic.h
@@ -206,12 +206,43 @@ const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index,
/* Free corpus. */
void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus);
+/* The TF-IDF terms of one function: one entry per distinct corpus token whose
+ * idf is above 0, weighted term frequency x idf, in ascending token-index
+ * order (the order the TF-IDF cosine merges on). `token_index[t]` is the
+ * corpus index of tokens[t] (-1 when unknown); NULL looks each token up by
+ * name. `indices` and `weights` hold at least `count` entries. Returns the
+ * number of terms written. */
+int cbm_sem_tfidf_terms(const cbm_sem_corpus_t *corpus, char *const *tokens, const int *token_index,
+ int count, int *indices, float *weights);
+
/* ── Combined scoring ────────────────────────────────────────────── */
/* Compute combined similarity score between two functions. */
float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
const cbm_sem_config_t *cfg);
+/* The signal values cbm_sem_combined_score weighs, for measuring them one by
+ * one: each similarity in [0, 1] (minhash 0 when either side has none),
+ * proximity a multiplier in [1.0, 1.10]. `near_copy`: the pair is a SIMILAR_TO
+ * copy, which the combined score sets to 0. */
+typedef struct {
+ float tfidf;
+ float ri;
+ float minhash;
+ float api;
+ float type;
+ float decorator;
+ float struct_profile;
+ float proximity;
+ bool near_copy;
+} cbm_sem_signals_t;
+
+void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
+ cbm_sem_signals_t *out);
+
+/* cbm_sem_combined_score from the signal values (the same arithmetic). */
+float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg);
+
/* Module proximity multiplier based on file paths. */
float cbm_sem_proximity(const char *path_a, const char *path_b);
diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c
index a2861d054..ed7cb7099 100644
--- a/tests/test_pipeline.c
+++ b/tests/test_pipeline.c
@@ -18810,6 +18810,92 @@ TEST(pipeline_semantic_batched_matches_unbatched) {
PASS();
}
+/* CBM_SEM_PAIR_SIGNALS (+ _FLOOR) is the semantic engine's measuring
+ * harness: it records each scored pair with its signal values, from a floor
+ * below the threshold up. Recording must never change the graph: the same
+ * SEMANTICALLY_RELATED rows with and without it, one admitted line per edge,
+ * and every pair under the threshold recorded but never admitted. */
+TEST(pipeline_semantic_pair_signals_never_change_the_graph) {
+ char tmp[256];
+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_semsig_XXXXXX");
+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
+ static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"};
+ for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) {
+ char file[64];
+ snprintf(file, sizeof(file), "%s_config.go", subjects[i]);
+ write_sem_family(tmp, file, subjects[i]);
+ }
+ char db_plain[512];
+ char db_signals[512];
+ char tsv[512];
+ snprintf(db_plain, sizeof(db_plain), "%s/plain.db", tmp);
+ snprintf(db_signals, sizeof(db_signals), "%s/signals.db", tmp);
+ snprintf(tsv, sizeof(tsv), "%s/pairs.tsv", tmp);
+
+ cbm_setenv("CBM_SEMANTIC_THRESHOLD", "0.3", 1); /* as above: admits the near-duplicates */
+ cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_plain, CBM_MODE_FULL);
+ ASSERT_NOT_NULL(p);
+ ASSERT_EQ(cbm_pipeline_run(p), 0);
+ cbm_pipeline_free(p);
+
+ cbm_setenv("CBM_SEM_PAIR_SIGNALS", tsv, 1);
+ cbm_setenv("CBM_SEM_PAIR_SIGNALS_FLOOR", "0.05", 1);
+ p = cbm_pipeline_new(tmp, db_signals, CBM_MODE_FULL);
+ ASSERT_NOT_NULL(p);
+ int rc = cbm_pipeline_run(p);
+ cbm_unsetenv("CBM_SEM_PAIR_SIGNALS");
+ cbm_unsetenv("CBM_SEM_PAIR_SIGNALS_FLOOR");
+ cbm_unsetenv("CBM_SEMANTIC_THRESHOLD");
+ cbm_pipeline_free(p);
+ ASSERT_EQ(rc, 0);
+
+ cbm_store_t *sp = cbm_store_open_path(db_plain);
+ cbm_store_t *ss = cbm_store_open_path(db_signals);
+ ASSERT_NOT_NULL(sp);
+ ASSERT_NOT_NULL(ss);
+ int edges = 0;
+ ASSERT_TRUE(
+ sem_same_rows(cbm_store_get_db(sp), cbm_store_get_db(ss),
+ "SELECT s.qualified_name, t.qualified_name, e.properties FROM edges e "
+ "JOIN nodes s ON s.id = e.source_id JOIN nodes t ON t.id = e.target_id "
+ "WHERE e.type = 'SEMANTICALLY_RELATED' ORDER BY 1, 2",
+ &edges));
+ ASSERT_GT(edges, 0);
+ cbm_store_close(sp);
+ cbm_store_close(ss);
+
+ /* qa qb score tfidf ri minhash api type decorator struct proximity
+ * same_file admitted loc_a loc_b */
+ FILE *f = cbm_fopen(tsv, "r");
+ ASSERT_NOT_NULL(f);
+ char line[4096];
+ int lines = 0;
+ int admitted = 0;
+ int below = 0;
+ while (fgets(line, sizeof(line), f)) {
+ char *field[16];
+ int n = 0;
+ for (char *tok = strtok(line, "\t\n"); tok && n < 16; tok = strtok(NULL, "\t\n")) {
+ field[n++] = tok;
+ }
+ ASSERT_EQ(n, 15);
+ float score = strtof(field[2], NULL);
+ bool is_admitted = strcmp(field[12], "1") == 0;
+ if (score < 0.3F) {
+ below++;
+ ASSERT_FALSE(is_admitted);
+ }
+ admitted += is_admitted ? 1 : 0;
+ lines++;
+ }
+ fclose(f);
+ ASSERT_EQ(admitted, edges);
+ ASSERT_GT(below, 0); /* the floor recorded pairs under the threshold */
+ ASSERT_GT(lines, admitted);
+ th_rmtree(tmp);
+ PASS();
+}
+
TEST(pipeline_semantic_edges_no_functions) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX");
@@ -19722,6 +19808,7 @@ SUITE(pipeline) {
RUN_TEST(pipeline_markdown_and_config_prose_reaches_fts_body);
RUN_TEST(pipeline_semantic_edges_no_functions);
RUN_TEST(pipeline_semantic_batched_matches_unbatched);
+ RUN_TEST(pipeline_semantic_pair_signals_never_change_the_graph);
RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153);
RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153);
RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153);
diff --git a/tests/test_semantic.c b/tests/test_semantic.c
index e56c02c23..87bc4dcec 100644
--- a/tests/test_semantic.c
+++ b/tests/test_semantic.c
@@ -355,6 +355,111 @@ TEST(sem_corpus_index_accessors_match_name_lookups) {
PASS();
}
+/* ── TF-IDF terms and the combined score ─────────────────────────── */
+
+static int sem_terms_of(const cbm_sem_corpus_t *c, char **tokens, int n, cbm_sem_func_t *f,
+ int *indices, float *weights) {
+ memset(f, 0, sizeof(*f));
+ f->tfidf_indices = indices;
+ f->tfidf_weights = weights;
+ f->tfidf_len = cbm_sem_tfidf_terms(c, tokens, NULL, n, indices, weights);
+ return f->tfidf_len;
+}
+
+/* Two functions share a TF-IDF term only when they share the word. Until
+ * 2026-10 the term key was the token's POSITION: any two functions of equal
+ * length matched on every term, so the signal measured length (a held-out
+ * judged sample: admitted-edge precision 0.713 -> 0.742 with the fix). */
+TEST(sem_tfidf_terms_compare_vocabulary_not_positions) {
+ cbm_sem_corpus_t *c = cbm_sem_corpus_new();
+ ASSERT_NOT_NULL(c);
+ static char *parse[] = {"parse", "header", "value"};
+ static char *render[] = {"render", "widget", "frame"};
+ static char *reparse[] = {"value", "parse", "header", "parse", "absent"};
+ static char *other[] = {"queue", "drain"};
+ cbm_sem_corpus_add_doc(c, (const char **)parse, 3);
+ cbm_sem_corpus_add_doc(c, (const char **)render, 3);
+ cbm_sem_corpus_add_doc(c, (const char **)reparse, 4); /* "absent" stays out */
+ cbm_sem_corpus_add_doc(c, (const char **)other, 2);
+ cbm_sem_corpus_finalize(c);
+
+ cbm_sem_func_t fa;
+ cbm_sem_func_t fb;
+ cbm_sem_func_t fc;
+ int ia[3];
+ int ib[3];
+ int ic[5];
+ float wa[3];
+ float wb[3];
+ float wc[5];
+ ASSERT_EQ(sem_terms_of(c, parse, 3, &fa, ia, wa), 3);
+ ASSERT_EQ(sem_terms_of(c, render, 3, &fb, ib, wb), 3);
+ /* A repeated word is one term, its weight twice the idf; an unknown word
+ * is no term; terms ascend by token index. */
+ ASSERT_EQ(sem_terms_of(c, reparse, 5, &fc, ic, wc), 3);
+ ASSERT_TRUE(ic[0] < ic[1] && ic[1] < ic[2]);
+ int at_parse = ic[0] == cbm_sem_corpus_token_index(c, "parse") ? 0
+ : ic[1] == cbm_sem_corpus_token_index(c, "parse") ? 1
+ : 2;
+ ASSERT_EQ(ic[at_parse], cbm_sem_corpus_token_index(c, "parse"));
+ ASSERT_FLOAT_EQ(wc[at_parse], 2.0F * cbm_sem_corpus_idf(c, "parse"), 1e-6);
+
+ cbm_sem_signals_t s;
+ cbm_sem_signal_values(&fa, &fb, &s);
+ ASSERT_FLOAT_EQ(s.tfidf, 0.0F, 1e-6); /* same length, no shared word */
+ cbm_sem_signal_values(&fa, &fc, &s);
+ ASSERT_TRUE(s.tfidf > 0.9F); /* the same words, other order and length */
+ cbm_sem_corpus_free(c);
+ PASS();
+}
+
+/* cbm_sem_combine is the weighted sum of the signals, times the proximity
+ * multiplier, clamped to [0, 1]; a SIMILAR_TO near copy scores 0. And
+ * cbm_sem_combined_score is exactly that, applied to cbm_sem_signal_values. */
+TEST(sem_combine_weighs_signals) {
+ cbm_sem_config_t cfg = cbm_sem_get_config();
+ cbm_sem_signals_t s = {.tfidf = 0.5F, .ri = 0.25F, .proximity = 1.0F};
+ ASSERT_FLOAT_EQ(cbm_sem_combine(&s, &cfg), cfg.w_tfidf * 0.5F + cfg.w_ri * 0.25F, 1e-6);
+ s.proximity = 1.10F;
+ ASSERT_FLOAT_EQ(cbm_sem_combine(&s, &cfg), (cfg.w_tfidf * 0.5F + cfg.w_ri * 0.25F) * 1.10F,
+ 1e-6);
+ cbm_sem_signals_t all = {.tfidf = 1.0F,
+ .ri = 1.0F,
+ .minhash = 1.0F,
+ .api = 1.0F,
+ .type = 1.0F,
+ .decorator = 1.0F,
+ .struct_profile = 1.0F,
+ .proximity = 1.10F};
+ ASSERT_FLOAT_EQ(cbm_sem_combine(&all, &cfg), 1.0F, 0.0); /* clamped */
+ all.near_copy = true;
+ ASSERT_FLOAT_EQ(cbm_sem_combine(&all, &cfg), 0.0F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_combine(NULL, &cfg), 0.0F, 0.0);
+
+ cbm_sem_corpus_t *c = cbm_sem_corpus_new();
+ ASSERT_NOT_NULL(c);
+ static char *one[] = {"open", "socket", "port"};
+ static char *two[] = {"open", "socket", "host", "retry"};
+ cbm_sem_corpus_add_doc(c, (const char **)one, 3);
+ cbm_sem_corpus_add_doc(c, (const char **)two, 4);
+ cbm_sem_corpus_finalize(c);
+ cbm_sem_func_t fa;
+ cbm_sem_func_t fb;
+ int ia[3];
+ int ib[4];
+ float wa[3];
+ float wb[4];
+ sem_terms_of(c, one, 3, &fa, ia, wa);
+ sem_terms_of(c, two, 4, &fb, ib, wb);
+ fa.file_path = "net/dial.go";
+ fb.file_path = "net/listen.go";
+ cbm_sem_signals_t v;
+ cbm_sem_signal_values(&fa, &fb, &v);
+ ASSERT_FLOAT_EQ(cbm_sem_combined_score(&fa, &fb, &cfg), cbm_sem_combine(&v, &cfg), 0.0);
+ cbm_sem_corpus_free(c);
+ PASS();
+}
+
TEST(sem_corpus_add_null_doc) {
cbm_sem_corpus_t *c = cbm_sem_corpus_new();
ASSERT_NOT_NULL(c);
@@ -531,6 +636,8 @@ SUITE(semantic) {
RUN_TEST(sem_corpus_add_one_doc);
RUN_TEST(sem_corpus_idf);
RUN_TEST(sem_corpus_index_accessors_match_name_lookups);
+ RUN_TEST(sem_tfidf_terms_compare_vocabulary_not_positions);
+ RUN_TEST(sem_combine_weighs_signals);
RUN_TEST(sem_corpus_add_null_doc);
RUN_TEST(sem_corpus_free_null);
RUN_TEST(sem_get_config_defaults);