Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(semantic): TF-IDF compares vocabulary, not token positions
Martin Vogel968d4a4be9f6
diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.cindex 14597289e..a0bcb636d 100644--- a/src/pipeline/pass_semantic_edges.c+++ b/src/pipeline/pass_semantic_edges.c@@ -71,6 +71,7 @@ typedef struct {int64_t target_id;float score;bool same_file;+ bool below; /* under the threshold: recorded for CBM_SEM_PAIR_SIGNALS only, never admitted *//* Canonical admission keys (determinism): func index of the discovering* side, its candidate rank, and the partner's func index. The sequential* admission pass replays pairs in (i, c) order so which pairs win the@@ -93,7 +94,7 @@ static void deferred_buf_init(deferred_edge_buf_t *buf) {}static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt, float score,- bool same_file, int i, int j, int c) {+ bool same_file, bool below, int i, int j, int c) {if (buf->count >= buf->cap) {int nc = buf->cap < CBM_SZ_256 ? CBM_SZ_256 : buf->cap * GROW;deferred_edge_t *grown =@@ -108,6 +109,7 @@ static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt.target_id = tgt,.score = score,.same_file = same_file,+ .below = below,.i = i,.j = j,.c = c};@@ -724,19 +726,10 @@ static void vec_build_worker(int worker_id, void *ctx_ptr) {token_index[t] = cbm_sem_corpus_token_index(vc->corpus, tokens[t]);}- /* TF-IDF weights */+ /* TF-IDF terms, keyed by corpus token (cbm_sem_tfidf_terms). */int *indices = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(int));float *weights = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(float));- int tfidf_len = 0;- for (int t = 0; t < tc; t++) {- float idf = token_index ? cbm_sem_corpus_idf_at(vc->corpus, token_index[t])- : cbm_sem_corpus_idf(vc->corpus, tokens[t]);- if (idf > 0.0F) {- indices[tfidf_len] = t;- weights[tfidf_len] = idf;- tfidf_len++;- }- }+ int tfidf_len = cbm_sem_tfidf_terms(vc->corpus, tokens, token_index, tc, indices, weights);vc->funcs[f].tfidf_indices = indices;vc->funcs[f].tfidf_weights = weights;vc->funcs[f].tfidf_len = tfidf_len;@@ -833,6 +826,7 @@ typedef struct {uint64_t *signatures;int *edge_counts; /* budget applied sequentially in phase6b (determinism) */cbm_sem_config_t cfg;+ float record_floor; /* kept from here up; under the threshold: signals dump only */int func_count;/* LSH buckets (read-only during scoring) */@@ -927,13 +921,13 @@ static void score_try_emit(score_ctx_t *sc, int i, int j, int c, deferred_edge_breturn;}float score = cbm_sem_combined_score(&sc->funcs[i], &sc->funcs[j], &sc->cfg);- if (score < sc->cfg.threshold) {+ if (score < sc->record_floor) {return;}bool same_file = sc->funcs[i].file_path && sc->funcs[j].file_path &&strcmp(sc->funcs[i].file_path, sc->funcs[j].file_path) == 0;- deferred_buf_push(my_buf, sc->funcs[i].node_id, sc->funcs[j].node_id, score, same_file, i, j,- c);+ deferred_buf_push(my_buf, sc->funcs[i].node_id, sc->funcs[j].node_id, score, same_file,+ score < sc->cfg.threshold, i, j, c);}static void score_worker(int worker_id, void *ctx_ptr) {@@ -1138,8 +1132,31 @@ static int cmp_deferred_edge_canonical(const void *pa, const void *pb) {* and apply the per-node max_edges budget HERE — single-threaded — so the* admitted edge set is a pure function of the (canonically sorted) inputs,* independent of worker count and scheduling. */+/* CBM_SEM_PAIR_SIGNALS=<path>: one line per recorded pair (from the record+ * floor up, CBM_SEM_PAIR_SIGNALS_FLOOR, default the threshold), in canonical+ * order: both qualified names, the score, every signal value and whether the+ * pair was admitted. The measuring harness for the semantic engine: judged+ * samples are drawn from it, stratified by score. Off unless set. */+static void pair_signals_line(FILE *f, const cbm_gbuf_t *gbuf, const cbm_sem_func_t *funcs,+ const deferred_edge_t *de, bool admitted) {+ cbm_sem_signals_t s;+ cbm_sem_signal_values(&funcs[de->i], &funcs[de->j], &s);+ const cbm_gbuf_node_t *a = cbm_gbuf_find_by_id(gbuf, de->source_id);+ const cbm_gbuf_node_t *b = cbm_gbuf_find_by_id(gbuf, de->target_id);+ (void)fprintf(+ f,+ "%s\t%s\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.4f\t%d\t%d"+ "\t%s:%d-%d\t%s:%d-%d\n",+ a && a->qualified_name ? a->qualified_name : "?",+ b && b->qualified_name ? b->qualified_name : "?", (double)de->score, (double)s.tfidf,+ (double)s.ri, (double)s.minhash, (double)s.api, (double)s.type, (double)s.decorator,+ (double)s.struct_profile, (double)s.proximity, de->same_file ? 1 : 0, admitted ? 1 : 0,+ a && a->file_path ? a->file_path : "?", a ? a->start_line : 0, a ? a->end_line : 0,+ b && b->file_path ? b->file_path : "?", b ? b->start_line : 0, b ? b->end_line : 0);+}+static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_bufs, int worker_count,- int *edge_counts, int max_edges) {+ int *edge_counts, int max_edges, const cbm_sem_func_t *funcs) {int total_pairs = 0;for (int w = 0; w < worker_count; w++) {total_pairs += worker_bufs[w].count;@@ -1175,20 +1192,34 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf* chase the near-threshold SEMANTICALLY_RELATED flicker: find the pair that* flips, then compare its score between runs. */FILE *pair_dump = NULL;+ FILE *signals = NULL;{char dump_path[CBM_SZ_1K];if (cbm_safe_getenv("CBM_SEM_PAIR_DUMP", dump_path, sizeof(dump_path), NULL)) {pair_dump = cbm_fopen(dump_path, "w");}+ if (funcs && cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS", dump_path, sizeof(dump_path), NULL)) {+ signals = cbm_fopen(dump_path, "w");+ }}int total_edges = 0;for (int e = 0; e < n; e++) {deferred_edge_t *de = &pairs[e];+ if (de->below) {+ if (signals) {+ pair_signals_line(signals, gbuf, funcs, de, false);+ }+ continue;+ }if (pair_dump) {fprintf(pair_dump, "%d %d %.9g\n", de->i, de->j, (double)de->score);}- if (edge_counts[de->i] >= max_edges || edge_counts[de->j] >= max_edges) {+ bool admitted = edge_counts[de->i] < max_edges && edge_counts[de->j] < max_edges;+ if (signals) {+ pair_signals_line(signals, gbuf, funcs, de, admitted);+ }+ if (!admitted) {continue;}char props[PROPS_BUF];@@ -1202,6 +1233,9 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_bufif (pair_dump) {(void)fclose(pair_dump);}+ if (signals) {+ (void)fclose(signals);+ }cbm_free(CBM_MEM_CLASS_SEMANTIC, pairs);return total_edges;}@@ -1542,13 +1576,14 @@ static void sem_corpus_dump(const cbm_sem_corpus_t *corpus, const char *suffix)/* Phase 6a: score candidate pairs in parallel and collect deferred edges. */static void phase6a_score_candidates(cbm_sem_func_t *funcs, uint64_t *signatures, int *edge_counts,sem_bucket_t **band_buckets, cbm_sem_config_t cfg,- deferred_edge_buf_t *worker_bufs, int func_count,- int worker_count) {+ float record_floor, deferred_edge_buf_t *worker_bufs,+ int func_count, int worker_count) {score_ctx_t sc = {.funcs = funcs,.signatures = signatures,.edge_counts = edge_counts,.cfg = cfg,+ .record_floor = record_floor,.func_count = func_count,.band_buckets = (void *)band_buckets,.worker_bufs = worker_bufs,@@ -1619,14 +1654,30 @@ static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t *deferred_buf_init(&worker_bufs[w]);}+ /* The measuring harness may record pairs below the threshold (never+ * admitted): CBM_SEM_PAIR_SIGNALS_FLOOR, only with CBM_SEM_PAIR_SIGNALS. */+ float record_floor = cfg.threshold;+ {+ char path[CBM_SZ_1K]; /* the dump path: a buffer too short reads as unset */+ char buf[CBM_SZ_64];+ if (cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS", path, sizeof(path), NULL) &&+ cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS_FLOOR", buf, sizeof(buf), NULL)) {+ float floor_value = strtof(buf, NULL);+ if (floor_value > 0.0F && floor_value < cfg.threshold) {+ record_floor = floor_value;+ }+ }+ }+CBM_PROF_START(t_phase6a);sem_state_dump(funcs, func_count);- phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, worker_bufs,- func_count, worker_count);+ phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, record_floor,+ worker_bufs, func_count, worker_count);CBM_PROF_END_N("semantic_edges", "6a_score_parallel", t_phase6a, func_count);CBM_PROF_START(t_phase6b);- int total = phase6b_merge_edges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges);+ int total =+ phase6b_merge_edges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges, funcs);CBM_PROF_END_N("semantic_edges", "6b_edge_merge_seq", t_phase6b, total);cbm_free(CBM_MEM_CLASS_SEMANTIC, worker_bufs);diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.hindex 87b9c160b..b00dcd699 100644--- a/src/pipeline/pipeline_internal.h+++ b/src/pipeline/pipeline_internal.h@@ -1274,8 +1274,13 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr* sections and PDF pages hold MENTIONS edges into the code, ADR records* are nodes, and AsciiDoc files, PDFs and a Sphinx project's extra* source suffixes are indexed. An index built before has none of this- * for its unchanged documents, so it rebuilds once on upgrade. */-enum { CBM_SEMANTIC_INDEX_VERSION = 6 };+ * for its unchanged documents, so it rebuilds once on upgrade.+ * 7: SEMANTICALLY_RELATED changed: TF-IDF compares vocabulary, string+ * properties are read with their escapes, admission is best-first and+ * every edge carries "p"; doc_link_candidates holds doc section -> code+ * candidates. An index built before has the old pairs and no+ * candidates, so it rebuilds once on upgrade. */+enum { CBM_SEMANTIC_INDEX_VERSION = 7 };typedef struct {cbm_gbuf_t *gbuf;diff --git a/src/semantic/semantic.c b/src/semantic/semantic.cindex 2aa1df5a3..c28615d93 100644--- a/src/semantic/semantic.c+++ b/src/semantic/semantic.c@@ -1619,6 +1619,47 @@ static float small_cosine(const float *a, const float *b, int dims) {return denom < CBM_SEM_DENOM_EPS ? 0.0F : dot / denom;}+static int cmp_int_asc(const void *pa, const void *pb) {+ int a = *(const int *)pa;+ int b = *(const int *)pb;+ return (a > b) - (a < b);+}++/* Terms are keyed by corpus token, so two functions share a term only when+ * they share the word. (Until 2026-10 the key was the token's POSITION in the+ * function: any two functions of similar length "shared" their first terms,+ * and the TF-IDF signal measured length, not vocabulary.) */+int cbm_sem_tfidf_terms(const cbm_sem_corpus_t *corpus, char *const *tokens, const int *token_index,+ int count, int *indices, float *weights) {+ if (!corpus || !indices || !weights || count <= 0) {+ return 0;+ }+ int known = 0;+ for (int t = 0; t < count; t++) {+ int idx = token_index ? token_index[t] : cbm_sem_corpus_token_index(corpus, tokens[t]);+ if (idx >= 0 && cbm_sem_corpus_idf_at(corpus, idx) > 0.0F) {+ indices[known++] = idx;+ }+ }+ if (known > 1) {+ qsort(indices, (size_t)known, sizeof(indices[0]), cmp_int_asc);+ }+ /* Each run of one index becomes one term; its weight is the idf added once+ * per occurrence (equal indices have equal idf). */+ int terms = 0;+ for (int k = 0; k < known; k++) {+ float idf = cbm_sem_corpus_idf_at(corpus, indices[k]);+ if (terms > 0 && indices[terms - SKIP_ONE] == indices[k]) {+ weights[terms - SKIP_ONE] += idf;+ } else {+ indices[terms] = indices[k];+ weights[terms] = idf;+ terms++;+ }+ }+ return terms;+}+/* Sparse cosine over two pre-sorted (index, weight) vectors. Returns 0 when* either side is empty or the magnitude product is below the epsilon guard. */static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t *b) {@@ -1651,49 +1692,47 @@ static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t *return denom > CBM_SEM_DENOM_EPS ? (dot / denom) : 0.0F;}-float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,- const cbm_sem_config_t *cfg) {- if (!a || !b || !cfg) {- return 0.0F;+void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b,+ cbm_sem_signals_t *out) {+ memset(out, 0, sizeof(*out));+ out->proximity = CBM_SEM_UNIT_POS;+ if (!a || !b) {+ return;}+ if (a->has_minhash && b->has_minhash) {+ double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash,+ (const cbm_minhash_t *)b->minhash);+ out->minhash = (float)j;+ out->near_copy = j >= CBM_MINHASH_JACCARD_THRESHOLD;+ }+ out->tfidf = sparse_tfidf_cosine(a, b);+ out->ri = cbm_rsq_ip(&a->ri_code, &b->ri_code); /* signal 2: Random Indexing */+ out->api = cbm_rsq_ip(&a->api_code, &b->api_code); /* signal 4: API signatures */+ out->type = cbm_rsq_ip(&a->type_code, &b->type_code); /* signal 5: type signatures */+ out->decorator = cbm_rsq_ip(&a->deco_code, &b->deco_code); /* signal 7: decorators */+ /* signals 8+9+11: structural profile + data flow + Halstead */+ out->struct_profile =+ small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS);+ out->proximity = cbm_sem_proximity(a->file_path, b->file_path); /* signal 6: multiplier */+}+float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg) {/* Short-circuit: if MinHash Jaccard is already above the SIMILAR_TO threshold,* the pass_similarity pipeline already emitted a SIMILAR_TO edge for this pair.* Returning 0 here avoids flooding top-k with cross-service copy-paste boilerplate* (logging_middleware, shared push/pull handlers) that SIMILAR_TO already covers,* freeing the edge budget for true semantic leaps (vocabulary-bridged relations). */- if (a->has_minhash && b->has_minhash) {- double early_j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash,- (const cbm_minhash_t *)b->minhash);- if (early_j >= CBM_MINHASH_JACCARD_THRESHOLD) {- return 0.0F;- }- }-- float score = cfg->w_tfidf * sparse_tfidf_cosine(a, b);-- /* Signal 2: Random Indexing */- score += cfg->w_ri * cbm_rsq_ip(&a->ri_code, &b->ri_code);-- /* Signal 3: MinHash Jaccard */- if (a->has_minhash && b->has_minhash) {- double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash,- (const cbm_minhash_t *)b->minhash);- score += cfg->w_minhash * (float)j;+ if (!s || !cfg || s->near_copy) {+ return 0.0F;}-- /* Signal 4: API Signatures */- score += cfg->w_api * cbm_rsq_ip(&a->api_code, &b->api_code);-- /* Signal 5: Type Signatures */- score += cfg->w_type * cbm_rsq_ip(&a->type_code, &b->type_code);-- /* Signal 7: Decorator Pattern */- score += cfg->w_decorator * cbm_rsq_ip(&a->deco_code, &b->deco_code);-- /* Signal 8+9+11: Structural profile + data flow + Halstead */- float sp_score = small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS);- score += cfg->w_struct_profile * sp_score;+ /* The terms in the historical order (float addition is not associative). */+ float score = cfg->w_tfidf * s->tfidf;+ score += cfg->w_ri * s->ri;+ score += cfg->w_minhash * s->minhash;+ score += cfg->w_api * s->api;+ score += cfg->w_type * s->type;+ score += cfg->w_decorator * s->decorator;+ score += cfg->w_struct_profile * s->struct_profile;/* Signal 6: Module proximity (multiplier, not additive).* Proximity returns [1.0, 1.10] — a same-file/same-dir boost for ranking.@@ -1701,17 +1740,26 @@ float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,* cosine-similarity range. Without the clamp, a 0.95 base × 1.10 proximity* would emit a 1.045 score which violates the semantic of "similarity" and* breaks downstream consumers that expect a normalized value. */- score *= cbm_sem_proximity(a->file_path, b->file_path);+ score *= s->proximity;if (score > CBM_SEM_UNIT_POS) {score = CBM_SEM_UNIT_POS;}if (score < 0.0F) {score = 0.0F;}-return score;}+float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,+ const cbm_sem_config_t *cfg) {+ if (!a || !b || !cfg) {+ return 0.0F;+ }+ cbm_sem_signals_t s;+ cbm_sem_signal_values(a, b, &s);+ return cbm_sem_combine(&s, cfg);+}+/* ── Graph diffusion ─────────────────────────────────────────────── */void cbm_sem_diffuse(cbm_sem_vec_t *combined, const cbm_sem_vec_t *neighbors, int neighbor_count,diff --git a/src/semantic/semantic.h b/src/semantic/semantic.hindex 1f99a54c0..34d738796 100644--- a/src/semantic/semantic.h+++ b/src/semantic/semantic.h@@ -206,12 +206,43 @@ const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index,/* Free corpus. */void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus);+/* The TF-IDF terms of one function: one entry per distinct corpus token whose+ * idf is above 0, weighted term frequency x idf, in ascending token-index+ * order (the order the TF-IDF cosine merges on). `token_index[t]` is the+ * corpus index of tokens[t] (-1 when unknown); NULL looks each token up by+ * name. `indices` and `weights` hold at least `count` entries. Returns the+ * number of terms written. */+int cbm_sem_tfidf_terms(const cbm_sem_corpus_t *corpus, char *const *tokens, const int *token_index,+ int count, int *indices, float *weights);+/* ── Combined scoring ────────────────────────────────────────────── *//* Compute combined similarity score between two functions. */float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,const cbm_sem_config_t *cfg);+/* The signal values cbm_sem_combined_score weighs, for measuring them one by+ * one: each similarity in [0, 1] (minhash 0 when either side has none),+ * proximity a multiplier in [1.0, 1.10]. `near_copy`: the pair is a SIMILAR_TO+ * copy, which the combined score sets to 0. */+typedef struct {+ float tfidf;+ float ri;+ float minhash;+ float api;+ float type;+ float decorator;+ float struct_profile;+ float proximity;+ bool near_copy;+} cbm_sem_signals_t;++void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b,+ cbm_sem_signals_t *out);++/* cbm_sem_combined_score from the signal values (the same arithmetic). */+float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg);+/* Module proximity multiplier based on file paths. */float cbm_sem_proximity(const char *path_a, const char *path_b);diff --git a/tests/test_pipeline.c b/tests/test_pipeline.cindex a2861d054..ed7cb7099 100644--- a/tests/test_pipeline.c+++ b/tests/test_pipeline.c@@ -18810,6 +18810,92 @@ TEST(pipeline_semantic_batched_matches_unbatched) {PASS();}+/* CBM_SEM_PAIR_SIGNALS (+ _FLOOR) is the semantic engine's measuring+ * harness: it records each scored pair with its signal values, from a floor+ * below the threshold up. Recording must never change the graph: the same+ * SEMANTICALLY_RELATED rows with and without it, one admitted line per edge,+ * and every pair under the threshold recorded but never admitted. */+TEST(pipeline_semantic_pair_signals_never_change_the_graph) {+ char tmp[256];+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_semsig_XXXXXX");+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));+ static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"};+ for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) {+ char file[64];+ snprintf(file, sizeof(file), "%s_config.go", subjects[i]);+ write_sem_family(tmp, file, subjects[i]);+ }+ char db_plain[512];+ char db_signals[512];+ char tsv[512];+ snprintf(db_plain, sizeof(db_plain), "%s/plain.db", tmp);+ snprintf(db_signals, sizeof(db_signals), "%s/signals.db", tmp);+ snprintf(tsv, sizeof(tsv), "%s/pairs.tsv", tmp);++ cbm_setenv("CBM_SEMANTIC_THRESHOLD", "0.3", 1); /* as above: admits the near-duplicates */+ cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_plain, CBM_MODE_FULL);+ ASSERT_NOT_NULL(p);+ ASSERT_EQ(cbm_pipeline_run(p), 0);+ cbm_pipeline_free(p);++ cbm_setenv("CBM_SEM_PAIR_SIGNALS", tsv, 1);+ cbm_setenv("CBM_SEM_PAIR_SIGNALS_FLOOR", "0.05", 1);+ p = cbm_pipeline_new(tmp, db_signals, CBM_MODE_FULL);+ ASSERT_NOT_NULL(p);+ int rc = cbm_pipeline_run(p);+ cbm_unsetenv("CBM_SEM_PAIR_SIGNALS");+ cbm_unsetenv("CBM_SEM_PAIR_SIGNALS_FLOOR");+ cbm_unsetenv("CBM_SEMANTIC_THRESHOLD");+ cbm_pipeline_free(p);+ ASSERT_EQ(rc, 0);++ cbm_store_t *sp = cbm_store_open_path(db_plain);+ cbm_store_t *ss = cbm_store_open_path(db_signals);+ ASSERT_NOT_NULL(sp);+ ASSERT_NOT_NULL(ss);+ int edges = 0;+ ASSERT_TRUE(+ sem_same_rows(cbm_store_get_db(sp), cbm_store_get_db(ss),+ "SELECT s.qualified_name, t.qualified_name, e.properties FROM edges e "+ "JOIN nodes s ON s.id = e.source_id JOIN nodes t ON t.id = e.target_id "+ "WHERE e.type = 'SEMANTICALLY_RELATED' ORDER BY 1, 2",+ &edges));+ ASSERT_GT(edges, 0);+ cbm_store_close(sp);+ cbm_store_close(ss);++ /* qa qb score tfidf ri minhash api type decorator struct proximity+ * same_file admitted loc_a loc_b */+ FILE *f = cbm_fopen(tsv, "r");+ ASSERT_NOT_NULL(f);+ char line[4096];+ int lines = 0;+ int admitted = 0;+ int below = 0;+ while (fgets(line, sizeof(line), f)) {+ char *field[16];+ int n = 0;+ for (char *tok = strtok(line, "\t\n"); tok && n < 16; tok = strtok(NULL, "\t\n")) {+ field[n++] = tok;+ }+ ASSERT_EQ(n, 15);+ float score = strtof(field[2], NULL);+ bool is_admitted = strcmp(field[12], "1") == 0;+ if (score < 0.3F) {+ below++;+ ASSERT_FALSE(is_admitted);+ }+ admitted += is_admitted ? 1 : 0;+ lines++;+ }+ fclose(f);+ ASSERT_EQ(admitted, edges);+ ASSERT_GT(below, 0); /* the floor recorded pairs under the threshold */+ ASSERT_GT(lines, admitted);+ th_rmtree(tmp);+ PASS();+}+TEST(pipeline_semantic_edges_no_functions) {char tmp[256];snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX");@@ -19722,6 +19808,7 @@ SUITE(pipeline) {RUN_TEST(pipeline_markdown_and_config_prose_reaches_fts_body);RUN_TEST(pipeline_semantic_edges_no_functions);RUN_TEST(pipeline_semantic_batched_matches_unbatched);+ RUN_TEST(pipeline_semantic_pair_signals_never_change_the_graph);RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153);RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153);RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153);diff --git a/tests/test_semantic.c b/tests/test_semantic.cindex e56c02c23..87bc4dcec 100644--- a/tests/test_semantic.c+++ b/tests/test_semantic.c@@ -355,6 +355,111 @@ TEST(sem_corpus_index_accessors_match_name_lookups) {PASS();}+/* ── TF-IDF terms and the combined score ─────────────────────────── */++static int sem_terms_of(const cbm_sem_corpus_t *c, char **tokens, int n, cbm_sem_func_t *f,+ int *indices, float *weights) {+ memset(f, 0, sizeof(*f));+ f->tfidf_indices = indices;+ f->tfidf_weights = weights;+ f->tfidf_len = cbm_sem_tfidf_terms(c, tokens, NULL, n, indices, weights);+ return f->tfidf_len;+}++/* Two functions share a TF-IDF term only when they share the word. Until+ * 2026-10 the term key was the token's POSITION: any two functions of equal+ * length matched on every term, so the signal measured length (a held-out+ * judged sample: admitted-edge precision 0.713 -> 0.742 with the fix). */+TEST(sem_tfidf_terms_compare_vocabulary_not_positions) {+ cbm_sem_corpus_t *c = cbm_sem_corpus_new();+ ASSERT_NOT_NULL(c);+ static char *parse[] = {"parse", "header", "value"};+ static char *render[] = {"render", "widget", "frame"};+ static char *reparse[] = {"value", "parse", "header", "parse", "absent"};+ static char *other[] = {"queue", "drain"};+ cbm_sem_corpus_add_doc(c, (const char **)parse, 3);+ cbm_sem_corpus_add_doc(c, (const char **)render, 3);+ cbm_sem_corpus_add_doc(c, (const char **)reparse, 4); /* "absent" stays out */+ cbm_sem_corpus_add_doc(c, (const char **)other, 2);+ cbm_sem_corpus_finalize(c);++ cbm_sem_func_t fa;+ cbm_sem_func_t fb;+ cbm_sem_func_t fc;+ int ia[3];+ int ib[3];+ int ic[5];+ float wa[3];+ float wb[3];+ float wc[5];+ ASSERT_EQ(sem_terms_of(c, parse, 3, &fa, ia, wa), 3);+ ASSERT_EQ(sem_terms_of(c, render, 3, &fb, ib, wb), 3);+ /* A repeated word is one term, its weight twice the idf; an unknown word+ * is no term; terms ascend by token index. */+ ASSERT_EQ(sem_terms_of(c, reparse, 5, &fc, ic, wc), 3);+ ASSERT_TRUE(ic[0] < ic[1] && ic[1] < ic[2]);+ int at_parse = ic[0] == cbm_sem_corpus_token_index(c, "parse") ? 0+ : ic[1] == cbm_sem_corpus_token_index(c, "parse") ? 1+ : 2;+ ASSERT_EQ(ic[at_parse], cbm_sem_corpus_token_index(c, "parse"));+ ASSERT_FLOAT_EQ(wc[at_parse], 2.0F * cbm_sem_corpus_idf(c, "parse"), 1e-6);++ cbm_sem_signals_t s;+ cbm_sem_signal_values(&fa, &fb, &s);+ ASSERT_FLOAT_EQ(s.tfidf, 0.0F, 1e-6); /* same length, no shared word */+ cbm_sem_signal_values(&fa, &fc, &s);+ ASSERT_TRUE(s.tfidf > 0.9F); /* the same words, other order and length */+ cbm_sem_corpus_free(c);+ PASS();+}++/* cbm_sem_combine is the weighted sum of the signals, times the proximity+ * multiplier, clamped to [0, 1]; a SIMILAR_TO near copy scores 0. And+ * cbm_sem_combined_score is exactly that, applied to cbm_sem_signal_values. */+TEST(sem_combine_weighs_signals) {+ cbm_sem_config_t cfg = cbm_sem_get_config();+ cbm_sem_signals_t s = {.tfidf = 0.5F, .ri = 0.25F, .proximity = 1.0F};+ ASSERT_FLOAT_EQ(cbm_sem_combine(&s, &cfg), cfg.w_tfidf * 0.5F + cfg.w_ri * 0.25F, 1e-6);+ s.proximity = 1.10F;+ ASSERT_FLOAT_EQ(cbm_sem_combine(&s, &cfg), (cfg.w_tfidf * 0.5F + cfg.w_ri * 0.25F) * 1.10F,+ 1e-6);+ cbm_sem_signals_t all = {.tfidf = 1.0F,+ .ri = 1.0F,+ .minhash = 1.0F,+ .api = 1.0F,+ .type = 1.0F,+ .decorator = 1.0F,+ .struct_profile = 1.0F,+ .proximity = 1.10F};+ ASSERT_FLOAT_EQ(cbm_sem_combine(&all, &cfg), 1.0F, 0.0); /* clamped */+ all.near_copy = true;+ ASSERT_FLOAT_EQ(cbm_sem_combine(&all, &cfg), 0.0F, 0.0);+ ASSERT_FLOAT_EQ(cbm_sem_combine(NULL, &cfg), 0.0F, 0.0);++ cbm_sem_corpus_t *c = cbm_sem_corpus_new();+ ASSERT_NOT_NULL(c);+ static char *one[] = {"open", "socket", "port"};+ static char *two[] = {"open", "socket", "host", "retry"};+ cbm_sem_corpus_add_doc(c, (const char **)one, 3);+ cbm_sem_corpus_add_doc(c, (const char **)two, 4);+ cbm_sem_corpus_finalize(c);+ cbm_sem_func_t fa;+ cbm_sem_func_t fb;+ int ia[3];+ int ib[4];+ float wa[3];+ float wb[4];+ sem_terms_of(c, one, 3, &fa, ia, wa);+ sem_terms_of(c, two, 4, &fb, ib, wb);+ fa.file_path = "net/dial.go";+ fb.file_path = "net/listen.go";+ cbm_sem_signals_t v;+ cbm_sem_signal_values(&fa, &fb, &v);+ ASSERT_FLOAT_EQ(cbm_sem_combined_score(&fa, &fb, &cfg), cbm_sem_combine(&v, &cfg), 0.0);+ cbm_sem_corpus_free(c);+ PASS();+}+TEST(sem_corpus_add_null_doc) {cbm_sem_corpus_t *c = cbm_sem_corpus_new();ASSERT_NOT_NULL(c);@@ -531,6 +636,8 @@ SUITE(semantic) {RUN_TEST(sem_corpus_add_one_doc);RUN_TEST(sem_corpus_idf);RUN_TEST(sem_corpus_index_accessors_match_name_lookups);+ RUN_TEST(sem_tfidf_terms_compare_vocabulary_not_positions);+ RUN_TEST(sem_combine_weighs_signals);RUN_TEST(sem_corpus_add_null_doc);RUN_TEST(sem_corpus_free_null);RUN_TEST(sem_get_config_defaults);