Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
/*
* semantic.c — Algorithmic code embeddings: TF-IDF, Random Indexing,
* API/Type/Decorator signatures, combined scoring, graph diffusion.
*
* All signals computed from graph buffer metadata — no source file reads.
* Uses xxHash for deterministic random vectors. Pure C, zero dependencies.
*/
#include "semantic/semantic.h"
#include "foundation/mem_core.h"
#include "foundation/constants.h"
#include "foundation/hash_table.h"
#include "foundation/log.h"
#include "foundation/profile.h"
#include "foundation/platform.h"
#include "foundation/compat_thread.h"
#include "pipeline/worker_pool.h"
#include "simhash/minhash.h"
#include "nomic/code_vectors.h"
#define XXH_INLINE_ALL
#include "xxhash/xxhash.h"
#include <ctype.h>
#include <math.h>
#include <stdatomic.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* ── Constants ───────────────────────────────────────────────────── */
enum {
TOKEN_BUF_LEN = 128,
CORPUS_INIT_CAP = 4096,
DOC_TOKENS_INIT = 64,
RI_SEED_BASE = 0x52494E44, /* "RIND" */
};
/* Default signal weights for cbm_sem_combined_score. Must sum to ~1.0;
* proximity is a multiplier applied on top. */
#define CBM_SEM_W_TFIDF 0.20F
#define CBM_SEM_W_RI 0.25F
#define CBM_SEM_W_MINHASH 0.10F
#define CBM_SEM_W_API 0.15F
#define CBM_SEM_W_TYPE 0.10F
#define CBM_SEM_W_DECORATOR 0.05F
#define CBM_SEM_W_STRUCT_PROFILE 0.10F
#define CBM_SEM_W_DATAFLOW 0.05F
/* Threshold bounds for CBM_SEMANTIC_THRESHOLD env override. */
#define CBM_SEM_THRESHOLD_MIN 0.0F
#define CBM_SEM_THRESHOLD_MAX 1.0F
/* Epsilon for denominator guards in cosine math. */
#define CBM_SEM_DENOM_EPS 1e-10F
/* Int8 quantization bounds for vector storage (maps [-1.0, 1.0] to [-127, 127]). */
#define CBM_SEM_INT8_MAX 127.0F
#define CBM_SEM_INT8_MIN (-127.0F)
/* Unit vector bounds (normalized similarity / cosine value limits). */
#define CBM_SEM_UNIT_POS 1.0F
#define CBM_SEM_UNIT_NEG (-1.0F)
/* Proximity boost: same-file gets 1.0 + CBM_SEM_PROX_MAX_BOOST, distant gets 1.0. */
#define CBM_SEM_PROX_MAX_BOOST 0.10F
/* Reflective Random Indexing blend factors for pass2 (context mixing). */
#define CBM_SEM_RRI_ALPHA 0.3F
#define CBM_SEM_RRI_BETA 0.7F
/* IDF smoothing constant (log base): log2(1 + docs / doc_freq). */
#define CBM_SEM_IDF_BASE 0.5F
/* Strings of exactly two parts / dimensions used for co-occurrence window math. */
#define CBM_SEM_COOCCUR_STRIDE 2
/* Worker tile/chunk sizes for parallel cooccurrence passes. */
enum {
CBM_SEM_WORKER_STACK_CAP = 256,
CBM_SEM_TILE_SIZE = 40,
CBM_SEM_SEEN_INIT_CAP = 256,
CBM_SEM_RESOLVE_CHUNK = 64,
CBM_SEM_COOCCUR_CHUNK = 32,
CBM_SEM_RRI_TILE = 128,
CBM_SEM_INT8_I_LO = -128,
CBM_SEM_INT8_I_HI = 127,
CBM_SEM_ATOMIC_INC = 1,
CBM_SEM_RI_NONZERO_COUNT = 8,
};
/* Mutex-init state machine for the lazy-initialized pretrained token map.
* State 0 = not started, 1 = initializing, 2 = initialized. */
enum {
MTX_STATE_UNINIT = 0,
MTX_STATE_INITIALIZING = 1,
MTX_STATE_READY = 2,
};
/* Pretrained map ready flag (signals a one-shot atomic transition 0 → 1). */
enum { MAP_READY = 1 };
/* Numeric conversion radix for strtol (base 10 decimal). */
enum { BASE_DECIMAL = 10 };
/* ── Configuration ───────────────────────────────────────────────── */
cbm_sem_config_t cbm_sem_get_config(void) {
cbm_sem_config_t cfg = {
.w_tfidf = CBM_SEM_W_TFIDF,
.w_ri = CBM_SEM_W_RI,
.w_minhash = CBM_SEM_W_MINHASH,
.w_api = CBM_SEM_W_API,
.w_type = CBM_SEM_W_TYPE,
.w_decorator = CBM_SEM_W_DECORATOR,
.w_struct_profile = CBM_SEM_W_STRUCT_PROFILE,
.w_dataflow = CBM_SEM_W_DATAFLOW,
.threshold = (float)CBM_SEM_EDGE_THRESHOLD,
.max_edges = CBM_SEM_MAX_EDGES,
};
const char *thresh = cbm_runtime_getenv("CBM_SEMANTIC_THRESHOLD");
if (thresh) {
/* strtod reports errors via endptr; reject non-numeric input silently. */
char *end = NULL;
double parsed = strtod(thresh, &end);
if (end != thresh && parsed > CBM_SEM_THRESHOLD_MIN && parsed <= CBM_SEM_THRESHOLD_MAX) {
cfg.threshold = (float)parsed;
}
}
return cfg;
}
bool cbm_sem_is_enabled(void) {
const char *val = cbm_runtime_getenv("CBM_SEMANTIC_ENABLED");
return val && val[0] == '1';
}
/* ── Token extraction ────────────────────────────────────────────── */
/* True for characters that terminate a token regardless of case. */
static bool is_token_delim(char c) {
return c == '.' || c == '/' || c == '_' || c == '-' || c == ' ' || c == '(' || c == ')' ||
c == ',' || c == ':';
}
/* True for a camelCase transition: uppercase letter preceded by a lowercase. */
static bool is_camel_break(const char *name, int i) {
if (i <= 0) {
return false;
}
char c = name[i];
char p = name[i - SKIP_ONE];
return c >= 'A' && c <= 'Z' && p >= 'a' && p <= 'z';
}
/* Flush the current buffer as a token into out[]. */
static void flush_token(char *buf, int *blen, char **out, int *count, int max_out) {
if (*blen > 0 && *count < max_out) {
buf[*blen] = '\0';
out[(*count)++] = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, buf);
}
*blen = 0;
}
int cbm_sem_tokenize(const char *name, char **out, int max_out) {
if (!name || !out || max_out <= 0) {
return 0;
}
int count = 0;
char buf[TOKEN_BUF_LEN];
int blen = 0;
for (int i = 0; name[i] && count < max_out; i++) {
char c = name[i];
bool split = is_token_delim(c);
bool camel = is_camel_break(name, i);
if (split || camel) {
flush_token(buf, &blen, out, &count, max_out);
if (split) {
continue;
}
}
if (blen < TOKEN_BUF_LEN - SKIP_ONE && isalnum((unsigned char)c)) {
buf[blen++] = (char)tolower((unsigned char)c);
}
}
flush_token(buf, &blen, out, &count, max_out);
/* Abbreviation expansion: add expanded forms for common code abbreviations.
* "err" → also add "error", "ctx" → "context", etc. */
/* Cross-language abbreviation table — covers Go, Python, JS/TS, Rust,
* Java, C/C++, Ruby, PHP, Kotlin, Swift, Scala, C#, and common patterns. */
static const struct {
const char *abbrev;
const char *expanded;
} abbrevs[] = {
/* Error/exception handling */
{"err", "error"},
{"exc", "exception"},
{"ex", "exception"},
/* Context/config */
{"ctx", "context"},
{"cfg", "config"},
{"conf", "configuration"},
{"env", "environment"},
{"opt", "option"},
{"opts", "options"},
/* Request/response (HTTP, RPC) */
{"req", "request"},
{"res", "response"},
{"resp", "response"},
{"rsp", "response"},
{"hdr", "header"},
{"hdrs", "headers"},
/* Strings/formatting */
{"str", "string"},
{"fmt", "format"},
{"msg", "message"},
{"txt", "text"},
{"lbl", "label"},
{"desc", "description"},
/* Data structures */
{"buf", "buffer"},
{"arr", "array"},
{"vec", "vector"},
{"lst", "list"},
{"dict", "dictionary"},
{"tbl", "table"},
{"stk", "stack"},
{"que", "queue"},
/* Functions/callbacks */
{"fn", "function"},
{"func", "function"},
{"cb", "callback"},
{"proc", "procedure"},
{"ctor", "constructor"},
{"dtor", "destructor"},
/* Database/storage */
{"db", "database"},
{"col", "column"},
{"tbl", "table"},
{"stmt", "statement"},
{"txn", "transaction"},
{"trx", "transaction"},
{"repo", "repository"},
/* Auth/security */
{"auth", "authentication"},
{"authz", "authorization"},
{"perm", "permission"},
{"cred", "credential"},
{"tok", "token"},
{"pwd", "password"},
/* Values/types */
{"val", "value"},
{"num", "number"},
{"int", "integer"},
{"bool", "boolean"},
{"flt", "float"},
{"dbl", "double"},
/* Indexing/iteration */
{"idx", "index"},
{"iter", "iterator"},
{"elem", "element"},
{"cnt", "count"},
{"len", "length"},
{"sz", "size"},
{"pos", "position"},
{"off", "offset"},
{"cap", "capacity"},
/* Lifecycle */
{"init", "initialize"},
{"deinit", "deinitialize"},
{"alloc", "allocate"},
{"dealloc", "deallocate"},
{"del", "delete"},
{"rm", "remove"},
/* Implementation/interface */
{"impl", "implementation"},
{"iface", "interface"},
{"abs", "abstract"},
{"decl", "declaration"},
/* Parameters/attributes */
{"param", "parameter"},
{"arg", "argument"},
{"attr", "attribute"},
{"prop", "property"},
{"ret", "return"},
/* Source/destination */
{"src", "source"},
{"dst", "destination"},
{"tgt", "target"},
{"orig", "original"},
{"prev", "previous"},
{"cur", "current"},
{"tmp", "temporary"},
{"temp", "temporary"},
/* Networking/IO */
{"conn", "connection"},
{"sess", "session"},
{"sock", "socket"},
{"addr", "address"},
{"url", "uniform"},
{"srv", "server"},
{"cli", "client"},
{"svc", "service"},
{"ep", "endpoint"},
/* Management */
{"mgr", "manager"},
{"ctrl", "controller"},
{"hdlr", "handler"},
{"sched", "scheduler"},
{"disp", "dispatcher"},
{"reg", "registry"},
/* Async/concurrent */
{"chan", "channel"},
{"sem", "semaphore"},
{"mtx", "mutex"},
{"wg", "waitgroup"},
{"sig", "signal"},
{"evt", "event"},
{"sub", "subscriber"},
{"pub", "publisher"},
/* Testing */
{"spec", "specification"},
{"mock", "mock"},
{"stub", "stub"},
{"assert", "assertion"},
/* Logging/monitoring */
{"log", "logging"},
{"lvl", "level"},
{"dbg", "debug"},
{"wrn", "warning"},
{"inf", "info"},
/* Time */
{"ts", "timestamp"},
{"dur", "duration"},
{"ttl", "timetolive"},
/* Miscellaneous */
{"ver", "version"},
{"ns", "namespace"},
{"pkg", "package"},
{"mod", "module"},
{"lib", "library"},
{"dep", "dependency"},
{"ref", "reference"},
{"ptr", "pointer"},
{"obj", "object"},
{"doc", "document"},
{"cmd", "command"},
{"ops", "operations"},
{"util", "utility"},
{"hlp", "helper"},
{"ext", "extension"},
{NULL, NULL},
};
int orig_count = count;
for (int t = 0; t < orig_count && count < max_out; t++) {
for (int a = 0; abbrevs[a].abbrev; a++) {
if (strcmp(out[t], abbrevs[a].abbrev) == 0) {
out[count++] = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, abbrevs[a].expanded);
break;
}
}
}
return count;
}
/* ── Dense vector operations ─────────────────────────────────────── */
float cbm_sem_cosine(const cbm_sem_vec_t *a, const cbm_sem_vec_t *b) {
if (!a || !b) {
return 0.0F;
}
float dot = 0.0F;
float mag_a = 0.0F;
float mag_b = 0.0F;
for (int i = 0; i < CBM_SEM_DIM; i++) {
dot += a->v[i] * b->v[i];
mag_a += a->v[i] * a->v[i];
mag_b += b->v[i] * b->v[i];
}
float denom = sqrtf(mag_a) * sqrtf(mag_b);
if (denom < CBM_SEM_DENOM_EPS) {
return 0.0F;
}
return dot / denom;
}
/* Pretrained token lookup table — built lazily on first use. */
static CBMHashTable *g_pretrained_map = NULL;
static _Atomic int g_pretrained_ready = 0;
static cbm_mutex_t g_pretrained_mtx;
static _Atomic int g_pretrained_mtx_init = 0;
/* Thread-safe lazy init of the pretrained token lookup map.
* Uses double-checked locking: fast path reads an atomic flag. */
static void ensure_pretrained_map(void) {
if (atomic_load_explicit(&g_pretrained_ready, memory_order_acquire)) {
return;
}
/* First-time init of the mutex itself (also needs to be thread-safe) */
int expected = MTX_STATE_UNINIT;
if (atomic_compare_exchange_strong_explicit(&g_pretrained_mtx_init, &expected,
MTX_STATE_INITIALIZING, memory_order_acq_rel,
memory_order_acquire)) {
cbm_mutex_init(&g_pretrained_mtx);
atomic_store_explicit(&g_pretrained_mtx_init, MTX_STATE_READY, memory_order_release);
} else {
/* Spin until another thread finishes initializing the mutex */
while (atomic_load_explicit(&g_pretrained_mtx_init, memory_order_acquire) !=
MTX_STATE_READY) {
/* brief spin */
}
}
cbm_mutex_lock(&g_pretrained_mtx);
if (!atomic_load_explicit(&g_pretrained_ready, memory_order_acquire)) {
g_pretrained_map = cbm_ht_create(PRETRAINED_TOKEN_COUNT);
char idx_buf[CBM_SZ_16];
for (int i = 0; i < PRETRAINED_TOKEN_COUNT; i++) {
const char *tok = PRETRAINED_TOKENS[i];
if (tok && tok[0]) {
snprintf(idx_buf, sizeof(idx_buf), "%d", i);
cbm_ht_set(g_pretrained_map, cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, tok),
cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, idx_buf));
}
}
atomic_store_explicit(&g_pretrained_ready, MAP_READY, memory_order_release);
}
cbm_mutex_unlock(&g_pretrained_mtx);
}
void cbm_sem_ensure_ready(void) {
ensure_pretrained_map();
}
void cbm_sem_random_index(const char *token, cbm_sem_vec_t *out) {
memset(out, 0, sizeof(*out));
if (!token) {
return;
}
/* Try pretrained nomic-embed-code vector first (768d, distilled from 7B). */
ensure_pretrained_map();
const char *idx_str = cbm_ht_get(g_pretrained_map, token);
if (idx_str) {
char *end = NULL;
long idx = strtol(idx_str, &end, BASE_DECIMAL);
if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) {
const int8_t *pvec = pretrained_vec_at((int)idx);
for (int d = 0; d < CBM_SEM_DIM && d < PRETRAINED_DIM; d++) {
out->v[d] = (float)pvec[d] / CBM_SEM_INT8_MAX;
}
return;
}
}
/* Fallback: sparse random vector for tokens not in pretrained vocab. */
uint64_t seed = XXH3_64bits(token, strlen(token));
for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) {
uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE);
int pos = (int)(h % CBM_SEM_DIM);
float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS;
out->v[pos] += sign;
}
}
void cbm_sem_normalize(cbm_sem_vec_t *v) {
if (!v) {
return;
}
float mag = 0.0F;
for (int i = 0; i < CBM_SEM_DIM; i++) {
mag += v->v[i] * v->v[i];
}
mag = sqrtf(mag);
if (mag < CBM_SEM_DENOM_EPS) {
return;
}
float inv = CBM_SEM_UNIT_POS / mag;
for (int i = 0; i < CBM_SEM_DIM; i++) {
v->v[i] *= inv;
}
}
void cbm_sem_vec_add_scaled(cbm_sem_vec_t *dst, const cbm_sem_vec_t *src, float scale) {
if (!dst || !src) {
return;
}
for (int i = 0; i < CBM_SEM_DIM; i++) {
dst->v[i] += scale * src->v[i];
}
}
/* ── Corpus (IDF + Random Indexing enrichment) ───────────────────── */
typedef struct {
char *token;
int doc_freq;
cbm_sem_vec_t enriched_vec; /* context-enriched via co-occurrence */
} corpus_entry_t;
struct cbm_sem_corpus {
CBMHashTable *token_map; /* token → index into entries[] */
corpus_entry_t *entries;
int entry_count;
int entry_cap;
int doc_count;
bool finalized;
/* Per-document token lists for co-occurrence pass */
int **doc_token_ids;
int *doc_token_counts;
int doc_cap;
};
static int corpus_get_or_add(cbm_sem_corpus_t *c, const char *token) {
char idx_buf[CBM_SZ_16];
const char *existing = cbm_ht_get(c->token_map, token);
if (existing) {
char *end = NULL;
long parsed = strtol(existing, &end, BASE_DECIMAL);
return (end != existing) ? (int)parsed : CBM_NOT_FOUND;
}
if (c->entry_count >= c->entry_cap) {
int new_cap = c->entry_cap < CORPUS_INIT_CAP ? CORPUS_INIT_CAP : c->entry_cap * PAIR_LEN;
corpus_entry_t *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, c->entries,
(size_t)new_cap * sizeof(corpus_entry_t));
if (!grown) {
return CBM_NOT_FOUND;
}
c->entries = grown;
c->entry_cap = new_cap;
}
int idx = c->entry_count++;
c->entries[idx].token = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, token);
c->entries[idx].doc_freq = 0;
memset(&c->entries[idx].enriched_vec, 0, sizeof(cbm_sem_vec_t));
snprintf(idx_buf, sizeof(idx_buf), "%d", idx);
cbm_ht_set(c->token_map, cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, token),
cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, idx_buf));
return idx;
}
cbm_sem_corpus_t *cbm_sem_corpus_new(void) {
cbm_sem_corpus_t *c = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, sizeof(cbm_sem_corpus_t));
if (c) {
c->token_map = cbm_ht_create(CORPUS_INIT_CAP);
}
return c;
}
void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int count) {
if (!corpus || !tokens || count <= 0) {
return;
}
/* Track document for co-occurrence pass */
if (corpus->doc_count >= corpus->doc_cap) {
int new_cap =
corpus->doc_cap < DOC_TOKENS_INIT ? DOC_TOKENS_INIT : corpus->doc_cap * PAIR_LEN;
int **grown_ids = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids,
(size_t)new_cap * sizeof(int *));
int *grown_counts = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_counts,
(size_t)new_cap * sizeof(int));
if (!grown_ids || !grown_counts) {
cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_ids);
cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_counts);
return;
}
corpus->doc_token_ids = grown_ids;
corpus->doc_token_counts = grown_counts;
corpus->doc_cap = new_cap;
}
int doc_idx = corpus->doc_count++;
corpus->doc_token_ids[doc_idx] = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)count * sizeof(int));
corpus->doc_token_counts[doc_idx] = count;
/* Per-doc unique set for IDF */
int *seen =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC,
((size_t)corpus->entry_cap + (size_t)count + CORPUS_INIT_CAP) * sizeof(int));
int seen_count = 0;
for (int i = 0; i < count; i++) {
int tid = corpus_get_or_add(corpus, tokens[i]);
corpus->doc_token_ids[doc_idx][i] = tid;
if (tid < 0) {
continue;
}
/* Check uniqueness for IDF (simple linear scan — tokens per doc is small) */
bool is_new = true;
for (int j = 0; j < seen_count; j++) {
if (seen[j] == tid) {
is_new = false;
break;
}
}
if (is_new) {
seen[seen_count++] = tid;
corpus->entries[tid].doc_freq++;
}
}
cbm_free(CBM_MEM_CLASS_SEMANTIC, seen);
}
/* ── Parallel corpus batch build ──────────────────────────────────── */
/* Strategy:
* Phase A (SEQUENTIAL): Scan all documents once to build the global
* token_map (inserts unique tokens, assigns global IDs). This is
* inherently sequential (hash table mutation), but much faster than
* the current per-doc add_doc because we avoid the per-doc malloc of
* the `seen` array and per-doc bookkeeping.
* Phase B (PARALLEL): Each worker processes a chunk of docs, translates
* tokens → global IDs via read-only token_map lookups, fills
* doc_token_ids[d], and accumulates doc_freq contributions via atomics.
*/
typedef struct {
cbm_sem_corpus_t *corpus;
char **all_tokens;
const size_t *offsets; /* document d = all_tokens[offsets[d] ..] */
const int *token_counts;
int doc_count;
_Atomic int *doc_freq_atomic; /* per-entry atomic counter (entry_count long) */
_Atomic int next_idx;
} batch_resolve_ctx_t;
/* Resolve one document: look up each token's global ID, fill the corpus
* doc_token_ids[d], and bump the per-token doc_freq counter atomically. The
* caller is responsible for ensuring `seen` has capacity for `count` ints
* before calling (the worker grows its per-thread scratch buffer). */
static void batch_resolve_one_doc(batch_resolve_ctx_t *bc, int doc_index, int *seen) {
int count = bc->token_counts[doc_index];
if (count <= 0) {
bc->corpus->doc_token_ids[doc_index] = NULL;
bc->corpus->doc_token_counts[doc_index] = 0;
return;
}
int *ids = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)count * sizeof(int));
bc->corpus->doc_token_ids[doc_index] = ids;
bc->corpus->doc_token_counts[doc_index] = count;
int seen_count = 0;
char **tokens = &bc->all_tokens[bc->offsets[doc_index]];
for (int i = 0; i < count; i++) {
const char *idx_str = cbm_ht_get(bc->corpus->token_map, tokens[i]);
int tid = CBM_NOT_FOUND;
if (idx_str) {
char *end = NULL;
long parsed = strtol(idx_str, &end, BASE_DECIMAL);
if (end != idx_str) {
tid = (int)parsed;
}
}
ids[i] = tid;
if (tid < 0) {
continue;
}
/* Unique-per-doc check for IDF */
bool is_new = true;
for (int j = 0; j < seen_count; j++) {
if (seen[j] == tid) {
is_new = false;
break;
}
}
if (is_new) {
seen[seen_count++] = tid;
atomic_fetch_add_explicit(&bc->doc_freq_atomic[tid], CBM_SEM_ATOMIC_INC,
memory_order_relaxed);
}
}
}
static void batch_resolve_worker(int worker_id, void *ctx_ptr) {
(void)worker_id;
batch_resolve_ctx_t *bc = ctx_ptr;
/* Per-worker scratch for unique-per-doc tracking */
int local_seen_cap = CBM_SEM_SEEN_INIT_CAP;
int *seen = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)local_seen_cap * sizeof(int));
if (!seen) {
return;
}
while (true) {
int start =
atomic_fetch_add_explicit(&bc->next_idx, CBM_SEM_RESOLVE_CHUNK, memory_order_relaxed);
if (start >= bc->doc_count) {
break;
}
int end = start + CBM_SEM_RESOLVE_CHUNK;
if (end > bc->doc_count) {
end = bc->doc_count;
}
for (int d = start; d < end; d++) {
int count = bc->token_counts[d];
if (count > local_seen_cap) {
int *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, seen, (size_t)count * sizeof(int));
if (!grown) {
continue;
}
seen = grown;
local_seen_cap = count;
}
batch_resolve_one_doc(bc, d, seen);
}
}
cbm_free(CBM_MEM_CLASS_SEMANTIC, seen);
}
void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens,
const size_t *offsets, const int *token_counts, int doc_count) {
if (!corpus || !all_tokens || !offsets || !token_counts || doc_count <= 0) {
return;
}
/* Phase A (SEQUENTIAL): Build token_map and allocate doc arrays.
* Hash table mutation can't be parallelized; strdup+insert is the cost. */
if (corpus->doc_cap < corpus->doc_count + doc_count) {
int new_cap = corpus->doc_count + doc_count;
int **grown_ids = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids,
(size_t)new_cap * sizeof(int *));
int *grown_counts = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_counts,
(size_t)new_cap * sizeof(int));
if (!grown_ids || !grown_counts) {
cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_ids);
cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_counts);
return;
}
corpus->doc_token_ids = grown_ids;
corpus->doc_token_counts = grown_counts;
corpus->doc_cap = new_cap;
}
int base_doc = corpus->doc_count;
corpus->doc_count += doc_count;
for (int d = 0; d < doc_count; d++) {
int count = token_counts[d];
char **tokens = &all_tokens[offsets[d]];
for (int i = 0; i < count; i++) {
/* Inserts token into token_map if new; we discard return here —
* Phase B will re-lookup in read-only mode to get the ID. */
(void)corpus_get_or_add(corpus, tokens[i]);
}
}
/* Phase B (PARALLEL): Resolve tokens → IDs and count doc_freq per entry.
* token_map is now read-only; each worker owns its doc range (no writes
* to shared state except atomic doc_freq counters). */
_Atomic int *doc_freq_atomic =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)corpus->entry_count * sizeof(_Atomic int));
if (!doc_freq_atomic) {
/* OOM fallback: sequential path. Roll back doc_count first since
* add_doc increments it itself. */
corpus->doc_count = base_doc;
for (int d = 0; d < doc_count; d++) {
int count = token_counts[d];
char **tokens = &all_tokens[offsets[d]];
cbm_sem_corpus_add_doc(corpus, (const char **)tokens, count);
}
return;
}
int worker_count = cbm_default_worker_count(false);
batch_resolve_ctx_t bc = {
.corpus = corpus,
.all_tokens = all_tokens,
.token_counts = token_counts,
.offsets = offsets,
.doc_count = doc_count,
.doc_freq_atomic = doc_freq_atomic,
};
atomic_init(&bc.next_idx, 0);
/* Temporarily re-base doc arrays so workers write to base_doc..base_doc+doc_count */
corpus->doc_token_ids += base_doc;
corpus->doc_token_counts += base_doc;
cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false};
cbm_parallel_for(worker_count, batch_resolve_worker, &bc, opts);
corpus->doc_token_ids -= base_doc;
corpus->doc_token_counts -= base_doc;
/* Phase C (SEQUENTIAL reduce): atomic counters → entries[].doc_freq */
for (int i = 0; i < corpus->entry_count; i++) {
corpus->entries[i].doc_freq +=
atomic_load_explicit(&doc_freq_atomic[i], memory_order_relaxed);
}
cbm_free(CBM_MEM_CLASS_SEMANTIC, doc_freq_atomic);
}
/* ── Parallel corpus_finalize ─────────────────────────────────────── */
/* Strategy:
* 1. Precompute base RI vectors into a shared array (eliminates ~333M
* redundant cbm_sem_random_index calls on kernel-scale corpora).
* 2. Co-occurrence passes: partition TARGET tokens across workers so each
* worker writes to a disjoint range of enriched_vec (zero contention).
* Each worker still scans all documents but only accumulates for targets
* in its range. Inner vector add is the parallelized work.
* 3. Normalize/blend loops are trivially parallel per-entry.
*/
/* Reverse index: for each token id, a list of (doc_id, position_in_doc) pairs.
* Built once, reused for both cooccur passes. Eliminates the O(num_chunks × doc_count)
* redundant outer scan in the old algorithm. */
typedef struct {
int32_t doc_id;
int32_t pos;
} cooccur_pos_t;
typedef struct {
int *offsets; /* offsets[entry_count + 1], prefix sum of occurrences */
cooccur_pos_t *flat; /* flat array of positions, total = offsets[entry_count] */
} reverse_index_t;
/* Tagged source vector: sparse (~30% of tokens, 8 inline nonzeros) or dense int8
* reference into PRETRAINED_VECTOR_BLOB (no copy). Dense path converts int8→float
* on the fly in the hot loop. Tested with both int8 and float32 blob storage
* formats — cooccur passes are memory-bandwidth-bound, so the int8 format (4x
* less source traffic) is equivalent in wall time despite the conversion cost,
* while saving 90 MB of binary size. */
typedef struct {
uint8_t is_sparse; /* 1 = sparse path, 0 = dense int8 reference */
uint8_t nnz; /* number of nonzeros used in sparse path */
uint16_t _pad;
uint16_t indices[CBM_SEM_SPARSE_NNZE]; /* 8 * 2 = 16 bytes */
float values[CBM_SEM_SPARSE_NNZE]; /* 8 * 4 = 32 bytes */
const int8_t *dense_int8; /* points into PRETRAINED_VECTOR_BLOB */
} cbm_sem_src_entry_t;
/* Inline helper: initialize a target vector from a sparse/dense source. */
static inline void sem_target_init_from_src(cbm_sem_vec_t *dst, const cbm_sem_src_entry_t *src) {
memset(dst, 0, sizeof(*dst));
if (src->is_sparse) {
for (int k = 0; k < src->nnz; k++) {
dst->v[src->indices[k]] = src->values[k];
}
} else {
const int8_t *s = src->dense_int8;
const float inv127 = CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX;
for (int d = 0; d < CBM_SEM_DIM; d++) {
dst->v[d] = inv127 * (float)s[d];
}
}
}
/* Inline helper: add weighted source into target.
* Sparse path: ~8 operations, ~48 bytes source memory traffic.
* Dense path: 768 mul-adds with int8→float conversion, ~768 bytes traffic. */
static inline void sem_vec_add_src_scaled(cbm_sem_vec_t *dst, const cbm_sem_src_entry_t *src,
float scale) {
if (src->is_sparse) {
for (int k = 0; k < src->nnz; k++) {
dst->v[src->indices[k]] += scale * src->values[k];
}
} else {
const int8_t *s = src->dense_int8;
const float mul = scale * (CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX);
for (int d = 0; d < CBM_SEM_DIM; d++) {
dst->v[d] += mul * (float)s[d];
}
}
}
/* Pass 1 context: uses sparse/int8 tagged sources (most memory-efficient). */
typedef struct {
corpus_entry_t *entries;
const cbm_sem_src_entry_t *src_entries; /* sparse or int8-dense per token */
int **doc_token_ids;
const int *doc_token_counts;
const reverse_index_t *rev;
int doc_count;
int entry_count;
_Atomic int next_chunk;
int num_chunks;
int chunk_size;
/* Cache-blocked tiling parameters */
int tile_size; /* targets per L2-resident tile */
} cooccur_sparse_ctx_t;
/* Accumulate co-occurrence context for a single target token into `target`.
* Reads neighbors within ±CBM_SEM_WINDOW positions across all documents that
* reference this token id via the reverse index. */
static void cooccur_sparse_one_target(cooccur_sparse_ctx_t *cc, int tid, cbm_sem_vec_t *target) {
int occ_start = cc->rev->offsets[tid];
int occ_end = cc->rev->offsets[tid + SKIP_ONE];
/* Frequent-token subsampling: stride over occurrences when a token is very
* common so the enriched vector's direction is preserved (it's normalized
* after) while bounding work to ~CBM_SEM_MAX_OCCUR samples. See semantic.h. */
int occ_step = SKIP_ONE;
if (occ_end - occ_start > CBM_SEM_MAX_OCCUR) {
occ_step = (occ_end - occ_start) / CBM_SEM_MAX_OCCUR;
}
for (int p = occ_start; p < occ_end; p += occ_step) {
int d = cc->rev->flat[p].doc_id;
int i = cc->rev->flat[p].pos;
int *ids = cc->doc_token_ids[d];
int len = cc->doc_token_counts[d];
for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) {
if (w == 0) {
continue;
}
int j = i + w;
if (j < 0 || j >= len) {
continue;
}
int nid = ids[j];
if (nid < 0) {
continue;
}
float weight = CBM_SEM_UNIT_POS / (float)abs(w);
sem_vec_add_src_scaled(target, &cc->src_entries[nid], weight);
}
}
}
static void cooccur_worker_sparse(int worker_id, void *ctx_ptr) {
(void)worker_id;
cooccur_sparse_ctx_t *cc = ctx_ptr;
while (true) {
int ci =
atomic_fetch_add_explicit(&cc->next_chunk, CBM_SEM_ATOMIC_INC, memory_order_relaxed);
if (ci >= cc->num_chunks) {
break;
}
int chunk_start = ci * cc->chunk_size;
int chunk_end = chunk_start + cc->chunk_size;
if (chunk_end > cc->entry_count) {
chunk_end = cc->entry_count;
}
/* Cache-blocked target tiling: process tile_size targets at a time so
* their vectors stay resident in L2 cache during their accumulation. */
for (int tile_start = chunk_start; tile_start < chunk_end; tile_start += cc->tile_size) {
int tile_end = tile_start + cc->tile_size;
if (tile_end > chunk_end) {
tile_end = chunk_end;
}
for (int tid = tile_start; tid < tile_end; tid++) {
sem_target_init_from_src(&cc->entries[tid].enriched_vec, &cc->src_entries[tid]);
cooccur_sparse_one_target(cc, tid, &cc->entries[tid].enriched_vec);
}
}
}
}
/* Pass 2 (RRI) context: uses int8-quantized pass1 vectors as source.
* Pass1 outputs are dense float32 post-normalization, values in [-1,1].
* We quantize to int8 once (×127) to cut source memory traffic 4x. */
typedef struct {
corpus_entry_t *entries;
const int8_t *pass1_q; /* [entry_count × CBM_SEM_DIM] int8 quantized pass1 */
int **doc_token_ids;
const int *doc_token_counts;
const reverse_index_t *rev;
int doc_count;
int entry_count;
_Atomic int next_chunk;
int num_chunks;
int chunk_size;
int tile_size;
} cooccur_int8_ctx_t;
static inline void sem_vec_add_int8_scaled(cbm_sem_vec_t *dst, const int8_t *src, float scale) {
const float mul = scale * (CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX);
for (int d = 0; d < CBM_SEM_DIM; d++) {
dst->v[d] += mul * (float)src[d];
}
}
/* RRI pass 2 accumulator for a single target token, reading int8-quantized
* pass1 vectors as the source. */
static void cooccur_int8_one_target(cooccur_int8_ctx_t *cc, int tid, cbm_sem_vec_t *target) {
int occ_start = cc->rev->offsets[tid];
int occ_end = cc->rev->offsets[tid + SKIP_ONE];
/* Frequent-token subsampling (see cooccur_sparse_one_target / semantic.h). */
int occ_step = SKIP_ONE;
if (occ_end - occ_start > CBM_SEM_MAX_OCCUR) {
occ_step = (occ_end - occ_start) / CBM_SEM_MAX_OCCUR;
}
for (int p = occ_start; p < occ_end; p += occ_step) {
int d = cc->rev->flat[p].doc_id;
int i = cc->rev->flat[p].pos;
int *ids = cc->doc_token_ids[d];
int len = cc->doc_token_counts[d];
for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) {
if (w == 0) {
continue;
}
int j = i + w;
if (j < 0 || j >= len) {
continue;
}
int nid = ids[j];
if (nid < 0) {
continue;
}
float weight = CBM_SEM_UNIT_POS / (float)abs(w);
sem_vec_add_int8_scaled(target, &cc->pass1_q[(size_t)nid * CBM_SEM_DIM], weight);
}
}
}
static void cooccur_worker_int8(int worker_id, void *ctx_ptr) {
(void)worker_id;
cooccur_int8_ctx_t *cc = ctx_ptr;
while (true) {
int ci =
atomic_fetch_add_explicit(&cc->next_chunk, CBM_SEM_ATOMIC_INC, memory_order_relaxed);
if (ci >= cc->num_chunks) {
break;
}
int chunk_start = ci * cc->chunk_size;
int chunk_end = chunk_start + cc->chunk_size;
if (chunk_end > cc->entry_count) {
chunk_end = cc->entry_count;
}
for (int tile_start = chunk_start; tile_start < chunk_end; tile_start += cc->tile_size) {
int tile_end = tile_start + cc->tile_size;
if (tile_end > chunk_end) {
tile_end = chunk_end;
}
for (int tid = tile_start; tid < tile_end; tid++) {
/* RRI pass 2 starts from zero (no self-init) */
memset(&cc->entries[tid].enriched_vec, 0, sizeof(cbm_sem_vec_t));
cooccur_int8_one_target(cc, tid, &cc->entries[tid].enriched_vec);
}
}
}
}
typedef struct {
corpus_entry_t *entries;
int entry_count;
_Atomic int next_idx;
} norm_ctx_t;
static void normalize_worker(int worker_id, void *ctx_ptr) {
(void)worker_id;
norm_ctx_t *nc = ctx_ptr;
while (true) {
int start = atomic_fetch_add_explicit(&nc->next_idx, CBM_SEM_WORKER_STACK_CAP,
memory_order_relaxed);
if (start >= nc->entry_count) {
break;
}
int end = start + CBM_SEM_WORKER_STACK_CAP;
if (end > nc->entry_count) {
end = nc->entry_count;
}
for (int i = start; i < end; i++) {
cbm_sem_normalize(&nc->entries[i].enriched_vec);
}
}
}
typedef struct {
corpus_entry_t *entries;
const cbm_sem_vec_t *pass1;
int entry_count;
_Atomic int next_idx;
} blend_ctx_t;
static void blend_worker(int worker_id, void *ctx_ptr) {
(void)worker_id;
blend_ctx_t *bc = ctx_ptr;
while (true) {
int start = atomic_fetch_add_explicit(&bc->next_idx, CBM_SEM_WORKER_STACK_CAP,
memory_order_relaxed);
if (start >= bc->entry_count) {
break;
}
int end = start + CBM_SEM_WORKER_STACK_CAP;
if (end > bc->entry_count) {
end = bc->entry_count;
}
for (int i = start; i < end; i++) {
cbm_sem_normalize(&bc->entries[i].enriched_vec);
for (int d = 0; d < CBM_SEM_DIM; d++) {
bc->entries[i].enriched_vec.v[d] =
(CBM_SEM_RRI_BETA * bc->pass1[i].v[d]) +
(CBM_SEM_RRI_ALPHA * bc->entries[i].enriched_vec.v[d]);
}
}
}
}
typedef struct {
corpus_entry_t *entries;
cbm_sem_src_entry_t *src_entries;
int entry_count;
_Atomic int next_idx;
} src_build_ctx_t;
/* Build one src_entry for a token: dense float32 reference if in nomic vocab,
* sparse inline representation otherwise. Collisions in the sparse hash are
* merged and zeros filtered so the final representation is exactly the same
* mathematical vector that the old dense path produced. */
static void build_src_entry(const char *token, cbm_sem_src_entry_t *out) {
memset(out, 0, sizeof(*out));
if (!token) {
out->is_sparse = SKIP_ONE;
out->nnz = 0;
return;
}
/* Dense path: direct int8 pointer into pretrained blob (zero-copy). */
const char *idx_str = cbm_ht_get(g_pretrained_map, token);
if (idx_str) {
char *end = NULL;
long idx = strtol(idx_str, &end, BASE_DECIMAL);
if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) {
out->is_sparse = 0;
out->dense_int8 = pretrained_vec_at((int)idx);
return;
}
}
/* Sparse path: compute 8 hash positions with collision merging. */
out->is_sparse = SKIP_ONE;
uint16_t tmp_idx[CBM_SEM_SPARSE_NNZE];
float tmp_val[CBM_SEM_SPARSE_NNZE];
int count = 0;
uint64_t seed = XXH3_64bits(token, strlen(token));
for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) {
uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE);
int pos = (int)(h % CBM_SEM_DIM);
float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS;
/* Merge collisions */
int found = CBM_NOT_FOUND;
for (int j = 0; j < count; j++) {
if (tmp_idx[j] == (uint16_t)pos) {
found = j;
break;
}
}
if (found >= 0) {
tmp_val[found] += sign;
} else {
tmp_idx[count] = (uint16_t)pos;
tmp_val[count] = sign;
count++;
}
}
/* Filter zeros */
int nnz = 0;
for (int j = 0; j < count; j++) {
if (tmp_val[j] != 0.0F) {
out->indices[nnz] = tmp_idx[j];
out->values[nnz] = tmp_val[j];
nnz++;
}
}
out->nnz = (uint8_t)nnz;
}
static void src_build_worker(int worker_id, void *ctx_ptr) {
(void)worker_id;
src_build_ctx_t *sc = ctx_ptr;
while (true) {
int start = atomic_fetch_add_explicit(&sc->next_idx, CBM_SEM_WORKER_STACK_CAP,
memory_order_relaxed);
if (start >= sc->entry_count) {
break;
}
int end = start + CBM_SEM_WORKER_STACK_CAP;
if (end > sc->entry_count) {
end = sc->entry_count;
}
for (int i = start; i < end; i++) {
build_src_entry(sc->entries[i].token, &sc->src_entries[i]);
}
}
}
/* Quantize pass1 float vectors to int8 (× 127) for use as pass2 source.
* Input vectors are unit-normalized, so values are in [-1, 1] → int8 preserves
* ~99% precision. 4x less memory traffic for pass2. */
typedef struct {
const corpus_entry_t *entries;
int8_t *pass1_q;
int entry_count;
_Atomic int next_idx;
} pass1_quant_ctx_t;
static void pass1_quantize_worker(int worker_id, void *ctx_ptr) {
(void)worker_id;
pass1_quant_ctx_t *qc = ctx_ptr;
while (true) {
int start =
atomic_fetch_add_explicit(&qc->next_idx, CBM_SEM_RRI_TILE, memory_order_relaxed);
if (start >= qc->entry_count) {
break;
}
int end = start + CBM_SEM_RRI_TILE;
if (end > qc->entry_count) {
end = qc->entry_count;
}
for (int i = start; i < end; i++) {
const cbm_sem_vec_t *src = &qc->entries[i].enriched_vec;
int8_t *dst = &qc->pass1_q[(size_t)i * CBM_SEM_DIM];
for (int d = 0; d < CBM_SEM_DIM; d++) {
float v = src->v[d] * CBM_SEM_INT8_MAX;
if (v > CBM_SEM_INT8_MAX) {
v = CBM_SEM_INT8_MAX;
}
if (v < -CBM_SEM_INT8_MAX) {
v = -CBM_SEM_INT8_MAX;
}
dst[d] = (int8_t)(v >= 0 ? v + CBM_SEM_IDF_BASE : v - CBM_SEM_IDF_BASE);
}
}
}
}
/* Build reverse index: token_id → list of (doc_id, position) pairs.
* SEQUENTIAL (fast: just pointer arithmetic + flat array fill). */
static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) {
reverse_index_t *rev = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, sizeof(reverse_index_t));
if (!rev) {
return NULL;
}
/* Phase A: count occurrences per token */
int *counts =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int));
if (!counts) {
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
return NULL;
}
long total = 0;
for (int d = 0; d < corpus->doc_count; d++) {
int *ids = corpus->doc_token_ids[d];
int len = corpus->doc_token_counts[d];
for (int i = 0; i < len; i++) {
int tid = ids[i];
if (tid >= 0 && tid < corpus->entry_count) {
counts[tid]++;
total++;
}
}
}
/* Phase B: exclusive prefix sum → offsets[] */
rev->offsets =
cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int));
if (!rev->offsets) {
cbm_free(CBM_MEM_CLASS_SEMANTIC, counts);
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
return NULL;
}
int running = 0;
for (int t = 0; t < corpus->entry_count; t++) {
rev->offsets[t] = running;
running += counts[t];
counts[t] = 0; /* reuse as per-token fill cursor */
}
rev->offsets[corpus->entry_count] = running;
/* Phase C: fill flat array. Ensure allocation size > 0 even for empty
* corpora (avoids cbm_alloc(CBM_MEM_CLASS_SEMANTIC, 0) which is implementation-defined). */
size_t flat_bytes = (total > 0 ? (size_t)total : SKIP_ONE) * sizeof(cooccur_pos_t);
rev->flat = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, flat_bytes);
if (!rev->flat) {
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev->offsets);
cbm_free(CBM_MEM_CLASS_SEMANTIC, counts);
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
return NULL;
}
for (int d = 0; d < corpus->doc_count; d++) {
int *ids = corpus->doc_token_ids[d];
int len = corpus->doc_token_counts[d];
for (int i = 0; i < len; i++) {
int tid = ids[i];
if (tid >= 0 && tid < corpus->entry_count) {
int slot = rev->offsets[tid] + counts[tid]++;
rev->flat[slot].doc_id = (int32_t)d;
rev->flat[slot].pos = (int32_t)i;
}
}
}
cbm_free(CBM_MEM_CLASS_SEMANTIC, counts);
return rev;
}
static void free_reverse_index(reverse_index_t *rev) {
if (!rev) {
return;
}
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev->offsets);
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev->flat);
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
}
/* Bundle of parameters shared by the finalize sub-phases. */
typedef struct {
cbm_sem_corpus_t *corpus;
reverse_index_t *rev;
cbm_sem_src_entry_t *src_entries;
int worker_count;
int num_chunks;
int chunk_size;
int tile_size;
cbm_parallel_for_opts_t opts;
} finalize_params_t;
/* Sub-phase 1: build tagged source vectors (sparse or dense-int8) in parallel. */
static void finalize_build_sources(finalize_params_t *p) {
src_build_ctx_t sc = {
.entries = p->corpus->entries,
.src_entries = p->src_entries,
.entry_count = p->corpus->entry_count,
};
atomic_init(&sc.next_idx, 0);
cbm_parallel_for(p->worker_count, src_build_worker, &sc, p->opts);
}
/* Sub-phases 2+3: co-occurrence pass 1 + normalize. */
static void finalize_pass1(finalize_params_t *p) {
cooccur_sparse_ctx_t cc = {
.entries = p->corpus->entries,
.src_entries = p->src_entries,
.doc_token_ids = p->corpus->doc_token_ids,
.doc_token_counts = p->corpus->doc_token_counts,
.rev = p->rev,
.doc_count = p->corpus->doc_count,
.entry_count = p->corpus->entry_count,
.num_chunks = p->num_chunks,
.chunk_size = p->chunk_size,
.tile_size = p->tile_size,
};
atomic_init(&cc.next_chunk, 0);
cbm_parallel_for(p->worker_count, cooccur_worker_sparse, &cc, p->opts);
norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count};
atomic_init(&nc.next_idx, 0);
cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts);
}
/* Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. */
static void finalize_pass2(finalize_params_t *p) {
int8_t *pass1_q = cbm_alloc(CBM_MEM_CLASS_SEMANTIC,
(size_t)p->corpus->entry_count * CBM_SEM_DIM * sizeof(int8_t));
if (pass1_q) {
pass1_quant_ctx_t qc = {
.entries = p->corpus->entries,
.pass1_q = pass1_q,
.entry_count = p->corpus->entry_count,
};
atomic_init(&qc.next_idx, 0);
cbm_parallel_for(p->worker_count, pass1_quantize_worker, &qc, p->opts);
}
cbm_sem_vec_t *pass1 =
cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)p->corpus->entry_count * sizeof(cbm_sem_vec_t));
if (pass1) {
for (int i = 0; i < p->corpus->entry_count; i++) {
pass1[i] = p->corpus->entries[i].enriched_vec;
}
}
if (pass1_q) {
cooccur_int8_ctx_t cc = {
.entries = p->corpus->entries,
.pass1_q = pass1_q,
.doc_token_ids = p->corpus->doc_token_ids,
.doc_token_counts = p->corpus->doc_token_counts,
.rev = p->rev,
.doc_count = p->corpus->doc_count,
.entry_count = p->corpus->entry_count,
.num_chunks = p->num_chunks,
.chunk_size = p->chunk_size,
.tile_size = p->tile_size,
};
atomic_init(&cc.next_chunk, 0);
cbm_parallel_for(p->worker_count, cooccur_worker_int8, &cc, p->opts);
}
if (pass1) {
blend_ctx_t bc = {
.entries = p->corpus->entries,
.pass1 = pass1,
.entry_count = p->corpus->entry_count,
};
atomic_init(&bc.next_idx, 0);
cbm_parallel_for(p->worker_count, blend_worker, &bc, p->opts);
cbm_free(CBM_MEM_CLASS_SEMANTIC, pass1);
}
cbm_free(CBM_MEM_CLASS_SEMANTIC, pass1_q);
norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count};
atomic_init(&nc.next_idx, 0);
cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts);
}
/* The per-document token id lists exist for the co-occurrence pass only;
* released at the end of finalize so the vector phase runs without them
* (783k small blocks on the kernel). */
static void corpus_release_docs(cbm_sem_corpus_t *corpus) {
for (int d = 0; d < corpus->doc_count; d++) {
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids[d]);
corpus->doc_token_ids[d] = NULL;
}
}
void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) {
if (!corpus || corpus->finalized) {
return;
}
/* Eager init before parallel dispatch to avoid lazy-init races */
ensure_pretrained_map();
int worker_count = cbm_default_worker_count(false);
cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false};
/* Finer chunks = better load balancing for skewed token distributions. */
int num_chunks = worker_count * CBM_SEM_COOCCUR_CHUNK;
if (num_chunks > corpus->entry_count) {
num_chunks = corpus->entry_count;
}
if (num_chunks < SKIP_ONE) {
num_chunks = SKIP_ONE;
}
int chunk_size = (corpus->entry_count + num_chunks - SKIP_ONE) / num_chunks;
reverse_index_t *rev = build_reverse_index(corpus);
if (!rev) {
corpus->finalized = true;
return;
}
cbm_sem_src_entry_t *src_entries = cbm_calloc(
CBM_MEM_CLASS_SEMANTIC, (size_t)corpus->entry_count * sizeof(cbm_sem_src_entry_t));
if (!src_entries) {
free_reverse_index(rev);
corpus->finalized = true;
return;
}
finalize_params_t params = {
.corpus = corpus,
.rev = rev,
.src_entries = src_entries,
.worker_count = worker_count,
.num_chunks = num_chunks,
.chunk_size = chunk_size,
.tile_size = CBM_SEM_TILE_SIZE,
.opts = opts,
};
finalize_build_sources(¶ms);
finalize_pass1(¶ms);
finalize_pass2(¶ms);
cbm_free(CBM_MEM_CLASS_SEMANTIC, src_entries);
free_reverse_index(rev);
corpus_release_docs(corpus);
corpus->finalized = true;
}
/* Parse a decimal index string via strtol, returning CBM_NOT_FOUND on parse
* failure. Centralizes the atoi-replacement pattern for token_map lookups. */
static int parse_token_index(const char *idx_str) {
if (!idx_str) {
return CBM_NOT_FOUND;
}
char *end = NULL;
long parsed = strtol(idx_str, &end, BASE_DECIMAL);
return (end != idx_str) ? (int)parsed : CBM_NOT_FOUND;
}
float cbm_sem_corpus_idf(const cbm_sem_corpus_t *corpus, const char *token) {
if (!corpus || !token || corpus->doc_count == 0) {
return 0.0F;
}
int idx = parse_token_index(cbm_ht_get(corpus->token_map, token));
if (idx < 0 || idx >= corpus->entry_count) {
return 0.0F;
}
int df = corpus->entries[idx].doc_freq;
if (df <= 0) {
return 0.0F;
}
return logf((float)corpus->doc_count / (float)df);
}
const cbm_sem_vec_t *cbm_sem_corpus_ri_vec(const cbm_sem_corpus_t *corpus, const char *token) {
if (!corpus || !token) {
return NULL;
}
int idx = parse_token_index(cbm_ht_get(corpus->token_map, token));
if (idx < 0 || idx >= corpus->entry_count) {
return NULL;
}
return &corpus->entries[idx].enriched_vec;
}
int cbm_sem_corpus_token_index(const cbm_sem_corpus_t *corpus, const char *token) {
if (!corpus || !token) {
return CBM_NOT_FOUND;
}
int idx = parse_token_index(cbm_ht_get(corpus->token_map, token));
return (idx < 0 || idx >= corpus->entry_count) ? CBM_NOT_FOUND : idx;
}
/* Same arithmetic as cbm_sem_corpus_idf, by index. */
float cbm_sem_corpus_idf_at(const cbm_sem_corpus_t *corpus, int index) {
if (!corpus || corpus->doc_count == 0 || index < 0 || index >= corpus->entry_count) {
return 0.0F;
}
int df = corpus->entries[index].doc_freq;
if (df <= 0) {
return 0.0F;
}
return logf((float)corpus->doc_count / (float)df);
}
const cbm_sem_vec_t *cbm_sem_corpus_ri_vec_at(const cbm_sem_corpus_t *corpus, int index) {
if (!corpus || index < 0 || index >= corpus->entry_count) {
return NULL;
}
return &corpus->entries[index].enriched_vec;
}
int cbm_sem_corpus_doc_count(const cbm_sem_corpus_t *corpus) {
return corpus ? corpus->doc_count : 0;
}
int cbm_sem_corpus_token_count(const cbm_sem_corpus_t *corpus) {
return corpus ? corpus->entry_count : 0;
}
const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index,
const cbm_sem_vec_t **out_vec, float *out_idf) {
if (!corpus || index < 0 || index >= corpus->entry_count) {
return NULL;
}
if (out_vec) {
*out_vec = &corpus->entries[index].enriched_vec;
}
if (out_idf && corpus->doc_count > 0) {
int df = corpus->entries[index].doc_freq;
*out_idf = df > 0 ? logf((float)corpus->doc_count / (float)df) : 0.0F;
}
return corpus->entries[index].token;
}
static void free_ht_kv(const char *key, void *value, void *userdata) {
(void)userdata;
cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)key);
cbm_free(CBM_MEM_CLASS_SEMANTIC, value);
}
void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus) {
if (!corpus) {
return;
}
for (int i = 0; i < corpus->entry_count; i++) {
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->entries[i].token);
}
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->entries);
for (int d = 0; d < corpus->doc_count; d++) {
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids[d]);
}
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids);
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_counts);
if (corpus->token_map) {
cbm_ht_foreach(corpus->token_map, free_ht_kv, NULL);
cbm_ht_free(corpus->token_map);
}
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus);
}
/* ── Combined scoring ────────────────────────────────────────────── */
float cbm_sem_proximity(const char *path_a, const char *path_b) {
if (!path_a || !path_b) {
return CBM_SEM_UNIT_POS;
}
/* Count shared directory components */
int shared = 0;
int total_a = 0;
int total_b = 0;
const char *a = path_a;
const char *b = path_b;
while (*a && *b && *a == *b) {
if (*a == '/') {
shared++;
}
a++;
b++;
}
for (const char *p = path_a; *p; p++) {
if (*p == '/') {
total_a++;
}
}
for (const char *p = path_b; *p; p++) {
if (*p == '/') {
total_b++;
}
}
int max_total = total_a > total_b ? total_a : total_b;
if (max_total == 0) {
return CBM_SEM_UNIT_POS;
}
float ratio = (float)shared / (float)max_total;
/* Same file = 1.10, same dir = 1.05, distant = 1.00 */
return CBM_SEM_UNIT_POS + (ratio * CBM_SEM_PROX_MAX_BOOST);
}
/* Cosine similarity for small float arrays (AST profile). */
static float small_cosine(const float *a, const float *b, int dims) {
float dot = 0.0F;
float ma = 0.0F;
float mb = 0.0F;
for (int i = 0; i < dims; i++) {
dot += a[i] * b[i];
ma += a[i] * a[i];
mb += b[i] * b[i];
}
float denom = sqrtf(ma) * sqrtf(mb);
return denom < CBM_SEM_DENOM_EPS ? 0.0F : dot / denom;
}
static int cmp_int_asc(const void *pa, const void *pb) {
int a = *(const int *)pa;
int b = *(const int *)pb;
return (a > b) - (a < b);
}
/* Terms are keyed by corpus token, so two functions share a term only when
* they share the word. (Until 2026-10 the key was the token's POSITION in the
* function: any two functions of similar length "shared" their first terms,
* and the TF-IDF signal measured length, not vocabulary.) */
int cbm_sem_tfidf_terms(const cbm_sem_corpus_t *corpus, char *const *tokens, const int *token_index,
int count, int *indices, float *weights) {
if (!corpus || !indices || !weights || count <= 0) {
return 0;
}
int known = 0;
for (int t = 0; t < count; t++) {
int idx = token_index ? token_index[t] : cbm_sem_corpus_token_index(corpus, tokens[t]);
if (idx >= 0 && cbm_sem_corpus_idf_at(corpus, idx) > 0.0F) {
indices[known++] = idx;
}
}
if (known > 1) {
qsort(indices, (size_t)known, sizeof(indices[0]), cmp_int_asc);
}
/* Each run of one index becomes one term; its weight is the idf added once
* per occurrence (equal indices have equal idf). */
int terms = 0;
for (int k = 0; k < known; k++) {
float idf = cbm_sem_corpus_idf_at(corpus, indices[k]);
if (terms > 0 && indices[terms - SKIP_ONE] == indices[k]) {
weights[terms - SKIP_ONE] += idf;
} else {
indices[terms] = indices[k];
weights[terms] = idf;
terms++;
}
}
return terms;
}
/* Sparse cosine over two pre-sorted (index, weight) vectors. Returns 0 when
* either side is empty or the magnitude product is below the epsilon guard. */
static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t *b) {
if (a->tfidf_len <= 0 || b->tfidf_len <= 0) {
return 0.0F;
}
float dot = 0.0F;
float ma = 0.0F;
float mb = 0.0F;
int ia = 0;
int ib = 0;
while (ia < a->tfidf_len && ib < b->tfidf_len) {
if (a->tfidf_indices[ia] == b->tfidf_indices[ib]) {
dot += a->tfidf_weights[ia] * b->tfidf_weights[ib];
ia++;
ib++;
} else if (a->tfidf_indices[ia] < b->tfidf_indices[ib]) {
ia++;
} else {
ib++;
}
}
for (int i = 0; i < a->tfidf_len; i++) {
ma += a->tfidf_weights[i] * a->tfidf_weights[i];
}
for (int i = 0; i < b->tfidf_len; i++) {
mb += b->tfidf_weights[i] * b->tfidf_weights[i];
}
float denom = sqrtf(ma) * sqrtf(mb);
return denom > CBM_SEM_DENOM_EPS ? (dot / denom) : 0.0F;
}
void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
cbm_sem_signals_t *out) {
memset(out, 0, sizeof(*out));
out->proximity = CBM_SEM_UNIT_POS;
if (!a || !b) {
return;
}
if (a->has_minhash && b->has_minhash) {
double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash,
(const cbm_minhash_t *)b->minhash);
out->minhash = (float)j;
out->near_copy = j >= CBM_MINHASH_JACCARD_THRESHOLD;
}
out->tfidf = sparse_tfidf_cosine(a, b);
out->ri = cbm_rsq_ip(&a->ri_code, &b->ri_code); /* signal 2: Random Indexing */
out->api = cbm_rsq_ip(&a->api_code, &b->api_code); /* signal 4: API signatures */
out->type = cbm_rsq_ip(&a->type_code, &b->type_code); /* signal 5: type signatures */
out->decorator = cbm_rsq_ip(&a->deco_code, &b->deco_code); /* signal 7: decorators */
/* signals 8+9+11: structural profile + data flow + Halstead */
out->struct_profile =
small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS);
out->proximity = cbm_sem_proximity(a->file_path, b->file_path); /* signal 6: multiplier */
}
float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg) {
/* Short-circuit: if MinHash Jaccard is already above the SIMILAR_TO threshold,
* the pass_similarity pipeline already emitted a SIMILAR_TO edge for this pair.
* Returning 0 here avoids flooding top-k with cross-service copy-paste boilerplate
* (logging_middleware, shared push/pull handlers) that SIMILAR_TO already covers,
* freeing the edge budget for true semantic leaps (vocabulary-bridged relations). */
if (!s || !cfg || s->near_copy) {
return 0.0F;
}
/* The terms in the historical order (float addition is not associative). */
float score = cfg->w_tfidf * s->tfidf;
score += cfg->w_ri * s->ri;
score += cfg->w_minhash * s->minhash;
score += cfg->w_api * s->api;
score += cfg->w_type * s->type;
score += cfg->w_decorator * s->decorator;
score += cfg->w_struct_profile * s->struct_profile;
/* Signal 6: Module proximity (multiplier, not additive).
* Proximity returns [1.0, 1.10] — a same-file/same-dir boost for ranking.
* Clamp the final product to [0, 1] so the output stays within valid
* cosine-similarity range. Without the clamp, a 0.95 base × 1.10 proximity
* would emit a 1.045 score which violates the semantic of "similarity" and
* breaks downstream consumers that expect a normalized value. */
score *= s->proximity;
if (score > CBM_SEM_UNIT_POS) {
score = CBM_SEM_UNIT_POS;
}
if (score < 0.0F) {
score = 0.0F;
}
return score;
}
typedef struct {
float score;
int pos;
} sem_rank_t;
static int cmp_rank_best_first(const void *pa, const void *pb) {
const sem_rank_t *a = pa;
const sem_rank_t *b = pb;
if (a->score != b->score) {
return a->score > b->score ? -1 : 1;
}
return (a->pos > b->pos) - (a->pos < b->pos);
}
/* Best-first, not first-come: when a function has more candidate partners
* than its budget, it keeps its best ones. Judged on held-out repositories
* (2026-10-07): the pairs first-come admitted over budget were 0 of 17
* related, the ones best-first admits instead 7 of 20. */
bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb,
const bool *eligible, int n, int max_edges, int *counts,
bool *admitted) {
if (n <= 0) {
return true;
}
memset(admitted, 0, (size_t)n * sizeof(admitted[0]));
sem_rank_t *order = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(sem_rank_t));
if (!order) {
return false;
}
int m = 0;
for (int k = 0; k < n; k++) {
if (eligible[k]) {
order[m++] = (sem_rank_t){.score = scores[k], .pos = k};
}
}
qsort(order, (size_t)m, sizeof(order[0]), cmp_rank_best_first);
for (int r = 0; r < m; r++) {
int k = order[r].pos;
if (counts[fa[k]] < max_edges && counts[fb[k]] < max_edges) {
counts[fa[k]]++;
counts[fb[k]]++;
admitted[k] = true;
}
}
cbm_free(CBM_MEM_CLASS_SEMANTIC, order);
return true;
}
/* Until 2026-10 the reader stopped at the first '"' and kept escapes raw:
* a docstring holding a quote lost everything after it, and "\nreturn" gave
* the token "nreturn". */
const char *cbm_sem_json_str(const char *json, const char *key, char *buf, int bufsize) {
if (!json || !key || !buf || bufsize <= 0) {
return NULL;
}
char search[CBM_SZ_64];
snprintf(search, sizeof(search), "\"%s\":\"", key);
const char *p = strstr(json, search);
if (!p) {
return NULL;
}
p += strlen(search);
int n = 0;
for (; *p && *p != '"' && n + SKIP_ONE < bufsize; p++) {
char c = *p;
if (c == '\\' && p[SKIP_ONE]) {
p++;
switch (*p) {
case 'n':
case 't':
case 'r':
case 'b':
case 'f':
c = ' ';
break;
case 'u':
c = ' ';
for (int k = 0; k < 4 && isxdigit((unsigned char)p[SKIP_ONE]); k++) {
p++;
}
break;
default:
c = *p; /* \" \\ \/ */
break;
}
}
buf[n++] = c;
}
buf[n] = '\0';
return buf;
}
/* Score bands at the quartiles of the admitted scores (0.773, 0.802, 0.838 at
* the stored score's three decimals, so one shown score always carries one p),
* each band's p the judged share of related pairs, pooled where a higher band
* judged lower: the upper three pool to one. Judged 2026-10-07 in eleven
* held-out repositories: the pairs this scoring admits that the previous one
* did not (100, blind, three reading rounds) and the earlier samples' pairs it
* keeps (65), each weighted by its share of the admitted set. Ships as the
* edge property "p". */
static const float SEM_P_CUTS[] = {0.773F};
static const float SEM_P_BANDS[] = {0.626F, 0.760F};
float cbm_sem_calibrated_p(float score) {
size_t band = 0;
while (band < sizeof(SEM_P_CUTS) / sizeof(SEM_P_CUTS[0]) && score >= SEM_P_CUTS[band]) {
band++;
}
return SEM_P_BANDS[band];
}
/* Doc -> code candidates by their tfidf: each band's p the judged share of
* sections that are about the function (2026-10-07: 274 blind-judged
* candidates -- a section's best five non-test functions -- from five held-out
* repositories, three reading rounds; the top three measured bands pooled to
* stay monotone). Below the first cut (judged 0.06) nothing is stored. */
enum { SEM_DOC_P_NBANDS = 4 };
static const float SEM_DOC_P_CUTS[SEM_DOC_P_NBANDS - 1] = {CBM_SEM_DOC_MIN_SCORE, 0.30F, 0.40F};
/* Per format, p below the first cut and in each band; 0 = not stored (a
* band judged below 0.20). Blind-judged, three reading rounds, pooled where
* a higher band judged lower:
* Markdown by kind and position since SEM7: SEM_DOC_P_MD_* below (its
* first curve, 2026-10-07: 274 candidates of five repositories,
* 0.06 / 0.36 / 0.42 / 0.55, did not separate positions).
* reST 2026-10-08, 120 candidates, three Sphinx projects; 0.20-0.30
* judged 0.125, so reST starts at 0.30.
* AsciiDoc 2026-10-08, 114 candidates, one repository (the only one of
* its search that qualified).
* PDF no qualifying repository: not measured, nothing stored. */
static const float SEM_DOC_P_BANDS[CBM_SEM_DOC_FMT_COUNT][SEM_DOC_P_NBANDS] = {
[CBM_SEM_DOC_FMT_OTHER] = {0.0F, 0.0F, 0.0F, 0.0F},
[CBM_SEM_DOC_FMT_MARKDOWN] = {0.0F, 0.0F, 0.0F, 0.0F}, /* SEM_DOC_P_MD_* */
[CBM_SEM_DOC_FMT_RST] = {0.0F, 0.0F, 0.35F, 0.625F},
[CBM_SEM_DOC_FMT_ADOC] = {0.0F, 0.225F, 0.25F, 0.50F},
[CBM_SEM_DOC_FMT_PDF] = {0.0F, 0.0F, 0.0F, 0.0F},
};
cbm_sem_doc_format_t cbm_sem_doc_format(const char *path) {
static const struct {
const char *ext;
cbm_sem_doc_format_t fmt;
} exts[] = {{".md", CBM_SEM_DOC_FMT_MARKDOWN}, {".mdx", CBM_SEM_DOC_FMT_MARKDOWN},
{".markdown", CBM_SEM_DOC_FMT_MARKDOWN}, {".rst", CBM_SEM_DOC_FMT_RST},
{".rest", CBM_SEM_DOC_FMT_RST}, {".txt", CBM_SEM_DOC_FMT_RST},
{".adoc", CBM_SEM_DOC_FMT_ADOC}, {".asciidoc", CBM_SEM_DOC_FMT_ADOC},
{".asc", CBM_SEM_DOC_FMT_ADOC}, {".pdf", CBM_SEM_DOC_FMT_PDF}};
const char *dot = path ? strrchr(path, '.') : NULL;
if (!dot || strchr(dot, '/')) {
return CBM_SEM_DOC_FMT_OTHER;
}
for (size_t i = 0; i < sizeof(exts) / sizeof(exts[0]); i++) {
if (strcasecmp(dot, exts[i].ext) == 0) {
return exts[i].fmt;
}
}
return CBM_SEM_DOC_FMT_OTHER;
}
double cbm_sem_p_2dp(float p) {
/* 1e-4 on the x100 scale: a float holds 0.325 as 0.32499998..., 1.2e-6
* below the half; judged values have at most three decimals, so the
* nearest one that must round down (x.xx4) is 0.1 away */
return (double)(long)((double)p * 100.0 + 0.5 + 1e-4) / 100.0;
}
/* Markdown by kind and position (SEM7, 2026-10-08: 490 blind-judged
* candidates from four monorepos with per-component docs, three reading
* rounds, pooled where a higher band judged lower). A doc's position decides
* most: functions outside the doc's home were right 6 of 90 (none stored);
* inside it 0.18 / 0.33 / 0.53. A doc of the whole project: SEM7's and
* sample 4's judged items pooled (the user's choice; 158 items, nine
* repositories): 0.17 / 0.22 / 0.51. Local extras (home functions below the
* top 5) 19 of 60 = 0.32 at every band; whole files 0.12 below 0.40, then
* 0.30. Below 0.20 nothing. */
static const float SEM_DOC_P_MD_FUNCTION[CBM_SEM_DOC_POS_COUNT][SEM_DOC_P_NBANDS] = {
[CBM_SEM_DOC_POS_GLOBAL] = {0.0F, 0.0F, 0.217F, 0.509F},
[CBM_SEM_DOC_POS_LOCAL] = {0.0F, 0.0F, 0.325F, 0.525F},
[CBM_SEM_DOC_POS_OUTSIDE] = {0.0F, 0.0F, 0.0F, 0.0F},
};
static const float SEM_DOC_P_MD_LOCAL[SEM_DOC_P_NBANDS] = {0.0F, 0.317F, 0.317F, 0.317F};
static const float SEM_DOC_P_MD_FILE[SEM_DOC_P_NBANDS] = {0.0F, 0.0F, 0.0F, 0.30F};
float cbm_sem_doc_calibrated_p(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind,
cbm_sem_doc_pos_t pos, float score) {
if ((int)fmt < 0 || fmt >= CBM_SEM_DOC_FMT_COUNT || (int)pos < 0 ||
pos >= CBM_SEM_DOC_POS_COUNT) {
return 0.0F;
}
size_t band = 0;
while (band < SEM_DOC_P_NBANDS - 1 && score >= SEM_DOC_P_CUTS[band]) {
band++;
}
if (fmt != CBM_SEM_DOC_FMT_MARKDOWN) {
/* measured for function candidates only, without position (SEM6) */
return kind == CBM_SEM_DOC_KIND_FUNCTION ? SEM_DOC_P_BANDS[fmt][band] : 0.0F;
}
switch (kind) {
case CBM_SEM_DOC_KIND_FUNCTION:
return SEM_DOC_P_MD_FUNCTION[pos][band];
case CBM_SEM_DOC_KIND_LOCAL:
return pos == CBM_SEM_DOC_POS_LOCAL ? SEM_DOC_P_MD_LOCAL[band] : 0.0F;
case CBM_SEM_DOC_KIND_FILE:
return SEM_DOC_P_MD_FILE[band];
default:
return 0.0F; /* a folder candidate: cbm_sem_doc_folder_p */
}
}
/* A doc's home folder as its candidate (SEM7): a README or index doc was
* right 26 of 30 (0.87); any other doc 5 of 30 (not stored). Markdown only. */
float cbm_sem_doc_folder_p(cbm_sem_doc_format_t fmt, const char *doc_path) {
if (fmt != CBM_SEM_DOC_FMT_MARKDOWN || !doc_path) {
return 0.0F;
}
const char *base = strrchr(doc_path, '/');
base = base ? base + 1 : doc_path;
const char *dot = strrchr(base, '.');
size_t n = dot ? (size_t)(dot - base) : strlen(base);
bool readme = (n == sizeof("readme") - 1 && strncasecmp(base, "readme", n) == 0) ||
(n == sizeof("index") - 1 && strncasecmp(base, "index", n) == 0);
return readme ? 0.867F : 0.0F;
}
cbm_sem_diag_family_t cbm_sem_diag_family(const char *format, const char *diagram_kind) {
static const char *const mermaid[] = {"sequence", "class", "er",
"c4_context", "c4_container", "c4_component",
"c4_dynamic", "c4_deployment", "architecture"};
static const char *const plantuml[] = {"c4", "class", "sequence", "usecase", "component"};
if (!format || !format[0]) {
return CBM_SEM_DIAG_NONE;
}
const char *const *structural = NULL;
size_t n = 0;
if (strcmp(format, "mermaid") == 0) {
structural = mermaid;
n = sizeof(mermaid) / sizeof(mermaid[0]);
} else if (strcmp(format, "plantuml") == 0) {
structural = plantuml;
n = sizeof(plantuml) / sizeof(plantuml[0]);
} else {
return CBM_SEM_DIAG_GRAPH;
}
for (size_t i = 0; diagram_kind && i < n; i++) {
if (strcmp(diagram_kind, structural[i]) == 0) {
return CBM_SEM_DIAG_STRUCTURAL;
}
}
return CBM_SEM_DIAG_PROSE;
}
/* Markdown sections that embed a diagram, by the section's family (DIAG-C,
* 2026-10-11: 553 blind-judged candidates from sections of 16 held-out
* repositories that embed a diagram, three reading rounds; function
* candidates at or inside the home pooled, as both curves rose by band).
* A diagram section is about its candidates more often than SEM7's Markdown
* curve says, so its first band is stored too:
* prose 0.24 / 0.28 / 0.45, local extras 11 of 40 = 0.275
* structural 0.32 / 0.46 / 0.57, local extras 7 of 9 = 0.78 (every
* local extra of the family; 95% interval 0.45-0.94)
* Graph diagrams (DOT, D2) were not found in Markdown sections: SEM7. */
static const struct {
bool fitted;
float function[SEM_DOC_P_NBANDS];
float local;
} SEM_DOC_P_MD_DIAGRAM[CBM_SEM_DIAG_COUNT] = {
[CBM_SEM_DIAG_PROSE] = {true, {0.0F, 0.24F, 0.28F, 0.45F}, 0.275F},
[CBM_SEM_DIAG_STRUCTURAL] = {true, {0.0F, 0.32F, 0.464F, 0.567F}, 0.778F},
};
float cbm_sem_doc_section_p(cbm_sem_doc_format_t fmt, cbm_sem_diag_family_t fam,
cbm_sem_doc_kind_t kind, cbm_sem_doc_pos_t pos, float score) {
bool fitted = fmt == CBM_SEM_DOC_FMT_MARKDOWN && (int)fam > 0 && fam < CBM_SEM_DIAG_COUNT &&
SEM_DOC_P_MD_DIAGRAM[fam].fitted;
bool home = pos == CBM_SEM_DOC_POS_GLOBAL || pos == CBM_SEM_DOC_POS_LOCAL;
if (!fitted || !((kind == CBM_SEM_DOC_KIND_FUNCTION && home) ||
(kind == CBM_SEM_DOC_KIND_LOCAL && pos == CBM_SEM_DOC_POS_LOCAL))) {
return cbm_sem_doc_calibrated_p(fmt, kind, pos, score);
}
size_t band = 0;
while (band < SEM_DOC_P_NBANDS - 1 && score >= SEM_DOC_P_CUTS[band]) {
band++;
}
if (band == 0) {
return 0.0F; /* below CBM_SEM_DOC_MIN_SCORE: never measured, nothing stored */
}
return kind == CBM_SEM_DOC_KIND_LOCAL ? SEM_DOC_P_MD_DIAGRAM[fam].local
: SEM_DOC_P_MD_DIAGRAM[fam].function[band];
}
float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
const cbm_sem_config_t *cfg) {
if (!a || !b || !cfg) {
return 0.0F;
}
cbm_sem_signals_t s;
cbm_sem_signal_values(a, b, &s);
return cbm_sem_combine(&s, cfg);
}
/* ── Graph diffusion ─────────────────────────────────────────────── */
void cbm_sem_diffuse(cbm_sem_vec_t *combined, const cbm_sem_vec_t *neighbors, int neighbor_count,
float alpha) {
if (!combined || !neighbors || neighbor_count <= 0) {
return;
}
/* Blend: combined = (1-α) × combined + α × mean(neighbors) */
cbm_sem_vec_t mean;
memset(&mean, 0, sizeof(mean));
for (int n = 0; n < neighbor_count; n++) {
for (int i = 0; i < CBM_SEM_DIM; i++) {
mean.v[i] += neighbors[n].v[i];
}
}
float inv_n = CBM_SEM_UNIT_POS / (float)neighbor_count;
float one_minus_alpha = CBM_SEM_UNIT_POS - alpha;
for (int i = 0; i < CBM_SEM_DIM; i++) {
combined->v[i] = (one_minus_alpha * combined->v[i]) + (alpha * mean.v[i] * inv_n);
}
cbm_sem_normalize(combined);
}