Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

74.9 KB
Raw Blame
/*
 * semantic.c — Algorithmic code embeddings: TF-IDF, Random Indexing,
 * API/Type/Decorator signatures, combined scoring, graph diffusion.
 *
 * All signals computed from graph buffer metadata — no source file reads.
 * Uses xxHash for deterministic random vectors. Pure C, zero dependencies.
 */
#include "semantic/semantic.h"
#include "foundation/mem_core.h"
#include "foundation/constants.h"
#include "foundation/hash_table.h"
#include "foundation/log.h"
#include "foundation/profile.h"
#include "foundation/platform.h"
#include "foundation/compat_thread.h"
#include "pipeline/worker_pool.h"
#include "simhash/minhash.h"
#include "nomic/code_vectors.h"

#define XXH_INLINE_ALL
#include "xxhash/xxhash.h"

#include <ctype.h>
#include <math.h>
#include <stdatomic.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

/* ── Constants ───────────────────────────────────────────────────── */

enum {
    TOKEN_BUF_LEN = 128,
    CORPUS_INIT_CAP = 4096,
    DOC_TOKENS_INIT = 64,
    RI_SEED_BASE = 0x52494E44, /* "RIND" */
};

/* Default signal weights for cbm_sem_combined_score.  Must sum to ~1.0;
 * proximity is a multiplier applied on top. */
#define CBM_SEM_W_TFIDF 0.20F
#define CBM_SEM_W_RI 0.25F
#define CBM_SEM_W_MINHASH 0.10F
#define CBM_SEM_W_API 0.15F
#define CBM_SEM_W_TYPE 0.10F
#define CBM_SEM_W_DECORATOR 0.05F
#define CBM_SEM_W_STRUCT_PROFILE 0.10F
#define CBM_SEM_W_DATAFLOW 0.05F

/* Threshold bounds for CBM_SEMANTIC_THRESHOLD env override. */
#define CBM_SEM_THRESHOLD_MIN 0.0F
#define CBM_SEM_THRESHOLD_MAX 1.0F

/* Epsilon for denominator guards in cosine math. */
#define CBM_SEM_DENOM_EPS 1e-10F

/* Int8 quantization bounds for vector storage (maps [-1.0, 1.0] to [-127, 127]). */
#define CBM_SEM_INT8_MAX 127.0F
#define CBM_SEM_INT8_MIN (-127.0F)

/* Unit vector bounds (normalized similarity / cosine value limits). */
#define CBM_SEM_UNIT_POS 1.0F
#define CBM_SEM_UNIT_NEG (-1.0F)

/* Proximity boost: same-file gets 1.0 + CBM_SEM_PROX_MAX_BOOST, distant gets 1.0. */
#define CBM_SEM_PROX_MAX_BOOST 0.10F

/* Reflective Random Indexing blend factors for pass2 (context mixing). */
#define CBM_SEM_RRI_ALPHA 0.3F
#define CBM_SEM_RRI_BETA 0.7F

/* IDF smoothing constant (log base): log2(1 + docs / doc_freq). */
#define CBM_SEM_IDF_BASE 0.5F

/* Strings of exactly two parts / dimensions used for co-occurrence window math. */
#define CBM_SEM_COOCCUR_STRIDE 2

/* Worker tile/chunk sizes for parallel cooccurrence passes. */
enum {
    CBM_SEM_WORKER_STACK_CAP = 256,
    CBM_SEM_TILE_SIZE = 40,
    CBM_SEM_SEEN_INIT_CAP = 256,
    CBM_SEM_RESOLVE_CHUNK = 64,
    CBM_SEM_COOCCUR_CHUNK = 32,
    CBM_SEM_RRI_TILE = 128,
    CBM_SEM_INT8_I_LO = -128,
    CBM_SEM_INT8_I_HI = 127,
    CBM_SEM_ATOMIC_INC = 1,
    CBM_SEM_RI_NONZERO_COUNT = 8,
};

/* Mutex-init state machine for the lazy-initialized pretrained token map.
 * State 0 = not started, 1 = initializing, 2 = initialized. */
enum {
    MTX_STATE_UNINIT = 0,
    MTX_STATE_INITIALIZING = 1,
    MTX_STATE_READY = 2,
};

/* Pretrained map ready flag (signals a one-shot atomic transition 0 → 1). */
enum { MAP_READY = 1 };

/* Numeric conversion radix for strtol (base 10 decimal). */
enum { BASE_DECIMAL = 10 };

/* ── Configuration ───────────────────────────────────────────────── */

cbm_sem_config_t cbm_sem_get_config(void) {
    cbm_sem_config_t cfg = {
        .w_tfidf = CBM_SEM_W_TFIDF,
        .w_ri = CBM_SEM_W_RI,
        .w_minhash = CBM_SEM_W_MINHASH,
        .w_api = CBM_SEM_W_API,
        .w_type = CBM_SEM_W_TYPE,
        .w_decorator = CBM_SEM_W_DECORATOR,
        .w_struct_profile = CBM_SEM_W_STRUCT_PROFILE,
        .w_dataflow = CBM_SEM_W_DATAFLOW,
        .threshold = (float)CBM_SEM_EDGE_THRESHOLD,
        .max_edges = CBM_SEM_MAX_EDGES,
    };
    const char *thresh = cbm_runtime_getenv("CBM_SEMANTIC_THRESHOLD");
    if (thresh) {
        /* strtod reports errors via endptr; reject non-numeric input silently. */
        char *end = NULL;
        double parsed = strtod(thresh, &end);
        if (end != thresh && parsed > CBM_SEM_THRESHOLD_MIN && parsed <= CBM_SEM_THRESHOLD_MAX) {
            cfg.threshold = (float)parsed;
        }
    }
    return cfg;
}

bool cbm_sem_is_enabled(void) {
    const char *val = cbm_runtime_getenv("CBM_SEMANTIC_ENABLED");
    return val && val[0] == '1';
}

/* ── Token extraction ────────────────────────────────────────────── */

/* True for characters that terminate a token regardless of case. */
static bool is_token_delim(char c) {
    return c == '.' || c == '/' || c == '_' || c == '-' || c == ' ' || c == '(' || c == ')' ||
           c == ',' || c == ':';
}

/* True for a camelCase transition: uppercase letter preceded by a lowercase. */
static bool is_camel_break(const char *name, int i) {
    if (i <= 0) {
        return false;
    }
    char c = name[i];
    char p = name[i - SKIP_ONE];
    return c >= 'A' && c <= 'Z' && p >= 'a' && p <= 'z';
}

/* Flush the current buffer as a token into out[]. */
static void flush_token(char *buf, int *blen, char **out, int *count, int max_out) {
    if (*blen > 0 && *count < max_out) {
        buf[*blen] = '\0';
        out[(*count)++] = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, buf);
    }
    *blen = 0;
}

int cbm_sem_tokenize(const char *name, char **out, int max_out) {
    if (!name || !out || max_out <= 0) {
        return 0;
    }
    int count = 0;
    char buf[TOKEN_BUF_LEN];
    int blen = 0;

    for (int i = 0; name[i] && count < max_out; i++) {
        char c = name[i];
        bool split = is_token_delim(c);
        bool camel = is_camel_break(name, i);
        if (split || camel) {
            flush_token(buf, &blen, out, &count, max_out);
            if (split) {
                continue;
            }
        }
        if (blen < TOKEN_BUF_LEN - SKIP_ONE && isalnum((unsigned char)c)) {
            buf[blen++] = (char)tolower((unsigned char)c);
        }
    }
    flush_token(buf, &blen, out, &count, max_out);

    /* Abbreviation expansion: add expanded forms for common code abbreviations.
     * "err" → also add "error", "ctx" → "context", etc. */
    /* Cross-language abbreviation table — covers Go, Python, JS/TS, Rust,
     * Java, C/C++, Ruby, PHP, Kotlin, Swift, Scala, C#, and common patterns. */
    static const struct {
        const char *abbrev;
        const char *expanded;
    } abbrevs[] = {
        /* Error/exception handling */
        {"err", "error"},
        {"exc", "exception"},
        {"ex", "exception"},
        /* Context/config */
        {"ctx", "context"},
        {"cfg", "config"},
        {"conf", "configuration"},
        {"env", "environment"},
        {"opt", "option"},
        {"opts", "options"},
        /* Request/response (HTTP, RPC) */
        {"req", "request"},
        {"res", "response"},
        {"resp", "response"},
        {"rsp", "response"},
        {"hdr", "header"},
        {"hdrs", "headers"},
        /* Strings/formatting */
        {"str", "string"},
        {"fmt", "format"},
        {"msg", "message"},
        {"txt", "text"},
        {"lbl", "label"},
        {"desc", "description"},
        /* Data structures */
        {"buf", "buffer"},
        {"arr", "array"},
        {"vec", "vector"},
        {"lst", "list"},
        {"dict", "dictionary"},
        {"tbl", "table"},
        {"stk", "stack"},
        {"que", "queue"},
        /* Functions/callbacks */
        {"fn", "function"},
        {"func", "function"},
        {"cb", "callback"},
        {"proc", "procedure"},
        {"ctor", "constructor"},
        {"dtor", "destructor"},
        /* Database/storage */
        {"db", "database"},
        {"col", "column"},
        {"tbl", "table"},
        {"stmt", "statement"},
        {"txn", "transaction"},
        {"trx", "transaction"},
        {"repo", "repository"},
        /* Auth/security */
        {"auth", "authentication"},
        {"authz", "authorization"},
        {"perm", "permission"},
        {"cred", "credential"},
        {"tok", "token"},
        {"pwd", "password"},
        /* Values/types */
        {"val", "value"},
        {"num", "number"},
        {"int", "integer"},
        {"bool", "boolean"},
        {"flt", "float"},
        {"dbl", "double"},
        /* Indexing/iteration */
        {"idx", "index"},
        {"iter", "iterator"},
        {"elem", "element"},
        {"cnt", "count"},
        {"len", "length"},
        {"sz", "size"},
        {"pos", "position"},
        {"off", "offset"},
        {"cap", "capacity"},
        /* Lifecycle */
        {"init", "initialize"},
        {"deinit", "deinitialize"},
        {"alloc", "allocate"},
        {"dealloc", "deallocate"},
        {"del", "delete"},
        {"rm", "remove"},
        /* Implementation/interface */
        {"impl", "implementation"},
        {"iface", "interface"},
        {"abs", "abstract"},
        {"decl", "declaration"},
        /* Parameters/attributes */
        {"param", "parameter"},
        {"arg", "argument"},
        {"attr", "attribute"},
        {"prop", "property"},
        {"ret", "return"},
        /* Source/destination */
        {"src", "source"},
        {"dst", "destination"},
        {"tgt", "target"},
        {"orig", "original"},
        {"prev", "previous"},
        {"cur", "current"},
        {"tmp", "temporary"},
        {"temp", "temporary"},
        /* Networking/IO */
        {"conn", "connection"},
        {"sess", "session"},
        {"sock", "socket"},
        {"addr", "address"},
        {"url", "uniform"},
        {"srv", "server"},
        {"cli", "client"},
        {"svc", "service"},
        {"ep", "endpoint"},
        /* Management */
        {"mgr", "manager"},
        {"ctrl", "controller"},
        {"hdlr", "handler"},
        {"sched", "scheduler"},
        {"disp", "dispatcher"},
        {"reg", "registry"},
        /* Async/concurrent */
        {"chan", "channel"},
        {"sem", "semaphore"},
        {"mtx", "mutex"},
        {"wg", "waitgroup"},
        {"sig", "signal"},
        {"evt", "event"},
        {"sub", "subscriber"},
        {"pub", "publisher"},
        /* Testing */
        {"spec", "specification"},
        {"mock", "mock"},
        {"stub", "stub"},
        {"assert", "assertion"},
        /* Logging/monitoring */
        {"log", "logging"},
        {"lvl", "level"},
        {"dbg", "debug"},
        {"wrn", "warning"},
        {"inf", "info"},
        /* Time */
        {"ts", "timestamp"},
        {"dur", "duration"},
        {"ttl", "timetolive"},
        /* Miscellaneous */
        {"ver", "version"},
        {"ns", "namespace"},
        {"pkg", "package"},
        {"mod", "module"},
        {"lib", "library"},
        {"dep", "dependency"},
        {"ref", "reference"},
        {"ptr", "pointer"},
        {"obj", "object"},
        {"doc", "document"},
        {"cmd", "command"},
        {"ops", "operations"},
        {"util", "utility"},
        {"hlp", "helper"},
        {"ext", "extension"},
        {NULL, NULL},
    };
    int orig_count = count;
    for (int t = 0; t < orig_count && count < max_out; t++) {
        for (int a = 0; abbrevs[a].abbrev; a++) {
            if (strcmp(out[t], abbrevs[a].abbrev) == 0) {
                out[count++] = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, abbrevs[a].expanded);
                break;
            }
        }
    }

    return count;
}

/* ── Dense vector operations ─────────────────────────────────────── */

float cbm_sem_cosine(const cbm_sem_vec_t *a, const cbm_sem_vec_t *b) {
    if (!a || !b) {
        return 0.0F;
    }
    float dot = 0.0F;
    float mag_a = 0.0F;
    float mag_b = 0.0F;
    for (int i = 0; i < CBM_SEM_DIM; i++) {
        dot += a->v[i] * b->v[i];
        mag_a += a->v[i] * a->v[i];
        mag_b += b->v[i] * b->v[i];
    }
    float denom = sqrtf(mag_a) * sqrtf(mag_b);
    if (denom < CBM_SEM_DENOM_EPS) {
        return 0.0F;
    }
    return dot / denom;
}

/* Pretrained token lookup table — built lazily on first use. */
static CBMHashTable *g_pretrained_map = NULL;
static _Atomic int g_pretrained_ready = 0;
static cbm_mutex_t g_pretrained_mtx;
static _Atomic int g_pretrained_mtx_init = 0;

/* Thread-safe lazy init of the pretrained token lookup map.
 * Uses double-checked locking: fast path reads an atomic flag. */
static void ensure_pretrained_map(void) {
    if (atomic_load_explicit(&g_pretrained_ready, memory_order_acquire)) {
        return;
    }
    /* First-time init of the mutex itself (also needs to be thread-safe) */
    int expected = MTX_STATE_UNINIT;
    if (atomic_compare_exchange_strong_explicit(&g_pretrained_mtx_init, &expected,
                                                MTX_STATE_INITIALIZING, memory_order_acq_rel,
                                                memory_order_acquire)) {
        cbm_mutex_init(&g_pretrained_mtx);
        atomic_store_explicit(&g_pretrained_mtx_init, MTX_STATE_READY, memory_order_release);
    } else {
        /* Spin until another thread finishes initializing the mutex */
        while (atomic_load_explicit(&g_pretrained_mtx_init, memory_order_acquire) !=
               MTX_STATE_READY) {
            /* brief spin */
        }
    }
    cbm_mutex_lock(&g_pretrained_mtx);
    if (!atomic_load_explicit(&g_pretrained_ready, memory_order_acquire)) {
        g_pretrained_map = cbm_ht_create(PRETRAINED_TOKEN_COUNT);
        char idx_buf[CBM_SZ_16];
        for (int i = 0; i < PRETRAINED_TOKEN_COUNT; i++) {
            const char *tok = PRETRAINED_TOKENS[i];
            if (tok && tok[0]) {
                snprintf(idx_buf, sizeof(idx_buf), "%d", i);
                cbm_ht_set(g_pretrained_map, cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, tok),
                           cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, idx_buf));
            }
        }
        atomic_store_explicit(&g_pretrained_ready, MAP_READY, memory_order_release);
    }
    cbm_mutex_unlock(&g_pretrained_mtx);
}

void cbm_sem_ensure_ready(void) {
    ensure_pretrained_map();
}

void cbm_sem_random_index(const char *token, cbm_sem_vec_t *out) {
    memset(out, 0, sizeof(*out));
    if (!token) {
        return;
    }

    /* Try pretrained nomic-embed-code vector first (768d, distilled from 7B). */
    ensure_pretrained_map();
    const char *idx_str = cbm_ht_get(g_pretrained_map, token);
    if (idx_str) {
        char *end = NULL;
        long idx = strtol(idx_str, &end, BASE_DECIMAL);
        if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) {
            const int8_t *pvec = pretrained_vec_at((int)idx);
            for (int d = 0; d < CBM_SEM_DIM && d < PRETRAINED_DIM; d++) {
                out->v[d] = (float)pvec[d] / CBM_SEM_INT8_MAX;
            }
            return;
        }
    }

    /* Fallback: sparse random vector for tokens not in pretrained vocab. */
    uint64_t seed = XXH3_64bits(token, strlen(token));
    for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) {
        uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE);
        int pos = (int)(h % CBM_SEM_DIM);
        float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS;
        out->v[pos] += sign;
    }
}

void cbm_sem_normalize(cbm_sem_vec_t *v) {
    if (!v) {
        return;
    }
    float mag = 0.0F;
    for (int i = 0; i < CBM_SEM_DIM; i++) {
        mag += v->v[i] * v->v[i];
    }
    mag = sqrtf(mag);
    if (mag < CBM_SEM_DENOM_EPS) {
        return;
    }
    float inv = CBM_SEM_UNIT_POS / mag;
    for (int i = 0; i < CBM_SEM_DIM; i++) {
        v->v[i] *= inv;
    }
}

void cbm_sem_vec_add_scaled(cbm_sem_vec_t *dst, const cbm_sem_vec_t *src, float scale) {
    if (!dst || !src) {
        return;
    }
    for (int i = 0; i < CBM_SEM_DIM; i++) {
        dst->v[i] += scale * src->v[i];
    }
}

/* ── Corpus (IDF + Random Indexing enrichment) ───────────────────── */

typedef struct {
    char *token;
    int doc_freq;
    cbm_sem_vec_t enriched_vec; /* context-enriched via co-occurrence */
} corpus_entry_t;

struct cbm_sem_corpus {
    CBMHashTable *token_map; /* token → index into entries[] */
    corpus_entry_t *entries;
    int entry_count;
    int entry_cap;
    int doc_count;
    bool finalized;

    /* Per-document token lists for co-occurrence pass */
    int **doc_token_ids;
    int *doc_token_counts;
    int doc_cap;
};

static int corpus_get_or_add(cbm_sem_corpus_t *c, const char *token) {
    char idx_buf[CBM_SZ_16];
    const char *existing = cbm_ht_get(c->token_map, token);
    if (existing) {
        char *end = NULL;
        long parsed = strtol(existing, &end, BASE_DECIMAL);
        return (end != existing) ? (int)parsed : CBM_NOT_FOUND;
    }
    if (c->entry_count >= c->entry_cap) {
        int new_cap = c->entry_cap < CORPUS_INIT_CAP ? CORPUS_INIT_CAP : c->entry_cap * PAIR_LEN;
        corpus_entry_t *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, c->entries,
                                            (size_t)new_cap * sizeof(corpus_entry_t));
        if (!grown) {
            return CBM_NOT_FOUND;
        }
        c->entries = grown;
        c->entry_cap = new_cap;
    }
    int idx = c->entry_count++;
    c->entries[idx].token = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, token);
    c->entries[idx].doc_freq = 0;
    memset(&c->entries[idx].enriched_vec, 0, sizeof(cbm_sem_vec_t));
    snprintf(idx_buf, sizeof(idx_buf), "%d", idx);
    cbm_ht_set(c->token_map, cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, token),
               cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, idx_buf));
    return idx;
}

cbm_sem_corpus_t *cbm_sem_corpus_new(void) {
    cbm_sem_corpus_t *c = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, sizeof(cbm_sem_corpus_t));
    if (c) {
        c->token_map = cbm_ht_create(CORPUS_INIT_CAP);
    }
    return c;
}

void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int count) {
    if (!corpus || !tokens || count <= 0) {
        return;
    }
    /* Track document for co-occurrence pass */
    if (corpus->doc_count >= corpus->doc_cap) {
        int new_cap =
            corpus->doc_cap < DOC_TOKENS_INIT ? DOC_TOKENS_INIT : corpus->doc_cap * PAIR_LEN;
        int **grown_ids = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids,
                                      (size_t)new_cap * sizeof(int *));
        int *grown_counts = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_counts,
                                        (size_t)new_cap * sizeof(int));
        if (!grown_ids || !grown_counts) {
            cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_ids);
            cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_counts);
            return;
        }
        corpus->doc_token_ids = grown_ids;
        corpus->doc_token_counts = grown_counts;
        corpus->doc_cap = new_cap;
    }
    int doc_idx = corpus->doc_count++;
    corpus->doc_token_ids[doc_idx] = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)count * sizeof(int));
    corpus->doc_token_counts[doc_idx] = count;

    /* Per-doc unique set for IDF */
    int *seen =
        cbm_calloc(CBM_MEM_CLASS_SEMANTIC,
                   ((size_t)corpus->entry_cap + (size_t)count + CORPUS_INIT_CAP) * sizeof(int));
    int seen_count = 0;

    for (int i = 0; i < count; i++) {
        int tid = corpus_get_or_add(corpus, tokens[i]);
        corpus->doc_token_ids[doc_idx][i] = tid;
        if (tid < 0) {
            continue;
        }
        /* Check uniqueness for IDF (simple linear scan — tokens per doc is small) */
        bool is_new = true;
        for (int j = 0; j < seen_count; j++) {
            if (seen[j] == tid) {
                is_new = false;
                break;
            }
        }
        if (is_new) {
            seen[seen_count++] = tid;
            corpus->entries[tid].doc_freq++;
        }
    }
    cbm_free(CBM_MEM_CLASS_SEMANTIC, seen);
}

/* ── Parallel corpus batch build ──────────────────────────────────── */
/* Strategy:
 *   Phase A (SEQUENTIAL): Scan all documents once to build the global
 *     token_map (inserts unique tokens, assigns global IDs). This is
 *     inherently sequential (hash table mutation), but much faster than
 *     the current per-doc add_doc because we avoid the per-doc malloc of
 *     the `seen` array and per-doc bookkeeping.
 *   Phase B (PARALLEL): Each worker processes a chunk of docs, translates
 *     tokens → global IDs via read-only token_map lookups, fills
 *     doc_token_ids[d], and accumulates doc_freq contributions via atomics.
 */

typedef struct {
    cbm_sem_corpus_t *corpus;
    char **all_tokens;
    const size_t *offsets; /* document d = all_tokens[offsets[d] ..] */
    const int *token_counts;
    int doc_count;
    _Atomic int *doc_freq_atomic; /* per-entry atomic counter (entry_count long) */
    _Atomic int next_idx;
} batch_resolve_ctx_t;

/* Resolve one document: look up each token's global ID, fill the corpus
 * doc_token_ids[d], and bump the per-token doc_freq counter atomically.  The
 * caller is responsible for ensuring `seen` has capacity for `count` ints
 * before calling (the worker grows its per-thread scratch buffer). */
static void batch_resolve_one_doc(batch_resolve_ctx_t *bc, int doc_index, int *seen) {
    int count = bc->token_counts[doc_index];
    if (count <= 0) {
        bc->corpus->doc_token_ids[doc_index] = NULL;
        bc->corpus->doc_token_counts[doc_index] = 0;
        return;
    }
    int *ids = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)count * sizeof(int));
    bc->corpus->doc_token_ids[doc_index] = ids;
    bc->corpus->doc_token_counts[doc_index] = count;

    int seen_count = 0;
    char **tokens = &bc->all_tokens[bc->offsets[doc_index]];
    for (int i = 0; i < count; i++) {
        const char *idx_str = cbm_ht_get(bc->corpus->token_map, tokens[i]);
        int tid = CBM_NOT_FOUND;
        if (idx_str) {
            char *end = NULL;
            long parsed = strtol(idx_str, &end, BASE_DECIMAL);
            if (end != idx_str) {
                tid = (int)parsed;
            }
        }
        ids[i] = tid;
        if (tid < 0) {
            continue;
        }
        /* Unique-per-doc check for IDF */
        bool is_new = true;
        for (int j = 0; j < seen_count; j++) {
            if (seen[j] == tid) {
                is_new = false;
                break;
            }
        }
        if (is_new) {
            seen[seen_count++] = tid;
            atomic_fetch_add_explicit(&bc->doc_freq_atomic[tid], CBM_SEM_ATOMIC_INC,
                                      memory_order_relaxed);
        }
    }
}

static void batch_resolve_worker(int worker_id, void *ctx_ptr) {
    (void)worker_id;
    batch_resolve_ctx_t *bc = ctx_ptr;
    /* Per-worker scratch for unique-per-doc tracking */
    int local_seen_cap = CBM_SEM_SEEN_INIT_CAP;
    int *seen = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)local_seen_cap * sizeof(int));
    if (!seen) {
        return;
    }

    while (true) {
        int start =
            atomic_fetch_add_explicit(&bc->next_idx, CBM_SEM_RESOLVE_CHUNK, memory_order_relaxed);
        if (start >= bc->doc_count) {
            break;
        }
        int end = start + CBM_SEM_RESOLVE_CHUNK;
        if (end > bc->doc_count) {
            end = bc->doc_count;
        }
        for (int d = start; d < end; d++) {
            int count = bc->token_counts[d];
            if (count > local_seen_cap) {
                int *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, seen, (size_t)count * sizeof(int));
                if (!grown) {
                    continue;
                }
                seen = grown;
                local_seen_cap = count;
            }
            batch_resolve_one_doc(bc, d, seen);
        }
    }
    cbm_free(CBM_MEM_CLASS_SEMANTIC, seen);
}

void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens,
                                   const size_t *offsets, const int *token_counts, int doc_count) {
    if (!corpus || !all_tokens || !offsets || !token_counts || doc_count <= 0) {
        return;
    }

    /* Phase A (SEQUENTIAL): Build token_map and allocate doc arrays.
     * Hash table mutation can't be parallelized; strdup+insert is the cost. */
    if (corpus->doc_cap < corpus->doc_count + doc_count) {
        int new_cap = corpus->doc_count + doc_count;
        int **grown_ids = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids,
                                      (size_t)new_cap * sizeof(int *));
        int *grown_counts = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_counts,
                                        (size_t)new_cap * sizeof(int));
        if (!grown_ids || !grown_counts) {
            cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_ids);
            cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_counts);
            return;
        }
        corpus->doc_token_ids = grown_ids;
        corpus->doc_token_counts = grown_counts;
        corpus->doc_cap = new_cap;
    }
    int base_doc = corpus->doc_count;
    corpus->doc_count += doc_count;

    for (int d = 0; d < doc_count; d++) {
        int count = token_counts[d];
        char **tokens = &all_tokens[offsets[d]];
        for (int i = 0; i < count; i++) {
            /* Inserts token into token_map if new; we discard return here —
             * Phase B will re-lookup in read-only mode to get the ID. */
            (void)corpus_get_or_add(corpus, tokens[i]);
        }
    }

    /* Phase B (PARALLEL): Resolve tokens → IDs and count doc_freq per entry.
     * token_map is now read-only; each worker owns its doc range (no writes
     * to shared state except atomic doc_freq counters). */
    _Atomic int *doc_freq_atomic =
        cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)corpus->entry_count * sizeof(_Atomic int));
    if (!doc_freq_atomic) {
        /* OOM fallback: sequential path. Roll back doc_count first since
         * add_doc increments it itself. */
        corpus->doc_count = base_doc;
        for (int d = 0; d < doc_count; d++) {
            int count = token_counts[d];
            char **tokens = &all_tokens[offsets[d]];
            cbm_sem_corpus_add_doc(corpus, (const char **)tokens, count);
        }
        return;
    }

    int worker_count = cbm_default_worker_count(false);
    batch_resolve_ctx_t bc = {
        .corpus = corpus,
        .all_tokens = all_tokens,
        .token_counts = token_counts,
        .offsets = offsets,
        .doc_count = doc_count,
        .doc_freq_atomic = doc_freq_atomic,
    };
    atomic_init(&bc.next_idx, 0);
    /* Temporarily re-base doc arrays so workers write to base_doc..base_doc+doc_count */
    corpus->doc_token_ids += base_doc;
    corpus->doc_token_counts += base_doc;
    cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false};
    cbm_parallel_for(worker_count, batch_resolve_worker, &bc, opts);
    corpus->doc_token_ids -= base_doc;
    corpus->doc_token_counts -= base_doc;

    /* Phase C (SEQUENTIAL reduce): atomic counters → entries[].doc_freq */
    for (int i = 0; i < corpus->entry_count; i++) {
        corpus->entries[i].doc_freq +=
            atomic_load_explicit(&doc_freq_atomic[i], memory_order_relaxed);
    }
    cbm_free(CBM_MEM_CLASS_SEMANTIC, doc_freq_atomic);
}

/* ── Parallel corpus_finalize ─────────────────────────────────────── */
/* Strategy:
 *   1. Precompute base RI vectors into a shared array (eliminates ~333M
 *      redundant cbm_sem_random_index calls on kernel-scale corpora).
 *   2. Co-occurrence passes: partition TARGET tokens across workers so each
 *      worker writes to a disjoint range of enriched_vec (zero contention).
 *      Each worker still scans all documents but only accumulates for targets
 *      in its range. Inner vector add is the parallelized work.
 *   3. Normalize/blend loops are trivially parallel per-entry.
 */

/* Reverse index: for each token id, a list of (doc_id, position_in_doc) pairs.
 * Built once, reused for both cooccur passes. Eliminates the O(num_chunks × doc_count)
 * redundant outer scan in the old algorithm. */
typedef struct {
    int32_t doc_id;
    int32_t pos;
} cooccur_pos_t;

typedef struct {
    int *offsets;        /* offsets[entry_count + 1], prefix sum of occurrences */
    cooccur_pos_t *flat; /* flat array of positions, total = offsets[entry_count] */
} reverse_index_t;

/* Tagged source vector: sparse (~30% of tokens, 8 inline nonzeros) or dense int8
 * reference into PRETRAINED_VECTOR_BLOB (no copy). Dense path converts int8→float
 * on the fly in the hot loop. Tested with both int8 and float32 blob storage
 * formats — cooccur passes are memory-bandwidth-bound, so the int8 format (4x
 * less source traffic) is equivalent in wall time despite the conversion cost,
 * while saving 90 MB of binary size. */
typedef struct {
    uint8_t is_sparse; /* 1 = sparse path, 0 = dense int8 reference */
    uint8_t nnz;       /* number of nonzeros used in sparse path */
    uint16_t _pad;
    uint16_t indices[CBM_SEM_SPARSE_NNZE]; /* 8 * 2 = 16 bytes */
    float values[CBM_SEM_SPARSE_NNZE];     /* 8 * 4 = 32 bytes */
    const int8_t *dense_int8;              /* points into PRETRAINED_VECTOR_BLOB */
} cbm_sem_src_entry_t;

/* Inline helper: initialize a target vector from a sparse/dense source. */
static inline void sem_target_init_from_src(cbm_sem_vec_t *dst, const cbm_sem_src_entry_t *src) {
    memset(dst, 0, sizeof(*dst));
    if (src->is_sparse) {
        for (int k = 0; k < src->nnz; k++) {
            dst->v[src->indices[k]] = src->values[k];
        }
    } else {
        const int8_t *s = src->dense_int8;
        const float inv127 = CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX;
        for (int d = 0; d < CBM_SEM_DIM; d++) {
            dst->v[d] = inv127 * (float)s[d];
        }
    }
}

/* Inline helper: add weighted source into target.
 * Sparse path: ~8 operations, ~48 bytes source memory traffic.
 * Dense path: 768 mul-adds with int8→float conversion, ~768 bytes traffic. */
static inline void sem_vec_add_src_scaled(cbm_sem_vec_t *dst, const cbm_sem_src_entry_t *src,
                                          float scale) {
    if (src->is_sparse) {
        for (int k = 0; k < src->nnz; k++) {
            dst->v[src->indices[k]] += scale * src->values[k];
        }
    } else {
        const int8_t *s = src->dense_int8;
        const float mul = scale * (CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX);
        for (int d = 0; d < CBM_SEM_DIM; d++) {
            dst->v[d] += mul * (float)s[d];
        }
    }
}

/* Pass 1 context: uses sparse/int8 tagged sources (most memory-efficient). */
typedef struct {
    corpus_entry_t *entries;
    const cbm_sem_src_entry_t *src_entries; /* sparse or int8-dense per token */
    int **doc_token_ids;
    const int *doc_token_counts;
    const reverse_index_t *rev;
    int doc_count;
    int entry_count;
    _Atomic int next_chunk;
    int num_chunks;
    int chunk_size;

    /* Cache-blocked tiling parameters */
    int tile_size; /* targets per L2-resident tile */
} cooccur_sparse_ctx_t;

/* Accumulate co-occurrence context for a single target token into `target`.
 * Reads neighbors within ±CBM_SEM_WINDOW positions across all documents that
 * reference this token id via the reverse index. */
static void cooccur_sparse_one_target(cooccur_sparse_ctx_t *cc, int tid, cbm_sem_vec_t *target) {
    int occ_start = cc->rev->offsets[tid];
    int occ_end = cc->rev->offsets[tid + SKIP_ONE];
    /* Frequent-token subsampling: stride over occurrences when a token is very
     * common so the enriched vector's direction is preserved (it's normalized
     * after) while bounding work to ~CBM_SEM_MAX_OCCUR samples. See semantic.h. */
    int occ_step = SKIP_ONE;
    if (occ_end - occ_start > CBM_SEM_MAX_OCCUR) {
        occ_step = (occ_end - occ_start) / CBM_SEM_MAX_OCCUR;
    }
    for (int p = occ_start; p < occ_end; p += occ_step) {
        int d = cc->rev->flat[p].doc_id;
        int i = cc->rev->flat[p].pos;
        int *ids = cc->doc_token_ids[d];
        int len = cc->doc_token_counts[d];
        for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) {
            if (w == 0) {
                continue;
            }
            int j = i + w;
            if (j < 0 || j >= len) {
                continue;
            }
            int nid = ids[j];
            if (nid < 0) {
                continue;
            }
            float weight = CBM_SEM_UNIT_POS / (float)abs(w);
            sem_vec_add_src_scaled(target, &cc->src_entries[nid], weight);
        }
    }
}

static void cooccur_worker_sparse(int worker_id, void *ctx_ptr) {
    (void)worker_id;
    cooccur_sparse_ctx_t *cc = ctx_ptr;
    while (true) {
        int ci =
            atomic_fetch_add_explicit(&cc->next_chunk, CBM_SEM_ATOMIC_INC, memory_order_relaxed);
        if (ci >= cc->num_chunks) {
            break;
        }
        int chunk_start = ci * cc->chunk_size;
        int chunk_end = chunk_start + cc->chunk_size;
        if (chunk_end > cc->entry_count) {
            chunk_end = cc->entry_count;
        }

        /* Cache-blocked target tiling: process tile_size targets at a time so
         * their vectors stay resident in L2 cache during their accumulation. */
        for (int tile_start = chunk_start; tile_start < chunk_end; tile_start += cc->tile_size) {
            int tile_end = tile_start + cc->tile_size;
            if (tile_end > chunk_end) {
                tile_end = chunk_end;
            }
            for (int tid = tile_start; tid < tile_end; tid++) {
                sem_target_init_from_src(&cc->entries[tid].enriched_vec, &cc->src_entries[tid]);
                cooccur_sparse_one_target(cc, tid, &cc->entries[tid].enriched_vec);
            }
        }
    }
}

/* Pass 2 (RRI) context: uses int8-quantized pass1 vectors as source.
 * Pass1 outputs are dense float32 post-normalization, values in [-1,1].
 * We quantize to int8 once (×127) to cut source memory traffic 4x. */
typedef struct {
    corpus_entry_t *entries;
    const int8_t *pass1_q; /* [entry_count × CBM_SEM_DIM] int8 quantized pass1 */
    int **doc_token_ids;
    const int *doc_token_counts;
    const reverse_index_t *rev;
    int doc_count;
    int entry_count;
    _Atomic int next_chunk;
    int num_chunks;
    int chunk_size;
    int tile_size;
} cooccur_int8_ctx_t;

static inline void sem_vec_add_int8_scaled(cbm_sem_vec_t *dst, const int8_t *src, float scale) {
    const float mul = scale * (CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX);
    for (int d = 0; d < CBM_SEM_DIM; d++) {
        dst->v[d] += mul * (float)src[d];
    }
}

/* RRI pass 2 accumulator for a single target token, reading int8-quantized
 * pass1 vectors as the source. */
static void cooccur_int8_one_target(cooccur_int8_ctx_t *cc, int tid, cbm_sem_vec_t *target) {
    int occ_start = cc->rev->offsets[tid];
    int occ_end = cc->rev->offsets[tid + SKIP_ONE];
    /* Frequent-token subsampling (see cooccur_sparse_one_target / semantic.h). */
    int occ_step = SKIP_ONE;
    if (occ_end - occ_start > CBM_SEM_MAX_OCCUR) {
        occ_step = (occ_end - occ_start) / CBM_SEM_MAX_OCCUR;
    }
    for (int p = occ_start; p < occ_end; p += occ_step) {
        int d = cc->rev->flat[p].doc_id;
        int i = cc->rev->flat[p].pos;
        int *ids = cc->doc_token_ids[d];
        int len = cc->doc_token_counts[d];
        for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) {
            if (w == 0) {
                continue;
            }
            int j = i + w;
            if (j < 0 || j >= len) {
                continue;
            }
            int nid = ids[j];
            if (nid < 0) {
                continue;
            }
            float weight = CBM_SEM_UNIT_POS / (float)abs(w);
            sem_vec_add_int8_scaled(target, &cc->pass1_q[(size_t)nid * CBM_SEM_DIM], weight);
        }
    }
}

static void cooccur_worker_int8(int worker_id, void *ctx_ptr) {
    (void)worker_id;
    cooccur_int8_ctx_t *cc = ctx_ptr;
    while (true) {
        int ci =
            atomic_fetch_add_explicit(&cc->next_chunk, CBM_SEM_ATOMIC_INC, memory_order_relaxed);
        if (ci >= cc->num_chunks) {
            break;
        }
        int chunk_start = ci * cc->chunk_size;
        int chunk_end = chunk_start + cc->chunk_size;
        if (chunk_end > cc->entry_count) {
            chunk_end = cc->entry_count;
        }

        for (int tile_start = chunk_start; tile_start < chunk_end; tile_start += cc->tile_size) {
            int tile_end = tile_start + cc->tile_size;
            if (tile_end > chunk_end) {
                tile_end = chunk_end;
            }
            for (int tid = tile_start; tid < tile_end; tid++) {
                /* RRI pass 2 starts from zero (no self-init) */
                memset(&cc->entries[tid].enriched_vec, 0, sizeof(cbm_sem_vec_t));
                cooccur_int8_one_target(cc, tid, &cc->entries[tid].enriched_vec);
            }
        }
    }
}

typedef struct {
    corpus_entry_t *entries;
    int entry_count;
    _Atomic int next_idx;
} norm_ctx_t;

static void normalize_worker(int worker_id, void *ctx_ptr) {
    (void)worker_id;
    norm_ctx_t *nc = ctx_ptr;
    while (true) {
        int start = atomic_fetch_add_explicit(&nc->next_idx, CBM_SEM_WORKER_STACK_CAP,
                                              memory_order_relaxed);
        if (start >= nc->entry_count) {
            break;
        }
        int end = start + CBM_SEM_WORKER_STACK_CAP;
        if (end > nc->entry_count) {
            end = nc->entry_count;
        }
        for (int i = start; i < end; i++) {
            cbm_sem_normalize(&nc->entries[i].enriched_vec);
        }
    }
}

typedef struct {
    corpus_entry_t *entries;
    const cbm_sem_vec_t *pass1;
    int entry_count;
    _Atomic int next_idx;
} blend_ctx_t;

static void blend_worker(int worker_id, void *ctx_ptr) {
    (void)worker_id;
    blend_ctx_t *bc = ctx_ptr;
    while (true) {
        int start = atomic_fetch_add_explicit(&bc->next_idx, CBM_SEM_WORKER_STACK_CAP,
                                              memory_order_relaxed);
        if (start >= bc->entry_count) {
            break;
        }
        int end = start + CBM_SEM_WORKER_STACK_CAP;
        if (end > bc->entry_count) {
            end = bc->entry_count;
        }
        for (int i = start; i < end; i++) {
            cbm_sem_normalize(&bc->entries[i].enriched_vec);
            for (int d = 0; d < CBM_SEM_DIM; d++) {
                bc->entries[i].enriched_vec.v[d] =
                    (CBM_SEM_RRI_BETA * bc->pass1[i].v[d]) +
                    (CBM_SEM_RRI_ALPHA * bc->entries[i].enriched_vec.v[d]);
            }
        }
    }
}

typedef struct {
    corpus_entry_t *entries;
    cbm_sem_src_entry_t *src_entries;
    int entry_count;
    _Atomic int next_idx;
} src_build_ctx_t;

/* Build one src_entry for a token: dense float32 reference if in nomic vocab,
 * sparse inline representation otherwise. Collisions in the sparse hash are
 * merged and zeros filtered so the final representation is exactly the same
 * mathematical vector that the old dense path produced. */
static void build_src_entry(const char *token, cbm_sem_src_entry_t *out) {
    memset(out, 0, sizeof(*out));
    if (!token) {
        out->is_sparse = SKIP_ONE;
        out->nnz = 0;
        return;
    }
    /* Dense path: direct int8 pointer into pretrained blob (zero-copy). */
    const char *idx_str = cbm_ht_get(g_pretrained_map, token);
    if (idx_str) {
        char *end = NULL;
        long idx = strtol(idx_str, &end, BASE_DECIMAL);
        if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) {
            out->is_sparse = 0;
            out->dense_int8 = pretrained_vec_at((int)idx);
            return;
        }
    }
    /* Sparse path: compute 8 hash positions with collision merging. */
    out->is_sparse = SKIP_ONE;
    uint16_t tmp_idx[CBM_SEM_SPARSE_NNZE];
    float tmp_val[CBM_SEM_SPARSE_NNZE];
    int count = 0;
    uint64_t seed = XXH3_64bits(token, strlen(token));
    for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) {
        uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE);
        int pos = (int)(h % CBM_SEM_DIM);
        float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS;
        /* Merge collisions */
        int found = CBM_NOT_FOUND;
        for (int j = 0; j < count; j++) {
            if (tmp_idx[j] == (uint16_t)pos) {
                found = j;
                break;
            }
        }
        if (found >= 0) {
            tmp_val[found] += sign;
        } else {
            tmp_idx[count] = (uint16_t)pos;
            tmp_val[count] = sign;
            count++;
        }
    }
    /* Filter zeros */
    int nnz = 0;
    for (int j = 0; j < count; j++) {
        if (tmp_val[j] != 0.0F) {
            out->indices[nnz] = tmp_idx[j];
            out->values[nnz] = tmp_val[j];
            nnz++;
        }
    }
    out->nnz = (uint8_t)nnz;
}

static void src_build_worker(int worker_id, void *ctx_ptr) {
    (void)worker_id;
    src_build_ctx_t *sc = ctx_ptr;
    while (true) {
        int start = atomic_fetch_add_explicit(&sc->next_idx, CBM_SEM_WORKER_STACK_CAP,
                                              memory_order_relaxed);
        if (start >= sc->entry_count) {
            break;
        }
        int end = start + CBM_SEM_WORKER_STACK_CAP;
        if (end > sc->entry_count) {
            end = sc->entry_count;
        }
        for (int i = start; i < end; i++) {
            build_src_entry(sc->entries[i].token, &sc->src_entries[i]);
        }
    }
}

/* Quantize pass1 float vectors to int8 (× 127) for use as pass2 source.
 * Input vectors are unit-normalized, so values are in [-1, 1] → int8 preserves
 * ~99% precision. 4x less memory traffic for pass2. */
typedef struct {
    const corpus_entry_t *entries;
    int8_t *pass1_q;
    int entry_count;
    _Atomic int next_idx;
} pass1_quant_ctx_t;

static void pass1_quantize_worker(int worker_id, void *ctx_ptr) {
    (void)worker_id;
    pass1_quant_ctx_t *qc = ctx_ptr;
    while (true) {
        int start =
            atomic_fetch_add_explicit(&qc->next_idx, CBM_SEM_RRI_TILE, memory_order_relaxed);
        if (start >= qc->entry_count) {
            break;
        }
        int end = start + CBM_SEM_RRI_TILE;
        if (end > qc->entry_count) {
            end = qc->entry_count;
        }
        for (int i = start; i < end; i++) {
            const cbm_sem_vec_t *src = &qc->entries[i].enriched_vec;
            int8_t *dst = &qc->pass1_q[(size_t)i * CBM_SEM_DIM];
            for (int d = 0; d < CBM_SEM_DIM; d++) {
                float v = src->v[d] * CBM_SEM_INT8_MAX;
                if (v > CBM_SEM_INT8_MAX) {
                    v = CBM_SEM_INT8_MAX;
                }
                if (v < -CBM_SEM_INT8_MAX) {
                    v = -CBM_SEM_INT8_MAX;
                }
                dst[d] = (int8_t)(v >= 0 ? v + CBM_SEM_IDF_BASE : v - CBM_SEM_IDF_BASE);
            }
        }
    }
}

/* Build reverse index: token_id → list of (doc_id, position) pairs.
 * SEQUENTIAL (fast: just pointer arithmetic + flat array fill). */
static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) {
    reverse_index_t *rev = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, sizeof(reverse_index_t));
    if (!rev) {
        return NULL;
    }
    /* Phase A: count occurrences per token */
    int *counts =
        cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int));
    if (!counts) {
        cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
        return NULL;
    }
    long total = 0;
    for (int d = 0; d < corpus->doc_count; d++) {
        int *ids = corpus->doc_token_ids[d];
        int len = corpus->doc_token_counts[d];
        for (int i = 0; i < len; i++) {
            int tid = ids[i];
            if (tid >= 0 && tid < corpus->entry_count) {
                counts[tid]++;
                total++;
            }
        }
    }
    /* Phase B: exclusive prefix sum → offsets[] */
    rev->offsets =
        cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int));
    if (!rev->offsets) {
        cbm_free(CBM_MEM_CLASS_SEMANTIC, counts);
        cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
        return NULL;
    }
    int running = 0;
    for (int t = 0; t < corpus->entry_count; t++) {
        rev->offsets[t] = running;
        running += counts[t];
        counts[t] = 0; /* reuse as per-token fill cursor */
    }
    rev->offsets[corpus->entry_count] = running;
    /* Phase C: fill flat array. Ensure allocation size > 0 even for empty
     * corpora (avoids cbm_alloc(CBM_MEM_CLASS_SEMANTIC, 0) which is implementation-defined). */
    size_t flat_bytes = (total > 0 ? (size_t)total : SKIP_ONE) * sizeof(cooccur_pos_t);
    rev->flat = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, flat_bytes);
    if (!rev->flat) {
        cbm_free(CBM_MEM_CLASS_SEMANTIC, rev->offsets);
        cbm_free(CBM_MEM_CLASS_SEMANTIC, counts);
        cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
        return NULL;
    }
    for (int d = 0; d < corpus->doc_count; d++) {
        int *ids = corpus->doc_token_ids[d];
        int len = corpus->doc_token_counts[d];
        for (int i = 0; i < len; i++) {
            int tid = ids[i];
            if (tid >= 0 && tid < corpus->entry_count) {
                int slot = rev->offsets[tid] + counts[tid]++;
                rev->flat[slot].doc_id = (int32_t)d;
                rev->flat[slot].pos = (int32_t)i;
            }
        }
    }
    cbm_free(CBM_MEM_CLASS_SEMANTIC, counts);
    return rev;
}

static void free_reverse_index(reverse_index_t *rev) {
    if (!rev) {
        return;
    }
    cbm_free(CBM_MEM_CLASS_SEMANTIC, rev->offsets);
    cbm_free(CBM_MEM_CLASS_SEMANTIC, rev->flat);
    cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
}

/* Bundle of parameters shared by the finalize sub-phases. */
typedef struct {
    cbm_sem_corpus_t *corpus;
    reverse_index_t *rev;
    cbm_sem_src_entry_t *src_entries;
    int worker_count;
    int num_chunks;
    int chunk_size;
    int tile_size;
    cbm_parallel_for_opts_t opts;
} finalize_params_t;

/* Sub-phase 1: build tagged source vectors (sparse or dense-int8) in parallel. */
static void finalize_build_sources(finalize_params_t *p) {
    src_build_ctx_t sc = {
        .entries = p->corpus->entries,
        .src_entries = p->src_entries,
        .entry_count = p->corpus->entry_count,
    };
    atomic_init(&sc.next_idx, 0);
    cbm_parallel_for(p->worker_count, src_build_worker, &sc, p->opts);
}

/* Sub-phases 2+3: co-occurrence pass 1 + normalize. */
static void finalize_pass1(finalize_params_t *p) {
    cooccur_sparse_ctx_t cc = {
        .entries = p->corpus->entries,
        .src_entries = p->src_entries,
        .doc_token_ids = p->corpus->doc_token_ids,
        .doc_token_counts = p->corpus->doc_token_counts,
        .rev = p->rev,
        .doc_count = p->corpus->doc_count,
        .entry_count = p->corpus->entry_count,
        .num_chunks = p->num_chunks,
        .chunk_size = p->chunk_size,
        .tile_size = p->tile_size,
    };
    atomic_init(&cc.next_chunk, 0);
    cbm_parallel_for(p->worker_count, cooccur_worker_sparse, &cc, p->opts);

    norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count};
    atomic_init(&nc.next_idx, 0);
    cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts);
}

/* Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. */
static void finalize_pass2(finalize_params_t *p) {
    int8_t *pass1_q = cbm_alloc(CBM_MEM_CLASS_SEMANTIC,
                                (size_t)p->corpus->entry_count * CBM_SEM_DIM * sizeof(int8_t));
    if (pass1_q) {
        pass1_quant_ctx_t qc = {
            .entries = p->corpus->entries,
            .pass1_q = pass1_q,
            .entry_count = p->corpus->entry_count,
        };
        atomic_init(&qc.next_idx, 0);
        cbm_parallel_for(p->worker_count, pass1_quantize_worker, &qc, p->opts);
    }

    cbm_sem_vec_t *pass1 =
        cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)p->corpus->entry_count * sizeof(cbm_sem_vec_t));
    if (pass1) {
        for (int i = 0; i < p->corpus->entry_count; i++) {
            pass1[i] = p->corpus->entries[i].enriched_vec;
        }
    }

    if (pass1_q) {
        cooccur_int8_ctx_t cc = {
            .entries = p->corpus->entries,
            .pass1_q = pass1_q,
            .doc_token_ids = p->corpus->doc_token_ids,
            .doc_token_counts = p->corpus->doc_token_counts,
            .rev = p->rev,
            .doc_count = p->corpus->doc_count,
            .entry_count = p->corpus->entry_count,
            .num_chunks = p->num_chunks,
            .chunk_size = p->chunk_size,
            .tile_size = p->tile_size,
        };
        atomic_init(&cc.next_chunk, 0);
        cbm_parallel_for(p->worker_count, cooccur_worker_int8, &cc, p->opts);
    }

    if (pass1) {
        blend_ctx_t bc = {
            .entries = p->corpus->entries,
            .pass1 = pass1,
            .entry_count = p->corpus->entry_count,
        };
        atomic_init(&bc.next_idx, 0);
        cbm_parallel_for(p->worker_count, blend_worker, &bc, p->opts);
        cbm_free(CBM_MEM_CLASS_SEMANTIC, pass1);
    }
    cbm_free(CBM_MEM_CLASS_SEMANTIC, pass1_q);

    norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count};
    atomic_init(&nc.next_idx, 0);
    cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts);
}

/* The per-document token id lists exist for the co-occurrence pass only;
 * released at the end of finalize so the vector phase runs without them
 * (783k small blocks on the kernel). */
static void corpus_release_docs(cbm_sem_corpus_t *corpus) {
    for (int d = 0; d < corpus->doc_count; d++) {
        cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids[d]);
        corpus->doc_token_ids[d] = NULL;
    }
}

void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) {
    if (!corpus || corpus->finalized) {
        return;
    }

    /* Eager init before parallel dispatch to avoid lazy-init races */
    ensure_pretrained_map();

    int worker_count = cbm_default_worker_count(false);
    cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false};

    /* Finer chunks = better load balancing for skewed token distributions. */
    int num_chunks = worker_count * CBM_SEM_COOCCUR_CHUNK;
    if (num_chunks > corpus->entry_count) {
        num_chunks = corpus->entry_count;
    }
    if (num_chunks < SKIP_ONE) {
        num_chunks = SKIP_ONE;
    }
    int chunk_size = (corpus->entry_count + num_chunks - SKIP_ONE) / num_chunks;

    reverse_index_t *rev = build_reverse_index(corpus);
    if (!rev) {
        corpus->finalized = true;
        return;
    }
    cbm_sem_src_entry_t *src_entries = cbm_calloc(
        CBM_MEM_CLASS_SEMANTIC, (size_t)corpus->entry_count * sizeof(cbm_sem_src_entry_t));
    if (!src_entries) {
        free_reverse_index(rev);
        corpus->finalized = true;
        return;
    }

    finalize_params_t params = {
        .corpus = corpus,
        .rev = rev,
        .src_entries = src_entries,
        .worker_count = worker_count,
        .num_chunks = num_chunks,
        .chunk_size = chunk_size,
        .tile_size = CBM_SEM_TILE_SIZE,
        .opts = opts,
    };
    finalize_build_sources(&params);
    finalize_pass1(&params);
    finalize_pass2(&params);

    cbm_free(CBM_MEM_CLASS_SEMANTIC, src_entries);
    free_reverse_index(rev);
    corpus_release_docs(corpus);
    corpus->finalized = true;
}

/* Parse a decimal index string via strtol, returning CBM_NOT_FOUND on parse
 * failure.  Centralizes the atoi-replacement pattern for token_map lookups. */
static int parse_token_index(const char *idx_str) {
    if (!idx_str) {
        return CBM_NOT_FOUND;
    }
    char *end = NULL;
    long parsed = strtol(idx_str, &end, BASE_DECIMAL);
    return (end != idx_str) ? (int)parsed : CBM_NOT_FOUND;
}

float cbm_sem_corpus_idf(const cbm_sem_corpus_t *corpus, const char *token) {
    if (!corpus || !token || corpus->doc_count == 0) {
        return 0.0F;
    }
    int idx = parse_token_index(cbm_ht_get(corpus->token_map, token));
    if (idx < 0 || idx >= corpus->entry_count) {
        return 0.0F;
    }
    int df = corpus->entries[idx].doc_freq;
    if (df <= 0) {
        return 0.0F;
    }
    return logf((float)corpus->doc_count / (float)df);
}

const cbm_sem_vec_t *cbm_sem_corpus_ri_vec(const cbm_sem_corpus_t *corpus, const char *token) {
    if (!corpus || !token) {
        return NULL;
    }
    int idx = parse_token_index(cbm_ht_get(corpus->token_map, token));
    if (idx < 0 || idx >= corpus->entry_count) {
        return NULL;
    }
    return &corpus->entries[idx].enriched_vec;
}

int cbm_sem_corpus_token_index(const cbm_sem_corpus_t *corpus, const char *token) {
    if (!corpus || !token) {
        return CBM_NOT_FOUND;
    }
    int idx = parse_token_index(cbm_ht_get(corpus->token_map, token));
    return (idx < 0 || idx >= corpus->entry_count) ? CBM_NOT_FOUND : idx;
}

/* Same arithmetic as cbm_sem_corpus_idf, by index. */
float cbm_sem_corpus_idf_at(const cbm_sem_corpus_t *corpus, int index) {
    if (!corpus || corpus->doc_count == 0 || index < 0 || index >= corpus->entry_count) {
        return 0.0F;
    }
    int df = corpus->entries[index].doc_freq;
    if (df <= 0) {
        return 0.0F;
    }
    return logf((float)corpus->doc_count / (float)df);
}

const cbm_sem_vec_t *cbm_sem_corpus_ri_vec_at(const cbm_sem_corpus_t *corpus, int index) {
    if (!corpus || index < 0 || index >= corpus->entry_count) {
        return NULL;
    }
    return &corpus->entries[index].enriched_vec;
}

int cbm_sem_corpus_doc_count(const cbm_sem_corpus_t *corpus) {
    return corpus ? corpus->doc_count : 0;
}

int cbm_sem_corpus_token_count(const cbm_sem_corpus_t *corpus) {
    return corpus ? corpus->entry_count : 0;
}

const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index,
                                    const cbm_sem_vec_t **out_vec, float *out_idf) {
    if (!corpus || index < 0 || index >= corpus->entry_count) {
        return NULL;
    }
    if (out_vec) {
        *out_vec = &corpus->entries[index].enriched_vec;
    }
    if (out_idf && corpus->doc_count > 0) {
        int df = corpus->entries[index].doc_freq;
        *out_idf = df > 0 ? logf((float)corpus->doc_count / (float)df) : 0.0F;
    }
    return corpus->entries[index].token;
}

static void free_ht_kv(const char *key, void *value, void *userdata) {
    (void)userdata;
    cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)key);
    cbm_free(CBM_MEM_CLASS_SEMANTIC, value);
}

void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus) {
    if (!corpus) {
        return;
    }
    for (int i = 0; i < corpus->entry_count; i++) {
        cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->entries[i].token);
    }
    cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->entries);
    for (int d = 0; d < corpus->doc_count; d++) {
        cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids[d]);
    }
    cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids);
    cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_counts);
    if (corpus->token_map) {
        cbm_ht_foreach(corpus->token_map, free_ht_kv, NULL);
        cbm_ht_free(corpus->token_map);
    }
    cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus);
}

/* ── Combined scoring ────────────────────────────────────────────── */

float cbm_sem_proximity(const char *path_a, const char *path_b) {
    if (!path_a || !path_b) {
        return CBM_SEM_UNIT_POS;
    }
    /* Count shared directory components */
    int shared = 0;
    int total_a = 0;
    int total_b = 0;
    const char *a = path_a;
    const char *b = path_b;
    while (*a && *b && *a == *b) {
        if (*a == '/') {
            shared++;
        }
        a++;
        b++;
    }
    for (const char *p = path_a; *p; p++) {
        if (*p == '/') {
            total_a++;
        }
    }
    for (const char *p = path_b; *p; p++) {
        if (*p == '/') {
            total_b++;
        }
    }
    int max_total = total_a > total_b ? total_a : total_b;
    if (max_total == 0) {
        return CBM_SEM_UNIT_POS;
    }
    float ratio = (float)shared / (float)max_total;
    /* Same file = 1.10, same dir = 1.05, distant = 1.00 */
    return CBM_SEM_UNIT_POS + (ratio * CBM_SEM_PROX_MAX_BOOST);
}

/* Cosine similarity for small float arrays (AST profile). */
static float small_cosine(const float *a, const float *b, int dims) {
    float dot = 0.0F;
    float ma = 0.0F;
    float mb = 0.0F;
    for (int i = 0; i < dims; i++) {
        dot += a[i] * b[i];
        ma += a[i] * a[i];
        mb += b[i] * b[i];
    }
    float denom = sqrtf(ma) * sqrtf(mb);
    return denom < CBM_SEM_DENOM_EPS ? 0.0F : dot / denom;
}

static int cmp_int_asc(const void *pa, const void *pb) {
    int a = *(const int *)pa;
    int b = *(const int *)pb;
    return (a > b) - (a < b);
}

/* Terms are keyed by corpus token, so two functions share a term only when
 * they share the word. (Until 2026-10 the key was the token's POSITION in the
 * function: any two functions of similar length "shared" their first terms,
 * and the TF-IDF signal measured length, not vocabulary.) */
int cbm_sem_tfidf_terms(const cbm_sem_corpus_t *corpus, char *const *tokens, const int *token_index,
                        int count, int *indices, float *weights) {
    if (!corpus || !indices || !weights || count <= 0) {
        return 0;
    }
    int known = 0;
    for (int t = 0; t < count; t++) {
        int idx = token_index ? token_index[t] : cbm_sem_corpus_token_index(corpus, tokens[t]);
        if (idx >= 0 && cbm_sem_corpus_idf_at(corpus, idx) > 0.0F) {
            indices[known++] = idx;
        }
    }
    if (known > 1) {
        qsort(indices, (size_t)known, sizeof(indices[0]), cmp_int_asc);
    }
    /* Each run of one index becomes one term; its weight is the idf added once
     * per occurrence (equal indices have equal idf). */
    int terms = 0;
    for (int k = 0; k < known; k++) {
        float idf = cbm_sem_corpus_idf_at(corpus, indices[k]);
        if (terms > 0 && indices[terms - SKIP_ONE] == indices[k]) {
            weights[terms - SKIP_ONE] += idf;
        } else {
            indices[terms] = indices[k];
            weights[terms] = idf;
            terms++;
        }
    }
    return terms;
}

/* Sparse cosine over two pre-sorted (index, weight) vectors.  Returns 0 when
 * either side is empty or the magnitude product is below the epsilon guard. */
static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t *b) {
    if (a->tfidf_len <= 0 || b->tfidf_len <= 0) {
        return 0.0F;
    }
    float dot = 0.0F;
    float ma = 0.0F;
    float mb = 0.0F;
    int ia = 0;
    int ib = 0;
    while (ia < a->tfidf_len && ib < b->tfidf_len) {
        if (a->tfidf_indices[ia] == b->tfidf_indices[ib]) {
            dot += a->tfidf_weights[ia] * b->tfidf_weights[ib];
            ia++;
            ib++;
        } else if (a->tfidf_indices[ia] < b->tfidf_indices[ib]) {
            ia++;
        } else {
            ib++;
        }
    }
    for (int i = 0; i < a->tfidf_len; i++) {
        ma += a->tfidf_weights[i] * a->tfidf_weights[i];
    }
    for (int i = 0; i < b->tfidf_len; i++) {
        mb += b->tfidf_weights[i] * b->tfidf_weights[i];
    }
    float denom = sqrtf(ma) * sqrtf(mb);
    return denom > CBM_SEM_DENOM_EPS ? (dot / denom) : 0.0F;
}

void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
                           cbm_sem_signals_t *out) {
    memset(out, 0, sizeof(*out));
    out->proximity = CBM_SEM_UNIT_POS;
    if (!a || !b) {
        return;
    }
    if (a->has_minhash && b->has_minhash) {
        double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash,
                                       (const cbm_minhash_t *)b->minhash);
        out->minhash = (float)j;
        out->near_copy = j >= CBM_MINHASH_JACCARD_THRESHOLD;
    }
    out->tfidf = sparse_tfidf_cosine(a, b);
    out->ri = cbm_rsq_ip(&a->ri_code, &b->ri_code);            /* signal 2: Random Indexing */
    out->api = cbm_rsq_ip(&a->api_code, &b->api_code);         /* signal 4: API signatures */
    out->type = cbm_rsq_ip(&a->type_code, &b->type_code);      /* signal 5: type signatures */
    out->decorator = cbm_rsq_ip(&a->deco_code, &b->deco_code); /* signal 7: decorators */
    /* signals 8+9+11: structural profile + data flow + Halstead */
    out->struct_profile =
        small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS);
    out->proximity = cbm_sem_proximity(a->file_path, b->file_path); /* signal 6: multiplier */
}

float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg) {
    /* Short-circuit: if MinHash Jaccard is already above the SIMILAR_TO threshold,
     * the pass_similarity pipeline already emitted a SIMILAR_TO edge for this pair.
     * Returning 0 here avoids flooding top-k with cross-service copy-paste boilerplate
     * (logging_middleware, shared push/pull handlers) that SIMILAR_TO already covers,
     * freeing the edge budget for true semantic leaps (vocabulary-bridged relations). */
    if (!s || !cfg || s->near_copy) {
        return 0.0F;
    }
    /* The terms in the historical order (float addition is not associative). */
    float score = cfg->w_tfidf * s->tfidf;
    score += cfg->w_ri * s->ri;
    score += cfg->w_minhash * s->minhash;
    score += cfg->w_api * s->api;
    score += cfg->w_type * s->type;
    score += cfg->w_decorator * s->decorator;
    score += cfg->w_struct_profile * s->struct_profile;

    /* Signal 6: Module proximity (multiplier, not additive).
     * Proximity returns [1.0, 1.10] — a same-file/same-dir boost for ranking.
     * Clamp the final product to [0, 1] so the output stays within valid
     * cosine-similarity range.  Without the clamp, a 0.95 base × 1.10 proximity
     * would emit a 1.045 score which violates the semantic of "similarity" and
     * breaks downstream consumers that expect a normalized value. */
    score *= s->proximity;
    if (score > CBM_SEM_UNIT_POS) {
        score = CBM_SEM_UNIT_POS;
    }
    if (score < 0.0F) {
        score = 0.0F;
    }
    return score;
}

typedef struct {
    float score;
    int pos;
} sem_rank_t;

static int cmp_rank_best_first(const void *pa, const void *pb) {
    const sem_rank_t *a = pa;
    const sem_rank_t *b = pb;
    if (a->score != b->score) {
        return a->score > b->score ? -1 : 1;
    }
    return (a->pos > b->pos) - (a->pos < b->pos);
}

/* Best-first, not first-come: when a function has more candidate partners
 * than its budget, it keeps its best ones. Judged on held-out repositories
 * (2026-10-07): the pairs first-come admitted over budget were 0 of 17
 * related, the ones best-first admits instead 7 of 20. */
bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb,
                              const bool *eligible, int n, int max_edges, int *counts,
                              bool *admitted) {
    if (n <= 0) {
        return true;
    }
    memset(admitted, 0, (size_t)n * sizeof(admitted[0]));
    sem_rank_t *order = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)n * sizeof(sem_rank_t));
    if (!order) {
        return false;
    }
    int m = 0;
    for (int k = 0; k < n; k++) {
        if (eligible[k]) {
            order[m++] = (sem_rank_t){.score = scores[k], .pos = k};
        }
    }
    qsort(order, (size_t)m, sizeof(order[0]), cmp_rank_best_first);
    for (int r = 0; r < m; r++) {
        int k = order[r].pos;
        if (counts[fa[k]] < max_edges && counts[fb[k]] < max_edges) {
            counts[fa[k]]++;
            counts[fb[k]]++;
            admitted[k] = true;
        }
    }
    cbm_free(CBM_MEM_CLASS_SEMANTIC, order);
    return true;
}

/* Until 2026-10 the reader stopped at the first '"' and kept escapes raw:
 * a docstring holding a quote lost everything after it, and "\nreturn" gave
 * the token "nreturn". */
const char *cbm_sem_json_str(const char *json, const char *key, char *buf, int bufsize) {
    if (!json || !key || !buf || bufsize <= 0) {
        return NULL;
    }
    char search[CBM_SZ_64];
    snprintf(search, sizeof(search), "\"%s\":\"", key);
    const char *p = strstr(json, search);
    if (!p) {
        return NULL;
    }
    p += strlen(search);
    int n = 0;
    for (; *p && *p != '"' && n + SKIP_ONE < bufsize; p++) {
        char c = *p;
        if (c == '\\' && p[SKIP_ONE]) {
            p++;
            switch (*p) {
            case 'n':
            case 't':
            case 'r':
            case 'b':
            case 'f':
                c = ' ';
                break;
            case 'u':
                c = ' ';
                for (int k = 0; k < 4 && isxdigit((unsigned char)p[SKIP_ONE]); k++) {
                    p++;
                }
                break;
            default:
                c = *p; /* \" \\ \/ */
                break;
            }
        }
        buf[n++] = c;
    }
    buf[n] = '\0';
    return buf;
}

/* Score bands at the quartiles of the admitted scores (0.773, 0.802, 0.838 at
 * the stored score's three decimals, so one shown score always carries one p),
 * each band's p the judged share of related pairs, pooled where a higher band
 * judged lower: the upper three pool to one. Judged 2026-10-07 in eleven
 * held-out repositories: the pairs this scoring admits that the previous one
 * did not (100, blind, three reading rounds) and the earlier samples' pairs it
 * keeps (65), each weighted by its share of the admitted set. Ships as the
 * edge property "p". */
static const float SEM_P_CUTS[] = {0.773F};
static const float SEM_P_BANDS[] = {0.626F, 0.760F};

float cbm_sem_calibrated_p(float score) {
    size_t band = 0;
    while (band < sizeof(SEM_P_CUTS) / sizeof(SEM_P_CUTS[0]) && score >= SEM_P_CUTS[band]) {
        band++;
    }
    return SEM_P_BANDS[band];
}

/* Doc -> code candidates by their tfidf: each band's p the judged share of
 * sections that are about the function (2026-10-07: 274 blind-judged
 * candidates -- a section's best five non-test functions -- from five held-out
 * repositories, three reading rounds; the top three measured bands pooled to
 * stay monotone). Below the first cut (judged 0.06) nothing is stored. */
enum { SEM_DOC_P_NBANDS = 4 };
static const float SEM_DOC_P_CUTS[SEM_DOC_P_NBANDS - 1] = {CBM_SEM_DOC_MIN_SCORE, 0.30F, 0.40F};
/* Per format, p below the first cut and in each band; 0 = not stored (a
 * band judged below 0.20). Blind-judged, three reading rounds, pooled where
 * a higher band judged lower:
 *   Markdown  by kind and position since SEM7: SEM_DOC_P_MD_* below (its
 *             first curve, 2026-10-07: 274 candidates of five repositories,
 *             0.06 / 0.36 / 0.42 / 0.55, did not separate positions).
 *   reST      2026-10-08, 120 candidates, three Sphinx projects; 0.20-0.30
 *             judged 0.125, so reST starts at 0.30.
 *   AsciiDoc  2026-10-08, 114 candidates, one repository (the only one of
 *             its search that qualified).
 *   PDF       no qualifying repository: not measured, nothing stored. */
static const float SEM_DOC_P_BANDS[CBM_SEM_DOC_FMT_COUNT][SEM_DOC_P_NBANDS] = {
    [CBM_SEM_DOC_FMT_OTHER] = {0.0F, 0.0F, 0.0F, 0.0F},
    [CBM_SEM_DOC_FMT_MARKDOWN] = {0.0F, 0.0F, 0.0F, 0.0F}, /* SEM_DOC_P_MD_* */
    [CBM_SEM_DOC_FMT_RST] = {0.0F, 0.0F, 0.35F, 0.625F},
    [CBM_SEM_DOC_FMT_ADOC] = {0.0F, 0.225F, 0.25F, 0.50F},
    [CBM_SEM_DOC_FMT_PDF] = {0.0F, 0.0F, 0.0F, 0.0F},
};

cbm_sem_doc_format_t cbm_sem_doc_format(const char *path) {
    static const struct {
        const char *ext;
        cbm_sem_doc_format_t fmt;
    } exts[] = {{".md", CBM_SEM_DOC_FMT_MARKDOWN},       {".mdx", CBM_SEM_DOC_FMT_MARKDOWN},
                {".markdown", CBM_SEM_DOC_FMT_MARKDOWN}, {".rst", CBM_SEM_DOC_FMT_RST},
                {".rest", CBM_SEM_DOC_FMT_RST},          {".txt", CBM_SEM_DOC_FMT_RST},
                {".adoc", CBM_SEM_DOC_FMT_ADOC},         {".asciidoc", CBM_SEM_DOC_FMT_ADOC},
                {".asc", CBM_SEM_DOC_FMT_ADOC},          {".pdf", CBM_SEM_DOC_FMT_PDF}};
    const char *dot = path ? strrchr(path, '.') : NULL;
    if (!dot || strchr(dot, '/')) {
        return CBM_SEM_DOC_FMT_OTHER;
    }
    for (size_t i = 0; i < sizeof(exts) / sizeof(exts[0]); i++) {
        if (strcasecmp(dot, exts[i].ext) == 0) {
            return exts[i].fmt;
        }
    }
    return CBM_SEM_DOC_FMT_OTHER;
}

double cbm_sem_p_2dp(float p) {
    /* 1e-4 on the x100 scale: a float holds 0.325 as 0.32499998..., 1.2e-6
     * below the half; judged values have at most three decimals, so the
     * nearest one that must round down (x.xx4) is 0.1 away */
    return (double)(long)((double)p * 100.0 + 0.5 + 1e-4) / 100.0;
}

/* Markdown by kind and position (SEM7, 2026-10-08: 490 blind-judged
 * candidates from four monorepos with per-component docs, three reading
 * rounds, pooled where a higher band judged lower). A doc's position decides
 * most: functions outside the doc's home were right 6 of 90 (none stored);
 * inside it 0.18 / 0.33 / 0.53. A doc of the whole project: SEM7's and
 * sample 4's judged items pooled (the user's choice; 158 items, nine
 * repositories): 0.17 / 0.22 / 0.51. Local extras (home functions below the
 * top 5) 19 of 60 = 0.32 at every band; whole files 0.12 below 0.40, then
 * 0.30. Below 0.20 nothing. */
static const float SEM_DOC_P_MD_FUNCTION[CBM_SEM_DOC_POS_COUNT][SEM_DOC_P_NBANDS] = {
    [CBM_SEM_DOC_POS_GLOBAL] = {0.0F, 0.0F, 0.217F, 0.509F},
    [CBM_SEM_DOC_POS_LOCAL] = {0.0F, 0.0F, 0.325F, 0.525F},
    [CBM_SEM_DOC_POS_OUTSIDE] = {0.0F, 0.0F, 0.0F, 0.0F},
};
static const float SEM_DOC_P_MD_LOCAL[SEM_DOC_P_NBANDS] = {0.0F, 0.317F, 0.317F, 0.317F};
static const float SEM_DOC_P_MD_FILE[SEM_DOC_P_NBANDS] = {0.0F, 0.0F, 0.0F, 0.30F};

float cbm_sem_doc_calibrated_p(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind,
                               cbm_sem_doc_pos_t pos, float score) {
    if ((int)fmt < 0 || fmt >= CBM_SEM_DOC_FMT_COUNT || (int)pos < 0 ||
        pos >= CBM_SEM_DOC_POS_COUNT) {
        return 0.0F;
    }
    size_t band = 0;
    while (band < SEM_DOC_P_NBANDS - 1 && score >= SEM_DOC_P_CUTS[band]) {
        band++;
    }
    if (fmt != CBM_SEM_DOC_FMT_MARKDOWN) {
        /* measured for function candidates only, without position (SEM6) */
        return kind == CBM_SEM_DOC_KIND_FUNCTION ? SEM_DOC_P_BANDS[fmt][band] : 0.0F;
    }
    switch (kind) {
    case CBM_SEM_DOC_KIND_FUNCTION:
        return SEM_DOC_P_MD_FUNCTION[pos][band];
    case CBM_SEM_DOC_KIND_LOCAL:
        return pos == CBM_SEM_DOC_POS_LOCAL ? SEM_DOC_P_MD_LOCAL[band] : 0.0F;
    case CBM_SEM_DOC_KIND_FILE:
        return SEM_DOC_P_MD_FILE[band];
    default:
        return 0.0F; /* a folder candidate: cbm_sem_doc_folder_p */
    }
}

/* A doc's home folder as its candidate (SEM7): a README or index doc was
 * right 26 of 30 (0.87); any other doc 5 of 30 (not stored). Markdown only. */
float cbm_sem_doc_folder_p(cbm_sem_doc_format_t fmt, const char *doc_path) {
    if (fmt != CBM_SEM_DOC_FMT_MARKDOWN || !doc_path) {
        return 0.0F;
    }
    const char *base = strrchr(doc_path, '/');
    base = base ? base + 1 : doc_path;
    const char *dot = strrchr(base, '.');
    size_t n = dot ? (size_t)(dot - base) : strlen(base);
    bool readme = (n == sizeof("readme") - 1 && strncasecmp(base, "readme", n) == 0) ||
                  (n == sizeof("index") - 1 && strncasecmp(base, "index", n) == 0);
    return readme ? 0.867F : 0.0F;
}

cbm_sem_diag_family_t cbm_sem_diag_family(const char *format, const char *diagram_kind) {
    static const char *const mermaid[] = {"sequence",   "class",         "er",
                                          "c4_context", "c4_container",  "c4_component",
                                          "c4_dynamic", "c4_deployment", "architecture"};
    static const char *const plantuml[] = {"c4", "class", "sequence", "usecase", "component"};
    if (!format || !format[0]) {
        return CBM_SEM_DIAG_NONE;
    }
    const char *const *structural = NULL;
    size_t n = 0;
    if (strcmp(format, "mermaid") == 0) {
        structural = mermaid;
        n = sizeof(mermaid) / sizeof(mermaid[0]);
    } else if (strcmp(format, "plantuml") == 0) {
        structural = plantuml;
        n = sizeof(plantuml) / sizeof(plantuml[0]);
    } else {
        return CBM_SEM_DIAG_GRAPH;
    }
    for (size_t i = 0; diagram_kind && i < n; i++) {
        if (strcmp(diagram_kind, structural[i]) == 0) {
            return CBM_SEM_DIAG_STRUCTURAL;
        }
    }
    return CBM_SEM_DIAG_PROSE;
}

/* Markdown sections that embed a diagram, by the section's family (DIAG-C,
 * 2026-10-11: 553 blind-judged candidates from sections of 16 held-out
 * repositories that embed a diagram, three reading rounds; function
 * candidates at or inside the home pooled, as both curves rose by band).
 * A diagram section is about its candidates more often than SEM7's Markdown
 * curve says, so its first band is stored too:
 *   prose       0.24 / 0.28 / 0.45, local extras 11 of 40 = 0.275
 *   structural  0.32 / 0.46 / 0.57, local extras  7 of  9 = 0.78 (every
 *               local extra of the family; 95% interval 0.45-0.94)
 * Graph diagrams (DOT, D2) were not found in Markdown sections: SEM7. */
static const struct {
    bool fitted;
    float function[SEM_DOC_P_NBANDS];
    float local;
} SEM_DOC_P_MD_DIAGRAM[CBM_SEM_DIAG_COUNT] = {
    [CBM_SEM_DIAG_PROSE] = {true, {0.0F, 0.24F, 0.28F, 0.45F}, 0.275F},
    [CBM_SEM_DIAG_STRUCTURAL] = {true, {0.0F, 0.32F, 0.464F, 0.567F}, 0.778F},
};

float cbm_sem_doc_section_p(cbm_sem_doc_format_t fmt, cbm_sem_diag_family_t fam,
                            cbm_sem_doc_kind_t kind, cbm_sem_doc_pos_t pos, float score) {
    bool fitted = fmt == CBM_SEM_DOC_FMT_MARKDOWN && (int)fam > 0 && fam < CBM_SEM_DIAG_COUNT &&
                  SEM_DOC_P_MD_DIAGRAM[fam].fitted;
    bool home = pos == CBM_SEM_DOC_POS_GLOBAL || pos == CBM_SEM_DOC_POS_LOCAL;
    if (!fitted || !((kind == CBM_SEM_DOC_KIND_FUNCTION && home) ||
                     (kind == CBM_SEM_DOC_KIND_LOCAL && pos == CBM_SEM_DOC_POS_LOCAL))) {
        return cbm_sem_doc_calibrated_p(fmt, kind, pos, score);
    }
    size_t band = 0;
    while (band < SEM_DOC_P_NBANDS - 1 && score >= SEM_DOC_P_CUTS[band]) {
        band++;
    }
    if (band == 0) {
        return 0.0F; /* below CBM_SEM_DOC_MIN_SCORE: never measured, nothing stored */
    }
    return kind == CBM_SEM_DOC_KIND_LOCAL ? SEM_DOC_P_MD_DIAGRAM[fam].local
                                          : SEM_DOC_P_MD_DIAGRAM[fam].function[band];
}

float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b,
                             const cbm_sem_config_t *cfg) {
    if (!a || !b || !cfg) {
        return 0.0F;
    }
    cbm_sem_signals_t s;
    cbm_sem_signal_values(a, b, &s);
    return cbm_sem_combine(&s, cfg);
}

/* ── Graph diffusion ─────────────────────────────────────────────── */

void cbm_sem_diffuse(cbm_sem_vec_t *combined, const cbm_sem_vec_t *neighbors, int neighbor_count,
                     float alpha) {
    if (!combined || !neighbors || neighbor_count <= 0) {
        return;
    }
    /* Blend: combined = (1-α) × combined + α × mean(neighbors) */
    cbm_sem_vec_t mean;
    memset(&mean, 0, sizeof(mean));
    for (int n = 0; n < neighbor_count; n++) {
        for (int i = 0; i < CBM_SEM_DIM; i++) {
            mean.v[i] += neighbors[n].v[i];
        }
    }
    float inv_n = CBM_SEM_UNIT_POS / (float)neighbor_count;
    float one_minus_alpha = CBM_SEM_UNIT_POS - alpha;
    for (int i = 0; i < CBM_SEM_DIM; i++) {
        combined->v[i] = (one_minus_alpha * combined->v[i]) + (alpha * mean.v[i] * inv_n);
    }
    cbm_sem_normalize(combined);
}