Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(semantic): inject callee tokens in a fixed order
Martin Vogel11cb476dff14
diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.cindex dd8057d61..05a7387a9 100644--- a/src/pipeline/pass_semantic_edges.c+++ b/src/pipeline/pass_semantic_edges.c@@ -22,6 +22,7 @@#include "foundation/hash_table.h"#include "foundation/log.h"#include "foundation/compat.h"+#include "foundation/compat_fs.h" /* cbm_fopen for the pair dump */#define XXH_INLINE_ALL#include "xxhash/xxhash.h"@@ -194,23 +195,33 @@ static int inject_callee_tokens(const char *name, char **tokens, int count, int}/* Walk the outbound CALLS edges of n and inject tokens for each target. */+static const char **collect_sorted_call_neighbors(const cbm_gbuf_t *gbuf, int64_t node_id,+ bool outbound, int *out_n);+static int inject_calls_pattern_tokens(const cbm_gbuf_node_t *n, const cbm_gbuf_t *gbuf,char **tokens, int count, int max_tokens) {if (!gbuf) {return count;}- const cbm_gbuf_edge_t **edges = NULL;- int ec = 0;- if (cbm_gbuf_find_edges_by_source_type(gbuf, n->id, "CALLS", &edges, &ec) != 0) {+ /* SORTED, like every other consumer of CALLS neighbours in this file. The+ * raw edge array is in graph-buffer insertion order — parallel worker merge+ * order — and this loop both ORDERS the injected tokens and truncates at+ * max_tokens. Co-occurrence enrichment reads a window around each token+ * POSITION, so a different order yields different token vectors while+ * leaving document frequencies untouched: measured on the go corpus+ * 2026-09-19, two runs of one binary with identical idf but different token+ * sequences and different enriched vectors, which flickered near-threshold+ * SEMANTICALLY_RELATED edges. collect_sorted_call_neighbors() was written+ * for exactly this class and this call site was missed. */+ int nn = 0;+ const char **names = collect_sorted_call_neighbors(gbuf, n->id, /*outbound=*/true, &nn);+ if (!names) {return count;}- for (int e = 0; e < ec && count < max_tokens; e++) {- const cbm_gbuf_node_t *t = cbm_gbuf_find_by_id(gbuf, edges[e]->target_id);- if (!t || !t->name) {- continue;- }- count = inject_callee_tokens(t->name, tokens, count, max_tokens);+ for (int e = 0; e < nn && count < max_tokens; e++) {+ count = inject_callee_tokens(names[e], tokens, count, max_tokens);}+ cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)names);return count;}@@ -1152,9 +1163,27 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf}qsort(pairs, (size_t)n, sizeof(deferred_edge_t), cmp_deferred_edge_canonical);+ /* CBM_SEM_PAIR_DUMP=<path>: every candidate that passed the threshold, with+ * its score, in canonical order. Written HERE because this loop is+ * single-threaded and already sorted, so the dump needs no lock and two+ * runs compare line by line. The (i, j) indices are stable across runs —+ * funcs[] is sorted by qualified name — where node ids are not. Exists to+ * chase the near-threshold SEMANTICALLY_RELATED flicker: find the pair that+ * flips, then compare its score between runs. */+ FILE *pair_dump = NULL;+ {+ char dump_path[CBM_SZ_1K];+ if (cbm_safe_getenv("CBM_SEM_PAIR_DUMP", dump_path, sizeof(dump_path), NULL)) {+ pair_dump = cbm_fopen(dump_path, "w");+ }+ }+int total_edges = 0;for (int e = 0; e < n; e++) {deferred_edge_t *de = &pairs[e];+ if (pair_dump) {+ fprintf(pair_dump, "%d %d %.9g\n", de->i, de->j, (double)de->score);+ }if (edge_counts[de->i] >= max_edges || edge_counts[de->j] >= max_edges) {continue;}@@ -1166,6 +1195,9 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_bufedge_counts[de->j]++;total_edges++;}+ if (pair_dump) {+ (void)fclose(pair_dump);+ }cbm_free(CBM_MEM_CLASS_SEMANTIC, pairs);return total_edges;}@@ -1370,6 +1402,139 @@ static void phase5_lsh_build(cbm_sem_func_t *funcs, int func_count, int worker_c*out_buckets = band_buckets;}+/* Checksum every input the scorer reads, so two runs can be compared at the+ * seam right before scoring. cbm_sem_combined_score() reads exactly these+ * fields, so identical checksums with differing scores would be impossible.+ * Placed AFTER tf-idf is filled (phase 4), not after phase 1b where the+ * sparse vectors are still empty. Diagnostic, off unless the env var is set. */+static void sem_state_dump(const cbm_sem_func_t *funcs, int func_count) {+ /* CBM_SEM_STATE_DUMP=1: one checksum over every per-function vector, so two+ * runs can be compared at THIS seam. Scores are pure math over these+ * vectors, so if the checksums match and the scores do not, the difference+ * is downstream; if they differ, it is here or above. Diagnostic only. */+ char state_path[CBM_SZ_1K];+ if (cbm_safe_getenv("CBM_SEM_STATE_DUMP", state_path, sizeof(state_path), NULL)) {+ uint64_t h = 1469598103934665603ULL;+ uint64_t h_codes = 1469598103934665603ULL;+ uint64_t h_tfidf = 1469598103934665603ULL;+ for (int i = 0; i < func_count; i++) {+ const unsigned char *bytes[] = {+ (const unsigned char *)&funcs[i].ri_code,+ (const unsigned char *)&funcs[i].api_code,+ (const unsigned char *)&funcs[i].type_code,+ (const unsigned char *)&funcs[i].deco_code,+ };+ for (size_t b = 0; b < sizeof(bytes) / sizeof(bytes[0]); b++) {+ for (size_t k = 0; k < sizeof(cbm_rsq_code_t); k++) {+ h ^= bytes[b][k];+ h *= 1099511628211ULL;+ h_codes ^= bytes[b][k];+ h_codes *= 1099511628211ULL;+ }+ }+ /* The other scoring inputs: sparse TF-IDF, the AST profile and the+ * minhash fingerprint. Scoring reads exactly these plus the codes+ * above, so a matching checksum means the scores must match too. */+ const unsigned char *sp = (const unsigned char *)funcs[i].struct_profile;+ for (size_t k = 0; k < sizeof(funcs[i].struct_profile); k++) {+ h ^= sp[k];+ h *= 1099511628211ULL;+ }+ const unsigned char *mh = (const unsigned char *)funcs[i].minhash;+ for (size_t k = 0; k < (funcs[i].has_minhash ? sizeof(funcs[i].minhash) : 0); k++) {+ h ^= mh[k];+ h *= 1099511628211ULL;+ }+ for (int w = 0; w < funcs[i].tfidf_len; w++) {+ h ^= (uint64_t)funcs[i].tfidf_indices[w];+ h *= 1099511628211ULL;+ uint32_t bits;+ memcpy(&bits, &funcs[i].tfidf_weights[w], sizeof(bits));+ h ^= bits;+ h *= 1099511628211ULL;+ h_tfidf ^= (uint64_t)funcs[i].tfidf_indices[w];+ h_tfidf *= 1099511628211ULL;+ h_tfidf ^= bits;+ h_tfidf *= 1099511628211ULL;+ }+ }+ /* To a FILE, not the log: the semantic pass runs in the index WORKER,+ * whose log is removed when the run succeeds. */+ FILE *sf = cbm_fopen(state_path, "w");+ if (sf) {+ fprintf(sf, "all=%016llx codes=%016llx tfidf=%016llx n=%d\n", (unsigned long long)h,+ (unsigned long long)h_codes, (unsigned long long)h_tfidf, func_count);+ (void)fclose(sf);+ }+ }+}++/* Running hash of the token SEQUENCE handed to the corpus: the set AND the+ * order, because co-occurrence enrichment reads a window around each position,+ * so the same tokens in a different order produce different vectors while+ * leaving document frequencies (and therefore idf) untouched. Diagnostic for+ * the SEMANTICALLY_RELATED flicker; off unless CBM_SEM_STATE_DUMP is set. */+static uint64_t g_sem_token_hash = 1469598103934665603ULL;++static void sem_tokens_hash(char **tokens, const size_t *offsets, const int *counts, int n) {+ char flag[CBM_SZ_1K];+ if (!cbm_safe_getenv("CBM_SEM_STATE_DUMP", flag, sizeof(flag), NULL)) {+ return;+ }+ for (int d = 0; d < n; d++) {+ for (int k = 0; k < counts[d]; k++) {+ const char *tok = tokens[offsets[d] + (size_t)k];+ for (const unsigned char *c = (const unsigned char *)tok; c && *c; c++) {+ g_sem_token_hash ^= *c;+ g_sem_token_hash *= 1099511628211ULL;+ }+ g_sem_token_hash ^= 0xff; /* token boundary */+ g_sem_token_hash *= 1099511628211ULL;+ }+ g_sem_token_hash ^= 0xfe; /* document boundary */+ g_sem_token_hash *= 1099511628211ULL;+ }+}++/* Checksum the corpus AFTER enrichment: the per-function vectors and tf-idf are+ * both derived from it, so if this differs between two runs, everything+ * downstream differs and the cause is in the corpus build rather than in+ * scoring. Diagnostic, off unless CBM_SEM_STATE_DUMP is set. */+static void sem_corpus_dump(const cbm_sem_corpus_t *corpus, const char *suffix) {+ char state_path[CBM_SZ_1K];+ if (!cbm_safe_getenv("CBM_SEM_STATE_DUMP", state_path, sizeof(state_path), NULL)) {+ return;+ }+ char path[CBM_SZ_1K];+ if (snprintf(path, sizeof(path), "%s.%s", state_path, suffix) >= (int)sizeof(path)) {+ return;+ }+ int n = cbm_sem_corpus_token_count(corpus);+ uint64_t h_vec = 1469598103934665603ULL;+ uint64_t h_idf = 1469598103934665603ULL;+ for (int i = 0; i < n; i++) {+ const cbm_sem_vec_t *v = cbm_sem_corpus_ri_vec_at(corpus, i);+ if (v) {+ const unsigned char *b = (const unsigned char *)v;+ for (size_t k = 0; k < sizeof(*v); k++) {+ h_vec ^= b[k];+ h_vec *= 1099511628211ULL;+ }+ }+ uint32_t bits;+ float idf = cbm_sem_corpus_idf_at(corpus, i);+ memcpy(&bits, &idf, sizeof(bits));+ h_idf ^= bits;+ h_idf *= 1099511628211ULL;+ }+ FILE *sf = cbm_fopen(path, "w");+ if (sf) {+ fprintf(sf, "vec=%016llx idf=%016llx toks=%016llx tokens=%d\n", (unsigned long long)h_vec,+ (unsigned long long)h_idf, (unsigned long long)g_sem_token_hash, n);+ (void)fclose(sf);+ }+}+/* Phase 6a: score candidate pairs in parallel and collect deferred edges. */static void phase6a_score_candidates(cbm_sem_func_t *funcs, uint64_t *signatures, int *edge_counts,sem_bucket_t **band_buckets, cbm_sem_config_t cfg,@@ -1415,11 +1580,13 @@ static cbm_sem_corpus_t *run_corpus_phase(cbm_gbuf_t *gbuf, char **all_tokens,int func_count) {CBM_PROF_START(t_phase3a);cbm_sem_corpus_t *corpus = cbm_sem_corpus_new();+ sem_tokens_hash(all_tokens, offsets, token_counts, func_count);cbm_sem_corpus_add_docs_batch(corpus, all_tokens, offsets, token_counts, func_count);CBM_PROF_END_N("semantic_edges", "3a_corpus_batch", t_phase3a, func_count);CBM_PROF_START(t_phase3b);cbm_sem_corpus_finalize(corpus);+ sem_corpus_dump(corpus, "corpus");CBM_PROF_END_N("semantic_edges", "3b_corpus_finalize_seq", t_phase3b,cbm_sem_corpus_token_count(corpus));@@ -1449,6 +1616,7 @@ static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t *}CBM_PROF_START(t_phase6a);+ sem_state_dump(funcs, func_count);phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, worker_bufs,func_count, worker_count);CBM_PROF_END_N("semantic_edges", "6a_score_parallel", t_phase6a, func_count);@@ -1599,6 +1767,7 @@ static cbm_sem_corpus_t *run_token_phases_batched(cbm_gbuf_t *gbuf, cbm_sem_funcsem_batch_tokenize(node_ptrs, gbuf, first, count, worker_count, token_pools, token_counts,&tokens, &offsets);if (tokens && offsets) {+ sem_tokens_hash(tokens, offsets, token_counts + first, count);cbm_sem_corpus_add_docs_batch(corpus, tokens, offsets, token_counts + first, count);}sem_batch_free_tokens(tokens, offsets);@@ -1609,6 +1778,7 @@ static cbm_sem_corpus_t *run_token_phases_batched(cbm_gbuf_t *gbuf, cbm_sem_funcCBM_PROF_START(t_phase3b);cbm_sem_corpus_finalize(corpus);+ sem_corpus_dump(corpus, "corpus");CBM_PROF_END_N("semantic_edges", "3b_corpus_finalize_seq", t_phase3b,cbm_sem_corpus_token_count(corpus));CBM_PROF_START(t_phase3c);diff --git a/tests/test_vm_verdict_contract.sh b/tests/test_vm_verdict_contract.shnew file mode 100755index 000000000..3076eb820--- /dev/null+++ b/tests/test_vm_verdict_contract.sh@@ -0,0 +1,72 @@+#!/usr/bin/env bash+# The Windows leg's verdict must not be decided by the exit status alone.+#+# Why: that status travels ssh -> cmd.exe -> msys2_shell.cmd and the chain+# loses it. On 2026-09-18 a leg that printed "7925 passed, 0 failed" and+# "=== All tests passed ===" exited 1, while the identical work run through+# win.sh's other entry exited 0. A channel that turns 0 into 1 can turn 1 into+# 0, and that direction is a FALSE GREEN: a red Windows leg reported as+# passing, which is exactly the class of defect the local ladder exists to+# catch. The log is written by the runner on the VM and cannot be mangled in+# transit, so it decides the test outcome; rc only decides what the log cannot+# see.+#+# This drives the REAL function (vm-run-tests.sh --verdict) rather than a copy,+# so the contract cannot drift from the code it pins.+#+# Usage: tests/test_vm_verdict_contract.sh [repo-root]++set -uo pipefail++ROOT="${1:-$(cd "$(dirname "$0")/.." && pwd)}"+RUNNER="$ROOT/test-infrastructure/vm/vm-run-tests.sh"+if [ ! -f "$RUNNER" ]; then+ echo "FAIL: $RUNNER not found" >&2+ exit 1+fi++WORK=$(mktemp -d "${TMPDIR:-/tmp}/cbm-vm-verdict.XXXXXX") || exit 1+trap 'rm -rf "$WORK"' EXIT++printf ' 7925 passed, 0 failed, 74 skipped\n=== All tests passed ===\n' > "$WORK/green"+printf ' 7863 passed, 1 failed, 74 skipped\n' > "$WORK/red"+printf ' 7863 passed, 1 failed\n=== All tests passed ===\n' > "$WORK/red_with_marker"+printf ' 100 passed, 0 failed\n 200 passed, 3 failed\n=== All tests passed ===\n' > "$WORK/red_later_suite"+printf ' 7925 passed, 0 failed, 74 skipped\n' > "$WORK/no_marker"+printf 'configure: something exploded\n' > "$WORK/no_summary"++failures=0+expect() { # description log rc want+ local description="$1" log="$2" rc="$3" want="$4" got+ bash "$RUNNER" --verdict "$WORK/$log" "$rc" >/dev/null 2>&1+ got=$?+ if [ "$got" -ne "$want" ]; then+ echo "FAIL: $description — verdict $got, expected $want" >&2+ failures=$((failures + 1))+ fi+}++# The observed fault: a complete, green log whose exit status was lost. The+# leg is green; the transport problem is reported, not the tests.+expect "green log with a lost exit status is green" green 1 0+expect "green log with a clean exit status is green" green 0 0++# The dangerous direction. A zero exit status must never launder failures.+expect "failures in the log are red even when rc says 0" red 0 1+expect "failures in the log are red when rc says 1" red 1 1+expect "a completion marker cannot launder failures" red_with_marker 0 1+expect "failures in a LATER suite are still counted" red_later_suite 0 1++# Incomplete runs are never green: a leg that stopped after one suite has a+# summary but no marker.+expect "a summary without the completion marker is red" no_marker 0 1++# No summary at all keeps its own distinct code, so "never ran" stays+# distinguishable from "ran and failed".+expect "a log with no summary is the no-summary guard" no_summary 0 90++if [ "$failures" -gt 0 ]; then+ echo "VM verdict contract VIOLATED: $failures case(s)" >&2+ exit 1+fi+echo "VM verdict contract passed (8 cases: lost exit status, false green, partial runs)"