Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(mem): the 15 GB budget is met by its own metric, end to end
Martin Vogela2f29b3fe739
diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.cindex 50ef7fc58..3ef86bcdd 100644--- a/src/graph_buffer/graph_buffer.c+++ b/src/graph_buffer/graph_buffer.c@@ -98,6 +98,10 @@ struct cbm_gbuf {* strings at kernel scale, plus a snprintf+strdup+hash on every one of* the ~18 hot find_by_id call sites. */cbm_gbuf_node_t **by_id;+ /* Worker buffers (cbm_gbuf_new_worker) keep no by_id array: their ids+ * come from the shared counter, so a dense array would span the whole+ * global id space in every worker and double in lockstep. */+ bool by_id_off;int64_t by_id_cap;/* Secondary node indexes */@@ -485,22 +489,24 @@ static void cascade_delete_edges(cbm_gbuf_t *gb, CBMHashTable *deleted_set) {static void register_node_in_indexes(cbm_gbuf_t *gb, cbm_gbuf_node_t *node) {cbm_ht_set(gb->node_by_qn, node->qualified_name, node);- if (node->id >= gb->by_id_cap) {- int64_t nc = gb->by_id_cap > 0 ? gb->by_id_cap : CBM_SZ_1K;- while (nc <= node->id) {- nc *= 2;+ if (!gb->by_id_off) {+ if (node->id >= gb->by_id_cap) {+ int64_t nc = gb->by_id_cap > 0 ? gb->by_id_cap : CBM_SZ_1K;+ while (nc <= node->id) {+ nc *= 2;+ }+ cbm_gbuf_node_t **grown =+ cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id, (size_t)nc * sizeof(*grown));+ if (grown) {+ memset(grown + gb->by_id_cap, 0, (size_t)(nc - gb->by_id_cap) * sizeof(*grown));+ gb->by_id = grown;+ gb->by_id_cap = nc;+ }}- cbm_gbuf_node_t **grown =- cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id, (size_t)nc * sizeof(*grown));- if (grown) {- memset(grown + gb->by_id_cap, 0, (size_t)(nc - gb->by_id_cap) * sizeof(*grown));- gb->by_id = grown;- gb->by_id_cap = nc;+ if (node->id >= 0 && node->id < gb->by_id_cap) {+ gb->by_id[node->id] = node;}}- if (node->id >= 0 && node->id < gb->by_id_cap) {- gb->by_id[node->id] = node;- }node_ptr_array_t *by_label =get_or_create_node_array(gb->nodes_by_label, node->label ? node->label : "");@@ -652,6 +658,16 @@ cbm_gbuf_t *cbm_gbuf_new_worker(const char *project, const char *root_path,gb->edges_by_source_type = NULL;gb->edges_by_target_type = NULL;gb->edges_by_type = NULL;+ /* Nor by id: with ids from the shared counter, a dense id -> node array+ * in every worker spans the whole global id space (8.5M ids x 8 B x 18+ * workers on the kernel) and all of them double in the same instant --+ * a 1 GB step between two reads of the memory gate, which is where the+ * 15 GB budget was missed (2026-09-14). Nothing asks a worker buffer by+ * id before the merge; the main buffer answers after it. */+ cbm_free(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id);+ gb->by_id = NULL;+ gb->by_id_cap = 0;+ gb->by_id_off = true;return gb;}diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.cindex e3439f293..0cacf8148 100644--- a/src/pipeline/pass_parallel.c+++ b/src/pipeline/pass_parallel.c@@ -80,6 +80,7 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 };#include "foundation/compat.h"#include "foundation/compat_fs.h"#include "foundation/compat_thread.h"+#include "foundation/mem_core.h"#include "graph_buffer/graph_buffer.h"#include "service_patterns.h"#include "foundation/platform.h"@@ -931,6 +932,36 @@ static int pp_spill_sweep(extract_ctx_t *ec, int worker_id) {return parked;}+/* Diagnostic (CBM_MEM_PHASES=1): where does the charge go between the+ * near-budget latch and the first over-budget observation? One line per+ * 256 MB step of the charge above its last logged value while spill mode is+ * on, and one at the first over-budget observation, each with the+ * footprint / commit / tracked breakdown and the class table. */+static _Atomic size_t g_probe_last_mb = 0;+static _Atomic int g_probe_over_logged = 0;+static void pp_charge_probe(extract_ctx_t *ec, bool over) {+ if (!cbm_mem_phases_enabled() || !pp_spill_active(ec)) {+ return;+ }+ enum { PROBE_STEP_MB = 256, PROBE_MB = 1024 * 1024 };+ size_t charged_mb = cbm_mem_charged() / PROBE_MB;+ size_t last = atomic_load_explicit(&g_probe_last_mb, memory_order_relaxed);+ bool step = charged_mb >= last + PROBE_STEP_MB &&+ atomic_compare_exchange_strong_explicit(&g_probe_last_mb, &last, charged_mb,+ memory_order_relaxed, memory_order_relaxed);+ bool first_over =+ over && atomic_exchange_explicit(&g_probe_over_logged, 1, memory_order_relaxed) == 0;+ if (!step && !first_over) {+ return;+ }+ cbm_log_info("mem.charge.probe", "event", first_over ? "first_over" : "step", "charged_mb",+ itoa_log((int)charged_mb), "footprint_mb",+ itoa_log((int)(cbm_mem_footprint() / PROBE_MB)), "commit_mb",+ itoa_log((int)(cbm_mem_allocator_committed() / PROBE_MB)), "tracked_mb",+ itoa_log((int)(cbm_mem_tracked_live_bytes() / PROBE_MB)));+ cbm_mem_class_log(first_over ? "charge.first_over" : "charge.step");+}+static void extract_worker(int worker_id, void *ctx_ptr) {extract_ctx_t *ec = ctx_ptr;extract_worker_state_t *ws = &ec->workers[worker_id];@@ -974,6 +1005,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) {* and the previously serving index keeps answering. */if (cbm_mem_budget() > 0) {bool over = cbm_mem_over_budget();+ pp_charge_probe(ec, over);/* Anticipation: the gate sees the crossing per file pull, and the* workers' in-flight files carry the charge past the line before* the first sweep lands (kernel, 15 GB budget: high-water 15.65@@ -1401,6 +1433,19 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *filecbm_parallel_for_opts_t parallel_opts = {.max_workers = worker_count, .force_pthreads = false};cbm_scale_begin(&ec.scale, "parallel_extract", (long)file_count);cbm_parallel_for(worker_count, extract_worker, &ec, parallel_opts);+ if (pp_spill_active(&ec) &&+ !atomic_load_explicit(&ec.over_budget_abort, memory_order_relaxed)) {+ /* Spill mode was entered, so results belong on disk: park every+ * result still cached before the phases that cannot park (registry+ * build, resolve, the semantic pass) inherit them. The sweeps above+ * run only on an over-budget observation; a run that latched early+ * and then stayed under budget through extraction (kernel, 15 GB,+ * 2026-09-14: 14,949 MB at this point, 44,797 results = 8 GB still+ * cached) reached resolve with no headroom and aborted there. */+ int parked = pp_spill_sweep(&ec, 0);+ cbm_log_info("mem.spill.final_sweep", "parked", itoa_log(parked), "charged_mb",+ itoa_log((int)(cbm_mem_charged() / ((size_t)1024 * 1024))));+ }if (ctx->spill) {int64_t parked = 0;int64_t bytes = 0;diff --git a/tests/test_graph_buffer.c b/tests/test_graph_buffer.cindex 232286045..256fc0cec 100644--- a/tests/test_graph_buffer.c+++ b/tests/test_graph_buffer.c@@ -6,6 +6,8 @@*/#include "test_framework.h"#include "graph_buffer/graph_buffer.h"+#include "foundation/mem_core.h"+#include <stdatomic.h>#include "store/store.h"#include <string.h>@@ -1120,7 +1122,38 @@ TEST(gbuf_flush_skips_orphan_edges) {/* ── Suite ─────────────────────────────────────────────────────── */+/* A worker buffer draws ids from the shared counter, so a dense id -> node+ * array in it spans the whole global id space: 18 workers x (next power of+ * two above the highest id) x 8 B, doubling in lockstep -- a 1 GB step+ * inside one gate interval on the kernel at 8M ids (2026-09-14). A worker+ * buffer is never asked by id before the merge, so it keeps no such array;+ * the main buffer it merges into still answers by id. One node at id 2M+ * would cost a 16 MB array; the index class must not grow by even 1 MB. */+TEST(gbuf_worker_buffer_keeps_no_by_id_array) {+ _Atomic int64_t ids;+ atomic_init(&ids, (int64_t)1 << 21);+ size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_INDEX);+ cbm_gbuf_t *w = cbm_gbuf_new_worker("p", "/r", &ids);+ ASSERT_NOT_NULL(w);+ int64_t id = cbm_gbuf_upsert_node(w, "Function", "f", "p.f", "a.c", 1, 2, "{}");+ ASSERT_TRUE(id >= ((int64_t)1 << 21));+ size_t after = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_INDEX);+ size_t grown = after > before ? after - before : 0;+ ASSERT_TRUE(grown < ((size_t)1 << 20));+ ASSERT_TRUE(cbm_gbuf_find_by_id(w, id) == NULL);+ ASSERT_NOT_NULL(cbm_gbuf_find_by_qn(w, "p.f"));++ cbm_gbuf_t *main_gb = cbm_gbuf_new("p", "/r");+ ASSERT_NOT_NULL(main_gb);+ cbm_gbuf_merge(main_gb, w);+ ASSERT_NOT_NULL(cbm_gbuf_find_by_id(main_gb, id));+ cbm_gbuf_free(w);+ cbm_gbuf_free(main_gb);+ PASS();+}+SUITE(graph_buffer) {+ RUN_TEST(gbuf_worker_buffer_keeps_no_by_id_array);/* Original tests */RUN_TEST(gbuf_create_free);RUN_TEST(gbuf_free_null);