Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(mem): the 15 GB budget is met by its own metric, end to end

Martin Vogela2f29b3fe739
diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c
index 50ef7fc58..3ef86bcdd 100644
--- a/src/graph_buffer/graph_buffer.c
+++ b/src/graph_buffer/graph_buffer.c
@@ -98,6 +98,10 @@ struct cbm_gbuf {
* strings at kernel scale, plus a snprintf+strdup+hash on every one of
* the ~18 hot find_by_id call sites. */
cbm_gbuf_node_t **by_id;
+ /* Worker buffers (cbm_gbuf_new_worker) keep no by_id array: their ids
+ * come from the shared counter, so a dense array would span the whole
+ * global id space in every worker and double in lockstep. */
+ bool by_id_off;
int64_t by_id_cap;
/* Secondary node indexes */
@@ -485,22 +489,24 @@ static void cascade_delete_edges(cbm_gbuf_t *gb, CBMHashTable *deleted_set) {
static void register_node_in_indexes(cbm_gbuf_t *gb, cbm_gbuf_node_t *node) {
cbm_ht_set(gb->node_by_qn, node->qualified_name, node);
- if (node->id >= gb->by_id_cap) {
- int64_t nc = gb->by_id_cap > 0 ? gb->by_id_cap : CBM_SZ_1K;
- while (nc <= node->id) {
- nc *= 2;
+ if (!gb->by_id_off) {
+ if (node->id >= gb->by_id_cap) {
+ int64_t nc = gb->by_id_cap > 0 ? gb->by_id_cap : CBM_SZ_1K;
+ while (nc <= node->id) {
+ nc *= 2;
+ }
+ cbm_gbuf_node_t **grown =
+ cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id, (size_t)nc * sizeof(*grown));
+ if (grown) {
+ memset(grown + gb->by_id_cap, 0, (size_t)(nc - gb->by_id_cap) * sizeof(*grown));
+ gb->by_id = grown;
+ gb->by_id_cap = nc;
+ }
}
- cbm_gbuf_node_t **grown =
- cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id, (size_t)nc * sizeof(*grown));
- if (grown) {
- memset(grown + gb->by_id_cap, 0, (size_t)(nc - gb->by_id_cap) * sizeof(*grown));
- gb->by_id = grown;
- gb->by_id_cap = nc;
+ if (node->id >= 0 && node->id < gb->by_id_cap) {
+ gb->by_id[node->id] = node;
}
}
- if (node->id >= 0 && node->id < gb->by_id_cap) {
- gb->by_id[node->id] = node;
- }
node_ptr_array_t *by_label =
get_or_create_node_array(gb->nodes_by_label, node->label ? node->label : "");
@@ -652,6 +658,16 @@ cbm_gbuf_t *cbm_gbuf_new_worker(const char *project, const char *root_path,
gb->edges_by_source_type = NULL;
gb->edges_by_target_type = NULL;
gb->edges_by_type = NULL;
+ /* Nor by id: with ids from the shared counter, a dense id -> node array
+ * in every worker spans the whole global id space (8.5M ids x 8 B x 18
+ * workers on the kernel) and all of them double in the same instant --
+ * a 1 GB step between two reads of the memory gate, which is where the
+ * 15 GB budget was missed (2026-09-14). Nothing asks a worker buffer by
+ * id before the merge; the main buffer answers after it. */
+ cbm_free(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id);
+ gb->by_id = NULL;
+ gb->by_id_cap = 0;
+ gb->by_id_off = true;
return gb;
}
diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c
index e3439f293..0cacf8148 100644
--- a/src/pipeline/pass_parallel.c
+++ b/src/pipeline/pass_parallel.c
@@ -80,6 +80,7 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 };
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
#include "foundation/compat_thread.h"
+#include "foundation/mem_core.h"
#include "graph_buffer/graph_buffer.h"
#include "service_patterns.h"
#include "foundation/platform.h"
@@ -931,6 +932,36 @@ static int pp_spill_sweep(extract_ctx_t *ec, int worker_id) {
return parked;
}
+/* Diagnostic (CBM_MEM_PHASES=1): where does the charge go between the
+ * near-budget latch and the first over-budget observation? One line per
+ * 256 MB step of the charge above its last logged value while spill mode is
+ * on, and one at the first over-budget observation, each with the
+ * footprint / commit / tracked breakdown and the class table. */
+static _Atomic size_t g_probe_last_mb = 0;
+static _Atomic int g_probe_over_logged = 0;
+static void pp_charge_probe(extract_ctx_t *ec, bool over) {
+ if (!cbm_mem_phases_enabled() || !pp_spill_active(ec)) {
+ return;
+ }
+ enum { PROBE_STEP_MB = 256, PROBE_MB = 1024 * 1024 };
+ size_t charged_mb = cbm_mem_charged() / PROBE_MB;
+ size_t last = atomic_load_explicit(&g_probe_last_mb, memory_order_relaxed);
+ bool step = charged_mb >= last + PROBE_STEP_MB &&
+ atomic_compare_exchange_strong_explicit(&g_probe_last_mb, &last, charged_mb,
+ memory_order_relaxed, memory_order_relaxed);
+ bool first_over =
+ over && atomic_exchange_explicit(&g_probe_over_logged, 1, memory_order_relaxed) == 0;
+ if (!step && !first_over) {
+ return;
+ }
+ cbm_log_info("mem.charge.probe", "event", first_over ? "first_over" : "step", "charged_mb",
+ itoa_log((int)charged_mb), "footprint_mb",
+ itoa_log((int)(cbm_mem_footprint() / PROBE_MB)), "commit_mb",
+ itoa_log((int)(cbm_mem_allocator_committed() / PROBE_MB)), "tracked_mb",
+ itoa_log((int)(cbm_mem_tracked_live_bytes() / PROBE_MB)));
+ cbm_mem_class_log(first_over ? "charge.first_over" : "charge.step");
+}
+
static void extract_worker(int worker_id, void *ctx_ptr) {
extract_ctx_t *ec = ctx_ptr;
extract_worker_state_t *ws = &ec->workers[worker_id];
@@ -974,6 +1005,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
* and the previously serving index keeps answering. */
if (cbm_mem_budget() > 0) {
bool over = cbm_mem_over_budget();
+ pp_charge_probe(ec, over);
/* Anticipation: the gate sees the crossing per file pull, and the
* workers' in-flight files carry the charge past the line before
* the first sweep lands (kernel, 15 GB budget: high-water 15.65
@@ -1401,6 +1433,19 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
cbm_parallel_for_opts_t parallel_opts = {.max_workers = worker_count, .force_pthreads = false};
cbm_scale_begin(&ec.scale, "parallel_extract", (long)file_count);
cbm_parallel_for(worker_count, extract_worker, &ec, parallel_opts);
+ if (pp_spill_active(&ec) &&
+ !atomic_load_explicit(&ec.over_budget_abort, memory_order_relaxed)) {
+ /* Spill mode was entered, so results belong on disk: park every
+ * result still cached before the phases that cannot park (registry
+ * build, resolve, the semantic pass) inherit them. The sweeps above
+ * run only on an over-budget observation; a run that latched early
+ * and then stayed under budget through extraction (kernel, 15 GB,
+ * 2026-09-14: 14,949 MB at this point, 44,797 results = 8 GB still
+ * cached) reached resolve with no headroom and aborted there. */
+ int parked = pp_spill_sweep(&ec, 0);
+ cbm_log_info("mem.spill.final_sweep", "parked", itoa_log(parked), "charged_mb",
+ itoa_log((int)(cbm_mem_charged() / ((size_t)1024 * 1024))));
+ }
if (ctx->spill) {
int64_t parked = 0;
int64_t bytes = 0;
diff --git a/tests/test_graph_buffer.c b/tests/test_graph_buffer.c
index 232286045..256fc0cec 100644
--- a/tests/test_graph_buffer.c
+++ b/tests/test_graph_buffer.c
@@ -6,6 +6,8 @@
*/
#include "test_framework.h"
#include "graph_buffer/graph_buffer.h"
+#include "foundation/mem_core.h"
+#include <stdatomic.h>
#include "store/store.h"
#include <string.h>
@@ -1120,7 +1122,38 @@ TEST(gbuf_flush_skips_orphan_edges) {
/* ── Suite ─────────────────────────────────────────────────────── */
+/* A worker buffer draws ids from the shared counter, so a dense id -> node
+ * array in it spans the whole global id space: 18 workers x (next power of
+ * two above the highest id) x 8 B, doubling in lockstep -- a 1 GB step
+ * inside one gate interval on the kernel at 8M ids (2026-09-14). A worker
+ * buffer is never asked by id before the merge, so it keeps no such array;
+ * the main buffer it merges into still answers by id. One node at id 2M
+ * would cost a 16 MB array; the index class must not grow by even 1 MB. */
+TEST(gbuf_worker_buffer_keeps_no_by_id_array) {
+ _Atomic int64_t ids;
+ atomic_init(&ids, (int64_t)1 << 21);
+ size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_INDEX);
+ cbm_gbuf_t *w = cbm_gbuf_new_worker("p", "/r", &ids);
+ ASSERT_NOT_NULL(w);
+ int64_t id = cbm_gbuf_upsert_node(w, "Function", "f", "p.f", "a.c", 1, 2, "{}");
+ ASSERT_TRUE(id >= ((int64_t)1 << 21));
+ size_t after = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_INDEX);
+ size_t grown = after > before ? after - before : 0;
+ ASSERT_TRUE(grown < ((size_t)1 << 20));
+ ASSERT_TRUE(cbm_gbuf_find_by_id(w, id) == NULL);
+ ASSERT_NOT_NULL(cbm_gbuf_find_by_qn(w, "p.f"));
+
+ cbm_gbuf_t *main_gb = cbm_gbuf_new("p", "/r");
+ ASSERT_NOT_NULL(main_gb);
+ cbm_gbuf_merge(main_gb, w);
+ ASSERT_NOT_NULL(cbm_gbuf_find_by_id(main_gb, id));
+ cbm_gbuf_free(w);
+ cbm_gbuf_free(main_gb);
+ PASS();
+}
+
SUITE(graph_buffer) {
+ RUN_TEST(gbuf_worker_buffer_keeps_no_by_id_array);
/* Original tests */
RUN_TEST(gbuf_create_free);
RUN_TEST(gbuf_free_null);