Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(mem): dedicated SQLite heap only in the worker; guard the committed stat

Martin Vogel9724d9030466
diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c
index 71bb4bb2f..52614d74c 100644
--- a/internal/cbm/cbm.c
+++ b/internal/cbm/cbm.c
@@ -300,6 +300,20 @@ static TSParser *get_thread_parser(const TSLanguage *ts_lang, CBMLanguage lang)
* to mimalloc would mismatch ASan/CRT frees — there these binds compile to
* no-ops and the build stays unchanged. */
+/* SQLite on a dedicated mimalloc heap per thread: ON only in the index worker,
+ * whose default heap holds the graph (SQLite churn on that heap paid a page
+ * walk per allocation: 132 s vs 9.7 s on the kernel's coverage publish). OFF
+ * everywhere else: the daemon runs a thread per connection, and a heap
+ * created per such thread pins every SQLite block the shared connection
+ * keeps (page cache, statement cache) to pages nobody's heap owns any more --
+ * the Linux soak grew 180 KB per query, 11 -> 144 MB in ten minutes, where
+ * the default thread heap had been flat (2026-09-14). */
+static _Atomic int g_sqlite_dedicated_heap;
+
+void cbm_sqlite_dedicated_heap(bool on) {
+ atomic_store_explicit(&g_sqlite_dedicated_heap, on ? 1 : 0, memory_order_relaxed);
+}
+
#if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR
#include <assert.h>
@@ -322,15 +336,20 @@ static TSParser *get_thread_parser(const TSLanguage *ts_lang, CBMLanguage lang)
* thread's heap is released with the thread. */
static _Thread_local mi_heap_t *tl_sqlite_heap;
+/* NULL = the calling thread's default heap (mi_malloc); see the switch above. */
static mi_heap_t *sqlite_heap(void) {
+ if (!atomic_load_explicit(&g_sqlite_dedicated_heap, memory_order_relaxed)) {
+ return NULL;
+ }
if (!tl_sqlite_heap) {
tl_sqlite_heap = mi_heap_new();
}
- return tl_sqlite_heap ? tl_sqlite_heap : mi_heap_main();
+ return tl_sqlite_heap;
}
static void *cbm_sqlite_malloc(int n) {
- void *block = mi_heap_malloc(sqlite_heap(), (size_t)n);
+ mi_heap_t *heap = sqlite_heap();
+ void *block = heap ? mi_heap_malloc(heap, (size_t)n) : mi_malloc((size_t)n);
if (block) {
cbm_mem_class_add_external(CBM_MEM_CLASS_STORE, mi_usable_size(block));
}
@@ -344,7 +363,8 @@ static void cbm_sqlite_free(void *p) {
}
static void *cbm_sqlite_realloc(void *p, int n) {
size_t old_size = p ? mi_usable_size(p) : 0;
- void *grown = mi_heap_realloc(sqlite_heap(), p, (size_t)n);
+ mi_heap_t *heap = sqlite_heap();
+ void *grown = heap ? mi_heap_realloc(heap, p, (size_t)n) : mi_realloc(p, (size_t)n);
if (grown) {
cbm_mem_class_remove_external(CBM_MEM_CLASS_STORE, old_size);
cbm_mem_class_add_external(CBM_MEM_CLASS_STORE, mi_usable_size(grown));
diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h
index c62d3f281..d875459c1 100644
--- a/internal/cbm/cbm.h
+++ b/internal/cbm/cbm.h
@@ -669,6 +669,12 @@ typedef struct {
// also calls it so non-main entry points (pipeline passes) still get the binds.
// In the test build (no CBM_BIND_TS_ALLOCATOR) this is a no-op.
void cbm_alloc_init(void);
+/* SQLite allocates from a dedicated mimalloc heap per thread while on; the
+ * index worker turns it on (its default heap holds the graph). Off elsewhere:
+ * a thread-per-connection daemon would pin connection-lifetime blocks to
+ * dead threads. The switch exists in every build; it changes nothing where
+ * the allocator binds are compiled out. */
+void cbm_sqlite_dedicated_heap(bool on);
// Initialize the library. Call once at startup. Returns 0 on success.
int cbm_init(void);
diff --git a/src/foundation/diagnostics.c b/src/foundation/diagnostics.c
index 5b2ee2cb8..77f16c5bc 100644
--- a/src/foundation/diagnostics.c
+++ b/src/foundation/diagnostics.c
@@ -23,6 +23,7 @@
#include <mimalloc.h>
#include <stdatomic.h>
#include <stdio.h>
+#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
@@ -538,6 +539,20 @@ static void write_diagnostics(void) {
* macOS/Windows, and peak is reconciled to never undercut current. */
current_rss = cbm_mem_rss();
peak_rss = cbm_mem_peak_rss();
+ /* The committed pair has the same failure: the counter is signed and
+ * merged per thread at thread exit, so a thread-per-connection daemon
+ * reads it negative once short-lived threads have freed what a long-lived
+ * one committed (query-leak soak, Linux 2026-09-14: heap_committed_bytes
+ * printed 2^64 - 121 MB from the second sample on). cbm_mem_allocator_
+ * committed reports 0 for a negative reading; the peak never undercuts
+ * the current value and never carries a wrapped one. */
+ current_commit = cbm_mem_allocator_committed();
+ if (peak_commit > (SIZE_MAX >> 1)) {
+ peak_commit = 0;
+ }
+ if (peak_commit < current_commit) {
+ peak_commit = current_commit;
+ }
int fds = count_open_fds();
time_t now = time(NULL);
diff --git a/src/foundation/mem.c b/src/foundation/mem.c
index 1c044c071..753e101f9 100644
--- a/src/foundation/mem.c
+++ b/src/foundation/mem.c
@@ -515,6 +515,15 @@ void cbm_mem_set_budget_for_tests(size_t bytes) {
size_t cbm_mem_allocator_committed(void) {
size_t commit = 0;
mi_process_info(NULL, NULL, NULL, NULL, NULL, &commit, NULL, NULL);
+ /* The statistic behind this is a signed counter merged per thread at
+ * thread exit; a process whose long-lived thread commits what its
+ * short-lived threads free reads it NEGATIVE, cast to size_t here. The
+ * daemon's query-leak soak reported 2^64 - 121 MB from the second sample
+ * on (Linux, 2026-09-14). A negative reading is no reading: report 0 so
+ * the charge falls back to the OS number instead of a 16 EB budget breach. */
+ if (commit > (SIZE_MAX >> 1)) {
+ return 0;
+ }
return commit;
}
diff --git a/src/main.c b/src/main.c
index 284a899cb..202aa6a18 100644
--- a/src/main.c
+++ b/src/main.c
@@ -854,6 +854,11 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo
const char *worker_marker = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_MARKER_ARG);
const char *worker_quarantine =
cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_QUARANTINE_ARG);
+ if (index_worker) {
+ /* The graph lives on this process's heaps: SQLite gets heaps of its
+ * own here, and only here (cbm_sqlite_dedicated_heap). */
+ cbm_sqlite_dedicated_heap(true);
+ }
cbm_index_set_worker_role_options(index_worker, response_out, worker_single_thread,
worker_marker, worker_quarantine,
cbm_index_worker_memory_budget_bytes());