Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
/*
* mem.c — Unified memory management via mimalloc.
*
* Budget tracking based on actual RSS via mi_process_info().
* When MI_OVERRIDE=0 (ASan builds), falls back to OS-specific
* RSS queries (task_info on macOS, /proc/self/statm on Linux,
* GetProcessMemoryInfo on Windows).
*/
#include "foundation/mem_events.h"
#include "mem.h"
#include "mem_core.h" /* cbm_mem_tracked_live_bytes */
#include "platform.h"
#include "log.h"
#include "compat_fs.h"
#include "compat_thread.h"
#include "foundation/constants.h"
#define MAX_RAM_FRACTION 1.0
#define DEFAULT_RAM_FRACTION 0.5
#include <errno.h>
#include <mimalloc.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef _WIN32
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h>
#include <psapi.h>
#elif defined(__APPLE__)
#include <mach/mach.h>
#include <malloc/malloc.h> /* malloc_zone_pressure_relief */
#else
#include <unistd.h>
#endif
/* Does THIS build ask mimalloc to replace ordinary malloc process-wide?
*
* Set by Makefile.cbm alongside MI_MALLOC_OVERRIDE itself — the two are
* switched together in one place, because MI_MALLOC_OVERRIDE reaches only the
* mimalloc translation unit and this one needs the same answer. Do NOT infer it
* from the platform here: the override is compiled into PROD builds only
* (MIMALLOC_CFLAGS_TEST uses -DMI_OVERRIDE=0 and no override define), so a
* Linux test binary genuinely has no global override, and a platform-based
* guess would make every test run warn about a build that is correct.
*
* macOS never gets it, and that is not a defect to warn about: enabling the
* override there compiles alloc-override.c's forwarding definitions, and under
* the two-level namespace this binary's free becomes mi_free while system
* libraries keep allocating from the system allocator, so the first pointer
* crossing that boundary aborts with "mi_free: invalid pointer". ELF's flat
* namespace has no such split, which is why Linux can have it and macOS cannot.
*
* The shipped artifact's actual wiring is pinned by scripts/smoke-test.sh
* Phase 1b, which fails in BOTH directions on the real binary — the only place
* this can be checked honestly, since a from-source test build never has it. */
#if defined(CBM_MEM_GLOBAL_OVERRIDE)
#define CBM_MEM_EXPECT_GLOBAL_OVERRIDE 1
#else
#define CBM_MEM_EXPECT_GLOBAL_OVERRIDE 0
#endif
#ifdef _WIN32
/* mimalloc internals. Not in the public header, but the allocator is compiled
* into this binary as a unity object, so the symbols are present.
*
* WHY WE REACH FOR THEM: arena creation (arena.c:515) and arena purging
* (arena.c:2060) are BOTH gated on `_mi_preloading()`. If that flag never
* clears, no arena is ever created and purge never runs — memory comes
* straight from the OS and is never given back, which is exactly the Windows
* behaviour in #581 and why no amount of purge configuration helped. The flag
* is cleared only by `_mi_auto_process_init()`, which upstream invokes from a
* `__attribute__((constructor))` for clang/gcc and from `.CRT$XIU` TLS
* callbacks for MSVC. Whether that constructor actually runs in our static
* MinGW link is the open question, so ask the allocator rather than assume. */
extern bool _mi_preloading(void);
extern void _mi_auto_process_init(void);
#endif
/* ── Static state ─────────────────────────────────────────────── */
static size_t g_budget; /* budget in bytes */
static atomic_int g_initialized; /* init guard */
static atomic_int g_was_over; /* pressure hysteresis */
#define MB_DIVISOR ((size_t)(CBM_SZ_1K * CBM_SZ_1K))
/* ── OS fallback for RSS (ASan builds where MI_OVERRIDE=0) ──── */
static size_t os_rss(void) {
#ifdef _WIN32
PROCESS_MEMORY_COUNTERS pmc;
if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc))) {
return (size_t)pmc.WorkingSetSize;
}
return 0;
#elif defined(__APPLE__)
struct mach_task_basic_info info = {0};
mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT;
if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &count) ==
KERN_SUCCESS) {
return (size_t)info.resident_size;
}
return 0;
#else
FILE *f = fopen("/proc/self/statm", "r");
if (!f) {
return 0;
}
unsigned long pages = 0;
unsigned long rss_pages = 0;
if (fscanf(f, "%lu %lu", &pages, &rss_pages) != 2) {
rss_pages = 0;
}
(void)fclose(f);
long ps = sysconf(_SC_PAGESIZE);
return rss_pages * (ps > 0 ? (size_t)ps : CBM_SZ_4K);
#endif
}
/* ── Pressure logging (hysteresis) ────────────────────────────── */
static void check_pressure(size_t rss) {
if (g_budget == 0) {
return;
}
bool over = rss > g_budget;
int was = atomic_load(&g_was_over);
if (over && !was) {
atomic_store(&g_was_over, 1);
char rss_mb[CBM_SZ_32];
char budget_mb[CBM_SZ_32];
char pct_str[CBM_SZ_16];
snprintf(rss_mb, sizeof(rss_mb), "%zu", rss / MB_DIVISOR);
snprintf(budget_mb, sizeof(budget_mb), "%zu", g_budget / MB_DIVISOR);
snprintf(pct_str, sizeof(pct_str), "%zu",
g_budget > 0 ? (rss * CBM_PERCENT) / g_budget : 0);
cbm_log_warn("mem.pressure.warn", "rss_mb", rss_mb, "budget_mb", budget_mb, "pct", pct_str);
} else if (!over && was) {
atomic_store(&g_was_over, 0);
char rss_mb[CBM_SZ_32];
char budget_mb[CBM_SZ_32];
char pct_str[CBM_SZ_16];
snprintf(rss_mb, sizeof(rss_mb), "%zu", rss / MB_DIVISOR);
snprintf(budget_mb, sizeof(budget_mb), "%zu", g_budget / MB_DIVISOR);
snprintf(pct_str, sizeof(pct_str), "%zu",
g_budget > 0 ? (rss * CBM_PERCENT) / g_budget : 0);
cbm_log_info("mem.pressure.ok", "rss_mb", rss_mb, "budget_mb", budget_mb, "pct", pct_str);
}
}
/* Allocator setup is not best-effort. A misconfigured allocator does not
* degrade gracefully — it grows without bound until the OS kills the process
* (#581 ran for months exactly this way, silently). So every step of the setup
* is VERIFIED and a failure is fatal here rather than a slow death later.
* There is deliberately no timeout or retry: these checks are synchronous, so
* "not true yet" and "never going to be true" are the same answer. */
static void mem_setup_fatal(const char *what, const char *detail) {
cbm_log_error("mem.allocator.setup_failed", "check", what, "detail", detail);
/* Never continue with an allocator that cannot return memory: unbounded
* growth is a worse failure than refusing to start. */
abort();
}
/* Set an option and prove it took effect; a silently-ignored option is the
* failure mode that made #581 invisible. */
static void mem_option_set_verified(mi_option_t option, long value, const char *name) {
mi_option_set(option, value);
if (mi_option_get(option) != value) {
char detail[CBM_SZ_128];
snprintf(detail, sizeof(detail), "%s did not take effect (wanted %ld, got %ld)", name,
value, mi_option_get(option));
mem_setup_fatal("option", detail);
}
}
/* ── Public API ────────────────────────────────────────────────── */
#define RAM_FRACTION_DEFAULT 0.5
#define RAM_FRACTION_16GB 0.25
#define RAM_FRACTION_32GB 0.35
#define RAM_BYTES_PER_GB (1024ULL * 1024 * 1024)
/* Bounds for the free-memory clamp (cbm_mem_clamp_to_available). */
#define CBM_MEM_AVAIL_HEADROOM_MIN ((size_t)RAM_BYTES_PER_GB)
#define CBM_MEM_AVAIL_HEADROOM_MAX ((size_t)(8ULL * RAM_BYTES_PER_GB))
#define CBM_MEM_BUDGET_FLOOR ((size_t)(RAM_BYTES_PER_GB / 2)) /* 512 MB */
double cbm_mem_ram_fraction_for_total(size_t total_ram_bytes) {
if (total_ram_bytes <= 16ULL * RAM_BYTES_PER_GB) {
return RAM_FRACTION_16GB;
}
if (total_ram_bytes <= 32ULL * RAM_BYTES_PER_GB) {
return RAM_FRACTION_32GB;
}
return RAM_FRACTION_DEFAULT;
}
cbm_mem_budget_t cbm_mem_resolve_budget(size_t total_ram, double ram_fraction,
const char *budget_mb) {
if (ram_fraction <= 0.0 || ram_fraction > MAX_RAM_FRACTION) {
ram_fraction = DEFAULT_RAM_FRACTION;
}
cbm_mem_budget_t result = {
.budget = (size_t)((double)total_ram * ram_fraction),
.source = "ram_fraction",
.clamped = false,
.invalid = false,
.hard_capped = false,
};
if (budget_mb == NULL || budget_mb[0] == '\0') {
return result; /* no override → fraction-derived budget */
}
/* Strict parse, matching the src/foundation/limits.c convention: reject
* trailing garbage (`*end`), overflow (errno==ERANGE), and non-positive
* values. This turns a fat-fingered value (e.g. "8GB", or a 20-digit typo)
* into a clean fallback-with-warning rather than a silently wrong budget. */
errno = 0;
char *end = NULL;
long long want_mb = strtoll(budget_mb, &end, CBM_DECIMAL_BASE);
if (errno != 0 || end == budget_mb || *end != '\0' || want_mb <= 0) {
result.invalid = true; /* keep the fraction-derived budget */
return result;
}
result.source = "CBM_MEM_BUDGET_MB";
size_t want = (size_t)want_mb;
if (total_ram > 0) {
/* Compare in MiB space so a valid-but-huge request (e.g. 2^44 MiB, which
* would overflow the ×MiB byte multiply) clamps cleanly to total_ram. */
if (want > total_ram / MB_DIVISOR) {
result.budget = total_ram;
result.clamped = true;
} else {
result.budget = want * MB_DIVISOR;
}
} else if (want > SIZE_MAX / MB_DIVISOR) {
/* RAM detection failed (no clamp target) and the request is
* astronomically large — cap at SIZE_MAX rather than wrap. */
result.budget = SIZE_MAX;
} else {
result.budget = want * MB_DIVISOR;
}
return result;
}
/* A budget derived from TOTAL ram plans to use memory that may already belong
* to something else. Measured 2026-09-18 on a 48 GB host: the default budget
* (50% = 24 GB) was sized while a 12 GiB VM and a second VM were running, the
* kernel index took its full 24.5 GB RSS, and the machine ran out — the same
* run completed at the same budget once the VM was stopped. Total RAM is the
* CEILING; what is free right now is the CONSTRAINT.
*
* Headroom is a quarter of what is free, held between 1 and 8 GB: enough that
* the OS, its file cache and the user's editor are not squeezed out, without
* making a large-memory machine behave like a small one. A budget is never
* clamped below CBM_MEM_BUDGET_FLOOR — below that nothing indexes at all, and
* refusing to start is worse than trying and spilling. available == 0 means
* the platform could not answer, and a guess is not better than the ceiling. */
size_t cbm_mem_clamp_to_available(size_t budget, size_t available) {
if (available == 0) {
return budget;
}
size_t headroom = available / 4;
if (headroom < CBM_MEM_AVAIL_HEADROOM_MIN) {
headroom = CBM_MEM_AVAIL_HEADROOM_MIN;
} else if (headroom > CBM_MEM_AVAIL_HEADROOM_MAX) {
headroom = CBM_MEM_AVAIL_HEADROOM_MAX;
}
size_t cap = available > headroom ? available - headroom : 0;
if (cap < CBM_MEM_BUDGET_FLOOR) {
cap = CBM_MEM_BUDGET_FLOOR;
}
return budget < cap ? budget : cap;
}
cbm_mem_budget_t cbm_mem_resolve_budget_capped(size_t total_ram, double ram_fraction,
const char *budget_mb, size_t hard_cap_bytes) {
cbm_mem_budget_t result = cbm_mem_resolve_budget(total_ram, ram_fraction, budget_mb);
/* The parent already divided the aggregate budget (env override or
* ram_fraction) across job slots. That per-slot share is the hard cap:
* N workers × a per-worker absolute override would oversubscribe the host
* (#1654). A lower explicit value still wins. Keep CBM_MEM_BUDGET_MB as
* the source when the env discriminator fired so the ceiling is visible
* as the user's aggregate, not as a silent daemon_worker_cap rewrite. */
if (hard_cap_bytes > 0 && (result.budget == 0 || result.budget > hard_cap_bytes)) {
bool explicit_override =
result.source != NULL && strcmp(result.source, "CBM_MEM_BUDGET_MB") == 0;
result.budget = hard_cap_bytes;
if (!explicit_override) {
result.source = "daemon_worker_cap";
}
result.hard_capped = true;
}
return result;
}
void cbm_mem_init_with_cap(double ram_fraction, size_t hard_cap_bytes) {
int expected = 0;
if (!atomic_compare_exchange_strong(&g_initialized, &expected, 1)) {
return;
}
if (ram_fraction <= 0.0 || ram_fraction > MAX_RAM_FRACTION) {
ram_fraction = DEFAULT_RAM_FRACTION;
}
/* Reduce upfront memory: don't eagerly commit arenas.
* Force decommit on purge (MADV_FREE_REUSABLE on macOS) so RSS
* drops immediately instead of staying high until memory pressure. */
#ifdef _WIN32
/* Decisive check: is the allocator still "preloading"? If so its arena
* machinery — including every purge path — is inert, and completing init
* here is what turns the options above from decoration into behaviour. */
if (_mi_preloading()) {
_mi_auto_process_init();
if (_mi_preloading()) {
/* Arena creation and every purge path stay disabled while this is
* set, so the process would grow until the OS killed it. Refuse. */
mem_setup_fatal("preloading",
"allocator remained in preloading state after explicit process "
"init: arena creation and purging are disabled, memory would "
"never be returned to the OS");
}
cbm_log_warn("mem.allocator.preloading_completed", "still_preloading", "false", "detail",
"allocator was still preloading, so arena creation and purging were "
"disabled; process init completed explicitly");
} else {
cbm_log_info("mem.allocator.preloading", "state", "already_complete");
}
#endif
/* Lazy arena commit costs address-space FRAGMENTATION, and on Linux we now
* pay it for every allocation in the process. mimalloc commits a range with
* mprotect(PROT_READ|PROT_WRITE) over a sub-range of a PROT_NONE reservation
* (prim/unix/prim.c), and each partial commit SPLITS the reserved VMA. While
* mimalloc only served the bound populations (sqlite, tree_sitter) that was a
* handful of mappings. Since #1360 routed ordinary malloc/new through
* mimalloc on Linux, an index worker peaked at ~22k mappings against
* v0.9.0's 10 (Go corpus, 18 workers). The count tracks CONCURRENCY, not
* corpus size — sampled mid-run it was 999 at 1 worker, 8460 at 4 and 11965
* at 18, while v0.9.0 stayed at 10 regardless — so every worker thread
* fragments the address space independently.
*
* Two consequences, both reported as #1654 on a 96-CPU/376 GB host: the
* mmap/mprotect churn serialises on the kernel's per-process mmap_lock (the
* reporter saw 1.2% of extraction in 45 minutes, where v0.9.0 finished the
* tree in ~13), and the VMA count climbs toward vm.max_map_count, after
* which mmap fails for ANY size — hence mimalloc reporting it "cannot
* allocate" 10 KB while `free -g` still showed 246 GB available.
*
* mimalloc's own default is 2, meaning "eager-commit arenas only on an OS
* with overcommit (i.e. linux)" — precisely because commit is free there
* until the pages are touched. Overriding it to 0 opted Linux out of the
* default written for it. Restore the default on Linux; keep the lazy
* setting everywhere else, where commit is NOT free and the upfront-memory
* reason for it still holds (Windows especially — see #581). */
#if defined(__linux__)
mem_option_set_verified(mi_option_arena_eager_commit, 2, "arena_eager_commit");
#else
mem_option_set_verified(mi_option_arena_eager_commit, 0, "arena_eager_commit");
#endif
mem_option_set_verified(mi_option_purge_decommits, SKIP_ONE, "purge_decommits");
mem_option_set_verified(mi_option_purge_delay, 0, "purge_delay"); /* immediate */
/* v3 (#832): reclaim abandoned pages on ANY thread's free (=1), restoring the
* v2 behaviour. mimalloc v3 defaults page_reclaim_on_free=0, so pages a worker
* thread abandons at exit are NOT reclaimed when the main thread later frees
* their blocks (and mi_collect cannot touch abandoned pages) — RSS then
* ratchets across repeated in-process index cycles. The supervised subprocess
* is the primary cure (the child returns 100% RSS on exit); this is the
* in-process fallback for any path that stays in-process (kill switch,
* spawn-fail degrade, embedders). */
mem_option_set_verified(mi_option_page_reclaim_on_free, 1, "page_reclaim_on_free");
/* Every option above is inert unless ordinary malloc actually reaches this
* allocator. That silently stopped being true on Windows — mimalloc's
* static override is gated on _MSC_VER, which clang/MinGW never defines —
* and nothing checked, so a long-lived daemon ratcheted committed memory
* for months (#581). Probe it once, out loud: a real malloc asked whether
* mimalloc owns it. Anything but true means the tuning here is decoration
* and freed pages will not come back.
*
* ...but "anything but true" only means that where the build actually ASKED
* for a global override: prod builds on Windows and Linux. Everywhere else —
* macOS, and every test binary on any platform — the override body is
* compiled out, so ordinary malloc is SUPPOSED to reach libc and the probe
* is answering a question this build never posed. Same measurement,
* different meaning per build config, hence the split below. */
cbm_mem_ownership_audit_t audit;
cbm_mem_audit_ownership(&audit);
char owned_str[CBM_SZ_32];
snprintf(owned_str, sizeof(owned_str), "%d/%d", audit.owned_count, audit.probed_count);
if (!audit.all_owned && !CBM_MEM_EXPECT_GLOBAL_OVERRIDE) {
/* Expected: this build never asked mimalloc to replace ordinary malloc,
* so ordinary malloc reaching libc is the design, not a fault. Say what
* IS allocator-served instead, because the interesting question here is
* "are the bound populations bound", not "did the override fire".
*
* Reported at INFO deliberately. A warning that fires on every run of a
* correctly configured build is not a tripwire, it is background noise —
* it trained readers to ignore the one line that catches #581, and cost
* a user the time to file and self-close #1360. */
cbm_log_info("mem.allocator.bound_populations_only", "owned_classes", owned_str,
"populations", "sqlite,tree_sitter", "detail",
"ordinary malloc is served by the system allocator in this build by "
"design; allocator tuning applies to the bound populations");
} else if (!audit.all_owned) {
/* Name the classes that escaped, because "not owned" is actionable
* only if you know WHICH sizes. A class listed here either bypasses
* the override or is misclassified by the routing predicate, and both
* mean freed pages will not come back. */
char classes[CBM_SZ_256];
int length = 0;
for (int i = 0;
i < CBM_MEM_OWNERSHIP_CLASSES && length >= 0 && (size_t)length < sizeof(classes);
i++) {
if (audit.allocated[i] && !audit.owned[i]) {
int written = snprintf(classes + length, sizeof(classes) - (size_t)length, "%s%zu",
length ? "," : "", audit.probe_bytes[i]);
if (written < 0) {
break;
}
length += written;
}
}
cbm_log_warn("mem.allocator.not_owned", "owned_classes", owned_str, "unowned_bytes",
length > 0 ? classes : "?", "detail",
"allocations in these size classes are not allocator-owned: purge and "
"reclaim options do not apply to them and freed pages stay committed");
} else {
cbm_log_info("mem.allocator.owned", "classes", "all");
}
/* CBM_MEM_BUDGET_MB env override (memory analogue of CBM_WORKERS).
* Lets users cap the budget directly without an enclosing cgroup —
* useful on bare-metal hosts where cgroup memory limits are absent
* (#363). Explicit override > implicit RAM/cgroup detection. The budget
* math (fraction default, override, clamp-to-total) lives in the pure,
* testable cbm_mem_resolve_budget(); this path only reads the env and
* emits the log/warn lines. */
cbm_system_info_t info = cbm_system_info();
char env_buf[CBM_SZ_32];
const char *env = cbm_safe_getenv("CBM_MEM_BUDGET_MB", env_buf, sizeof(env_buf), NULL);
cbm_mem_budget_t resolved =
cbm_mem_resolve_budget_capped(info.total_ram, ram_fraction, env, hard_cap_bytes);
/* A budget derived from total RAM is a plan made in ignorance of what the
* machine is already doing. Clamp the DERIVED budget to what is actually
* free; an explicit CBM_MEM_BUDGET_MB is the user's deliberate choice and
* still wins, but it is warned about when it exceeds what is free, because
* the failure it buys (the OS killing the worker) reads like a product bug
* rather than a setting. */
size_t available = cbm_system_available_ram();
bool explicit_budget =
resolved.source != NULL && strcmp(resolved.source, "CBM_MEM_BUDGET_MB") == 0;
if (!explicit_budget) {
size_t clamped = cbm_mem_clamp_to_available(resolved.budget, available);
if (clamped < resolved.budget) {
char want_mb[CBM_SZ_32];
char avail_mb[CBM_SZ_32];
snprintf(want_mb, sizeof(want_mb), "%zu", resolved.budget / MB_DIVISOR);
snprintf(avail_mb, sizeof(avail_mb), "%zu", available / MB_DIVISOR);
cbm_log_info("mem.budget.available_clamp", "from_mb", want_mb, "available_mb", avail_mb,
"detail", "budget reduced to fit memory that is actually free");
resolved.budget = clamped;
resolved.source = "available_ram";
}
} else if (available > 0 && resolved.budget > available) {
char want_mb[CBM_SZ_32];
char avail_mb[CBM_SZ_32];
snprintf(want_mb, sizeof(want_mb), "%zu", resolved.budget / MB_DIVISOR);
snprintf(avail_mb, sizeof(avail_mb), "%zu", available / MB_DIVISOR);
cbm_log_warn("mem.budget.over_available", "budget_mb", want_mb, "available_mb", avail_mb,
"detail",
"explicit budget exceeds free memory: the OS may kill this process before "
"the budget is ever reached");
}
g_budget = resolved.budget;
/* The resolver is the single source of truth for the parse + clamp; this
* path only surfaces its outcome as log lines. */
if (resolved.invalid) {
cbm_log_warn("mem.budget.env.invalid", "value", env, "fallback", "ram_fraction");
} else if (resolved.clamped) {
char cap_mb[CBM_SZ_32];
snprintf(cap_mb, sizeof(cap_mb), "%zu", info.total_ram / MB_DIVISOR);
cbm_log_warn("mem.budget.clamped", "requested_mb", env, "cap_mb", cap_mb);
}
if (resolved.hard_capped) {
char cap_bytes[CBM_SZ_32];
snprintf(cap_bytes, sizeof(cap_bytes), "%zu", hard_cap_bytes);
cbm_log_info("mem.budget.worker_cap", "cap_bytes", cap_bytes);
}
char budget_mb[CBM_SZ_32];
char ram_mb[CBM_SZ_32];
snprintf(budget_mb, sizeof(budget_mb), "%zu", g_budget / MB_DIVISOR);
snprintf(ram_mb, sizeof(ram_mb), "%zu", info.total_ram / MB_DIVISOR);
cbm_log_info("mem.init", "budget_mb", budget_mb, "total_ram_mb", ram_mb, "source",
resolved.source);
}
void cbm_mem_init(double ram_fraction) {
cbm_mem_init_with_cap(ram_fraction, 0);
}
size_t cbm_mem_rss(void) {
#if defined(__linux__)
/* Linux: mimalloc's _mi_prim_process_info() (vendored/mimalloc/src/prim/
* unix/prim.c) never sets pinfo->current_rss on Linux — it only sets
* peak_rss (from getrusage's ru_maxrss). current_rss therefore keeps
* mi_process_info()'s default of pinfo.current_commit: mimalloc's OWN
* committed-page counter, which this project deliberately tunes low via
* mi_option_arena_eager_commit=0 + purge_decommits=1 + purge_delay=0
* (cbm_mem_init) to reduce upfront memory. So on Linux "current_rss" is a
* low-biased mimalloc-internal metric, not true RSS: under concurrent
* large-file parsing it can read a few MB while real RSS is multiple GB,
* silently blinding cbm_mem_over_budget()'s backpressure to real memory
* pressure (small-but-nonzero, so the `current_rss > 0` guard below never
* catches it). os_rss() reads /proc/self/statm — authoritative OS RSS,
* unaffected by mimalloc's accounting — so it is the PRIMARY source on
* Linux, not a last-resort fallback. macOS/Windows are unaffected:
* mimalloc sets current_rss correctly there via task_info /
* GetProcessMemoryInfo. */
size_t proc_rss = os_rss();
if (proc_rss > 0) {
return proc_rss;
}
/* Extremely unlikely (/proc unavailable) — fall through to mimalloc. */
#endif
size_t current_rss = 0;
size_t peak_rss = 0;
mi_process_info(NULL, NULL, NULL, ¤t_rss, &peak_rss, NULL, NULL, NULL);
if (current_rss > 0) {
return current_rss;
}
/* Fallback for ASan builds (MI_OVERRIDE=0) and any platform where
* mimalloc's current_rss is unavailable/zero. */
return os_rss();
}
size_t cbm_mem_peak_rss(void) {
size_t peak_rss = 0;
mi_process_info(NULL, NULL, NULL, NULL, &peak_rss, NULL, NULL, NULL);
/* Peak RSS is by definition >= current RSS. On Linux cbm_mem_rss() reads the
* live /proc/self/statm value (page-granular), while mimalloc's peak_rss
* comes from getrusage's ru_maxrss (KB-granular, and it can lag the live
* statm read by a few pages). Reading the two sources independently lets a
* fresh current read momentarily exceed the reported peak, breaking the
* peak >= current invariant. Reconcile them: the true peak is at least the
* current RSS. (Not observable on macOS, where both come from mimalloc.) */
size_t current = cbm_mem_rss();
if (current > peak_rss) {
peak_rss = current;
}
if (peak_rss > 0) {
return peak_rss;
}
/* No OS fallback for peak — return current as best approximation */
return os_rss();
}
size_t cbm_mem_budget(void) {
return g_budget;
}
void cbm_mem_set_budget_for_tests(size_t bytes) {
g_budget = bytes;
}
size_t cbm_mem_allocator_committed(void) {
size_t commit = 0;
mi_process_info(NULL, NULL, NULL, NULL, NULL, &commit, NULL, NULL);
/* The statistic behind this is a signed counter merged per thread at
* thread exit; a process whose long-lived thread commits what its
* short-lived threads free reads it NEGATIVE, cast to size_t here. The
* daemon's query-leak soak reported 2^64 - 121 MB from the second sample
* on (Linux, 2026-09-14). A negative reading is no reading: report 0 so
* the charge falls back to the OS number instead of a 16 EB budget breach. */
if (commit > (SIZE_MAX >> 1)) {
return 0;
}
return commit;
}
static _Atomic size_t g_peak_charged;
static _Atomic size_t g_charged_for_tests; /* 0 = live reading */
void cbm_mem_set_charged_for_tests(size_t bytes) {
atomic_store_explicit(&g_charged_for_tests, bytes, memory_order_relaxed);
}
size_t cbm_mem_charged(void) {
size_t pinned = atomic_load_explicit(&g_charged_for_tests, memory_order_relaxed);
if (pinned != 0) {
return pinned;
}
/* The OS number (phys_footprint on macOS, RSS elsewhere) is the charge for
* everything the process maps, and the memory core's own live bytes are its
* floor: macOS was measured under-reporting the OS number after
* MADV_FREE_REUSABLE cycles (kernel extraction: 4.1 GB charged, 13.4 GB
* tracked live), so the larger of the two is the honest reading.
*
* NOT the allocator's committed bytes. mimalloc only decrements that
* counter when a decommit will need a matching recommit, and in a RELEASE
* build it never will (prim/unix/prim.c: `#if !MI_DEBUG && MI_SECURE<=2`
* sets needs_recommit=false), so a purge hands the pages back to the OS and
* leaves the counter where it was. Measured on the kernel, 2026-09-17: 77 GB
* purged, the counter flat at 14.8 GB, macOS phys_footprint 5.2 GB. Reading
* it as the charge pinned the budget at its limit -- the spill sweep freed
* 10 GB and the number did not move, so the semantic pass ran with zero
* headroom and the run reported a 32 % budget overshoot that never happened.
* It stays in the logs (commit_mb) as a diagnostic. */
#if defined(__APPLE__)
size_t os_charge = cbm_mem_footprint();
if (os_charge == 0) {
os_charge = cbm_mem_rss(); /* footprint unavailable: fall back */
}
#else
size_t os_charge = cbm_mem_rss();
#endif
size_t tracked = cbm_mem_tracked_live_bytes();
size_t charged = tracked > os_charge ? tracked : os_charge;
/* High-water mark of the charge itself, at the granularity of the gate
* that reads it (every file pull, every phase mark). RSS high-water
* counts pages already purged to the OS but not yet reclaimed
* (MADV_FREE); this is the number the budget is measured against. */
size_t seen = atomic_load_explicit(&g_peak_charged, memory_order_relaxed);
while (charged > seen &&
!atomic_compare_exchange_weak_explicit(&g_peak_charged, &seen, charged,
memory_order_relaxed, memory_order_relaxed)) {}
return charged;
}
size_t cbm_mem_peak_charged(void) {
return atomic_load_explicit(&g_peak_charged, memory_order_relaxed);
}
bool cbm_mem_over_budget(void) {
size_t charged = cbm_mem_charged();
check_pressure(charged);
return charged > g_budget;
}
/* "Give memory back now" — true when our own budget is exceeded OR the MACHINE
* is short, whichever comes first.
*
* Deliberately separate from cbm_mem_over_budget(): the budget decides whether
* the run may CONTINUE (and, as a last resort, aborts it), while this decides
* whether to spill and reclaim. System pressure must never abort a run — it is
* someone else's allocation spike as often as ours, and the honest response is
* to hand memory back, not to fail. Keeping the two apart is what lets the
* relief path be aggressive without making the failure path trigger-happy.
*
* Why it is needed at all: the budget is charged against OUR accounting, which
* on 2026-09-18 read 22 GB against a 24 GB budget while the host had nothing
* left and killed the worker. Our own number can be comfortably under budget
* while the machine is dying. */
bool cbm_mem_should_relieve(void) {
return cbm_mem_over_budget() || cbm_mem_system_under_pressure();
}
/* Reclaimable memory below this share of total RAM is where paging starts to
* hurt, so it is the point at which pressing on stops being reasonable. */
enum { MEM_PRESSURE_AVAIL_DIVISOR = 8 }; /* 12.5% of total RAM */
bool cbm_mem_system_under_pressure(void) {
size_t available = cbm_system_available_ram();
if (available == 0) {
return false; /* platform cannot answer - do not abort on a guess */
}
cbm_system_info_t info = cbm_system_info();
if (info.total_ram == 0) {
return false;
}
return available < info.total_ram / MEM_PRESSURE_AVAIL_DIVISOR;
}
size_t cbm_mem_worker_budget(int num_workers) {
if (num_workers <= 0) {
num_workers = SKIP_ONE;
}
return g_budget / (size_t)num_workers;
}
void cbm_mem_collect(void) {
mi_collect(true);
}
size_t cbm_mem_footprint(void) {
#if defined(__APPLE__)
task_vm_info_data_t vm = {0};
mach_msg_type_number_t count = TASK_VM_INFO_COUNT;
if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&vm, &count) == KERN_SUCCESS) {
return (size_t)vm.phys_footprint;
}
return 0;
#else
return os_rss();
#endif
}
void cbm_mem_release_to_os(void) {
/* mi_collect is the release: on Linux and Windows mimalloc owns malloc, so
* there is no libc heap to trim. glibc's malloc_trim was called here once
* and cost the static Linux release its link: the reference pulls
* libc.a(malloc.o) in beside mimalloc's malloc/free (multiple definition,
* release run 34948714902, 2026-09-15). macOS keeps the system heap for
* everything outside the core, hence the pressure-relief call there. */
mi_collect(true);
#if defined(__APPLE__)
(void)malloc_zone_pressure_relief(NULL, 0);
#endif
}
/* ── Memory map (see mem.h for how to read the triple) ─────────────── */
static const size_t MEM_MAP_BUCKET_LIMITS[CBM_MEM_MAP_BUCKETS] = {
64, 256, 1024, 4096, 16384, 65536, 1048576, 0, /* 0 = open-ended tail */
};
size_t cbm_mem_map_bucket_limit(int bucket) {
if (bucket < 0 || bucket >= CBM_MEM_MAP_BUCKETS) {
return 0;
}
return MEM_MAP_BUCKET_LIMITS[bucket];
}
static int mem_map_bucket_of(size_t block_size) {
for (int i = 0; i < CBM_MEM_MAP_BUCKETS - 1; i++) {
if (block_size <= MEM_MAP_BUCKET_LIMITS[i]) {
return i;
}
}
return CBM_MEM_MAP_BUCKETS - 1;
}
/* Area visitor: visit_blocks=false means one call per area, so `used` (live
* block count) x `block_size` is the live byte total without touching blocks. */
static bool mem_map_visit_area(const mi_heap_t *heap, const mi_heap_area_t *area, void *block,
size_t block_size, void *arg) {
(void)heap;
(void)block;
(void)block_size;
cbm_mem_map_t *map = arg;
if (!area || !map) {
return true;
}
size_t live = area->used * area->block_size;
int bucket = mem_map_bucket_of(area->block_size);
map->live_bytes += live;
map->live_blocks += area->used;
map->bucket_bytes[bucket] += live;
map->bucket_blocks[bucket] += area->used;
map->area_committed_bytes += area->committed;
map->area_reserved_bytes += area->reserved;
return true; /* keep walking */
}
static bool mem_map_collect_impl(cbm_mem_map_t *out, bool walk_allocator);
bool cbm_mem_map_collect_os(cbm_mem_map_t *out) {
return mem_map_collect_impl(out, false);
}
bool cbm_mem_map_collect(cbm_mem_map_t *out) {
return mem_map_collect_impl(out, true);
}
static bool mem_map_collect_impl(cbm_mem_map_t *out, bool walk_allocator) {
if (!out) {
return false;
}
memset(out, 0, sizeof(*out));
size_t elapsed_ms = 0;
size_t user_ms = 0;
size_t sys_ms = 0;
size_t current_rss = 0;
size_t peak_rss = 0;
size_t current_commit = 0;
size_t peak_commit = 0;
size_t page_faults = 0;
mi_process_info(&elapsed_ms, &user_ms, &sys_ms, ¤t_rss, &peak_rss, ¤t_commit,
&peak_commit, &page_faults);
out->os_rss_bytes = current_rss ? current_rss : cbm_mem_rss();
out->os_committed_bytes = current_commit;
/* Does plain malloc actually land in the allocator's regions? This single
* bit distinguishes "nothing is live" from "the allocator does not own
* this build's allocations", which are opposite conclusions from the same
* zero. Probe with a real malloc/free pair rather than inferring from
* build flags, because the Windows static-CRT override is defined at
* compile time yet can still fail to take effect at link time. */
void *probe = malloc(CBM_SZ_64);
if (probe) {
out->malloc_is_allocator_owned = mi_is_in_heap_region(probe);
free(probe);
}
/* Walk only heaps this thread may safely read.
*
* mi_heap_main() is the process-wide aggregate: any other thread can be
* allocating into it while the walk runs, which is a data race by
* construction and TSan reports it as one (init.c:452 in mi_heap_main).
* A diagnostic must not introduce a race into the code it measures, so the
* aggregate walk is gone. What remains is safe by ownership: this thread's
* own theap, plus abandoned pages, which by definition have no owning
* thread left to race with.
*
* That includes mi_heap_main() itself: merely CALLING it reads the
* allocator's main-heap pointer, which a thread exiting concurrently
* rewrites from _mi_thread_done -> _mi_theap_default_set. TSan caught
* exactly that pairing on macOS, so the abandoned-page walk goes too --
* it could only be reached through mi_heap_main().
*
* The cost is coverage -- other threads' live blocks and abandoned pages
* are not attributed -- and that is exactly what the residual in mem.h
* exists to carry. An unmeasured map must never read as an empty one. */
if (walk_allocator) {
(void)mi_theap_visit_blocks(mi_theap_get_default(), false, mem_map_visit_area, out);
}
return true;
}
/* ── Allocator ownership audit (see mem.h) ─────────────────────────── */
static atomic_size_t g_foreign_pointers = ATOMIC_VAR_INIT(0);
static atomic_size_t g_owned_pointers = ATOMIC_VAR_INIT(0);
void cbm_mem_record_pointer_ownership(bool owned) {
atomic_fetch_add_explicit(owned ? &g_owned_pointers : &g_foreign_pointers, 1,
memory_order_relaxed);
}
size_t cbm_mem_foreign_pointer_count(void) {
return atomic_load_explicit(&g_foreign_pointers, memory_order_relaxed);
}
size_t cbm_mem_owned_pointer_count(void) {
return atomic_load_explicit(&g_owned_pointers, memory_order_relaxed);
}
void cbm_mem_audit_ownership(cbm_mem_ownership_audit_t *out) {
if (!out) {
return;
}
memset(out, 0, sizeof(*out));
/* Span the classes an allocator treats differently: small (bins), page
* scale, medium, large, and past the point where allocators typically stop
* using their own pages and go to the OS directly — the class most likely
* to escape both an override and an ownership predicate. */
static const size_t PROBES[CBM_MEM_OWNERSHIP_CLASSES] = {
64, 4096, 65536, 262144, 1048576, 8388608,
};
for (int i = 0; i < CBM_MEM_OWNERSHIP_CLASSES; i++) {
out->probe_bytes[i] = PROBES[i];
void *block = malloc(PROBES[i]);
if (!block) {
continue;
}
((char *)block)[0] = 1; /* touch it so it is really backed */
out->allocated[i] = true;
out->probed_count++;
out->owned[i] = mi_is_in_heap_region(block);
if (out->owned[i]) {
out->owned_count++;
}
free(block);
}
out->all_owned = out->probed_count > 0 && out->owned_count == out->probed_count;
}
/* ── Phase map (see mem.h for how to read it, and its limits) ──────── */
enum { MEM_PHASE_SLOTS = 24 };
typedef struct {
const char *label; /* static string, compared by pointer then by text */
int64_t bytes;
int64_t hits;
} mem_phase_slot_t;
static mem_phase_slot_t g_mem_phases[MEM_PHASE_SLOTS];
static int g_mem_phase_count;
static const char *g_mem_phase_open;
static size_t g_mem_phase_baseline;
static cbm_mutex_t g_mem_phase_mutex;
static atomic_int g_mem_phase_enabled = ATOMIC_VAR_INIT(-1); /* -1 = undecided */
static size_t mem_phase_committed(void) {
size_t elapsed_ms = 0;
size_t user_ms = 0;
size_t sys_ms = 0;
size_t rss = 0;
size_t peak_rss = 0;
size_t commit = 0;
size_t peak_commit = 0;
size_t faults = 0;
mi_process_info(&elapsed_ms, &user_ms, &sys_ms, &rss, &peak_rss, &commit, &peak_commit,
&faults);
return commit;
}
static bool mem_phase_enabled(void) {
int state = atomic_load_explicit(&g_mem_phase_enabled, memory_order_acquire);
if (state >= 0) {
return state == 1;
}
char buf[CBM_SZ_16];
bool on = cbm_safe_getenv("CBM_MEM_PHASES", buf, sizeof(buf), NULL) != NULL && buf[0] == '1';
if (on) {
cbm_mutex_init(&g_mem_phase_mutex);
}
atomic_store_explicit(&g_mem_phase_enabled, on ? 1 : 0, memory_order_release);
return on;
}
bool cbm_mem_phases_enabled(void) {
return mem_phase_enabled();
}
/* One line of mimalloc's stats table, logged as it comes. */
static void mem_allocator_stats_line(const char *msg, void *arg) {
if (!msg || !msg[0]) {
return;
}
char line[512];
size_t n = 0;
for (const char *p = msg; *p && n + 1 < sizeof(line); p++) {
line[n++] = (*p == '\n' || *p == '\r' || *p == '\t') ? ' ' : *p;
}
line[n] = '\0';
/* trailing blanks make the log unreadable; trim */
while (n > 0 && line[n - 1] == ' ') {
line[--n] = '\0';
}
if (line[0] == '\0') {
return;
}
cbm_log_info("mem.allocator.stats", "tag", (const char *)arg, "line", line);
}
void cbm_mem_allocator_stats_log(const char *tag) {
char enabled[CBM_SZ_16];
if (cbm_safe_getenv("CBM_MEM_ALLOCATOR_STATS", enabled, sizeof(enabled), NULL) == NULL) {
return;
}
if (enabled[0] == '0' && enabled[1] == '\0') {
return;
}
mi_stats_print_out(mem_allocator_stats_line, (void *)(tag ? tag : "-"));
}
void cbm_mem_phase_mark(const char *label) {
cbm_memev_phase(label); /* waste sanitizer: dormant unless CBM_MEMWASTE=1 */
if (!mem_phase_enabled()) {
return;
}
size_t now = mem_phase_committed();
cbm_mutex_lock(&g_mem_phase_mutex);
if (g_mem_phase_open) {
int slot = -1;
for (int i = 0; i < g_mem_phase_count; i++) {
if (g_mem_phases[i].label == g_mem_phase_open ||
strcmp(g_mem_phases[i].label, g_mem_phase_open) == 0) {
slot = i;
break;
}
}
if (slot < 0 && g_mem_phase_count < MEM_PHASE_SLOTS) {
slot = g_mem_phase_count++;
g_mem_phases[slot].label = g_mem_phase_open;
g_mem_phases[slot].bytes = 0;
g_mem_phases[slot].hits = 0;
}
if (slot >= 0) {
g_mem_phases[slot].bytes += (int64_t)now - (int64_t)g_mem_phase_baseline;
g_mem_phases[slot].hits++;
}
}
g_mem_phase_open = label;
g_mem_phase_baseline = now;
cbm_mutex_unlock(&g_mem_phase_mutex);
}
void cbm_mem_phase_reset(void) {
if (!mem_phase_enabled()) {
return;
}
cbm_mutex_lock(&g_mem_phase_mutex);
memset(g_mem_phases, 0, sizeof(g_mem_phases));
g_mem_phase_count = 0;
g_mem_phase_open = NULL;
g_mem_phase_baseline = mem_phase_committed();
cbm_mutex_unlock(&g_mem_phase_mutex);
}
int cbm_mem_phase_report_json(char *out, size_t size) {
if (!out || size == 0) {
return 0;
}
out[0] = '\0';
if (!mem_phase_enabled()) {
return 0;
}
mem_phase_slot_t copy[MEM_PHASE_SLOTS];
int count = 0;
cbm_mutex_lock(&g_mem_phase_mutex);
count = g_mem_phase_count;
memcpy(copy, g_mem_phases, sizeof(copy));
cbm_mutex_unlock(&g_mem_phase_mutex);
/* Biggest retainer first — insertion sort, count <= MEM_PHASE_SLOTS. */
for (int i = 1; i < count; i++) {
mem_phase_slot_t key = copy[i];
int j = i - 1;
while (j >= 0 && copy[j].bytes < key.bytes) {
copy[j + 1] = copy[j];
j--;
}
copy[j + 1] = key;
}
int length = 0;
for (int i = 0; i < count && length >= 0 && (size_t)length < size; i++) {
int written = snprintf(out + length, size - (size_t)length,
"%s{\"label\": \"%s\", \"bytes\": %lld, \"hits\": %lld}",
i == 0 ? "" : ", ", copy[i].label ? copy[i].label : "?",
(long long)copy[i].bytes, (long long)copy[i].hits);
if (written < 0) {
break;
}
length += written;
}
return length > 0 && (size_t)length < size ? length : 0;
}
/* ── Windows region census (see mem.h) ─────────────────────────────── */
static atomic_int g_mem_census_enabled = -1;
bool cbm_mem_census_enabled(void) {
int state = atomic_load_explicit(&g_mem_census_enabled, memory_order_acquire);
if (state >= 0) {
return state == 1;
}
char buf[CBM_SZ_16];
bool on = cbm_safe_getenv("CBM_MEM_CENSUS", buf, sizeof(buf), NULL) != NULL && buf[0] == '1';
atomic_store_explicit(&g_mem_census_enabled, on ? 1 : 0, memory_order_release);
return on;
}
bool cbm_mem_win_census(cbm_mem_win_census_t *out) {
if (!out) {
return false;
}
memset(out, 0, sizeof(*out));
#ifndef _WIN32
return false;
#else
/* Walk the whole user address space once. VirtualQuery reports the largest
* run of pages sharing a state/type, so the region count also tells us how
* fragmented the space has become. */
MEMORY_BASIC_INFORMATION mbi;
unsigned char *address = NULL;
while (VirtualQuery(address, &mbi, sizeof(mbi)) == sizeof(mbi)) {
out->regions++;
if (mbi.State == MEM_COMMIT) {
out->committed_total += mbi.RegionSize;
switch (mbi.Type) {
case MEM_PRIVATE:
out->committed_private += mbi.RegionSize;
if (mbi.RegionSize < (size_t)CBM_SZ_64K) {
out->private_small_bytes += mbi.RegionSize;
out->private_small_count++;
} else if (mbi.RegionSize < (size_t)CBM_SZ_1K * CBM_SZ_1K) {
out->private_medium_bytes += mbi.RegionSize;
out->private_medium_count++;
} else {
out->private_large_bytes += mbi.RegionSize;
out->private_large_count++;
if (out->large_tracked < CBM_MEM_TRACKED_REGIONS) {
out->large_base[out->large_tracked] = mbi.BaseAddress;
out->large_size[out->large_tracked] = mbi.RegionSize;
out->large_tracked++;
}
}
if (mbi.RegionSize > out->largest_private_region) {
out->largest_private_region = mbi.RegionSize;
out->largest_private_base = mbi.BaseAddress;
}
if (mbi.RegionSize >= (size_t)CBM_SZ_1K * CBM_SZ_1K) {
out->private_regions_1mb++;
}
break;
case MEM_MAPPED:
out->committed_mapped += mbi.RegionSize;
break;
case MEM_IMAGE:
out->committed_image += mbi.RegionSize;
break;
default:
break;
}
} else if (mbi.State == MEM_RESERVE) {
out->reserved_total += mbi.RegionSize;
}
unsigned char *next = (unsigned char *)mbi.BaseAddress + mbi.RegionSize;
if (next <= address) {
break; /* no forward progress: stop rather than spin */
}
address = next;
}
/* The CRT heaps are the pool --wrap cannot reach: anything ucrtbase or a
* system DLL allocates internally lands here, and Windows heaps do not
* decommit on free. HeapWalk needs the heap locked to be coherent. */
HANDLE heaps[64];
DWORD count = GetProcessHeaps((DWORD)(sizeof(heaps) / sizeof(heaps[0])), heaps);
if (count > 0) {
out->heap_walk_ok = true;
out->crt_heap_count = (unsigned)(count > 64 ? 64 : count);
for (DWORD i = 0; i < out->crt_heap_count; i++) {
if (!HeapLock(heaps[i])) {
continue;
}
PROCESS_HEAP_ENTRY entry;
memset(&entry, 0, sizeof(entry));
while (HeapWalk(heaps[i], &entry)) {
if ((entry.wFlags & PROCESS_HEAP_REGION) != 0) {
out->crt_heap_committed += entry.Region.dwCommittedSize;
} else if ((entry.wFlags & PROCESS_HEAP_ENTRY_BUSY) != 0) {
out->crt_heap_busy += entry.cbData + entry.cbOverhead;
}
}
HeapUnlock(heaps[i]);
}
}
return true;
#endif
}
/* mi_stats_print_out hands us one chunk at a time; append each to the file. */
static void mem_stats_writer(const char *text, void *arg) {
FILE *out = arg;
if (out && text) {
(void)fputs(text, out);
}
}
void cbm_mem_census_log(const char *tag) {
if (!cbm_mem_census_enabled()) {
return;
}
cbm_mem_win_census_t census;
/* A false return means the OS-region walk is unavailable (non-Windows), not
* that there is nothing to report: the allocation-site half is the whole
* point of the cross-platform comparison, so it still runs and the pool
* fields simply read zero. */
(void)cbm_mem_win_census(&census);
/* Kilobytes throughout: the CRT heap measured 0 at megabyte resolution,
* which reads as "empty" when it may simply be small — a diagnostic that
* rounds its own evidence away is worse than none. */
char priv_kb[CBM_SZ_32];
char crt_kb[CBM_SZ_32];
char busy_kb[CBM_SZ_32];
char mapped_kb[CBM_SZ_32];
char rss_kb[CBM_SZ_32];
char biggest_kb[CBM_SZ_32];
char regions[CBM_SZ_32];
char base[CBM_SZ_32];
char big_regions[CBM_SZ_32];
char psmall[CBM_SZ_32];
char pmed[CBM_SZ_32];
char plarge[CBM_SZ_32];
char pcounts[CBM_SZ_64];
char large_map[CBM_SZ_256];
char heaps[CBM_SZ_32];
(void)snprintf(priv_kb, sizeof(priv_kb), "%zu", census.committed_private / CBM_SZ_1K);
(void)snprintf(crt_kb, sizeof(crt_kb), "%zu", census.crt_heap_committed / CBM_SZ_1K);
(void)snprintf(busy_kb, sizeof(busy_kb), "%zu", census.crt_heap_busy / CBM_SZ_1K);
(void)snprintf(mapped_kb, sizeof(mapped_kb), "%zu", census.committed_mapped / CBM_SZ_1K);
(void)snprintf(rss_kb, sizeof(rss_kb), "%zu", cbm_mem_rss() / CBM_SZ_1K);
(void)snprintf(biggest_kb, sizeof(biggest_kb), "%zu",
census.largest_private_region / CBM_SZ_1K);
(void)snprintf(regions, sizeof(regions), "%u", census.regions);
(void)snprintf(base, sizeof(base), "%p", census.largest_private_base);
(void)snprintf(big_regions, sizeof(big_regions), "%u", census.private_regions_1mb);
(void)snprintf(psmall, sizeof(psmall), "%zu", census.private_small_bytes / CBM_SZ_1K);
(void)snprintf(pmed, sizeof(pmed), "%zu", census.private_medium_bytes / CBM_SZ_1K);
(void)snprintf(plarge, sizeof(plarge), "%zu", census.private_large_bytes / CBM_SZ_1K);
(void)snprintf(pcounts, sizeof(pcounts), "%u/%u/%u", census.private_small_count,
census.private_medium_count, census.private_large_count);
large_map[0] = '\0';
for (unsigned r = 0; r < census.large_tracked; r++) {
char one[CBM_SZ_64];
(void)snprintf(one, sizeof(one), "%s%p:%zuM", r == 0 ? "" : ",", census.large_base[r],
census.large_size[r] / ((size_t)CBM_SZ_1K * CBM_SZ_1K));
(void)strncat(large_map, one, sizeof(large_map) - strlen(large_map) - 1);
}
(void)snprintf(heaps, sizeof(heaps), "%u", census.heap_walk_ok ? census.crt_heap_count : 0);
/* Attribution alongside the pool totals: the census says which pool holds
* the memory, the profile says which call sites put it there. Emitting
* both from one place keeps the two views on the same sample. */
/* The allocator's OWN view, next to the OS view. If mimalloc's committed
* area tracks the process's private commit, the memory is the allocator's
* and it is holding pages against tiny live data; if it stays flat while
* private climbs, the growth is outside the allocator entirely and no
* allocator option will ever move it (thirteen of them did not). */
cbm_mem_map_t map;
(void)cbm_mem_map_collect(&map);
char map_live_kb[CBM_SZ_32];
char map_area_kb[CBM_SZ_32];
(void)snprintf(map_live_kb, sizeof(map_live_kb), "%zu", map.live_bytes / CBM_SZ_1K);
(void)snprintf(map_area_kb, sizeof(map_area_kb), "%zu", map.area_committed_bytes / CBM_SZ_1K);
cbm_log_info("mem.census", "at", tag ? tag : "?", "mi_area_kb", map_area_kb, "mi_live_kb",
map_live_kb, "priv_kb", priv_kb, "crt_kb", crt_kb, "crt_busy_kb", busy_kb,
"mapped_kb", mapped_kb, "rss_kb", rss_kb, "biggest_kb", biggest_kb, "regions",
regions, "big_regions", big_regions, "heaps", heaps, "biggest_base", base,
"priv_small_kb", psmall, "priv_med_kb", pmed, "priv_large_kb", plarge,
"priv_counts", pcounts, "large_map", large_map);
/* The phase table answers WHICH bracketed path the committed bytes stayed
* in. Emitted once at the end rather than per request: it is cumulative,
* so only the final table carries the verdict. */
if (tag && strcmp(tag, "connection.exit") == 0) {
/* Dump the allocator's own stats to a known path. The theap count is
* the question: if it tracks the request count, each request is
* getting a fresh thread-heap whose pages are abandoned on exit --
* invisible to mi_heap_visit, which only sees main and current. */
char stats_path[CBM_SZ_1K];
if (cbm_safe_getenv("CBM_MEM_STATS_OUT", stats_path, sizeof(stats_path), NULL) != NULL) {
FILE *stats = cbm_fopen(stats_path, "wb");
if (stats) {
mi_stats_print_out(mem_stats_writer, stats);
(void)fclose(stats);
}
}
char phases[CBM_SZ_1K];
if (cbm_mem_phase_report_json(phases, sizeof(phases)) > 0) {
cbm_log_info("mem.phases", "table", phases);
}
}
}