Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
/*
* mem_events.c — see mem_events.h.
*
* Layout:
* sites[] global, open-addressed, keyed by return address. Keys are
* claimed with one CAS; counters are touched only under the
* merge lock, from per-thread deltas.
* shards[] the pointer table, 256 independently locked open-addressed
* maps. A block is routinely freed by a thread that did not
* allocate it, so the pointer table cannot be per-thread.
* work[] CPU work counters keyed by (return address, kind), merged
* the same way as the sites.
* tl_state per-thread deltas for every site and work row the thread
* touched, dirty lists so a flush walks only those, the core
* handshake, the re-entrancy guard and the repeat caches.
* access lane (CBM_MEMWASTE_ACCESS) a shadow map from every 16-byte slot of
* a tracked block to a per-block record the compiler's load and
* store callbacks update.
*
* Everything the layer owns comes from raw virtual memory: it runs INSIDE the
* allocator it observes and must never call it.
*/
#include "foundation/mem_events.h"
#if (defined(CBM_MEMWASTE) && CBM_MEMWASTE) || \
(defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS)
#include "foundation/mem.h"
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef _WIN32
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <fcntl.h> /* _O_WRONLY, _O_CREAT, _O_APPEND, _O_BINARY */
#include <io.h>
#include <windows.h>
#else
#include <fcntl.h>
#include <pthread.h>
#include <sys/mman.h>
#include <unistd.h>
#if defined(__linux__)
#include <link.h> /* dl_iterate_phdr: the load bias of the main program */
#endif
#endif
/* The layer's own code is never instrumented by the access lane: its tables
* are not program data, and a hook that reached instrumented code would pay
* for itself on every table probe. */
#if defined(__clang__)
#pragma clang attribute push(__attribute__((no_sanitize("coverage"))), apply_to = function)
#endif
enum {
MEMEV_SITE_BITS = 16,
MEMEV_SITE_CAP = 1 << MEMEV_SITE_BITS,
MEMEV_WORK_BITS = 17,
MEMEV_WORK_CAP = 1 << MEMEV_WORK_BITS,
MEMEV_SHARD_BITS = 8,
MEMEV_SHARDS = 1 << MEMEV_SHARD_BITS,
MEMEV_SHARD_INIT_BITS = 10,
MEMEV_SHARD_MAX_BITS = 26,
MEMEV_LOAD_NUM = 7, /* grow at 70 % occupancy (live + tombstones) */
MEMEV_LOAD_DEN = 10,
MEMEV_SEQ_BITS = 24,
MEMEV_SEQ_MASK = (1 << MEMEV_SEQ_BITS) - 1,
MEMEV_PHASE_CAP = 63, /* stored in 6 bits */
MEMEV_PHASE_LABEL = 48,
MEMEV_LINE = 2048, /* a site line with every field at 20 digits is ~1,150 bytes */
MEMEV_PTR_SHIFT = 4, /* allocators align to 16: the low bits carry no entropy */
MEMEV_REPEAT_SLOTS = 256,
MEMEV_PATH_BITS = 20,
MEMEV_PATH_CAP = 1 << MEMEV_PATH_BITS,
MEMEV_RUN_MIN = 8, /* a run of the fill byte this long was never written */
MEMEV_FILL_BYTE = 0xA5,
};
#define MEMEV_NO_INDEX UINT32_MAX
#define MEMEV_TOMBSTONE ((uintptr_t)1)
#define MEMEV_FILL_CAP ((size_t)256 * 1024 * 1024) /* never fill (or scan) beyond this */
#define MEMEV_SITE_MASK ((uintptr_t)0x00FFFFFFFFFFFFFFULL)
/* ── raw virtual memory ───────────────────────────────────────────────*/
/* The layer's own footprint, reported in every dump header: a sanitizer that
* cannot say what it costs cannot be trusted to measure anybody else's waste.
* On Windows every byte here is committed; on POSIX it is lazily backed. */
static _Atomic uint64_t g_layer_vm_bytes;
static _Atomic int64_t g_layer_threads;
static void *vm_get(size_t bytes) {
#ifdef _WIN32
void *p = VirtualAlloc(NULL, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
#else
void *p = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
if (p == MAP_FAILED) {
p = NULL;
}
#endif
if (p) {
atomic_fetch_add_explicit(&g_layer_vm_bytes, bytes, memory_order_relaxed);
}
return p;
}
static void vm_put(void *p, size_t bytes) {
if (!p) {
return;
}
atomic_fetch_sub_explicit(&g_layer_vm_bytes, bytes, memory_order_relaxed);
#ifdef _WIN32
(void)bytes;
VirtualFree(p, 0, MEM_RELEASE);
#else
munmap(p, bytes);
#endif
}
/* ── spin lock ────────────────────────────────────────────────────────
* Held for a handful of instructions around one table probe. A mutex would
* have to be initialised, and initialisation may allocate. */
typedef atomic_flag memev_lock_t;
static void lock_take(memev_lock_t *l) {
while (atomic_flag_test_and_set_explicit(l, memory_order_acquire)) {}
}
static void lock_drop(memev_lock_t *l) {
atomic_flag_clear_explicit(l, memory_order_release);
}
/* ── tables ───────────────────────────────────────────────────────────*/
typedef struct {
uint64_t allocs;
uint64_t frees;
uint64_t reallocs;
uint64_t requested_bytes;
uint64_t usable_bytes;
uint64_t realloc_copy_bytes;
uint64_t short_lived;
uint64_t raw_bytes;
uint64_t scanned_blocks;
uint64_t never_written_bytes;
uint64_t over_requested_bytes;
uint64_t zero_untouched_bytes;
uint64_t acc_untouched_blocks;
uint64_t acc_untouched_bytes;
uint64_t acc_dead_blocks;
uint64_t acc_dead_bytes;
uint64_t acc_uninit_reads;
uint64_t acc_idle_bytes;
uint64_t acc_opaque_blocks;
int64_t live_bytes;
int64_t live_blocks;
} site_counters_t;
typedef struct {
uint64_t calls;
uint64_t bytes;
uint64_t aux1;
uint64_t aux2;
uint64_t repeats;
uint64_t peak;
} work_counters_t;
enum {
META_PHASE_MASK = 0x3F,
META_FILLED = 0x40,
META_ZEROED = 0x80,
};
typedef struct {
uintptr_t block; /* 0 empty, MEMEV_TOMBSTONE deleted */
uint64_t usable;
uint64_t requested;
uint32_t seq; /* thread slot << MEMEV_SEQ_BITS | that thread's alloc counter */
uint16_t site;
int8_t mem_class;
uint8_t meta; /* phase (6 bits) | FILLED | ZEROED */
} ptr_entry_t;
typedef struct {
memev_lock_t lock;
ptr_entry_t *entries;
uint32_t bits;
uint32_t used; /* live + tombstones */
uint32_t live;
} shard_t;
typedef struct {
const void *table;
uint64_t key_hash;
uint64_t generation;
} ht_pair_t;
typedef struct {
const char *ptr;
size_t len;
uint64_t content_hash;
} strlen_pair_t;
typedef struct {
site_counters_t *delta; /* MEMEV_SITE_CAP entries */
uint32_t *dirty; /* site ids touched since the last flush */
uint8_t *is_dirty;
uint32_t dirty_count;
work_counters_t *work_delta; /* MEMEV_WORK_CAP entries */
uint32_t *work_dirty;
uint8_t *work_is_dirty;
uint32_t work_dirty_count;
uint32_t slot; /* thread slot, 8 bits */
uint32_t alloc_seq; /* this thread's allocation counter */
uint64_t untracked_frees;
uint64_t pointer_table_full;
uint64_t site_table_full;
uint64_t work_table_full;
strlen_pair_t strlen_cache[MEMEV_REPEAT_SLOTS];
ht_pair_t ht_cache[MEMEV_REPEAT_SLOTS];
/* The handshake and the re-entrancy guard live HERE, not in _Thread_local
* variables. On macOS the first touch of a _Thread_local in a thread
* allocates its storage THROUGH MALLOC: inside an allocation hook that
* re-enters the hook, which touches the variable again, until the stack is
* gone (SIGSEGV at start-up, 2026-09-17; MinGW's emulated TLS allocates the
* same way). A pthread key / FLS slot never allocates, and the state it
* points at comes from raw virtual memory. */
void *hint_site;
int hint_class;
bool hint_set;
bool busy; /* the layer is invisible to itself */
uint32_t backing; /* depth of backing allocator calls (access lane) */
uint64_t ops; /* allocations + work notes by this thread */
uint32_t library_depth;
void *library_site;
#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS
uint32_t acc_ids[512];
uint32_t acc_id_count;
#endif
} tl_state_t;
static _Atomic uintptr_t g_site_keys[MEMEV_SITE_CAP];
static site_counters_t g_site_counters[MEMEV_SITE_CAP];
static uint64_t g_snap_retained[MEMEV_SITE_CAP];
static uint64_t g_snap_dup_bytes[MEMEV_SITE_CAP];
static uint64_t g_snap_dup_blocks[MEMEV_SITE_CAP];
static _Atomic uintptr_t g_work_keys[MEMEV_WORK_CAP];
static work_counters_t g_work_counters[MEMEV_WORK_CAP];
static _Atomic uint64_t g_path_hashes[MEMEV_PATH_CAP];
static memev_lock_t g_merge_lock = ATOMIC_FLAG_INIT;
static shard_t g_shards[MEMEV_SHARDS];
static _Atomic uint64_t g_untracked_frees;
static _Atomic uint64_t g_foreign_frees;
static _Atomic uint64_t g_pointer_table_full;
static _Atomic uint64_t g_site_table_full;
static _Atomic uint64_t g_work_table_full;
static _Atomic uint32_t g_next_slot;
static _Atomic uint32_t g_phase;
static char g_phase_label[MEMEV_PHASE_LABEL] = "start";
static _Atomic int g_enabled = -1;
static _Atomic int g_fill = -1;
static _Atomic int g_dupes = -1;
static _Atomic bool g_observer_installed;
static _Atomic uintptr_t g_image_base;
static const char *const g_work_names[CBM_WORK_KIND_COUNT] = {
"memcpy", "memmove", "memset", "memcmp", "strlen", "strcmp", "strncmp",
"read", "write", "pread", "pwrite", "open", "stat", "fopen",
"fread", "fwrite", "opendir", "readdir", "mutex", "parallel_for", "pool_ops",
"ht_get", "ht_set", "dead_write", "ct_arena", "ct_dyn_array", "ct_hash_table",
};
const char *cbm_work_kind_name(cbm_work_kind_t kind) {
if ((int)kind < 0 || kind >= CBM_WORK_KIND_COUNT) {
return "invalid";
}
return g_work_names[kind];
}
static int env_flag(const char *name, int dflt) {
const char *v = getenv(name);
if (!v || !v[0]) {
return dflt;
}
return (v[0] == '1' && v[1] == '\0') ? 1 : 0;
}
#if defined(__linux__)
extern char **environ;
#endif
bool cbm_memev_enabled(void) {
int state = atomic_load_explicit(&g_enabled, memory_order_acquire);
if (state >= 0) {
return state == 1;
}
#if defined(__linux__)
/* Code can run before libc has set up the environment: the loader's
* pre-init hooks (the UBSan runtime that -fsanitize-coverage links, which
* allocates through dlsym) reach this through the malloc and mutex wrappers.
* getenv answers NULL there, and caching that answer switched the layer
* off for the whole process (Linux access lane, no dump, 2026-09-17).
* Decide only once the environment exists. */
if (!environ) {
return false;
}
#endif
int on = env_flag("CBM_MEMWASTE", 0);
atomic_store_explicit(&g_enabled, on, memory_order_release);
return on == 1;
}
static bool fill_enabled(void) {
int state = atomic_load_explicit(&g_fill, memory_order_relaxed);
if (state < 0) {
state = env_flag("CBM_MEMWASTE_FILL", 1);
atomic_store_explicit(&g_fill, state, memory_order_relaxed);
}
return state == 1;
}
static bool dupes_enabled(void) {
int state = atomic_load_explicit(&g_dupes, memory_order_relaxed);
if (state < 0) {
state = env_flag("CBM_MEMWASTE_DUPES", 1);
atomic_store_explicit(&g_dupes, state, memory_order_relaxed);
}
return state == 1;
}
void cbm_memev_scan_for_tests(bool fill, bool dupes) {
atomic_store_explicit(&g_fill, fill ? 1 : 0, memory_order_relaxed);
atomic_store_explicit(&g_dupes, dupes ? 1 : 0, memory_order_relaxed);
}
void cbm_memev_force_for_tests(bool on) {
atomic_store_explicit(&g_enabled, on ? 1 : 0, memory_order_release);
}
void cbm_memev_set_image_base(uintptr_t base) {
atomic_store_explicit(&g_image_base, base, memory_order_release);
}
void cbm_memev_set_observer_installed(bool installed) {
atomic_store_explicit(&g_observer_installed, installed, memory_order_release);
}
bool cbm_memev_observer_installed(void) {
return atomic_load_explicit(&g_observer_installed, memory_order_acquire);
}
/* ── per-thread slot: a key, never _Thread_local (see tl_state_t) ───────*/
static _Atomic int g_key_state; /* 0 none, 1 being created, 2 ready */
#ifdef _WIN32
static DWORD g_key;
#else
static pthread_key_t g_key;
#endif
static size_t tl_bytes(void);
static void tl_release(void *raw);
#ifdef _WIN32
#endif
static bool key_ready(void) {
int state = atomic_load_explicit(&g_key_state, memory_order_acquire);
while (state != 2) {
int expect = 0;
if (state == 0 &&
atomic_compare_exchange_strong_explicit(&g_key_state, &expect, 1, memory_order_acq_rel,
memory_order_acquire)) {
#ifdef _WIN32
/* A plain TLS slot, released from the loader's thread-detach
* callback (cbm_memev_thread_end, compat_thread.c). An FLS
* destructor never ran in the static MinGW link, and by the time
* the detach callback runs the FLS value is already gone -- the
* state leaked 22 MB per thread (71 threads by the similarity pass,
* then commit exhaustion, 2026-09-17). A TLS slot is still readable
* there. */
g_key = TlsAlloc();
bool ok = g_key != TLS_OUT_OF_INDEXES;
#else
bool ok = pthread_key_create(&g_key, tl_release) == 0;
#endif
atomic_store_explicit(&g_key_state, ok ? 2 : 0, memory_order_release);
if (!ok) {
return false;
}
}
state = atomic_load_explicit(&g_key_state, memory_order_acquire);
}
return true;
}
/* A thread whose state was already released keeps this marker in its slot. Its
* teardown still frees (and on Windows allocates): without the marker every such
* call would create a fresh 22 MB state that no destructor ever releases --
* Windows runs an FLS callback once per thread, it does not re-run it the way
* POSIX re-runs key destructors. Observed as commit exhaustion in the Windows
* index worker, 2026-09-17. */
static char g_tl_exited_marker;
#define TL_EXITED ((void *)&g_tl_exited_marker)
static void *tl_slot_value(void) {
#ifdef _WIN32
return TlsGetValue(g_key);
#else
return pthread_getspecific(g_key);
#endif
}
static tl_state_t *tl_peek(void) {
if (!key_ready()) {
return NULL;
}
void *v = tl_slot_value();
return v == TL_EXITED ? NULL : (tl_state_t *)v;
}
static size_t tl_bytes(void) {
return sizeof(tl_state_t) + ((size_t)MEMEV_SITE_CAP * sizeof(site_counters_t)) +
((size_t)MEMEV_SITE_CAP * sizeof(uint32_t)) + (size_t)MEMEV_SITE_CAP +
((size_t)MEMEV_WORK_CAP * sizeof(work_counters_t)) +
((size_t)MEMEV_WORK_CAP * sizeof(uint32_t)) + (size_t)MEMEV_WORK_CAP;
}
static tl_state_t *tl_get(void) {
if (!key_ready()) {
return NULL;
}
void *v = tl_slot_value();
if (v == TL_EXITED) {
return NULL; /* this thread is ending: observe nothing more */
}
if (v) {
return (tl_state_t *)v;
}
char *raw = (char *)vm_get(tl_bytes());
if (!raw) {
return NULL;
}
tl_state_t *st = (tl_state_t *)raw;
raw += sizeof(tl_state_t);
st->delta = (site_counters_t *)raw;
raw += (size_t)MEMEV_SITE_CAP * sizeof(site_counters_t);
st->dirty = (uint32_t *)raw;
raw += (size_t)MEMEV_SITE_CAP * sizeof(uint32_t);
st->is_dirty = (uint8_t *)raw;
raw += (size_t)MEMEV_SITE_CAP;
st->work_delta = (work_counters_t *)raw;
raw += (size_t)MEMEV_WORK_CAP * sizeof(work_counters_t);
st->work_dirty = (uint32_t *)raw;
raw += (size_t)MEMEV_WORK_CAP * sizeof(uint32_t);
st->work_is_dirty = (uint8_t *)raw;
atomic_fetch_add_explicit(&g_layer_threads, 1, memory_order_relaxed);
st->slot = atomic_fetch_add_explicit(&g_next_slot, 1, memory_order_relaxed) &
((1U << MEMEV_SHARD_BITS) - 1U);
#ifdef _WIN32
if (!TlsSetValue(g_key, st)) {
vm_put(st, tl_bytes());
return NULL;
}
#else
if (pthread_setspecific(g_key, st) != 0) {
vm_put(st, tl_bytes());
return NULL;
}
#endif
return st;
}
static site_counters_t *tl_site(tl_state_t *st, uint32_t site) {
if (!st->is_dirty[site]) {
st->is_dirty[site] = 1;
st->dirty[st->dirty_count++] = site;
}
return &st->delta[site];
}
static work_counters_t *tl_work(tl_state_t *st, uint32_t idx) {
if (!st->work_is_dirty[idx]) {
st->work_is_dirty[idx] = 1;
st->work_dirty[st->work_dirty_count++] = idx;
}
return &st->work_delta[idx];
}
/* ── keyed tables ─────────────────────────────────────────────────────*/
static uint64_t mix(uint64_t x) {
x ^= x >> 33;
x *= 0xff51afd7ed558ccdULL;
x ^= x >> 33;
x *= 0xc4ceb9fe1a85ec53ULL;
x ^= x >> 33;
return x;
}
/* Index of `key` in a CAS-claimed open-addressed key table, claiming a slot on
* first sight. MEMEV_NO_INDEX when the table is full — counted by the caller,
* never silently dropped. */
static uint32_t key_index(_Atomic uintptr_t *keys, uint32_t cap, uintptr_t key) {
uint32_t idx = (uint32_t)(mix((uint64_t)key) & (cap - 1));
for (uint32_t probe = 0; probe < cap; probe++) {
uintptr_t seen = atomic_load_explicit(&keys[idx], memory_order_acquire);
if (seen == key) {
return idx;
}
if (seen == 0) {
uintptr_t expect = 0;
if (atomic_compare_exchange_strong_explicit(
&keys[idx], &expect, key, memory_order_acq_rel, memory_order_acquire) ||
expect == key) {
return idx;
}
}
idx = (idx + 1) & (cap - 1);
}
return MEMEV_NO_INDEX;
}
static uint32_t site_index(uintptr_t site) {
site &= MEMEV_SITE_MASK;
if (site == 0) {
site = (uintptr_t)1; /* "unknown caller" is a site like any other */
}
return key_index(g_site_keys, MEMEV_SITE_CAP, site);
}
static uint32_t work_index(cbm_work_kind_t kind, uintptr_t site) {
uintptr_t key = (site & MEMEV_SITE_MASK) | ((uintptr_t)((unsigned)kind + 1U) << 56);
return key_index(g_work_keys, MEMEV_WORK_CAP, key);
}
/* ── pointer shards ───────────────────────────────────────────────────*/
static shard_t *shard_of(uintptr_t block) {
return &g_shards[mix((uint64_t)(block >> MEMEV_PTR_SHIFT)) >> (64 - MEMEV_SHARD_BITS)];
}
static uint32_t slot_of(uintptr_t block, uint32_t bits) {
return (uint32_t)(mix((uint64_t)(block >> MEMEV_PTR_SHIFT)) & ((1U << bits) - 1U));
}
static bool shard_resize(shard_t *sh, uint32_t bits) {
if (bits > MEMEV_SHARD_MAX_BITS) {
return false;
}
size_t bytes = ((size_t)1 << bits) * sizeof(ptr_entry_t);
ptr_entry_t *next = (ptr_entry_t *)vm_get(bytes);
if (!next) {
return false;
}
uint32_t live = 0;
if (sh->entries) {
uint32_t old_cap = 1U << sh->bits;
for (uint32_t i = 0; i < old_cap; i++) {
ptr_entry_t *e = &sh->entries[i];
if (e->block == 0 || e->block == MEMEV_TOMBSTONE) {
continue;
}
uint32_t s = slot_of(e->block, bits);
while (next[s].block != 0) {
s = (s + 1) & ((1U << bits) - 1U);
}
next[s] = *e;
live++;
}
vm_put(sh->entries, ((size_t)1 << sh->bits) * sizeof(ptr_entry_t));
}
sh->entries = next;
sh->bits = bits;
sh->used = live;
sh->live = live;
return true;
}
/* Caller holds the shard lock. */
static ptr_entry_t *shard_insert(shard_t *sh, uintptr_t block) {
if (!sh->entries && !shard_resize(sh, MEMEV_SHARD_INIT_BITS)) {
return NULL;
}
uint32_t cap = 1U << sh->bits;
if ((uint64_t)(sh->used + 1) * MEMEV_LOAD_DEN > (uint64_t)cap * MEMEV_LOAD_NUM) {
/* Mostly tombstones: rehash in place at the same size; otherwise double. */
uint32_t want = ((uint64_t)sh->live * 2 * MEMEV_LOAD_DEN > (uint64_t)cap * MEMEV_LOAD_NUM)
? sh->bits + 1
: sh->bits;
if (!shard_resize(sh, want)) {
return NULL;
}
cap = 1U << sh->bits;
}
uint32_t s = slot_of(block, sh->bits);
ptr_entry_t *grave = NULL;
for (uint32_t probe = 0; probe < cap; probe++) {
ptr_entry_t *e = &sh->entries[s];
if (e->block == block) {
return e; /* address reuse without a free we saw: overwrite */
}
if (e->block == MEMEV_TOMBSTONE && !grave) {
grave = e;
} else if (e->block == 0) {
ptr_entry_t *dst = grave ? grave : e;
if (!grave) {
sh->used++;
}
sh->live++;
dst->block = block;
return dst;
}
s = (s + 1) & (cap - 1);
}
return NULL;
}
/* Caller holds the shard lock. */
static ptr_entry_t *shard_find(shard_t *sh, uintptr_t block) {
if (!sh->entries) {
return NULL;
}
uint32_t cap = 1U << sh->bits;
uint32_t s = slot_of(block, sh->bits);
for (uint32_t probe = 0; probe < cap; probe++) {
ptr_entry_t *e = &sh->entries[s];
if (e->block == block) {
return e;
}
if (e->block == 0) {
return NULL;
}
s = (s + 1) & (cap - 1);
}
return NULL;
}
/* ── fill pattern and scan ────────────────────────────────────────────
* A new block is filled with MEMEV_FILL_BYTE. At free, a run of at least
* MEMEV_RUN_MIN fill bytes is memory nobody ever wrote, and the last byte that
* is NOT the fill byte is the high-water mark of what was written. Runs are
* counted at word granularity (aligned 8-byte words), which is within 7 bytes
* per run edge and fast enough for hundreds of gigabytes. A calloc block is
* scanned the same way for zero bytes, which is an upper bound: a written zero
* is indistinguishable from an untouched one. */
typedef struct {
uint64_t run_bytes;
uint64_t high_water;
} scan_t;
static scan_t scan_block(const uint8_t *p, size_t n, uint8_t pat) {
scan_t out = {0, 0};
uint64_t patw = 0x0101010101010101ULL * pat;
size_t hw = n;
while (hw > 0 && (hw & 7U) != 0 && p[hw - 1] == pat) {
hw--;
}
while (hw >= 8) {
uint64_t w;
memcpy(&w, p + hw - 8, 8);
if (w != patw) {
break;
}
hw -= 8;
}
while (hw > 0 && p[hw - 1] == pat) {
hw--;
}
size_t tail = n - hw;
if (tail < MEMEV_RUN_MIN) {
hw = n; /* a short tail of the fill byte is as likely written as not */
tail = 0;
}
uint64_t runs = tail;
size_t i = 0;
while (i + 8 <= hw) {
uint64_t w;
memcpy(&w, p + i, 8);
if (w == patw) {
runs += 8;
}
i += 8;
}
out.run_bytes = runs;
out.high_water = hw;
return out;
}
/* ── access lane ──────────────────────────────────────────────────────*/
#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS
enum {
/* 48-bit user address space: arm64 Linux maps the program and its heap
* above 2^47 (0xaaaa... and 0xffff...), which a 47-bit map silently ignored. */
ACC_L1_BITS = 16,
ACC_L1_CAP = 1 << ACC_L1_BITS,
ACC_SLOT_SHIFT = 4,
ACC_REGION_SLOTS = 1 << 28, /* one 4 GB region in 16-byte slots */
ACC_RECS_BITS = 27,
ACC_WRITTEN = 1,
ACC_READ = 2,
ACC_READ_BEFORE_WRITE = 4,
ACC_ID_BATCH = 256,
};
typedef struct {
_Atomic uint8_t flags;
uint8_t last_phase;
uint8_t zeroed;
uint8_t filled; /* holds the fill pattern: a byte that differs was written by someone */
uint32_t site;
uint64_t usable;
_Atomic uintptr_t writer;
} acc_rec_t;
static _Atomic(uint32_t *) g_acc_l2[ACC_L1_CAP];
static acc_rec_t *g_acc_recs;
static uint32_t *g_acc_free;
static uint32_t g_acc_free_count;
static uint32_t g_acc_next = 1;
static memev_lock_t g_acc_lock = ATOMIC_FLAG_INIT;
static _Atomic int g_acc_ready;
static bool acc_init(void) {
if (atomic_load_explicit(&g_acc_ready, memory_order_acquire) == 2) {
return true;
}
lock_take(&g_acc_lock);
if (atomic_load_explicit(&g_acc_ready, memory_order_relaxed) != 2) {
g_acc_recs = (acc_rec_t *)vm_get(((size_t)1 << ACC_RECS_BITS) * sizeof(acc_rec_t));
g_acc_free = (uint32_t *)vm_get(((size_t)1 << ACC_RECS_BITS) * sizeof(uint32_t));
atomic_store_explicit(&g_acc_ready, (g_acc_recs && g_acc_free) ? 2 : 1,
memory_order_release);
}
lock_drop(&g_acc_lock);
return atomic_load_explicit(&g_acc_ready, memory_order_acquire) == 2;
}
static uint32_t *acc_region(uint32_t l1, bool create) {
uint32_t *l2 = atomic_load_explicit(&g_acc_l2[l1], memory_order_acquire);
if (l2 || !create) {
return l2;
}
uint32_t *fresh = (uint32_t *)vm_get((size_t)ACC_REGION_SLOTS * sizeof(uint32_t));
if (!fresh) {
return NULL;
}
uint32_t *expect = NULL;
if (!atomic_compare_exchange_strong_explicit(&g_acc_l2[l1], &expect, fresh,
memory_order_acq_rel, memory_order_acquire)) {
vm_put(fresh, (size_t)ACC_REGION_SLOTS * sizeof(uint32_t));
return expect;
}
return fresh;
}
/* Write `id` into every slot of [a, a + n). */
static void acc_map(uintptr_t a, uint64_t n, uint32_t id) {
uintptr_t end = a + (uintptr_t)(n ? n : 1);
while (a < end) {
uint32_t l1 = (uint32_t)(a >> 32);
if (l1 >= ACC_L1_CAP) {
return;
}
uint32_t *l2 = acc_region(l1, id != 0);
uintptr_t region_end = ((uintptr_t)l1 + 1) << 32;
uintptr_t stop = end < region_end ? end : region_end;
if (l2) {
uint64_t first = ((uint32_t)a) >> ACC_SLOT_SHIFT;
uint64_t last = (stop == region_end) ? (uint64_t)ACC_REGION_SLOTS
: ((((uint32_t)(stop - 1)) >> ACC_SLOT_SHIFT) + 1);
for (uint64_t s = first; s < last; s++) {
l2[s] = id;
}
}
a = stop;
}
}
static uint32_t acc_lookup(uintptr_t a) {
uint32_t l1 = (uint32_t)(a >> 32);
if (l1 >= ACC_L1_CAP) {
return 0;
}
uint32_t *l2 = atomic_load_explicit(&g_acc_l2[l1], memory_order_relaxed);
return l2 ? l2[((uint32_t)a) >> ACC_SLOT_SHIFT] : 0;
}
static uint32_t acc_take_id(tl_state_t *st) {
if (st->acc_id_count == 0) {
lock_take(&g_acc_lock);
while (st->acc_id_count < ACC_ID_BATCH) {
uint32_t id;
if (g_acc_free_count > 0) {
id = g_acc_free[--g_acc_free_count];
} else if (g_acc_next < (1U << ACC_RECS_BITS)) {
id = g_acc_next++;
} else {
break;
}
st->acc_ids[st->acc_id_count++] = id;
}
lock_drop(&g_acc_lock);
}
return st->acc_id_count ? st->acc_ids[--st->acc_id_count] : 0;
}
static void acc_give_id(tl_state_t *st, uint32_t id) {
if (st->acc_id_count < (uint32_t)(sizeof(st->acc_ids) / sizeof(st->acc_ids[0]))) {
st->acc_ids[st->acc_id_count++] = id;
return;
}
lock_take(&g_acc_lock);
g_acc_free[g_acc_free_count++] = id;
while (st->acc_id_count > ACC_ID_BATCH) {
g_acc_free[g_acc_free_count++] = st->acc_ids[--st->acc_id_count];
}
lock_drop(&g_acc_lock);
}
static void acc_track(tl_state_t *st, uintptr_t block, uint64_t usable, uint32_t site, bool zeroed,
bool filled, bool written) {
if (!acc_init()) {
return;
}
uint32_t id = acc_take_id(st);
if (!id) {
return;
}
acc_rec_t *r = &g_acc_recs[id];
/* A block that already holds content when the layer hears of it (the
* Linux/Windows strdup wrappers report AFTER the copy) was written by the
* allocation itself: without this every such block read a first time was a
* read-before-write -- 16.5 M false verdicts on Linux, 12.6 M of them the
* reach-cache key copies (2026-09-17). */
atomic_store_explicit(&r->flags, written ? ACC_WRITTEN : 0, memory_order_relaxed);
atomic_store_explicit(&r->writer, 0, memory_order_relaxed);
r->last_phase = (uint8_t)atomic_load_explicit(&g_phase, memory_order_relaxed);
r->zeroed = zeroed ? 1 : 0;
r->filled = filled ? 1 : 0;
r->site = site;
r->usable = usable;
acc_map(block, usable, id);
}
static void work_add(tl_state_t *st, cbm_work_kind_t kind, uintptr_t site, uint64_t bytes,
uint64_t aux1, uint64_t aux2, uint64_t repeats);
/* The verdict on a block that is going away: what its life was spent on.
* `scan_written` is the fill scan's word on it (1 written, 0 not, -1 no scan):
* a library call the compiler did not instrument (snprintf, strcpy, qsort)
* writes without a store callback, and the pattern still shows it. */
static void acc_untrack(tl_state_t *st, uintptr_t block, uint64_t usable, int scan_written) {
if (atomic_load_explicit(&g_acc_ready, memory_order_relaxed) != 2) {
return;
}
uint32_t id = acc_lookup(block);
if (!id) {
return;
}
acc_rec_t *r = &g_acc_recs[id];
uint8_t f = atomic_load_explicit(&r->flags, memory_order_relaxed);
site_counters_t *c = tl_site(st, r->site);
/* OPAQUE: the fill pattern shows writes, yet no callback ever fired. The
* block was used only by code the lane cannot see -- an uninstrumented
* library, or an aggregate copy (`*dst = *src` of a struct) that the
* compiler expands inline after instrumentation, emitting no load or store
* callback. Its reads are invisible too, so neither "untouched" nor
* "written, never read" nor "idle" can be claimed about it. */
bool opaque = scan_written == 1 && !(f & (ACC_WRITTEN | ACC_READ));
if (opaque) {
c->acc_opaque_blocks++;
} else if (!(f & (ACC_WRITTEN | ACC_READ))) {
c->acc_untouched_blocks++;
c->acc_untouched_bytes += r->usable;
} else if ((f & ACC_WRITTEN) && !(f & ACC_READ)) {
/* Seen written, never seen read. A read through an inline struct copy is
* invisible, so this is "no instrumented read", named as such in reports. */
c->acc_dead_blocks++;
c->acc_dead_bytes += r->usable;
uintptr_t writer = atomic_load_explicit(&r->writer, memory_order_relaxed);
if (writer) { /* 0: only a library wrote it -- counted at the site, no writer to name */
work_add(st, CBM_WORK_DEAD_WRITE, writer, r->usable, 1, 0, 0);
}
}
if ((f & ACC_READ_BEFORE_WRITE) && !r->zeroed) {
c->acc_uninit_reads++;
}
if (!opaque && r->last_phase < (uint8_t)atomic_load_explicit(&g_phase, memory_order_relaxed)) {
c->acc_idle_bytes += r->usable;
}
acc_map(block, usable, 0);
acc_give_id(st, id);
}
static uint32_t acc_move(uintptr_t old_block, uint64_t old_usable, uintptr_t grown,
uint64_t usable) {
if (atomic_load_explicit(&g_acc_ready, memory_order_relaxed) != 2) {
return 0;
}
uint32_t id = acc_lookup(old_block);
if (!id) {
return 0;
}
acc_map(old_block, old_usable, 0);
g_acc_recs[id].usable = usable;
acc_map(grown, usable, id);
return id;
}
static inline void acc_touch(uintptr_t a, bool store, uintptr_t pc) {
uint32_t id = acc_lookup(a);
if (!id) {
return;
}
acc_rec_t *r = &g_acc_recs[id];
uint8_t f = atomic_load_explicit(&r->flags, memory_order_relaxed);
if (store) {
if (!(f & ACC_WRITTEN)) {
atomic_fetch_or_explicit(&r->flags, ACC_WRITTEN, memory_order_relaxed);
atomic_store_explicit(&r->writer, pc, memory_order_relaxed);
}
} else if (!(f & ACC_READ)) {
uint8_t mark = ACC_READ;
if (!(f & ACC_WRITTEN)) {
/* The program is about to read this byte, so looking at it is safe. */
bool library_wrote = r->filled && *(const uint8_t *)a != MEMEV_FILL_BYTE;
mark |= library_wrote ? ACC_WRITTEN : ACC_READ_BEFORE_WRITE;
}
atomic_fetch_or_explicit(&r->flags, mark, memory_order_relaxed);
}
uint8_t ph = (uint8_t)atomic_load_explicit(&g_phase, memory_order_relaxed);
if (r->last_phase != ph) {
r->last_phase = ph;
}
}
#define ACC_PC() ((uintptr_t)__builtin_return_address(0))
void __sanitizer_cov_load1(uint8_t *a) {
acc_touch((uintptr_t)a, false, 0);
}
void __sanitizer_cov_load2(uint16_t *a) {
acc_touch((uintptr_t)a, false, 0);
}
void __sanitizer_cov_load4(uint32_t *a) {
acc_touch((uintptr_t)a, false, 0);
}
void __sanitizer_cov_load8(uint64_t *a) {
acc_touch((uintptr_t)a, false, 0);
}
void __sanitizer_cov_load16(__int128 *a) {
acc_touch((uintptr_t)a, false, 0);
}
void __sanitizer_cov_store1(uint8_t *a) {
acc_touch((uintptr_t)a, true, ACC_PC());
}
void __sanitizer_cov_store2(uint16_t *a) {
acc_touch((uintptr_t)a, true, ACC_PC());
}
void __sanitizer_cov_store4(uint32_t *a) {
acc_touch((uintptr_t)a, true, ACC_PC());
}
void __sanitizer_cov_store8(uint64_t *a) {
acc_touch((uintptr_t)a, true, ACC_PC());
}
void __sanitizer_cov_store16(__int128 *a) {
acc_touch((uintptr_t)a, true, ACC_PC());
}
void __sanitizer_cov_bool_flag_init(uint8_t *start, uint8_t *stop) {
(void)start;
(void)stop;
}
/* Verdicts are per block, so touching both ends of the range marks every block
* it can belong to (a range never spans two allocations in a correct program). */
void cbm_memev_access(const void *p, size_t n, bool store, void *pc) {
if (!p || n == 0 || atomic_load_explicit(&g_acc_ready, memory_order_relaxed) != 2) {
return;
}
tl_state_t *st = tl_get();
if (!st || st->busy || st->backing) {
return;
}
uintptr_t a = (uintptr_t)p;
acc_touch(a, store, (uintptr_t)pc);
if (n > 1) {
acc_touch(a + (uintptr_t)(n - 1), store, (uintptr_t)pc);
}
}
void cbm_memev_backing(int delta) {
tl_state_t *st = tl_get();
if (st) {
st->backing = (uint32_t)((int)st->backing + delta);
}
}
#else
static void work_add(tl_state_t *st, cbm_work_kind_t kind, uintptr_t site, uint64_t bytes,
uint64_t aux1, uint64_t aux2, uint64_t repeats);
#endif /* CBM_MEMWASTE_ACCESS */
/* ── the handshake ────────────────────────────────────────────────────*/
void cbm_memev_hint(void *site, int mem_class) {
if (!cbm_memev_enabled()) {
return;
}
tl_state_t *st = tl_get();
if (st) {
st->hint_site = site;
st->hint_class = mem_class;
st->hint_set = true;
}
}
bool cbm_memev_hint_pending(void) {
if (!cbm_memev_enabled()) {
return false;
}
const tl_state_t *st = tl_peek();
return st && st->hint_set;
}
/* ── events ───────────────────────────────────────────────────────────*/
static void record_alloc(tl_state_t *st, uintptr_t block, size_t requested, size_t usable,
uintptr_t site, int mem_class, unsigned flags) {
st->ops++;
uint32_t sid = site_index(site);
if (sid == MEMEV_NO_INDEX) {
st->site_table_full++;
return;
}
uint64_t charged = usable ? usable : requested;
st->alloc_seq = (st->alloc_seq + 1) & (uint32_t)MEMEV_SEQ_MASK;
bool zeroed = (flags & CBM_MEMEV_ZEROED) != 0;
bool filled = false;
if (!zeroed && !(flags & (CBM_MEMEV_FOREIGN | CBM_MEMEV_WRITTEN)) &&
charged <= MEMEV_FILL_CAP && fill_enabled()) {
memset((void *)block, MEMEV_FILL_BYTE, (size_t)charged);
filled = true;
}
shard_t *sh = shard_of(block);
lock_take(&sh->lock);
ptr_entry_t *e = shard_insert(sh, block);
if (e) {
e->usable = charged;
e->requested = requested;
e->seq = (st->slot << MEMEV_SEQ_BITS) | st->alloc_seq;
e->site = (uint16_t)sid;
e->mem_class = (int8_t)mem_class;
e->meta =
(uint8_t)((atomic_load_explicit(&g_phase, memory_order_relaxed) & META_PHASE_MASK) |
(filled ? META_FILLED : 0) | (zeroed ? META_ZEROED : 0));
}
lock_drop(&sh->lock);
if (!e) {
st->pointer_table_full++;
}
site_counters_t *c = tl_site(st, sid);
c->allocs++;
c->requested_bytes += requested;
c->usable_bytes += charged;
if (mem_class == CBM_MEMEV_CLASS_RAW) {
c->raw_bytes += charged;
}
if (e) {
c->live_bytes += (int64_t)charged;
c->live_blocks++;
#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS
if (!(flags & (CBM_MEMEV_FOREIGN | CBM_MEMEV_LIBRARY))) {
acc_track(st, block, charged, sid, zeroed, filled, (flags & CBM_MEMEV_WRITTEN) != 0);
}
#endif
}
}
void cbm_memev_alloc_ex(void *block, size_t requested, size_t usable, void *site, unsigned flags) {
if (!cbm_memev_enabled()) {
return;
}
tl_state_t *st = tl_get();
if (!st || st->busy) {
return;
}
bool hinted = st->hint_set;
st->hint_set = false; /* consumed even when the allocation failed */
if (!block) {
return;
}
st->busy = true;
record_alloc(st, (uintptr_t)block, requested, usable,
(uintptr_t)(hinted ? st->hint_site : site),
hinted ? st->hint_class : CBM_MEMEV_CLASS_RAW,
hinted ? (flags & ~(unsigned)CBM_MEMEV_FOREIGN) : flags);
st->busy = false;
}
void cbm_memev_alloc(void *block, size_t requested, size_t usable, void *site) {
cbm_memev_alloc_ex(block, requested, usable, site, 0);
}
/* Removes the record of `block`. Returns false when the layer never saw it. */
static bool take_entry(uintptr_t block, ptr_entry_t *out) {
shard_t *sh = shard_of(block);
lock_take(&sh->lock);
ptr_entry_t *e = shard_find(sh, block);
bool found = e != NULL;
if (found) {
*out = *e;
e->block = MEMEV_TOMBSTONE;
sh->live--;
}
lock_drop(&sh->lock);
return found;
}
void cbm_memev_free(void *block) {
if (!block || !cbm_memev_enabled()) {
return;
}
tl_state_t *st = tl_get();
if (!st || st->busy) {
return;
}
st->busy = true;
ptr_entry_t e;
if (take_entry((uintptr_t)block, &e)) {
site_counters_t *c = tl_site(st, e.site);
c->frees++;
c->live_bytes -= (int64_t)e.usable;
c->live_blocks--;
uint32_t slot = e.seq >> MEMEV_SEQ_BITS;
uint32_t age =
(st->alloc_seq - (e.seq & (uint32_t)MEMEV_SEQ_MASK)) & (uint32_t)MEMEV_SEQ_MASK;
if (slot == st->slot && age <= CBM_MEMEV_CHURN_WINDOW) {
c->short_lived++;
}
int scan_written = -1;
if ((e.meta & (META_FILLED | META_ZEROED)) && e.usable <= MEMEV_FILL_CAP) {
bool zeroed = (e.meta & META_ZEROED) != 0;
scan_t sc =
scan_block((const uint8_t *)block, (size_t)e.usable, zeroed ? 0 : MEMEV_FILL_BYTE);
scan_written = sc.high_water > 0 ? 1 : 0;
c->scanned_blocks++;
if (zeroed) {
c->zero_untouched_bytes += e.usable - sc.high_water;
} else {
c->never_written_bytes += sc.run_bytes;
if (e.requested > sc.high_water) {
c->over_requested_bytes += e.requested - sc.high_water;
}
}
}
#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS
acc_untrack(st, (uintptr_t)block, e.usable, scan_written);
#else
(void)scan_written;
#endif
} else {
st->untracked_frees++;
}
st->busy = false;
}
void cbm_memev_free_foreign(void *block) {
if (!block || !cbm_memev_enabled()) {
return;
}
atomic_fetch_add_explicit(&g_foreign_frees, 1, memory_order_relaxed);
}
void cbm_memev_realloc(void *old_block, void *grown, size_t requested, size_t usable, void *site) {
if (!cbm_memev_enabled()) {
return;
}
tl_state_t *st = tl_get();
if (!st || st->busy) {
return;
}
bool hinted = st->hint_set;
void *hint_site = st->hint_site;
int hint_class = st->hint_class;
st->hint_set = false;
if (!grown) {
return;
}
st->busy = true;
ptr_entry_t e;
bool had = old_block && take_entry((uintptr_t)old_block, &e);
if (had) {
/* The record keeps its ORIGINAL site: a realloc chain is one object
* growing, and its copies are that object's cost. */
site_counters_t *c = tl_site(st, e.site);
c->reallocs++;
c->live_bytes -= (int64_t)e.usable;
c->live_blocks--;
if (grown != old_block) {
c->realloc_copy_bytes += e.usable < requested ? e.usable : requested;
}
uint64_t charged = usable ? usable : requested;
uint8_t meta = e.meta;
if ((meta & META_FILLED) && charged > e.usable && charged <= MEMEV_FILL_CAP) {
/* the new tail is uninitialised: it gets the pattern too */
memset((char *)grown + e.usable, MEMEV_FILL_BYTE, (size_t)(charged - e.usable));
} else if (charged > e.usable) {
meta &= (uint8_t)~(META_FILLED | META_ZEROED); /* mixed content: stop scanning */
}
shard_t *sh = shard_of((uintptr_t)grown);
lock_take(&sh->lock);
ptr_entry_t *n = shard_insert(sh, (uintptr_t)grown);
if (n) {
n->usable = charged;
n->requested = requested;
n->seq = e.seq;
n->site = e.site;
n->mem_class = e.mem_class;
n->meta = meta;
}
lock_drop(&sh->lock);
c->requested_bytes += requested;
c->usable_bytes += charged;
if (e.mem_class == CBM_MEMEV_CLASS_RAW) {
c->raw_bytes += charged;
}
if (n) {
c->live_bytes += (int64_t)charged;
c->live_blocks++;
#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS
(void)acc_move((uintptr_t)old_block, e.usable, (uintptr_t)grown, charged);
#endif
} else {
st->pointer_table_full++;
}
} else {
/* The old block was never seen (allocated before the observer, or by a
* path it does not cover): the grown block already holds the program's
* data, so it is recorded without the fill. realloc(NULL, n) is a fresh
* allocation and gets the fill like any malloc. */
record_alloc(st, (uintptr_t)grown, requested, usable,
(uintptr_t)(hinted ? hint_site : site),
hinted ? hint_class : CBM_MEMEV_CLASS_RAW,
old_block ? CBM_MEMEV_WRITTEN : 0); /* realloc(NULL, n) is a malloc */
}
st->busy = false;
}
/* ── CPU work ─────────────────────────────────────────────────────────*/
static void work_add(tl_state_t *st, cbm_work_kind_t kind, uintptr_t site, uint64_t bytes,
uint64_t aux1, uint64_t aux2, uint64_t repeats) {
st->ops++;
uint32_t idx = work_index(kind, site);
if (idx == MEMEV_NO_INDEX) {
st->work_table_full++;
return;
}
work_counters_t *w = tl_work(st, idx);
w->calls++;
w->bytes += bytes;
w->aux1 += aux1;
w->aux2 += aux2;
w->repeats += repeats;
bool container = kind == CBM_WORK_CT_ARENA || kind == CBM_WORK_CT_DYN_ARRAY ||
kind == CBM_WORK_CT_HASH_TABLE;
uint64_t single = container ? (bytes > aux1 ? bytes - aux1 : 0) : bytes;
if (single > w->peak) {
w->peak = single;
}
}
void cbm_memev_library_enter(void *site) {
tl_state_t *st = cbm_memev_enabled() ? tl_get() : NULL;
if (st && st->library_depth++ == 0) {
st->library_site = site;
}
}
void cbm_memev_library_leave(void) {
tl_state_t *st = cbm_memev_enabled() ? tl_get() : NULL;
if (st && st->library_depth > 0 && --st->library_depth == 0) {
st->library_site = NULL;
}
}
void *cbm_memev_library_site(void) {
tl_state_t *st = cbm_memev_enabled() ? tl_get() : NULL;
return st ? st->library_site : NULL;
}
#if defined(CBM_MEMWASTE_PROFILE) && CBM_MEMWASTE_PROFILE
void __llvm_profile_set_filename(const char *pattern);
int __llvm_profile_write_file(void);
#endif
void cbm_memev_process_role(const char *role) {
#if defined(CBM_MEMWASTE_PROFILE) && CBM_MEMWASTE_PROFILE
const char *dir = getenv("CBM_PROFILE_DIR");
if (!dir || !dir[0] || !role) {
return;
}
/* The runtime keeps the pointer, not a copy: the pattern must outlive us. */
static char pattern[1024];
int n = snprintf(pattern, sizeof(pattern), "%s/%s-%%p.profraw", dir, role);
if (n > 0 && (size_t)n < sizeof(pattern)) {
__llvm_profile_set_filename(pattern);
}
#else
(void)role;
#endif
}
void cbm_memev_process_exit(void) {
if (cbm_memev_enabled()) {
const char *out = getenv("CBM_MEMWASTE_OUT");
if (out && out[0]) {
(void)cbm_memev_dump(out, "exit");
}
}
#if defined(CBM_MEMWASTE_PROFILE) && CBM_MEMWASTE_PROFILE
(void)__llvm_profile_write_file();
#endif
}
uint64_t cbm_memev_thread_ops(void) {
if (!cbm_memev_enabled()) {
return 0;
}
tl_state_t *st = tl_get();
return st ? st->ops : 0;
}
void cbm_work_note(cbm_work_kind_t kind, void *site, uint64_t bytes, uint64_t aux1, uint64_t aux2) {
if ((int)kind < 0 || kind >= CBM_WORK_KIND_COUNT || !cbm_memev_enabled()) {
return;
}
tl_state_t *st = tl_get();
if (!st || st->busy) {
return;
}
st->busy = true;
work_add(st, kind, (uintptr_t)site, bytes, aux1, aux2, 0);
st->busy = false;
}
__attribute__((noinline)) void cbm_memev_da_note(size_t cap_bytes, size_t used_bytes,
size_t cap_slots) {
if (!cbm_memev_enabled() || cap_bytes == 0) {
return;
}
uint64_t growths = 0;
for (size_t c = 8; c < cap_slots; c <<= 1) {
growths++;
}
cbm_work_note(CBM_WORK_CT_DYN_ARRAY, __builtin_return_address(0), cap_bytes, used_bytes,
growths);
}
/* FNV-1a over the first bytes of a string of known length; the length is
* mixed in, so two strings that share a long prefix still differ. */
enum { MEMEV_CONTENT_HASH_MAX = 4096 };
static uint64_t bytes_hash(const char *s, size_t len) {
uint64_t h = 0xcbf29ce484222325ULL ^ (uint64_t)len;
size_t n = len < MEMEV_CONTENT_HASH_MAX ? len : MEMEV_CONTENT_HASH_MAX;
for (size_t i = 0; s && i < n; i++) {
h ^= (uint8_t)s[i];
h *= 0x100000001b3ULL;
}
return h;
}
void cbm_work_note_strlen(void *site, const char *s, size_t len) {
if (!cbm_memev_enabled()) {
return;
}
tl_state_t *st = tl_get();
if (!st || st->busy) {
return;
}
st->busy = true;
strlen_pair_t *slot = &st->strlen_cache[mix((uint64_t)(uintptr_t)s) & (MEMEV_REPEAT_SLOTS - 1)];
uint64_t content = bytes_hash(s, len);
uint64_t repeat = (slot->ptr == s && slot->len == len && slot->content_hash == content) ? 1 : 0;
slot->ptr = s;
slot->len = len;
slot->content_hash = content;
work_add(st, CBM_WORK_STRLEN, (uintptr_t)site, len, 0, 0, repeat);
st->busy = false;
}
/* FNV-1a over a NUL-terminated path, without calling the strlen we wrap. */
static uint64_t path_hash(const char *p) {
uint64_t h = 0xcbf29ce484222325ULL;
for (; p && *p; p++) {
h ^= (uint8_t)*p;
h *= 0x100000001b3ULL;
}
return h;
}
void cbm_work_note_path(cbm_work_kind_t kind, void *site, const char *path, bool failed) {
if (!cbm_memev_enabled()) {
return;
}
tl_state_t *st = tl_get();
if (!st || st->busy) {
return;
}
st->busy = true;
uint64_t h = mix(path_hash(path) ^ ((uint64_t)kind << 58)) | 1U;
uint64_t repeat = 0;
uint32_t idx = (uint32_t)(h & (MEMEV_PATH_CAP - 1));
for (uint32_t probe = 0; probe < MEMEV_PATH_CAP; probe++) {
uint64_t seen = atomic_load_explicit(&g_path_hashes[idx], memory_order_relaxed);
if (seen == h) {
repeat = 1;
break;
}
if (seen == 0) {
uint64_t expect = 0;
if (atomic_compare_exchange_strong_explicit(
&g_path_hashes[idx], &expect, h, memory_order_relaxed, memory_order_relaxed)) {
break;
}
if (expect == h) {
repeat = 1;
break;
}
}
idx = (idx + 1) & (MEMEV_PATH_CAP - 1);
}
work_add(st, kind, (uintptr_t)site, 0, failed ? 1 : 0, 0, repeat);
st->busy = false;
}
void cbm_work_note_ht(cbm_work_kind_t kind, void *site, const void *table, const char *key,
uint64_t generation, bool hit) {
if (!cbm_memev_enabled()) {
return;
}
tl_state_t *st = tl_get();
if (!st || st->busy) {
return;
}
st->busy = true;
uint64_t repeat = 0;
if (kind == CBM_WORK_HT_GET) {
uint64_t key_hash = path_hash(key);
ht_pair_t *slot =
&st->ht_cache[mix((uint64_t)(uintptr_t)table ^ (key_hash * 0x9e3779b97f4a7c15ULL)) &
(MEMEV_REPEAT_SLOTS - 1)];
repeat =
(slot->table == table && slot->key_hash == key_hash && slot->generation == generation)
? 1
: 0;
slot->table = table;
slot->key_hash = key_hash;
slot->generation = generation;
}
work_add(st, kind, (uintptr_t)site, 0, (kind == CBM_WORK_HT_GET && !hit) ? 1 : 0, 0, repeat);
st->busy = false;
}
/* ── merge ────────────────────────────────────────────────────────────*/
static void flush_state(tl_state_t *st) {
lock_take(&g_merge_lock);
for (uint32_t i = 0; i < st->dirty_count; i++) {
uint32_t sid = st->dirty[i];
site_counters_t *d = &st->delta[sid];
site_counters_t *g = &g_site_counters[sid];
g->allocs += d->allocs;
g->frees += d->frees;
g->reallocs += d->reallocs;
g->requested_bytes += d->requested_bytes;
g->usable_bytes += d->usable_bytes;
g->realloc_copy_bytes += d->realloc_copy_bytes;
g->short_lived += d->short_lived;
g->raw_bytes += d->raw_bytes;
g->scanned_blocks += d->scanned_blocks;
g->never_written_bytes += d->never_written_bytes;
g->over_requested_bytes += d->over_requested_bytes;
g->zero_untouched_bytes += d->zero_untouched_bytes;
g->acc_untouched_blocks += d->acc_untouched_blocks;
g->acc_untouched_bytes += d->acc_untouched_bytes;
g->acc_dead_blocks += d->acc_dead_blocks;
g->acc_dead_bytes += d->acc_dead_bytes;
g->acc_uninit_reads += d->acc_uninit_reads;
g->acc_idle_bytes += d->acc_idle_bytes;
g->acc_opaque_blocks += d->acc_opaque_blocks;
g->live_bytes += d->live_bytes;
g->live_blocks += d->live_blocks;
memset(d, 0, sizeof(*d));
st->is_dirty[sid] = 0;
}
st->dirty_count = 0;
for (uint32_t i = 0; i < st->work_dirty_count; i++) {
uint32_t idx = st->work_dirty[i];
work_counters_t *d = &st->work_delta[idx];
work_counters_t *g = &g_work_counters[idx];
g->calls += d->calls;
g->bytes += d->bytes;
g->aux1 += d->aux1;
g->aux2 += d->aux2;
g->repeats += d->repeats;
if (d->peak > g->peak) {
g->peak = d->peak;
}
memset(d, 0, sizeof(*d));
st->work_is_dirty[idx] = 0;
}
st->work_dirty_count = 0;
lock_drop(&g_merge_lock);
atomic_fetch_add_explicit(&g_untracked_frees, st->untracked_frees, memory_order_relaxed);
atomic_fetch_add_explicit(&g_pointer_table_full, st->pointer_table_full, memory_order_relaxed);
atomic_fetch_add_explicit(&g_site_table_full, st->site_table_full, memory_order_relaxed);
atomic_fetch_add_explicit(&g_work_table_full, st->work_table_full, memory_order_relaxed);
st->untracked_frees = 0;
st->pointer_table_full = 0;
st->site_table_full = 0;
st->work_table_full = 0;
}
void cbm_memev_flush_thread(void) {
if (!cbm_memev_enabled()) {
return;
}
tl_state_t *st = tl_peek();
if (!st || st->busy) {
return;
}
st->busy = true;
flush_state(st);
st->busy = false;
}
/* Thread exit (pthread key / FLS destructor): hand over what is pending, then
* give the raw memory back. */
static void tl_release(void *raw);
void cbm_memev_thread_end(void) {
if (atomic_load_explicit(&g_key_state, memory_order_acquire) != 2) {
return; /* no thread ever had state: nothing to release */
}
void *v = tl_slot_value();
if (v && v != TL_EXITED) {
tl_release(v);
}
}
static void tl_release(void *raw) {
tl_state_t *st = (tl_state_t *)raw;
if (!st || raw == TL_EXITED) {
return;
}
st->busy = true;
flush_state(st);
/* Mark the slot before the memory goes: whatever the rest of the thread's
* teardown allocates or frees must find the marker, neither a freed pointer
* nor an empty slot that would build a new state. */
#ifdef _WIN32
(void)TlsSetValue(g_key, TL_EXITED);
#else
(void)pthread_setspecific(g_key, TL_EXITED);
#endif
atomic_fetch_sub_explicit(&g_layer_threads, 1, memory_order_relaxed);
vm_put(st, tl_bytes());
}
/* ── phase snapshot ───────────────────────────────────────────────────
* Live blocks allocated in an earlier phase are RETAINED: the question is
* whether the phase that is running still needs them. Small live blocks with
* identical content are DUPLICATES: one interned copy would do. Both are
* snapshots of the live set taken after the workers joined. */
static uint64_t content_hash(const uint8_t *p, size_t n) {
uint64_t h = 0xcbf29ce484222325ULL ^ ((uint64_t)n * 0x9e3779b97f4a7c15ULL);
for (size_t i = 0; i < n; i++) {
h ^= p[i];
h *= 0x100000001b3ULL;
}
return h | 1U;
}
static void snapshot_phase(uint32_t phase_now) {
memset(g_snap_retained, 0, sizeof(g_snap_retained));
memset(g_snap_dup_bytes, 0, sizeof(g_snap_dup_bytes));
memset(g_snap_dup_blocks, 0, sizeof(g_snap_dup_blocks));
bool dupes = dupes_enabled();
uint64_t live_total = 0;
for (uint32_t i = 0; i < MEMEV_SHARDS; i++) {
live_total += g_shards[i].live;
}
size_t set_cap = 1024;
while (set_cap < live_total * 2 && set_cap < ((size_t)1 << 30)) {
set_cap <<= 1;
}
uint64_t *set = dupes ? (uint64_t *)vm_get(set_cap * sizeof(uint64_t)) : NULL;
for (uint32_t i = 0; i < MEMEV_SHARDS; i++) {
shard_t *sh = &g_shards[i];
lock_take(&sh->lock);
uint32_t cap = sh->entries ? (1U << sh->bits) : 0;
for (uint32_t j = 0; j < cap; j++) {
const ptr_entry_t *e = &sh->entries[j];
if (e->block == 0 || e->block == MEMEV_TOMBSTONE) {
continue;
}
if ((uint32_t)(e->meta & META_PHASE_MASK) < phase_now) {
g_snap_retained[e->site] += e->usable;
}
if (!set || e->usable > CBM_MEMEV_DUP_MAX || !(e->meta & (META_FILLED | META_ZEROED))) {
continue;
}
const uint8_t *p = (const uint8_t *)e->block;
scan_t sc =
scan_block(p, (size_t)e->usable, (e->meta & META_ZEROED) ? 0 : MEMEV_FILL_BYTE);
if (sc.high_water == 0) {
continue; /* nothing written: that is never-written waste, not a duplicate */
}
uint64_t h = content_hash(p, (size_t)sc.high_water);
size_t s = (size_t)(h & (set_cap - 1));
for (;;) {
if (set[s] == 0) {
set[s] = h;
break;
}
if (set[s] == h) {
g_snap_dup_bytes[e->site] += e->usable;
g_snap_dup_blocks[e->site]++;
break;
}
s = (s + 1) & (set_cap - 1);
}
}
lock_drop(&sh->lock);
}
vm_put(set, set_cap * sizeof(uint64_t));
}
void cbm_memev_phase(const char *label) {
if (!cbm_memev_enabled()) {
return;
}
tl_state_t *st = tl_get();
if (!st || st->busy) {
return;
}
st->busy = true;
flush_state(st);
uint32_t next = atomic_load_explicit(&g_phase, memory_order_relaxed) + 1;
if (next <= MEMEV_PHASE_CAP) {
atomic_store_explicit(&g_phase, next, memory_order_relaxed);
}
lock_take(&g_merge_lock);
snprintf(g_phase_label, sizeof(g_phase_label), "%s", label ? label : "-");
lock_drop(&g_merge_lock);
snapshot_phase(atomic_load_explicit(&g_phase, memory_order_relaxed));
st->busy = false;
const char *out = getenv("CBM_MEMWASTE_OUT");
if (out && out[0]) {
char why[MEMEV_PHASE_LABEL + 8];
snprintf(why, sizeof(why), "phase:%s", label ? label : "-");
(void)cbm_memev_dump(out, why);
}
}
/* ── report ───────────────────────────────────────────────────────────*/
void cbm_memev_totals(cbm_memev_totals_t *out) {
if (!out) {
return;
}
memset(out, 0, sizeof(*out));
cbm_memev_flush_thread();
lock_take(&g_merge_lock);
for (uint32_t i = 0; i < MEMEV_SITE_CAP; i++) {
if (atomic_load_explicit(&g_site_keys[i], memory_order_relaxed) == 0) {
continue;
}
const site_counters_t *g = &g_site_counters[i];
out->sites++;
out->allocs += g->allocs;
out->frees += g->frees;
out->reallocs += g->reallocs;
out->requested_bytes += g->requested_bytes;
out->usable_bytes += g->usable_bytes;
out->realloc_copy_bytes += g->realloc_copy_bytes;
out->raw_bytes += g->raw_bytes;
if (g->live_bytes > 0) {
out->live_bytes += (uint64_t)g->live_bytes;
}
if (g->live_blocks > 0) {
out->live_blocks += (uint64_t)g->live_blocks;
}
}
lock_drop(&g_merge_lock);
out->untracked_frees = atomic_load_explicit(&g_untracked_frees, memory_order_relaxed);
out->foreign_frees = atomic_load_explicit(&g_foreign_frees, memory_order_relaxed);
out->pointer_table_full = atomic_load_explicit(&g_pointer_table_full, memory_order_relaxed);
out->site_table_full = atomic_load_explicit(&g_site_table_full, memory_order_relaxed);
out->work_table_full = atomic_load_explicit(&g_work_table_full, memory_order_relaxed);
}
static void site_row_of(uint32_t i, uintptr_t key, cbm_memev_site_t *row) {
const site_counters_t *g = &g_site_counters[i];
*row = (cbm_memev_site_t){
.site = key,
.allocs = g->allocs,
.frees = g->frees,
.reallocs = g->reallocs,
.requested_bytes = g->requested_bytes,
.usable_bytes = g->usable_bytes,
.realloc_copy_bytes = g->realloc_copy_bytes,
.live_bytes = g->live_bytes > 0 ? (uint64_t)g->live_bytes : 0,
.live_blocks = g->live_blocks > 0 ? (uint64_t)g->live_blocks : 0,
.short_lived = g->short_lived,
.raw_bytes = g->raw_bytes,
.scanned_blocks = g->scanned_blocks,
.never_written_bytes = g->never_written_bytes,
.over_requested_bytes = g->over_requested_bytes,
.zero_untouched_bytes = g->zero_untouched_bytes,
.retained_bytes = g_snap_retained[i],
.dup_bytes = g_snap_dup_bytes[i],
.dup_blocks = g_snap_dup_blocks[i],
.acc_untouched_blocks = g->acc_untouched_blocks,
.acc_untouched_bytes = g->acc_untouched_bytes,
.acc_dead_blocks = g->acc_dead_blocks,
.acc_dead_bytes = g->acc_dead_bytes,
.acc_uninit_reads = g->acc_uninit_reads,
.acc_idle_bytes = g->acc_idle_bytes,
.acc_opaque_blocks = g->acc_opaque_blocks,
};
}
static int by_usable_desc(const void *a, const void *b) {
const cbm_memev_site_t *x = (const cbm_memev_site_t *)a;
const cbm_memev_site_t *y = (const cbm_memev_site_t *)b;
if (x->usable_bytes != y->usable_bytes) {
return x->usable_bytes < y->usable_bytes ? 1 : -1;
}
/* Equal bytes: order by address, so the report is a function of the data. */
return x->site < y->site ? -1 : (x->site > y->site ? 1 : 0);
}
/* Top `max` sites by cumulative usable bytes, selected without allocating. */
size_t cbm_memev_sites(cbm_memev_site_t *out, size_t max) {
if (!out || max == 0) {
return 0;
}
cbm_memev_flush_thread();
size_t n = 0;
lock_take(&g_merge_lock);
for (uint32_t i = 0; i < MEMEV_SITE_CAP; i++) {
uintptr_t key = atomic_load_explicit(&g_site_keys[i], memory_order_relaxed);
if (key == 0) {
continue;
}
cbm_memev_site_t row;
site_row_of(i, key, &row);
if (n < max) {
out[n++] = row;
continue;
}
size_t smallest = 0;
for (size_t k = 1; k < n; k++) {
if (by_usable_desc(&out[k], &out[smallest]) > 0) {
smallest = k;
}
}
if (by_usable_desc(&row, &out[smallest]) < 0) {
out[smallest] = row;
}
}
lock_drop(&g_merge_lock);
tl_state_t *st = tl_get();
bool was_busy = st && st->busy;
if (st) {
st->busy = true; /* qsort is libc; keep anything it does out of the tables */
}
qsort(out, n, sizeof(out[0]), by_usable_desc);
if (st) {
st->busy = was_busy;
}
return n;
}
size_t cbm_work_rows(cbm_work_row_t *out, size_t max) {
if (!out || max == 0) {
return 0;
}
cbm_memev_flush_thread();
size_t n = 0;
lock_take(&g_merge_lock);
for (uint32_t i = 0; i < MEMEV_WORK_CAP && n < max; i++) {
uintptr_t key = atomic_load_explicit(&g_work_keys[i], memory_order_relaxed);
if (key == 0) {
continue;
}
const work_counters_t *g = &g_work_counters[i];
out[n++] = (cbm_work_row_t){
.site = key & MEMEV_SITE_MASK,
.kind = (cbm_work_kind_t)((key >> 56) - 1),
.calls = g->calls,
.bytes = g->bytes,
.aux1 = g->aux1,
.aux2 = g->aux2,
.repeats = g->repeats,
.peak = g->peak,
};
}
lock_drop(&g_merge_lock);
return n;
}
static bool write_all(int fd, const char *buf, size_t len) {
while (len > 0) {
#ifdef _WIN32
int put = _write(fd, buf, (unsigned)len);
#else
ssize_t put = write(fd, buf, len);
#endif
if (put <= 0) {
return false;
}
buf += put;
len -= (size_t)put;
}
return true;
}
#if defined(__linux__)
static int first_object_bias(struct dl_phdr_info *info, size_t size, void *out) {
(void)size;
*(uintptr_t *)out = (uintptr_t)info->dlpi_addr;
return 1; /* the first object is the main program */
}
#endif
/* Sites are raw return addresses; the report subtracts the image base to get
* the address the symboliser expects. macOS records it when the zone observer
* installs; Linux (the load bias: 0 for a non-PIE binary) and Windows (the
* runtime module base) record it here, once, before the first dump. */
static void ensure_image_base(void) {
if (atomic_load_explicit(&g_image_base, memory_order_relaxed) != 0) {
return;
}
#if defined(__linux__)
uintptr_t bias = 0;
dl_iterate_phdr(first_object_bias, &bias);
atomic_store_explicit(&g_image_base, bias, memory_order_relaxed);
#elif defined(_WIN32)
atomic_store_explicit(&g_image_base, (uintptr_t)GetModuleHandleW(NULL), memory_order_relaxed);
#endif
}
static bool dump_locked(const char *path, const char *why) {
ensure_image_base();
cbm_memev_totals_t t;
cbm_memev_totals(&t);
#ifdef _WIN32
int fd = _open(path, _O_WRONLY | _O_CREAT | _O_APPEND | _O_BINARY, 0600);
int pid = (int)GetCurrentProcessId();
#else
int fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0600);
int pid = (int)getpid();
#endif
if (fd < 0) {
return false;
}
char label[MEMEV_PHASE_LABEL];
lock_take(&g_merge_lock);
snprintf(label, sizeof(label), "%s", g_phase_label);
lock_drop(&g_merge_lock);
char line[MEMEV_LINE];
int len = snprintf(
line, sizeof(line),
"{\"memwaste\":2,\"why\":\"%s\",\"pid\":%d,\"image_base\":\"0x%llx\",\"phase\":%u,"
"\"phase_label\":\"%s\",\"rss_bytes\":%llu,\"allocs\":%llu,\"frees\":%llu,"
"\"untracked_frees\":%llu,\"foreign_frees\":%llu,\"reallocs\":%llu,\"requested_bytes\":%"
"llu,\"usable_bytes\":%llu,"
"\"realloc_copy_bytes\":%llu,\"live_bytes\":%llu,\"live_blocks\":%llu,\"raw_bytes\":%llu,"
"\"sites\":%llu,\"site_table_full\":%llu,\"pointer_table_full\":%llu,"
"\"work_table_full\":%llu,\"layer_vm_bytes\":%llu,\"layer_threads\":%lld,"
"\"fill\":%d,\"access\":%d}\n",
why ? why : "-", pid,
(unsigned long long)atomic_load_explicit(&g_image_base, memory_order_relaxed),
atomic_load_explicit(&g_phase, memory_order_relaxed), label,
(unsigned long long)cbm_mem_rss(), (unsigned long long)t.allocs,
(unsigned long long)t.frees, (unsigned long long)t.untracked_frees,
(unsigned long long)t.foreign_frees, (unsigned long long)t.reallocs,
(unsigned long long)t.requested_bytes, (unsigned long long)t.usable_bytes,
(unsigned long long)t.realloc_copy_bytes, (unsigned long long)t.live_bytes,
(unsigned long long)t.live_blocks, (unsigned long long)t.raw_bytes,
(unsigned long long)t.sites, (unsigned long long)t.site_table_full,
(unsigned long long)t.pointer_table_full, (unsigned long long)t.work_table_full,
(unsigned long long)atomic_load_explicit(&g_layer_vm_bytes, memory_order_relaxed),
(long long)atomic_load_explicit(&g_layer_threads, memory_order_relaxed),
fill_enabled() ? 1 : 0,
#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS
1
#else
0
#endif
);
bool ok = len > 0 && (size_t)len < sizeof(line) && write_all(fd, line, (size_t)len);
lock_take(&g_merge_lock);
for (uint32_t i = 0; ok && i < MEMEV_SITE_CAP; i++) {
uintptr_t key = atomic_load_explicit(&g_site_keys[i], memory_order_relaxed);
if (key == 0) {
continue;
}
cbm_memev_site_t r;
site_row_of(i, key, &r);
len = snprintf(
line, sizeof(line),
"{\"site\":\"0x%llx\",\"allocs\":%llu,\"frees\":%llu,\"reallocs\":%llu,"
"\"requested_bytes\":%llu,\"usable_bytes\":%llu,\"realloc_copy_bytes\":%llu,"
"\"live_bytes\":%llu,\"live_blocks\":%llu,\"short_lived\":%llu,\"raw_bytes\":%llu,"
"\"scanned_blocks\":%llu,\"never_written_bytes\":%llu,\"over_requested_bytes\":%llu,"
"\"zero_untouched_bytes\":%llu,\"retained_bytes\":%llu,\"dup_bytes\":%llu,"
"\"dup_blocks\":%llu,\"acc_untouched_blocks\":%llu,\"acc_untouched_bytes\":%llu,"
"\"acc_dead_blocks\":%llu,\"acc_dead_bytes\":%llu,\"acc_uninit_reads\":%llu,"
"\"acc_idle_bytes\":%llu,\"acc_opaque_blocks\":%llu}\n",
(unsigned long long)r.site, (unsigned long long)r.allocs, (unsigned long long)r.frees,
(unsigned long long)r.reallocs, (unsigned long long)r.requested_bytes,
(unsigned long long)r.usable_bytes, (unsigned long long)r.realloc_copy_bytes,
(unsigned long long)r.live_bytes, (unsigned long long)r.live_blocks,
(unsigned long long)r.short_lived, (unsigned long long)r.raw_bytes,
(unsigned long long)r.scanned_blocks, (unsigned long long)r.never_written_bytes,
(unsigned long long)r.over_requested_bytes, (unsigned long long)r.zero_untouched_bytes,
(unsigned long long)r.retained_bytes, (unsigned long long)r.dup_bytes,
(unsigned long long)r.dup_blocks, (unsigned long long)r.acc_untouched_blocks,
(unsigned long long)r.acc_untouched_bytes, (unsigned long long)r.acc_dead_blocks,
(unsigned long long)r.acc_dead_bytes, (unsigned long long)r.acc_uninit_reads,
(unsigned long long)r.acc_idle_bytes, (unsigned long long)r.acc_opaque_blocks);
ok = len > 0 && (size_t)len < sizeof(line) && write_all(fd, line, (size_t)len);
}
for (uint32_t i = 0; ok && i < MEMEV_WORK_CAP; i++) {
uintptr_t key = atomic_load_explicit(&g_work_keys[i], memory_order_relaxed);
if (key == 0) {
continue;
}
const work_counters_t *g = &g_work_counters[i];
if (g->calls == 0) {
continue;
}
len = snprintf(line, sizeof(line),
"{\"work\":\"%s\",\"site\":\"0x%llx\",\"calls\":%llu,\"bytes\":%llu,"
"\"aux1\":%llu,\"aux2\":%llu,\"repeats\":%llu,\"peak\":%llu}\n",
cbm_work_kind_name((cbm_work_kind_t)((key >> 56) - 1)),
(unsigned long long)(key & MEMEV_SITE_MASK), (unsigned long long)g->calls,
(unsigned long long)g->bytes, (unsigned long long)g->aux1,
(unsigned long long)g->aux2, (unsigned long long)g->repeats,
(unsigned long long)g->peak);
ok = len > 0 && (size_t)len < sizeof(line) && write_all(fd, line, (size_t)len);
}
lock_drop(&g_merge_lock);
#ifdef _WIN32
_close(fd);
#else
close(fd);
#endif
return ok;
}
bool cbm_memev_dump(const char *path, const char *why) {
if (!path || !path[0]) {
return false;
}
static memev_lock_t dump_lock = ATOMIC_FLAG_INIT;
tl_state_t *st = tl_get();
bool was_busy = st && st->busy;
if (st && !was_busy) {
flush_state(st); /* this thread's pending events belong in the dump */
}
if (st) {
st->busy = true; /* the dump's own writes and allocations are not the program's work */
}
lock_take(&dump_lock);
bool ok = dump_locked(path, why);
lock_drop(&dump_lock);
if (st) {
st->busy = was_busy;
}
return ok;
}
void cbm_memev_reset_for_tests(void) {
cbm_memev_flush_thread();
lock_take(&g_merge_lock);
for (uint32_t i = 0; i < MEMEV_SITE_CAP; i++) {
atomic_store_explicit(&g_site_keys[i], 0, memory_order_relaxed);
}
for (uint32_t i = 0; i < MEMEV_WORK_CAP; i++) {
atomic_store_explicit(&g_work_keys[i], 0, memory_order_relaxed);
}
for (uint32_t i = 0; i < MEMEV_PATH_CAP; i++) {
atomic_store_explicit(&g_path_hashes[i], 0, memory_order_relaxed);
}
memset(g_site_counters, 0, sizeof(g_site_counters));
memset(g_work_counters, 0, sizeof(g_work_counters));
memset(g_snap_retained, 0, sizeof(g_snap_retained));
memset(g_snap_dup_bytes, 0, sizeof(g_snap_dup_bytes));
memset(g_snap_dup_blocks, 0, sizeof(g_snap_dup_blocks));
lock_drop(&g_merge_lock);
for (uint32_t i = 0; i < MEMEV_SHARDS; i++) {
shard_t *sh = &g_shards[i];
lock_take(&sh->lock);
vm_put(sh->entries, sh->entries ? ((size_t)1 << sh->bits) * sizeof(ptr_entry_t) : 0);
sh->entries = NULL;
sh->bits = 0;
sh->used = 0;
sh->live = 0;
lock_drop(&sh->lock);
}
atomic_store_explicit(&g_untracked_frees, 0, memory_order_relaxed);
atomic_store_explicit(&g_foreign_frees, 0, memory_order_relaxed);
atomic_store_explicit(&g_pointer_table_full, 0, memory_order_relaxed);
atomic_store_explicit(&g_site_table_full, 0, memory_order_relaxed);
atomic_store_explicit(&g_work_table_full, 0, memory_order_relaxed);
atomic_store_explicit(&g_phase, 0, memory_order_relaxed);
atomic_store_explicit(&g_fill, 0, memory_order_relaxed); /* tests use synthetic addresses */
tl_state_t *st = tl_peek();
if (st) {
st->hint_set = false;
memset(st->strlen_cache, 0, sizeof(st->strlen_cache));
memset(st->ht_cache, 0, sizeof(st->ht_cache));
}
}
#if defined(__clang__)
#pragma clang attribute pop
#endif
#else /* layer compiled out: every entry point is an empty stub */
bool cbm_memev_enabled(void) {
return false;
}
void cbm_memev_alloc(void *block, size_t requested, size_t usable, void *site) {
(void)block;
(void)requested;
(void)usable;
(void)site;
}
void cbm_memev_alloc_ex(void *block, size_t requested, size_t usable, void *site, unsigned flags) {
(void)block;
(void)requested;
(void)usable;
(void)site;
(void)flags;
}
void cbm_memev_free(void *block) {
(void)block;
}
void cbm_memev_free_foreign(void *block) {
(void)block;
}
void cbm_memev_realloc(void *old_block, void *grown, size_t requested, size_t usable, void *site) {
(void)old_block;
(void)grown;
(void)requested;
(void)usable;
(void)site;
}
void cbm_memev_hint(void *site, int mem_class) {
(void)site;
(void)mem_class;
}
bool cbm_memev_hint_pending(void) {
return false;
}
void cbm_memev_set_image_base(uintptr_t base) {
(void)base;
}
void cbm_memev_set_observer_installed(bool installed) {
(void)installed;
}
bool cbm_memev_observer_installed(void) {
return false;
}
void cbm_memev_flush_thread(void) {}
void cbm_memev_phase(const char *label) {
(void)label;
}
const char *cbm_work_kind_name(cbm_work_kind_t kind) {
(void)kind;
return "off";
}
uint64_t cbm_memev_thread_ops(void) {
return 0;
}
void cbm_memev_process_role(const char *role) {
(void)role;
}
void cbm_memev_thread_end(void) {}
void cbm_memev_process_exit(void) {}
void cbm_memev_library_enter(void *site) {
(void)site;
}
void cbm_memev_library_leave(void) {}
void *cbm_memev_library_site(void) {
return NULL;
}
void cbm_work_note(cbm_work_kind_t kind, void *site, uint64_t bytes, uint64_t aux1, uint64_t aux2) {
(void)kind;
(void)site;
(void)bytes;
(void)aux1;
(void)aux2;
}
void cbm_memev_da_note(size_t cap_bytes, size_t used_bytes, size_t cap_slots) {
(void)cap_bytes;
(void)used_bytes;
(void)cap_slots;
}
void cbm_work_note_strlen(void *site, const char *s, size_t len) {
(void)site;
(void)s;
(void)len;
}
void cbm_work_note_path(cbm_work_kind_t kind, void *site, const char *path, bool failed) {
(void)kind;
(void)site;
(void)path;
(void)failed;
}
void cbm_work_note_ht(cbm_work_kind_t kind, void *site, const void *table, const char *key,
uint64_t generation, bool hit) {
(void)kind;
(void)site;
(void)table;
(void)key;
(void)generation;
(void)hit;
}
void cbm_memev_totals(cbm_memev_totals_t *out) {
if (out) {
*out = (cbm_memev_totals_t){0};
}
}
size_t cbm_memev_sites(cbm_memev_site_t *out, size_t max) {
(void)out;
(void)max;
return 0;
}
size_t cbm_work_rows(cbm_work_row_t *out, size_t max) {
(void)out;
(void)max;
return 0;
}
bool cbm_memev_dump(const char *path, const char *why) {
(void)path;
(void)why;
return false;
}
void cbm_memev_reset_for_tests(void) {}
void cbm_memev_scan_for_tests(bool fill, bool dupes) {
(void)fill;
(void)dupes;
}
void cbm_memev_force_for_tests(bool on) {
(void)on;
}
#endif