Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

feat(qa): memory + CPU waste sanitizer over the memory core

Martin Vogel81264814f162
diff --git a/.github/workflows/_memwaste.yml b/.github/workflows/_memwaste.yml
new file mode 100644
index 000000000..ab9e1e3c6
--- /dev/null
+++ b/.github/workflows/_memwaste.yml
@@ -0,0 +1,163 @@
+# Reusable: the waste sanitizer (memory + CPU waste), REPORT-ONLY.
+#
+# What runs (scripts/memwaste.sh, memwaste-scaling.py, memwaste-ratchet.py):
+# mode=pr Linux: the event lane (memory waste + CPU work) indexing this
+# repository at the checked-out commit, and the CPU scaling lane
+# (src/ replicated 2x against 4x, per-function counter ratios).
+# mode=full the same, plus the event lane on macOS and Windows and the access
+# lane (read/write verdicts per allocation) on Linux.
+#
+# The corpus is the repository itself: pinned by the commit, no network, and a
+# real mix of C, C++, Python, JavaScript and vendored code.
+#
+# REPORT-ONLY: every job uploads its report and prints the ratchet comparison
+# against scripts/memwaste-baseline.json, but nothing here can turn a PR red --
+# the ratchet runs without --enforce and the PR's ci-ok gate does not depend on
+# these jobs. A job fails only when the MEASUREMENT is unsound (a table
+# overflowed, frees the layer never saw): that is a defect of the tool, and a
+# red job is how it gets noticed. Switching the ratchet to --enforce is a
+# separate, deliberate change once the numbers have been stable across reruns.
+name: Memwaste
+
+on:
+ workflow_call:
+ inputs:
+ mode:
+ description: "pr (Linux event + scaling) or full (all platforms, access lane)"
+ type: string
+ default: pr
+
+permissions:
+ contents: read
+
+jobs:
+ event-linux:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - name: Install deps
+ run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm
+ - name: Event lane on this repository
+ run: |
+ mkdir -p "$RUNNER_TEMP/memwaste-event"
+ scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \
+ | tee "$RUNNER_TEMP/memwaste-event/report.txt"
+ - name: Ratchet (report-only)
+ run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue ${{ inputs.mode }}-linux
+ - name: Upload report
+ if: ${{ always() }}
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: memwaste-event-linux-${{ inputs.mode }}
+ path: |
+ ${{ runner.temp }}/memwaste-event/report.txt
+ ${{ runner.temp }}/memwaste-event/memwaste.json
+ ${{ runner.temp }}/memwaste-event/index.json
+ if-no-files-found: warn
+
+ scaling-linux:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - name: Install deps
+ run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm
+ - name: CPU scaling lane (src/ at 2x against 4x)
+ # Report-only: the script exits 1 when it flags a super-linear function.
+ continue-on-error: true
+ run: |
+ mkdir -p "$RUNNER_TEMP/memwaste-scaling"
+ python3 scripts/memwaste-scaling.py --corpus src --replicas 2 \
+ --out "$RUNNER_TEMP/memwaste-scaling" | tee "$RUNNER_TEMP/memwaste-scaling/report.txt"
+ - name: Upload report
+ if: ${{ always() }}
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: memwaste-scaling-linux-${{ inputs.mode }}
+ path: ${{ runner.temp }}/memwaste-scaling/report.txt
+ if-no-files-found: warn
+
+ access-linux:
+ if: ${{ inputs.mode == 'full' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 120
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - name: Install deps
+ run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm
+ - name: Access lane on this repository
+ run: |
+ mkdir -p "$RUNNER_TEMP/memwaste-access"
+ scripts/memwaste.sh "$GITHUB_WORKSPACE" --lane access --top 40 --sort untouched \
+ --out "$RUNNER_TEMP/memwaste-access" | tee "$RUNNER_TEMP/memwaste-access/report.txt"
+ - name: Upload report
+ if: ${{ always() }}
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: memwaste-access-linux-${{ inputs.mode }}
+ path: |
+ ${{ runner.temp }}/memwaste-access/report.txt
+ ${{ runner.temp }}/memwaste-access/memwaste.json
+ if-no-files-found: warn
+
+ event-macos:
+ if: ${{ inputs.mode == 'full' }}
+ runs-on: macos-14
+ timeout-minutes: 60
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - name: Event lane on this repository
+ run: |
+ mkdir -p "$RUNNER_TEMP/memwaste-event"
+ scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \
+ | tee "$RUNNER_TEMP/memwaste-event/report.txt"
+ - name: Ratchet (report-only)
+ run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue full-macos
+ - name: Upload report
+ if: ${{ always() }}
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: memwaste-event-macos-${{ inputs.mode }}
+ path: |
+ ${{ runner.temp }}/memwaste-event/report.txt
+ ${{ runner.temp }}/memwaste-event/memwaste.json
+ if-no-files-found: warn
+
+ event-windows:
+ if: ${{ inputs.mode == 'full' }}
+ runs-on: windows-latest
+ timeout-minutes: 90
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2
+ with:
+ msystem: CLANG64
+ path-type: inherit
+ install: >-
+ mingw-w64-clang-x86_64-clang
+ mingw-w64-clang-x86_64-llvm
+ mingw-w64-clang-x86_64-zlib
+ mingw-w64-clang-x86_64-python3
+ make
+ git
+ coreutils
+ - name: Event lane on this repository
+ shell: msys2 {0}
+ run: |
+ out="$(cygpath -u "$RUNNER_TEMP")/memwaste-event"
+ mkdir -p "$out"
+ CC=clang CXX=clang++ scripts/memwaste.sh "$(cygpath -u "$GITHUB_WORKSPACE")" --top 40 --out "$out" \
+ | tee "$out/report.txt"
+ - name: Ratchet (report-only)
+ shell: msys2 {0}
+ run: python3 scripts/memwaste-ratchet.py --out "$(cygpath -u "$RUNNER_TEMP")/memwaste-event" --venue full-windows
+ - name: Upload report
+ if: ${{ always() }}
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: memwaste-event-windows-${{ inputs.mode }}
+ path: |
+ ${{ runner.temp }}/memwaste-event/report.txt
+ ${{ runner.temp }}/memwaste-event/memwaste.json
+ if-no-files-found: warn
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 4fef6ea68..0cfc796be 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -154,6 +154,17 @@ jobs:
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 1000M
+ # ── Waste sanitizer, REPORT-ONLY: memory + CPU waste on this repository and
+ # the per-function scaling lane (Linux). Deliberately NOT in ci-ok's needs:
+ # it uploads evidence and prints the ratchet, it never blocks a merge. Only
+ # product-code PRs pay for it. ──
+ memwaste:
+ needs: [changes]
+ if: ${{ !cancelled() && needs.changes.outputs.product == 'true' }}
+ uses: ./.github/workflows/_memwaste.yml
+ with:
+ mode: pr
+
ci-ok:
# The one required context (besides dco) — fails unless every PR stage
# succeeded, so matrix renames can never silently deadlock merges. `skipped`
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 15f19f41f..c0e796364 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -145,6 +145,15 @@ jobs:
version: ${{ inputs.version }}
use_release_artifacts: true
+ # ── Waste sanitizer, REPORT-ONLY ────────────────────────────────
+ # The same full run as the dry run, on the release commit. Not in
+ # release-draft's needs: the report is evidence for the release notes, never
+ # a publication gate.
+ memwaste:
+ uses: ./.github/workflows/_memwaste.yml
+ with:
+ mode: full
+
# ── 6. Create DRAFT release ────────────────────────────────────
# Requires smoke to have actually SUCCEEDED, not merely "not failed". The bare
# !cancelled() && !failure() form is fail-OPEN: a skipped smoke is neither
diff --git a/Makefile.cbm b/Makefile.cbm
index 066d3375c..317b51411 100644
--- a/Makefile.cbm
+++ b/Makefile.cbm
@@ -76,8 +76,8 @@ endif
# the prod build uses mimalloc (MI_OVERRIDE=1); the test build is CRT+ASan, where
# binding would create an alloc/free mismatch, so the guard is prod-only.
CFLAGS_PROD = $(CFLAGS_COMMON) -O2 -DCBM_BIND_TS_ALLOCATOR=1 $(TEST_SEAM_DEFINE) \
- $(GLOBAL_OVERRIDE_DEFINE) $(CFLAGS_EXTRA)
-CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2
+ $(GLOBAL_OVERRIDE_DEFINE) $(MEMWASTE_CFLAGS) $(CFLAGS_EXTRA)
+CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2 $(MEMWASTE_VENDOR_CFLAGS)
# Test flags: debug + sanitizers (override SANITIZE= to disable on Windows)
SANITIZE = -fsanitize=address,undefined -fno-omit-frame-pointer
@@ -216,6 +216,75 @@ ELF_HARDENING_FLAGS := -Wl,-z,noexecstack -Wl,-z,separate-code \
endif
endif
+# MEMWASTE=1: the waste-sanitizer flavour (src/foundation/mem_events.h). Built
+# like production -- -O2, mimalloc, the real allocator routing -- plus the event
+# layer and its observers, because waste measured under ASan's allocator would
+# describe ASan. Frame pointers stay so the macOS observer can walk to the
+# caller. On Linux this is also what finally LINKS the --wrap observation shim:
+# its flags and source were defined here but never reached a link line, so the
+# Linux profiler had been reporting nothing.
+#
+# Every object is built this way, VENDORED ones included (grammars, the
+# tree-sitter runtime, SQLite, tre, lz4, zstd, the C++ preprocessor): waste in
+# a vendored parser is waste in this program. mimalloc alone is left as it is --
+# its reads and writes are the allocator's own bookkeeping, not program data.
+#
+# MEMWASTE_ACCESS=1 adds the access lane (reads and writes of every tracked
+# block through the compiler's load/store callbacks; clang only). It builds at
+# -O1 without vectorisation: at -O2 the callbacks describe the OPTIMISED program,
+# and reads the optimiser removed would be reported as never read.
+# MEMWASTE_PROFILE=1 adds clang's execution counters (-fprofile-instr-generate)
+# for the CPU scaling lane (scripts/memwaste-scaling.py).
+# Each flavour needs its own BUILD_DIR: vendored objects are not rebuilt when
+# only the flavour changes.
+#
+# The libc wrappers (mem_override_libc.c) need --wrap on Linux and Windows;
+# macOS defines the symbols instead. STATIC=1 (musl) is not a memwaste target.
+MEMWASTE ?= 0
+MEMWASTE_ACCESS ?= 0
+MEMWASTE_PROFILE ?= 0
+ifeq ($(MEMWASTE_ACCESS),1)
+override MEMWASTE := 1
+endif
+ifeq ($(MEMWASTE_PROFILE),1)
+override MEMWASTE := 1
+endif
+MEMWASTE_CFLAGS :=
+MEMWASTE_VENDOR_CFLAGS :=
+MEMWASTE_LDFLAGS :=
+MEMWASTE_SRCS :=
+MEMWASTE_ALLOC_SYMS_LINUX := malloc calloc realloc free strdup strndup posix_memalign \
+ aligned_alloc memalign
+MEMWASTE_LIBC_SYMS_LINUX := memcpy memmove memset memcmp strlen strcmp strncmp \
+ __memcpy_chk __memmove_chk __memset_chk read __read_chk write \
+ pread pwrite pread64 pwrite64 open open64 __open_2 __open64_2 \
+ stat lstat fopen fopen64 fread __fread_chk fwrite opendir readdir \
+ pthread_mutex_lock
+MEMWASTE_LIBC_SYMS_WIN := memcpy memmove memset memcmp strlen strcmp strncmp _wfopen fread fwrite
+ifeq ($(MEMWASTE),1)
+MEMWASTE_CFLAGS := -DCBM_MEMWASTE=1 -fno-omit-frame-pointer
+MEMWASTE_VENDOR_CFLAGS := -DCBM_MEMWASTE=1 -fno-omit-frame-pointer
+ifeq ($(IS_LINUX),yes)
+MEMWASTE_LDFLAGS := $(foreach sym,$(MEMWASTE_ALLOC_SYMS_LINUX) $(MEMWASTE_LIBC_SYMS_LINUX),-Wl,--wrap=$(sym))
+MEMWASTE_SRCS := src/foundation/mem_override_posix.c
+endif
+ifeq ($(IS_MINGW),yes)
+MEMWASTE_LDFLAGS := $(foreach sym,$(MEMWASTE_LIBC_SYMS_WIN),-Wl,--wrap=$(sym))
+endif
+endif
+ifeq ($(MEMWASTE_ACCESS),1)
+MEMWASTE_ACCESS_FLAGS := -DCBM_MEMWASTE_ACCESS=1 -O1 -fno-vectorize -fno-slp-vectorize \
+ -fsanitize-coverage=inline-bool-flag,trace-loads,trace-stores
+MEMWASTE_CFLAGS += $(MEMWASTE_ACCESS_FLAGS)
+MEMWASTE_VENDOR_CFLAGS += $(MEMWASTE_ACCESS_FLAGS)
+endif
+ifeq ($(MEMWASTE_PROFILE),1)
+MEMWASTE_PROFILE_FLAGS := -DCBM_MEMWASTE_PROFILE=1 -fprofile-instr-generate -fprofile-update=atomic
+MEMWASTE_CFLAGS += $(MEMWASTE_PROFILE_FLAGS)
+MEMWASTE_VENDOR_CFLAGS += $(MEMWASTE_PROFILE_FLAGS)
+MEMWASTE_LDFLAGS += $(MEMWASTE_PROFILE_FLAGS)
+endif
+
# The POSIX wrap shim exists only so the profiler can observe allocations. It
# must never reach a sanitized build: the Linux/macOS test builds are CRT+ASan,
# and redirecting malloc into mimalloc underneath ASan's own interception mixes
@@ -225,7 +294,7 @@ endif
# instead of the system libstdc++ (scripts/msan.sh overrides both vars).
CXX_STDLIB ?= -lstdc++
CXX_STDLIB_FLAGS ?=
-LDFLAGS = -lm $(CXX_STDLIB) -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS)
+LDFLAGS = -lm $(CXX_STDLIB) -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS) $(MIMALLOC_WRAP_FLAGS) $(MEMWASTE_LDFLAGS) $(ELF_HARDENING_FLAGS)
LDFLAGS_TEST = -lm $(CXX_STDLIB) -lpthread -lz $(SANITIZE) $(WIN32_LIBS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS)
LDFLAGS_TSAN = -lm $(CXX_STDLIB) -lpthread -lz $(TSAN_SANITIZE) $(WIN32_LIBS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS)
@@ -249,6 +318,9 @@ FOUNDATION_SRCS = \
src/foundation/compat_regex.c \
src/foundation/mem.c \
src/foundation/mem_core.c \
+ src/foundation/mem_events.c \
+ src/foundation/mem_override_darwin.c \
+ src/foundation/mem_override_libc.c \
src/foundation/diagnostics.c \
src/foundation/profile.c \
src/foundation/dump_verify.c \
@@ -472,10 +544,19 @@ ifneq ($(filter yes,$(IS_MINGW) $(IS_LINUX)),)
MIMALLOC_OVERRIDE_DEFINE := -DMI_MALLOC_OVERRIDE=1
GLOBAL_OVERRIDE_DEFINE := -DCBM_MEM_GLOBAL_OVERRIDE=1
endif
+# MI_DEBUG=0: the production allocator is mimalloc's RELEASE build. Its header
+# defaults to MI_DEBUG=2 unless NDEBUG is defined, and nothing here defined it,
+# so every shipped binary up to v0.11.0 ran mimalloc's debug build: internal
+# assertions, fine-grained statistics, padding on every block and a memset of
+# every freed block. The waste sanitizer measured it on the Go corpus as 92 GB
+# (macOS) and 444 GB (Linux, where mimalloc also serves libc's malloc) of byte
+# work inside the allocator (2026-09-17). Committed-bytes accounting, which the
+# memory budget reads, is kept in every build (subproc stats, not MI_STAT).
+# The TEST build below keeps mimalloc's debug checks.
MIMALLOC_CFLAGS = -std=c11 -O2 -w \
-Ivendored/mimalloc/include \
-Ivendored/mimalloc/src \
- -DMI_OVERRIDE=1 \
+ -DMI_OVERRIDE=1 -DMI_DEBUG=0 \
$(MIMALLOC_OVERRIDE_DEFINE)
MIMALLOC_CFLAGS_TEST = -std=c11 -g -O1 -w \
-Ivendored/mimalloc/include \
@@ -496,7 +577,7 @@ MIMALLOC_CFLAGS_TEST = -std=c11 -g -O1 -w \
SQLITE3_OMIT_FLAGS = -DSQLITE_OMIT_LOAD_EXTENSION
SQLITE3_SRC = vendored/sqlite3/sqlite3.c
SQLITE3_CFLAGS = -std=c11 -O2 -w -DSQLITE_DQS=0 -DSQLITE_THREADSAFE=1 -DSQLITE_ENABLE_FTS5 \
- $(SQLITE3_OMIT_FLAGS)
+ $(SQLITE3_OMIT_FLAGS) $(MEMWASTE_VENDOR_CFLAGS)
SQLITE3_CFLAGS_TEST = -std=c11 -g -O1 -w -DSQLITE_DQS=0 -DSQLITE_THREADSAFE=1 -DSQLITE_ENABLE_FTS5 \
$(SQLITE3_OMIT_FLAGS)
@@ -508,7 +589,7 @@ TRE_CFLAGS = -std=c11 -g -O1 -w -Ivendored/tre
YYJSON_SRC = vendored/yyjson/yyjson.c
# All production sources
-PROD_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC)
+PROD_SRCS = $(MEMWASTE_SRCS) $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC)
EXISTING_C_SRCS = $(EXTRACTION_SRCS) $(LSP_SRCS) $(TS_RUNTIME_SRC) \
$(GRAMMAR_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC)
@@ -641,7 +722,7 @@ TEST_CLI_SRCS = tests/test_cli.c tests/test_agent_clients.c tests/test_agent_pro
tests/test_config_toml_edit.c tests/test_config_yaml_edit.c tests/test_config_text_edit.c \
tests/test_activation_transaction.c
-TEST_MEM_SRCS = tests/test_mem.c
+TEST_MEM_SRCS = tests/test_mem.c tests/test_mem_events.c
TEST_UI_SRCS = tests/test_ui.c
TEST_HTTPD_SRCS = tests/test_httpd.c
@@ -746,7 +827,8 @@ BUILD_DIR = build/c
# ── Object file compilation (grammars need relaxed warnings) ─────
# Grammar + tree-sitter runtime: compiled without -Werror (upstream code has warnings)
-GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC)
+GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
+ $(MEMWASTE_VENDOR_CFLAGS)
GRAMMAR_CFLAGS_TEST = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
-DCBM_ENABLE_TEST_SEAMS=1 $(SANITIZED_DEFINE) $(SANITIZE)
GRAMMAR_CFLAGS_TSAN = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
@@ -764,7 +846,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o
# ── Targets ──────────────────────────────────────────────────────
-.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security
+.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security fuzz
$(BUILD_DIR):
mkdir -p $(BUILD_DIR)
@@ -868,7 +950,7 @@ $(BUILD_DIR)/tsan_tre.o: $(TRE_SRC) | $(BUILD_DIR)
$(CC) $(TRE_CFLAGS) $(TSAN_SANITIZE) -c -o $@ $<
$(BUILD_DIR)/prod_tre.o: $(TRE_SRC) | $(BUILD_DIR)
- $(CC) $(TRE_CFLAGS) -O2 -c -o $@ $<
+ $(CC) $(TRE_CFLAGS) -O2 $(MEMWASTE_VENDOR_CFLAGS) -c -o $@ $<
endif
# Vendored LZ4 (test build)
@@ -1036,14 +1118,14 @@ $(BUILD_DIR)/prod_preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)
# Vendored LZ4 (compiled separately, not unity-built via lz4_store.c)
LZ4_OBJ_PROD = $(BUILD_DIR)/prod_lz4.o $(BUILD_DIR)/prod_lz4hc.o
$(BUILD_DIR)/prod_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR)
- $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -c -o $@ $<
+ $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR) -c -o $@ $<
$(BUILD_DIR)/prod_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR)
- $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<
+ $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<
# Vendored zstd (compiled separately, not unity-built via zstd_store.c)
ZSTD_OBJ_PROD = $(BUILD_DIR)/prod_zstd.o
$(BUILD_DIR)/prod_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR)
- $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $<
+ $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR)/vendored/zstd -c -o $@ $<
OBJS_VENDORED_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LSP_OBJ_PROD) $(PP_OBJ_PROD) $(LZ4_OBJ_PROD) $(ZSTD_OBJ_PROD) $(UNIXCODER_OBJ)
@@ -1065,7 +1147,7 @@ MAIN_SRC = src/main.c
# for trap debugging -- silently re-run the previously instrumented binary,
# which during this bisect produced a "crashes without sanitizers too" result
# that was pure artifact and sent the investigation down the wrong path.
-BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)
+BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)|MEMWASTE=$(MEMWASTE)|MEMWASTE_ACCESS=$(MEMWASTE_ACCESS)|MEMWASTE_PROFILE=$(MEMWASTE_PROFILE)
.PHONY: build-config-check
build-config-check:
$(BUILD_DIR)/.build-config: build-config-check | $(BUILD_DIR)
@@ -1261,6 +1343,28 @@ test-lsan:
cd $(CURDIR) && ASAN_OPTIONS="detect_leaks=1:halt_on_error=1" \
./build/lsan/test-runner $(LSAN_SUITES)
+# ── Coverage-guided fuzzing (libFuzzer) ─────────────────────────
+# One binary, three targets (tests/fuzz/cbm_fuzz.c): the per-file extractor,
+# the Cypher parser, the TOML/YAML/JSON config editors. ASan+UBSan as in the
+# test lane, plus fuzzer-no-link on EVERY object -- the vendored grammars and
+# the tree-sitter runtime feed coverage back, which is where the parser state
+# space lives. Needs a clang that ships libFuzzer (Homebrew LLVM on macOS; Apple
+# clang does not). Run through scripts/fuzz.sh.
+FUZZ_CC ?= clang
+FUZZ_CXX ?= clang++
+FUZZ_SANITIZE = -fsanitize=address,undefined,fuzzer-no-link -fno-omit-frame-pointer
+FUZZ_SRC = tests/fuzz/cbm_fuzz.c
+fuzz:
+ $(MAKE) -f Makefile.cbm build/fuzz/cbm-fuzz BUILD_DIR=build/fuzz \
+ CC=$(FUZZ_CC) CXX=$(FUZZ_CXX) SANITIZE="$(FUZZ_SANITIZE)"
+
+$(BUILD_DIR)/cbm-fuzz: $(FUZZ_SRC) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) $(PROJECT_HDRS) | $(BUILD_DIR)
+ $(CC) $(CFLAGS_TEST) -fsanitize=fuzzer -o $@ \
+ $(FUZZ_SRC) $(PROD_SRCS) \
+ $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \
+ $(OBJS_VENDORED_TEST) \
+ $(LDFLAGS_TEST)
+
# Path-sensitive memory analysis: leak paths, null derefs, uninitialized reads.
# Non-gating: triage findings against the code (this lane's first run produced
# 9 real fixes and 6 recorded false positives — see the analyzer batch commit).
diff --git a/scripts/memlab-drive.py b/scripts/memlab-drive.py
index cef508810..bac474126 100644
--- a/scripts/memlab-drive.py
+++ b/scripts/memlab-drive.py
@@ -32,6 +32,16 @@ def rpc(proc, payload, timeout_note):
return message
+def tool_failed(reply):
+ """A JSON-RPC error, or a tool result the server marked isError: MCP reports
+ a failed tool call as a normal result, so checking only "error" counts every
+ failure as served (a lab measuring nothing looked healthy)."""
+ if "error" in reply:
+ return True
+ result = reply.get("result")
+ return isinstance(result, dict) and bool(result.get("isError"))
+
+
def main():
parser = argparse.ArgumentParser()
parser.add_argument("binary")
@@ -63,14 +73,16 @@ def main():
"params": {"protocolVersion": "2024-11-05", "capabilities": {},
"clientInfo": {"name": "memlab", "version": "1"}}},
"initialize")
+ project = None
if not args.skip_index:
indexed = rpc(proc, {"jsonrpc": "2.0", "id": 1, "method": "tools/call",
"params": {"name": "index_repository",
- "arguments": {"path": args.corpus}}},
+ "arguments": {"repo_path": args.corpus}}},
"index_repository")
- if "error" in indexed:
- print(f"index failed: {indexed['error']}", file=sys.stderr)
+ if tool_failed(indexed):
+ print(f"index failed: {indexed.get('error') or indexed.get('result')}", file=sys.stderr)
return 3
+ project = (indexed.get("result", {}).get("structuredContent") or {}).get("project")
halfway = 2 + args.requests // 2
for i in range(2, args.requests + 2):
@@ -84,11 +96,15 @@ def main():
"list_projects": {},
"get_graph_schema": {},
"search_code": {"pattern": "Widget"}}.get(args.tool, {})
+ if project and args.tool != "list_projects":
+ arguments = dict(arguments, project=project)
reply = rpc(proc, {"jsonrpc": "2.0", "id": i, "method": "tools/call",
"params": {"name": args.tool, "arguments": arguments}},
f"request {i}")
served += 1
- if "error" in reply:
+ if tool_failed(reply):
+ if failures == 0:
+ print(f"first failure (request {i}): {json.dumps(reply)[:600]}", file=sys.stderr, flush=True)
failures += 1
finally:
try:
diff --git a/scripts/memlab-report.py b/scripts/memlab-report.py
index cf538fa7d..eae4bccf8 100644
--- a/scripts/memlab-report.py
+++ b/scripts/memlab-report.py
@@ -1,19 +1,20 @@
#!/usr/bin/env python3
"""Analyse #581 memory-lab output: attribute growth to allocation sites.
-Reads the JSONL the profiler appends (CBM_MEM_PROFILE_OUT) and the mem.census
-lines from a daemon log, and prints:
+Reads the waste layer's dumps (CBM_MEMWASTE_OUT, one at every request-stage phase
+mark; see scripts/memlab.sh) and the mem.census lines from a daemon log, and
+prints:
* retention per request, as a linear fit rather than first-vs-last, so noise
does not masquerade as a trend
* the call sites whose live bytes grow across the run, ranked by growth
- * an explicit unattributed remainder: process growth the profiler could not
- account for. That number is the honesty check — a report that attributed
- everything while tracking only >=32 KiB allocations would be lying.
+ * an explicit unattributed remainder: process growth the layer could not
+ account for (allocator metadata, fragmentation, memory outside malloc).
+ That number is the honesty check.
Usage:
- memlab-report.py <profile.jsonl> [--census <daemon.log>] [--top N]
- memlab-report.py <a.jsonl> --diff <b.jsonl> # platform comparison
+ memlab-report.py <dump.jsonl> [--census <daemon.log>] [--binary BIN] [--top N]
+ memlab-report.py <a.jsonl> --diff <b.jsonl> [--binary BIN] # platform comparison
"""
import argparse
import json
@@ -23,9 +24,11 @@ from collections import defaultdict
def load_profile(path):
- """Return (summaries, site_series). Records are appended over time, so the
- Nth occurrence of a site is its Nth sample."""
- summaries, series = [], defaultdict(list)
+ """Return (summaries, site_series) for the process with the most dumps.
+ Every dump lists every site, so the Nth occurrence of a site is its Nth
+ sample; a site first seen late is padded with zeros so series align."""
+ procs = defaultdict(lambda: {"summaries": [], "series": defaultdict(list)})
+ cur = None
with open(path, "r", errors="replace") as handle:
for line in handle:
line = line.strip()
@@ -35,11 +38,31 @@ def load_profile(path):
record = json.loads(line)
except json.JSONDecodeError:
continue
- if "site" in record:
- series[record["site"]].append(record)
- else:
- summaries.append(record)
- return summaries, series
+ if "memwaste" in record:
+ cur = procs[record.get("pid", 0)]
+ cur["summaries"].append(record)
+ elif cur is not None and "site" in record and "work" not in record:
+ samples = cur["series"][record["site"]]
+ while len(samples) < len(cur["summaries"]) - 1:
+ samples.append({"live_bytes": 0})
+ samples.append(record)
+ if not procs:
+ return [], {}
+ best = max(procs.values(), key=lambda p: len(p["summaries"]))
+ return best["summaries"], best["series"]
+
+
+def load_symbols(binary, summaries, sites):
+ """Symbolise site addresses with the waste report's own symboliser."""
+ if not binary or not summaries:
+ return {}
+ import importlib.util
+ import os
+ spec = importlib.util.spec_from_file_location(
+ "memwaste_report", os.path.join(os.path.dirname(os.path.abspath(__file__)), "memwaste-report.py"))
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ return mod.symbolise(binary, summaries[-1].get("image_base", "0x0"), list(sites))
def linear_slope(values):
@@ -57,7 +80,7 @@ def linear_slope(values):
def parse_census(path):
"""Pull the pool totals out of mem.census log lines."""
- fields = ("priv_kb", "rss_kb", "crt_kb", "biggest_kb", "prof_live_kb")
+ fields = ("priv_kb", "rss_kb", "crt_kb", "biggest_kb")
rows = []
pattern = re.compile(r"(\w+)=(\S+)")
with open(path, "r", errors="replace") as handle:
@@ -71,16 +94,16 @@ def parse_census(path):
return rows
-def report(path, census_path, top):
+def report(path, census_path, top, binary=None):
summaries, series = load_profile(path)
if not summaries and not series:
- print(f"{path}: no profiler records — was CBM_MEM_PROFILE=1 set?")
+ print(f"{path}: no waste-layer records -- was the memwaste flavour run with CBM_MEMWASTE=1?")
return
+ names = load_symbols(binary, summaries, series.keys())
last = summaries[-1] if summaries else {}
print(f"=== {path} ===")
if last:
- print(f"threshold : {last.get('threshold', 0)} bytes "
- f"(allocations below this are invisible by design)")
+ print(f"samples : {len(summaries)} dumps (every allocation size is tracked)")
print(f"sites seen : {last.get('sites', 0)}")
print(f"attributed live: {last.get('live_bytes', 0)/1048576:.1f} MB "
f"in {last.get('live_blocks', 0)} blocks")
@@ -94,23 +117,23 @@ def report(path, census_path, top):
slope = linear_slope(live)
if slope <= 0 and live[-1] <= live[0]:
continue
- growth.append((slope, live[-1] - live[0], live[-1], samples[-1]))
+ growth.append((slope, live[-1] - live[0], live[-1], names.get(site, site)))
growth.sort(key=lambda row: row[0], reverse=True)
print(f"\n--- sites whose retained bytes GROW (top {top}) ---")
if not growth:
print("none — no tracked site retains more over time.")
- for slope, delta, live, sample in growth[:top]:
- frames = " <- ".join(sample.get("frames", [])[:4])
+ for slope, delta, live, name in growth[:top]:
print(f" +{slope/1024:8.1f} KB/sample total +{delta/1048576:6.2f} MB "
f"live {live/1048576:6.2f} MB")
- print(f" {frames}")
+ print(f" {name}")
if census_path:
rows = parse_census(census_path)
if rows:
priv = [r["priv_kb"] for r in rows if "priv_kb" in r]
- prof = [r["prof_live_kb"] for r in rows if "prof_live_kb" in r]
+ # attributed = what the waste layer holds live, sampled at its own dumps
+ prof = [int(h.get("live_bytes", 0)) // 1024 for h in summaries]
print(f"\n--- pools vs attribution ({len(rows)} samples) ---")
if priv:
print(f" private committed : {priv[0]/1024:.1f} -> {priv[-1]/1024:.1f} MB "
@@ -119,24 +142,27 @@ def report(path, census_path, top):
unattributed = (priv[-1] - priv[0]) - (prof[-1] - prof[0])
print(f" attributed growth : {(prof[-1]-prof[0])/1024:+.1f} MB")
print(f" UNATTRIBUTED : {unattributed/1024:+.1f} MB "
- "<- below-threshold allocations, allocator metadata, or fragmentation")
+ "<- allocator metadata, fragmentation, or memory outside malloc (mmap, stacks)")
-def diff(path_a, path_b, top):
- """Same sites, two platforms: what does one retain that the other does not?"""
- _, series_a = load_profile(path_a)
+def diff(path_a, path_b, top, binary=None):
+ """Same sites, two platforms: what does one retain that the other does not?
+ Raw addresses differ between binaries, so sites are matched by symbol name
+ when --binary is given (the same binary for both runs) and by address otherwise."""
+ summaries_a, series_a = load_profile(path_a)
_, series_b = load_profile(path_b)
+ names = load_symbols(binary, summaries_a, set(series_a) | set(series_b))
rows = []
for site, samples in series_a.items():
a_live = samples[-1]["live_bytes"]
b_live = series_b[site][-1]["live_bytes"] if site in series_b else 0
if a_live - b_live > 0:
- rows.append((a_live - b_live, a_live, b_live, samples[-1]))
+ rows.append((a_live - b_live, a_live, b_live, site))
rows.sort(key=lambda row: row[0], reverse=True)
print(f"=== {path_a} minus {path_b} (top {top}) ===")
- for delta, a_live, b_live, sample in rows[:top]:
+ for delta, a_live, b_live, site in rows[:top]:
print(f" +{delta/1048576:7.2f} MB A={a_live/1048576:.2f} MB B={b_live/1048576:.2f} MB")
- print(f" {' <- '.join(sample.get('frames', [])[:4])}")
+ print(f" {names.get(site, site)}")
def main():
@@ -145,11 +171,12 @@ def main():
parser.add_argument("--census")
parser.add_argument("--diff")
parser.add_argument("--top", type=int, default=12)
+ parser.add_argument("--binary", help="the memwaste binary, for symbol names")
args = parser.parse_args()
if args.diff:
- diff(args.profile, args.diff, args.top)
+ diff(args.profile, args.diff, args.top, args.binary)
else:
- report(args.profile, args.census, args.top)
+ report(args.profile, args.census, args.top, args.binary)
return 0
diff --git a/scripts/memlab.sh b/scripts/memlab.sh
index 373742556..537738d60 100644
--- a/scripts/memlab.sh
+++ b/scripts/memlab.sh
@@ -10,9 +10,14 @@
#
# Usage: scripts/memlab.sh <binary> [requests] [label]
#
-# Output: memlab-<label>.jsonl (profiler records, one block per sample)
+# <binary> must be the waste-sanitizer flavour (make -f Makefile.cbm cbm
+# MEMWASTE=1 BUILD_DIR=build/memwaste): the attribution comes from its event
+# layer, which observes every allocation on Linux, macOS and Windows alike.
+#
+# Output: memlab-<label>.jsonl (a waste-layer dump at every request-stage phase mark)
# memlab-<label>.log (daemon log with mem.census lines)
-# Analyse with: scripts/memlab-report.py memlab-<label>.jsonl --census memlab-<label>.log
+# Analyse with: scripts/memlab-report.py memlab-<label>.jsonl --census memlab-<label>.log \
+# --binary <binary>
set -u
BINARY="${1:?usage: memlab.sh <binary> [requests] [label]}"
@@ -115,8 +120,13 @@ if command -v cygpath >/dev/null 2>&1 && ! command -v winepath >/dev/null 2>&1;
else
export CBM_CACHE_DIR="$WORK/cache"
fi
-export CBM_MEM_PROFILE=1
-export CBM_MEM_PROFILE_OUT="$PROFILE_OUT"
+export CBM_MEMWASTE=1
+export CBM_MEMWASTE_OUT="$PROFILE_OUT"
+# The server marks a memory phase at every request stage, and the layer dumps
+# at each mark: live bytes per site become a series over identical requests. The
+# series needs neither the fill scan nor the duplicate hash, which only cost time.
+export CBM_MEMWASTE_FILL=0
+export CBM_MEMWASTE_DUPES=0
export CBM_MEM_CENSUS=1
export CBM_LOG_LEVEL=info
export CBM_LOG_FORMAT=text
@@ -155,9 +165,8 @@ if [ "$CENSUS" -eq 0 ]; then
echo "WARN: no census samples — check CBM_MEM_CENSUS wiring" >&2
fi
if [ "$SITES" -eq 0 ]; then
- # Not fatal, but never silent: on macOS there is no --wrap, so the
- # profiler legitimately has no observation point.
- echo "WARN: no profiler records — expected on macOS (no --wrap); a gap anywhere else" >&2
+ # Never silent: an empty dump means the binary is not the memwaste flavour.
+ echo "WARN: no waste-layer records -- is $BINARY built with MEMWASTE=1?" >&2
fi
echo "profile: $PROFILE_OUT"
echo "log: $RUN_LOG"
diff --git a/scripts/memwaste-padding.py b/scripts/memwaste-padding.py
new file mode 100644
index 000000000..0921ef462
--- /dev/null
+++ b/scripts/memwaste-padding.py
@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+"""
+memwaste-padding.py — the layout lane of the waste sanitizer: bytes every
+instance of a struct carries for alignment alone.
+
+Padding is waste multiplied by the instance count: 4 bytes in a record the graph
+buffer holds 15 million of is 57 MB. The compiler knows every hole; this lane
+asks it, for EVERY translation unit of the production build -- vendored code
+included (grammars, the tree-sitter runtime, SQLite, mimalloc, tre, lz4, zstd,
+the C++ preprocessor) -- by replaying the build's own compile commands with
+`-fsyntax-only -Wno-everything -Wpadded`.
+
+Per struct it reports the interior holes (removable by ordering fields from the
+largest alignment down), the tail padding (removable only when the interior
+reordering frees enough bytes), and how many allocation sites name the type in a
+sizeof, as a first hint of instance counts. The event lane's per-site requested
+sizes are the runtime evidence to pair it with.
+
+Usage:
+ scripts/memwaste-padding.py # every TU, top 40 by interior padding
+ scripts/memwaste-padding.py --project-only # skip vendored TUs
+ scripts/memwaste-padding.py --json
+"""
+from __future__ import annotations
+
+import argparse
+import concurrent.futures as cf
+import json
+import os
+import re
+import shlex
+import subprocess
+import sys
+from collections import defaultdict
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+SRC_EXT = (".c", ".cc", ".cpp")
+KEEP_WITH_ARG = {"-include", "-isystem", "-iquote", "-arch", "-target"}
+WARN_RE = re.compile(
+ r"^(?P<file>[^:]+):(?P<line>\d+):\d+: warning: padding (?:struct|class|size of) "
+ r"'(?P<type>[^']+)' with (?P<bytes>\d+) bytes? to (?P<what>align '(?P<field>[^']*)'|alignment boundary)")
+
+
+def build_commands(cc: str, cxx: str) -> list[tuple[str, list[str], str]]:
+ dry = subprocess.run(["make", "-f", "Makefile.cbm", "-n", "cbm", "BUILD_DIR=build/memwaste-padding-dry",
+ f"CC={cc}", f"CXX={cxx}", "MEMWASTE=1"],
+ cwd=ROOT, capture_output=True, text=True, check=False)
+ text = dry.stdout.replace("\\\n", " ")
+ jobs = []
+ seen = set()
+ for line in text.splitlines():
+ try:
+ argv = shlex.split(line)
+ except ValueError:
+ continue
+ if not argv or os.path.basename(argv[0]) not in (cc, cxx, os.path.basename(cc), os.path.basename(cxx)):
+ continue
+ flags = []
+ sources = []
+ i = 1
+ while i < len(argv):
+ a = argv[i]
+ if a in KEEP_WITH_ARG and i + 1 < len(argv):
+ flags += [a, argv[i + 1]]
+ i += 2
+ continue
+ if a == "-o":
+ i += 2
+ continue
+ if a.startswith(("-I", "-D", "-U", "-std=")):
+ flags.append(a)
+ elif a.endswith(SRC_EXT) and os.path.isfile(os.path.join(ROOT, a)):
+ sources.append(a)
+ i += 1
+ for s in sources:
+ key = (s, tuple(flags))
+ if key not in seen:
+ seen.add(key)
+ jobs.append((argv[0], flags, s))
+ return jobs
+
+
+def run_one(job: tuple[str, list[str], str]) -> tuple[str, str]:
+ compiler, flags, source = job
+ res = subprocess.run([compiler, "-fsyntax-only", "-Wno-everything", "-Wpadded"] + flags + [source],
+ cwd=ROOT, capture_output=True, text=True, check=False)
+ return source, res.stderr
+
+
+def is_vendored(path: str) -> bool:
+ return "/vendored/" in f"/{path}" or path.startswith("vendored/")
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--cc", default="clang")
+ ap.add_argument("--cxx", default="clang++")
+ ap.add_argument("--top", type=int, default=40)
+ ap.add_argument("--project-only", action="store_true")
+ ap.add_argument("--jobs", type=int, default=os.cpu_count() or 4)
+ ap.add_argument("--json", action="store_true")
+ args = ap.parse_args()
+
+ jobs = build_commands(args.cc, args.cxx)
+ if args.project_only:
+ jobs = [j for j in jobs if not is_vendored(j[2])]
+ if not jobs:
+ sys.exit("memwaste-padding: no compile commands found (make -n printed none for the compiler)")
+
+ holes: dict[str, dict] = {}
+ with cf.ThreadPoolExecutor(max_workers=args.jobs) as pool:
+ for source, stderr in pool.map(run_one, jobs):
+ for line in stderr.splitlines():
+ m = WARN_RE.match(line)
+ if not m:
+ continue
+ file = os.path.relpath(os.path.join(ROOT, m.group("file")), ROOT) \
+ if not os.path.isabs(m.group("file")) else os.path.relpath(m.group("file"), ROOT)
+ t = holes.setdefault(m.group("type"), {"type": m.group("type"), "interior": {}, "tail": 0,
+ "where": f"{file}:{m.group('line')}",
+ "vendored": is_vendored(file)})
+ n = int(m.group("bytes"))
+ if m.group("field") is not None:
+ t["interior"][m.group("field")] = n # a header seen by many TUs reports the same hole
+ else:
+ t["tail"] = n
+ t["where"] = f"{file}:{m.group('line')}"
+
+ # Allocation hint: how many allocation expressions size by this type.
+ sizeof_sites: dict[str, int] = defaultdict(int)
+ names = {t.split(" ", 1)[-1] for t in holes}
+ alloc_re = re.compile(r"(?:alloc|calloc|realloc|arena_alloc|CBM_DA_PUSH|cbm_da_push)\w*\s*\([^;]*sizeof\s*\(\s*"
+ r"(?:struct\s+)?(\w+)\s*\)")
+ for base in ("src", "internal/cbm"):
+ for dirpath, _, filenames in os.walk(os.path.join(ROOT, base)):
+ if "/vendored" in dirpath:
+ continue
+ for fn in filenames:
+ if not fn.endswith((".c", ".h")):
+ continue
+ try:
+ text = open(os.path.join(dirpath, fn), encoding="utf-8", errors="replace").read()
+ except OSError:
+ continue
+ for m in alloc_re.finditer(text):
+ if m.group(1) in names:
+ sizeof_sites[m.group(1)] += 1
+
+ rows = []
+ for t in holes.values():
+ interior = sum(t["interior"].values())
+ short = t["type"].split(" ", 1)[-1]
+ rows.append({"type": t["type"], "interior_bytes": interior, "tail_bytes": t["tail"],
+ "holes": len(t["interior"]), "alloc_sites": sizeof_sites.get(short, 0),
+ "where": t["where"], "vendored": t["vendored"]})
+ rows.sort(key=lambda r: (-(r["alloc_sites"] > 0), -r["interior_bytes"], r["type"]))
+
+ if args.json:
+ json.dump({"translation_units": len(jobs), "structs": rows}, sys.stdout, indent=1)
+ print()
+ return 0
+ print(f"translation units checked: {len(jobs)} structs with padding: {len(rows)} "
+ f"(vendored: {sum(1 for r in rows if r['vendored'])})")
+ print()
+ print(f" {'interior B':>10} {'tail B':>6} {'holes':>5} {'alloc sites':>11} struct (declared at)")
+ for r in rows[: args.top]:
+ tag = " [vendored]" if r["vendored"] else ""
+ print(f" {r['interior_bytes']:>10} {r['tail_bytes']:>6} {r['holes']:>5} {r['alloc_sites']:>11} "
+ f"{r['type']}{tag} ({r['where']})")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/memwaste-ratchet.py b/scripts/memwaste-ratchet.py
new file mode 100644
index 000000000..2c886ee54
--- /dev/null
+++ b/scripts/memwaste-ratchet.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""
+memwaste-ratchet.py — hold the waste numbers where they are, lower them as they fall.
+
+Same shape as the memory-core linter's ratchet: a checked-in baseline per venue
+(scripts/memwaste-baseline.json), every waste class normalised PER THOUSAND GRAPH
+NODES so a corpus that grows does not look like a regression. A class that grows
+past its baseline by more than --tolerance fails under --enforce; without it the
+comparison is printed and the exit is 0 (report-only rollout). A class that
+FALLS is reported so the baseline can be lowered with --record.
+
+The inputs are what scripts/memwaste.sh leaves in its --out directory:
+memwaste.json (the report's --json) and index.json (the index result, for the
+node count).
+
+Usage:
+ memwaste-ratchet.py --out DIR --venue pr-linux # compare, report-only
+ memwaste-ratchet.py --out DIR --venue pr-linux --enforce # compare, gate
+ memwaste-ratchet.py --out DIR --venue pr-linux --record # write the baseline
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+BASELINE = os.path.join(ROOT, "scripts", "memwaste-baseline.json")
+
+# Classes the ratchet holds. Left out on purpose: allocator-internal byte work
+# (it moves with allocator settings), the access-lane classes (one lane only),
+# and contended mutex acquisitions and pool imbalance -- those depend on thread
+# scheduling, and a gate on them would be a lottery (O9). They stay in the report.
+CLASSES = [
+ "slack_bytes", "never_written_bytes", "over_requested_bytes", "realloc_copy_bytes",
+ "short_lived", "dup_bytes_max", "retained_bytes_max", "raw_bytes",
+ "container_idle_peak_bytes", "byte_work_bytes", "byte_work_repeats", "io_tiny_calls",
+ "path_repeats", "ht_repeat_lookups", "ht_growths",
+]
+
+
+def current(out_dir: str) -> dict:
+ report = json.load(open(os.path.join(out_dir, "memwaste.json")))
+ index = json.load(open(os.path.join(out_dir, "index.json")))
+ nodes = int(index.get("nodes") or 0)
+ if nodes <= 0:
+ sys.exit("memwaste-ratchet: index.json has no node count")
+ # The index worker is the process that allocated the most.
+ worker = max(report["processes"].values(), key=lambda p: p["header"].get("usable_bytes", 0))
+ summary = worker["summary"]
+ return {
+ "nodes": nodes,
+ "per_1k_nodes": {c: round(summary.get(c, 0) * 1000.0 / nodes, 3) for c in CLASSES},
+ }
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--out", required=True, help="memwaste.sh --out directory")
+ ap.add_argument("--venue", required=True, help="baseline key, e.g. pr-linux, dryrun-macos")
+ ap.add_argument("--tolerance", type=float, default=0.05, help="allowed growth before a class fails")
+ ap.add_argument("--enforce", action="store_true")
+ ap.add_argument("--record", action="store_true")
+ args = ap.parse_args()
+
+ cur = current(args.out)
+ base_all = json.load(open(BASELINE)) if os.path.exists(BASELINE) else {}
+ if args.record:
+ base_all[args.venue] = cur
+ with open(BASELINE, "w") as fh:
+ json.dump(base_all, fh, indent=1, sort_keys=True)
+ fh.write("\n")
+ print(f"baseline recorded for {args.venue} ({cur['nodes']:,} nodes)")
+ return 0
+ base = base_all.get(args.venue)
+ if not base:
+ print(f"no baseline for {args.venue}: nothing to hold yet (record one with --record)")
+ return 0
+
+ worse, better = [], []
+ print(f"{'class':<28} {'baseline/1k':>14} {'now/1k':>14} {'delta':>8}")
+ for c in CLASSES:
+ b = base["per_1k_nodes"].get(c, 0.0)
+ n = cur["per_1k_nodes"].get(c, 0.0)
+ delta = (n - b) / b if b else (0.0 if n == 0 else float("inf"))
+ mark = ""
+ if delta > args.tolerance:
+ worse.append(c)
+ mark = " WORSE"
+ elif delta < -args.tolerance:
+ better.append(c)
+ mark = " better"
+ shown = "new" if delta == float("inf") else f"{delta * 100:+.1f}%"
+ print(f"{c:<28} {b:>14,.3f} {n:>14,.3f} {shown:>8}{mark}")
+ if better:
+ print(f"\n{len(better)} class(es) improved beyond tolerance: lower the baseline with --record")
+ if worse:
+ print(f"\n{len(worse)} class(es) grew beyond {args.tolerance * 100:.0f}%: {', '.join(worse)}")
+ return 1 if args.enforce else 0
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/memwaste-report.py b/scripts/memwaste-report.py
new file mode 100644
index 000000000..1764a4e46
--- /dev/null
+++ b/scripts/memwaste-report.py
@@ -0,0 +1,392 @@
+#!/usr/bin/env python3
+"""
+memwaste-report.py — turn waste-sanitizer dumps into ranked, symbolised evidence.
+
+The sanitizer (src/foundation/mem_events.h) appends JSON lines at every phase
+mark: a header per dump, then one object per allocation site and one per CPU
+work row, all with RAW return addresses. Symbolisation is offline so the hot
+path never loads a symbol handler. This tool reads every dump, keeps the LAST
+dump of each process for the cumulative counters, the MAXIMUM over its dumps for
+the phase snapshots (retained, duplicates), resolves the addresses against the
+binary that produced them, and prints:
+
+ 1. a waste summary per process — one line per waste class, in bytes/calls;
+ 2. the memory table — allocation sites ranked by a chosen waste metric;
+ 3. containers — arenas, dynamic arrays, hash tables: capacity against use;
+ 4. CPU work — byte work, I/O, contention, pool imbalance, repeated work.
+
+Usage:
+ memwaste-report.py DUMP.jsonl --binary build/memwaste/codebase-memory-mcp
+ memwaste-report.py DUMP.jsonl --binary ... --sort never_written --top 30
+ memwaste-report.py DUMP.jsonl --binary ... --json (for the ratchet)
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import platform
+import re
+import shutil
+import subprocess
+import sys
+from collections import defaultdict
+
+MB = 1024.0 * 1024.0
+
+SNAPSHOT_FIELDS = ("retained_bytes", "dup_bytes", "dup_blocks")
+
+WASTE_CLASSES = [
+ # (key, label, unit)
+ ("slack_bytes", "allocator slack (usable - requested)", "bytes"),
+ ("never_written_bytes", "never-written bytes (fill pattern intact at free)", "bytes"),
+ ("over_requested_bytes", "requested beyond the last written byte", "bytes"),
+ ("zero_untouched_bytes", "calloc tail still zero at free (upper bound)", "bytes"),
+ ("realloc_copy_bytes", "bytes copied by realloc growth", "bytes"),
+ ("short_lived", "short-lived allocations (churn)", "calls"),
+ ("dup_bytes_max", "duplicate small blocks (peak over phases)", "bytes"),
+ ("retained_bytes_max", "live across a phase boundary (peak)", "bytes"),
+ ("raw_bytes", "bytes that never went through the memory core", "bytes"),
+ ("acc_untouched_bytes", "blocks never read or written (access lane)", "bytes"),
+ ("acc_dead_bytes", "blocks written, never read by instrumented code (access lane)", "bytes"),
+ ("acc_opaque_blocks", "blocks used only by code the access lane cannot see (no verdict)", "blocks"),
+ ("acc_idle_bytes", "blocks idle for at least one phase before free (access lane)", "bytes"),
+ ("acc_uninit_reads", "reads before any write, not calloc (access lane)", "blocks"),
+ ("container_idle_peak_bytes", "container capacity left unused (largest instance, per site)", "bytes"),
+ ("byte_work_bytes", "bytes moved/compared by mem*/str* calls in program code", "bytes"),
+ ("allocator_byte_work_bytes", "bytes moved/compared inside the allocator (mimalloc)", "bytes"),
+ ("byte_work_repeats", "strlen of a string just measured", "calls"),
+ ("io_tiny_calls", "I/O calls under 512 bytes", "calls"),
+ ("path_repeats", "open/stat of a path already opened/statted", "calls"),
+ ("mutex_contended", "contended mutex acquisitions", "calls"),
+ ("pool_imbalance_items", "work items an even pool would have spread", "items"),
+ ("pool_ops_imbalance", "worker events an even pool would have spread (work done)", "events"),
+ ("ht_repeat_lookups", "hash lookups of the key object just looked up", "calls"),
+ ("ht_growths", "hash-table growths", "calls"),
+]
+
+BYTE_KINDS = {"memcpy", "memmove", "memset", "memcmp", "strlen", "strcmp", "strncmp"}
+IO_KINDS = {"read", "write", "pread", "pwrite", "fread", "fwrite"}
+PATH_KINDS = {"open", "stat", "fopen", "opendir"}
+CONTAINER_KINDS = {"ct_arena", "ct_dyn_array", "ct_hash_table"}
+# Symbols of the vendored allocator. Its byte work is real CPU, but the fix is in
+# how the allocator is built or configured, not at a program call site.
+ALLOCATOR_SYMBOL = re.compile(r"^_?mi_")
+
+
+def load(path: str):
+ """{pid: {"header": last header, "headers": [...], "sites": {addr: row}, "snap_max": {addr: {...}},
+ "work": {(kind, addr): row}}} — sites/work from the LAST dump of the process."""
+ procs: dict[int, dict] = {}
+ cur = None
+ with open(path, encoding="utf-8", errors="replace") as fh:
+ for line in fh:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ obj = json.loads(line)
+ except json.JSONDecodeError:
+ continue # a torn line from a process that died mid-write
+ if "memwaste" in obj:
+ pid = int(obj.get("pid", 0))
+ p = procs.setdefault(pid, {"headers": [], "snap_max": defaultdict(dict)})
+ p["headers"].append(obj)
+ p["header"] = obj
+ p["sites"] = {}
+ p["work"] = {}
+ cur = p
+ elif cur is None:
+ continue
+ elif "work" in obj:
+ cur["work"][(obj["work"], obj["site"])] = obj
+ elif "site" in obj:
+ cur["sites"][obj["site"]] = obj
+ snap = cur["snap_max"][obj["site"]]
+ for f in SNAPSHOT_FIELDS:
+ v = obj.get(f, 0)
+ if v > snap.get(f, 0):
+ snap[f] = v
+ snap[f + "_phase"] = cur["header"].get("phase_label", "?")
+ return procs
+
+
+def pe_image_base(binary: str) -> int | None:
+ """The preferred ImageBase of a PE file, or None for anything else. A PE
+ symboliser wants VA = ImageBase + RVA, and the dump gives RVA + runtime base."""
+ try:
+ with open(binary, "rb") as fh:
+ head = fh.read(4096)
+ except OSError:
+ return None
+ if head[:2] != b"MZ" or len(head) < 0x40:
+ return None
+ pe = int.from_bytes(head[0x3C:0x40], "little")
+ if head[pe:pe + 4] != b"PE\0\0":
+ return None
+ opt = pe + 24
+ magic = int.from_bytes(head[opt:opt + 2], "little")
+ if magic == 0x20B: # PE32+
+ return int.from_bytes(head[opt + 24:opt + 32], "little")
+ if magic == 0x10B: # PE32
+ return int.from_bytes(head[opt + 28:opt + 32], "little")
+ return None
+
+
+def symbolise(binary: str | None, base: str, addrs: list[str]) -> dict[str, str]:
+ if not binary or not addrs:
+ return {}
+ if not os.path.exists(binary) and os.path.exists(binary + ".exe"):
+ binary = binary + ".exe"
+ out: dict[str, str] = {}
+ uniq = sorted(set(addrs))
+ if platform.system() == "Darwin" and shutil.which("atos"):
+ for i in range(0, len(uniq), 2000):
+ chunk = uniq[i:i + 2000]
+ res = subprocess.run(["atos", "-o", binary, "-l", base] + chunk,
+ capture_output=True, text=True, check=False)
+ for addr, name in zip(chunk, res.stdout.splitlines()):
+ out[addr] = name.strip()
+ elif shutil.which("addr2line") or shutil.which("llvm-addr2line"):
+ tool = shutil.which("llvm-addr2line") or shutil.which("addr2line")
+ b = int(base, 16)
+ preferred = pe_image_base(binary) or 0
+ for i in range(0, len(uniq), 2000):
+ chunk = uniq[i:i + 2000]
+ rel = [hex(int(a, 16) - b + preferred) if b else a for a in chunk]
+ res = subprocess.run([tool, "-f", "-C", "-p", "-e", binary] + rel,
+ capture_output=True, text=True, check=False)
+ for addr, name in zip(chunk, res.stdout.splitlines()):
+ out[addr] = name.strip()
+ return out
+
+
+def pct(part: float, whole: float) -> float:
+ return 100.0 * part / whole if whole else 0.0
+
+
+def summarise(p: dict, names: dict[str, str]) -> dict:
+ s = defaultdict(int)
+ for addr, r in p["sites"].items():
+ s["slack_bytes"] += max(r["usable_bytes"] - r["requested_bytes"], 0)
+ for k in ("never_written_bytes", "over_requested_bytes", "zero_untouched_bytes",
+ "realloc_copy_bytes", "short_lived", "raw_bytes", "acc_untouched_bytes",
+ "acc_dead_bytes", "acc_idle_bytes", "acc_uninit_reads", "acc_opaque_blocks"):
+ s[k] += r.get(k, 0)
+ snap = p["snap_max"].get(addr, {})
+ s["dup_bytes_max"] += snap.get("dup_bytes", 0)
+ s["retained_bytes_max"] += snap.get("retained_bytes", 0)
+ for (kind, addr), w in p["work"].items():
+ if kind in CONTAINER_KINDS:
+ s["container_idle_peak_bytes"] += w.get("peak", max(w["bytes"] - w["aux1"], 0))
+ if kind == "ct_hash_table":
+ s["ht_growths"] += w["aux2"]
+ elif kind in BYTE_KINDS:
+ if ALLOCATOR_SYMBOL.match(names.get(addr, "")):
+ s["allocator_byte_work_bytes"] += w["bytes"]
+ else:
+ s["byte_work_bytes"] += w["bytes"]
+ s["byte_work_repeats"] += w["repeats"]
+ elif kind in IO_KINDS:
+ s["io_tiny_calls"] += w["aux1"]
+ elif kind in PATH_KINDS:
+ s["path_repeats"] += w["repeats"]
+ elif kind == "mutex":
+ s["mutex_contended"] += w["aux1"]
+ elif kind == "parallel_for":
+ s["pool_imbalance_items"] += w["aux1"]
+ elif kind == "pool_ops":
+ s["pool_ops_imbalance"] += w["aux1"]
+ elif kind == "ht_get":
+ s["ht_repeat_lookups"] += w["repeats"]
+ return dict(s)
+
+
+def fmt(v: float, unit: str) -> str:
+ if unit == "bytes":
+ return f"{v / MB:,.1f} MB"
+ return f"{v:,.0f} {unit}"
+
+
+SITE_SORTS = {
+ "bytes": lambda r, s: r["usable_bytes"],
+ "slack": lambda r, s: r["usable_bytes"] - r["requested_bytes"],
+ "never_written": lambda r, s: r.get("never_written_bytes", 0),
+ "over_requested": lambda r, s: r.get("over_requested_bytes", 0),
+ "churn": lambda r, s: r["short_lived"],
+ "copy": lambda r, s: r["realloc_copy_bytes"],
+ "live": lambda r, s: r["live_bytes"],
+ "allocs": lambda r, s: r["allocs"],
+ "dups": lambda r, s: s.get("dup_bytes", 0),
+ "retained": lambda r, s: s.get("retained_bytes", 0),
+ "dead": lambda r, s: r.get("acc_dead_bytes", 0),
+ "untouched": lambda r, s: r.get("acc_untouched_bytes", 0),
+ "idle": lambda r, s: r.get("acc_idle_bytes", 0),
+ "uninit": lambda r, s: r.get("acc_uninit_reads", 0),
+}
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("dump")
+ ap.add_argument("--binary", help="the binary that produced the dump, for symbolisation")
+ ap.add_argument("--top", type=int, default=20)
+ ap.add_argument("--sort", choices=sorted(SITE_SORTS), default="bytes")
+ ap.add_argument("--json", action="store_true", help="emit the merged tables as JSON")
+ args = ap.parse_args()
+
+ procs = load(args.dump)
+ if not procs:
+ print("no waste-sanitizer dump found in", args.dump, file=sys.stderr)
+ return 2
+
+ names_by_pid = {}
+ for pid, p in procs.items():
+ addrs = list(p["sites"].keys()) + [a for (_, a) in p["work"].keys()]
+ names_by_pid[pid] = symbolise(args.binary, p["header"].get("image_base", "0x0"), addrs)
+
+ if args.json:
+ out = {"processes": {}}
+ for pid, p in procs.items():
+ names = names_by_pid[pid]
+ out["processes"][str(pid)] = {
+ "header": p["header"],
+ "phases": [h.get("phase_label") for h in p["headers"]],
+ "summary": summarise(p, names),
+ "sites": [dict(r, name=names.get(a, a),
+ **{(k if k.endswith("_phase") else k + "_max"): v
+ for k, v in p["snap_max"].get(a, {}).items()})
+ for a, r in p["sites"].items()],
+ "work": [dict(w, name=names.get(a, a)) for (_, a), w in p["work"].items()],
+ }
+ json.dump(out, sys.stdout, indent=1)
+ print()
+ return 0
+
+ # Work-heavy processes first: the index worker is the one that matters.
+ order = sorted(procs.items(), key=lambda kv: -kv[1]["header"].get("usable_bytes", 0))
+ for pid, p in order:
+ h = p["header"]
+ names = names_by_pid[pid]
+ s = summarise(p, names)
+ print("=" * 110)
+ print(f"process {pid} last dump={h.get('why')} phases={len(p['headers'])} allocs={h['allocs']:,} "
+ f"cum={h['usable_bytes'] / MB:,.0f} MB live={h['live_bytes'] / MB:,.0f} MB "
+ f"rss={h.get('rss_bytes', 0) / MB:,.0f} MB sites={h['sites']:,} fill={h.get('fill')} access={h.get('access')} "
+ f"layer={h.get('layer_vm_bytes', 0) / MB:,.0f} MB/{h.get('layer_threads', 0)} threads")
+ blind = {k: h.get(k, 0) for k in ("site_table_full", "pointer_table_full", "work_table_full", "untracked_frees")}
+ if any(blind.values()):
+ print(" blind spots: " + " ".join(f"{k}={v:,}" for k, v in blind.items()))
+ if h.get("foreign_frees", 0):
+ print(f" outside the shim: {h['foreign_frees']:,} frees of blocks another allocator made "
+ "(C runtime / system DLL; their allocations are invisible to the layer)")
+ print()
+ print(" WASTE SUMMARY")
+ for key, label, unit in WASTE_CLASSES:
+ v = s.get(key, 0)
+ if v:
+ print(f" {fmt(v, unit):>18} {label}")
+ if len(p["sites"]) == 0 and len(p["work"]) == 0:
+ continue
+
+ print()
+ print(f" MEMORY — top {args.top} sites by {args.sort}")
+ print(f" {'cum MB':>9} {'allocs':>12} {'slack%':>6} {'churn%':>6} {'copy MB':>8} {'unwrit MB':>9} "
+ f"{'overreq MB':>10} {'dup MB':>7} {'retain MB':>9} {'raw%':>5} {'live MB':>8} site")
+ key = SITE_SORTS[args.sort]
+ rows = sorted(p["sites"].items(), key=lambda kv: (-key(kv[1], p["snap_max"].get(kv[0], {})), kv[0]))
+ for addr, r in rows[:args.top]:
+ snap = p["snap_max"].get(addr, {})
+ print(f" {r['usable_bytes'] / MB:>9,.1f} {r['allocs']:>12,} "
+ f"{pct(r['usable_bytes'] - r['requested_bytes'], r['usable_bytes']):>6.1f} "
+ f"{pct(r['short_lived'], r['allocs']):>6.1f} {r['realloc_copy_bytes'] / MB:>8,.1f} "
+ f"{r.get('never_written_bytes', 0) / MB:>9,.1f} {r.get('over_requested_bytes', 0) / MB:>10,.1f} "
+ f"{snap.get('dup_bytes', 0) / MB:>7,.1f} {snap.get('retained_bytes', 0) / MB:>9,.1f} "
+ f"{pct(r['raw_bytes'], r['usable_bytes']):>5.0f} {r['live_bytes'] / MB:>8,.1f} "
+ f"{names.get(addr, addr)[:90]}")
+
+ if h.get("access"):
+ # The access lane's verdicts per allocation site. Library-owned blocks (a FILE
+ # buffer) are not tracked; bytes a library wrote without a store callback are
+ # recognised through the fill pattern.
+ acc_cols = [("untouched", "acc_untouched_bytes", "never read or written"),
+ ("dead", "acc_dead_bytes", "written, never read"),
+ ("idle", "acc_idle_bytes", "idle for a phase before free")]
+ for sort_key, field, label in acc_cols:
+ rows = [(a, r) for a, r in p["sites"].items() if r.get(field, 0) > 0]
+ if not rows:
+ continue
+ print()
+ print(f" ACCESS — top {args.top} sites by bytes {label}")
+ print(f" {'MB':>9} {'blocks':>11} {'allocs':>12} {'uninit rd':>10} site")
+ blocks_field = field.replace("_bytes", "_blocks")
+ for a, r in sorted(rows, key=lambda t: (-t[1][field], t[0]))[:args.top]:
+ print(f" {r[field] / MB:>9,.1f} {r.get(blocks_field, 0):>11,} {r['allocs']:>12,} "
+ f"{r.get('acc_uninit_reads', 0):>10,} {names.get(a, a)[:90]}")
+ rows = [(a, r) for a, r in p["sites"].items() if r.get("acc_uninit_reads", 0) > 0]
+ if rows:
+ print()
+ print(f" ACCESS — top {args.top} sites by blocks read before any write (not calloc)")
+ print(f" {'blocks':>11} {'allocs':>12} site")
+ for a, r in sorted(rows, key=lambda t: (-t[1]["acc_uninit_reads"], t[0]))[:args.top]:
+ print(f" {r['acc_uninit_reads']:>11,} {r['allocs']:>12,} {names.get(a, a)[:90]}")
+
+ ct = [(k, a, w) for (k, a), w in p["work"].items() if k in CONTAINER_KINDS]
+ if ct:
+ print()
+ # A container reports at every reset: calls are USES, cap/used sum over uses.
+ # "peak idle" is the largest capacity a single instance left unused.
+ print(f" CONTAINERS — top {args.top} by largest unused capacity of one instance")
+ print(f" {'kind':<10} {'uses':>9} {'avg cap MB':>10} {'avg used MB':>11} {'unused%':>7} "
+ f"{'peak idle MB':>12} {'growths':>9} created at")
+ for k, a, w in sorted(ct, key=lambda t: (-t[2].get("peak", 0), t[1]))[:args.top]:
+ uses = max(w["calls"], 1)
+ print(f" {k[3:]:<10} {w['calls']:>9,} {w['bytes'] / uses / MB:>10,.2f} {w['aux1'] / uses / MB:>11,.2f} "
+ f"{pct(w['bytes'] - w['aux1'], w['bytes']):>7.1f} {w.get('peak', 0) / MB:>12,.1f} "
+ f"{w['aux2']:>9,} {names.get(a, a)[:90]}")
+
+ def section(title, kinds, sort_key, cols):
+ rows = [(k, a, w) for (k, a), w in p["work"].items() if k in kinds]
+ if not rows:
+ return
+ print()
+ print(f" CPU — {title}")
+ print(" " + " ".join(f"{c[0]:>{c[1]}}" for c in cols) + " site")
+ for k, a, w in sorted(rows, key=lambda t: (-sort_key(t[2]), t[1]))[:args.top]:
+ cells = []
+ for c in cols:
+ v = c[2](k, w)
+ cells.append(f"{v:>{c[1]}}" if isinstance(v, str) else f"{v:>{c[1]},.0f}")
+ print(" " + " ".join(cells) + f" {names.get(a, a)[:90]}")
+
+ section("byte work (mem*/str*) by bytes", BYTE_KINDS, lambda w: w["bytes"],
+ [("kind", 8, lambda k, w: k), ("calls", 12, lambda k, w: w["calls"]),
+ ("MB", 10, lambda k, w: w["bytes"] / MB), ("repeats", 10, lambda k, w: w["repeats"]),
+ ("max call", 10, lambda k, w: w.get("peak", 0))])
+ section("I/O by calls", IO_KINDS, lambda w: w["calls"],
+ [("kind", 8, lambda k, w: k), ("calls", 12, lambda k, w: w["calls"]),
+ ("MB", 10, lambda k, w: w["bytes"] / MB), ("tiny", 10, lambda k, w: w["aux1"])])
+ section("paths opened/statted again", PATH_KINDS, lambda w: w["repeats"],
+ [("kind", 8, lambda k, w: k), ("calls", 12, lambda k, w: w["calls"]),
+ ("repeats", 10, lambda k, w: w["repeats"]), ("failed", 10, lambda k, w: w["aux1"])])
+ section("mutex contention", {"mutex"}, lambda w: w["aux1"],
+ [("calls", 12, lambda k, w: w["calls"]), ("contended", 10, lambda k, w: w["aux1"])])
+ section("parallel_for imbalance", {"parallel_for"}, lambda w: w["aux1"],
+ [("calls", 8, lambda k, w: w["calls"]), ("items", 10, lambda k, w: w["bytes"]),
+ ("imbalance", 10, lambda k, w: w["aux1"])])
+ # imbalance / (work + imbalance): the share of the pool's capacity that sat idle
+ # while the busiest worker finished (0% = perfectly even).
+ section("pool balance by work done (per-worker events)", {"pool_ops"}, lambda w: w["aux1"],
+ [("calls", 8, lambda k, w: w["calls"]), ("events", 14, lambda k, w: w["bytes"]),
+ ("idle%", 6, lambda k, w: f"{pct(w['aux1'], w['bytes'] + w['aux1']):.1f}"),
+ ("workers", 8, lambda k, w: w["aux2"] / max(w["calls"], 1))])
+ section("hash-table lookups", {"ht_get"}, lambda w: w["repeats"],
+ [("gets", 12, lambda k, w: w["calls"]), ("misses", 12, lambda k, w: w["aux1"]),
+ ("repeats", 12, lambda k, w: w["repeats"])])
+ section("dead writes (access lane)", {"dead_write"}, lambda w: w["bytes"],
+ [("blocks", 10, lambda k, w: w["aux1"]), ("MB", 10, lambda k, w: w["bytes"] / MB)])
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/memwaste-scaling.py b/scripts/memwaste-scaling.py
new file mode 100644
index 000000000..ea8f1b060
--- /dev/null
+++ b/scripts/memwaste-scaling.py
@@ -0,0 +1,262 @@
+#!/usr/bin/env python3
+"""
+memwaste-scaling.py — the CPU scaling lane of the waste sanitizer.
+
+Work that grows faster than the input is the costliest CPU waste there is, and a
+single run cannot see it: a function that burns 5 % of the time on the Go corpus
+may burn 60 % on a corpus four times larger. This lane indexes the SAME real
+source tree replicated K times and 2K times, with clang's execution counters in
+every object (vendored ones included), and compares, function by function, how
+much work each leg did. Linear work doubles; a function whose counters grow by
+more than --threshold (default 2.3, i.e. 15 % above linear) while doing at least
+--min-excess extra counter increments is reported as super-linear.
+
+The counters are COUNTS, not time: the verdict is a function of the code and the
+input, so it can gate (O9). Wall time is printed as information only.
+
+Replicas are hard links (copies when the file system refuses) under
+<out>/leg-<n>/r<i>/, so every replica is a separate directory -- a separate Go
+package, a separate module -- and the registry, namespace and resolution paths
+see N times the same names: exactly the coupling behind the #1669 class of bugs.
+
+Usage:
+ scripts/memwaste-scaling.py --corpus ~/perf-bench/go/src/net --replicas 2
+ scripts/memwaste-scaling.py --corpus DIR --replicas 3 --no-build --json
+
+Build: MEMWASTE_PROFILE=1 into build/memwaste-profile with clang (the driver does
+it unless --no-build). llvm-profdata is taken from the directory of the clang
+that built the binary, so the profile format always matches.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+BUILD_DIR = "build/memwaste-profile"
+
+
+def find_profdata(cc: str) -> str:
+ cc_path = shutil.which(cc)
+ if cc_path:
+ cand = os.path.join(os.path.dirname(os.path.realpath(cc_path)), "llvm-profdata")
+ if os.access(cand, os.X_OK):
+ return cand
+ for name in ("llvm-profdata",) + tuple(f"llvm-profdata-{v}" for v in range(30, 13, -1)):
+ p = shutil.which(name)
+ if p:
+ return p
+ if shutil.which("xcrun"):
+ res = subprocess.run(["xcrun", "--find", "llvm-profdata"], capture_output=True, text=True, check=False)
+ if res.returncode == 0 and res.stdout.strip():
+ return res.stdout.strip()
+ sys.exit("memwaste-scaling: llvm-profdata not found next to the compiler or on PATH")
+
+
+def replicate(src: str, dst: str, copies: int) -> int:
+ """Hard-link `copies` replicas of src under dst/r<i>; returns the file count."""
+ files = 0
+ for i in range(copies):
+ base = os.path.join(dst, f"r{i}")
+ for dirpath, dirnames, filenames in os.walk(src):
+ dirnames[:] = [d for d in dirnames if d not in (".git", ".hg", ".svn")]
+ rel = os.path.relpath(dirpath, src)
+ out_dir = os.path.join(base, rel)
+ os.makedirs(out_dir, exist_ok=True)
+ for name in filenames:
+ s = os.path.join(dirpath, name)
+ if os.path.islink(s) or not os.path.isfile(s):
+ continue
+ d = os.path.join(out_dir, name)
+ try:
+ os.link(s, d)
+ except OSError:
+ shutil.copy2(s, d)
+ files += 1
+ return files
+
+
+def run_leg(binary: str, profdata: str, corpus: str, work: str, copies: int) -> tuple[dict, dict]:
+ leg = os.path.join(work, f"leg-{copies}")
+ tree = os.path.join(leg, "corpus")
+ os.makedirs(tree, exist_ok=True)
+ nfiles = replicate(corpus, tree, copies)
+ raw_dir = os.path.join(leg, "profraw")
+ os.makedirs(raw_dir, exist_ok=True)
+ runtime = tempfile.mkdtemp(prefix="cbm-sc.", dir="/tmp")
+ os.chmod(runtime, 0o700)
+ env = dict(os.environ)
+ env.update({
+ "LLVM_PROFILE_FILE": os.path.join(raw_dir, "cbm-%p.profraw"),
+ "CBM_PROFILE_DIR": raw_dir, # the index worker names its own: worker-<pid>.profraw
+ "CBM_CACHE_DIR": os.path.join(leg, "cache"),
+ "CBM_RUNTIME_DIR": runtime,
+ "CBM_LOG_LEVEL": "error",
+ })
+ env.pop("CBM_MEMWASTE", None) # production code paths: the event layer stays dormant
+ t0 = time.monotonic()
+ try:
+ res = subprocess.run([binary, "cli", "index_repository", "--repo-path", tree, "--mode", "full"],
+ env=env, capture_output=True, text=True, check=False)
+ finally:
+ shutil.rmtree(runtime, ignore_errors=True)
+ wall = time.monotonic() - t0
+ if res.returncode != 0:
+ sys.exit(f"memwaste-scaling: index of leg {copies} failed (rc={res.returncode}):\n{res.stderr[-2000:]}")
+ raws = sorted(os.path.join(raw_dir, f) for f in os.listdir(raw_dir) if f.endswith(".profraw"))
+ workers = [r for r in raws if os.path.basename(r).startswith("worker-")]
+ if not workers:
+ sys.exit("memwaste-scaling: the index worker wrote no profile -- is the binary built with "
+ "MEMWASTE_PROFILE=1, and does it call cbm_memev_process_exit before _Exit?")
+ # The comparison is the WORKER's work: the daemon and CLI processes differ in
+ # number between legs (and hash the binary at start-up), which is not scaling.
+ info = {"copies": copies, "files": nfiles, "wall_s": round(wall, 1), "profiles": len(raws)}
+ return merge_counts(profdata, workers, os.path.join(leg, "worker.profdata")), \
+ dict(info, other=merge_counts(profdata, [r for r in raws if r not in workers],
+ os.path.join(leg, "other.profdata")))
+
+
+def merge_counts(profdata: str, raws: list[str], merged: str) -> dict[str, int]:
+ if not raws:
+ return {}
+ subprocess.run([profdata, "merge", "-o", merged] + raws, check=True)
+ shown = subprocess.run([profdata, "show", "--all-functions", "--counts", merged],
+ capture_output=True, text=True, check=True).stdout
+ return parse_counts(shown)
+
+
+FUNC_RE = re.compile(r"^ (\S.*):$")
+
+
+def parse_counts(text: str) -> dict[str, int]:
+ """Total counter increments per function from `llvm-profdata show --counts`."""
+ out: dict[str, int] = {}
+ name = None
+ for line in text.splitlines():
+ m = FUNC_RE.match(line)
+ if m:
+ name = m.group(1)
+ continue
+ if name is None:
+ continue
+ s = line.strip()
+ if s.startswith("Function count:"):
+ out[name] = out.get(name, 0) + int(s.split(":", 1)[1])
+ elif s.startswith("Block counts:"):
+ vals = s.split(":", 1)[1].strip().strip("[]")
+ if vals:
+ out[name] = out.get(name, 0) + sum(int(v) for v in vals.split(",") if v.strip())
+ return out
+
+
+def display_name(raw: str) -> str:
+ # static functions are "file.c;name" (or "file.c:name" in older formats)
+ for sep in (";", ":"):
+ if sep in raw:
+ file_part, fn = raw.rsplit(sep, 1)
+ return f"{fn} ({os.path.basename(file_part)})"
+ return raw
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--corpus", required=True, help="a real source tree to replicate")
+ ap.add_argument("--replicas", type=int, default=2, help="K: legs index K and 2K replicas")
+ ap.add_argument("--threshold", type=float, default=2.3, help="2K/K counter ratio that is super-linear")
+ ap.add_argument("--min-excess", type=int, default=10_000_000,
+ help="ignore functions whose work beyond linear is below this many counter increments")
+ ap.add_argument("--top", type=int, default=30)
+ ap.add_argument("--out", help="work directory (default: a new temp dir)")
+ ap.add_argument("--no-build", action="store_true")
+ ap.add_argument("--cc", default="clang")
+ ap.add_argument("--json", action="store_true")
+ args = ap.parse_args()
+
+ corpus = os.path.abspath(os.path.expanduser(args.corpus))
+ if not os.path.isdir(corpus):
+ sys.exit(f"memwaste-scaling: {corpus} is not a directory")
+ binary = os.path.join(ROOT, BUILD_DIR, "codebase-memory-mcp")
+ if not args.no_build:
+ cxx = "clang++" if args.cc == "clang" else args.cc.replace("clang", "clang++")
+ subprocess.run([os.path.join(ROOT, "scripts", "build.sh"), f"BUILD_DIR={BUILD_DIR}",
+ "MEMWASTE_PROFILE=1", f"CC={args.cc}", f"CXX={cxx}"],
+ check=True, stdout=subprocess.DEVNULL)
+ if not os.access(binary, os.X_OK):
+ sys.exit(f"memwaste-scaling: {binary} is missing (build it, or drop --no-build)")
+ profdata = find_profdata(args.cc)
+ work = os.path.abspath(args.out) if args.out else tempfile.mkdtemp(prefix="cbm-scaling.")
+ os.makedirs(work, exist_ok=True)
+ for sub in os.listdir(work):
+ if sub.startswith("leg-"):
+ shutil.rmtree(os.path.join(work, sub), ignore_errors=True)
+
+ k = args.replicas
+ small, info_k = run_leg(binary, profdata, corpus, work, k)
+ large, info_2k = run_leg(binary, profdata, corpus, work, 2 * k)
+
+ total_k = sum(small.values())
+ total_2k = sum(large.values())
+ rows = []
+ for name, c2 in large.items():
+ c1 = small.get(name, 0)
+ excess = c2 - 2 * c1
+ ratio = (c2 / c1) if c1 else float("inf")
+ rows.append({"function": name, "k": c1, "2k": c2, "ratio": ratio, "excess": excess})
+ flagged = [r for r in rows if r["ratio"] > args.threshold and r["excess"] >= args.min_excess]
+ flagged.sort(key=lambda r: -r["excess"])
+ heaviest = sorted(rows, key=lambda r: -r["2k"])[: args.top]
+
+ for info in (info_k, info_2k):
+ other = info.pop("other", {}) or {}
+ info["other_total"] = sum(other.values())
+ info["other_top"] = sorted(other.items(), key=lambda kv: -kv[1])[:10]
+ report = {
+ "corpus": corpus, "legs": [info_k, info_2k], "threshold": args.threshold,
+ "min_excess": args.min_excess, "total_counts": {"k": total_k, "2k": total_2k},
+ "total_ratio": (total_2k / total_k) if total_k else None,
+ "superlinear": flagged,
+ "work_dir": work,
+ }
+ if args.json:
+ json.dump(report, sys.stdout, indent=1, default=lambda o: None if o == float("inf") else o)
+ print()
+ return 1 if flagged else 0
+
+ print(f"corpus {corpus}")
+ for info in (info_k, info_2k):
+ print(f" leg {info['copies']:>2} replicas: {info['files']:,} files, {info['profiles']} process profile(s), "
+ f"{info['wall_s']} s wall (information only)")
+ print(f" worker counter increments: k={total_k:,} 2k={total_2k:,} ratio={report['total_ratio']:.3f}")
+ for info in (info_k, info_2k):
+ other = info.get("other") or {}
+ top = sorted(other.items(), key=lambda kv: -kv[1])[:3]
+ print(f" other processes at {info['copies']} replicas: {sum(other.values()):,} increments; heaviest: "
+ + ", ".join(f"{display_name(n)} {c:,}" for n, c in top))
+ print()
+ print(f" SUPER-LINEAR — ratio > {args.threshold} and >= {args.min_excess:,} increments beyond linear")
+ if not flagged:
+ print(" none")
+ else:
+ print(f" {'ratio':>7} {'excess':>16} {'k':>16} {'2k':>16} function")
+ for r in flagged[: args.top]:
+ ratio = "new" if r["ratio"] == float("inf") else f"{r['ratio']:.2f}"
+ print(f" {ratio:>7} {r['excess']:>16,} {r['k']:>16,} {r['2k']:>16,} {display_name(r['function'])[:100]}")
+ print()
+ print(f" HEAVIEST at 2k — top {args.top} by counter increments (share of all work, scaling ratio)")
+ for r in heaviest:
+ ratio = "new" if r["ratio"] == float("inf") else f"{r['ratio']:.2f}"
+ print(f" {100.0 * r['2k'] / total_2k:>6.2f}% x{ratio:>5} {display_name(r['function'])[:100]}")
+ print()
+ print(f"work dir: {work}")
+ return 1 if flagged else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/memwaste.sh b/scripts/memwaste.sh
new file mode 100644
index 000000000..8d1935d7e
--- /dev/null
+++ b/scripts/memwaste.sh
@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# memwaste.sh — run the waste sanitizer on a repository and print the ranked report.
+#
+# The sanitizer (src/foundation/mem_events.h) measures memory that is CORRECT but
+# unnecessary: allocator slack, churn, realloc copying, memory that never reaches
+# the memory core. This driver is the one entry point for every venue: a local
+# run on a bench corpus, the PR-CI fixture run, the dry-run and release runs.
+#
+# Usage:
+# scripts/memwaste.sh <repo-path> [--lane event|access] [--top N] [--sort METRIC]
+# [--out DIR] [--no-build] [--json] [--binary PATH]
+#
+# --binary runs a kept copy of a flavour binary (a baseline, a replay) instead of
+# the one in the build directory; it implies --no-build.
+#
+# What it does
+# 1. builds the `memwaste` flavour (production flags + the event layer) into
+# build/memwaste, unless --no-build;
+# 2. indexes <repo-path> with CBM_MEMWASTE=1 into a private cache and a private,
+# SHORT runtime directory (the coordination socket has a 104-byte path limit);
+# 3. prints the symbolised report and writes the raw dump + JSON next to it;
+# 4. FAILS when the measurement itself is unsound: a table overflowed, or more
+# than 0.1 % of frees were of blocks the layer never saw. A sanitizer that
+# silently loses records produces false confidence.
+#
+# Nothing here reads a clock to decide anything: wall time is printed as
+# information only. Gates are counts.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+REPO=""
+TOP=25
+SORT=bytes
+OUT=""
+BUILD=1
+JSON=0
+LANE=event
+BINARY_OVERRIDE=""
+
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --binary) BINARY_OVERRIDE="$2"; BUILD=0; shift 2 ;;
+ --top) TOP="$2"; shift 2 ;;
+ --sort) SORT="$2"; shift 2 ;;
+ --out) OUT="$2"; shift 2 ;;
+ --no-build) BUILD=0; shift ;;
+ --json) JSON=1; shift ;;
+ --lane) LANE="$2"; shift 2 ;;
+ -h | --help) sed -n '2,24p' "$0"; exit 0 ;;
+ # A typo'd flag must never be read as the repository path — it would index
+ # nothing and report an empty lane as a clean one.
+ -*) echo "memwaste.sh: unknown argument $1. Please consult --help." >&2; exit 2 ;;
+ *) REPO="$1"; shift ;;
+ esac
+done
+
+if [ -z "$REPO" ] || [ ! -d "$REPO" ]; then
+ echo "memwaste.sh: a repository directory is required. Please consult --help." >&2
+ exit 2
+fi
+
+case "$LANE" in
+event) BUILD_ARGS=(BUILD_DIR=build/memwaste MEMWASTE=1 ${CC:+CC=$CC} ${CXX:+CXX=$CXX})
+ BIN="$ROOT/build/memwaste/codebase-memory-mcp" ;;
+access) BUILD_ARGS=(BUILD_DIR=build/memwaste-access MEMWASTE_ACCESS=1 CC=clang CXX=clang++)
+ BIN="$ROOT/build/memwaste-access/codebase-memory-mcp" ;;
+*) echo "memwaste.sh: --lane must be event or access" >&2; exit 2 ;;
+esac
+if [ "$BUILD" -eq 1 ]; then
+ "$ROOT/scripts/build.sh" "${BUILD_ARGS[@]}" >/dev/null
+fi
+if [ -n "$BINARY_OVERRIDE" ]; then
+ BIN="$BINARY_OVERRIDE"
+fi
+if [ ! -x "$BIN" ]; then
+ echo "memwaste.sh: $BIN is missing (build failed, or --no-build without a build)" >&2
+ exit 2
+fi
+
+if [ -z "$OUT" ]; then
+ OUT="$(mktemp -d "${TMPDIR:-/tmp}/cbm-memwaste.XXXXXX")"
+fi
+mkdir -p "$OUT"
+DUMP="$OUT/memwaste.jsonl"
+rm -f "$DUMP"
+
+RUN_BIN="$BIN"
+if command -v cygpath >/dev/null 2>&1 && ! command -v winepath >/dev/null 2>&1; then
+ # Native Windows: the server walks the ancestor chain of its binary, cache and
+ # runtime directories and refuses any that grant mutation to untrusted SIDs.
+ # The checkout and msys /tmp both do, so the run happens from a root under
+ # USERPROFILE stamped to the current user, SYSTEM and Administrators -- by
+ # SID, the way scripts/memlab.sh does it (#1532).
+ PROFILE_DIR="$(cygpath -u "$USERPROFILE")"
+ WORK="$(mktemp -d "$PROFILE_DIR/cbm-mw.XXXXXX")"
+ WORK_W="$(cygpath -w "$WORK")"
+ SID_QUERY='[System.Security.Principal.WindowsIdentity]::GetCurrent().User.Value'
+ ME="$(powershell.exe -NoProfile -NonInteractive -Command "$SID_QUERY" 2>/dev/null | tr -d '\r\n')"
+ case "$ME" in
+ S-1-*) ME="*$ME" ;;
+ *) echo "memwaste.sh: could not resolve the current user's SID" >&2; exit 2 ;;
+ esac
+ if ! MSYS2_ARG_CONV_EXCL='*' icacls "$WORK_W" /inheritance:r \
+ /grant:r "${ME}:(OI)(CI)F" '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' /Q >/dev/null 2>&1; then
+ echo "memwaste.sh: could not stamp $WORK" >&2
+ exit 2
+ fi
+ mkdir -p "$WORK/cache" "$WORK/rt"
+ cp "$BIN.exe" "$WORK/codebase-memory-mcp.exe" 2>/dev/null || cp "$BIN" "$WORK/codebase-memory-mcp.exe"
+ RUN_BIN="$WORK/codebase-memory-mcp.exe"
+ CACHE_DIR="$WORK_W\\cache"
+ RUNTIME="$WORK_W\\rt"
+ DUMP_ENV="$(cygpath -w "$DUMP")"
+else
+ WORK="$(mktemp -d /tmp/cbm-mw.XXXXXX)"
+ chmod 700 "$WORK"
+ # A fresh cache every run: an existing index turns --mode full into an
+ # incremental pass that measures hashing instead of indexing.
+ rm -rf "$OUT/cache"
+ mkdir -p "$OUT/cache"
+ CACHE_DIR="$OUT/cache"
+ RUNTIME="$WORK"
+ DUMP_ENV="$DUMP"
+fi
+
+# CBM_MEMWASTE_KEEP=1 keeps the private root (binary copy, cache, logs) for inspection.
+cleanup() {
+ if [ "${CBM_MEMWASTE_KEEP:-0}" != 1 ]; then
+ rm -rf "$WORK"
+ fi
+}
+trap cleanup EXIT
+
+INDEX_RC=0
+CBM_MEMWASTE=1 CBM_MEMWASTE_OUT="$DUMP_ENV" CBM_CACHE_DIR="$CACHE_DIR" CBM_RUNTIME_DIR="$RUNTIME" \
+ CBM_LOG_LEVEL=error "$RUN_BIN" cli index_repository --repo-path "$REPO" --mode full \
+ >"$OUT/index.json" || INDEX_RC=$?
+if [ "$INDEX_RC" -ne 0 ]; then
+ # A failed index is a finding about the flavour, never a silent partial report.
+ echo "memwaste.sh: the index failed (rc=$INDEX_RC); its logs follow" >&2
+ for log in "$WORK"/cache/logs/* "$WORK"/cache/logs/.worker-log-* "$OUT"/cache/logs/* "$OUT"/cache/logs/.worker-log-*; do
+ if [ -f "$log" ]; then
+ echo "--- $log" >&2
+ tail -n 60 "$log" >&2
+ fi
+ done
+ exit 1
+fi
+
+if [ ! -s "$DUMP" ]; then
+ echo "memwaste.sh: the sanitizer wrote no dump -- is the flavour built with MEMWASTE=1?" >&2
+ exit 1
+fi
+
+python3 "$ROOT/scripts/memwaste-report.py" "$DUMP" --binary "$BIN" --json >"$OUT/memwaste.json"
+if [ "$JSON" -eq 1 ]; then
+ cat "$OUT/memwaste.json"
+else
+ python3 "$ROOT/scripts/memwaste-report.py" "$DUMP" --binary "$BIN" --top "$TOP" --sort "$SORT"
+ echo
+ echo "raw dump: $DUMP"
+ echo "json: $OUT/memwaste.json"
+fi
+
+# Soundness of the measurement itself.
+python3 - "$OUT/memwaste.json" <<'PY'
+import json, sys
+procs = json.load(open(sys.argv[1]))["processes"]
+bad = []
+for pid, proc in procs.items():
+ h = proc["header"]
+ if h.get("site_table_full", 0) or h.get("pointer_table_full", 0):
+ bad.append(f"process {pid}: a table overflowed (sites={h.get('site_table_full')}, pointers={h.get('pointer_table_full')})")
+ frees = h.get("frees", 0) + h.get("untracked_frees", 0)
+ if frees and h.get("untracked_frees", 0) * 1000 > frees:
+ bad.append(f"process {pid}: {h['untracked_frees']:,} of {frees:,} frees were of blocks the layer never saw (> 0.1 %)")
+if bad:
+ print("memwaste.sh: UNSOUND MEASUREMENT", file=sys.stderr)
+ for b in bad:
+ print(" " + b, file=sys.stderr)
+ sys.exit(1)
+PY
diff --git a/scripts/sanitizer-coverage-map.py b/scripts/sanitizer-coverage-map.py
new file mode 100644
index 000000000..d961ec976
--- /dev/null
+++ b/scripts/sanitizer-coverage-map.py
@@ -0,0 +1,306 @@
+#!/usr/bin/env python3
+"""
+sanitizer-coverage-map.py — what our sanitizer lanes instrument, what they
+actually execute, and every hole, vendored code included.
+
+A sanitizer only checks code that runs under it. A function no suite reaches is
+unchecked in every lane, and until now nobody could name those functions. This
+tool answers three questions and keeps the answers from regressing:
+
+ static Which translation units does each lane COMPILE with its sanitizer,
+ per the build's own compile commands (make -n)? Lists every TU no
+ dynamic lane instruments and every TU the static lanes filter out.
+ run Build one lane with clang source coverage on top of its sanitizer
+ flags, run its test runner (suites optional), and export per-function
+ execution counts (lcov) -- vendored objects included, because the
+ flags ride on the lane's own sanitizer variable, which the vendored
+ rules already use.
+ union Merge the per-lane exports: the functions that ran under NO lane,
+ grouped by file, vendored code flagged.
+ ratchet Compare a union against the checked-in baseline: a function that
+ joins the never-executed list fails; one that leaves it is reported
+ so the baseline can be lowered.
+ map Validate and print scripts/sanitizer-map.json: every defect class x
+ platform cell must be covered, partial, or a named limit with a reason.
+
+`run` executes test suites, so it belongs to the lane's own venue (local CI, the
+CI job); `static`, `union`, `ratchet` and `map` run anywhere in seconds.
+
+Usage:
+ scripts/sanitizer-coverage-map.py static
+ scripts/sanitizer-coverage-map.py run --lane asan --out build/covmap [SUITE...]
+ scripts/sanitizer-coverage-map.py union --out build/covmap
+ scripts/sanitizer-coverage-map.py ratchet --out build/covmap
+ scripts/sanitizer-coverage-map.py map
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+from collections import defaultdict
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+BASELINE = os.path.join(ROOT, "scripts", "sanitizer-coverage-baseline.txt")
+MAP_FILE = os.path.join(ROOT, "scripts", "sanitizer-map.json")
+COVERAGE_FLAGS = "-fprofile-instr-generate -fcoverage-mapping"
+SRC_EXT = (".c", ".cc", ".cpp")
+
+# lane -> how the lane builds and runs. `var` is the make variable carrying the
+# lane's sanitizer flags; the coverage flags are appended to it.
+LANES = {
+ "asan": {"target": "test-runner", "var": "SANITIZE",
+ "value": "-fsanitize=address,undefined -fno-omit-frame-pointer", "env": {}},
+ "tsan": {"target": "test-runner-tsan", "var": "TSAN_SANITIZE",
+ "value": "-fsanitize=thread -fno-omit-frame-pointer",
+ "env": {"TSAN_OPTIONS": "halt_on_error=1:report_thread_leaks=0"}},
+ "lsan": {"target": "test-runner", "var": "SANITIZE",
+ "value": "-fsanitize=address,undefined -fno-omit-frame-pointer",
+ "env": {"ASAN_OPTIONS": "detect_leaks=1:halt_on_error=1"}},
+}
+DYNAMIC_FLAGS = ("-fsanitize=address", "-fsanitize=thread", "-fsanitize=memory", "-fsanitize=undefined",
+ "-fsanitize=address,undefined")
+
+
+def is_vendored(path: str) -> bool:
+ # grammar_<lang>.c and the runtime/unity wrappers only #include vendored sources
+ base = os.path.basename(path)
+ return "vendored/" in path or base.startswith(("grammar_", "ts_runtime")) or base in (
+ "sqlite3.c", "mimalloc.c", "unixcoder_blob.c")
+
+
+def dry_commands(target: str, extra: list[str]) -> list[list[str]]:
+ res = subprocess.run(["make", "-f", "Makefile.cbm", "-n", "-B", target] + extra,
+ cwd=ROOT, capture_output=True, text=True, check=False)
+ cmds = []
+ for line in res.stdout.replace("\\\n", " ").splitlines():
+ try:
+ argv = shlex.split(line)
+ except ValueError:
+ continue
+ if argv and any(a.endswith(SRC_EXT) for a in argv) and ("-c" in argv or "-o" in argv):
+ cmds.append(argv)
+ return cmds
+
+
+def make_var(name: str) -> list[str]:
+ """The fully expanded value of a Makefile.cbm variable (works with make 3.81: no --eval)."""
+ with tempfile.NamedTemporaryFile("w", suffix=".mk", delete=False) as mk:
+ mk.write("print-cbm-var-%:\n\t@echo $($*)\n")
+ helper = mk.name
+ try:
+ res = subprocess.run(["make", "-s", "-f", "Makefile.cbm", "-f", helper, f"print-cbm-var-{name}"],
+ cwd=ROOT, capture_output=True, text=True, check=False)
+ finally:
+ os.unlink(helper)
+ return res.stdout.split() if res.returncode == 0 else []
+
+
+def cmd_static(args) -> int:
+ instrumented: dict[str, set[str]] = defaultdict(set)
+ compiled: set[str] = set()
+ for lane, spec in {"asan": LANES["asan"], "tsan": LANES["tsan"]}.items():
+ bdir = f"build/covmap-static-{lane}"
+ for argv in dry_commands(f"{bdir}/{spec['target']}", [f"BUILD_DIR={bdir}"]):
+ flags = " ".join(argv)
+ for a in argv:
+ if a.endswith(SRC_EXT) and os.path.isfile(os.path.join(ROOT, a)):
+ compiled.add(a)
+ if any(f in flags for f in ("-fsanitize=address", "-fsanitize=thread")):
+ instrumented[a].add(lane)
+ uninstrumented = sorted(s for s in compiled if not instrumented.get(s))
+ lint_srcs = set(make_var("LINT_SRCS"))
+ static_skipped = sorted(s for s in compiled if s not in lint_srcs)
+ out = {
+ "translation_units": len(compiled),
+ "not_instrumented_by_any_dynamic_lane": uninstrumented,
+ "not_in_static_lanes": static_skipped,
+ "per_tu": {s: sorted(instrumented.get(s, ())) for s in sorted(compiled)},
+ }
+ if args.json:
+ json.dump(out, sys.stdout, indent=1)
+ print()
+ return 0
+ print(f"translation units in the test builds: {len(compiled)}")
+ print(f" instrumented by ASan+UBSan: {sum(1 for v in instrumented.values() if 'asan' in v)}"
+ f" by TSan: {sum(1 for v in instrumented.values() if 'tsan' in v)}")
+ print()
+ print(f" NOT instrumented by any dynamic lane ({len(uninstrumented)}):")
+ for s in uninstrumented:
+ print(f" {s}{' [vendored]' if is_vendored(s) else ''}")
+ print()
+ print(f" outside the static lanes' source list (cppcheck, clang-analyzer) ({len(static_skipped)}):")
+ vend = [s for s in static_skipped if is_vendored(s)]
+ own = [s for s in static_skipped if not is_vendored(s) and not s.startswith("tests/")]
+ print(f" vendored: {len(vend)} project: {len(own)}")
+ for s in own:
+ print(f" {s}")
+ return 0
+
+
+def find_llvm_tool(name: str, cc: str) -> str:
+ cc_path = shutil.which(cc)
+ if cc_path:
+ cand = os.path.join(os.path.dirname(os.path.realpath(cc_path)), name)
+ if os.access(cand, os.X_OK):
+ return cand
+ p = shutil.which(name)
+ if p:
+ return p
+ if shutil.which("xcrun"):
+ res = subprocess.run(["xcrun", "--find", name], capture_output=True, text=True, check=False)
+ if res.returncode == 0:
+ return res.stdout.strip()
+ sys.exit(f"sanitizer-coverage-map: {name} not found next to {cc} or on PATH")
+
+
+def cmd_run(args) -> int:
+ spec = LANES[args.lane]
+ out = os.path.abspath(args.out)
+ os.makedirs(out, exist_ok=True)
+ bdir = f"build/covmap-{args.lane}"
+ cxx = "clang++" if args.cc == "clang" else args.cc.replace("clang", "clang++")
+ flags = f"{spec['value']} {COVERAGE_FLAGS}"
+ subprocess.run(["make", "-f", "Makefile.cbm", f"-j{os.cpu_count() or 4}", f"{bdir}/{spec['target']}",
+ f"BUILD_DIR={bdir}", f"CC={args.cc}", f"CXX={cxx}", f"{spec['var']}={flags}"],
+ cwd=ROOT, check=True)
+ raw_dir = os.path.join(out, f"{args.lane}-profraw")
+ shutil.rmtree(raw_dir, ignore_errors=True)
+ os.makedirs(raw_dir)
+ env = dict(os.environ)
+ env.update(spec["env"])
+ env["LLVM_PROFILE_FILE"] = os.path.join(raw_dir, "%p-%m.profraw")
+ runner = os.path.join(ROOT, bdir, spec["target"])
+ rc = subprocess.run([runner] + args.suites, cwd=ROOT, env=env, check=False).returncode
+ raws = [os.path.join(raw_dir, f) for f in os.listdir(raw_dir) if f.endswith(".profraw")]
+ if not raws:
+ sys.exit("sanitizer-coverage-map: the runner wrote no profile")
+ profdata = find_llvm_tool("llvm-profdata", args.cc)
+ cov = find_llvm_tool("llvm-cov", args.cc)
+ merged = os.path.join(out, f"{args.lane}.profdata")
+ subprocess.run([profdata, "merge", "-sparse", "-o", merged] + raws, check=True)
+ with open(os.path.join(out, f"{args.lane}.lcov"), "w") as fh:
+ subprocess.run([cov, "export", "-format=lcov", "-skip-expansions", f"-instr-profile={merged}", runner],
+ cwd=ROOT, stdout=fh, check=True)
+ print(f"{args.lane}: runner rc={rc}, {len(raws)} profile(s) -> {out}/{args.lane}.lcov")
+ return 0 if rc == 0 else rc
+
+
+def read_lcov(path: str) -> dict[tuple[str, str], int]:
+ counts: dict[tuple[str, str], int] = {}
+ src = None
+ for line in open(path, encoding="utf-8", errors="replace"):
+ if line.startswith("SF:"):
+ src = os.path.relpath(line[3:].strip(), ROOT)
+ elif line.startswith("FNDA:") and src:
+ count, name = line[5:].strip().split(",", 1)
+ key = (src, name)
+ counts[key] = max(counts.get(key, 0), int(count))
+ return counts
+
+
+def cmd_union(args) -> int:
+ out = os.path.abspath(args.out)
+ lanes = [f[:-5] for f in sorted(os.listdir(out)) if f.endswith(".lcov")]
+ if not lanes:
+ sys.exit(f"sanitizer-coverage-map: no <lane>.lcov under {out} (run `run` first)")
+ total: dict[tuple[str, str], int] = {}
+ for lane in lanes:
+ for key, c in read_lcov(os.path.join(out, f"{lane}.lcov")).items():
+ total[key] = max(total.get(key, 0), c)
+ never = sorted(k for k, c in total.items() if c == 0 and not k[0].startswith(("tests/", "..")))
+ with open(os.path.join(out, "never-executed.txt"), "w") as fh:
+ for src, fn in never:
+ fh.write(f"{src}\t{fn}\n")
+ by_file = defaultdict(int)
+ for src, _ in never:
+ by_file[src] += 1
+ functions = sum(1 for k in total if not k[0].startswith(("tests/", "..")))
+ print(f"lanes merged: {', '.join(lanes)}")
+ print(f"functions: {functions:,} never executed under any lane: {len(never):,} "
+ f"(vendored: {sum(1 for s, _ in never if is_vendored(s)):,})")
+ for src, n in sorted(by_file.items(), key=lambda kv: (-kv[1], kv[0]))[: args.top]:
+ print(f" {n:>6} {src}{' [vendored]' if is_vendored(src) else ''}")
+ print(f"list: {out}/never-executed.txt")
+ return 0
+
+
+def cmd_ratchet(args) -> int:
+ current_path = os.path.join(os.path.abspath(args.out), "never-executed.txt")
+ current = set(open(current_path).read().splitlines())
+ if not os.path.exists(BASELINE):
+ print(f"no baseline yet: copy {current_path} to {BASELINE} to arm the ratchet")
+ return 0
+ base = set(l for l in open(BASELINE).read().splitlines() if l and not l.startswith("#"))
+ new = sorted(current - base)
+ gone = sorted(base - current)
+ for line in new:
+ print(f"NEW never-executed: {line}")
+ if gone:
+ print(f"{len(gone)} function(s) now executed: lower the baseline (remove them from {BASELINE})")
+ return 1 if new else 0
+
+
+def cmd_map(args) -> int:
+ data = json.load(open(MAP_FILE))
+ platforms = data["platforms"]
+ bad = []
+ for cls in data["classes"]:
+ cells = cls["cells"]
+ for plat in platforms:
+ cell = cells.get(plat)
+ if not cell:
+ bad.append(f"{cls['name']} / {plat}: missing")
+ elif cell.get("status") not in ("covered", "partial", "limit"):
+ bad.append(f"{cls['name']} / {plat}: status must be covered, partial or limit")
+ elif cell["status"] != "covered" and not cell.get("why"):
+ bad.append(f"{cls['name']} / {plat}: {cell['status']} needs a why")
+ if not args.quiet:
+ width = max(len(c["name"]) for c in data["classes"])
+ print(f"{'defect class':<{width}} " + " ".join(f"{p:<14}" for p in platforms))
+ for cls in data["classes"]:
+ row = []
+ for plat in platforms:
+ cell = cls["cells"].get(plat, {})
+ row.append(f"{cell.get('status', '?') + ':' + cell.get('lane', ''):<14}"[:14])
+ print(f"{cls['name']:<{width}} " + " ".join(row))
+ print()
+ for cls in data["classes"]:
+ for plat in platforms:
+ cell = cls["cells"].get(plat, {})
+ if cell.get("status") in ("partial", "limit"):
+ print(f" {cell['status']:<7} {cls['name']} / {plat}: {cell.get('why', '')}")
+ for b in bad:
+ print(f"MAP DEFECT: {b}", file=sys.stderr)
+ return 1 if bad else 0
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ sub = ap.add_subparsers(dest="cmd", required=True)
+ s = sub.add_parser("static")
+ s.add_argument("--json", action="store_true")
+ r = sub.add_parser("run")
+ r.add_argument("--lane", choices=sorted(LANES), required=True)
+ r.add_argument("--out", default="build/covmap")
+ r.add_argument("--cc", default="clang")
+ r.add_argument("suites", nargs="*")
+ u = sub.add_parser("union")
+ u.add_argument("--out", default="build/covmap")
+ u.add_argument("--top", type=int, default=40)
+ t = sub.add_parser("ratchet")
+ t.add_argument("--out", default="build/covmap")
+ m = sub.add_parser("map")
+ m.add_argument("--quiet", action="store_true")
+ args = ap.parse_args()
+ return {"static": cmd_static, "run": cmd_run, "union": cmd_union,
+ "ratchet": cmd_ratchet, "map": cmd_map}[args.cmd](args)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/sanitizer-map.json b/scripts/sanitizer-map.json
new file mode 100644
index 000000000..d2ee15fb6
--- /dev/null
+++ b/scripts/sanitizer-map.json
@@ -0,0 +1,126 @@
+{
+ "about": "Defect class x platform coverage of the sanitizer lanes. Every cell is covered, partial (with why), or a named limit (with why). Validated by scripts/sanitizer-coverage-map.py map. Which functions each lane actually EXECUTES is the run/union/ratchet part of the same tool.",
+ "platforms": ["linux-x64", "linux-arm64", "macos-arm64", "windows-x64", "windows-arm64"],
+ "classes": [
+ {
+ "name": "heap/stack memory errors",
+ "cells": {
+ "linux-x64": {"status": "covered", "lane": "asan", "venue": "PR CI"},
+ "linux-arm64": {"status": "covered", "lane": "asan", "venue": "local container"},
+ "macos-arm64": {"status": "partial", "lane": "asan+diag", "why": "use-after-return and use-after-scope only on the Homebrew-LLVM diag lane, which is local-only"},
+ "windows-x64": {"status": "covered", "lane": "asan", "venue": "PR CI"},
+ "windows-arm64": {"status": "limit", "lane": "none", "why": "no ASan runtime for Windows ARM64; the class is covered for Windows by the x64 leg"}
+ }
+ },
+ {
+ "name": "allocator-path memory errors",
+ "cells": {
+ "linux-x64": {"status": "partial", "lane": "asan", "why": "test builds use the C runtime allocator: mimalloc bindings, the --wrap shims and the slab never run sanitized; mimalloc itself is built without ASan"},
+ "linux-arm64": {"status": "partial", "lane": "asan", "why": "same as linux-x64"},
+ "macos-arm64": {"status": "partial", "lane": "asan", "why": "same as linux-x64; the zone-observer path runs only in the memwaste flavours"},
+ "windows-x64": {"status": "partial", "lane": "asan", "why": "same as linux-x64"},
+ "windows-arm64": {"status": "limit", "lane": "none", "why": "no ASan runtime for Windows ARM64"}
+ }
+ },
+ {
+ "name": "undefined behaviour",
+ "cells": {
+ "linux-x64": {"status": "partial", "lane": "ubsan", "why": "tre (regex) is built with -fno-sanitize=alignment"},
+ "linux-arm64": {"status": "partial", "lane": "ubsan", "why": "tre (regex) is built with -fno-sanitize=alignment"},
+ "macos-arm64": {"status": "partial", "lane": "ubsan", "why": "tre (regex) is built with -fno-sanitize=alignment"},
+ "windows-x64": {"status": "partial", "lane": "ubsan", "why": "tre (regex) is built with -fno-sanitize=alignment"},
+ "windows-arm64": {"status": "partial", "lane": "ubsan-trap", "why": "trap mode: a violation stops the run without a message"}
+ }
+ },
+ {
+ "name": "leaks",
+ "cells": {
+ "linux-x64": {"status": "covered", "lane": "lsan", "venue": "PR CI (on by default under ASan)"},
+ "linux-arm64": {"status": "covered", "lane": "lsan", "venue": "local container"},
+ "macos-arm64": {"status": "covered", "lane": "test-lsan", "venue": "local (Homebrew LLVM; Apple clang refuses detect_leaks)"},
+ "windows-x64": {"status": "partial", "lane": "memwaste-event", "why": "no leak sanitizer on Windows; the waste lane's live-per-site report at the last phase and the soak slope stand in"},
+ "windows-arm64": {"status": "partial", "lane": "memwaste-event", "why": "no leak sanitizer on Windows; the waste lane's live-per-site report at the last phase and the soak slope stand in"}
+ }
+ },
+ {
+ "name": "uninitialised reads",
+ "cells": {
+ "linux-x64": {"status": "partial", "lane": "msan", "why": "CI only; deep-recursion suites need larger thread stacks under MSan"},
+ "linux-arm64": {"status": "partial", "lane": "msan", "why": "locally red: three deep-recursion suites overflow their thread stacks"},
+ "macos-arm64": {"status": "partial", "lane": "memwaste-access", "why": "no MSan on macOS; the access lane's read-before-write verdict on a real corpus is the second net, per block, not per byte"},
+ "windows-x64": {"status": "partial", "lane": "memwaste-access", "why": "no MSan on Windows; the access lane (clang) is the second net"},
+ "windows-arm64": {"status": "partial", "lane": "memwaste-access", "why": "no MSan on Windows; the access lane (clang) is the second net"}
+ }
+ },
+ {
+ "name": "data races",
+ "cells": {
+ "linux-x64": {"status": "partial", "lane": "tsan", "why": "a subset of suites runs under TSan"},
+ "linux-arm64": {"status": "partial", "lane": "tsan", "why": "a subset of suites runs under TSan"},
+ "macos-arm64": {"status": "partial", "lane": "tsan", "why": "a subset of suites runs under TSan"},
+ "windows-x64": {"status": "limit", "lane": "none", "why": "no TSan on Windows"},
+ "windows-arm64": {"status": "limit", "lane": "none", "why": "no TSan on Windows"}
+ }
+ },
+ {
+ "name": "static analysis",
+ "cells": {
+ "linux-x64": {"status": "partial", "lane": "cppcheck+lint-mem+codeql", "why": "vendored trees are filtered out of cppcheck and the analyzer gate"},
+ "linux-arm64": {"status": "partial", "lane": "cppcheck+lint-mem", "why": "vendored trees are filtered out"},
+ "macos-arm64": {"status": "partial", "lane": "lint-mem", "why": "vendored trees are filtered out"},
+ "windows-x64": {"status": "limit", "lane": "none", "why": "static lanes are platform-independent source passes; Windows-only code is analysed by CodeQL only"},
+ "windows-arm64": {"status": "limit", "lane": "none", "why": "static lanes are platform-independent source passes"}
+ }
+ },
+ {
+ "name": "hostile input",
+ "cells": {
+ "linux-x64": {"status": "partial", "lane": "security-fuzz+libfuzzer", "why": "fixed MCP payloads in CI; libFuzzer targets for extractors, Cypher and config editors run in dry-run/nightly"},
+ "linux-arm64": {"status": "partial", "lane": "libfuzzer", "why": "local only"},
+ "macos-arm64": {"status": "partial", "lane": "libfuzzer", "why": "local only (Homebrew LLVM ships libFuzzer; Apple clang does not)"},
+ "windows-x64": {"status": "limit", "lane": "none", "why": "libFuzzer is not built for the MinGW toolchain"},
+ "windows-arm64": {"status": "limit", "lane": "none", "why": "libFuzzer is not built for the MinGW toolchain"}
+ }
+ },
+ {
+ "name": "algorithmic blow-up",
+ "cells": {
+ "linux-x64": {"status": "covered", "lane": "complexity+memwaste-scaling"},
+ "linux-arm64": {"status": "covered", "lane": "complexity+memwaste-scaling"},
+ "macos-arm64": {"status": "covered", "lane": "complexity+memwaste-scaling"},
+ "windows-x64": {"status": "partial", "lane": "complexity", "why": "the per-function scaling lane needs clang's profile runtime; whole-pipeline counters only"},
+ "windows-arm64": {"status": "partial", "lane": "complexity", "why": "the per-function scaling lane needs clang's profile runtime; whole-pipeline counters only"}
+ }
+ },
+ {
+ "name": "memory waste",
+ "cells": {
+ "linux-x64": {"status": "covered", "lane": "memwaste-event+access"},
+ "linux-arm64": {"status": "covered", "lane": "memwaste-event+access"},
+ "macos-arm64": {"status": "covered", "lane": "memwaste-event+access"},
+ "windows-x64": {"status": "partial", "lane": "memwaste-event", "why": "libc wrappers cover mem*/str* and FILE I/O only; the access lane needs a clang build"},
+ "windows-arm64": {"status": "partial", "lane": "memwaste-event", "why": "libc wrappers cover mem*/str* and FILE I/O only"}
+ }
+ },
+ {
+ "name": "CPU waste",
+ "cells": {
+ "linux-x64": {"status": "covered", "lane": "memwaste-cpu+scaling"},
+ "linux-arm64": {"status": "covered", "lane": "memwaste-cpu+scaling"},
+ "macos-arm64": {"status": "covered", "lane": "memwaste-cpu+scaling"},
+ "windows-x64": {"status": "partial", "lane": "memwaste-cpu", "why": "no system-call or pthread lock wrappers on Windows: Win32 file and lock APIs are imports --wrap cannot redirect"},
+ "windows-arm64": {"status": "partial", "lane": "memwaste-cpu", "why": "no system-call or pthread lock wrappers on Windows: Win32 file and lock APIs are imports --wrap cannot redirect"}
+ }
+ },
+ {
+ "name": "executed-code coverage of the lanes",
+ "cells": {
+ "linux-x64": {"status": "partial", "lane": "sanitizer-coverage-map", "why": "needs a clang job; the gcc legs are not measured"},
+ "linux-arm64": {"status": "covered", "lane": "sanitizer-coverage-map"},
+ "macos-arm64": {"status": "covered", "lane": "sanitizer-coverage-map"},
+ "windows-x64": {"status": "limit", "lane": "none", "why": "no sanitizer lane on Windows executes under clang source coverage yet"},
+ "windows-arm64": {"status": "limit", "lane": "none", "why": "no sanitizer lane on Windows executes under clang source coverage yet"}
+ }
+ }
+ ]
+}
diff --git a/src/foundation/mem_core.c b/src/foundation/mem_core.c
index dee1fb759..425e617fa 100644
--- a/src/foundation/mem_core.c
+++ b/src/foundation/mem_core.c
@@ -3,6 +3,7 @@
*/
#include "foundation/mem_core.h"
#include "foundation/mem.h"
+#include "foundation/mem_events.h"
/* Ownership check for blocks handed back to the core (defined with cbm_free). */
static void check_owned(const void *block, const char *op);
@@ -152,6 +153,7 @@ static void class_flush_one(cbm_mem_class_t cls) {
}
void cbm_mem_class_flush_thread(void) {
+ cbm_memev_flush_thread(); /* same seam: work-item end, thread end, every reader */
for (int i = 0; i < CBM_MEM_CLASS_COUNT; i++) {
class_flush_one((cbm_mem_class_t)i);
}
@@ -205,8 +207,62 @@ size_t cbm_mem_usable_size(const void *block) {
}
#endif
+/* -- Waste-sanitizer hooks (mem_events.h) ---------------------------------
+ * Compiled to nothing outside the `memwaste` flavour. The core names the
+ * caller's site and the class; who REPORTS the event depends on whether an
+ * observer sees the backing allocator:
+ * - backing is mimalloc called directly (CBM_BIND_TS_ALLOCATOR): no observer
+ * ever sees it, the core reports everything itself;
+ * - backing is plain malloc and an observer is installed: the observer
+ * reports, the hint hands it the site and the class;
+ * - backing is plain malloc, no observer: the core reports.
+ * Frees are reported BEFORE the block goes back (see mem_events.h). */
+#if defined(CBM_MEMWASTE) && CBM_MEMWASTE
+static bool core_reports_frees(void) {
+#if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR
+ return true;
+#else
+ return !cbm_memev_observer_installed();
+#endif
+}
+#define MEMEV_HINT(cls) \
+ do { \
+ if (cbm_memev_enabled() && !cbm_memev_hint_pending()) { \
+ cbm_memev_hint(__builtin_return_address(0), (int)(cls)); \
+ } \
+ } while (0)
+#define MEMEV_ALLOCATED(block, bytes, flags) \
+ do { \
+ if (cbm_memev_hint_pending()) { \
+ cbm_memev_alloc_ex((block), (bytes), \
+ (block) ? charge_size((block), (bytes)) : 0, NULL, \
+ (flags)); \
+ } \
+ } while (0)
+#define MEMEV_REALLOCATED(old_block, grown, bytes) \
+ do { \
+ if (cbm_memev_hint_pending()) { \
+ cbm_memev_realloc((old_block), (grown), (bytes), \
+ (grown) ? charge_size((grown), (bytes)) : 0, NULL); \
+ } \
+ } while (0)
+#define MEMEV_FREEING(block) \
+ do { \
+ if (cbm_memev_enabled() && core_reports_frees()) { \
+ cbm_memev_free(block); \
+ } \
+ } while (0)
+#else
+#define MEMEV_HINT(cls) ((void)0)
+#define MEMEV_ALLOCATED(block, bytes, flags) ((void)0)
+#define MEMEV_REALLOCATED(old_block, grown, bytes) ((void)0)
+#define MEMEV_FREEING(block) ((void)0)
+#endif
+
void *cbm_alloc(cbm_mem_class_t cls, size_t bytes) {
+ MEMEV_HINT(cls);
void *block = CBM_BACKING_MALLOC(bytes ? bytes : CBM_ALLOC_ONE);
+ MEMEV_ALLOCATED(block, bytes, 0);
if (!block) {
return NULL;
}
@@ -215,7 +271,9 @@ void *cbm_alloc(cbm_mem_class_t cls, size_t bytes) {
}
void *cbm_calloc(cbm_mem_class_t cls, size_t bytes) {
+ MEMEV_HINT(cls);
void *block = CBM_BACKING_CALLOC(bytes ? bytes : CBM_ALLOC_ONE);
+ MEMEV_ALLOCATED(block, bytes, CBM_MEMEV_ZEROED);
if (!block) {
return NULL;
}
@@ -231,7 +289,11 @@ void *cbm_realloc(cbm_mem_class_t cls, void *block, size_t bytes) {
/* Measure BEFORE: after realloc the old block is gone and its size is
* unknowable, so the decrement has to be computed first. */
size_t old = charge_size(block, 0);
+ MEMEV_HINT(cls);
+ CBM_MEMEV_BACKING(1);
void *next = CBM_BACKING_REALLOC(block, bytes ? bytes : CBM_ALLOC_ONE);
+ CBM_MEMEV_BACKING(-1);
+ MEMEV_REALLOCATED(block, next, bytes);
if (!next) {
return NULL; /* original intact and still charged - correct */
}
@@ -245,6 +307,7 @@ char *cbm_mem_strdup(cbm_mem_class_t cls, const char *s) {
return NULL;
}
size_t len = strlen(s) + CBM_ALLOC_ONE;
+ MEMEV_HINT(cls); /* the site is OUR caller, not cbm_alloc's */
char *copy = (char *)cbm_alloc(cls, len);
if (!copy) {
return NULL;
@@ -279,6 +342,7 @@ void cbm_free(cbm_mem_class_t cls, void *block) {
}
check_owned(block, "free");
class_sub(cls, charge_size(block, 0), CBM_ALLOC_ONE);
+ MEMEV_FREEING(block);
CBM_BACKING_FREE(block);
}
diff --git a/src/foundation/mem_events.c b/src/foundation/mem_events.c
new file mode 100644
index 000000000..80b4ee19f
--- /dev/null
+++ b/src/foundation/mem_events.c
@@ -0,0 +1,2178 @@
+/*
+ * mem_events.c — see mem_events.h.
+ *
+ * Layout:
+ * sites[] global, open-addressed, keyed by return address. Keys are
+ * claimed with one CAS; counters are touched only under the
+ * merge lock, from per-thread deltas.
+ * shards[] the pointer table, 256 independently locked open-addressed
+ * maps. A block is routinely freed by a thread that did not
+ * allocate it, so the pointer table cannot be per-thread.
+ * work[] CPU work counters keyed by (return address, kind), merged
+ * the same way as the sites.
+ * tl_state per-thread deltas for every site and work row the thread
+ * touched, dirty lists so a flush walks only those, the core
+ * handshake, the re-entrancy guard and the repeat caches.
+ * access lane (CBM_MEMWASTE_ACCESS) a shadow map from every 16-byte slot of
+ * a tracked block to a per-block record the compiler's load and
+ * store callbacks update.
+ *
+ * Everything the layer owns comes from raw virtual memory: it runs INSIDE the
+ * allocator it observes and must never call it.
+ */
+#include "foundation/mem_events.h"
+
+#if (defined(CBM_MEMWASTE) && CBM_MEMWASTE) || \
+ (defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS)
+
+#include "foundation/mem.h"
+
+#include <stdatomic.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _WIN32
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#include <fcntl.h> /* _O_WRONLY, _O_CREAT, _O_APPEND, _O_BINARY */
+#include <io.h>
+#include <windows.h>
+#else
+#include <fcntl.h>
+#include <pthread.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#if defined(__linux__)
+#include <link.h> /* dl_iterate_phdr: the load bias of the main program */
+#endif
+#endif
+
+/* The layer's own code is never instrumented by the access lane: its tables
+ * are not program data, and a hook that reached instrumented code would pay
+ * for itself on every table probe. */
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((no_sanitize("coverage"))), apply_to = function)
+#endif
+
+enum {
+ MEMEV_SITE_BITS = 16,
+ MEMEV_SITE_CAP = 1 << MEMEV_SITE_BITS,
+ MEMEV_WORK_BITS = 17,
+ MEMEV_WORK_CAP = 1 << MEMEV_WORK_BITS,
+ MEMEV_SHARD_BITS = 8,
+ MEMEV_SHARDS = 1 << MEMEV_SHARD_BITS,
+ MEMEV_SHARD_INIT_BITS = 10,
+ MEMEV_SHARD_MAX_BITS = 26,
+ MEMEV_LOAD_NUM = 7, /* grow at 70 % occupancy (live + tombstones) */
+ MEMEV_LOAD_DEN = 10,
+ MEMEV_SEQ_BITS = 24,
+ MEMEV_SEQ_MASK = (1 << MEMEV_SEQ_BITS) - 1,
+ MEMEV_PHASE_CAP = 63, /* stored in 6 bits */
+ MEMEV_PHASE_LABEL = 48,
+ MEMEV_LINE = 2048, /* a site line with every field at 20 digits is ~1,150 bytes */
+ MEMEV_PTR_SHIFT = 4, /* allocators align to 16: the low bits carry no entropy */
+ MEMEV_REPEAT_SLOTS = 256,
+ MEMEV_PATH_BITS = 20,
+ MEMEV_PATH_CAP = 1 << MEMEV_PATH_BITS,
+ MEMEV_RUN_MIN = 8, /* a run of the fill byte this long was never written */
+ MEMEV_FILL_BYTE = 0xA5,
+};
+
+#define MEMEV_NO_INDEX UINT32_MAX
+#define MEMEV_TOMBSTONE ((uintptr_t)1)
+#define MEMEV_FILL_CAP ((size_t)256 * 1024 * 1024) /* never fill (or scan) beyond this */
+#define MEMEV_SITE_MASK ((uintptr_t)0x00FFFFFFFFFFFFFFULL)
+
+/* ── raw virtual memory ───────────────────────────────────────────────*/
+/* The layer's own footprint, reported in every dump header: a sanitizer that
+ * cannot say what it costs cannot be trusted to measure anybody else's waste.
+ * On Windows every byte here is committed; on POSIX it is lazily backed. */
+static _Atomic uint64_t g_layer_vm_bytes;
+static _Atomic int64_t g_layer_threads;
+
+static void *vm_get(size_t bytes) {
+#ifdef _WIN32
+ void *p = VirtualAlloc(NULL, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+#else
+ void *p = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
+ if (p == MAP_FAILED) {
+ p = NULL;
+ }
+#endif
+ if (p) {
+ atomic_fetch_add_explicit(&g_layer_vm_bytes, bytes, memory_order_relaxed);
+ }
+ return p;
+}
+
+static void vm_put(void *p, size_t bytes) {
+ if (!p) {
+ return;
+ }
+ atomic_fetch_sub_explicit(&g_layer_vm_bytes, bytes, memory_order_relaxed);
+#ifdef _WIN32
+ (void)bytes;
+ VirtualFree(p, 0, MEM_RELEASE);
+#else
+ munmap(p, bytes);
+#endif
+}
+
+/* ── spin lock ────────────────────────────────────────────────────────
+ * Held for a handful of instructions around one table probe. A mutex would
+ * have to be initialised, and initialisation may allocate. */
+typedef atomic_flag memev_lock_t;
+
+static void lock_take(memev_lock_t *l) {
+ while (atomic_flag_test_and_set_explicit(l, memory_order_acquire)) {
+ }
+}
+
+static void lock_drop(memev_lock_t *l) {
+ atomic_flag_clear_explicit(l, memory_order_release);
+}
+
+/* ── tables ───────────────────────────────────────────────────────────*/
+typedef struct {
+ uint64_t allocs;
+ uint64_t frees;
+ uint64_t reallocs;
+ uint64_t requested_bytes;
+ uint64_t usable_bytes;
+ uint64_t realloc_copy_bytes;
+ uint64_t short_lived;
+ uint64_t raw_bytes;
+ uint64_t scanned_blocks;
+ uint64_t never_written_bytes;
+ uint64_t over_requested_bytes;
+ uint64_t zero_untouched_bytes;
+ uint64_t acc_untouched_blocks;
+ uint64_t acc_untouched_bytes;
+ uint64_t acc_dead_blocks;
+ uint64_t acc_dead_bytes;
+ uint64_t acc_uninit_reads;
+ uint64_t acc_idle_bytes;
+ uint64_t acc_opaque_blocks;
+ int64_t live_bytes;
+ int64_t live_blocks;
+} site_counters_t;
+
+typedef struct {
+ uint64_t calls;
+ uint64_t bytes;
+ uint64_t aux1;
+ uint64_t aux2;
+ uint64_t repeats;
+ uint64_t peak;
+} work_counters_t;
+
+enum {
+ META_PHASE_MASK = 0x3F,
+ META_FILLED = 0x40,
+ META_ZEROED = 0x80,
+};
+
+typedef struct {
+ uintptr_t block; /* 0 empty, MEMEV_TOMBSTONE deleted */
+ uint64_t usable;
+ uint64_t requested;
+ uint32_t seq; /* thread slot << MEMEV_SEQ_BITS | that thread's alloc counter */
+ uint16_t site;
+ int8_t mem_class;
+ uint8_t meta; /* phase (6 bits) | FILLED | ZEROED */
+} ptr_entry_t;
+
+typedef struct {
+ memev_lock_t lock;
+ ptr_entry_t *entries;
+ uint32_t bits;
+ uint32_t used; /* live + tombstones */
+ uint32_t live;
+} shard_t;
+
+typedef struct {
+ const void *table;
+ uint64_t key_hash;
+ uint64_t generation;
+} ht_pair_t;
+
+typedef struct {
+ const char *ptr;
+ size_t len;
+ uint64_t content_hash;
+} strlen_pair_t;
+
+typedef struct {
+ site_counters_t *delta; /* MEMEV_SITE_CAP entries */
+ uint32_t *dirty; /* site ids touched since the last flush */
+ uint8_t *is_dirty;
+ uint32_t dirty_count;
+ work_counters_t *work_delta; /* MEMEV_WORK_CAP entries */
+ uint32_t *work_dirty;
+ uint8_t *work_is_dirty;
+ uint32_t work_dirty_count;
+ uint32_t slot; /* thread slot, 8 bits */
+ uint32_t alloc_seq; /* this thread's allocation counter */
+ uint64_t untracked_frees;
+ uint64_t pointer_table_full;
+ uint64_t site_table_full;
+ uint64_t work_table_full;
+ strlen_pair_t strlen_cache[MEMEV_REPEAT_SLOTS];
+ ht_pair_t ht_cache[MEMEV_REPEAT_SLOTS];
+ /* The handshake and the re-entrancy guard live HERE, not in _Thread_local
+ * variables. On macOS the first touch of a _Thread_local in a thread
+ * allocates its storage THROUGH MALLOC: inside an allocation hook that
+ * re-enters the hook, which touches the variable again, until the stack is
+ * gone (SIGSEGV at start-up, 2026-09-17; MinGW's emulated TLS allocates the
+ * same way). A pthread key / FLS slot never allocates, and the state it
+ * points at comes from raw virtual memory. */
+ void *hint_site;
+ int hint_class;
+ bool hint_set;
+ bool busy; /* the layer is invisible to itself */
+ uint32_t backing; /* depth of backing allocator calls (access lane) */
+ uint64_t ops; /* allocations + work notes by this thread */
+ uint32_t library_depth;
+ void *library_site;
+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS
+ uint32_t acc_ids[512];
+ uint32_t acc_id_count;
+#endif
+} tl_state_t;
+
+static _Atomic uintptr_t g_site_keys[MEMEV_SITE_CAP];
+static site_counters_t g_site_counters[MEMEV_SITE_CAP];
+static uint64_t g_snap_retained[MEMEV_SITE_CAP];
+static uint64_t g_snap_dup_bytes[MEMEV_SITE_CAP];
+static uint64_t g_snap_dup_blocks[MEMEV_SITE_CAP];
+static _Atomic uintptr_t g_work_keys[MEMEV_WORK_CAP];
+static work_counters_t g_work_counters[MEMEV_WORK_CAP];
+static _Atomic uint64_t g_path_hashes[MEMEV_PATH_CAP];
+static memev_lock_t g_merge_lock = ATOMIC_FLAG_INIT;
+static shard_t g_shards[MEMEV_SHARDS];
+static _Atomic uint64_t g_untracked_frees;
+static _Atomic uint64_t g_foreign_frees;
+static _Atomic uint64_t g_pointer_table_full;
+static _Atomic uint64_t g_site_table_full;
+static _Atomic uint64_t g_work_table_full;
+static _Atomic uint32_t g_next_slot;
+static _Atomic uint32_t g_phase;
+static char g_phase_label[MEMEV_PHASE_LABEL] = "start";
+static _Atomic int g_enabled = -1;
+static _Atomic int g_fill = -1;
+static _Atomic int g_dupes = -1;
+static _Atomic bool g_observer_installed;
+static _Atomic uintptr_t g_image_base;
+
+static const char *const g_work_names[CBM_WORK_KIND_COUNT] = {
+ "memcpy", "memmove", "memset", "memcmp", "strlen", "strcmp", "strncmp",
+ "read", "write", "pread", "pwrite", "open", "stat", "fopen",
+ "fread", "fwrite", "opendir", "readdir", "mutex", "parallel_for", "pool_ops",
+ "ht_get", "ht_set", "dead_write", "ct_arena", "ct_dyn_array", "ct_hash_table",
+};
+
+const char *cbm_work_kind_name(cbm_work_kind_t kind) {
+ if ((int)kind < 0 || kind >= CBM_WORK_KIND_COUNT) {
+ return "invalid";
+ }
+ return g_work_names[kind];
+}
+
+static int env_flag(const char *name, int dflt) {
+ const char *v = getenv(name);
+ if (!v || !v[0]) {
+ return dflt;
+ }
+ return (v[0] == '1' && v[1] == '\0') ? 1 : 0;
+}
+
+#if defined(__linux__)
+extern char **environ;
+#endif
+
+bool cbm_memev_enabled(void) {
+ int state = atomic_load_explicit(&g_enabled, memory_order_acquire);
+ if (state >= 0) {
+ return state == 1;
+ }
+#if defined(__linux__)
+ /* Code can run before libc has set up the environment: the loader's
+ * pre-init hooks (the UBSan runtime that -fsanitize-coverage links, which
+ * allocates through dlsym) reach this through the malloc and mutex wrappers.
+ * getenv answers NULL there, and caching that answer switched the layer
+ * off for the whole process (Linux access lane, no dump, 2026-09-17).
+ * Decide only once the environment exists. */
+ if (!environ) {
+ return false;
+ }
+#endif
+ int on = env_flag("CBM_MEMWASTE", 0);
+ atomic_store_explicit(&g_enabled, on, memory_order_release);
+ return on == 1;
+}
+
+static bool fill_enabled(void) {
+ int state = atomic_load_explicit(&g_fill, memory_order_relaxed);
+ if (state < 0) {
+ state = env_flag("CBM_MEMWASTE_FILL", 1);
+ atomic_store_explicit(&g_fill, state, memory_order_relaxed);
+ }
+ return state == 1;
+}
+
+static bool dupes_enabled(void) {
+ int state = atomic_load_explicit(&g_dupes, memory_order_relaxed);
+ if (state < 0) {
+ state = env_flag("CBM_MEMWASTE_DUPES", 1);
+ atomic_store_explicit(&g_dupes, state, memory_order_relaxed);
+ }
+ return state == 1;
+}
+
+void cbm_memev_scan_for_tests(bool fill, bool dupes) {
+ atomic_store_explicit(&g_fill, fill ? 1 : 0, memory_order_relaxed);
+ atomic_store_explicit(&g_dupes, dupes ? 1 : 0, memory_order_relaxed);
+}
+
+void cbm_memev_force_for_tests(bool on) {
+ atomic_store_explicit(&g_enabled, on ? 1 : 0, memory_order_release);
+}
+
+void cbm_memev_set_image_base(uintptr_t base) {
+ atomic_store_explicit(&g_image_base, base, memory_order_release);
+}
+
+void cbm_memev_set_observer_installed(bool installed) {
+ atomic_store_explicit(&g_observer_installed, installed, memory_order_release);
+}
+
+bool cbm_memev_observer_installed(void) {
+ return atomic_load_explicit(&g_observer_installed, memory_order_acquire);
+}
+
+/* ── per-thread slot: a key, never _Thread_local (see tl_state_t) ───────*/
+static _Atomic int g_key_state; /* 0 none, 1 being created, 2 ready */
+#ifdef _WIN32
+static DWORD g_key;
+#else
+static pthread_key_t g_key;
+#endif
+
+static size_t tl_bytes(void);
+static void tl_release(void *raw);
+#ifdef _WIN32
+#endif
+
+static bool key_ready(void) {
+ int state = atomic_load_explicit(&g_key_state, memory_order_acquire);
+ while (state != 2) {
+ int expect = 0;
+ if (state == 0 && atomic_compare_exchange_strong_explicit(&g_key_state, &expect, 1,
+ memory_order_acq_rel,
+ memory_order_acquire)) {
+#ifdef _WIN32
+ /* A plain TLS slot, released from the loader's thread-detach
+ * callback (cbm_memev_thread_end, compat_thread.c). An FLS
+ * destructor never ran in the static MinGW link, and by the time
+ * the detach callback runs the FLS value is already gone -- the
+ * state leaked 22 MB per thread (71 threads by the similarity pass,
+ * then commit exhaustion, 2026-09-17). A TLS slot is still readable
+ * there. */
+ g_key = TlsAlloc();
+ bool ok = g_key != TLS_OUT_OF_INDEXES;
+#else
+ bool ok = pthread_key_create(&g_key, tl_release) == 0;
+#endif
+ atomic_store_explicit(&g_key_state, ok ? 2 : 0, memory_order_release);
+ if (!ok) {
+ return false;
+ }
+ }
+ state = atomic_load_explicit(&g_key_state, memory_order_acquire);
+ }
+ return true;
+}
+
+/* A thread whose state was already released keeps this marker in its slot. Its
+ * teardown still frees (and on Windows allocates): without the marker every such
+ * call would create a fresh 22 MB state that no destructor ever releases --
+ * Windows runs an FLS callback once per thread, it does not re-run it the way
+ * POSIX re-runs key destructors. Observed as commit exhaustion in the Windows
+ * index worker, 2026-09-17. */
+static char g_tl_exited_marker;
+#define TL_EXITED ((void *)&g_tl_exited_marker)
+
+static void *tl_slot_value(void) {
+#ifdef _WIN32
+ return TlsGetValue(g_key);
+#else
+ return pthread_getspecific(g_key);
+#endif
+}
+
+static tl_state_t *tl_peek(void) {
+ if (!key_ready()) {
+ return NULL;
+ }
+ void *v = tl_slot_value();
+ return v == TL_EXITED ? NULL : (tl_state_t *)v;
+}
+
+static size_t tl_bytes(void) {
+ return sizeof(tl_state_t) + ((size_t)MEMEV_SITE_CAP * sizeof(site_counters_t)) +
+ ((size_t)MEMEV_SITE_CAP * sizeof(uint32_t)) + (size_t)MEMEV_SITE_CAP +
+ ((size_t)MEMEV_WORK_CAP * sizeof(work_counters_t)) +
+ ((size_t)MEMEV_WORK_CAP * sizeof(uint32_t)) + (size_t)MEMEV_WORK_CAP;
+}
+
+static tl_state_t *tl_get(void) {
+ if (!key_ready()) {
+ return NULL;
+ }
+ void *v = tl_slot_value();
+ if (v == TL_EXITED) {
+ return NULL; /* this thread is ending: observe nothing more */
+ }
+ if (v) {
+ return (tl_state_t *)v;
+ }
+ char *raw = (char *)vm_get(tl_bytes());
+ if (!raw) {
+ return NULL;
+ }
+ tl_state_t *st = (tl_state_t *)raw;
+ raw += sizeof(tl_state_t);
+ st->delta = (site_counters_t *)raw;
+ raw += (size_t)MEMEV_SITE_CAP * sizeof(site_counters_t);
+ st->dirty = (uint32_t *)raw;
+ raw += (size_t)MEMEV_SITE_CAP * sizeof(uint32_t);
+ st->is_dirty = (uint8_t *)raw;
+ raw += (size_t)MEMEV_SITE_CAP;
+ st->work_delta = (work_counters_t *)raw;
+ raw += (size_t)MEMEV_WORK_CAP * sizeof(work_counters_t);
+ st->work_dirty = (uint32_t *)raw;
+ raw += (size_t)MEMEV_WORK_CAP * sizeof(uint32_t);
+ st->work_is_dirty = (uint8_t *)raw;
+ atomic_fetch_add_explicit(&g_layer_threads, 1, memory_order_relaxed);
+ st->slot = atomic_fetch_add_explicit(&g_next_slot, 1, memory_order_relaxed) &
+ ((1U << MEMEV_SHARD_BITS) - 1U);
+#ifdef _WIN32
+ if (!TlsSetValue(g_key, st)) {
+ vm_put(st, tl_bytes());
+ return NULL;
+ }
+#else
+ if (pthread_setspecific(g_key, st) != 0) {
+ vm_put(st, tl_bytes());
+ return NULL;
+ }
+#endif
+ return st;
+}
+
+static site_counters_t *tl_site(tl_state_t *st, uint32_t site) {
+ if (!st->is_dirty[site]) {
+ st->is_dirty[site] = 1;
+ st->dirty[st->dirty_count++] = site;
+ }
+ return &st->delta[site];
+}
+
+static work_counters_t *tl_work(tl_state_t *st, uint32_t idx) {
+ if (!st->work_is_dirty[idx]) {
+ st->work_is_dirty[idx] = 1;
+ st->work_dirty[st->work_dirty_count++] = idx;
+ }
+ return &st->work_delta[idx];
+}
+
+/* ── keyed tables ─────────────────────────────────────────────────────*/
+static uint64_t mix(uint64_t x) {
+ x ^= x >> 33;
+ x *= 0xff51afd7ed558ccdULL;
+ x ^= x >> 33;
+ x *= 0xc4ceb9fe1a85ec53ULL;
+ x ^= x >> 33;
+ return x;
+}
+
+/* Index of `key` in a CAS-claimed open-addressed key table, claiming a slot on
+ * first sight. MEMEV_NO_INDEX when the table is full — counted by the caller,
+ * never silently dropped. */
+static uint32_t key_index(_Atomic uintptr_t *keys, uint32_t cap, uintptr_t key) {
+ uint32_t idx = (uint32_t)(mix((uint64_t)key) & (cap - 1));
+ for (uint32_t probe = 0; probe < cap; probe++) {
+ uintptr_t seen = atomic_load_explicit(&keys[idx], memory_order_acquire);
+ if (seen == key) {
+ return idx;
+ }
+ if (seen == 0) {
+ uintptr_t expect = 0;
+ if (atomic_compare_exchange_strong_explicit(&keys[idx], &expect, key,
+ memory_order_acq_rel,
+ memory_order_acquire) ||
+ expect == key) {
+ return idx;
+ }
+ }
+ idx = (idx + 1) & (cap - 1);
+ }
+ return MEMEV_NO_INDEX;
+}
+
+static uint32_t site_index(uintptr_t site) {
+ site &= MEMEV_SITE_MASK;
+ if (site == 0) {
+ site = (uintptr_t)1; /* "unknown caller" is a site like any other */
+ }
+ return key_index(g_site_keys, MEMEV_SITE_CAP, site);
+}
+
+static uint32_t work_index(cbm_work_kind_t kind, uintptr_t site) {
+ uintptr_t key = (site & MEMEV_SITE_MASK) | ((uintptr_t)((unsigned)kind + 1U) << 56);
+ return key_index(g_work_keys, MEMEV_WORK_CAP, key);
+}
+
+/* ── pointer shards ───────────────────────────────────────────────────*/
+static shard_t *shard_of(uintptr_t block) {
+ return &g_shards[mix((uint64_t)(block >> MEMEV_PTR_SHIFT)) >> (64 - MEMEV_SHARD_BITS)];
+}
+
+static uint32_t slot_of(uintptr_t block, uint32_t bits) {
+ return (uint32_t)(mix((uint64_t)(block >> MEMEV_PTR_SHIFT)) & ((1U << bits) - 1U));
+}
+
+static bool shard_resize(shard_t *sh, uint32_t bits) {
+ if (bits > MEMEV_SHARD_MAX_BITS) {
+ return false;
+ }
+ size_t bytes = ((size_t)1 << bits) * sizeof(ptr_entry_t);
+ ptr_entry_t *next = (ptr_entry_t *)vm_get(bytes);
+ if (!next) {
+ return false;
+ }
+ uint32_t live = 0;
+ if (sh->entries) {
+ uint32_t old_cap = 1U << sh->bits;
+ for (uint32_t i = 0; i < old_cap; i++) {
+ ptr_entry_t *e = &sh->entries[i];
+ if (e->block == 0 || e->block == MEMEV_TOMBSTONE) {
+ continue;
+ }
+ uint32_t s = slot_of(e->block, bits);
+ while (next[s].block != 0) {
+ s = (s + 1) & ((1U << bits) - 1U);
+ }
+ next[s] = *e;
+ live++;
+ }
+ vm_put(sh->entries, ((size_t)1 << sh->bits) * sizeof(ptr_entry_t));
+ }
+ sh->entries = next;
+ sh->bits = bits;
+ sh->used = live;
+ sh->live = live;
+ return true;
+}
+
+/* Caller holds the shard lock. */
+static ptr_entry_t *shard_insert(shard_t *sh, uintptr_t block) {
+ if (!sh->entries && !shard_resize(sh, MEMEV_SHARD_INIT_BITS)) {
+ return NULL;
+ }
+ uint32_t cap = 1U << sh->bits;
+ if ((uint64_t)(sh->used + 1) * MEMEV_LOAD_DEN > (uint64_t)cap * MEMEV_LOAD_NUM) {
+ /* Mostly tombstones: rehash in place at the same size; otherwise double. */
+ uint32_t want = ((uint64_t)sh->live * 2 * MEMEV_LOAD_DEN > (uint64_t)cap * MEMEV_LOAD_NUM)
+ ? sh->bits + 1
+ : sh->bits;
+ if (!shard_resize(sh, want)) {
+ return NULL;
+ }
+ cap = 1U << sh->bits;
+ }
+ uint32_t s = slot_of(block, sh->bits);
+ ptr_entry_t *grave = NULL;
+ for (uint32_t probe = 0; probe < cap; probe++) {
+ ptr_entry_t *e = &sh->entries[s];
+ if (e->block == block) {
+ return e; /* address reuse without a free we saw: overwrite */
+ }
+ if (e->block == MEMEV_TOMBSTONE && !grave) {
+ grave = e;
+ } else if (e->block == 0) {
+ ptr_entry_t *dst = grave ? grave : e;
+ if (!grave) {
+ sh->used++;
+ }
+ sh->live++;
+ dst->block = block;
+ return dst;
+ }
+ s = (s + 1) & (cap - 1);
+ }
+ return NULL;
+}
+
+/* Caller holds the shard lock. */
+static ptr_entry_t *shard_find(shard_t *sh, uintptr_t block) {
+ if (!sh->entries) {
+ return NULL;
+ }
+ uint32_t cap = 1U << sh->bits;
+ uint32_t s = slot_of(block, sh->bits);
+ for (uint32_t probe = 0; probe < cap; probe++) {
+ ptr_entry_t *e = &sh->entries[s];
+ if (e->block == block) {
+ return e;
+ }
+ if (e->block == 0) {
+ return NULL;
+ }
+ s = (s + 1) & (cap - 1);
+ }
+ return NULL;
+}
+
+/* ── fill pattern and scan ────────────────────────────────────────────
+ * A new block is filled with MEMEV_FILL_BYTE. At free, a run of at least
+ * MEMEV_RUN_MIN fill bytes is memory nobody ever wrote, and the last byte that
+ * is NOT the fill byte is the high-water mark of what was written. Runs are
+ * counted at word granularity (aligned 8-byte words), which is within 7 bytes
+ * per run edge and fast enough for hundreds of gigabytes. A calloc block is
+ * scanned the same way for zero bytes, which is an upper bound: a written zero
+ * is indistinguishable from an untouched one. */
+typedef struct {
+ uint64_t run_bytes;
+ uint64_t high_water;
+} scan_t;
+
+static scan_t scan_block(const uint8_t *p, size_t n, uint8_t pat) {
+ scan_t out = {0, 0};
+ uint64_t patw = 0x0101010101010101ULL * pat;
+ size_t hw = n;
+ while (hw > 0 && (hw & 7U) != 0 && p[hw - 1] == pat) {
+ hw--;
+ }
+ while (hw >= 8) {
+ uint64_t w;
+ memcpy(&w, p + hw - 8, 8);
+ if (w != patw) {
+ break;
+ }
+ hw -= 8;
+ }
+ while (hw > 0 && p[hw - 1] == pat) {
+ hw--;
+ }
+ size_t tail = n - hw;
+ if (tail < MEMEV_RUN_MIN) {
+ hw = n; /* a short tail of the fill byte is as likely written as not */
+ tail = 0;
+ }
+ uint64_t runs = tail;
+ size_t i = 0;
+ while (i + 8 <= hw) {
+ uint64_t w;
+ memcpy(&w, p + i, 8);
+ if (w == patw) {
+ runs += 8;
+ }
+ i += 8;
+ }
+ out.run_bytes = runs;
+ out.high_water = hw;
+ return out;
+}
+
+/* ── access lane ──────────────────────────────────────────────────────*/
+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS
+enum {
+ /* 48-bit user address space: arm64 Linux maps the program and its heap
+ * above 2^47 (0xaaaa... and 0xffff...), which a 47-bit map silently ignored. */
+ ACC_L1_BITS = 16,
+ ACC_L1_CAP = 1 << ACC_L1_BITS,
+ ACC_SLOT_SHIFT = 4,
+ ACC_REGION_SLOTS = 1 << 28, /* one 4 GB region in 16-byte slots */
+ ACC_RECS_BITS = 27,
+ ACC_WRITTEN = 1,
+ ACC_READ = 2,
+ ACC_READ_BEFORE_WRITE = 4,
+ ACC_ID_BATCH = 256,
+};
+
+typedef struct {
+ _Atomic uint8_t flags;
+ uint8_t last_phase;
+ uint8_t zeroed;
+ uint8_t filled; /* holds the fill pattern: a byte that differs was written by someone */
+ uint32_t site;
+ uint64_t usable;
+ _Atomic uintptr_t writer;
+} acc_rec_t;
+
+static _Atomic(uint32_t *) g_acc_l2[ACC_L1_CAP];
+static acc_rec_t *g_acc_recs;
+static uint32_t *g_acc_free;
+static uint32_t g_acc_free_count;
+static uint32_t g_acc_next = 1;
+static memev_lock_t g_acc_lock = ATOMIC_FLAG_INIT;
+static _Atomic int g_acc_ready;
+
+static bool acc_init(void) {
+ if (atomic_load_explicit(&g_acc_ready, memory_order_acquire) == 2) {
+ return true;
+ }
+ lock_take(&g_acc_lock);
+ if (atomic_load_explicit(&g_acc_ready, memory_order_relaxed) != 2) {
+ g_acc_recs = (acc_rec_t *)vm_get(((size_t)1 << ACC_RECS_BITS) * sizeof(acc_rec_t));
+ g_acc_free = (uint32_t *)vm_get(((size_t)1 << ACC_RECS_BITS) * sizeof(uint32_t));
+ atomic_store_explicit(&g_acc_ready, (g_acc_recs && g_acc_free) ? 2 : 1,
+ memory_order_release);
+ }
+ lock_drop(&g_acc_lock);
+ return atomic_load_explicit(&g_acc_ready, memory_order_acquire) == 2;
+}
+
+static uint32_t *acc_region(uint32_t l1, bool create) {
+ uint32_t *l2 = atomic_load_explicit(&g_acc_l2[l1], memory_order_acquire);
+ if (l2 || !create) {
+ return l2;
+ }
+ uint32_t *fresh = (uint32_t *)vm_get((size_t)ACC_REGION_SLOTS * sizeof(uint32_t));
+ if (!fresh) {
+ return NULL;
+ }
+ uint32_t *expect = NULL;
+ if (!atomic_compare_exchange_strong_explicit(&g_acc_l2[l1], &expect, fresh,
+ memory_order_acq_rel, memory_order_acquire)) {
+ vm_put(fresh, (size_t)ACC_REGION_SLOTS * sizeof(uint32_t));
+ return expect;
+ }
+ return fresh;
+}
+
+/* Write `id` into every slot of [a, a + n). */
+static void acc_map(uintptr_t a, uint64_t n, uint32_t id) {
+ uintptr_t end = a + (uintptr_t)(n ? n : 1);
+ while (a < end) {
+ uint32_t l1 = (uint32_t)(a >> 32);
+ if (l1 >= ACC_L1_CAP) {
+ return;
+ }
+ uint32_t *l2 = acc_region(l1, id != 0);
+ uintptr_t region_end = ((uintptr_t)l1 + 1) << 32;
+ uintptr_t stop = end < region_end ? end : region_end;
+ if (l2) {
+ uint64_t first = ((uint32_t)a) >> ACC_SLOT_SHIFT;
+ uint64_t last = (stop == region_end) ? (uint64_t)ACC_REGION_SLOTS
+ : ((((uint32_t)(stop - 1)) >> ACC_SLOT_SHIFT) + 1);
+ for (uint64_t s = first; s < last; s++) {
+ l2[s] = id;
+ }
+ }
+ a = stop;
+ }
+}
+
+static uint32_t acc_lookup(uintptr_t a) {
+ uint32_t l1 = (uint32_t)(a >> 32);
+ if (l1 >= ACC_L1_CAP) {
+ return 0;
+ }
+ uint32_t *l2 = atomic_load_explicit(&g_acc_l2[l1], memory_order_relaxed);
+ return l2 ? l2[((uint32_t)a) >> ACC_SLOT_SHIFT] : 0;
+}
+
+static uint32_t acc_take_id(tl_state_t *st) {
+ if (st->acc_id_count == 0) {
+ lock_take(&g_acc_lock);
+ while (st->acc_id_count < ACC_ID_BATCH) {
+ uint32_t id;
+ if (g_acc_free_count > 0) {
+ id = g_acc_free[--g_acc_free_count];
+ } else if (g_acc_next < (1U << ACC_RECS_BITS)) {
+ id = g_acc_next++;
+ } else {
+ break;
+ }
+ st->acc_ids[st->acc_id_count++] = id;
+ }
+ lock_drop(&g_acc_lock);
+ }
+ return st->acc_id_count ? st->acc_ids[--st->acc_id_count] : 0;
+}
+
+static void acc_give_id(tl_state_t *st, uint32_t id) {
+ if (st->acc_id_count < (uint32_t)(sizeof(st->acc_ids) / sizeof(st->acc_ids[0]))) {
+ st->acc_ids[st->acc_id_count++] = id;
+ return;
+ }
+ lock_take(&g_acc_lock);
+ g_acc_free[g_acc_free_count++] = id;
+ while (st->acc_id_count > ACC_ID_BATCH) {
+ g_acc_free[g_acc_free_count++] = st->acc_ids[--st->acc_id_count];
+ }
+ lock_drop(&g_acc_lock);
+}
+
+static void acc_track(tl_state_t *st, uintptr_t block, uint64_t usable, uint32_t site,
+ bool zeroed, bool filled, bool written) {
+ if (!acc_init()) {
+ return;
+ }
+ uint32_t id = acc_take_id(st);
+ if (!id) {
+ return;
+ }
+ acc_rec_t *r = &g_acc_recs[id];
+ /* A block that already holds content when the layer hears of it (the
+ * Linux/Windows strdup wrappers report AFTER the copy) was written by the
+ * allocation itself: without this every such block read a first time was a
+ * read-before-write -- 16.5 M false verdicts on Linux, 12.6 M of them the
+ * reach-cache key copies (2026-09-17). */
+ atomic_store_explicit(&r->flags, written ? ACC_WRITTEN : 0, memory_order_relaxed);
+ atomic_store_explicit(&r->writer, 0, memory_order_relaxed);
+ r->last_phase = (uint8_t)atomic_load_explicit(&g_phase, memory_order_relaxed);
+ r->zeroed = zeroed ? 1 : 0;
+ r->filled = filled ? 1 : 0;
+ r->site = site;
+ r->usable = usable;
+ acc_map(block, usable, id);
+}
+
+static void work_add(tl_state_t *st, cbm_work_kind_t kind, uintptr_t site, uint64_t bytes,
+ uint64_t aux1, uint64_t aux2, uint64_t repeats);
+
+/* The verdict on a block that is going away: what its life was spent on.
+ * `scan_written` is the fill scan's word on it (1 written, 0 not, -1 no scan):
+ * a library call the compiler did not instrument (snprintf, strcpy, qsort)
+ * writes without a store callback, and the pattern still shows it. */
+static void acc_untrack(tl_state_t *st, uintptr_t block, uint64_t usable, int scan_written) {
+ if (atomic_load_explicit(&g_acc_ready, memory_order_relaxed) != 2) {
+ return;
+ }
+ uint32_t id = acc_lookup(block);
+ if (!id) {
+ return;
+ }
+ acc_rec_t *r = &g_acc_recs[id];
+ uint8_t f = atomic_load_explicit(&r->flags, memory_order_relaxed);
+ site_counters_t *c = tl_site(st, r->site);
+ /* OPAQUE: the fill pattern shows writes, yet no callback ever fired. The
+ * block was used only by code the lane cannot see -- an uninstrumented
+ * library, or an aggregate copy (`*dst = *src` of a struct) that the
+ * compiler expands inline after instrumentation, emitting no load or store
+ * callback. Its reads are invisible too, so neither "untouched" nor
+ * "written, never read" nor "idle" can be claimed about it. */
+ bool opaque = scan_written == 1 && !(f & (ACC_WRITTEN | ACC_READ));
+ if (opaque) {
+ c->acc_opaque_blocks++;
+ } else if (!(f & (ACC_WRITTEN | ACC_READ))) {
+ c->acc_untouched_blocks++;
+ c->acc_untouched_bytes += r->usable;
+ } else if ((f & ACC_WRITTEN) && !(f & ACC_READ)) {
+ /* Seen written, never seen read. A read through an inline struct copy is
+ * invisible, so this is "no instrumented read", named as such in reports. */
+ c->acc_dead_blocks++;
+ c->acc_dead_bytes += r->usable;
+ uintptr_t writer = atomic_load_explicit(&r->writer, memory_order_relaxed);
+ if (writer) { /* 0: only a library wrote it -- counted at the site, no writer to name */
+ work_add(st, CBM_WORK_DEAD_WRITE, writer, r->usable, 1, 0, 0);
+ }
+ }
+ if ((f & ACC_READ_BEFORE_WRITE) && !r->zeroed) {
+ c->acc_uninit_reads++;
+ }
+ if (!opaque && r->last_phase < (uint8_t)atomic_load_explicit(&g_phase, memory_order_relaxed)) {
+ c->acc_idle_bytes += r->usable;
+ }
+ acc_map(block, usable, 0);
+ acc_give_id(st, id);
+}
+
+static uint32_t acc_move(uintptr_t old_block, uint64_t old_usable, uintptr_t grown,
+ uint64_t usable) {
+ if (atomic_load_explicit(&g_acc_ready, memory_order_relaxed) != 2) {
+ return 0;
+ }
+ uint32_t id = acc_lookup(old_block);
+ if (!id) {
+ return 0;
+ }
+ acc_map(old_block, old_usable, 0);
+ g_acc_recs[id].usable = usable;
+ acc_map(grown, usable, id);
+ return id;
+}
+
+static inline void acc_touch(uintptr_t a, bool store, uintptr_t pc) {
+ uint32_t id = acc_lookup(a);
+ if (!id) {
+ return;
+ }
+ acc_rec_t *r = &g_acc_recs[id];
+ uint8_t f = atomic_load_explicit(&r->flags, memory_order_relaxed);
+ if (store) {
+ if (!(f & ACC_WRITTEN)) {
+ atomic_fetch_or_explicit(&r->flags, ACC_WRITTEN, memory_order_relaxed);
+ atomic_store_explicit(&r->writer, pc, memory_order_relaxed);
+ }
+ } else if (!(f & ACC_READ)) {
+ uint8_t mark = ACC_READ;
+ if (!(f & ACC_WRITTEN)) {
+ /* The program is about to read this byte, so looking at it is safe. */
+ bool library_wrote = r->filled && *(const uint8_t *)a != MEMEV_FILL_BYTE;
+ mark |= library_wrote ? ACC_WRITTEN : ACC_READ_BEFORE_WRITE;
+ }
+ atomic_fetch_or_explicit(&r->flags, mark, memory_order_relaxed);
+ }
+ uint8_t ph = (uint8_t)atomic_load_explicit(&g_phase, memory_order_relaxed);
+ if (r->last_phase != ph) {
+ r->last_phase = ph;
+ }
+}
+
+#define ACC_PC() ((uintptr_t)__builtin_return_address(0))
+void __sanitizer_cov_load1(uint8_t *a) {
+ acc_touch((uintptr_t)a, false, 0);
+}
+void __sanitizer_cov_load2(uint16_t *a) {
+ acc_touch((uintptr_t)a, false, 0);
+}
+void __sanitizer_cov_load4(uint32_t *a) {
+ acc_touch((uintptr_t)a, false, 0);
+}
+void __sanitizer_cov_load8(uint64_t *a) {
+ acc_touch((uintptr_t)a, false, 0);
+}
+void __sanitizer_cov_load16(__int128 *a) {
+ acc_touch((uintptr_t)a, false, 0);
+}
+void __sanitizer_cov_store1(uint8_t *a) {
+ acc_touch((uintptr_t)a, true, ACC_PC());
+}
+void __sanitizer_cov_store2(uint16_t *a) {
+ acc_touch((uintptr_t)a, true, ACC_PC());
+}
+void __sanitizer_cov_store4(uint32_t *a) {
+ acc_touch((uintptr_t)a, true, ACC_PC());
+}
+void __sanitizer_cov_store8(uint64_t *a) {
+ acc_touch((uintptr_t)a, true, ACC_PC());
+}
+void __sanitizer_cov_store16(__int128 *a) {
+ acc_touch((uintptr_t)a, true, ACC_PC());
+}
+void __sanitizer_cov_bool_flag_init(uint8_t *start, uint8_t *stop) {
+ (void)start;
+ (void)stop;
+}
+
+/* Verdicts are per block, so touching both ends of the range marks every block
+ * it can belong to (a range never spans two allocations in a correct program). */
+void cbm_memev_access(const void *p, size_t n, bool store, void *pc) {
+ if (!p || n == 0 || atomic_load_explicit(&g_acc_ready, memory_order_relaxed) != 2) {
+ return;
+ }
+ tl_state_t *st = tl_get();
+ if (!st || st->busy || st->backing) {
+ return;
+ }
+ uintptr_t a = (uintptr_t)p;
+ acc_touch(a, store, (uintptr_t)pc);
+ if (n > 1) {
+ acc_touch(a + (uintptr_t)(n - 1), store, (uintptr_t)pc);
+ }
+}
+
+void cbm_memev_backing(int delta) {
+ tl_state_t *st = tl_get();
+ if (st) {
+ st->backing = (uint32_t)((int)st->backing + delta);
+ }
+}
+#else
+static void work_add(tl_state_t *st, cbm_work_kind_t kind, uintptr_t site, uint64_t bytes,
+ uint64_t aux1, uint64_t aux2, uint64_t repeats);
+#endif /* CBM_MEMWASTE_ACCESS */
+
+/* ── the handshake ────────────────────────────────────────────────────*/
+void cbm_memev_hint(void *site, int mem_class) {
+ if (!cbm_memev_enabled()) {
+ return;
+ }
+ tl_state_t *st = tl_get();
+ if (st) {
+ st->hint_site = site;
+ st->hint_class = mem_class;
+ st->hint_set = true;
+ }
+}
+
+bool cbm_memev_hint_pending(void) {
+ if (!cbm_memev_enabled()) {
+ return false;
+ }
+ const tl_state_t *st = tl_peek();
+ return st && st->hint_set;
+}
+
+/* ── events ───────────────────────────────────────────────────────────*/
+static void record_alloc(tl_state_t *st, uintptr_t block, size_t requested, size_t usable,
+ uintptr_t site, int mem_class, unsigned flags) {
+ st->ops++;
+ uint32_t sid = site_index(site);
+ if (sid == MEMEV_NO_INDEX) {
+ st->site_table_full++;
+ return;
+ }
+ uint64_t charged = usable ? usable : requested;
+ st->alloc_seq = (st->alloc_seq + 1) & (uint32_t)MEMEV_SEQ_MASK;
+
+ bool zeroed = (flags & CBM_MEMEV_ZEROED) != 0;
+ bool filled = false;
+ if (!zeroed && !(flags & (CBM_MEMEV_FOREIGN | CBM_MEMEV_WRITTEN)) && charged <= MEMEV_FILL_CAP &&
+ fill_enabled()) {
+ memset((void *)block, MEMEV_FILL_BYTE, (size_t)charged);
+ filled = true;
+ }
+
+ shard_t *sh = shard_of(block);
+ lock_take(&sh->lock);
+ ptr_entry_t *e = shard_insert(sh, block);
+ if (e) {
+ e->usable = charged;
+ e->requested = requested;
+ e->seq = (st->slot << MEMEV_SEQ_BITS) | st->alloc_seq;
+ e->site = (uint16_t)sid;
+ e->mem_class = (int8_t)mem_class;
+ e->meta = (uint8_t)((atomic_load_explicit(&g_phase, memory_order_relaxed) &
+ META_PHASE_MASK) |
+ (filled ? META_FILLED : 0) | (zeroed ? META_ZEROED : 0));
+ }
+ lock_drop(&sh->lock);
+ if (!e) {
+ st->pointer_table_full++;
+ }
+
+ site_counters_t *c = tl_site(st, sid);
+ c->allocs++;
+ c->requested_bytes += requested;
+ c->usable_bytes += charged;
+ if (mem_class == CBM_MEMEV_CLASS_RAW) {
+ c->raw_bytes += charged;
+ }
+ if (e) {
+ c->live_bytes += (int64_t)charged;
+ c->live_blocks++;
+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS
+ if (!(flags & (CBM_MEMEV_FOREIGN | CBM_MEMEV_LIBRARY))) {
+ acc_track(st, block, charged, sid, zeroed, filled,
+ (flags & CBM_MEMEV_WRITTEN) != 0);
+ }
+#endif
+ }
+}
+
+void cbm_memev_alloc_ex(void *block, size_t requested, size_t usable, void *site,
+ unsigned flags) {
+ if (!cbm_memev_enabled()) {
+ return;
+ }
+ tl_state_t *st = tl_get();
+ if (!st || st->busy) {
+ return;
+ }
+ bool hinted = st->hint_set;
+ st->hint_set = false; /* consumed even when the allocation failed */
+ if (!block) {
+ return;
+ }
+ st->busy = true;
+ record_alloc(st, (uintptr_t)block, requested, usable,
+ (uintptr_t)(hinted ? st->hint_site : site),
+ hinted ? st->hint_class : CBM_MEMEV_CLASS_RAW,
+ hinted ? (flags & ~(unsigned)CBM_MEMEV_FOREIGN) : flags);
+ st->busy = false;
+}
+
+void cbm_memev_alloc(void *block, size_t requested, size_t usable, void *site) {
+ cbm_memev_alloc_ex(block, requested, usable, site, 0);
+}
+
+/* Removes the record of `block`. Returns false when the layer never saw it. */
+static bool take_entry(uintptr_t block, ptr_entry_t *out) {
+ shard_t *sh = shard_of(block);
+ lock_take(&sh->lock);
+ ptr_entry_t *e = shard_find(sh, block);
+ bool found = e != NULL;
+ if (found) {
+ *out = *e;
+ e->block = MEMEV_TOMBSTONE;
+ sh->live--;
+ }
+ lock_drop(&sh->lock);
+ return found;
+}
+
+void cbm_memev_free(void *block) {
+ if (!block || !cbm_memev_enabled()) {
+ return;
+ }
+ tl_state_t *st = tl_get();
+ if (!st || st->busy) {
+ return;
+ }
+ st->busy = true;
+ ptr_entry_t e;
+ if (take_entry((uintptr_t)block, &e)) {
+ site_counters_t *c = tl_site(st, e.site);
+ c->frees++;
+ c->live_bytes -= (int64_t)e.usable;
+ c->live_blocks--;
+ uint32_t slot = e.seq >> MEMEV_SEQ_BITS;
+ uint32_t age =
+ (st->alloc_seq - (e.seq & (uint32_t)MEMEV_SEQ_MASK)) & (uint32_t)MEMEV_SEQ_MASK;
+ if (slot == st->slot && age <= CBM_MEMEV_CHURN_WINDOW) {
+ c->short_lived++;
+ }
+ int scan_written = -1;
+ if ((e.meta & (META_FILLED | META_ZEROED)) && e.usable <= MEMEV_FILL_CAP) {
+ bool zeroed = (e.meta & META_ZEROED) != 0;
+ scan_t sc = scan_block((const uint8_t *)block, (size_t)e.usable,
+ zeroed ? 0 : MEMEV_FILL_BYTE);
+ scan_written = sc.high_water > 0 ? 1 : 0;
+ c->scanned_blocks++;
+ if (zeroed) {
+ c->zero_untouched_bytes += e.usable - sc.high_water;
+ } else {
+ c->never_written_bytes += sc.run_bytes;
+ if (e.requested > sc.high_water) {
+ c->over_requested_bytes += e.requested - sc.high_water;
+ }
+ }
+ }
+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS
+ acc_untrack(st, (uintptr_t)block, e.usable, scan_written);
+#else
+ (void)scan_written;
+#endif
+ } else {
+ st->untracked_frees++;
+ }
+ st->busy = false;
+}
+
+void cbm_memev_free_foreign(void *block) {
+ if (!block || !cbm_memev_enabled()) {
+ return;
+ }
+ atomic_fetch_add_explicit(&g_foreign_frees, 1, memory_order_relaxed);
+}
+
+void cbm_memev_realloc(void *old_block, void *grown, size_t requested, size_t usable,
+ void *site) {
+ if (!cbm_memev_enabled()) {
+ return;
+ }
+ tl_state_t *st = tl_get();
+ if (!st || st->busy) {
+ return;
+ }
+ bool hinted = st->hint_set;
+ void *hint_site = st->hint_site;
+ int hint_class = st->hint_class;
+ st->hint_set = false;
+ if (!grown) {
+ return;
+ }
+ st->busy = true;
+ ptr_entry_t e;
+ bool had = old_block && take_entry((uintptr_t)old_block, &e);
+ if (had) {
+ /* The record keeps its ORIGINAL site: a realloc chain is one object
+ * growing, and its copies are that object's cost. */
+ site_counters_t *c = tl_site(st, e.site);
+ c->reallocs++;
+ c->live_bytes -= (int64_t)e.usable;
+ c->live_blocks--;
+ if (grown != old_block) {
+ c->realloc_copy_bytes += e.usable < requested ? e.usable : requested;
+ }
+ uint64_t charged = usable ? usable : requested;
+ uint8_t meta = e.meta;
+ if ((meta & META_FILLED) && charged > e.usable && charged <= MEMEV_FILL_CAP) {
+ /* the new tail is uninitialised: it gets the pattern too */
+ memset((char *)grown + e.usable, MEMEV_FILL_BYTE, (size_t)(charged - e.usable));
+ } else if (charged > e.usable) {
+ meta &= (uint8_t)~(META_FILLED | META_ZEROED); /* mixed content: stop scanning */
+ }
+ shard_t *sh = shard_of((uintptr_t)grown);
+ lock_take(&sh->lock);
+ ptr_entry_t *n = shard_insert(sh, (uintptr_t)grown);
+ if (n) {
+ n->usable = charged;
+ n->requested = requested;
+ n->seq = e.seq;
+ n->site = e.site;
+ n->mem_class = e.mem_class;
+ n->meta = meta;
+ }
+ lock_drop(&sh->lock);
+ c->requested_bytes += requested;
+ c->usable_bytes += charged;
+ if (e.mem_class == CBM_MEMEV_CLASS_RAW) {
+ c->raw_bytes += charged;
+ }
+ if (n) {
+ c->live_bytes += (int64_t)charged;
+ c->live_blocks++;
+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS
+ (void)acc_move((uintptr_t)old_block, e.usable, (uintptr_t)grown, charged);
+#endif
+ } else {
+ st->pointer_table_full++;
+ }
+ } else {
+ /* The old block was never seen (allocated before the observer, or by a
+ * path it does not cover): the grown block already holds the program's
+ * data, so it is recorded without the fill. realloc(NULL, n) is a fresh
+ * allocation and gets the fill like any malloc. */
+ record_alloc(st, (uintptr_t)grown, requested, usable,
+ (uintptr_t)(hinted ? hint_site : site),
+ hinted ? hint_class : CBM_MEMEV_CLASS_RAW,
+ old_block ? CBM_MEMEV_WRITTEN : 0); /* realloc(NULL, n) is a malloc */
+ }
+ st->busy = false;
+}
+
+/* ── CPU work ─────────────────────────────────────────────────────────*/
+static void work_add(tl_state_t *st, cbm_work_kind_t kind, uintptr_t site, uint64_t bytes,
+ uint64_t aux1, uint64_t aux2, uint64_t repeats) {
+ st->ops++;
+ uint32_t idx = work_index(kind, site);
+ if (idx == MEMEV_NO_INDEX) {
+ st->work_table_full++;
+ return;
+ }
+ work_counters_t *w = tl_work(st, idx);
+ w->calls++;
+ w->bytes += bytes;
+ w->aux1 += aux1;
+ w->aux2 += aux2;
+ w->repeats += repeats;
+ bool container = kind == CBM_WORK_CT_ARENA || kind == CBM_WORK_CT_DYN_ARRAY ||
+ kind == CBM_WORK_CT_HASH_TABLE;
+ uint64_t single = container ? (bytes > aux1 ? bytes - aux1 : 0) : bytes;
+ if (single > w->peak) {
+ w->peak = single;
+ }
+}
+
+void cbm_memev_library_enter(void *site) {
+ tl_state_t *st = cbm_memev_enabled() ? tl_get() : NULL;
+ if (st && st->library_depth++ == 0) {
+ st->library_site = site;
+ }
+}
+
+void cbm_memev_library_leave(void) {
+ tl_state_t *st = cbm_memev_enabled() ? tl_get() : NULL;
+ if (st && st->library_depth > 0 && --st->library_depth == 0) {
+ st->library_site = NULL;
+ }
+}
+
+void *cbm_memev_library_site(void) {
+ tl_state_t *st = cbm_memev_enabled() ? tl_get() : NULL;
+ return st ? st->library_site : NULL;
+}
+
+#if defined(CBM_MEMWASTE_PROFILE) && CBM_MEMWASTE_PROFILE
+void __llvm_profile_set_filename(const char *pattern);
+int __llvm_profile_write_file(void);
+#endif
+
+void cbm_memev_process_role(const char *role) {
+#if defined(CBM_MEMWASTE_PROFILE) && CBM_MEMWASTE_PROFILE
+ const char *dir = getenv("CBM_PROFILE_DIR");
+ if (!dir || !dir[0] || !role) {
+ return;
+ }
+ /* The runtime keeps the pointer, not a copy: the pattern must outlive us. */
+ static char pattern[1024];
+ int n = snprintf(pattern, sizeof(pattern), "%s/%s-%%p.profraw", dir, role);
+ if (n > 0 && (size_t)n < sizeof(pattern)) {
+ __llvm_profile_set_filename(pattern);
+ }
+#else
+ (void)role;
+#endif
+}
+
+void cbm_memev_process_exit(void) {
+ if (cbm_memev_enabled()) {
+ const char *out = getenv("CBM_MEMWASTE_OUT");
+ if (out && out[0]) {
+ (void)cbm_memev_dump(out, "exit");
+ }
+ }
+#if defined(CBM_MEMWASTE_PROFILE) && CBM_MEMWASTE_PROFILE
+ (void)__llvm_profile_write_file();
+#endif
+}
+
+uint64_t cbm_memev_thread_ops(void) {
+ if (!cbm_memev_enabled()) {
+ return 0;
+ }
+ tl_state_t *st = tl_get();
+ return st ? st->ops : 0;
+}
+
+void cbm_work_note(cbm_work_kind_t kind, void *site, uint64_t bytes, uint64_t aux1,
+ uint64_t aux2) {
+ if ((int)kind < 0 || kind >= CBM_WORK_KIND_COUNT || !cbm_memev_enabled()) {
+ return;
+ }
+ tl_state_t *st = tl_get();
+ if (!st || st->busy) {
+ return;
+ }
+ st->busy = true;
+ work_add(st, kind, (uintptr_t)site, bytes, aux1, aux2, 0);
+ st->busy = false;
+}
+
+__attribute__((noinline)) void cbm_memev_da_note(size_t cap_bytes, size_t used_bytes,
+ size_t cap_slots) {
+ if (!cbm_memev_enabled() || cap_bytes == 0) {
+ return;
+ }
+ uint64_t growths = 0;
+ for (size_t c = 8; c < cap_slots; c <<= 1) {
+ growths++;
+ }
+ cbm_work_note(CBM_WORK_CT_DYN_ARRAY, __builtin_return_address(0), cap_bytes, used_bytes,
+ growths);
+}
+
+/* FNV-1a over the first bytes of a string of known length; the length is
+ * mixed in, so two strings that share a long prefix still differ. */
+enum { MEMEV_CONTENT_HASH_MAX = 4096 };
+static uint64_t bytes_hash(const char *s, size_t len) {
+ uint64_t h = 0xcbf29ce484222325ULL ^ (uint64_t)len;
+ size_t n = len < MEMEV_CONTENT_HASH_MAX ? len : MEMEV_CONTENT_HASH_MAX;
+ for (size_t i = 0; s && i < n; i++) {
+ h ^= (uint8_t)s[i];
+ h *= 0x100000001b3ULL;
+ }
+ return h;
+}
+
+void cbm_work_note_strlen(void *site, const char *s, size_t len) {
+ if (!cbm_memev_enabled()) {
+ return;
+ }
+ tl_state_t *st = tl_get();
+ if (!st || st->busy) {
+ return;
+ }
+ st->busy = true;
+ strlen_pair_t *slot =
+ &st->strlen_cache[mix((uint64_t)(uintptr_t)s) & (MEMEV_REPEAT_SLOTS - 1)];
+ uint64_t content = bytes_hash(s, len);
+ uint64_t repeat =
+ (slot->ptr == s && slot->len == len && slot->content_hash == content) ? 1 : 0;
+ slot->ptr = s;
+ slot->len = len;
+ slot->content_hash = content;
+ work_add(st, CBM_WORK_STRLEN, (uintptr_t)site, len, 0, 0, repeat);
+ st->busy = false;
+}
+
+/* FNV-1a over a NUL-terminated path, without calling the strlen we wrap. */
+static uint64_t path_hash(const char *p) {
+ uint64_t h = 0xcbf29ce484222325ULL;
+ for (; p && *p; p++) {
+ h ^= (uint8_t)*p;
+ h *= 0x100000001b3ULL;
+ }
+ return h;
+}
+
+void cbm_work_note_path(cbm_work_kind_t kind, void *site, const char *path, bool failed) {
+ if (!cbm_memev_enabled()) {
+ return;
+ }
+ tl_state_t *st = tl_get();
+ if (!st || st->busy) {
+ return;
+ }
+ st->busy = true;
+ uint64_t h = mix(path_hash(path) ^ ((uint64_t)kind << 58)) | 1U;
+ uint64_t repeat = 0;
+ uint32_t idx = (uint32_t)(h & (MEMEV_PATH_CAP - 1));
+ for (uint32_t probe = 0; probe < MEMEV_PATH_CAP; probe++) {
+ uint64_t seen = atomic_load_explicit(&g_path_hashes[idx], memory_order_relaxed);
+ if (seen == h) {
+ repeat = 1;
+ break;
+ }
+ if (seen == 0) {
+ uint64_t expect = 0;
+ if (atomic_compare_exchange_strong_explicit(&g_path_hashes[idx], &expect, h,
+ memory_order_relaxed,
+ memory_order_relaxed)) {
+ break;
+ }
+ if (expect == h) {
+ repeat = 1;
+ break;
+ }
+ }
+ idx = (idx + 1) & (MEMEV_PATH_CAP - 1);
+ }
+ work_add(st, kind, (uintptr_t)site, 0, failed ? 1 : 0, 0, repeat);
+ st->busy = false;
+}
+
+void cbm_work_note_ht(cbm_work_kind_t kind, void *site, const void *table, const char *key,
+ uint64_t generation, bool hit) {
+ if (!cbm_memev_enabled()) {
+ return;
+ }
+ tl_state_t *st = tl_get();
+ if (!st || st->busy) {
+ return;
+ }
+ st->busy = true;
+ uint64_t repeat = 0;
+ if (kind == CBM_WORK_HT_GET) {
+ uint64_t key_hash = path_hash(key);
+ ht_pair_t *slot =
+ &st->ht_cache[mix((uint64_t)(uintptr_t)table ^ (key_hash * 0x9e3779b97f4a7c15ULL)) &
+ (MEMEV_REPEAT_SLOTS - 1)];
+ repeat = (slot->table == table && slot->key_hash == key_hash &&
+ slot->generation == generation)
+ ? 1
+ : 0;
+ slot->table = table;
+ slot->key_hash = key_hash;
+ slot->generation = generation;
+ }
+ work_add(st, kind, (uintptr_t)site, 0, (kind == CBM_WORK_HT_GET && !hit) ? 1 : 0, 0, repeat);
+ st->busy = false;
+}
+
+/* ── merge ────────────────────────────────────────────────────────────*/
+static void flush_state(tl_state_t *st) {
+ lock_take(&g_merge_lock);
+ for (uint32_t i = 0; i < st->dirty_count; i++) {
+ uint32_t sid = st->dirty[i];
+ site_counters_t *d = &st->delta[sid];
+ site_counters_t *g = &g_site_counters[sid];
+ g->allocs += d->allocs;
+ g->frees += d->frees;
+ g->reallocs += d->reallocs;
+ g->requested_bytes += d->requested_bytes;
+ g->usable_bytes += d->usable_bytes;
+ g->realloc_copy_bytes += d->realloc_copy_bytes;
+ g->short_lived += d->short_lived;
+ g->raw_bytes += d->raw_bytes;
+ g->scanned_blocks += d->scanned_blocks;
+ g->never_written_bytes += d->never_written_bytes;
+ g->over_requested_bytes += d->over_requested_bytes;
+ g->zero_untouched_bytes += d->zero_untouched_bytes;
+ g->acc_untouched_blocks += d->acc_untouched_blocks;
+ g->acc_untouched_bytes += d->acc_untouched_bytes;
+ g->acc_dead_blocks += d->acc_dead_blocks;
+ g->acc_dead_bytes += d->acc_dead_bytes;
+ g->acc_uninit_reads += d->acc_uninit_reads;
+ g->acc_idle_bytes += d->acc_idle_bytes;
+ g->acc_opaque_blocks += d->acc_opaque_blocks;
+ g->live_bytes += d->live_bytes;
+ g->live_blocks += d->live_blocks;
+ memset(d, 0, sizeof(*d));
+ st->is_dirty[sid] = 0;
+ }
+ st->dirty_count = 0;
+ for (uint32_t i = 0; i < st->work_dirty_count; i++) {
+ uint32_t idx = st->work_dirty[i];
+ work_counters_t *d = &st->work_delta[idx];
+ work_counters_t *g = &g_work_counters[idx];
+ g->calls += d->calls;
+ g->bytes += d->bytes;
+ g->aux1 += d->aux1;
+ g->aux2 += d->aux2;
+ g->repeats += d->repeats;
+ if (d->peak > g->peak) {
+ g->peak = d->peak;
+ }
+ memset(d, 0, sizeof(*d));
+ st->work_is_dirty[idx] = 0;
+ }
+ st->work_dirty_count = 0;
+ lock_drop(&g_merge_lock);
+ atomic_fetch_add_explicit(&g_untracked_frees, st->untracked_frees, memory_order_relaxed);
+ atomic_fetch_add_explicit(&g_pointer_table_full, st->pointer_table_full,
+ memory_order_relaxed);
+ atomic_fetch_add_explicit(&g_site_table_full, st->site_table_full, memory_order_relaxed);
+ atomic_fetch_add_explicit(&g_work_table_full, st->work_table_full, memory_order_relaxed);
+ st->untracked_frees = 0;
+ st->pointer_table_full = 0;
+ st->site_table_full = 0;
+ st->work_table_full = 0;
+}
+
+void cbm_memev_flush_thread(void) {
+ if (!cbm_memev_enabled()) {
+ return;
+ }
+ tl_state_t *st = tl_peek();
+ if (!st || st->busy) {
+ return;
+ }
+ st->busy = true;
+ flush_state(st);
+ st->busy = false;
+}
+
+/* Thread exit (pthread key / FLS destructor): hand over what is pending, then
+ * give the raw memory back. */
+static void tl_release(void *raw);
+
+void cbm_memev_thread_end(void) {
+ if (atomic_load_explicit(&g_key_state, memory_order_acquire) != 2) {
+ return; /* no thread ever had state: nothing to release */
+ }
+ void *v = tl_slot_value();
+ if (v && v != TL_EXITED) {
+ tl_release(v);
+ }
+}
+
+static void tl_release(void *raw) {
+ tl_state_t *st = (tl_state_t *)raw;
+ if (!st || raw == TL_EXITED) {
+ return;
+ }
+ st->busy = true;
+ flush_state(st);
+ /* Mark the slot before the memory goes: whatever the rest of the thread's
+ * teardown allocates or frees must find the marker, neither a freed pointer
+ * nor an empty slot that would build a new state. */
+#ifdef _WIN32
+ (void)TlsSetValue(g_key, TL_EXITED);
+#else
+ (void)pthread_setspecific(g_key, TL_EXITED);
+#endif
+ atomic_fetch_sub_explicit(&g_layer_threads, 1, memory_order_relaxed);
+ vm_put(st, tl_bytes());
+}
+
+/* ── phase snapshot ───────────────────────────────────────────────────
+ * Live blocks allocated in an earlier phase are RETAINED: the question is
+ * whether the phase that is running still needs them. Small live blocks with
+ * identical content are DUPLICATES: one interned copy would do. Both are
+ * snapshots of the live set taken after the workers joined. */
+static uint64_t content_hash(const uint8_t *p, size_t n) {
+ uint64_t h = 0xcbf29ce484222325ULL ^ ((uint64_t)n * 0x9e3779b97f4a7c15ULL);
+ for (size_t i = 0; i < n; i++) {
+ h ^= p[i];
+ h *= 0x100000001b3ULL;
+ }
+ return h | 1U;
+}
+
+static void snapshot_phase(uint32_t phase_now) {
+ memset(g_snap_retained, 0, sizeof(g_snap_retained));
+ memset(g_snap_dup_bytes, 0, sizeof(g_snap_dup_bytes));
+ memset(g_snap_dup_blocks, 0, sizeof(g_snap_dup_blocks));
+ bool dupes = dupes_enabled();
+ uint64_t live_total = 0;
+ for (uint32_t i = 0; i < MEMEV_SHARDS; i++) {
+ live_total += g_shards[i].live;
+ }
+ size_t set_cap = 1024;
+ while (set_cap < live_total * 2 && set_cap < ((size_t)1 << 30)) {
+ set_cap <<= 1;
+ }
+ uint64_t *set = dupes ? (uint64_t *)vm_get(set_cap * sizeof(uint64_t)) : NULL;
+ for (uint32_t i = 0; i < MEMEV_SHARDS; i++) {
+ shard_t *sh = &g_shards[i];
+ lock_take(&sh->lock);
+ uint32_t cap = sh->entries ? (1U << sh->bits) : 0;
+ for (uint32_t j = 0; j < cap; j++) {
+ const ptr_entry_t *e = &sh->entries[j];
+ if (e->block == 0 || e->block == MEMEV_TOMBSTONE) {
+ continue;
+ }
+ if ((uint32_t)(e->meta & META_PHASE_MASK) < phase_now) {
+ g_snap_retained[e->site] += e->usable;
+ }
+ if (!set || e->usable > CBM_MEMEV_DUP_MAX || !(e->meta & (META_FILLED | META_ZEROED))) {
+ continue;
+ }
+ const uint8_t *p = (const uint8_t *)e->block;
+ scan_t sc = scan_block(p, (size_t)e->usable,
+ (e->meta & META_ZEROED) ? 0 : MEMEV_FILL_BYTE);
+ if (sc.high_water == 0) {
+ continue; /* nothing written: that is never-written waste, not a duplicate */
+ }
+ uint64_t h = content_hash(p, (size_t)sc.high_water);
+ size_t s = (size_t)(h & (set_cap - 1));
+ for (;;) {
+ if (set[s] == 0) {
+ set[s] = h;
+ break;
+ }
+ if (set[s] == h) {
+ g_snap_dup_bytes[e->site] += e->usable;
+ g_snap_dup_blocks[e->site]++;
+ break;
+ }
+ s = (s + 1) & (set_cap - 1);
+ }
+ }
+ lock_drop(&sh->lock);
+ }
+ vm_put(set, set_cap * sizeof(uint64_t));
+}
+
+void cbm_memev_phase(const char *label) {
+ if (!cbm_memev_enabled()) {
+ return;
+ }
+ tl_state_t *st = tl_get();
+ if (!st || st->busy) {
+ return;
+ }
+ st->busy = true;
+ flush_state(st);
+ uint32_t next = atomic_load_explicit(&g_phase, memory_order_relaxed) + 1;
+ if (next <= MEMEV_PHASE_CAP) {
+ atomic_store_explicit(&g_phase, next, memory_order_relaxed);
+ }
+ lock_take(&g_merge_lock);
+ snprintf(g_phase_label, sizeof(g_phase_label), "%s", label ? label : "-");
+ lock_drop(&g_merge_lock);
+ snapshot_phase(atomic_load_explicit(&g_phase, memory_order_relaxed));
+ st->busy = false;
+ const char *out = getenv("CBM_MEMWASTE_OUT");
+ if (out && out[0]) {
+ char why[MEMEV_PHASE_LABEL + 8];
+ snprintf(why, sizeof(why), "phase:%s", label ? label : "-");
+ (void)cbm_memev_dump(out, why);
+ }
+}
+
+/* ── report ───────────────────────────────────────────────────────────*/
+void cbm_memev_totals(cbm_memev_totals_t *out) {
+ if (!out) {
+ return;
+ }
+ memset(out, 0, sizeof(*out));
+ cbm_memev_flush_thread();
+ lock_take(&g_merge_lock);
+ for (uint32_t i = 0; i < MEMEV_SITE_CAP; i++) {
+ if (atomic_load_explicit(&g_site_keys[i], memory_order_relaxed) == 0) {
+ continue;
+ }
+ const site_counters_t *g = &g_site_counters[i];
+ out->sites++;
+ out->allocs += g->allocs;
+ out->frees += g->frees;
+ out->reallocs += g->reallocs;
+ out->requested_bytes += g->requested_bytes;
+ out->usable_bytes += g->usable_bytes;
+ out->realloc_copy_bytes += g->realloc_copy_bytes;
+ out->raw_bytes += g->raw_bytes;
+ if (g->live_bytes > 0) {
+ out->live_bytes += (uint64_t)g->live_bytes;
+ }
+ if (g->live_blocks > 0) {
+ out->live_blocks += (uint64_t)g->live_blocks;
+ }
+ }
+ lock_drop(&g_merge_lock);
+ out->untracked_frees = atomic_load_explicit(&g_untracked_frees, memory_order_relaxed);
+ out->foreign_frees = atomic_load_explicit(&g_foreign_frees, memory_order_relaxed);
+ out->pointer_table_full = atomic_load_explicit(&g_pointer_table_full, memory_order_relaxed);
+ out->site_table_full = atomic_load_explicit(&g_site_table_full, memory_order_relaxed);
+ out->work_table_full = atomic_load_explicit(&g_work_table_full, memory_order_relaxed);
+}
+
+static void site_row_of(uint32_t i, uintptr_t key, cbm_memev_site_t *row) {
+ const site_counters_t *g = &g_site_counters[i];
+ *row = (cbm_memev_site_t){
+ .site = key,
+ .allocs = g->allocs,
+ .frees = g->frees,
+ .reallocs = g->reallocs,
+ .requested_bytes = g->requested_bytes,
+ .usable_bytes = g->usable_bytes,
+ .realloc_copy_bytes = g->realloc_copy_bytes,
+ .live_bytes = g->live_bytes > 0 ? (uint64_t)g->live_bytes : 0,
+ .live_blocks = g->live_blocks > 0 ? (uint64_t)g->live_blocks : 0,
+ .short_lived = g->short_lived,
+ .raw_bytes = g->raw_bytes,
+ .scanned_blocks = g->scanned_blocks,
+ .never_written_bytes = g->never_written_bytes,
+ .over_requested_bytes = g->over_requested_bytes,
+ .zero_untouched_bytes = g->zero_untouched_bytes,
+ .retained_bytes = g_snap_retained[i],
+ .dup_bytes = g_snap_dup_bytes[i],
+ .dup_blocks = g_snap_dup_blocks[i],
+ .acc_untouched_blocks = g->acc_untouched_blocks,
+ .acc_untouched_bytes = g->acc_untouched_bytes,
+ .acc_dead_blocks = g->acc_dead_blocks,
+ .acc_dead_bytes = g->acc_dead_bytes,
+ .acc_uninit_reads = g->acc_uninit_reads,
+ .acc_idle_bytes = g->acc_idle_bytes,
+ .acc_opaque_blocks = g->acc_opaque_blocks,
+ };
+}
+
+static int by_usable_desc(const void *a, const void *b) {
+ const cbm_memev_site_t *x = (const cbm_memev_site_t *)a;
+ const cbm_memev_site_t *y = (const cbm_memev_site_t *)b;
+ if (x->usable_bytes != y->usable_bytes) {
+ return x->usable_bytes < y->usable_bytes ? 1 : -1;
+ }
+ /* Equal bytes: order by address, so the report is a function of the data. */
+ return x->site < y->site ? -1 : (x->site > y->site ? 1 : 0);
+}
+
+/* Top `max` sites by cumulative usable bytes, selected without allocating. */
+size_t cbm_memev_sites(cbm_memev_site_t *out, size_t max) {
+ if (!out || max == 0) {
+ return 0;
+ }
+ cbm_memev_flush_thread();
+ size_t n = 0;
+ lock_take(&g_merge_lock);
+ for (uint32_t i = 0; i < MEMEV_SITE_CAP; i++) {
+ uintptr_t key = atomic_load_explicit(&g_site_keys[i], memory_order_relaxed);
+ if (key == 0) {
+ continue;
+ }
+ cbm_memev_site_t row;
+ site_row_of(i, key, &row);
+ if (n < max) {
+ out[n++] = row;
+ continue;
+ }
+ size_t smallest = 0;
+ for (size_t k = 1; k < n; k++) {
+ if (by_usable_desc(&out[k], &out[smallest]) > 0) {
+ smallest = k;
+ }
+ }
+ if (by_usable_desc(&row, &out[smallest]) < 0) {
+ out[smallest] = row;
+ }
+ }
+ lock_drop(&g_merge_lock);
+ tl_state_t *st = tl_get();
+ bool was_busy = st && st->busy;
+ if (st) {
+ st->busy = true; /* qsort is libc; keep anything it does out of the tables */
+ }
+ qsort(out, n, sizeof(out[0]), by_usable_desc);
+ if (st) {
+ st->busy = was_busy;
+ }
+ return n;
+}
+
+size_t cbm_work_rows(cbm_work_row_t *out, size_t max) {
+ if (!out || max == 0) {
+ return 0;
+ }
+ cbm_memev_flush_thread();
+ size_t n = 0;
+ lock_take(&g_merge_lock);
+ for (uint32_t i = 0; i < MEMEV_WORK_CAP && n < max; i++) {
+ uintptr_t key = atomic_load_explicit(&g_work_keys[i], memory_order_relaxed);
+ if (key == 0) {
+ continue;
+ }
+ const work_counters_t *g = &g_work_counters[i];
+ out[n++] = (cbm_work_row_t){
+ .site = key & MEMEV_SITE_MASK,
+ .kind = (cbm_work_kind_t)((key >> 56) - 1),
+ .calls = g->calls,
+ .bytes = g->bytes,
+ .aux1 = g->aux1,
+ .aux2 = g->aux2,
+ .repeats = g->repeats,
+ .peak = g->peak,
+ };
+ }
+ lock_drop(&g_merge_lock);
+ return n;
+}
+
+static bool write_all(int fd, const char *buf, size_t len) {
+ while (len > 0) {
+#ifdef _WIN32
+ int put = _write(fd, buf, (unsigned)len);
+#else
+ ssize_t put = write(fd, buf, len);
+#endif
+ if (put <= 0) {
+ return false;
+ }
+ buf += put;
+ len -= (size_t)put;
+ }
+ return true;
+}
+
+#if defined(__linux__)
+static int first_object_bias(struct dl_phdr_info *info, size_t size, void *out) {
+ (void)size;
+ *(uintptr_t *)out = (uintptr_t)info->dlpi_addr;
+ return 1; /* the first object is the main program */
+}
+#endif
+
+/* Sites are raw return addresses; the report subtracts the image base to get
+ * the address the symboliser expects. macOS records it when the zone observer
+ * installs; Linux (the load bias: 0 for a non-PIE binary) and Windows (the
+ * runtime module base) record it here, once, before the first dump. */
+static void ensure_image_base(void) {
+ if (atomic_load_explicit(&g_image_base, memory_order_relaxed) != 0) {
+ return;
+ }
+#if defined(__linux__)
+ uintptr_t bias = 0;
+ dl_iterate_phdr(first_object_bias, &bias);
+ atomic_store_explicit(&g_image_base, bias, memory_order_relaxed);
+#elif defined(_WIN32)
+ atomic_store_explicit(&g_image_base, (uintptr_t)GetModuleHandleW(NULL), memory_order_relaxed);
+#endif
+}
+
+static bool dump_locked(const char *path, const char *why) {
+ ensure_image_base();
+ cbm_memev_totals_t t;
+ cbm_memev_totals(&t);
+#ifdef _WIN32
+ int fd = _open(path, _O_WRONLY | _O_CREAT | _O_APPEND | _O_BINARY, 0600);
+ int pid = (int)GetCurrentProcessId();
+#else
+ int fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0600);
+ int pid = (int)getpid();
+#endif
+ if (fd < 0) {
+ return false;
+ }
+ char label[MEMEV_PHASE_LABEL];
+ lock_take(&g_merge_lock);
+ snprintf(label, sizeof(label), "%s", g_phase_label);
+ lock_drop(&g_merge_lock);
+ char line[MEMEV_LINE];
+ int len = snprintf(
+ line, sizeof(line),
+ "{\"memwaste\":2,\"why\":\"%s\",\"pid\":%d,\"image_base\":\"0x%llx\",\"phase\":%u,"
+ "\"phase_label\":\"%s\",\"rss_bytes\":%llu,\"allocs\":%llu,\"frees\":%llu,"
+ "\"untracked_frees\":%llu,\"foreign_frees\":%llu,\"reallocs\":%llu,\"requested_bytes\":%llu,\"usable_bytes\":%llu,"
+ "\"realloc_copy_bytes\":%llu,\"live_bytes\":%llu,\"live_blocks\":%llu,\"raw_bytes\":%llu,"
+ "\"sites\":%llu,\"site_table_full\":%llu,\"pointer_table_full\":%llu,"
+ "\"work_table_full\":%llu,\"layer_vm_bytes\":%llu,\"layer_threads\":%lld,"
+ "\"fill\":%d,\"access\":%d}\n",
+ why ? why : "-", pid,
+ (unsigned long long)atomic_load_explicit(&g_image_base, memory_order_relaxed),
+ atomic_load_explicit(&g_phase, memory_order_relaxed), label,
+ (unsigned long long)cbm_mem_rss(), (unsigned long long)t.allocs,
+ (unsigned long long)t.frees, (unsigned long long)t.untracked_frees,
+ (unsigned long long)t.foreign_frees,
+ (unsigned long long)t.reallocs, (unsigned long long)t.requested_bytes,
+ (unsigned long long)t.usable_bytes, (unsigned long long)t.realloc_copy_bytes,
+ (unsigned long long)t.live_bytes, (unsigned long long)t.live_blocks,
+ (unsigned long long)t.raw_bytes, (unsigned long long)t.sites,
+ (unsigned long long)t.site_table_full, (unsigned long long)t.pointer_table_full,
+ (unsigned long long)t.work_table_full,
+ (unsigned long long)atomic_load_explicit(&g_layer_vm_bytes, memory_order_relaxed),
+ (long long)atomic_load_explicit(&g_layer_threads, memory_order_relaxed),
+ fill_enabled() ? 1 : 0,
+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS
+ 1
+#else
+ 0
+#endif
+ );
+ bool ok = len > 0 && (size_t)len < sizeof(line) && write_all(fd, line, (size_t)len);
+
+ lock_take(&g_merge_lock);
+ for (uint32_t i = 0; ok && i < MEMEV_SITE_CAP; i++) {
+ uintptr_t key = atomic_load_explicit(&g_site_keys[i], memory_order_relaxed);
+ if (key == 0) {
+ continue;
+ }
+ cbm_memev_site_t r;
+ site_row_of(i, key, &r);
+ len = snprintf(
+ line, sizeof(line),
+ "{\"site\":\"0x%llx\",\"allocs\":%llu,\"frees\":%llu,\"reallocs\":%llu,"
+ "\"requested_bytes\":%llu,\"usable_bytes\":%llu,\"realloc_copy_bytes\":%llu,"
+ "\"live_bytes\":%llu,\"live_blocks\":%llu,\"short_lived\":%llu,\"raw_bytes\":%llu,"
+ "\"scanned_blocks\":%llu,\"never_written_bytes\":%llu,\"over_requested_bytes\":%llu,"
+ "\"zero_untouched_bytes\":%llu,\"retained_bytes\":%llu,\"dup_bytes\":%llu,"
+ "\"dup_blocks\":%llu,\"acc_untouched_blocks\":%llu,\"acc_untouched_bytes\":%llu,"
+ "\"acc_dead_blocks\":%llu,\"acc_dead_bytes\":%llu,\"acc_uninit_reads\":%llu,"
+ "\"acc_idle_bytes\":%llu,\"acc_opaque_blocks\":%llu}\n",
+ (unsigned long long)r.site, (unsigned long long)r.allocs, (unsigned long long)r.frees,
+ (unsigned long long)r.reallocs, (unsigned long long)r.requested_bytes,
+ (unsigned long long)r.usable_bytes, (unsigned long long)r.realloc_copy_bytes,
+ (unsigned long long)r.live_bytes, (unsigned long long)r.live_blocks,
+ (unsigned long long)r.short_lived, (unsigned long long)r.raw_bytes,
+ (unsigned long long)r.scanned_blocks, (unsigned long long)r.never_written_bytes,
+ (unsigned long long)r.over_requested_bytes,
+ (unsigned long long)r.zero_untouched_bytes, (unsigned long long)r.retained_bytes,
+ (unsigned long long)r.dup_bytes, (unsigned long long)r.dup_blocks,
+ (unsigned long long)r.acc_untouched_blocks, (unsigned long long)r.acc_untouched_bytes,
+ (unsigned long long)r.acc_dead_blocks, (unsigned long long)r.acc_dead_bytes,
+ (unsigned long long)r.acc_uninit_reads, (unsigned long long)r.acc_idle_bytes,
+ (unsigned long long)r.acc_opaque_blocks);
+ ok = len > 0 && (size_t)len < sizeof(line) && write_all(fd, line, (size_t)len);
+ }
+ for (uint32_t i = 0; ok && i < MEMEV_WORK_CAP; i++) {
+ uintptr_t key = atomic_load_explicit(&g_work_keys[i], memory_order_relaxed);
+ if (key == 0) {
+ continue;
+ }
+ const work_counters_t *g = &g_work_counters[i];
+ if (g->calls == 0) {
+ continue;
+ }
+ len = snprintf(line, sizeof(line),
+ "{\"work\":\"%s\",\"site\":\"0x%llx\",\"calls\":%llu,\"bytes\":%llu,"
+ "\"aux1\":%llu,\"aux2\":%llu,\"repeats\":%llu,\"peak\":%llu}\n",
+ cbm_work_kind_name((cbm_work_kind_t)((key >> 56) - 1)),
+ (unsigned long long)(key & MEMEV_SITE_MASK), (unsigned long long)g->calls,
+ (unsigned long long)g->bytes, (unsigned long long)g->aux1,
+ (unsigned long long)g->aux2, (unsigned long long)g->repeats,
+ (unsigned long long)g->peak);
+ ok = len > 0 && (size_t)len < sizeof(line) && write_all(fd, line, (size_t)len);
+ }
+ lock_drop(&g_merge_lock);
+#ifdef _WIN32
+ _close(fd);
+#else
+ close(fd);
+#endif
+ return ok;
+}
+
+bool cbm_memev_dump(const char *path, const char *why) {
+ if (!path || !path[0]) {
+ return false;
+ }
+ static memev_lock_t dump_lock = ATOMIC_FLAG_INIT;
+ tl_state_t *st = tl_get();
+ bool was_busy = st && st->busy;
+ if (st && !was_busy) {
+ flush_state(st); /* this thread's pending events belong in the dump */
+ }
+ if (st) {
+ st->busy = true; /* the dump's own writes and allocations are not the program's work */
+ }
+ lock_take(&dump_lock);
+ bool ok = dump_locked(path, why);
+ lock_drop(&dump_lock);
+ if (st) {
+ st->busy = was_busy;
+ }
+ return ok;
+}
+
+void cbm_memev_reset_for_tests(void) {
+ cbm_memev_flush_thread();
+ lock_take(&g_merge_lock);
+ for (uint32_t i = 0; i < MEMEV_SITE_CAP; i++) {
+ atomic_store_explicit(&g_site_keys[i], 0, memory_order_relaxed);
+ }
+ for (uint32_t i = 0; i < MEMEV_WORK_CAP; i++) {
+ atomic_store_explicit(&g_work_keys[i], 0, memory_order_relaxed);
+ }
+ for (uint32_t i = 0; i < MEMEV_PATH_CAP; i++) {
+ atomic_store_explicit(&g_path_hashes[i], 0, memory_order_relaxed);
+ }
+ memset(g_site_counters, 0, sizeof(g_site_counters));
+ memset(g_work_counters, 0, sizeof(g_work_counters));
+ memset(g_snap_retained, 0, sizeof(g_snap_retained));
+ memset(g_snap_dup_bytes, 0, sizeof(g_snap_dup_bytes));
+ memset(g_snap_dup_blocks, 0, sizeof(g_snap_dup_blocks));
+ lock_drop(&g_merge_lock);
+ for (uint32_t i = 0; i < MEMEV_SHARDS; i++) {
+ shard_t *sh = &g_shards[i];
+ lock_take(&sh->lock);
+ vm_put(sh->entries, sh->entries ? ((size_t)1 << sh->bits) * sizeof(ptr_entry_t) : 0);
+ sh->entries = NULL;
+ sh->bits = 0;
+ sh->used = 0;
+ sh->live = 0;
+ lock_drop(&sh->lock);
+ }
+ atomic_store_explicit(&g_untracked_frees, 0, memory_order_relaxed);
+ atomic_store_explicit(&g_foreign_frees, 0, memory_order_relaxed);
+ atomic_store_explicit(&g_pointer_table_full, 0, memory_order_relaxed);
+ atomic_store_explicit(&g_site_table_full, 0, memory_order_relaxed);
+ atomic_store_explicit(&g_work_table_full, 0, memory_order_relaxed);
+ atomic_store_explicit(&g_phase, 0, memory_order_relaxed);
+ atomic_store_explicit(&g_fill, 0, memory_order_relaxed); /* tests use synthetic addresses */
+ tl_state_t *st = tl_peek();
+ if (st) {
+ st->hint_set = false;
+ memset(st->strlen_cache, 0, sizeof(st->strlen_cache));
+ memset(st->ht_cache, 0, sizeof(st->ht_cache));
+ }
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#endif
+
+#else /* layer compiled out: every entry point is an empty stub */
+
+bool cbm_memev_enabled(void) {
+ return false;
+}
+void cbm_memev_alloc(void *block, size_t requested, size_t usable, void *site) {
+ (void)block;
+ (void)requested;
+ (void)usable;
+ (void)site;
+}
+void cbm_memev_alloc_ex(void *block, size_t requested, size_t usable, void *site,
+ unsigned flags) {
+ (void)block;
+ (void)requested;
+ (void)usable;
+ (void)site;
+ (void)flags;
+}
+void cbm_memev_free(void *block) {
+ (void)block;
+}
+void cbm_memev_free_foreign(void *block) {
+ (void)block;
+}
+void cbm_memev_realloc(void *old_block, void *grown, size_t requested, size_t usable,
+ void *site) {
+ (void)old_block;
+ (void)grown;
+ (void)requested;
+ (void)usable;
+ (void)site;
+}
+void cbm_memev_hint(void *site, int mem_class) {
+ (void)site;
+ (void)mem_class;
+}
+bool cbm_memev_hint_pending(void) {
+ return false;
+}
+void cbm_memev_set_image_base(uintptr_t base) {
+ (void)base;
+}
+void cbm_memev_set_observer_installed(bool installed) {
+ (void)installed;
+}
+bool cbm_memev_observer_installed(void) {
+ return false;
+}
+void cbm_memev_flush_thread(void) {
+}
+void cbm_memev_phase(const char *label) {
+ (void)label;
+}
+const char *cbm_work_kind_name(cbm_work_kind_t kind) {
+ (void)kind;
+ return "off";
+}
+uint64_t cbm_memev_thread_ops(void) {
+ return 0;
+}
+void cbm_memev_process_role(const char *role) {
+ (void)role;
+}
+void cbm_memev_thread_end(void) {}
+void cbm_memev_process_exit(void) {}
+void cbm_memev_library_enter(void *site) {
+ (void)site;
+}
+void cbm_memev_library_leave(void) {}
+void *cbm_memev_library_site(void) {
+ return NULL;
+}
+void cbm_work_note(cbm_work_kind_t kind, void *site, uint64_t bytes, uint64_t aux1,
+ uint64_t aux2) {
+ (void)kind;
+ (void)site;
+ (void)bytes;
+ (void)aux1;
+ (void)aux2;
+}
+void cbm_memev_da_note(size_t cap_bytes, size_t used_bytes, size_t cap_slots) {
+ (void)cap_bytes;
+ (void)used_bytes;
+ (void)cap_slots;
+}
+void cbm_work_note_strlen(void *site, const char *s, size_t len) {
+ (void)site;
+ (void)s;
+ (void)len;
+}
+void cbm_work_note_path(cbm_work_kind_t kind, void *site, const char *path, bool failed) {
+ (void)kind;
+ (void)site;
+ (void)path;
+ (void)failed;
+}
+void cbm_work_note_ht(cbm_work_kind_t kind, void *site, const void *table, const char *key,
+ uint64_t generation, bool hit) {
+ (void)kind;
+ (void)site;
+ (void)table;
+ (void)key;
+ (void)generation;
+ (void)hit;
+}
+void cbm_memev_totals(cbm_memev_totals_t *out) {
+ if (out) {
+ *out = (cbm_memev_totals_t){0};
+ }
+}
+size_t cbm_memev_sites(cbm_memev_site_t *out, size_t max) {
+ (void)out;
+ (void)max;
+ return 0;
+}
+size_t cbm_work_rows(cbm_work_row_t *out, size_t max) {
+ (void)out;
+ (void)max;
+ return 0;
+}
+bool cbm_memev_dump(const char *path, const char *why) {
+ (void)path;
+ (void)why;
+ return false;
+}
+void cbm_memev_reset_for_tests(void) {
+}
+void cbm_memev_scan_for_tests(bool fill, bool dupes) {
+ (void)fill;
+ (void)dupes;
+}
+void cbm_memev_force_for_tests(bool on) {
+ (void)on;
+}
+
+#endif
diff --git a/src/foundation/mem_events.h b/src/foundation/mem_events.h
new file mode 100644
index 000000000..30eb05088
--- /dev/null
+++ b/src/foundation/mem_events.h
@@ -0,0 +1,359 @@
+/*
+ * mem_events.h — the waste sanitizer: memory and CPU work that is correct but
+ * unnecessary, attributed to the code that caused it.
+ *
+ * ASan, MSan, TSan and LSan find memory that is WRONG. Nothing in the tree
+ * measured memory or work that is merely WASTED — allocator slack, blocks
+ * bigger than what is written into them, churn an arena would have served,
+ * realloc chains copying the same bytes again, duplicate strings, memory held
+ * across phases that never touch it, the same string measured twice, the same
+ * file opened twice. The memory-core rounds found such waste BY HAND; this
+ * layer turns it into a ranked report.
+ *
+ * On 2026-09-17 a forwarding malloc-zone observer showed the Go indexing worker
+ * making 92.3 million raw allocations OUTSIDE the memory core on macOS — 128 GB
+ * requested cumulatively in a 41 s run — that no instrument in the tree saw.
+ *
+ * WHERE IT LISTENS
+ * - the memory core (mem_core.c) announces its own calls with a site + class;
+ * - platform observers see the malloc family underneath: the Linux and
+ * Windows --wrap shims, the macOS malloc-zone observer;
+ * - bound allocators that bypass both (SQLite, tree-sitter -> mimalloc) report
+ * directly;
+ * - containers (arena, dynamic array, hash table) report capacity vs use;
+ * - libc wrappers (mem_override_libc.c) report byte and system-call work;
+ * - the ACCESS lane (-fsanitize-coverage=trace-loads,trace-stores) reports
+ * reads and writes of every tracked block.
+ *
+ * COSTS NOTHING UNLESS ASKED FOR. Compiled in only under -DCBM_MEMWASTE=1 (the
+ * `memwaste` build flavour, `make MEMWASTE=1`) and dormant unless CBM_MEMWASTE=1
+ * is set. The access lane additionally needs -DCBM_MEMWASTE_ACCESS=1
+ * (`make MEMWASTE_ACCESS=1`). Release binaries contain none of it.
+ *
+ * NO PER-BLOCK HEADER (see mem_core.h: 16 bytes x ~100 M kernel blocks is
+ * 1.6 GB). Block state lives in side tables backed by raw virtual memory, so
+ * the layer never calls the allocator it is observing.
+ *
+ * DETERMINISM (O9). Every quantity a gate may read is a SUM over events —
+ * per-site totals — or a snapshot taken at a phase mark after the workers
+ * joined. Nothing here reads a clock.
+ *
+ * ENVIRONMENT
+ * CBM_MEMWASTE=1 wake the layer
+ * CBM_MEMWASTE_OUT=path append JSON-lines dumps at every phase mark
+ * CBM_MEMWASTE_FILL=0 do not fill new blocks with the never-written pattern
+ * CBM_MEMWASTE_DUPES=0 skip the duplicate-content scan at phase marks
+ */
+#ifndef CBM_MEM_EVENTS_H
+#define CBM_MEM_EVENTS_H
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/* Class value for blocks that never passed through the memory core. */
+#define CBM_MEMEV_CLASS_RAW (-1)
+
+/* Allocation flags. */
+enum {
+ CBM_MEMEV_ZEROED = 1u, /* calloc: the allocator zeroed it, never fill it */
+ CBM_MEMEV_FOREIGN = 2u, /* a system library allocating for itself: observe, never fill */
+ CBM_MEMEV_LIBRARY = 4u, /* allocated inside a wrapped library call, charged to its caller */
+ /* The block already holds its content when it is reported (a strdup copy, a
+ * realloc whose old block the layer never saw): observe, NEVER fill -- the
+ * fill pattern would overwrite live data. */
+ CBM_MEMEV_WRITTEN = 8u,
+};
+
+/* Blocks at or below this size take part in the duplicate-content scan. */
+enum { CBM_MEMEV_DUP_MAX = 256 };
+
+/* Freed by the allocating thread within this many of its own allocations:
+ * churn an arena or a stack buffer would have served. */
+enum { CBM_MEMEV_CHURN_WINDOW = 64 };
+
+/* True when the layer is compiled in AND CBM_MEMWASTE=1. Cached after the
+ * first call; safe from inside an allocator hook (it never allocates). */
+bool cbm_memev_enabled(void);
+
+/* ── Allocation events ────────────────────────────────────────────────
+ * `site` is a return address inside the code that asked for the memory.
+ * `usable` is what the allocator actually handed out (0 when unknown: the
+ * layer then charges `requested`). All return immediately when dormant. */
+void cbm_memev_alloc(void *block, size_t requested, size_t usable, void *site);
+void cbm_memev_alloc_ex(void *block, size_t requested, size_t usable, void *site, unsigned flags);
+void cbm_memev_free(void *block);
+/* A free of a block another allocator made (Windows: the C runtime or a system
+ * DLL allocating where --wrap cannot reach). Counted apart from untracked
+ * frees: its allocation was never observable, so it says what the layer cannot
+ * see, not that the layer missed something it should have seen. */
+void cbm_memev_free_foreign(void *block);
+/* One event for a realloc: the old block's record moves to `grown`, the copy
+ * the allocator had to make is charged to the ORIGINAL allocation site. */
+void cbm_memev_realloc(void *old_block, void *grown, size_t requested, size_t usable, void *site);
+
+/* ── Core <-> observer handshake ──────────────────────────────────────
+ * The memory core and a platform observer can both see one allocation. The
+ * core announces itself first; the observer consumes the hint and the event
+ * carries the CORE's caller as site and the class attached. When nothing
+ * consumed the hint, the core reports the event itself:
+ *
+ * cbm_memev_hint(site, cls);
+ * block = backing_malloc(n);
+ * if (cbm_memev_hint_pending()) {
+ * cbm_memev_alloc(block, n, usable, NULL);
+ * }
+ *
+ * FREES HAVE NO HANDSHAKE, on purpose: a free event must be recorded BEFORE
+ * the block goes back to the allocator, or another thread can be handed the
+ * same address, record its allocation, and have it erased by the late free.
+ * Where an observer sees the backing allocator it reports frees; everywhere
+ * else the core reports them itself, first. */
+void cbm_memev_hint(void *site, int mem_class);
+bool cbm_memev_hint_pending(void);
+void cbm_memev_set_observer_installed(bool installed);
+bool cbm_memev_observer_installed(void);
+
+/* Load address of the main image, written into every report header: sites are
+ * raw addresses in a randomised process, and offline symbolisation needs the
+ * base they are relative to. 0 = unknown. */
+void cbm_memev_set_image_base(uintptr_t base);
+
+/* ── Merge points ─────────────────────────────────────────────────────
+ * Per-thread counters are pushed to the shared tables here: work-item end,
+ * thread end, before any reader. */
+void cbm_memev_flush_thread(void);
+
+/* This thread is ending: merge its counters and release its state now. POSIX
+ * does this from a pthread key destructor. A static MinGW link runs no FLS or
+ * key destructors at thread exit (the same reason compat_thread.c releases the
+ * mimalloc thread heap from a loader TLS callback, #581), so Windows threads
+ * left their 22 MB state behind -- 71 of them by the similarity pass on the
+ * Go corpus, then commit exhaustion (2026-09-17). Safe to call twice. */
+void cbm_memev_thread_end(void);
+
+/* A pipeline phase boundary. Blocks allocated from now on carry the new phase;
+ * the live set is snapshotted (retained-across-phases, duplicate content) and
+ * a dump is appended to CBM_MEMWASTE_OUT. Called after the workers joined. */
+void cbm_memev_phase(const char *label);
+
+/* ── Process role and exit ────────────────────────────────────────────
+ * The index worker leaves with _Exit (no exit handlers: multi-GB teardown is
+ * skipped), which also skips every at-exit writer -- the waste dump and clang's
+ * profile runtime. cbm_memev_process_exit writes both first; it is a no-op when
+ * neither lane is compiled in or active.
+ * cbm_memev_process_role names this process's execution-count profile
+ * <CBM_PROFILE_DIR>/<role>-<pid>.profraw, so the scaling lane can compare the
+ * worker's work without the daemon and CLI processes mixed in. */
+void cbm_memev_process_role(const char *role);
+void cbm_memev_process_exit(void);
+
+/* ── Access lane: what the compiler cannot see ─────────────────────────
+ * The access lane learns about reads and writes from compiler callbacks, and a
+ * library call is not compiled with them: a block filled by memcpy or read(2)
+ * would look never written. The library wrappers report those ranges here.
+ * While the allocator itself runs (a realloc copying the old block), its reads
+ * and writes are not the program's: the shims bracket every backing call that
+ * can copy. Both vanish outside the access lane. */
+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS
+void cbm_memev_access(const void *p, size_t n, bool store, void *pc);
+void cbm_memev_backing(int delta);
+#define CBM_MEMEV_ACCESS(p, n, store, pc) cbm_memev_access((p), (n), (store), (pc))
+#define CBM_MEMEV_BACKING(delta) cbm_memev_backing(delta)
+#else
+#define CBM_MEMEV_ACCESS(p, n, store, pc) ((void)0)
+#define CBM_MEMEV_BACKING(delta) ((void)0)
+#endif
+
+/* A wrapped library call (fopen, fread, opendir...) is running on this thread.
+ * Memory the library allocates for itself meanwhile -- a FILE buffer, a DIR
+ * stream -- is charged to `site` (the wrapper's caller) with CBM_MEMEV_LIBRARY:
+ * it is counted like any memory, but the access lane leaves it alone, because
+ * only the uninstrumented library ever reads it. Nestable; the outermost site
+ * wins. */
+void cbm_memev_library_enter(void *site);
+void cbm_memev_library_leave(void);
+void *cbm_memev_library_site(void);
+
+/* ── CPU work ─────────────────────────────────────────────────────────
+ * Work is COUNTED, never timed: a count is a function of the code and the
+ * input, so it can gate. Semantics of the counters per kind:
+ *
+ * kind bytes aux1 aux2
+ * memcpy..strncmp bytes processed - -
+ * read, write,
+ * pread, pwrite,
+ * fread, fwrite bytes transferred tiny calls (<512 B) -
+ * open, stat,
+ * fopen, opendir - failures -
+ * readdir - - -
+ * mutex - contended -
+ * parallel_for items imbalance items workers
+ * pool_ops events by workers imbalance events workers
+ * ht_get - misses -
+ * ht_set - table growths -
+ * dead_write bytes never read blocks -
+ * ct_arena,
+ * ct_dyn_array,
+ * ct_hash_table capacity bytes used bytes growths
+ *
+ * `repeats` counts the same work done again, where the answer cannot have
+ * changed: strlen of the same pointer holding the same bytes (recent, same
+ * thread), open/stat of the same path (process-wide), a hash-table lookup of
+ * the same key TEXT in the same table with no write to that table in between
+ * (recent, same thread). A reused buffer holding a different string is not a
+ * repeat.
+ *
+ * `peak` is the largest single note: the biggest call for byte work and I/O,
+ * and for containers the largest capacity one instance left unused. Containers
+ * report at every reset, so `bytes` and `aux1` sum over USES (a per-thread arena
+ * rewound for 28,000 files reports 28,000 times); `peak` is the memory figure. */
+typedef enum {
+ CBM_WORK_MEMCPY = 0,
+ CBM_WORK_MEMMOVE,
+ CBM_WORK_MEMSET,
+ CBM_WORK_MEMCMP,
+ CBM_WORK_STRLEN,
+ CBM_WORK_STRCMP,
+ CBM_WORK_STRNCMP,
+ CBM_WORK_READ,
+ CBM_WORK_WRITE,
+ CBM_WORK_PREAD,
+ CBM_WORK_PWRITE,
+ CBM_WORK_OPEN,
+ CBM_WORK_STAT,
+ CBM_WORK_FOPEN,
+ CBM_WORK_FREAD,
+ CBM_WORK_FWRITE,
+ CBM_WORK_OPENDIR,
+ CBM_WORK_READDIR,
+ CBM_WORK_MUTEX,
+ CBM_WORK_PARALLEL_FOR,
+ CBM_WORK_POOL_OPS,
+ CBM_WORK_HT_GET,
+ CBM_WORK_HT_SET,
+ CBM_WORK_DEAD_WRITE,
+ CBM_WORK_CT_ARENA,
+ CBM_WORK_CT_DYN_ARRAY,
+ CBM_WORK_CT_HASH_TABLE,
+ CBM_WORK_KIND_COUNT
+} cbm_work_kind_t;
+
+enum { CBM_WORK_TINY_IO = 512 };
+
+const char *cbm_work_kind_name(cbm_work_kind_t kind);
+void cbm_work_note(cbm_work_kind_t kind, void *site, uint64_t bytes, uint64_t aux1,
+ uint64_t aux2);
+void cbm_work_note_strlen(void *site, const char *s, size_t len);
+
+/* Events (allocations and work notes) this thread has produced so far. A pool
+ * compares its workers by this: a pool over N work SLOTS whose workers each pull
+ * their own items shows its real imbalance only in the work done. */
+uint64_t cbm_memev_thread_ops(void);
+
+#if defined(CBM_MEMWASTE) && CBM_MEMWASTE && !defined(_WIN32)
+/* The one counting lock behind every pthread_mutex_lock in the flavour
+ * (mem_override_libc.c): try first, count a contended acquisition at `site`,
+ * then block. cbm_mutex_lock passes its own caller as the site. */
+int cbm_memev_mutex_lock(void *pthread_mutex, void *site);
+#endif
+void cbm_work_note_path(cbm_work_kind_t kind, void *site, const char *path, bool failed);
+/* `generation` is the table's write counter: a lookup repeats only when the
+ * table has not been written since the same key was last looked up. */
+void cbm_work_note_ht(cbm_work_kind_t kind, void *site, const void *table, const char *key,
+ uint64_t generation, bool hit);
+
+/* Containers report capacity against use when they are freed or reset.
+ * `site` is where the container was created. */
+static inline void cbm_memev_container(cbm_work_kind_t kind, void *site, uint64_t capacity,
+ uint64_t used, uint64_t growths) {
+ cbm_work_note(kind, site, capacity, used, growths);
+}
+
+/* A dynamic array being freed: capacity against count, in bytes. The site is
+ * the function that freed it (the helper is never inlined, so its return
+ * address lands there). Growths are the doublings from 8 slots to `cap`. */
+void cbm_memev_da_note(size_t cap_bytes, size_t used_bytes, size_t cap_slots);
+
+/* ── Report ───────────────────────────────────────────────────────────*/
+typedef struct {
+ uint64_t allocs; /* alloc events seen */
+ uint64_t frees; /* free events matched to a tracked block */
+ uint64_t untracked_frees; /* frees of blocks the layer never saw */
+ uint64_t foreign_frees; /* frees of blocks another allocator made */
+ uint64_t reallocs;
+ uint64_t requested_bytes; /* cumulative */
+ uint64_t usable_bytes; /* cumulative */
+ uint64_t realloc_copy_bytes; /* bytes the allocator had to move */
+ uint64_t live_bytes; /* usable bytes outstanding now */
+ uint64_t live_blocks;
+ uint64_t raw_bytes; /* cumulative usable bytes never tagged by the core */
+ uint64_t sites; /* distinct allocation sites */
+ /* Records the fixed tables could not hold. A profiler that silently loses
+ * records produces false confidence; these make the blind spot a number. */
+ uint64_t site_table_full;
+ uint64_t pointer_table_full;
+ uint64_t work_table_full;
+} cbm_memev_totals_t;
+
+void cbm_memev_totals(cbm_memev_totals_t *out);
+
+typedef struct {
+ uintptr_t site;
+ uint64_t allocs;
+ uint64_t frees;
+ uint64_t reallocs;
+ uint64_t requested_bytes;
+ uint64_t usable_bytes;
+ uint64_t realloc_copy_bytes;
+ uint64_t live_bytes;
+ uint64_t live_blocks;
+ uint64_t short_lived;
+ uint64_t raw_bytes;
+ /* fill-pattern scan at free */
+ uint64_t scanned_blocks;
+ uint64_t never_written_bytes; /* bytes still holding the fill pattern */
+ uint64_t over_requested_bytes; /* requested beyond the last written byte */
+ uint64_t zero_untouched_bytes; /* calloc: trailing zero bytes (upper bound) */
+ /* phase snapshot (the most recent phase mark) */
+ uint64_t retained_bytes; /* live, allocated in an EARLIER phase */
+ uint64_t dup_bytes; /* live small blocks identical to another */
+ uint64_t dup_blocks;
+ /* access lane */
+ uint64_t acc_untouched_blocks;
+ uint64_t acc_untouched_bytes;
+ uint64_t acc_dead_blocks; /* written, never read */
+ uint64_t acc_dead_bytes;
+ uint64_t acc_uninit_reads; /* read before any write, not calloc */
+ uint64_t acc_idle_bytes; /* last accessed in an earlier phase than freed */
+ uint64_t acc_opaque_blocks; /* written (fill scan) with no callback: no verdict possible */
+} cbm_memev_site_t;
+
+size_t cbm_memev_sites(cbm_memev_site_t *out, size_t max);
+
+typedef struct {
+ uintptr_t site;
+ cbm_work_kind_t kind;
+ uint64_t calls;
+ uint64_t bytes;
+ uint64_t aux1;
+ uint64_t aux2;
+ uint64_t repeats;
+ uint64_t peak;
+} cbm_work_row_t;
+
+size_t cbm_work_rows(cbm_work_row_t *out, size_t max);
+
+/* Append the report as JSON lines to `path`: one header object, one object per
+ * allocation site, one per work row. Raw addresses; symbolisation is offline
+ * (scripts/memwaste-report.py). `why` labels the dump. */
+bool cbm_memev_dump(const char *path, const char *why);
+
+/* Test seams. Never called in production. */
+void cbm_memev_reset_for_tests(void);
+void cbm_memev_force_for_tests(bool on);
+/* Tests that hand the layer REAL memory turn the fill pattern and the duplicate
+ * scan on (reset turns fill off: most tests use synthetic addresses). */
+void cbm_memev_scan_for_tests(bool fill, bool dupes);
+
+#endif /* CBM_MEM_EVENTS_H */
diff --git a/src/foundation/mem_override_darwin.c b/src/foundation/mem_override_darwin.c
new file mode 100644
index 000000000..370e6428a
--- /dev/null
+++ b/src/foundation/mem_override_darwin.c
@@ -0,0 +1,267 @@
+/*
+ * mem_override_darwin.c — the macOS allocation observer of the waste sanitizer.
+ *
+ * WHY THIS FILE EXISTS
+ *
+ * Linux and Windows observe the malloc family through the linker's --wrap
+ * (mem_override_posix.c, mem_override_win.c). macOS ld has no --wrap, and the
+ * mimalloc global override is off there permanently: the two-level namespace
+ * splits this binary's free from the system libraries' malloc, and a pointer
+ * crossing that boundary aborts. So on macOS nothing saw raw malloc at all —
+ * the platform the project is developed on was the one platform that could not
+ * be measured.
+ *
+ * HOW IT WORKS
+ *
+ * A malloc ZONE that forwards every entry to the original default zone and
+ * reports the event. Forwarding is the whole safety argument: a block never
+ * changes allocator, so there is no boundary for it to cross. The zone claims
+ * ownership through size(), which is what routes free() back through it.
+ *
+ * Proven on 2026-09-17 inside the real indexer (Go corpus): no abort, a block
+ * allocated BEFORE installation frees cleanly after it, wall time 41.2 s ->
+ * 41.3 s, and 92.3 million raw allocations that nothing had ever counted.
+ *
+ * THREE TRAPS, each one hit during that proof:
+ * 1. malloc_default_zone() returns a VIRTUAL wrapper that dispatches to
+ * whatever is default at call time. Forwarding to it recurses forever once
+ * this zone is the default. The real slot 0 comes from
+ * malloc_get_all_zones().
+ * 2. Registration appends. The default is slot 0, and the only way in is the
+ * rotation jemalloc uses: unregister slot 0 (the last zone moves into it),
+ * re-register it (appended), until this zone is first.
+ * 3. free() arrives through free_definite_size, not through free.
+ *
+ * Compiled only into the `memwaste` flavour; dormant unless CBM_MEMWASTE=1.
+ */
+#if defined(__APPLE__) && defined(CBM_MEMWASTE) && CBM_MEMWASTE
+
+#include "foundation/mem_events.h"
+
+#include <mach-o/dyld.h>
+#include <mach-o/getsect.h>
+#include <mach/mach.h>
+#include <malloc/malloc.h>
+#include <pthread.h>
+#include <stdint.h>
+#include <string.h>
+
+enum { OBS_ZONE_VERSION = 9, OBS_MAX_ROTATIONS = 16, OBS_MAX_FRAMES = 24 };
+
+static malloc_zone_t *g_orig;
+static malloc_zone_t g_obs;
+static uintptr_t g_text_lo;
+static uintptr_t g_text_hi;
+
+/* The site of a raw allocation is the first return address inside OUR image:
+ * above the zone entry sit libmalloc's own frames, and often libc's (strdup,
+ * getpwuid), before the code of ours that caused the call. Frames are walked
+ * through the frame-pointer chain, bounded by this thread's stack, so a
+ * foreign or corrupt frame ends the walk instead of faulting. */
+static void *caller_site(void *frame, unsigned *flags) {
+ *flags = 0;
+ if (cbm_memev_hint_pending()) {
+ return NULL; /* the memory core already named the site */
+ }
+ void *library_caller = cbm_memev_library_site();
+ if (library_caller) {
+ *flags = CBM_MEMEV_LIBRARY; /* a FILE buffer, a DIR stream: the library's own */
+ return library_caller;
+ }
+ pthread_t self = pthread_self();
+ uintptr_t hi = (uintptr_t)pthread_get_stackaddr_np(self);
+ uintptr_t lo = hi - pthread_get_stacksize_np(self);
+ uintptr_t *fp = (uintptr_t *)frame;
+ void *first = NULL;
+ for (int depth = 0; depth < OBS_MAX_FRAMES; depth++) {
+ if ((uintptr_t)fp < lo || (uintptr_t)fp + (2 * sizeof(uintptr_t)) > hi ||
+ ((uintptr_t)fp & (sizeof(uintptr_t) - 1)) != 0) {
+ break;
+ }
+ uintptr_t ret = fp[1];
+ uintptr_t *next = (uintptr_t *)fp[0];
+ if (ret == 0) {
+ break;
+ }
+ if (!first) {
+ first = (void *)ret;
+ }
+ if (ret >= g_text_lo && ret < g_text_hi) {
+ return (void *)ret;
+ }
+ if (next <= fp) {
+ break; /* the chain must climb */
+ }
+ fp = next;
+ }
+ *flags = CBM_MEMEV_FOREIGN;
+ return first; /* a system library allocating for itself: observe, never fill */
+}
+
+static size_t obs_size(malloc_zone_t *z, const void *p) {
+ (void)z;
+ return g_orig->size(g_orig, p);
+}
+
+static void *obs_malloc(malloc_zone_t *z, size_t n) {
+ (void)z;
+ void *p = g_orig->malloc(g_orig, n);
+ if (p && cbm_memev_enabled()) {
+ unsigned flags;
+ void *site = caller_site(__builtin_frame_address(0), &flags);
+ cbm_memev_alloc_ex(p, n, g_orig->size(g_orig, p), site, flags);
+ }
+ return p;
+}
+
+static void *obs_calloc(malloc_zone_t *z, size_t count, size_t n) {
+ (void)z;
+ void *p = g_orig->calloc(g_orig, count, n);
+ if (p && cbm_memev_enabled()) {
+ unsigned flags;
+ void *site = caller_site(__builtin_frame_address(0), &flags);
+ cbm_memev_alloc_ex(p, count * n, g_orig->size(g_orig, p), site,
+ flags | CBM_MEMEV_ZEROED);
+ }
+ return p;
+}
+
+static void *obs_valloc(malloc_zone_t *z, size_t n) {
+ (void)z;
+ void *p = g_orig->valloc(g_orig, n);
+ if (p && cbm_memev_enabled()) {
+ unsigned flags;
+ void *site = caller_site(__builtin_frame_address(0), &flags);
+ cbm_memev_alloc_ex(p, n, g_orig->size(g_orig, p), site, flags);
+ }
+ return p;
+}
+
+static void *obs_memalign(malloc_zone_t *z, size_t align, size_t n) {
+ (void)z;
+ void *p = g_orig->memalign(g_orig, align, n);
+ if (p && cbm_memev_enabled()) {
+ unsigned flags;
+ void *site = caller_site(__builtin_frame_address(0), &flags);
+ cbm_memev_alloc_ex(p, n, g_orig->size(g_orig, p), site, flags);
+ }
+ return p;
+}
+
+static void *obs_realloc(malloc_zone_t *z, void *p, size_t n) {
+ (void)z;
+ unsigned flags = 0;
+ void *site = cbm_memev_enabled() ? caller_site(__builtin_frame_address(0), &flags) : NULL;
+ CBM_MEMEV_BACKING(1);
+ void *q = g_orig->realloc(g_orig, p, n);
+ CBM_MEMEV_BACKING(-1);
+ if (q && cbm_memev_enabled()) {
+ cbm_memev_realloc(p, q, n, g_orig->size(g_orig, q), site);
+ }
+ return q;
+}
+
+static void obs_free(malloc_zone_t *z, void *p) {
+ (void)z;
+ cbm_memev_free(p);
+ g_orig->free(g_orig, p);
+}
+
+static void obs_free_definite_size(malloc_zone_t *z, void *p, size_t n) {
+ (void)z;
+ cbm_memev_free(p);
+ if (g_orig->free_definite_size) {
+ g_orig->free_definite_size(g_orig, p, n);
+ } else {
+ g_orig->free(g_orig, p);
+ }
+}
+
+static unsigned obs_batch_malloc(malloc_zone_t *z, size_t size, void **results, unsigned count) {
+ (void)z;
+ unsigned got = g_orig->batch_malloc ? g_orig->batch_malloc(g_orig, size, results, count) : 0;
+ if (cbm_memev_enabled()) {
+ unsigned flags;
+ void *site = caller_site(__builtin_frame_address(0), &flags);
+ for (unsigned i = 0; i < got; i++) {
+ cbm_memev_alloc_ex(results[i], size, g_orig->size(g_orig, results[i]), site, flags);
+ }
+ }
+ return got;
+}
+
+static void obs_batch_free(malloc_zone_t *z, void **blocks, unsigned count) {
+ (void)z;
+ for (unsigned i = 0; i < count; i++) {
+ cbm_memev_free(blocks[i]);
+ }
+ if (g_orig->batch_free) {
+ g_orig->batch_free(g_orig, blocks, count);
+ }
+}
+
+static void obs_destroy(malloc_zone_t *z) {
+ (void)z;
+}
+
+static size_t obs_pressure_relief(malloc_zone_t *z, size_t goal) {
+ (void)z;
+ return g_orig->pressure_relief ? g_orig->pressure_relief(g_orig, goal) : 0;
+}
+
+static malloc_zone_t *slot_zero(void) {
+ vm_address_t *zones = NULL;
+ unsigned count = 0;
+ if (malloc_get_all_zones(mach_task_self(), NULL, &zones, &count) != KERN_SUCCESS ||
+ count == 0) {
+ return NULL;
+ }
+ return (malloc_zone_t *)zones[0];
+}
+
+__attribute__((constructor)) static void cbm_mem_observe_install(void) {
+ if (!cbm_memev_enabled()) {
+ return;
+ }
+ const struct mach_header_64 *hdr = (const struct mach_header_64 *)_dyld_get_image_header(0);
+ unsigned long text_size = 0;
+ uint8_t *text = hdr ? getsegmentdata(hdr, "__TEXT", &text_size) : NULL;
+ g_text_lo = (uintptr_t)text;
+ g_text_hi = g_text_lo + text_size;
+ cbm_memev_set_image_base((uintptr_t)hdr);
+
+ g_orig = slot_zero();
+ if (!g_orig) {
+ return;
+ }
+ memset(&g_obs, 0, sizeof(g_obs));
+ g_obs.size = obs_size;
+ g_obs.malloc = obs_malloc;
+ g_obs.calloc = obs_calloc;
+ g_obs.valloc = obs_valloc;
+ g_obs.free = obs_free;
+ g_obs.realloc = obs_realloc;
+ g_obs.destroy = obs_destroy;
+ g_obs.zone_name = "cbm_memwaste_observer";
+ g_obs.batch_malloc = obs_batch_malloc;
+ g_obs.batch_free = obs_batch_free;
+ g_obs.introspect = g_orig->introspect; /* leaks/heap keep working */
+ g_obs.version = OBS_ZONE_VERSION;
+ g_obs.memalign = obs_memalign;
+ g_obs.free_definite_size = obs_free_definite_size;
+ g_obs.pressure_relief = obs_pressure_relief;
+ malloc_zone_register(&g_obs);
+ for (int spin = 0; spin < OBS_MAX_ROTATIONS; spin++) {
+ malloc_zone_t *first = slot_zero();
+ if (!first || first == &g_obs) {
+ break;
+ }
+ malloc_zone_unregister(first);
+ malloc_zone_register(first);
+ }
+ cbm_memev_set_observer_installed(slot_zero() == &g_obs);
+}
+
+#else
+typedef int cbm_mem_override_darwin_unused_t;
+#endif
diff --git a/src/foundation/mem_override_libc.c b/src/foundation/mem_override_libc.c
new file mode 100644
index 000000000..b86b8e0d1
--- /dev/null
+++ b/src/foundation/mem_override_libc.c
@@ -0,0 +1,738 @@
+/*
+ * mem_override_libc.c — the CPU lane of the waste sanitizer: byte work and
+ * system calls, counted per call site.
+ *
+ * WHAT IT COUNTS. memcpy, memmove, memset, memcmp, strlen, strcmp, strncmp
+ * (calls and bytes), read/write/pread/pwrite/fread/fwrite (bytes and tiny
+ * calls), open/stat/fopen/opendir (failures and the same path again), readdir,
+ * and pthread_mutex_lock (contended acquisitions; POSIX only -- Windows locks
+ * are counted where cbm_mutex_lock takes them).
+ * Calls the compiler inlines (small constant-size copies) are not calls and are
+ * not counted: what is counted is exactly the work that reaches the library.
+ *
+ * HOW IT LISTENS, per platform
+ * Linux, Windows the linker's --wrap redirects every reference in OUR
+ * objects to __wrap_<name>; __real_<name> is the original.
+ * The flavour adds the flags (Makefile.cbm, MEMWASTE=1).
+ * macOS ld has no --wrap and dyld ignores __interpose tuples in the
+ * main executable (proven 2026-09-17). Instead this file
+ * DEFINES the symbols: references in our objects bind to the
+ * definitions in our own image, and each definition reaches
+ * libSystem's through dlsym(RTLD_NEXT). Calls made inside
+ * libSystem keep libSystem's own functions. Fortified builds
+ * call __memcpy_chk and friends, so those are defined too.
+ *
+ * The CALLER's return address is the site: this file is the observation point.
+ * Compiled only into the `memwaste` flavour; dormant unless CBM_MEMWASTE=1.
+ * Everything here forwards unconditionally: counting can only ever be skipped,
+ * never the work.
+ */
+#if defined(CBM_MEMWASTE) && CBM_MEMWASTE
+
+#include "foundation/mem_events.h"
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#if !defined(_WIN32)
+#include <pthread.h>
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((no_sanitize("coverage"))), apply_to = function)
+#endif
+
+#define SITE() __builtin_return_address(0)
+
+static inline void note_io(cbm_work_kind_t kind, void *site, long long got, size_t asked) {
+ if (!cbm_memev_enabled()) {
+ return;
+ }
+ uint64_t bytes = got > 0 ? (uint64_t)got : 0;
+ cbm_work_note(kind, site, bytes, (asked > 0 && asked < CBM_WORK_TINY_IO) ? 1 : 0, 0);
+}
+
+/* Access lane: a library call's reads and writes, charged to its caller. */
+#define ACC_LOAD(p, n) CBM_MEMEV_ACCESS((p), (n), false, SITE())
+#define ACC_STORE(p, n) CBM_MEMEV_ACCESS((p), (n), true, SITE())
+
+static inline void note_bytes(cbm_work_kind_t kind, void *site, size_t n) {
+ if (cbm_memev_enabled()) {
+ cbm_work_note(kind, site, n, 0, 0);
+ }
+}
+
+/* ══════════════════════════════════════════════════════════════════════ */
+#if defined(__APPLE__)
+
+#include <dirent.h>
+#include <dlfcn.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#undef memcpy
+#undef memmove
+#undef memset
+#undef strlen
+#undef strcmp
+#undef strncmp
+#undef memcmp
+
+/* Resolved in a constructor. Until then the byte functions run a plain loop
+ * (dlsym and the dynamic loader call them before any constructor), marked
+ * no_builtin so the optimiser cannot turn the loop back into a call to itself. */
+static void *(*r_memcpy)(void *, const void *, size_t);
+static void *(*r_memmove)(void *, const void *, size_t);
+static void *(*r_memset)(void *, int, size_t);
+static int (*r_memcmp)(const void *, const void *, size_t);
+static size_t (*r_strlen)(const char *);
+static int (*r_strcmp)(const char *, const char *);
+static int (*r_strncmp)(const char *, const char *, size_t);
+static void *(*r_memcpy_chk)(void *, const void *, size_t, size_t);
+static void *(*r_memmove_chk)(void *, const void *, size_t, size_t);
+static void *(*r_memset_chk)(void *, int, size_t, size_t);
+static ssize_t (*r_read)(int, void *, size_t);
+static ssize_t (*r_write)(int, const void *, size_t);
+static ssize_t (*r_pread)(int, void *, size_t, off_t);
+static ssize_t (*r_pwrite)(int, const void *, size_t, off_t);
+static int (*r_open)(const char *, int, ...);
+static int (*r_stat)(const char *, struct stat *);
+static int (*r_lstat)(const char *, struct stat *);
+static FILE *(*r_fopen)(const char *, const char *);
+static size_t (*r_fread)(void *, size_t, size_t, FILE *);
+static size_t (*r_fwrite)(const void *, size_t, size_t, FILE *);
+static DIR *(*r_opendir)(const char *);
+static struct dirent *(*r_readdir)(DIR *);
+static int (*r_mutex_lock)(pthread_mutex_t *);
+static int (*r_mutex_trylock)(pthread_mutex_t *);
+
+__attribute__((constructor(101))) static void libc_resolve(void) {
+ r_memcpy = dlsym(RTLD_NEXT, "memcpy");
+ r_memmove = dlsym(RTLD_NEXT, "memmove");
+ r_memset = dlsym(RTLD_NEXT, "memset");
+ r_memcmp = dlsym(RTLD_NEXT, "memcmp");
+ r_strlen = dlsym(RTLD_NEXT, "strlen");
+ r_strcmp = dlsym(RTLD_NEXT, "strcmp");
+ r_strncmp = dlsym(RTLD_NEXT, "strncmp");
+ r_memcpy_chk = dlsym(RTLD_NEXT, "__memcpy_chk");
+ r_memmove_chk = dlsym(RTLD_NEXT, "__memmove_chk");
+ r_memset_chk = dlsym(RTLD_NEXT, "__memset_chk");
+ r_read = dlsym(RTLD_NEXT, "read");
+ r_write = dlsym(RTLD_NEXT, "write");
+ r_pread = dlsym(RTLD_NEXT, "pread");
+ r_pwrite = dlsym(RTLD_NEXT, "pwrite");
+ r_open = dlsym(RTLD_NEXT, "open");
+ r_stat = dlsym(RTLD_NEXT, "stat");
+ r_lstat = dlsym(RTLD_NEXT, "lstat");
+ r_fopen = dlsym(RTLD_NEXT, "fopen");
+ r_fread = dlsym(RTLD_NEXT, "fread");
+ r_fwrite = dlsym(RTLD_NEXT, "fwrite");
+ r_opendir = dlsym(RTLD_NEXT, "opendir");
+ r_readdir = dlsym(RTLD_NEXT, "readdir");
+ r_mutex_lock = dlsym(RTLD_NEXT, "pthread_mutex_lock");
+ r_mutex_trylock = dlsym(RTLD_NEXT, "pthread_mutex_trylock");
+}
+
+/* System calls are only reachable after libc_resolve; a call from an earlier
+ * constructor resolves its own target once. */
+#define RESOLVED(ptr, name) ((ptr) ? (ptr) : ((ptr) = dlsym(RTLD_NEXT, name)))
+
+__attribute__((no_builtin("memcpy", "memmove", "memset")))
+static void *slow_move(void *d, const void *s, size_t n) {
+ unsigned char *dd = (unsigned char *)d;
+ const unsigned char *ss = (const unsigned char *)s;
+ if (dd < ss) {
+ for (size_t i = 0; i < n; i++) {
+ dd[i] = ss[i];
+ }
+ } else {
+ for (size_t i = n; i > 0; i--) {
+ dd[i - 1] = ss[i - 1];
+ }
+ }
+ return d;
+}
+
+__attribute__((no_builtin("memset")))
+static void *slow_set(void *d, int c, size_t n) {
+ unsigned char *dd = (unsigned char *)d;
+ for (size_t i = 0; i < n; i++) {
+ dd[i] = (unsigned char)c;
+ }
+ return d;
+}
+
+__attribute__((no_builtin("strlen")))
+static size_t slow_strlen(const char *s) {
+ size_t i = 0;
+ while (s[i]) {
+ i++;
+ }
+ return i;
+}
+
+__attribute__((no_builtin("memcmp", "strcmp", "strncmp")))
+static int slow_ncmp(const char *a, const char *b, size_t n, bool stop_at_nul) {
+ for (size_t i = 0; i < n; i++) {
+ unsigned char x = (unsigned char)a[i];
+ unsigned char y = (unsigned char)b[i];
+ if (x != y) {
+ return x < y ? -1 : 1;
+ }
+ if (stop_at_nul && x == 0) {
+ return 0;
+ }
+ }
+ return 0;
+}
+
+void *(memcpy)(void *restrict d, const void *restrict s, size_t n) {
+ note_bytes(CBM_WORK_MEMCPY, SITE(), n);
+ ACC_LOAD(s, n);
+ ACC_STORE(d, n);
+ return r_memcpy ? r_memcpy(d, s, n) : slow_move(d, s, n);
+}
+
+void *(memmove)(void *d, const void *s, size_t n) {
+ note_bytes(CBM_WORK_MEMMOVE, SITE(), n);
+ ACC_LOAD(s, n);
+ ACC_STORE(d, n);
+ return r_memmove ? r_memmove(d, s, n) : slow_move(d, s, n);
+}
+
+void *(memset)(void *d, int c, size_t n) {
+ note_bytes(CBM_WORK_MEMSET, SITE(), n);
+ ACC_STORE(d, n);
+ return r_memset ? r_memset(d, c, n) : slow_set(d, c, n);
+}
+
+int (memcmp)(const void *a, const void *b, size_t n) {
+ note_bytes(CBM_WORK_MEMCMP, SITE(), n);
+ ACC_LOAD(a, n);
+ ACC_LOAD(b, n);
+ return r_memcmp ? r_memcmp(a, b, n) : slow_ncmp((const char *)a, (const char *)b, n, false);
+}
+
+size_t (strlen)(const char *s) {
+ size_t n = r_strlen ? r_strlen(s) : slow_strlen(s);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_strlen(SITE(), s, n);
+ ACC_LOAD(s, n + 1);
+ }
+ return n;
+}
+
+int (strcmp)(const char *a, const char *b) {
+ note_bytes(CBM_WORK_STRCMP, SITE(), 0);
+ ACC_LOAD(a, 1);
+ ACC_LOAD(b, 1);
+ return r_strcmp ? r_strcmp(a, b) : slow_ncmp(a, b, (size_t)-1, true);
+}
+
+int (strncmp)(const char *a, const char *b, size_t n) {
+ note_bytes(CBM_WORK_STRNCMP, SITE(), n);
+ ACC_LOAD(a, 1);
+ ACC_LOAD(b, 1);
+ return r_strncmp ? r_strncmp(a, b, n) : slow_ncmp(a, b, n, true);
+}
+
+void *__memcpy_chk(void *d, const void *s, size_t n, size_t dn) {
+ note_bytes(CBM_WORK_MEMCPY, SITE(), n);
+ ACC_LOAD(s, n);
+ ACC_STORE(d, n);
+ return RESOLVED(r_memcpy_chk, "__memcpy_chk")(d, s, n, dn);
+}
+
+void *__memmove_chk(void *d, const void *s, size_t n, size_t dn) {
+ note_bytes(CBM_WORK_MEMMOVE, SITE(), n);
+ ACC_LOAD(s, n);
+ ACC_STORE(d, n);
+ return RESOLVED(r_memmove_chk, "__memmove_chk")(d, s, n, dn);
+}
+
+void *__memset_chk(void *d, int c, size_t n, size_t dn) {
+ note_bytes(CBM_WORK_MEMSET, SITE(), n);
+ ACC_STORE(d, n);
+ return RESOLVED(r_memset_chk, "__memset_chk")(d, c, n, dn);
+}
+
+ssize_t read(int fd, void *buf, size_t n) {
+ ssize_t got = RESOLVED(r_read, "read")(fd, buf, n);
+ note_io(CBM_WORK_READ, SITE(), (long long)got, n);
+ ACC_STORE(buf, got > 0 ? (size_t)got : 0);
+ return got;
+}
+
+ssize_t write(int fd, const void *buf, size_t n) {
+ ssize_t put = RESOLVED(r_write, "write")(fd, buf, n);
+ note_io(CBM_WORK_WRITE, SITE(), (long long)put, n);
+ ACC_LOAD(buf, put > 0 ? (size_t)put : 0);
+ return put;
+}
+
+ssize_t pread(int fd, void *buf, size_t n, off_t off) {
+ ssize_t got = RESOLVED(r_pread, "pread")(fd, buf, n, off);
+ note_io(CBM_WORK_PREAD, SITE(), (long long)got, n);
+ ACC_STORE(buf, got > 0 ? (size_t)got : 0);
+ return got;
+}
+
+ssize_t pwrite(int fd, const void *buf, size_t n, off_t off) {
+ ssize_t put = RESOLVED(r_pwrite, "pwrite")(fd, buf, n, off);
+ note_io(CBM_WORK_PWRITE, SITE(), (long long)put, n);
+ ACC_LOAD(buf, put > 0 ? (size_t)put : 0);
+ return put;
+}
+
+int open(const char *path, int flags, ...) {
+ int mode = 0;
+ if (flags & O_CREAT) {
+ va_list ap;
+ va_start(ap, flags);
+ mode = va_arg(ap, int);
+ va_end(ap);
+ }
+ int fd = RESOLVED(r_open, "open")(path, flags, mode);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_OPEN, SITE(), path, fd < 0);
+ }
+ return fd;
+}
+
+int stat(const char *path, struct stat *st) {
+ int rc = RESOLVED(r_stat, "stat")(path, st);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_STAT, SITE(), path, rc != 0);
+ }
+ return rc;
+}
+
+int lstat(const char *path, struct stat *st) {
+ int rc = RESOLVED(r_lstat, "lstat")(path, st);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_STAT, SITE(), path, rc != 0);
+ }
+ return rc;
+}
+
+FILE *fopen(const char *restrict path, const char *restrict mode) {
+ cbm_memev_library_enter(SITE());
+ FILE *f = RESOLVED(r_fopen, "fopen")(path, mode);
+ cbm_memev_library_leave();
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_FOPEN, SITE(), path, f == NULL);
+ }
+ return f;
+}
+
+size_t fread(void *restrict buf, size_t size, size_t count, FILE *restrict f) {
+ cbm_memev_library_enter(SITE());
+ size_t got = RESOLVED(r_fread, "fread")(buf, size, count, f);
+ cbm_memev_library_leave();
+ note_io(CBM_WORK_FREAD, SITE(), (long long)(got * size), size * count);
+ ACC_STORE(buf, got * size);
+ return got;
+}
+
+size_t fwrite(const void *restrict buf, size_t size, size_t count, FILE *restrict f) {
+ cbm_memev_library_enter(SITE());
+ size_t put = RESOLVED(r_fwrite, "fwrite")(buf, size, count, f);
+ cbm_memev_library_leave();
+ note_io(CBM_WORK_FWRITE, SITE(), (long long)(put * size), size * count);
+ ACC_LOAD(buf, put * size);
+ return put;
+}
+
+DIR *opendir(const char *path) {
+ cbm_memev_library_enter(SITE());
+ DIR *d = RESOLVED(r_opendir, "opendir")(path);
+ cbm_memev_library_leave();
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_OPENDIR, SITE(), path, d == NULL);
+ }
+ return d;
+}
+
+struct dirent *readdir(DIR *d) {
+ note_bytes(CBM_WORK_READDIR, SITE(), 0);
+ cbm_memev_library_enter(SITE());
+ struct dirent *ent = RESOLVED(r_readdir, "readdir")(d);
+ cbm_memev_library_leave();
+ return ent;
+}
+
+int cbm_memev_mutex_lock(void *pthread_mutex, void *site) {
+ pthread_mutex_t *m = (pthread_mutex_t *)pthread_mutex;
+ int (*lock)(pthread_mutex_t *) = RESOLVED(r_mutex_lock, "pthread_mutex_lock");
+ if (!cbm_memev_enabled()) {
+ return lock(m);
+ }
+ int rc = RESOLVED(r_mutex_trylock, "pthread_mutex_trylock")(m);
+ bool contended = rc == EBUSY;
+ if (contended) {
+ rc = lock(m);
+ }
+ cbm_work_note(CBM_WORK_MUTEX, site, 0, contended ? 1 : 0, 0);
+ return rc;
+}
+
+int pthread_mutex_lock(pthread_mutex_t *m) {
+ return cbm_memev_mutex_lock(m, SITE());
+}
+
+/* ══════════════════════════════════════════════════════════════════════ */
+#elif defined(__linux__)
+
+#include <dirent.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+void *__real_memcpy(void *, const void *, size_t);
+void *__real_memmove(void *, const void *, size_t);
+void *__real_memset(void *, int, size_t);
+int __real_memcmp(const void *, const void *, size_t);
+size_t __real_strlen(const char *);
+int __real_strcmp(const char *, const char *);
+int __real_strncmp(const char *, const char *, size_t);
+void *__real___memcpy_chk(void *, const void *, size_t, size_t);
+void *__real___memmove_chk(void *, const void *, size_t, size_t);
+void *__real___memset_chk(void *, int, size_t, size_t);
+ssize_t __real_read(int, void *, size_t);
+ssize_t __real___read_chk(int, void *, size_t, size_t);
+ssize_t __real_write(int, const void *, size_t);
+ssize_t __real_pread(int, void *, size_t, off_t);
+ssize_t __real_pwrite(int, const void *, size_t, off_t);
+ssize_t __real_pread64(int, void *, size_t, off_t);
+ssize_t __real_pwrite64(int, const void *, size_t, off_t);
+int __real_open(const char *, int, ...);
+int __real_open64(const char *, int, ...);
+int __real___open_2(const char *, int);
+int __real___open64_2(const char *, int);
+int __real_stat(const char *, struct stat *);
+int __real_lstat(const char *, struct stat *);
+FILE *__real_fopen(const char *, const char *);
+FILE *__real_fopen64(const char *, const char *);
+size_t __real_fread(void *, size_t, size_t, FILE *);
+size_t __real___fread_chk(void *, size_t, size_t, size_t, FILE *);
+size_t __real_fwrite(const void *, size_t, size_t, FILE *);
+DIR *__real_opendir(const char *);
+struct dirent *__real_readdir(DIR *);
+int __real_pthread_mutex_lock(pthread_mutex_t *);
+
+void *__wrap_memcpy(void *d, const void *s, size_t n) {
+ note_bytes(CBM_WORK_MEMCPY, SITE(), n);
+ ACC_LOAD(s, n);
+ ACC_STORE(d, n);
+ return __real_memcpy(d, s, n);
+}
+void *__wrap_memmove(void *d, const void *s, size_t n) {
+ note_bytes(CBM_WORK_MEMMOVE, SITE(), n);
+ ACC_LOAD(s, n);
+ ACC_STORE(d, n);
+ return __real_memmove(d, s, n);
+}
+void *__wrap_memset(void *d, int c, size_t n) {
+ note_bytes(CBM_WORK_MEMSET, SITE(), n);
+ ACC_STORE(d, n);
+ return __real_memset(d, c, n);
+}
+int __wrap_memcmp(const void *a, const void *b, size_t n) {
+ note_bytes(CBM_WORK_MEMCMP, SITE(), n);
+ ACC_LOAD(a, n);
+ ACC_LOAD(b, n);
+ return __real_memcmp(a, b, n);
+}
+size_t __wrap_strlen(const char *s) {
+ size_t n = __real_strlen(s);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_strlen(SITE(), s, n);
+ ACC_LOAD(s, n + 1);
+ }
+ return n;
+}
+int __wrap_strcmp(const char *a, const char *b) {
+ note_bytes(CBM_WORK_STRCMP, SITE(), 0);
+ ACC_LOAD(a, 1);
+ ACC_LOAD(b, 1);
+ return __real_strcmp(a, b);
+}
+int __wrap_strncmp(const char *a, const char *b, size_t n) {
+ note_bytes(CBM_WORK_STRNCMP, SITE(), n);
+ ACC_LOAD(a, 1);
+ ACC_LOAD(b, 1);
+ return __real_strncmp(a, b, n);
+}
+void *__wrap___memcpy_chk(void *d, const void *s, size_t n, size_t dn) {
+ note_bytes(CBM_WORK_MEMCPY, SITE(), n);
+ ACC_LOAD(s, n);
+ ACC_STORE(d, n);
+ return __real___memcpy_chk(d, s, n, dn);
+}
+void *__wrap___memmove_chk(void *d, const void *s, size_t n, size_t dn) {
+ note_bytes(CBM_WORK_MEMMOVE, SITE(), n);
+ ACC_LOAD(s, n);
+ ACC_STORE(d, n);
+ return __real___memmove_chk(d, s, n, dn);
+}
+void *__wrap___memset_chk(void *d, int c, size_t n, size_t dn) {
+ note_bytes(CBM_WORK_MEMSET, SITE(), n);
+ ACC_STORE(d, n);
+ return __real___memset_chk(d, c, n, dn);
+}
+ssize_t __wrap_read(int fd, void *buf, size_t n) {
+ ssize_t got = __real_read(fd, buf, n);
+ note_io(CBM_WORK_READ, SITE(), (long long)got, n);
+ ACC_STORE(buf, got > 0 ? (size_t)got : 0);
+ return got;
+}
+ssize_t __wrap___read_chk(int fd, void *buf, size_t n, size_t bn) {
+ ssize_t got = __real___read_chk(fd, buf, n, bn);
+ note_io(CBM_WORK_READ, SITE(), (long long)got, n);
+ ACC_STORE(buf, got > 0 ? (size_t)got : 0);
+ return got;
+}
+ssize_t __wrap_write(int fd, const void *buf, size_t n) {
+ ssize_t put = __real_write(fd, buf, n);
+ note_io(CBM_WORK_WRITE, SITE(), (long long)put, n);
+ ACC_LOAD(buf, put > 0 ? (size_t)put : 0);
+ return put;
+}
+ssize_t __wrap_pread(int fd, void *buf, size_t n, off_t off) {
+ ssize_t got = __real_pread(fd, buf, n, off);
+ note_io(CBM_WORK_PREAD, SITE(), (long long)got, n);
+ ACC_STORE(buf, got > 0 ? (size_t)got : 0);
+ return got;
+}
+ssize_t __wrap_pwrite(int fd, const void *buf, size_t n, off_t off) {
+ ssize_t put = __real_pwrite(fd, buf, n, off);
+ note_io(CBM_WORK_PWRITE, SITE(), (long long)put, n);
+ ACC_LOAD(buf, put > 0 ? (size_t)put : 0);
+ return put;
+}
+ssize_t __wrap_pread64(int fd, void *buf, size_t n, off_t off) {
+ ssize_t got = __real_pread64(fd, buf, n, off);
+ note_io(CBM_WORK_PREAD, SITE(), (long long)got, n);
+ ACC_STORE(buf, got > 0 ? (size_t)got : 0);
+ return got;
+}
+ssize_t __wrap_pwrite64(int fd, const void *buf, size_t n, off_t off) {
+ ssize_t put = __real_pwrite64(fd, buf, n, off);
+ note_io(CBM_WORK_PWRITE, SITE(), (long long)put, n);
+ ACC_LOAD(buf, put > 0 ? (size_t)put : 0);
+ return put;
+}
+static int open_mode(int flags, va_list ap) {
+ return (flags & O_CREAT) ? va_arg(ap, int) : 0;
+}
+int __wrap_open(const char *path, int flags, ...) {
+ va_list ap;
+ va_start(ap, flags);
+ int mode = open_mode(flags, ap);
+ va_end(ap);
+ int fd = __real_open(path, flags, mode);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_OPEN, SITE(), path, fd < 0);
+ }
+ return fd;
+}
+int __wrap_open64(const char *path, int flags, ...) {
+ va_list ap;
+ va_start(ap, flags);
+ int mode = open_mode(flags, ap);
+ va_end(ap);
+ int fd = __real_open64(path, flags, mode);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_OPEN, SITE(), path, fd < 0);
+ }
+ return fd;
+}
+int __wrap___open_2(const char *path, int flags) {
+ int fd = __real___open_2(path, flags);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_OPEN, SITE(), path, fd < 0);
+ }
+ return fd;
+}
+int __wrap___open64_2(const char *path, int flags) {
+ int fd = __real___open64_2(path, flags);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_OPEN, SITE(), path, fd < 0);
+ }
+ return fd;
+}
+int __wrap_stat(const char *path, struct stat *st) {
+ int rc = __real_stat(path, st);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_STAT, SITE(), path, rc != 0);
+ }
+ return rc;
+}
+int __wrap_lstat(const char *path, struct stat *st) {
+ int rc = __real_lstat(path, st);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_STAT, SITE(), path, rc != 0);
+ }
+ return rc;
+}
+FILE *__wrap_fopen(const char *path, const char *mode) {
+ FILE *f = __real_fopen(path, mode);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_FOPEN, SITE(), path, f == NULL);
+ }
+ return f;
+}
+FILE *__wrap_fopen64(const char *path, const char *mode) {
+ FILE *f = __real_fopen64(path, mode);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_FOPEN, SITE(), path, f == NULL);
+ }
+ return f;
+}
+size_t __wrap_fread(void *buf, size_t size, size_t count, FILE *f) {
+ size_t got = __real_fread(buf, size, count, f);
+ note_io(CBM_WORK_FREAD, SITE(), (long long)(got * size), size * count);
+ ACC_STORE(buf, got * size);
+ return got;
+}
+size_t __wrap___fread_chk(void *buf, size_t bn, size_t size, size_t count, FILE *f) {
+ size_t got = __real___fread_chk(buf, bn, size, count, f);
+ note_io(CBM_WORK_FREAD, SITE(), (long long)(got * size), size * count);
+ ACC_STORE(buf, got * size);
+ return got;
+}
+size_t __wrap_fwrite(const void *buf, size_t size, size_t count, FILE *f) {
+ size_t put = __real_fwrite(buf, size, count, f);
+ note_io(CBM_WORK_FWRITE, SITE(), (long long)(put * size), size * count);
+ ACC_LOAD(buf, put * size);
+ return put;
+}
+DIR *__wrap_opendir(const char *path) {
+ DIR *d = __real_opendir(path);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_path(CBM_WORK_OPENDIR, SITE(), path, d == NULL);
+ }
+ return d;
+}
+struct dirent *__wrap_readdir(DIR *d) {
+ note_bytes(CBM_WORK_READDIR, SITE(), 0);
+ return __real_readdir(d);
+}
+
+int cbm_memev_mutex_lock(void *pthread_mutex, void *site) {
+ pthread_mutex_t *m = (pthread_mutex_t *)pthread_mutex;
+ if (!cbm_memev_enabled()) {
+ return __real_pthread_mutex_lock(m);
+ }
+ int rc = pthread_mutex_trylock(m);
+ bool contended = rc == EBUSY;
+ if (contended) {
+ rc = __real_pthread_mutex_lock(m);
+ }
+ cbm_work_note(CBM_WORK_MUTEX, site, 0, contended ? 1 : 0, 0);
+ return rc;
+}
+
+int __wrap_pthread_mutex_lock(pthread_mutex_t *m) {
+ return cbm_memev_mutex_lock(m, SITE());
+}
+
+/* ══════════════════════════════════════════════════════════════════════ */
+#elif defined(_WIN32)
+
+#include <wchar.h>
+
+void *__real_memcpy(void *, const void *, size_t);
+void *__real_memmove(void *, const void *, size_t);
+void *__real_memset(void *, int, size_t);
+int __real_memcmp(const void *, const void *, size_t);
+size_t __real_strlen(const char *);
+int __real_strcmp(const char *, const char *);
+int __real_strncmp(const char *, const char *, size_t);
+FILE *__real__wfopen(const wchar_t *, const wchar_t *);
+size_t __real_fread(void *, size_t, size_t, FILE *);
+size_t __real_fwrite(const void *, size_t, size_t, FILE *);
+
+void *__wrap_memcpy(void *d, const void *s, size_t n) {
+ note_bytes(CBM_WORK_MEMCPY, SITE(), n);
+ ACC_LOAD(s, n);
+ ACC_STORE(d, n);
+ return __real_memcpy(d, s, n);
+}
+void *__wrap_memmove(void *d, const void *s, size_t n) {
+ note_bytes(CBM_WORK_MEMMOVE, SITE(), n);
+ ACC_LOAD(s, n);
+ ACC_STORE(d, n);
+ return __real_memmove(d, s, n);
+}
+void *__wrap_memset(void *d, int c, size_t n) {
+ note_bytes(CBM_WORK_MEMSET, SITE(), n);
+ ACC_STORE(d, n);
+ return __real_memset(d, c, n);
+}
+int __wrap_memcmp(const void *a, const void *b, size_t n) {
+ note_bytes(CBM_WORK_MEMCMP, SITE(), n);
+ ACC_LOAD(a, n);
+ ACC_LOAD(b, n);
+ return __real_memcmp(a, b, n);
+}
+size_t __wrap_strlen(const char *s) {
+ size_t n = __real_strlen(s);
+ if (cbm_memev_enabled()) {
+ cbm_work_note_strlen(SITE(), s, n);
+ ACC_LOAD(s, n + 1);
+ }
+ return n;
+}
+int __wrap_strcmp(const char *a, const char *b) {
+ note_bytes(CBM_WORK_STRCMP, SITE(), 0);
+ ACC_LOAD(a, 1);
+ ACC_LOAD(b, 1);
+ return __real_strcmp(a, b);
+}
+int __wrap_strncmp(const char *a, const char *b, size_t n) {
+ note_bytes(CBM_WORK_STRNCMP, SITE(), n);
+ ACC_LOAD(a, 1);
+ ACC_LOAD(b, 1);
+ return __real_strncmp(a, b, n);
+}
+/* A wide path's hash is taken over its UTF-16 units squeezed to bytes: equal
+ * paths hash equal, which is all the repeat detector needs. */
+FILE *__wrap__wfopen(const wchar_t *path, const wchar_t *mode) {
+ FILE *f = __real__wfopen(path, mode);
+ if (cbm_memev_enabled()) {
+ char narrow[1024];
+ size_t i = 0;
+ for (; path && path[i] && i + 1 < sizeof(narrow); i++) {
+ narrow[i] = (char)(path[i] & 0x7F ? path[i] : '?');
+ }
+ narrow[i] = '\0';
+ cbm_work_note_path(CBM_WORK_FOPEN, SITE(), narrow, f == NULL);
+ }
+ return f;
+}
+size_t __wrap_fread(void *buf, size_t size, size_t count, FILE *f) {
+ size_t got = __real_fread(buf, size, count, f);
+ note_io(CBM_WORK_FREAD, SITE(), (long long)(got * size), size * count);
+ ACC_STORE(buf, got * size);
+ return got;
+}
+size_t __wrap_fwrite(const void *buf, size_t size, size_t count, FILE *f) {
+ size_t put = __real_fwrite(buf, size, count, f);
+ note_io(CBM_WORK_FWRITE, SITE(), (long long)(put * size), size * count);
+ ACC_LOAD(buf, put * size);
+ return put;
+}
+
+#endif /* platform */
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#endif
+
+#else
+typedef int cbm_mem_override_libc_unused_t;
+#endif /* CBM_MEMWASTE */
diff --git a/src/foundation/mem_override_posix.c b/src/foundation/mem_override_posix.c
index 008d8010d..af340fd23 100644
--- a/src/foundation/mem_override_posix.c
+++ b/src/foundation/mem_override_posix.c
@@ -32,36 +32,61 @@
* and a difference in the output is a difference in behaviour rather than an
* artifact of two different instrumentation schemes.
*
- * Linux only: --wrap is a GNU ld/lld feature. macOS ld has no equivalent, so
- * macOS runs census-only and the profiler reports nothing there. That gap is
- * stated in the harness spec rather than hidden behind zeros that look like
- * "no allocations".
+ * Linux only: --wrap is a GNU ld/lld feature. macOS ld has no equivalent; there
+ * the waste layer observes through a forwarding malloc zone instead
+ * (mem_override_darwin.c), so all three platforms report the same site tables.
*/
-#include "foundation/mem_profile.h"
-
#include <mimalloc.h>
#include <stddef.h>
+#include <string.h>
#if defined(__linux__)
+/* Waste sanitizer (mem_events.h). The wrapper IS the observation point, so the
+ * caller's return address is the allocation site. Compiled to nothing outside
+ * the `memwaste` flavour. Frees are reported BEFORE the block goes back. */
+#if defined(CBM_MEMWASTE) && CBM_MEMWASTE
+#include "foundation/mem_events.h"
+#define OBS_ALLOC_F(block, requested, flags) \
+ do { \
+ if ((block) && cbm_memev_enabled()) { \
+ cbm_memev_alloc_ex((block), (requested), mi_usable_size(block), \
+ __builtin_return_address(0), (unsigned)(flags)); \
+ } \
+ } while (0)
+#define OBS_ALLOC(block, requested) OBS_ALLOC_F((block), (requested), 0)
+#define OBS_REALLOC(old_block, grown, requested) \
+ do { \
+ if ((grown) && cbm_memev_enabled()) { \
+ cbm_memev_realloc((old_block), (grown), (requested), mi_usable_size(grown), \
+ __builtin_return_address(0)); \
+ } \
+ } while (0)
+#define OBS_FREE(block) cbm_memev_free(block)
+#else
+#define OBS_ALLOC(block, requested) ((void)0)
+#define OBS_ALLOC_F(block, requested, flags) ((void)0)
+#define OBS_REALLOC(old_block, grown, requested) ((void)0)
+#define OBS_FREE(block) ((void)0)
+#endif
+
void *__wrap_malloc(size_t size) {
void *block = mi_malloc(size);
- cbm_mem_profile_alloc(block, size);
+ OBS_ALLOC(block, size);
return block;
}
void *__wrap_calloc(size_t count, size_t size) {
void *block = mi_calloc(count, size);
- cbm_mem_profile_alloc(block, count * size);
+ OBS_ALLOC_F(block, count * size, CBM_MEMEV_ZEROED);
return block;
}
void *__wrap_realloc(void *block, size_t size) {
- if (block) {
- cbm_mem_profile_free(block);
- }
+ CBM_MEMEV_BACKING(1);
void *grown = mi_realloc(block, size);
- cbm_mem_profile_alloc(grown, size);
+ CBM_MEMEV_BACKING(-1);
+ OBS_REALLOC(block, grown, size);
return grown;
}
@@ -71,16 +96,43 @@ void __wrap_free(void *block) {
}
/* Unlike Windows there is no foreign-allocator hazard to route around:
* every malloc in this image is already mimalloc's. */
- cbm_mem_profile_free(block);
+ OBS_FREE(block);
mi_free(block);
}
char *__wrap_strdup(const char *text) {
char *copy = mi_strdup(text);
- if (copy) {
- cbm_mem_profile_alloc(copy, mi_usable_size(copy));
- }
+ OBS_ALLOC_F(copy, copy ? strlen(copy) + 1 : 0, CBM_MEMEV_WRITTEN);
return copy;
}
+/* The aligned family and strndup reach mimalloc too; without these a block
+ * from posix_memalign (the slab allocator's pages) is invisible, and its
+ * free reads as untracked. */
+char *__wrap_strndup(const char *text, size_t length) {
+ char *copy = mi_strndup(text, length);
+ OBS_ALLOC_F(copy, copy ? strlen(copy) + 1 : 0, CBM_MEMEV_WRITTEN);
+ return copy;
+}
+
+int __wrap_posix_memalign(void **out, size_t alignment, size_t size) {
+ int rc = mi_posix_memalign(out, alignment, size);
+ if (rc == 0 && out) {
+ OBS_ALLOC(*out, size);
+ }
+ return rc;
+}
+
+void *__wrap_aligned_alloc(size_t alignment, size_t size) {
+ void *block = mi_aligned_alloc(alignment, size);
+ OBS_ALLOC(block, size);
+ return block;
+}
+
+void *__wrap_memalign(size_t alignment, size_t size) {
+ void *block = mi_memalign(alignment, size);
+ OBS_ALLOC(block, size);
+ return block;
+}
+
#endif /* __linux__ */
diff --git a/src/foundation/mem_override_win.c b/src/foundation/mem_override_win.c
index 8b51cb690..cf7e1a790 100644
--- a/src/foundation/mem_override_win.c
+++ b/src/foundation/mem_override_win.c
@@ -42,6 +42,7 @@
* mimalloc is always safe.
*/
#include "foundation/mem.h"
+#include "foundation/mem_events.h" /* CBM_MEMEV_BACKING: empty outside the access lane */
#include <mimalloc.h>
#include <stddef.h>
@@ -68,32 +69,73 @@ static inline bool mem_override_is_ours(const void *block) {
return mi_is_in_heap_region(block);
}
+/* Waste sanitizer (mem_events.h). The wrapper IS the observation point, so the
+ * caller's return address is the allocation site. Compiled to nothing outside
+ * the `memwaste` flavour. Frees are reported BEFORE the block goes back. */
+#if defined(CBM_MEMWASTE) && CBM_MEMWASTE
+#define OBS_ALLOC_F(block, requested, flags) \
+ do { \
+ if ((block) && cbm_memev_enabled()) { \
+ cbm_memev_alloc_ex((block), (requested), mi_usable_size(block), \
+ __builtin_return_address(0), (unsigned)(flags)); \
+ } \
+ } while (0)
+#define OBS_ALLOC(block, requested) OBS_ALLOC_F((block), (requested), 0)
+#define OBS_REALLOC(old_block, grown, requested) \
+ do { \
+ if ((grown) && cbm_memev_enabled()) { \
+ cbm_memev_realloc((old_block), (grown), (requested), mi_usable_size(grown), \
+ __builtin_return_address(0)); \
+ } \
+ } while (0)
+/* A block mimalloc did not make came from the CRT or a system DLL: the layer
+ * could never have seen its allocation, so it is not an untracked free. */
+#define OBS_FREE(block) \
+ (mem_override_is_ours(block) ? cbm_memev_free(block) : cbm_memev_free_foreign(block))
+#else
+#define OBS_ALLOC(block, requested) ((void)0)
+#define OBS_ALLOC_F(block, requested, flags) ((void)0)
+#define OBS_REALLOC(old_block, grown, requested) ((void)0)
+#define OBS_FREE(block) ((void)0)
+#endif
+
void *__wrap_malloc(size_t size) {
- return mi_malloc(size);
+ void *block = mi_malloc(size);
+ OBS_ALLOC(block, size);
+ return block;
}
void *__wrap_calloc(size_t count, size_t size) {
- return mi_calloc(count, size);
+ void *block = mi_calloc(count, size);
+ OBS_ALLOC_F(block, count * size, CBM_MEMEV_ZEROED);
+ return block;
}
char *__wrap_strdup(const char *text) {
- return mi_strdup(text);
+ char *copy = mi_strdup(text);
+ OBS_ALLOC_F(copy, copy ? strlen(copy) + 1 : 0, CBM_MEMEV_WRITTEN);
+ return copy;
}
char *__wrap_strndup(const char *text, size_t length) {
- return mi_strndup(text, length);
+ char *copy = mi_strndup(text, length);
+ OBS_ALLOC_F(copy, copy ? strlen(copy) + 1 : 0, CBM_MEMEV_WRITTEN);
+ return copy;
}
void *__wrap__aligned_malloc(size_t size, size_t alignment) {
/* Windows spells aligned allocation _aligned_malloc(size, alignment) —
* note the argument order is the reverse of mi_malloc_aligned's. */
- return mi_malloc_aligned(size, alignment);
+ void *block = mi_malloc_aligned(size, alignment);
+ OBS_ALLOC(block, size);
+ return block;
}
void __wrap_free(void *block) {
if (!block) {
return;
}
+ OBS_FREE(block);
if (mem_override_is_ours(block)) {
mi_free(block);
return;
@@ -108,6 +150,7 @@ void __wrap__aligned_free(void *block) {
if (!block) {
return;
}
+ OBS_FREE(block);
if (mem_override_is_ours(block)) {
mi_free(block);
return;
@@ -117,10 +160,15 @@ void __wrap__aligned_free(void *block) {
void *__wrap_realloc(void *block, size_t size) {
if (!block) {
- return mi_malloc(size);
+ void *fresh = mi_malloc(size);
+ OBS_ALLOC(fresh, size);
+ return fresh;
}
if (mem_override_is_ours(block)) {
+ CBM_MEMEV_BACKING(1);
void *grown = mi_realloc(block, size);
+ CBM_MEMEV_BACKING(-1);
+ OBS_REALLOC(block, grown, size);
return grown;
}
/* A CRT block cannot be handed to mi_realloc, and the CRT's realloc must
@@ -134,6 +182,7 @@ void *__wrap_realloc(void *block, size_t size) {
return NULL; /* original stays valid, as realloc requires */
}
memcpy(moved, block, old_size < size ? old_size : size);
+ OBS_REALLOC(block, moved, size);
__real_free(block);
return moved;
}
diff --git a/src/foundation/mem_profile.c b/src/foundation/mem_profile.c
deleted file mode 100644
index f0c43b745..000000000
--- a/src/foundation/mem_profile.c
+++ /dev/null
@@ -1,325 +0,0 @@
-/*
- * mem_profile.c — see mem_profile.h.
- *
- * Two fixed tables, both open-addressed with linear probing:
- * sites[] stack hash -> {frames, live bytes, live blocks, totals}
- * pointers[] block address -> {site index, size}
- *
- * A single mutex guards both. That is heavy for a hot path and entirely
- * acceptable here: profiling is opt-in, and a diagnostic that is fast but
- * racy would produce numbers nobody can trust — which is the failure mode
- * this whole harness exists to correct.
- */
-#include "mem_profile.h"
-
-#include "compat_fs.h"
-#include "compat_thread.h"
-#include "foundation/constants.h"
-#include "platform.h"
-
-#include <stdatomic.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#ifdef _WIN32
-#ifndef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#endif
-#include <windows.h>
-#else
-#include <execinfo.h>
-#endif
-
-enum {
- PROFILE_FRAMES = 8, /* stack depth per site */
- PROFILE_SKIP_FRAMES = 2, /* this file's own frames */
- PROFILE_SITES = 4096, /* distinct call sites */
- PROFILE_POINTERS = 262144, /* tracked live blocks */
- PROFILE_DEFAULT_MIN = 32768,
-};
-
-typedef struct {
- uint64_t hash; /* 0 = empty slot */
- void *frames[PROFILE_FRAMES];
- int frame_count;
- size_t live_bytes;
- size_t live_blocks;
- size_t total_bytes;
- size_t total_blocks;
- size_t peak_live_bytes;
-} profile_site_t;
-
-typedef struct {
- void *block; /* NULL = empty slot */
- size_t size;
- size_t site;
-} profile_pointer_t;
-
-/* The profiler runs INSIDE the allocator: every hook here is reached from
- * malloc/free. Anything it calls that itself allocates — reading an env var,
- * the logger, even a probe — re-enters the hook, which recurses until the stack
- * dies (silently: the process produces no stdout, no stderr and no log) and
- * would deadlock the non-recursive mutex on the way. This guard makes the
- * profiler invisible to itself. */
-static _Thread_local bool g_profile_reentrant;
-
-static cbm_mutex_t g_profile_mutex;
-static profile_site_t g_sites[PROFILE_SITES];
-static profile_pointer_t g_pointers[PROFILE_POINTERS];
-static cbm_mem_profile_totals_t g_totals;
-static atomic_int g_profile_enabled = -1;
-static size_t g_profile_min = PROFILE_DEFAULT_MIN;
-
-size_t cbm_mem_profile_threshold(void) {
- (void)cbm_mem_profile_enabled();
- return g_profile_min;
-}
-
-bool cbm_mem_profile_enabled(void) {
- int state = atomic_load_explicit(&g_profile_enabled, memory_order_acquire);
- if (state >= 0) {
- return state == 1;
- }
- char buf[CBM_SZ_32];
- bool on = cbm_safe_getenv("CBM_MEM_PROFILE", buf, sizeof(buf), NULL) != NULL && buf[0] == '1';
- if (on) {
- char min_buf[CBM_SZ_32];
- if (cbm_safe_getenv("CBM_MEM_PROFILE_MIN", min_buf, sizeof(min_buf), NULL) != NULL) {
- long parsed = strtol(min_buf, NULL, 10);
- if (parsed > 0) {
- g_profile_min = (size_t)parsed;
- }
- }
- cbm_mutex_init(&g_profile_mutex);
- }
- atomic_store_explicit(&g_profile_enabled, on ? 1 : 0, memory_order_release);
- return on;
-}
-
-/* FNV-1a over the return addresses: cheap, and collisions merely merge two
- * sites rather than corrupting counts. */
-static uint64_t profile_hash(void *const *frames, int count) {
- uint64_t hash = 1469598103934665603ULL;
- for (int i = 0; i < count; i++) {
- uintptr_t value = (uintptr_t)frames[i];
- for (size_t byte = 0; byte < sizeof(value); byte++) {
- hash ^= (value >> (byte * 8)) & 0xFFU;
- hash *= 1099511628211ULL;
- }
- }
- return hash == 0 ? 1 : hash; /* 0 marks an empty slot */
-}
-
-static int profile_capture(void **frames, void *caller) {
-#ifdef _WIN32
- int captured = (int)CaptureStackBackTrace(PROFILE_SKIP_FRAMES, PROFILE_FRAMES, frames, NULL);
- if (captured > 0) {
- return captured;
- }
- /* Fall back to the caller the hook supplied. Using a return address from
- * THIS frame would name the profiler, not the allocating code. */
- if (caller) {
- frames[0] = caller;
- return 1;
- }
- frames[0] = __builtin_return_address(0);
- return frames[0] != NULL ? 1 : 0;
-#else
- (void)caller;
- void *raw[PROFILE_FRAMES + PROFILE_SKIP_FRAMES];
- int got = backtrace(raw, PROFILE_FRAMES + PROFILE_SKIP_FRAMES);
- int count = got - PROFILE_SKIP_FRAMES;
- if (count <= 0) {
- return 0;
- }
- if (count > PROFILE_FRAMES) {
- count = PROFILE_FRAMES;
- }
- memcpy(frames, raw + PROFILE_SKIP_FRAMES, (size_t)count * sizeof(*frames));
- return count;
-#endif
-}
-
-/* Returns SIZE_MAX when the table is full, so the caller can count the loss. */
-static size_t profile_intern_site_locked(void **frames, int count) {
- uint64_t hash = profile_hash(frames, count);
- size_t index = (size_t)(hash % PROFILE_SITES);
- for (size_t probe = 0; probe < PROFILE_SITES; probe++) {
- profile_site_t *site = &g_sites[index];
- if (site->hash == hash) {
- return index;
- }
- if (site->hash == 0) {
- site->hash = hash;
- site->frame_count = count;
- memcpy(site->frames, frames, (size_t)count * sizeof(*frames));
- g_totals.sites++;
- return index;
- }
- index = (index + 1) % PROFILE_SITES;
- }
- return SIZE_MAX;
-}
-
-static size_t profile_pointer_slot(const void *block) {
- uintptr_t value = (uintptr_t)block;
- /* Block addresses are page-aligned at these sizes, so mix the high bits
- * down before masking or every entry lands in the same few buckets. */
- value ^= value >> 20;
- value *= 2654435761U;
- return (size_t)(value % PROFILE_POINTERS);
-}
-
-void cbm_mem_profile_alloc(void *block, size_t size) {
- cbm_mem_profile_alloc_at(block, size, NULL);
-}
-
-void cbm_mem_profile_alloc_at(void *block, size_t size, void *caller) {
- if (!block || g_profile_reentrant) {
- return;
- }
- g_profile_reentrant = true;
- if (!cbm_mem_profile_enabled() || size < g_profile_min) {
- g_profile_reentrant = false;
- return;
- }
- void *frames[PROFILE_FRAMES];
- int count = profile_capture(frames, caller);
- if (count <= 0) {
- cbm_mutex_lock(&g_profile_mutex);
- g_totals.capture_failed++;
- cbm_mutex_unlock(&g_profile_mutex);
- g_profile_reentrant = false;
- return;
- }
- cbm_mutex_lock(&g_profile_mutex);
- size_t site_index = profile_intern_site_locked(frames, count);
- if (site_index == SIZE_MAX) {
- g_totals.site_table_full++;
- cbm_mutex_unlock(&g_profile_mutex);
- g_profile_reentrant = false;
- return;
- }
- size_t slot = profile_pointer_slot(block);
- bool stored = false;
- for (size_t probe = 0; probe < PROFILE_POINTERS; probe++) {
- if (g_pointers[slot].block == NULL) {
- g_pointers[slot].block = block;
- g_pointers[slot].size = size;
- g_pointers[slot].site = site_index;
- stored = true;
- break;
- }
- slot = (slot + 1) % PROFILE_POINTERS;
- }
- if (!stored) {
- g_totals.pointer_table_full++;
- cbm_mutex_unlock(&g_profile_mutex);
- g_profile_reentrant = false;
- return;
- }
- profile_site_t *site = &g_sites[site_index];
- site->live_bytes += size;
- site->live_blocks++;
- site->total_bytes += size;
- site->total_blocks++;
- if (site->live_bytes > site->peak_live_bytes) {
- site->peak_live_bytes = site->live_bytes;
- }
- g_totals.live_bytes += size;
- g_totals.live_blocks++;
- g_totals.total_bytes += size;
- cbm_mutex_unlock(&g_profile_mutex);
- g_profile_reentrant = false;
-}
-
-void cbm_mem_profile_free(void *block) {
- if (!block || g_profile_reentrant) {
- return;
- }
- g_profile_reentrant = true;
- if (!cbm_mem_profile_enabled()) {
- g_profile_reentrant = false;
- return;
- }
- cbm_mutex_lock(&g_profile_mutex);
- size_t slot = profile_pointer_slot(block);
- for (size_t probe = 0; probe < PROFILE_POINTERS; probe++) {
- profile_pointer_t *entry = &g_pointers[slot];
- if (entry->block == block) {
- profile_site_t *site = &g_sites[entry->site];
- site->live_bytes -= entry->size;
- site->live_blocks--;
- g_totals.live_bytes -= entry->size;
- g_totals.live_blocks--;
- entry->block = NULL;
- entry->size = 0;
- entry->site = 0;
- cbm_mutex_unlock(&g_profile_mutex);
- g_profile_reentrant = false;
- return;
- }
- if (entry->block == NULL) {
- break; /* linear probe ended: never tracked (below threshold) */
- }
- slot = (slot + 1) % PROFILE_POINTERS;
- }
- /* Expected for every sub-threshold allocation, so this is a scale check,
- * not an error: it should track the small-allocation rate, not grow with
- * retained bytes. */
- g_totals.untracked_frees++;
- cbm_mutex_unlock(&g_profile_mutex);
- g_profile_reentrant = false;
-}
-
-void cbm_mem_profile_totals(cbm_mem_profile_totals_t *out) {
- if (!out) {
- return;
- }
- memset(out, 0, sizeof(*out));
- if (!cbm_mem_profile_enabled()) {
- return;
- }
- cbm_mutex_lock(&g_profile_mutex);
- *out = g_totals;
- cbm_mutex_unlock(&g_profile_mutex);
-}
-
-bool cbm_mem_profile_dump(const char *path, const char *label) {
- if (!path || !cbm_mem_profile_enabled()) {
- return false;
- }
- FILE *out = cbm_fopen(path, "ab");
- if (!out) {
- return false;
- }
- cbm_mutex_lock(&g_profile_mutex);
- (void)fprintf(out,
- "{\"label\":\"%s\",\"threshold\":%zu,\"sites\":%zu,\"live_bytes\":%zu,"
- "\"live_blocks\":%zu,\"total_bytes\":%zu,\"untracked_frees\":%zu,"
- "\"site_table_full\":%zu,\"pointer_table_full\":%zu}\n",
- label ? label : "", g_profile_min, g_totals.sites, g_totals.live_bytes,
- g_totals.live_blocks, g_totals.total_bytes, g_totals.untracked_frees,
- g_totals.site_table_full, g_totals.pointer_table_full);
- for (size_t i = 0; i < PROFILE_SITES; i++) {
- profile_site_t *site = &g_sites[i];
- if (site->hash == 0 || site->total_blocks == 0) {
- continue;
- }
- (void)fprintf(out,
- "{\"label\":\"%s\",\"site\":%llu,\"live_bytes\":%zu,\"live_blocks\":%zu,"
- "\"total_bytes\":%zu,\"total_blocks\":%zu,\"peak_live\":%zu,\"frames\":[",
- label ? label : "", (unsigned long long)site->hash, site->live_bytes,
- site->live_blocks, site->total_bytes, site->total_blocks,
- site->peak_live_bytes);
- for (int f = 0; f < site->frame_count; f++) {
- (void)fprintf(out, "%s\"%p\"", f == 0 ? "" : ",", site->frames[f]);
- }
- (void)fprintf(out, "]}\n");
- }
- cbm_mutex_unlock(&g_profile_mutex);
- (void)fclose(out);
- return true;
-}
diff --git a/src/foundation/mem_profile.h b/src/foundation/mem_profile.h
deleted file mode 100644
index 44613a45b..000000000
--- a/src/foundation/mem_profile.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * mem_profile.h — Which allocation sites retain memory?
- *
- * The census answers "which POOL holds the memory"; it cannot answer "which
- * allocation put it there". Without that, every hypothesis about #581 needed a
- * code change plus a soak to test, so the investigation could only ask one
- * yes/no question per eight minutes — and two mechanisms that looked confirmed
- * were later falsified. This records the missing half.
- *
- * Only allocations at or above a size threshold are tracked (default 32 KiB).
- * That is deliberate: arena fragmentation is driven by large transient runs,
- * and tracking every small allocation would cost more than the signal is worth.
- * The threshold is reported alongside the data so the blind spot is explicit.
- *
- * Off unless CBM_MEM_PROFILE=1. All tables are fixed-capacity and allocated
- * once; when a table fills, the overflow is COUNTED and reported rather than
- * silently dropped, because a profiler that quietly loses records produces
- * exactly the false confidence this is meant to end.
- */
-#ifndef CBM_MEM_PROFILE_H
-#define CBM_MEM_PROFILE_H
-
-#include <stdbool.h>
-#include <stddef.h>
-
-/* Gate: CBM_MEM_PROFILE=1. Read once, cached. */
-bool cbm_mem_profile_enabled(void);
-
-/* Minimum allocation size recorded, from CBM_MEM_PROFILE_MIN (bytes). */
-size_t cbm_mem_profile_threshold(void);
-
-/* Record an allocation / release. Safe to call unconditionally: both return
- * immediately when profiling is off or the size is below the threshold. */
-void cbm_mem_profile_alloc(void *block, size_t size);
-
-/* Same, but with the caller's return address supplied by the hook itself.
- * Required on Windows ARM64, where CaptureStackBackTrace returns nothing: a
- * return address captured inside the capture helper names the helper, so every
- * allocation collapses onto a handful of hook addresses and the attribution
- * column becomes useless. Each hook must evaluate __builtin_return_address(0)
- * in its OWN frame and pass it here. */
-void cbm_mem_profile_alloc_at(void *block, size_t size, void *caller);
-void cbm_mem_profile_free(void *block);
-
-typedef struct {
- size_t sites; /* distinct call sites seen */
- size_t live_bytes; /* currently retained, attributed */
- size_t live_blocks;
- size_t total_bytes; /* cumulative allocated at/above threshold */
- size_t untracked_frees; /* frees whose pointer was not in the table */
- size_t site_table_full; /* records lost: site table exhausted */
- size_t pointer_table_full; /* records lost: pointer table exhausted */
- /* Stack capture returned nothing. On Windows ARM64 CaptureStackBackTrace
- * can legitimately fail, and dropping those samples silently makes a
- * working profiler look like an empty one. */
- size_t capture_failed;
-} cbm_mem_profile_totals_t;
-
-void cbm_mem_profile_totals(cbm_mem_profile_totals_t *out);
-
-/* Append one JSON object per site to `path`, biggest live first, with the raw
- * return addresses of each site's stack. Symbolisation is deliberately left to
- * the offline analyser so the hot path never loads a symbol handler. */
-bool cbm_mem_profile_dump(const char *path, const char *label);
-
-#endif /* CBM_MEM_PROFILE_H */
diff --git a/src/main.c b/src/main.c
index 202aa6a18..52113bdcc 100644
--- a/src/main.c
+++ b/src/main.c
@@ -71,6 +71,7 @@ enum {
#include "foundation/compat_fs.h"
#include "foundation/compat_thread.h"
#include "foundation/mem.h"
+#include "foundation/mem_events.h"
#include "foundation/profile.h"
#include "foundation/sha256.h"
#include "foundation/secure_random.h"
@@ -858,6 +859,7 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo
/* The graph lives on this process's heaps: SQLite gets heaps of its
* own here, and only here (cbm_sqlite_dedicated_heap). */
cbm_sqlite_dedicated_heap(true);
+ cbm_memev_process_role("worker");
}
cbm_index_set_worker_role_options(index_worker, response_out, worker_single_thread,
worker_marker, worker_quarantine,
@@ -1024,6 +1026,12 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo
/* Supervised worker: hand the full result string to the parent via the
* response file before printing (parent reads it back on a clean exit). */
const char *ro = cbm_index_worker_response_out();
+ if (cbm_index_worker_active()) {
+ /* Waste dump + execution counts BEFORE the response: the parent may
+ * end the worker as soon as the response file is complete, and the
+ * fast exit below skips every at-exit writer anyway. */
+ cbm_memev_process_exit();
+ }
bool worker_response_written = false;
if (ro) {
FILE *rf = cbm_fopen(ro, "wb");
diff --git a/tests/test_mem_events.c b/tests/test_mem_events.c
new file mode 100644
index 000000000..a3ddfd05c
--- /dev/null
+++ b/tests/test_mem_events.c
@@ -0,0 +1,510 @@
+/*
+ * test_mem_events.c — the allocation-event layer of the waste sanitizer.
+ *
+ * The layer never dereferences a block, so these tests feed it synthetic
+ * addresses: every assertion is about ACCOUNTING, and the verdicts are pure
+ * functions of the event sequence (O9) — no allocator, no clock, no timing.
+ */
+#include "test_framework.h"
+#include "../src/foundation/compat.h"
+#include "../src/foundation/compat_fs.h"
+#include "../src/foundation/compat_thread.h"
+#include "../src/foundation/mem_events.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#ifndef _WIN32
+#include <unistd.h>
+#else
+#include <process.h>
+#define getpid _getpid
+#endif
+
+#define BLK(n) ((void *)(uintptr_t)(0x100000u + ((uintptr_t)(n) << 4)))
+#define SITE_A ((void *)(uintptr_t)0xA000)
+#define SITE_B ((void *)(uintptr_t)0xB000)
+#define SITE_C ((void *)(uintptr_t)0xC000)
+
+static void fresh(void) {
+ cbm_memev_force_for_tests(true);
+ cbm_memev_reset_for_tests();
+}
+
+static bool site_row(void *site, cbm_memev_site_t *out) {
+ static cbm_memev_site_t rows[64];
+ size_t n = cbm_memev_sites(rows, 64);
+ for (size_t i = 0; i < n; i++) {
+ if (rows[i].site == (uintptr_t)site) {
+ *out = rows[i];
+ return true;
+ }
+ }
+ return false;
+}
+
+/* Requested vs usable is the allocator's slack, and it belongs to the site
+ * that asked: 3 x (100 requested, 112 handed out) = 36 bytes nobody can use. */
+TEST(memev_slack_is_charged_to_the_asking_site) {
+ fresh();
+ for (int i = 0; i < 3; i++) {
+ cbm_memev_alloc(BLK(i), 100, 112, SITE_A);
+ }
+ cbm_memev_alloc(BLK(9), 64, 64, SITE_B);
+ cbm_memev_site_t a;
+ ASSERT_TRUE(site_row(SITE_A, &a));
+ ASSERT_EQ(a.allocs, 3);
+ ASSERT_EQ(a.requested_bytes, 300);
+ ASSERT_EQ(a.usable_bytes, 336);
+ ASSERT_EQ(a.live_bytes, 336);
+ ASSERT_EQ(a.live_blocks, 3);
+ cbm_memev_free(BLK(1));
+ ASSERT_TRUE(site_row(SITE_A, &a));
+ ASSERT_EQ(a.frees, 1);
+ ASSERT_EQ(a.live_bytes, 224);
+ ASSERT_EQ(a.live_blocks, 2);
+ cbm_memev_totals_t t;
+ cbm_memev_totals(&t);
+ ASSERT_EQ(t.allocs, 4);
+ ASSERT_EQ(t.sites, 2);
+ ASSERT_EQ(t.live_bytes, 288);
+ ASSERT_EQ(t.untracked_frees, 0);
+ PASS();
+}
+
+/* A block freed by its own thread within a few allocations is churn: an
+ * arena or a stack buffer would have served. One that outlives the window
+ * is not. */
+TEST(memev_short_lived_blocks_are_counted_as_churn) {
+ fresh();
+ for (int i = 0; i < 10; i++) {
+ cbm_memev_alloc(BLK(i), 32, 32, SITE_A);
+ cbm_memev_free(BLK(i));
+ }
+ cbm_memev_alloc(BLK(100), 32, 32, SITE_B); /* lives across the window */
+ for (int i = 0; i < CBM_MEMEV_CHURN_WINDOW + 8; i++) {
+ cbm_memev_alloc(BLK(200 + i), 8, 8, SITE_C);
+ }
+ cbm_memev_free(BLK(100));
+ cbm_memev_site_t a;
+ cbm_memev_site_t b;
+ ASSERT_TRUE(site_row(SITE_A, &a));
+ ASSERT_TRUE(site_row(SITE_B, &b));
+ ASSERT_EQ(a.short_lived, 10);
+ ASSERT_EQ(b.frees, 1);
+ ASSERT_EQ(b.short_lived, 0);
+ PASS();
+}
+
+/* A realloc chain is ONE object growing. The bytes the allocator had to move
+ * are that object's cost, charged to the site that created it — not to the
+ * realloc call. Growth in place moves nothing. */
+TEST(memev_realloc_copy_bytes_go_to_the_original_site) {
+ fresh();
+ cbm_memev_alloc(BLK(1), 64, 64, SITE_A);
+ cbm_memev_realloc(BLK(1), BLK(2), 128, 128, SITE_B); /* moved: 64 bytes copied */
+ cbm_memev_realloc(BLK(2), BLK(2), 200, 256, SITE_B); /* in place: nothing copied */
+ cbm_memev_realloc(BLK(2), BLK(3), 512, 512, SITE_B); /* moved: 256 bytes copied */
+ cbm_memev_site_t a;
+ cbm_memev_site_t b;
+ ASSERT_TRUE(site_row(SITE_A, &a));
+ ASSERT_FALSE(site_row(SITE_B, &b)); /* the realloc call site owns nothing */
+ ASSERT_EQ(a.reallocs, 3);
+ ASSERT_EQ(a.realloc_copy_bytes, 64 + 256);
+ ASSERT_EQ(a.live_blocks, 1);
+ ASSERT_EQ(a.live_bytes, 512);
+ cbm_memev_free(BLK(3));
+ ASSERT_TRUE(site_row(SITE_A, &a));
+ ASSERT_EQ(a.live_bytes, 0);
+ cbm_memev_totals_t t;
+ cbm_memev_totals(&t);
+ ASSERT_EQ(t.untracked_frees, 0);
+ PASS();
+}
+
+/* The memory core announces itself before calling its backing allocator. An
+ * observer underneath consumes the hint: one event, the CORE's caller as the
+ * site, the class attached. Without an observer the hint stays pending and the
+ * core emits the event itself. Either way the block is counted once. */
+TEST(memev_hint_gives_one_event_with_the_cores_site_and_class) {
+ fresh();
+ /* observed path: hint, then the observer's event (which passes its own frame) */
+ cbm_memev_hint(SITE_A, 5);
+ ASSERT_TRUE(cbm_memev_hint_pending());
+ cbm_memev_alloc(BLK(1), 40, 48, SITE_C);
+ ASSERT_FALSE(cbm_memev_hint_pending());
+ /* unobserved path: nobody consumed it, the core emits */
+ cbm_memev_hint(SITE_A, 5);
+ if (cbm_memev_hint_pending()) {
+ cbm_memev_alloc(BLK(2), 40, 48, NULL);
+ }
+ /* an allocation that never touched the core */
+ cbm_memev_alloc(BLK(3), 40, 48, SITE_B);
+
+ cbm_memev_site_t a;
+ cbm_memev_site_t b;
+ cbm_memev_site_t c;
+ ASSERT_TRUE(site_row(SITE_A, &a));
+ ASSERT_TRUE(site_row(SITE_B, &b));
+ ASSERT_FALSE(site_row(SITE_C, &c)); /* the observer's frame never becomes a site */
+ ASSERT_EQ(a.allocs, 2);
+ ASSERT_EQ(a.raw_bytes, 0); /* tagged by the core */
+ ASSERT_EQ(b.raw_bytes, 48); /* the unmigrated surface, as a number */
+ cbm_memev_totals_t t;
+ cbm_memev_totals(&t);
+ ASSERT_EQ(t.raw_bytes, 48);
+
+ /* frees carry no handshake: whoever is responsible reports BEFORE the block goes back */
+ cbm_memev_free(BLK(1));
+ cbm_memev_free(BLK(2));
+ cbm_memev_totals(&t);
+ ASSERT_EQ(t.frees, 2);
+ ASSERT_EQ(t.untracked_frees, 0);
+ PASS();
+}
+
+/* A free of a block the layer never saw is a NUMBER, not a silent no-op: it
+ * measures how much traffic predates or bypasses the observers. */
+TEST(memev_untracked_free_is_counted_not_dropped) {
+ fresh();
+ cbm_memev_free(BLK(77));
+ cbm_memev_free(NULL);
+ cbm_memev_totals_t t;
+ cbm_memev_totals(&t);
+ ASSERT_EQ(t.untracked_frees, 1);
+ ASSERT_EQ(t.frees, 0);
+ PASS();
+}
+
+/* A block another allocator made (the Windows C runtime, a system DLL) is not a
+ * block the layer missed: it is counted apart, so the soundness gate that reads
+ * untracked frees sees only real misses (the Windows daemon's 14 CRT blocks
+ * tripped it, 2026-09-17). */
+TEST(memev_foreign_free_is_counted_apart_from_untracked) {
+ fresh();
+ cbm_memev_free_foreign(BLK(78));
+ cbm_memev_free_foreign(NULL);
+ cbm_memev_totals_t t;
+ cbm_memev_totals(&t);
+ ASSERT_EQ(t.foreign_frees, 1);
+ ASSERT_EQ(t.untracked_frees, 0);
+ ASSERT_EQ(t.frees, 0);
+ PASS();
+}
+
+typedef struct {
+ int first;
+ int count;
+ void *site;
+} span_t;
+
+static void *alloc_span(void *arg) {
+ const span_t *s = (const span_t *)arg;
+ for (int i = 0; i < s->count; i++) {
+ cbm_memev_alloc(BLK(s->first + i), 48, 64, s->site);
+ }
+ cbm_memev_flush_thread();
+ return NULL;
+}
+
+static void *free_span(void *arg) {
+ const span_t *s = (const span_t *)arg;
+ for (int i = 0; i < s->count; i++) {
+ cbm_memev_free(BLK(s->first + i));
+ }
+ cbm_memev_flush_thread();
+ return NULL;
+}
+
+enum { MEMEV_T = 4, MEMEV_PER_T = 50000 };
+
+/* Blocks are routinely freed by a thread that did not allocate them, and the
+ * per-site totals are SUMS — so they cannot depend on which thread did what or
+ * in which order the threads ran. 200,000 live blocks also walks every shard
+ * through several growth steps. */
+TEST(memev_cross_thread_frees_balance_and_totals_are_order_independent) {
+ fresh();
+ cbm_thread_t th[MEMEV_T];
+ span_t spans[MEMEV_T];
+ for (int i = 0; i < MEMEV_T; i++) {
+ spans[i] = (span_t){
+ .first = i * MEMEV_PER_T, .count = MEMEV_PER_T, .site = (i % 2) ? SITE_A : SITE_B};
+ ASSERT_EQ(cbm_thread_create(&th[i], 0, alloc_span, &spans[i]), 0);
+ }
+ for (int i = 0; i < MEMEV_T; i++) {
+ cbm_thread_join(&th[i]);
+ }
+ cbm_memev_totals_t t;
+ cbm_memev_totals(&t);
+ ASSERT_EQ(t.allocs, (uint64_t)MEMEV_T * MEMEV_PER_T);
+ ASSERT_EQ(t.live_blocks, (uint64_t)MEMEV_T * MEMEV_PER_T);
+ ASSERT_EQ(t.live_bytes, (uint64_t)MEMEV_T * MEMEV_PER_T * 64);
+ ASSERT_EQ(t.pointer_table_full, 0);
+
+ /* every span is freed by a DIFFERENT thread than the one that allocated it */
+ span_t rotated[MEMEV_T];
+ for (int i = 0; i < MEMEV_T; i++) {
+ rotated[i] = spans[(i + 1) % MEMEV_T];
+ ASSERT_EQ(cbm_thread_create(&th[i], 0, free_span, &rotated[i]), 0);
+ }
+ for (int i = 0; i < MEMEV_T; i++) {
+ cbm_thread_join(&th[i]);
+ }
+ cbm_memev_totals(&t);
+ ASSERT_EQ(t.frees, (uint64_t)MEMEV_T * MEMEV_PER_T);
+ ASSERT_EQ(t.live_blocks, 0);
+ ASSERT_EQ(t.live_bytes, 0);
+ ASSERT_EQ(t.untracked_frees, 0);
+ cbm_memev_site_t a;
+ cbm_memev_site_t b;
+ ASSERT_TRUE(site_row(SITE_A, &a));
+ ASSERT_TRUE(site_row(SITE_B, &b));
+ ASSERT_EQ(a.allocs, (uint64_t)(MEMEV_T / 2) * MEMEV_PER_T);
+ ASSERT_EQ(b.usable_bytes, (uint64_t)(MEMEV_T / 2) * MEMEV_PER_T * 64);
+ ASSERT_EQ(a.short_lived, 0); /* freed by another thread: never churn */
+ PASS();
+}
+
+/* Dormant means dormant: with the layer off nothing is recorded and a hint
+ * never lingers to mis-attribute a later event. */
+TEST(memev_dormant_layer_records_nothing) {
+ fresh();
+ cbm_memev_force_for_tests(false);
+ cbm_memev_hint(SITE_A, 3);
+ cbm_memev_alloc(BLK(1), 10, 16, SITE_A);
+ ASSERT_FALSE(cbm_memev_hint_pending());
+ cbm_memev_free(BLK(1));
+ cbm_memev_force_for_tests(true);
+ cbm_memev_totals_t t;
+ cbm_memev_totals(&t);
+ ASSERT_EQ(t.allocs, 0);
+ ASSERT_EQ(t.untracked_frees, 0);
+ ASSERT_EQ(t.sites, 0);
+ PASS();
+}
+
+/* The report is JSON lines: one header, then one line per site and one per
+ * work row. Ordering is the report tool's job; the dump only has to be
+ * complete. */
+TEST(memev_dump_writes_header_sites_and_work) {
+ fresh();
+ cbm_memev_alloc(BLK(1), 10, 16, SITE_A);
+ cbm_memev_alloc(BLK(2), 900, 1024, SITE_B);
+ cbm_work_note(CBM_WORK_MEMCPY, SITE_C, 4096, 0, 0);
+ char path[512];
+ snprintf(path, sizeof(path), "%s/cbm_memev_dump_%d.jsonl", cbm_tmpdir(), (int)getpid());
+ (void)remove(path);
+ ASSERT_TRUE(cbm_memev_dump(path, "test"));
+ FILE *f = cbm_fopen(path, "r");
+ ASSERT_NOT_NULL(f);
+ char line[4096];
+ int n = 0;
+ bool header = false;
+ bool site_a = false;
+ bool site_b = false;
+ bool work = false;
+ while (fgets(line, sizeof(line), f)) {
+ if (n++ == 0) {
+ header = strstr(line, "\"memwaste\":2") && strstr(line, "\"why\":\"test\"") &&
+ strstr(line, "\"allocs\":2");
+ }
+ site_a = site_a || strstr(line, "\"site\":\"0xa000\",\"allocs\":1");
+ site_b = site_b || (strstr(line, "\"site\":\"0xb000\"") && strstr(line, "\"usable_bytes\":1024"));
+ work = work || (strstr(line, "\"work\":\"memcpy\"") && strstr(line, "\"site\":\"0xc000\"") &&
+ strstr(line, "\"bytes\":4096") && strstr(line, "\"peak\":4096"));
+ }
+ fclose(f);
+ (void)remove(path);
+ ASSERT_TRUE(header);
+ ASSERT_TRUE(site_a);
+ ASSERT_TRUE(site_b);
+ ASSERT_TRUE(work);
+ ASSERT_EQ(n, 4);
+ PASS();
+}
+
+static bool work_row(cbm_work_kind_t kind, void *site, cbm_work_row_t *out) {
+ static cbm_work_row_t rows[128];
+ size_t n = cbm_work_rows(rows, 128);
+ for (size_t i = 0; i < n; i++) {
+ if (rows[i].kind == kind && rows[i].site == (uintptr_t)site) {
+ *out = rows[i];
+ return true;
+ }
+ }
+ return false;
+}
+
+/* The fill pattern is how the layer sees memory nobody wrote. A 4 KB block
+ * that got 100 bytes: 3,996 bytes never written, and of the 1,000 requested
+ * 900 lie beyond the last write. A hole in the middle is never-written memory
+ * too, but it is not over-requested: the bytes after it were used. */
+TEST(memev_fill_scan_measures_never_written_and_over_requested) {
+ fresh();
+ cbm_memev_scan_for_tests(true, false);
+ static uint8_t big[4096];
+ static uint8_t holed[128];
+ cbm_memev_alloc(big, 1000, sizeof(big), SITE_A);
+ ASSERT_EQ(big[4095], 0xA5); /* the layer filled it */
+ memset(big, 1, 100);
+ cbm_memev_free(big);
+
+ cbm_memev_alloc(holed, 80, sizeof(holed), SITE_B);
+ memset(holed, 2, 16); /* [0, 16) written */
+ memset(holed + 64, 3, 16); /* [64, 80) written: [16, 64) is a 48-byte hole */
+ cbm_memev_free(holed);
+
+ cbm_memev_site_t a;
+ cbm_memev_site_t b;
+ ASSERT_TRUE(site_row(SITE_A, &a));
+ ASSERT_TRUE(site_row(SITE_B, &b));
+ ASSERT_EQ(a.scanned_blocks, 1);
+ ASSERT_EQ(a.never_written_bytes, 3996);
+ ASSERT_EQ(a.over_requested_bytes, 900);
+ ASSERT_EQ(b.never_written_bytes, 48 + 48); /* the hole + the unwritten tail */
+ ASSERT_EQ(b.over_requested_bytes, 0);
+ PASS();
+}
+
+/* A calloc'd block is not filled -- its zeros are the contract. Its zero tail
+ * at free is an UPPER bound on untouched memory (a written zero looks the same). */
+TEST(memev_calloc_tail_is_scanned_for_zeros) {
+ fresh();
+ cbm_memev_scan_for_tests(true, false);
+ static uint8_t block[256];
+ memset(block, 0, sizeof(block)); /* what calloc hands out */
+ cbm_memev_alloc_ex(block, 256, sizeof(block), SITE_A, CBM_MEMEV_ZEROED);
+ ASSERT_EQ(block[200], 0); /* never filled */
+ memset(block, 7, 32);
+ cbm_memev_free(block);
+ cbm_memev_site_t a;
+ ASSERT_TRUE(site_row(SITE_A, &a));
+ ASSERT_EQ(a.zero_untouched_bytes, 224);
+ ASSERT_EQ(a.never_written_bytes, 0);
+ PASS();
+}
+
+/* A phase boundary is a snapshot: what each site still holds, and which small
+ * live blocks carry the same bytes as another -- memory an interning table or
+ * a shared constant would have served once. */
+TEST(memev_phase_snapshot_finds_retained_memory_and_duplicates) {
+ fresh();
+ cbm_memev_scan_for_tests(true, true);
+ static uint8_t blocks[4][64];
+ for (int i = 0; i < 3; i++) {
+ cbm_memev_alloc(blocks[i], 16, sizeof(blocks[i]), SITE_A);
+ memcpy(blocks[i], "same bytes here!", 16);
+ }
+ cbm_memev_alloc(blocks[3], 16, sizeof(blocks[3]), SITE_B);
+ memcpy(blocks[3], "different bytes!", 16);
+ cbm_memev_phase("snapshot");
+ cbm_memev_site_t a;
+ cbm_memev_site_t b;
+ ASSERT_TRUE(site_row(SITE_A, &a));
+ ASSERT_TRUE(site_row(SITE_B, &b));
+ ASSERT_EQ(a.retained_bytes, 3 * 64);
+ ASSERT_EQ(a.dup_blocks, 2); /* the first copy is the original */
+ ASSERT_EQ(a.dup_bytes, 2 * 64);
+ ASSERT_EQ(b.retained_bytes, 64);
+ ASSERT_EQ(b.dup_blocks, 0);
+ for (int i = 0; i < 4; i++) {
+ cbm_memev_free(blocks[i]);
+ }
+ PASS();
+}
+
+/* Work rows sum per (kind, site); `peak` keeps the largest single note. For a
+ * container that is the biggest capacity one instance left unused -- the sums
+ * run over USES, the peak is the memory figure. */
+TEST(memev_work_rows_sum_and_keep_the_peak) {
+ fresh();
+ cbm_work_note(CBM_WORK_MEMCPY, SITE_A, 100, 0, 0);
+ cbm_work_note(CBM_WORK_MEMCPY, SITE_A, 5000, 0, 0);
+ cbm_memev_container(CBM_WORK_CT_ARENA, SITE_B, 1000, 100, 2);
+ cbm_memev_container(CBM_WORK_CT_ARENA, SITE_B, 500, 450, 0);
+ cbm_work_row_t w;
+ ASSERT_TRUE(work_row(CBM_WORK_MEMCPY, SITE_A, &w));
+ ASSERT_EQ(w.calls, 2);
+ ASSERT_EQ(w.bytes, 5100);
+ ASSERT_EQ(w.peak, 5000);
+ ASSERT_TRUE(work_row(CBM_WORK_CT_ARENA, SITE_B, &w));
+ ASSERT_EQ(w.calls, 2);
+ ASSERT_EQ(w.bytes, 1500);
+ ASSERT_EQ(w.aux1, 550);
+ ASSERT_EQ(w.aux2, 2);
+ ASSERT_EQ(w.peak, 900);
+ PASS();
+}
+
+/* A repeat is work whose answer could not have changed. strlen of the same
+ * buffer holding a DIFFERENT string of the same length is new work; a lookup
+ * of the same key text repeats even from another buffer, unless the table was
+ * written in between. */
+TEST(memev_repeats_require_unchanged_input) {
+ fresh();
+ char buf[16];
+ memcpy(buf, "alpha", 6);
+ cbm_work_note_strlen(SITE_A, buf, 5);
+ cbm_work_note_strlen(SITE_A, buf, 5); /* repeat */
+ memcpy(buf, "omega", 6);
+ cbm_work_note_strlen(SITE_A, buf, 5); /* same pointer, same length, new text */
+ cbm_work_row_t w;
+ ASSERT_TRUE(work_row(CBM_WORK_STRLEN, SITE_A, &w));
+ ASSERT_EQ(w.calls, 3);
+ ASSERT_EQ(w.repeats, 1);
+
+ static const int table = 0;
+ char k1[8];
+ char k2[8];
+ memcpy(k1, "key", 4);
+ memcpy(k2, "key", 4);
+ cbm_work_note_ht(CBM_WORK_HT_GET, SITE_B, &table, k1, 0, true);
+ cbm_work_note_ht(CBM_WORK_HT_GET, SITE_B, &table, k2, 0, true); /* same text: repeat */
+ cbm_work_note_ht(CBM_WORK_HT_GET, SITE_B, &table, k2, 1, true); /* table written since */
+ cbm_work_note_ht(CBM_WORK_HT_GET, SITE_B, &table, "other", 1, false);
+ ASSERT_TRUE(work_row(CBM_WORK_HT_GET, SITE_B, &w));
+ ASSERT_EQ(w.calls, 4);
+ ASSERT_EQ(w.repeats, 1);
+ ASSERT_EQ(w.aux1, 1); /* one miss */
+
+ cbm_work_note_path(CBM_WORK_STAT, SITE_A, "/tmp/cbm-memev-path", false);
+ cbm_work_note_path(CBM_WORK_STAT, SITE_C, "/tmp/cbm-memev-path", false); /* process-wide */
+ cbm_work_note_path(CBM_WORK_OPEN, SITE_C, "/tmp/cbm-memev-path", true); /* other kind */
+ ASSERT_TRUE(work_row(CBM_WORK_STAT, SITE_C, &w));
+ ASSERT_EQ(w.repeats, 1);
+ ASSERT_TRUE(work_row(CBM_WORK_OPEN, SITE_C, &w));
+ ASSERT_EQ(w.repeats, 0);
+ ASSERT_EQ(w.aux1, 1); /* a failure */
+ PASS();
+}
+
+/* Pools compare their workers by the events each produced. */
+TEST(memev_thread_ops_count_this_threads_events) {
+ fresh();
+ uint64_t before = cbm_memev_thread_ops();
+ cbm_memev_alloc(BLK(1), 8, 8, SITE_A);
+ cbm_memev_alloc(BLK(2), 8, 8, SITE_A);
+ cbm_work_note(CBM_WORK_MEMSET, SITE_B, 8, 0, 0);
+ ASSERT_EQ(cbm_memev_thread_ops() - before, 3);
+ cbm_memev_free(BLK(1));
+ cbm_memev_free(BLK(2));
+ PASS();
+}
+
+SUITE(mem_events) {
+ RUN_TEST(memev_slack_is_charged_to_the_asking_site);
+ RUN_TEST(memev_short_lived_blocks_are_counted_as_churn);
+ RUN_TEST(memev_realloc_copy_bytes_go_to_the_original_site);
+ RUN_TEST(memev_hint_gives_one_event_with_the_cores_site_and_class);
+ RUN_TEST(memev_untracked_free_is_counted_not_dropped);
+ RUN_TEST(memev_foreign_free_is_counted_apart_from_untracked);
+ RUN_TEST(memev_cross_thread_frees_balance_and_totals_are_order_independent);
+ RUN_TEST(memev_dormant_layer_records_nothing);
+ RUN_TEST(memev_dump_writes_header_sites_and_work);
+ RUN_TEST(memev_fill_scan_measures_never_written_and_over_requested);
+ RUN_TEST(memev_calloc_tail_is_scanned_for_zeros);
+ RUN_TEST(memev_phase_snapshot_finds_retained_memory_and_duplicates);
+ RUN_TEST(memev_work_rows_sum_and_keep_the_peak);
+ RUN_TEST(memev_repeats_require_unchanged_input);
+ RUN_TEST(memev_thread_ops_count_this_threads_events);
+}