Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
feat(qa): memory + CPU waste sanitizer over the memory core
Martin Vogel81264814f162
diff --git a/.github/workflows/_memwaste.yml b/.github/workflows/_memwaste.ymlnew file mode 100644index 000000000..ab9e1e3c6--- /dev/null+++ b/.github/workflows/_memwaste.yml@@ -0,0 +1,163 @@+# Reusable: the waste sanitizer (memory + CPU waste), REPORT-ONLY.+#+# What runs (scripts/memwaste.sh, memwaste-scaling.py, memwaste-ratchet.py):+# mode=pr Linux: the event lane (memory waste + CPU work) indexing this+# repository at the checked-out commit, and the CPU scaling lane+# (src/ replicated 2x against 4x, per-function counter ratios).+# mode=full the same, plus the event lane on macOS and Windows and the access+# lane (read/write verdicts per allocation) on Linux.+#+# The corpus is the repository itself: pinned by the commit, no network, and a+# real mix of C, C++, Python, JavaScript and vendored code.+#+# REPORT-ONLY: every job uploads its report and prints the ratchet comparison+# against scripts/memwaste-baseline.json, but nothing here can turn a PR red --+# the ratchet runs without --enforce and the PR's ci-ok gate does not depend on+# these jobs. A job fails only when the MEASUREMENT is unsound (a table+# overflowed, frees the layer never saw): that is a defect of the tool, and a+# red job is how it gets noticed. Switching the ratchet to --enforce is a+# separate, deliberate change once the numbers have been stable across reruns.+name: Memwaste++on:+ workflow_call:+ inputs:+ mode:+ description: "pr (Linux event + scaling) or full (all platforms, access lane)"+ type: string+ default: pr++permissions:+ contents: read++jobs:+ event-linux:+ runs-on: ubuntu-latest+ timeout-minutes: 60+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - name: Install deps+ run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm+ - name: Event lane on this repository+ run: |+ mkdir -p "$RUNNER_TEMP/memwaste-event"+ scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \+ | tee "$RUNNER_TEMP/memwaste-event/report.txt"+ - name: Ratchet (report-only)+ run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue ${{ inputs.mode }}-linux+ - name: Upload report+ if: ${{ always() }}+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: memwaste-event-linux-${{ inputs.mode }}+ path: |+ ${{ runner.temp }}/memwaste-event/report.txt+ ${{ runner.temp }}/memwaste-event/memwaste.json+ ${{ runner.temp }}/memwaste-event/index.json+ if-no-files-found: warn++ scaling-linux:+ runs-on: ubuntu-latest+ timeout-minutes: 60+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - name: Install deps+ run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm+ - name: CPU scaling lane (src/ at 2x against 4x)+ # Report-only: the script exits 1 when it flags a super-linear function.+ continue-on-error: true+ run: |+ mkdir -p "$RUNNER_TEMP/memwaste-scaling"+ python3 scripts/memwaste-scaling.py --corpus src --replicas 2 \+ --out "$RUNNER_TEMP/memwaste-scaling" | tee "$RUNNER_TEMP/memwaste-scaling/report.txt"+ - name: Upload report+ if: ${{ always() }}+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: memwaste-scaling-linux-${{ inputs.mode }}+ path: ${{ runner.temp }}/memwaste-scaling/report.txt+ if-no-files-found: warn++ access-linux:+ if: ${{ inputs.mode == 'full' }}+ runs-on: ubuntu-latest+ timeout-minutes: 120+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - name: Install deps+ run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm+ - name: Access lane on this repository+ run: |+ mkdir -p "$RUNNER_TEMP/memwaste-access"+ scripts/memwaste.sh "$GITHUB_WORKSPACE" --lane access --top 40 --sort untouched \+ --out "$RUNNER_TEMP/memwaste-access" | tee "$RUNNER_TEMP/memwaste-access/report.txt"+ - name: Upload report+ if: ${{ always() }}+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: memwaste-access-linux-${{ inputs.mode }}+ path: |+ ${{ runner.temp }}/memwaste-access/report.txt+ ${{ runner.temp }}/memwaste-access/memwaste.json+ if-no-files-found: warn++ event-macos:+ if: ${{ inputs.mode == 'full' }}+ runs-on: macos-14+ timeout-minutes: 60+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - name: Event lane on this repository+ run: |+ mkdir -p "$RUNNER_TEMP/memwaste-event"+ scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \+ | tee "$RUNNER_TEMP/memwaste-event/report.txt"+ - name: Ratchet (report-only)+ run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue full-macos+ - name: Upload report+ if: ${{ always() }}+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: memwaste-event-macos-${{ inputs.mode }}+ path: |+ ${{ runner.temp }}/memwaste-event/report.txt+ ${{ runner.temp }}/memwaste-event/memwaste.json+ if-no-files-found: warn++ event-windows:+ if: ${{ inputs.mode == 'full' }}+ runs-on: windows-latest+ timeout-minutes: 90+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2+ with:+ msystem: CLANG64+ path-type: inherit+ install: >-+ mingw-w64-clang-x86_64-clang+ mingw-w64-clang-x86_64-llvm+ mingw-w64-clang-x86_64-zlib+ mingw-w64-clang-x86_64-python3+ make+ git+ coreutils+ - name: Event lane on this repository+ shell: msys2 {0}+ run: |+ out="$(cygpath -u "$RUNNER_TEMP")/memwaste-event"+ mkdir -p "$out"+ CC=clang CXX=clang++ scripts/memwaste.sh "$(cygpath -u "$GITHUB_WORKSPACE")" --top 40 --out "$out" \+ | tee "$out/report.txt"+ - name: Ratchet (report-only)+ shell: msys2 {0}+ run: python3 scripts/memwaste-ratchet.py --out "$(cygpath -u "$RUNNER_TEMP")/memwaste-event" --venue full-windows+ - name: Upload report+ if: ${{ always() }}+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: memwaste-event-windows-${{ inputs.mode }}+ path: |+ ${{ runner.temp }}/memwaste-event/report.txt+ ${{ runner.temp }}/memwaste-event/memwaste.json+ if-no-files-found: warndiff --git a/.github/workflows/pr.yml b/.github/workflows/pr.ymlindex 4fef6ea68..0cfc796be 100644--- a/.github/workflows/pr.yml+++ b/.github/workflows/pr.yml@@ -154,6 +154,17 @@ jobs:CCACHE_DIR: ${{ github.workspace }}/.ccacheCCACHE_MAXSIZE: 1000M+ # ── Waste sanitizer, REPORT-ONLY: memory + CPU waste on this repository and+ # the per-function scaling lane (Linux). Deliberately NOT in ci-ok's needs:+ # it uploads evidence and prints the ratchet, it never blocks a merge. Only+ # product-code PRs pay for it. ──+ memwaste:+ needs: [changes]+ if: ${{ !cancelled() && needs.changes.outputs.product == 'true' }}+ uses: ./.github/workflows/_memwaste.yml+ with:+ mode: pr+ci-ok:# The one required context (besides dco) — fails unless every PR stage# succeeded, so matrix renames can never silently deadlock merges. `skipped`diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 15f19f41f..c0e796364 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -145,6 +145,15 @@ jobs:version: ${{ inputs.version }}use_release_artifacts: true+ # ── Waste sanitizer, REPORT-ONLY ────────────────────────────────+ # The same full run as the dry run, on the release commit. Not in+ # release-draft's needs: the report is evidence for the release notes, never+ # a publication gate.+ memwaste:+ uses: ./.github/workflows/_memwaste.yml+ with:+ mode: full+# ── 6. Create DRAFT release ────────────────────────────────────# Requires smoke to have actually SUCCEEDED, not merely "not failed". The bare# !cancelled() && !failure() form is fail-OPEN: a skipped smoke is neitherdiff --git a/Makefile.cbm b/Makefile.cbmindex 066d3375c..317b51411 100644--- a/Makefile.cbm+++ b/Makefile.cbm@@ -76,8 +76,8 @@ endif# the prod build uses mimalloc (MI_OVERRIDE=1); the test build is CRT+ASan, where# binding would create an alloc/free mismatch, so the guard is prod-only.CFLAGS_PROD = $(CFLAGS_COMMON) -O2 -DCBM_BIND_TS_ALLOCATOR=1 $(TEST_SEAM_DEFINE) \- $(GLOBAL_OVERRIDE_DEFINE) $(CFLAGS_EXTRA)-CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2+ $(GLOBAL_OVERRIDE_DEFINE) $(MEMWASTE_CFLAGS) $(CFLAGS_EXTRA)+CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2 $(MEMWASTE_VENDOR_CFLAGS)# Test flags: debug + sanitizers (override SANITIZE= to disable on Windows)SANITIZE = -fsanitize=address,undefined -fno-omit-frame-pointer@@ -216,6 +216,75 @@ ELF_HARDENING_FLAGS := -Wl,-z,noexecstack -Wl,-z,separate-code \endifendif+# MEMWASTE=1: the waste-sanitizer flavour (src/foundation/mem_events.h). Built+# like production -- -O2, mimalloc, the real allocator routing -- plus the event+# layer and its observers, because waste measured under ASan's allocator would+# describe ASan. Frame pointers stay so the macOS observer can walk to the+# caller. On Linux this is also what finally LINKS the --wrap observation shim:+# its flags and source were defined here but never reached a link line, so the+# Linux profiler had been reporting nothing.+#+# Every object is built this way, VENDORED ones included (grammars, the+# tree-sitter runtime, SQLite, tre, lz4, zstd, the C++ preprocessor): waste in+# a vendored parser is waste in this program. mimalloc alone is left as it is --+# its reads and writes are the allocator's own bookkeeping, not program data.+#+# MEMWASTE_ACCESS=1 adds the access lane (reads and writes of every tracked+# block through the compiler's load/store callbacks; clang only). It builds at+# -O1 without vectorisation: at -O2 the callbacks describe the OPTIMISED program,+# and reads the optimiser removed would be reported as never read.+# MEMWASTE_PROFILE=1 adds clang's execution counters (-fprofile-instr-generate)+# for the CPU scaling lane (scripts/memwaste-scaling.py).+# Each flavour needs its own BUILD_DIR: vendored objects are not rebuilt when+# only the flavour changes.+#+# The libc wrappers (mem_override_libc.c) need --wrap on Linux and Windows;+# macOS defines the symbols instead. STATIC=1 (musl) is not a memwaste target.+MEMWASTE ?= 0+MEMWASTE_ACCESS ?= 0+MEMWASTE_PROFILE ?= 0+ifeq ($(MEMWASTE_ACCESS),1)+override MEMWASTE := 1+endif+ifeq ($(MEMWASTE_PROFILE),1)+override MEMWASTE := 1+endif+MEMWASTE_CFLAGS :=+MEMWASTE_VENDOR_CFLAGS :=+MEMWASTE_LDFLAGS :=+MEMWASTE_SRCS :=+MEMWASTE_ALLOC_SYMS_LINUX := malloc calloc realloc free strdup strndup posix_memalign \+ aligned_alloc memalign+MEMWASTE_LIBC_SYMS_LINUX := memcpy memmove memset memcmp strlen strcmp strncmp \+ __memcpy_chk __memmove_chk __memset_chk read __read_chk write \+ pread pwrite pread64 pwrite64 open open64 __open_2 __open64_2 \+ stat lstat fopen fopen64 fread __fread_chk fwrite opendir readdir \+ pthread_mutex_lock+MEMWASTE_LIBC_SYMS_WIN := memcpy memmove memset memcmp strlen strcmp strncmp _wfopen fread fwrite+ifeq ($(MEMWASTE),1)+MEMWASTE_CFLAGS := -DCBM_MEMWASTE=1 -fno-omit-frame-pointer+MEMWASTE_VENDOR_CFLAGS := -DCBM_MEMWASTE=1 -fno-omit-frame-pointer+ifeq ($(IS_LINUX),yes)+MEMWASTE_LDFLAGS := $(foreach sym,$(MEMWASTE_ALLOC_SYMS_LINUX) $(MEMWASTE_LIBC_SYMS_LINUX),-Wl,--wrap=$(sym))+MEMWASTE_SRCS := src/foundation/mem_override_posix.c+endif+ifeq ($(IS_MINGW),yes)+MEMWASTE_LDFLAGS := $(foreach sym,$(MEMWASTE_LIBC_SYMS_WIN),-Wl,--wrap=$(sym))+endif+endif+ifeq ($(MEMWASTE_ACCESS),1)+MEMWASTE_ACCESS_FLAGS := -DCBM_MEMWASTE_ACCESS=1 -O1 -fno-vectorize -fno-slp-vectorize \+ -fsanitize-coverage=inline-bool-flag,trace-loads,trace-stores+MEMWASTE_CFLAGS += $(MEMWASTE_ACCESS_FLAGS)+MEMWASTE_VENDOR_CFLAGS += $(MEMWASTE_ACCESS_FLAGS)+endif+ifeq ($(MEMWASTE_PROFILE),1)+MEMWASTE_PROFILE_FLAGS := -DCBM_MEMWASTE_PROFILE=1 -fprofile-instr-generate -fprofile-update=atomic+MEMWASTE_CFLAGS += $(MEMWASTE_PROFILE_FLAGS)+MEMWASTE_VENDOR_CFLAGS += $(MEMWASTE_PROFILE_FLAGS)+MEMWASTE_LDFLAGS += $(MEMWASTE_PROFILE_FLAGS)+endif+# The POSIX wrap shim exists only so the profiler can observe allocations. It# must never reach a sanitized build: the Linux/macOS test builds are CRT+ASan,# and redirecting malloc into mimalloc underneath ASan's own interception mixes@@ -225,7 +294,7 @@ endif# instead of the system libstdc++ (scripts/msan.sh overrides both vars).CXX_STDLIB ?= -lstdc++CXX_STDLIB_FLAGS ?=-LDFLAGS = -lm $(CXX_STDLIB) -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS)+LDFLAGS = -lm $(CXX_STDLIB) -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS) $(MIMALLOC_WRAP_FLAGS) $(MEMWASTE_LDFLAGS) $(ELF_HARDENING_FLAGS)LDFLAGS_TEST = -lm $(CXX_STDLIB) -lpthread -lz $(SANITIZE) $(WIN32_LIBS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS)LDFLAGS_TSAN = -lm $(CXX_STDLIB) -lpthread -lz $(TSAN_SANITIZE) $(WIN32_LIBS) $(MIMALLOC_WRAP_FLAGS) $(ELF_HARDENING_FLAGS)@@ -249,6 +318,9 @@ FOUNDATION_SRCS = \src/foundation/compat_regex.c \src/foundation/mem.c \src/foundation/mem_core.c \+ src/foundation/mem_events.c \+ src/foundation/mem_override_darwin.c \+ src/foundation/mem_override_libc.c \src/foundation/diagnostics.c \src/foundation/profile.c \src/foundation/dump_verify.c \@@ -472,10 +544,19 @@ ifneq ($(filter yes,$(IS_MINGW) $(IS_LINUX)),)MIMALLOC_OVERRIDE_DEFINE := -DMI_MALLOC_OVERRIDE=1GLOBAL_OVERRIDE_DEFINE := -DCBM_MEM_GLOBAL_OVERRIDE=1endif+# MI_DEBUG=0: the production allocator is mimalloc's RELEASE build. Its header+# defaults to MI_DEBUG=2 unless NDEBUG is defined, and nothing here defined it,+# so every shipped binary up to v0.11.0 ran mimalloc's debug build: internal+# assertions, fine-grained statistics, padding on every block and a memset of+# every freed block. The waste sanitizer measured it on the Go corpus as 92 GB+# (macOS) and 444 GB (Linux, where mimalloc also serves libc's malloc) of byte+# work inside the allocator (2026-09-17). Committed-bytes accounting, which the+# memory budget reads, is kept in every build (subproc stats, not MI_STAT).+# The TEST build below keeps mimalloc's debug checks.MIMALLOC_CFLAGS = -std=c11 -O2 -w \-Ivendored/mimalloc/include \-Ivendored/mimalloc/src \- -DMI_OVERRIDE=1 \+ -DMI_OVERRIDE=1 -DMI_DEBUG=0 \$(MIMALLOC_OVERRIDE_DEFINE)MIMALLOC_CFLAGS_TEST = -std=c11 -g -O1 -w \-Ivendored/mimalloc/include \@@ -496,7 +577,7 @@ MIMALLOC_CFLAGS_TEST = -std=c11 -g -O1 -w \SQLITE3_OMIT_FLAGS = -DSQLITE_OMIT_LOAD_EXTENSIONSQLITE3_SRC = vendored/sqlite3/sqlite3.cSQLITE3_CFLAGS = -std=c11 -O2 -w -DSQLITE_DQS=0 -DSQLITE_THREADSAFE=1 -DSQLITE_ENABLE_FTS5 \- $(SQLITE3_OMIT_FLAGS)+ $(SQLITE3_OMIT_FLAGS) $(MEMWASTE_VENDOR_CFLAGS)SQLITE3_CFLAGS_TEST = -std=c11 -g -O1 -w -DSQLITE_DQS=0 -DSQLITE_THREADSAFE=1 -DSQLITE_ENABLE_FTS5 \$(SQLITE3_OMIT_FLAGS)@@ -508,7 +589,7 @@ TRE_CFLAGS = -std=c11 -g -O1 -w -Ivendored/treYYJSON_SRC = vendored/yyjson/yyjson.c# All production sources-PROD_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC)+PROD_SRCS = $(MEMWASTE_SRCS) $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC)EXISTING_C_SRCS = $(EXTRACTION_SRCS) $(LSP_SRCS) $(TS_RUNTIME_SRC) \$(GRAMMAR_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC)@@ -641,7 +722,7 @@ TEST_CLI_SRCS = tests/test_cli.c tests/test_agent_clients.c tests/test_agent_protests/test_config_toml_edit.c tests/test_config_yaml_edit.c tests/test_config_text_edit.c \tests/test_activation_transaction.c-TEST_MEM_SRCS = tests/test_mem.c+TEST_MEM_SRCS = tests/test_mem.c tests/test_mem_events.cTEST_UI_SRCS = tests/test_ui.cTEST_HTTPD_SRCS = tests/test_httpd.c@@ -746,7 +827,8 @@ BUILD_DIR = build/c# ── Object file compilation (grammars need relaxed warnings) ─────# Grammar + tree-sitter runtime: compiled without -Werror (upstream code has warnings)-GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC)+GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \+ $(MEMWASTE_VENDOR_CFLAGS)GRAMMAR_CFLAGS_TEST = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \-DCBM_ENABLE_TEST_SEAMS=1 $(SANITIZED_DEFINE) $(SANITIZE)GRAMMAR_CFLAGS_TSAN = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \@@ -764,7 +846,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o# ── Targets ──────────────────────────────────────────────────────-.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security+.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security fuzz$(BUILD_DIR):mkdir -p $(BUILD_DIR)@@ -868,7 +950,7 @@ $(BUILD_DIR)/tsan_tre.o: $(TRE_SRC) | $(BUILD_DIR)$(CC) $(TRE_CFLAGS) $(TSAN_SANITIZE) -c -o $@ $<$(BUILD_DIR)/prod_tre.o: $(TRE_SRC) | $(BUILD_DIR)- $(CC) $(TRE_CFLAGS) -O2 -c -o $@ $<+ $(CC) $(TRE_CFLAGS) -O2 $(MEMWASTE_VENDOR_CFLAGS) -c -o $@ $<endif# Vendored LZ4 (test build)@@ -1036,14 +1118,14 @@ $(BUILD_DIR)/prod_preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)# Vendored LZ4 (compiled separately, not unity-built via lz4_store.c)LZ4_OBJ_PROD = $(BUILD_DIR)/prod_lz4.o $(BUILD_DIR)/prod_lz4hc.o$(BUILD_DIR)/prod_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR)- $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -c -o $@ $<+ $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR) -c -o $@ $<$(BUILD_DIR)/prod_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR)- $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<+ $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<# Vendored zstd (compiled separately, not unity-built via zstd_store.c)ZSTD_OBJ_PROD = $(BUILD_DIR)/prod_zstd.o$(BUILD_DIR)/prod_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR)- $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $<+ $(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR)/vendored/zstd -c -o $@ $<OBJS_VENDORED_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LSP_OBJ_PROD) $(PP_OBJ_PROD) $(LZ4_OBJ_PROD) $(ZSTD_OBJ_PROD) $(UNIXCODER_OBJ)@@ -1065,7 +1147,7 @@ MAIN_SRC = src/main.c# for trap debugging -- silently re-run the previously instrumented binary,# which during this bisect produced a "crashes without sanitizers too" result# that was pure artifact and sent the investigation down the wrong path.-BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)+BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)|MEMWASTE=$(MEMWASTE)|MEMWASTE_ACCESS=$(MEMWASTE_ACCESS)|MEMWASTE_PROFILE=$(MEMWASTE_PROFILE).PHONY: build-config-checkbuild-config-check:$(BUILD_DIR)/.build-config: build-config-check | $(BUILD_DIR)@@ -1261,6 +1343,28 @@ test-lsan:cd $(CURDIR) && ASAN_OPTIONS="detect_leaks=1:halt_on_error=1" \./build/lsan/test-runner $(LSAN_SUITES)+# ── Coverage-guided fuzzing (libFuzzer) ─────────────────────────+# One binary, three targets (tests/fuzz/cbm_fuzz.c): the per-file extractor,+# the Cypher parser, the TOML/YAML/JSON config editors. ASan+UBSan as in the+# test lane, plus fuzzer-no-link on EVERY object -- the vendored grammars and+# the tree-sitter runtime feed coverage back, which is where the parser state+# space lives. Needs a clang that ships libFuzzer (Homebrew LLVM on macOS; Apple+# clang does not). Run through scripts/fuzz.sh.+FUZZ_CC ?= clang+FUZZ_CXX ?= clang+++FUZZ_SANITIZE = -fsanitize=address,undefined,fuzzer-no-link -fno-omit-frame-pointer+FUZZ_SRC = tests/fuzz/cbm_fuzz.c+fuzz:+ $(MAKE) -f Makefile.cbm build/fuzz/cbm-fuzz BUILD_DIR=build/fuzz \+ CC=$(FUZZ_CC) CXX=$(FUZZ_CXX) SANITIZE="$(FUZZ_SANITIZE)"++$(BUILD_DIR)/cbm-fuzz: $(FUZZ_SRC) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) $(PROJECT_HDRS) | $(BUILD_DIR)+ $(CC) $(CFLAGS_TEST) -fsanitize=fuzzer -o $@ \+ $(FUZZ_SRC) $(PROD_SRCS) \+ $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \+ $(OBJS_VENDORED_TEST) \+ $(LDFLAGS_TEST)+# Path-sensitive memory analysis: leak paths, null derefs, uninitialized reads.# Non-gating: triage findings against the code (this lane's first run produced# 9 real fixes and 6 recorded false positives — see the analyzer batch commit).diff --git a/scripts/memlab-drive.py b/scripts/memlab-drive.pyindex cef508810..bac474126 100644--- a/scripts/memlab-drive.py+++ b/scripts/memlab-drive.py@@ -32,6 +32,16 @@ def rpc(proc, payload, timeout_note):return message+def tool_failed(reply):+ """A JSON-RPC error, or a tool result the server marked isError: MCP reports+ a failed tool call as a normal result, so checking only "error" counts every+ failure as served (a lab measuring nothing looked healthy)."""+ if "error" in reply:+ return True+ result = reply.get("result")+ return isinstance(result, dict) and bool(result.get("isError"))++def main():parser = argparse.ArgumentParser()parser.add_argument("binary")@@ -63,14 +73,16 @@ def main():"params": {"protocolVersion": "2024-11-05", "capabilities": {},"clientInfo": {"name": "memlab", "version": "1"}}},"initialize")+ project = Noneif not args.skip_index:indexed = rpc(proc, {"jsonrpc": "2.0", "id": 1, "method": "tools/call","params": {"name": "index_repository",- "arguments": {"path": args.corpus}}},+ "arguments": {"repo_path": args.corpus}}},"index_repository")- if "error" in indexed:- print(f"index failed: {indexed['error']}", file=sys.stderr)+ if tool_failed(indexed):+ print(f"index failed: {indexed.get('error') or indexed.get('result')}", file=sys.stderr)return 3+ project = (indexed.get("result", {}).get("structuredContent") or {}).get("project")halfway = 2 + args.requests // 2for i in range(2, args.requests + 2):@@ -84,11 +96,15 @@ def main():"list_projects": {},"get_graph_schema": {},"search_code": {"pattern": "Widget"}}.get(args.tool, {})+ if project and args.tool != "list_projects":+ arguments = dict(arguments, project=project)reply = rpc(proc, {"jsonrpc": "2.0", "id": i, "method": "tools/call","params": {"name": args.tool, "arguments": arguments}},f"request {i}")served += 1- if "error" in reply:+ if tool_failed(reply):+ if failures == 0:+ print(f"first failure (request {i}): {json.dumps(reply)[:600]}", file=sys.stderr, flush=True)failures += 1finally:try:diff --git a/scripts/memlab-report.py b/scripts/memlab-report.pyindex cf538fa7d..eae4bccf8 100644--- a/scripts/memlab-report.py+++ b/scripts/memlab-report.py@@ -1,19 +1,20 @@#!/usr/bin/env python3"""Analyse #581 memory-lab output: attribute growth to allocation sites.-Reads the JSONL the profiler appends (CBM_MEM_PROFILE_OUT) and the mem.census-lines from a daemon log, and prints:+Reads the waste layer's dumps (CBM_MEMWASTE_OUT, one at every request-stage phase+mark; see scripts/memlab.sh) and the mem.census lines from a daemon log, and+prints:* retention per request, as a linear fit rather than first-vs-last, so noisedoes not masquerade as a trend* the call sites whose live bytes grow across the run, ranked by growth- * an explicit unattributed remainder: process growth the profiler could not- account for. That number is the honesty check — a report that attributed- everything while tracking only >=32 KiB allocations would be lying.+ * an explicit unattributed remainder: process growth the layer could not+ account for (allocator metadata, fragmentation, memory outside malloc).+ That number is the honesty check.Usage:- memlab-report.py <profile.jsonl> [--census <daemon.log>] [--top N]- memlab-report.py <a.jsonl> --diff <b.jsonl> # platform comparison+ memlab-report.py <dump.jsonl> [--census <daemon.log>] [--binary BIN] [--top N]+ memlab-report.py <a.jsonl> --diff <b.jsonl> [--binary BIN] # platform comparison"""import argparseimport json@@ -23,9 +24,11 @@ from collections import defaultdictdef load_profile(path):- """Return (summaries, site_series). Records are appended over time, so the- Nth occurrence of a site is its Nth sample."""- summaries, series = [], defaultdict(list)+ """Return (summaries, site_series) for the process with the most dumps.+ Every dump lists every site, so the Nth occurrence of a site is its Nth+ sample; a site first seen late is padded with zeros so series align."""+ procs = defaultdict(lambda: {"summaries": [], "series": defaultdict(list)})+ cur = Nonewith open(path, "r", errors="replace") as handle:for line in handle:line = line.strip()@@ -35,11 +38,31 @@ def load_profile(path):record = json.loads(line)except json.JSONDecodeError:continue- if "site" in record:- series[record["site"]].append(record)- else:- summaries.append(record)- return summaries, series+ if "memwaste" in record:+ cur = procs[record.get("pid", 0)]+ cur["summaries"].append(record)+ elif cur is not None and "site" in record and "work" not in record:+ samples = cur["series"][record["site"]]+ while len(samples) < len(cur["summaries"]) - 1:+ samples.append({"live_bytes": 0})+ samples.append(record)+ if not procs:+ return [], {}+ best = max(procs.values(), key=lambda p: len(p["summaries"]))+ return best["summaries"], best["series"]+++def load_symbols(binary, summaries, sites):+ """Symbolise site addresses with the waste report's own symboliser."""+ if not binary or not summaries:+ return {}+ import importlib.util+ import os+ spec = importlib.util.spec_from_file_location(+ "memwaste_report", os.path.join(os.path.dirname(os.path.abspath(__file__)), "memwaste-report.py"))+ mod = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(mod)+ return mod.symbolise(binary, summaries[-1].get("image_base", "0x0"), list(sites))def linear_slope(values):@@ -57,7 +80,7 @@ def linear_slope(values):def parse_census(path):"""Pull the pool totals out of mem.census log lines."""- fields = ("priv_kb", "rss_kb", "crt_kb", "biggest_kb", "prof_live_kb")+ fields = ("priv_kb", "rss_kb", "crt_kb", "biggest_kb")rows = []pattern = re.compile(r"(\w+)=(\S+)")with open(path, "r", errors="replace") as handle:@@ -71,16 +94,16 @@ def parse_census(path):return rows-def report(path, census_path, top):+def report(path, census_path, top, binary=None):summaries, series = load_profile(path)if not summaries and not series:- print(f"{path}: no profiler records — was CBM_MEM_PROFILE=1 set?")+ print(f"{path}: no waste-layer records -- was the memwaste flavour run with CBM_MEMWASTE=1?")return+ names = load_symbols(binary, summaries, series.keys())last = summaries[-1] if summaries else {}print(f"=== {path} ===")if last:- print(f"threshold : {last.get('threshold', 0)} bytes "- f"(allocations below this are invisible by design)")+ print(f"samples : {len(summaries)} dumps (every allocation size is tracked)")print(f"sites seen : {last.get('sites', 0)}")print(f"attributed live: {last.get('live_bytes', 0)/1048576:.1f} MB "f"in {last.get('live_blocks', 0)} blocks")@@ -94,23 +117,23 @@ def report(path, census_path, top):slope = linear_slope(live)if slope <= 0 and live[-1] <= live[0]:continue- growth.append((slope, live[-1] - live[0], live[-1], samples[-1]))+ growth.append((slope, live[-1] - live[0], live[-1], names.get(site, site)))growth.sort(key=lambda row: row[0], reverse=True)print(f"\n--- sites whose retained bytes GROW (top {top}) ---")if not growth:print("none — no tracked site retains more over time.")- for slope, delta, live, sample in growth[:top]:- frames = " <- ".join(sample.get("frames", [])[:4])+ for slope, delta, live, name in growth[:top]:print(f" +{slope/1024:8.1f} KB/sample total +{delta/1048576:6.2f} MB "f"live {live/1048576:6.2f} MB")- print(f" {frames}")+ print(f" {name}")if census_path:rows = parse_census(census_path)if rows:priv = [r["priv_kb"] for r in rows if "priv_kb" in r]- prof = [r["prof_live_kb"] for r in rows if "prof_live_kb" in r]+ # attributed = what the waste layer holds live, sampled at its own dumps+ prof = [int(h.get("live_bytes", 0)) // 1024 for h in summaries]print(f"\n--- pools vs attribution ({len(rows)} samples) ---")if priv:print(f" private committed : {priv[0]/1024:.1f} -> {priv[-1]/1024:.1f} MB "@@ -119,24 +142,27 @@ def report(path, census_path, top):unattributed = (priv[-1] - priv[0]) - (prof[-1] - prof[0])print(f" attributed growth : {(prof[-1]-prof[0])/1024:+.1f} MB")print(f" UNATTRIBUTED : {unattributed/1024:+.1f} MB "- "<- below-threshold allocations, allocator metadata, or fragmentation")+ "<- allocator metadata, fragmentation, or memory outside malloc (mmap, stacks)")-def diff(path_a, path_b, top):- """Same sites, two platforms: what does one retain that the other does not?"""- _, series_a = load_profile(path_a)+def diff(path_a, path_b, top, binary=None):+ """Same sites, two platforms: what does one retain that the other does not?+ Raw addresses differ between binaries, so sites are matched by symbol name+ when --binary is given (the same binary for both runs) and by address otherwise."""+ summaries_a, series_a = load_profile(path_a)_, series_b = load_profile(path_b)+ names = load_symbols(binary, summaries_a, set(series_a) | set(series_b))rows = []for site, samples in series_a.items():a_live = samples[-1]["live_bytes"]b_live = series_b[site][-1]["live_bytes"] if site in series_b else 0if a_live - b_live > 0:- rows.append((a_live - b_live, a_live, b_live, samples[-1]))+ rows.append((a_live - b_live, a_live, b_live, site))rows.sort(key=lambda row: row[0], reverse=True)print(f"=== {path_a} minus {path_b} (top {top}) ===")- for delta, a_live, b_live, sample in rows[:top]:+ for delta, a_live, b_live, site in rows[:top]:print(f" +{delta/1048576:7.2f} MB A={a_live/1048576:.2f} MB B={b_live/1048576:.2f} MB")- print(f" {' <- '.join(sample.get('frames', [])[:4])}")+ print(f" {names.get(site, site)}")def main():@@ -145,11 +171,12 @@ def main():parser.add_argument("--census")parser.add_argument("--diff")parser.add_argument("--top", type=int, default=12)+ parser.add_argument("--binary", help="the memwaste binary, for symbol names")args = parser.parse_args()if args.diff:- diff(args.profile, args.diff, args.top)+ diff(args.profile, args.diff, args.top, args.binary)else:- report(args.profile, args.census, args.top)+ report(args.profile, args.census, args.top, args.binary)return 0diff --git a/scripts/memlab.sh b/scripts/memlab.shindex 373742556..537738d60 100644--- a/scripts/memlab.sh+++ b/scripts/memlab.sh@@ -10,9 +10,14 @@## Usage: scripts/memlab.sh <binary> [requests] [label]#-# Output: memlab-<label>.jsonl (profiler records, one block per sample)+# <binary> must be the waste-sanitizer flavour (make -f Makefile.cbm cbm+# MEMWASTE=1 BUILD_DIR=build/memwaste): the attribution comes from its event+# layer, which observes every allocation on Linux, macOS and Windows alike.+#+# Output: memlab-<label>.jsonl (a waste-layer dump at every request-stage phase mark)# memlab-<label>.log (daemon log with mem.census lines)-# Analyse with: scripts/memlab-report.py memlab-<label>.jsonl --census memlab-<label>.log+# Analyse with: scripts/memlab-report.py memlab-<label>.jsonl --census memlab-<label>.log \+# --binary <binary>set -uBINARY="${1:?usage: memlab.sh <binary> [requests] [label]}"@@ -115,8 +120,13 @@ if command -v cygpath >/dev/null 2>&1 && ! command -v winepath >/dev/null 2>&1;elseexport CBM_CACHE_DIR="$WORK/cache"fi-export CBM_MEM_PROFILE=1-export CBM_MEM_PROFILE_OUT="$PROFILE_OUT"+export CBM_MEMWASTE=1+export CBM_MEMWASTE_OUT="$PROFILE_OUT"+# The server marks a memory phase at every request stage, and the layer dumps+# at each mark: live bytes per site become a series over identical requests. The+# series needs neither the fill scan nor the duplicate hash, which only cost time.+export CBM_MEMWASTE_FILL=0+export CBM_MEMWASTE_DUPES=0export CBM_MEM_CENSUS=1export CBM_LOG_LEVEL=infoexport CBM_LOG_FORMAT=text@@ -155,9 +165,8 @@ if [ "$CENSUS" -eq 0 ]; thenecho "WARN: no census samples — check CBM_MEM_CENSUS wiring" >&2fiif [ "$SITES" -eq 0 ]; then- # Not fatal, but never silent: on macOS there is no --wrap, so the- # profiler legitimately has no observation point.- echo "WARN: no profiler records — expected on macOS (no --wrap); a gap anywhere else" >&2+ # Never silent: an empty dump means the binary is not the memwaste flavour.+ echo "WARN: no waste-layer records -- is $BINARY built with MEMWASTE=1?" >&2fiecho "profile: $PROFILE_OUT"echo "log: $RUN_LOG"diff --git a/scripts/memwaste-padding.py b/scripts/memwaste-padding.pynew file mode 100644index 000000000..0921ef462--- /dev/null+++ b/scripts/memwaste-padding.py@@ -0,0 +1,174 @@+#!/usr/bin/env python3+"""+memwaste-padding.py — the layout lane of the waste sanitizer: bytes every+instance of a struct carries for alignment alone.++Padding is waste multiplied by the instance count: 4 bytes in a record the graph+buffer holds 15 million of is 57 MB. The compiler knows every hole; this lane+asks it, for EVERY translation unit of the production build -- vendored code+included (grammars, the tree-sitter runtime, SQLite, mimalloc, tre, lz4, zstd,+the C++ preprocessor) -- by replaying the build's own compile commands with+`-fsyntax-only -Wno-everything -Wpadded`.++Per struct it reports the interior holes (removable by ordering fields from the+largest alignment down), the tail padding (removable only when the interior+reordering frees enough bytes), and how many allocation sites name the type in a+sizeof, as a first hint of instance counts. The event lane's per-site requested+sizes are the runtime evidence to pair it with.++Usage:+ scripts/memwaste-padding.py # every TU, top 40 by interior padding+ scripts/memwaste-padding.py --project-only # skip vendored TUs+ scripts/memwaste-padding.py --json+"""+from __future__ import annotations++import argparse+import concurrent.futures as cf+import json+import os+import re+import shlex+import subprocess+import sys+from collections import defaultdict++ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+SRC_EXT = (".c", ".cc", ".cpp")+KEEP_WITH_ARG = {"-include", "-isystem", "-iquote", "-arch", "-target"}+WARN_RE = re.compile(+ r"^(?P<file>[^:]+):(?P<line>\d+):\d+: warning: padding (?:struct|class|size of) "+ r"'(?P<type>[^']+)' with (?P<bytes>\d+) bytes? to (?P<what>align '(?P<field>[^']*)'|alignment boundary)")+++def build_commands(cc: str, cxx: str) -> list[tuple[str, list[str], str]]:+ dry = subprocess.run(["make", "-f", "Makefile.cbm", "-n", "cbm", "BUILD_DIR=build/memwaste-padding-dry",+ f"CC={cc}", f"CXX={cxx}", "MEMWASTE=1"],+ cwd=ROOT, capture_output=True, text=True, check=False)+ text = dry.stdout.replace("\\\n", " ")+ jobs = []+ seen = set()+ for line in text.splitlines():+ try:+ argv = shlex.split(line)+ except ValueError:+ continue+ if not argv or os.path.basename(argv[0]) not in (cc, cxx, os.path.basename(cc), os.path.basename(cxx)):+ continue+ flags = []+ sources = []+ i = 1+ while i < len(argv):+ a = argv[i]+ if a in KEEP_WITH_ARG and i + 1 < len(argv):+ flags += [a, argv[i + 1]]+ i += 2+ continue+ if a == "-o":+ i += 2+ continue+ if a.startswith(("-I", "-D", "-U", "-std=")):+ flags.append(a)+ elif a.endswith(SRC_EXT) and os.path.isfile(os.path.join(ROOT, a)):+ sources.append(a)+ i += 1+ for s in sources:+ key = (s, tuple(flags))+ if key not in seen:+ seen.add(key)+ jobs.append((argv[0], flags, s))+ return jobs+++def run_one(job: tuple[str, list[str], str]) -> tuple[str, str]:+ compiler, flags, source = job+ res = subprocess.run([compiler, "-fsyntax-only", "-Wno-everything", "-Wpadded"] + flags + [source],+ cwd=ROOT, capture_output=True, text=True, check=False)+ return source, res.stderr+++def is_vendored(path: str) -> bool:+ return "/vendored/" in f"/{path}" or path.startswith("vendored/")+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--cc", default="clang")+ ap.add_argument("--cxx", default="clang++")+ ap.add_argument("--top", type=int, default=40)+ ap.add_argument("--project-only", action="store_true")+ ap.add_argument("--jobs", type=int, default=os.cpu_count() or 4)+ ap.add_argument("--json", action="store_true")+ args = ap.parse_args()++ jobs = build_commands(args.cc, args.cxx)+ if args.project_only:+ jobs = [j for j in jobs if not is_vendored(j[2])]+ if not jobs:+ sys.exit("memwaste-padding: no compile commands found (make -n printed none for the compiler)")++ holes: dict[str, dict] = {}+ with cf.ThreadPoolExecutor(max_workers=args.jobs) as pool:+ for source, stderr in pool.map(run_one, jobs):+ for line in stderr.splitlines():+ m = WARN_RE.match(line)+ if not m:+ continue+ file = os.path.relpath(os.path.join(ROOT, m.group("file")), ROOT) \+ if not os.path.isabs(m.group("file")) else os.path.relpath(m.group("file"), ROOT)+ t = holes.setdefault(m.group("type"), {"type": m.group("type"), "interior": {}, "tail": 0,+ "where": f"{file}:{m.group('line')}",+ "vendored": is_vendored(file)})+ n = int(m.group("bytes"))+ if m.group("field") is not None:+ t["interior"][m.group("field")] = n # a header seen by many TUs reports the same hole+ else:+ t["tail"] = n+ t["where"] = f"{file}:{m.group('line')}"++ # Allocation hint: how many allocation expressions size by this type.+ sizeof_sites: dict[str, int] = defaultdict(int)+ names = {t.split(" ", 1)[-1] for t in holes}+ alloc_re = re.compile(r"(?:alloc|calloc|realloc|arena_alloc|CBM_DA_PUSH|cbm_da_push)\w*\s*\([^;]*sizeof\s*\(\s*"+ r"(?:struct\s+)?(\w+)\s*\)")+ for base in ("src", "internal/cbm"):+ for dirpath, _, filenames in os.walk(os.path.join(ROOT, base)):+ if "/vendored" in dirpath:+ continue+ for fn in filenames:+ if not fn.endswith((".c", ".h")):+ continue+ try:+ text = open(os.path.join(dirpath, fn), encoding="utf-8", errors="replace").read()+ except OSError:+ continue+ for m in alloc_re.finditer(text):+ if m.group(1) in names:+ sizeof_sites[m.group(1)] += 1++ rows = []+ for t in holes.values():+ interior = sum(t["interior"].values())+ short = t["type"].split(" ", 1)[-1]+ rows.append({"type": t["type"], "interior_bytes": interior, "tail_bytes": t["tail"],+ "holes": len(t["interior"]), "alloc_sites": sizeof_sites.get(short, 0),+ "where": t["where"], "vendored": t["vendored"]})+ rows.sort(key=lambda r: (-(r["alloc_sites"] > 0), -r["interior_bytes"], r["type"]))++ if args.json:+ json.dump({"translation_units": len(jobs), "structs": rows}, sys.stdout, indent=1)+ print()+ return 0+ print(f"translation units checked: {len(jobs)} structs with padding: {len(rows)} "+ f"(vendored: {sum(1 for r in rows if r['vendored'])})")+ print()+ print(f" {'interior B':>10} {'tail B':>6} {'holes':>5} {'alloc sites':>11} struct (declared at)")+ for r in rows[: args.top]:+ tag = " [vendored]" if r["vendored"] else ""+ print(f" {r['interior_bytes']:>10} {r['tail_bytes']:>6} {r['holes']:>5} {r['alloc_sites']:>11} "+ f"{r['type']}{tag} ({r['where']})")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/memwaste-ratchet.py b/scripts/memwaste-ratchet.pynew file mode 100644index 000000000..2c886ee54--- /dev/null+++ b/scripts/memwaste-ratchet.py@@ -0,0 +1,105 @@+#!/usr/bin/env python3+"""+memwaste-ratchet.py — hold the waste numbers where they are, lower them as they fall.++Same shape as the memory-core linter's ratchet: a checked-in baseline per venue+(scripts/memwaste-baseline.json), every waste class normalised PER THOUSAND GRAPH+NODES so a corpus that grows does not look like a regression. A class that grows+past its baseline by more than --tolerance fails under --enforce; without it the+comparison is printed and the exit is 0 (report-only rollout). A class that+FALLS is reported so the baseline can be lowered with --record.++The inputs are what scripts/memwaste.sh leaves in its --out directory:+memwaste.json (the report's --json) and index.json (the index result, for the+node count).++Usage:+ memwaste-ratchet.py --out DIR --venue pr-linux # compare, report-only+ memwaste-ratchet.py --out DIR --venue pr-linux --enforce # compare, gate+ memwaste-ratchet.py --out DIR --venue pr-linux --record # write the baseline+"""+from __future__ import annotations++import argparse+import json+import os+import sys++ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+BASELINE = os.path.join(ROOT, "scripts", "memwaste-baseline.json")++# Classes the ratchet holds. Left out on purpose: allocator-internal byte work+# (it moves with allocator settings), the access-lane classes (one lane only),+# and contended mutex acquisitions and pool imbalance -- those depend on thread+# scheduling, and a gate on them would be a lottery (O9). They stay in the report.+CLASSES = [+ "slack_bytes", "never_written_bytes", "over_requested_bytes", "realloc_copy_bytes",+ "short_lived", "dup_bytes_max", "retained_bytes_max", "raw_bytes",+ "container_idle_peak_bytes", "byte_work_bytes", "byte_work_repeats", "io_tiny_calls",+ "path_repeats", "ht_repeat_lookups", "ht_growths",+]+++def current(out_dir: str) -> dict:+ report = json.load(open(os.path.join(out_dir, "memwaste.json")))+ index = json.load(open(os.path.join(out_dir, "index.json")))+ nodes = int(index.get("nodes") or 0)+ if nodes <= 0:+ sys.exit("memwaste-ratchet: index.json has no node count")+ # The index worker is the process that allocated the most.+ worker = max(report["processes"].values(), key=lambda p: p["header"].get("usable_bytes", 0))+ summary = worker["summary"]+ return {+ "nodes": nodes,+ "per_1k_nodes": {c: round(summary.get(c, 0) * 1000.0 / nodes, 3) for c in CLASSES},+ }+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--out", required=True, help="memwaste.sh --out directory")+ ap.add_argument("--venue", required=True, help="baseline key, e.g. pr-linux, dryrun-macos")+ ap.add_argument("--tolerance", type=float, default=0.05, help="allowed growth before a class fails")+ ap.add_argument("--enforce", action="store_true")+ ap.add_argument("--record", action="store_true")+ args = ap.parse_args()++ cur = current(args.out)+ base_all = json.load(open(BASELINE)) if os.path.exists(BASELINE) else {}+ if args.record:+ base_all[args.venue] = cur+ with open(BASELINE, "w") as fh:+ json.dump(base_all, fh, indent=1, sort_keys=True)+ fh.write("\n")+ print(f"baseline recorded for {args.venue} ({cur['nodes']:,} nodes)")+ return 0+ base = base_all.get(args.venue)+ if not base:+ print(f"no baseline for {args.venue}: nothing to hold yet (record one with --record)")+ return 0++ worse, better = [], []+ print(f"{'class':<28} {'baseline/1k':>14} {'now/1k':>14} {'delta':>8}")+ for c in CLASSES:+ b = base["per_1k_nodes"].get(c, 0.0)+ n = cur["per_1k_nodes"].get(c, 0.0)+ delta = (n - b) / b if b else (0.0 if n == 0 else float("inf"))+ mark = ""+ if delta > args.tolerance:+ worse.append(c)+ mark = " WORSE"+ elif delta < -args.tolerance:+ better.append(c)+ mark = " better"+ shown = "new" if delta == float("inf") else f"{delta * 100:+.1f}%"+ print(f"{c:<28} {b:>14,.3f} {n:>14,.3f} {shown:>8}{mark}")+ if better:+ print(f"\n{len(better)} class(es) improved beyond tolerance: lower the baseline with --record")+ if worse:+ print(f"\n{len(worse)} class(es) grew beyond {args.tolerance * 100:.0f}%: {', '.join(worse)}")+ return 1 if args.enforce else 0+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/memwaste-report.py b/scripts/memwaste-report.pynew file mode 100644index 000000000..1764a4e46--- /dev/null+++ b/scripts/memwaste-report.py@@ -0,0 +1,392 @@+#!/usr/bin/env python3+"""+memwaste-report.py — turn waste-sanitizer dumps into ranked, symbolised evidence.++The sanitizer (src/foundation/mem_events.h) appends JSON lines at every phase+mark: a header per dump, then one object per allocation site and one per CPU+work row, all with RAW return addresses. Symbolisation is offline so the hot+path never loads a symbol handler. This tool reads every dump, keeps the LAST+dump of each process for the cumulative counters, the MAXIMUM over its dumps for+the phase snapshots (retained, duplicates), resolves the addresses against the+binary that produced them, and prints:++ 1. a waste summary per process — one line per waste class, in bytes/calls;+ 2. the memory table — allocation sites ranked by a chosen waste metric;+ 3. containers — arenas, dynamic arrays, hash tables: capacity against use;+ 4. CPU work — byte work, I/O, contention, pool imbalance, repeated work.++Usage:+ memwaste-report.py DUMP.jsonl --binary build/memwaste/codebase-memory-mcp+ memwaste-report.py DUMP.jsonl --binary ... --sort never_written --top 30+ memwaste-report.py DUMP.jsonl --binary ... --json (for the ratchet)+"""+from __future__ import annotations++import argparse+import json+import os+import platform+import re+import shutil+import subprocess+import sys+from collections import defaultdict++MB = 1024.0 * 1024.0++SNAPSHOT_FIELDS = ("retained_bytes", "dup_bytes", "dup_blocks")++WASTE_CLASSES = [+ # (key, label, unit)+ ("slack_bytes", "allocator slack (usable - requested)", "bytes"),+ ("never_written_bytes", "never-written bytes (fill pattern intact at free)", "bytes"),+ ("over_requested_bytes", "requested beyond the last written byte", "bytes"),+ ("zero_untouched_bytes", "calloc tail still zero at free (upper bound)", "bytes"),+ ("realloc_copy_bytes", "bytes copied by realloc growth", "bytes"),+ ("short_lived", "short-lived allocations (churn)", "calls"),+ ("dup_bytes_max", "duplicate small blocks (peak over phases)", "bytes"),+ ("retained_bytes_max", "live across a phase boundary (peak)", "bytes"),+ ("raw_bytes", "bytes that never went through the memory core", "bytes"),+ ("acc_untouched_bytes", "blocks never read or written (access lane)", "bytes"),+ ("acc_dead_bytes", "blocks written, never read by instrumented code (access lane)", "bytes"),+ ("acc_opaque_blocks", "blocks used only by code the access lane cannot see (no verdict)", "blocks"),+ ("acc_idle_bytes", "blocks idle for at least one phase before free (access lane)", "bytes"),+ ("acc_uninit_reads", "reads before any write, not calloc (access lane)", "blocks"),+ ("container_idle_peak_bytes", "container capacity left unused (largest instance, per site)", "bytes"),+ ("byte_work_bytes", "bytes moved/compared by mem*/str* calls in program code", "bytes"),+ ("allocator_byte_work_bytes", "bytes moved/compared inside the allocator (mimalloc)", "bytes"),+ ("byte_work_repeats", "strlen of a string just measured", "calls"),+ ("io_tiny_calls", "I/O calls under 512 bytes", "calls"),+ ("path_repeats", "open/stat of a path already opened/statted", "calls"),+ ("mutex_contended", "contended mutex acquisitions", "calls"),+ ("pool_imbalance_items", "work items an even pool would have spread", "items"),+ ("pool_ops_imbalance", "worker events an even pool would have spread (work done)", "events"),+ ("ht_repeat_lookups", "hash lookups of the key object just looked up", "calls"),+ ("ht_growths", "hash-table growths", "calls"),+]++BYTE_KINDS = {"memcpy", "memmove", "memset", "memcmp", "strlen", "strcmp", "strncmp"}+IO_KINDS = {"read", "write", "pread", "pwrite", "fread", "fwrite"}+PATH_KINDS = {"open", "stat", "fopen", "opendir"}+CONTAINER_KINDS = {"ct_arena", "ct_dyn_array", "ct_hash_table"}+# Symbols of the vendored allocator. Its byte work is real CPU, but the fix is in+# how the allocator is built or configured, not at a program call site.+ALLOCATOR_SYMBOL = re.compile(r"^_?mi_")+++def load(path: str):+ """{pid: {"header": last header, "headers": [...], "sites": {addr: row}, "snap_max": {addr: {...}},+ "work": {(kind, addr): row}}} — sites/work from the LAST dump of the process."""+ procs: dict[int, dict] = {}+ cur = None+ with open(path, encoding="utf-8", errors="replace") as fh:+ for line in fh:+ line = line.strip()+ if not line:+ continue+ try:+ obj = json.loads(line)+ except json.JSONDecodeError:+ continue # a torn line from a process that died mid-write+ if "memwaste" in obj:+ pid = int(obj.get("pid", 0))+ p = procs.setdefault(pid, {"headers": [], "snap_max": defaultdict(dict)})+ p["headers"].append(obj)+ p["header"] = obj+ p["sites"] = {}+ p["work"] = {}+ cur = p+ elif cur is None:+ continue+ elif "work" in obj:+ cur["work"][(obj["work"], obj["site"])] = obj+ elif "site" in obj:+ cur["sites"][obj["site"]] = obj+ snap = cur["snap_max"][obj["site"]]+ for f in SNAPSHOT_FIELDS:+ v = obj.get(f, 0)+ if v > snap.get(f, 0):+ snap[f] = v+ snap[f + "_phase"] = cur["header"].get("phase_label", "?")+ return procs+++def pe_image_base(binary: str) -> int | None:+ """The preferred ImageBase of a PE file, or None for anything else. A PE+ symboliser wants VA = ImageBase + RVA, and the dump gives RVA + runtime base."""+ try:+ with open(binary, "rb") as fh:+ head = fh.read(4096)+ except OSError:+ return None+ if head[:2] != b"MZ" or len(head) < 0x40:+ return None+ pe = int.from_bytes(head[0x3C:0x40], "little")+ if head[pe:pe + 4] != b"PE\0\0":+ return None+ opt = pe + 24+ magic = int.from_bytes(head[opt:opt + 2], "little")+ if magic == 0x20B: # PE32++ return int.from_bytes(head[opt + 24:opt + 32], "little")+ if magic == 0x10B: # PE32+ return int.from_bytes(head[opt + 28:opt + 32], "little")+ return None+++def symbolise(binary: str | None, base: str, addrs: list[str]) -> dict[str, str]:+ if not binary or not addrs:+ return {}+ if not os.path.exists(binary) and os.path.exists(binary + ".exe"):+ binary = binary + ".exe"+ out: dict[str, str] = {}+ uniq = sorted(set(addrs))+ if platform.system() == "Darwin" and shutil.which("atos"):+ for i in range(0, len(uniq), 2000):+ chunk = uniq[i:i + 2000]+ res = subprocess.run(["atos", "-o", binary, "-l", base] + chunk,+ capture_output=True, text=True, check=False)+ for addr, name in zip(chunk, res.stdout.splitlines()):+ out[addr] = name.strip()+ elif shutil.which("addr2line") or shutil.which("llvm-addr2line"):+ tool = shutil.which("llvm-addr2line") or shutil.which("addr2line")+ b = int(base, 16)+ preferred = pe_image_base(binary) or 0+ for i in range(0, len(uniq), 2000):+ chunk = uniq[i:i + 2000]+ rel = [hex(int(a, 16) - b + preferred) if b else a for a in chunk]+ res = subprocess.run([tool, "-f", "-C", "-p", "-e", binary] + rel,+ capture_output=True, text=True, check=False)+ for addr, name in zip(chunk, res.stdout.splitlines()):+ out[addr] = name.strip()+ return out+++def pct(part: float, whole: float) -> float:+ return 100.0 * part / whole if whole else 0.0+++def summarise(p: dict, names: dict[str, str]) -> dict:+ s = defaultdict(int)+ for addr, r in p["sites"].items():+ s["slack_bytes"] += max(r["usable_bytes"] - r["requested_bytes"], 0)+ for k in ("never_written_bytes", "over_requested_bytes", "zero_untouched_bytes",+ "realloc_copy_bytes", "short_lived", "raw_bytes", "acc_untouched_bytes",+ "acc_dead_bytes", "acc_idle_bytes", "acc_uninit_reads", "acc_opaque_blocks"):+ s[k] += r.get(k, 0)+ snap = p["snap_max"].get(addr, {})+ s["dup_bytes_max"] += snap.get("dup_bytes", 0)+ s["retained_bytes_max"] += snap.get("retained_bytes", 0)+ for (kind, addr), w in p["work"].items():+ if kind in CONTAINER_KINDS:+ s["container_idle_peak_bytes"] += w.get("peak", max(w["bytes"] - w["aux1"], 0))+ if kind == "ct_hash_table":+ s["ht_growths"] += w["aux2"]+ elif kind in BYTE_KINDS:+ if ALLOCATOR_SYMBOL.match(names.get(addr, "")):+ s["allocator_byte_work_bytes"] += w["bytes"]+ else:+ s["byte_work_bytes"] += w["bytes"]+ s["byte_work_repeats"] += w["repeats"]+ elif kind in IO_KINDS:+ s["io_tiny_calls"] += w["aux1"]+ elif kind in PATH_KINDS:+ s["path_repeats"] += w["repeats"]+ elif kind == "mutex":+ s["mutex_contended"] += w["aux1"]+ elif kind == "parallel_for":+ s["pool_imbalance_items"] += w["aux1"]+ elif kind == "pool_ops":+ s["pool_ops_imbalance"] += w["aux1"]+ elif kind == "ht_get":+ s["ht_repeat_lookups"] += w["repeats"]+ return dict(s)+++def fmt(v: float, unit: str) -> str:+ if unit == "bytes":+ return f"{v / MB:,.1f} MB"+ return f"{v:,.0f} {unit}"+++SITE_SORTS = {+ "bytes": lambda r, s: r["usable_bytes"],+ "slack": lambda r, s: r["usable_bytes"] - r["requested_bytes"],+ "never_written": lambda r, s: r.get("never_written_bytes", 0),+ "over_requested": lambda r, s: r.get("over_requested_bytes", 0),+ "churn": lambda r, s: r["short_lived"],+ "copy": lambda r, s: r["realloc_copy_bytes"],+ "live": lambda r, s: r["live_bytes"],+ "allocs": lambda r, s: r["allocs"],+ "dups": lambda r, s: s.get("dup_bytes", 0),+ "retained": lambda r, s: s.get("retained_bytes", 0),+ "dead": lambda r, s: r.get("acc_dead_bytes", 0),+ "untouched": lambda r, s: r.get("acc_untouched_bytes", 0),+ "idle": lambda r, s: r.get("acc_idle_bytes", 0),+ "uninit": lambda r, s: r.get("acc_uninit_reads", 0),+}+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("dump")+ ap.add_argument("--binary", help="the binary that produced the dump, for symbolisation")+ ap.add_argument("--top", type=int, default=20)+ ap.add_argument("--sort", choices=sorted(SITE_SORTS), default="bytes")+ ap.add_argument("--json", action="store_true", help="emit the merged tables as JSON")+ args = ap.parse_args()++ procs = load(args.dump)+ if not procs:+ print("no waste-sanitizer dump found in", args.dump, file=sys.stderr)+ return 2++ names_by_pid = {}+ for pid, p in procs.items():+ addrs = list(p["sites"].keys()) + [a for (_, a) in p["work"].keys()]+ names_by_pid[pid] = symbolise(args.binary, p["header"].get("image_base", "0x0"), addrs)++ if args.json:+ out = {"processes": {}}+ for pid, p in procs.items():+ names = names_by_pid[pid]+ out["processes"][str(pid)] = {+ "header": p["header"],+ "phases": [h.get("phase_label") for h in p["headers"]],+ "summary": summarise(p, names),+ "sites": [dict(r, name=names.get(a, a),+ **{(k if k.endswith("_phase") else k + "_max"): v+ for k, v in p["snap_max"].get(a, {}).items()})+ for a, r in p["sites"].items()],+ "work": [dict(w, name=names.get(a, a)) for (_, a), w in p["work"].items()],+ }+ json.dump(out, sys.stdout, indent=1)+ print()+ return 0++ # Work-heavy processes first: the index worker is the one that matters.+ order = sorted(procs.items(), key=lambda kv: -kv[1]["header"].get("usable_bytes", 0))+ for pid, p in order:+ h = p["header"]+ names = names_by_pid[pid]+ s = summarise(p, names)+ print("=" * 110)+ print(f"process {pid} last dump={h.get('why')} phases={len(p['headers'])} allocs={h['allocs']:,} "+ f"cum={h['usable_bytes'] / MB:,.0f} MB live={h['live_bytes'] / MB:,.0f} MB "+ f"rss={h.get('rss_bytes', 0) / MB:,.0f} MB sites={h['sites']:,} fill={h.get('fill')} access={h.get('access')} "+ f"layer={h.get('layer_vm_bytes', 0) / MB:,.0f} MB/{h.get('layer_threads', 0)} threads")+ blind = {k: h.get(k, 0) for k in ("site_table_full", "pointer_table_full", "work_table_full", "untracked_frees")}+ if any(blind.values()):+ print(" blind spots: " + " ".join(f"{k}={v:,}" for k, v in blind.items()))+ if h.get("foreign_frees", 0):+ print(f" outside the shim: {h['foreign_frees']:,} frees of blocks another allocator made "+ "(C runtime / system DLL; their allocations are invisible to the layer)")+ print()+ print(" WASTE SUMMARY")+ for key, label, unit in WASTE_CLASSES:+ v = s.get(key, 0)+ if v:+ print(f" {fmt(v, unit):>18} {label}")+ if len(p["sites"]) == 0 and len(p["work"]) == 0:+ continue++ print()+ print(f" MEMORY — top {args.top} sites by {args.sort}")+ print(f" {'cum MB':>9} {'allocs':>12} {'slack%':>6} {'churn%':>6} {'copy MB':>8} {'unwrit MB':>9} "+ f"{'overreq MB':>10} {'dup MB':>7} {'retain MB':>9} {'raw%':>5} {'live MB':>8} site")+ key = SITE_SORTS[args.sort]+ rows = sorted(p["sites"].items(), key=lambda kv: (-key(kv[1], p["snap_max"].get(kv[0], {})), kv[0]))+ for addr, r in rows[:args.top]:+ snap = p["snap_max"].get(addr, {})+ print(f" {r['usable_bytes'] / MB:>9,.1f} {r['allocs']:>12,} "+ f"{pct(r['usable_bytes'] - r['requested_bytes'], r['usable_bytes']):>6.1f} "+ f"{pct(r['short_lived'], r['allocs']):>6.1f} {r['realloc_copy_bytes'] / MB:>8,.1f} "+ f"{r.get('never_written_bytes', 0) / MB:>9,.1f} {r.get('over_requested_bytes', 0) / MB:>10,.1f} "+ f"{snap.get('dup_bytes', 0) / MB:>7,.1f} {snap.get('retained_bytes', 0) / MB:>9,.1f} "+ f"{pct(r['raw_bytes'], r['usable_bytes']):>5.0f} {r['live_bytes'] / MB:>8,.1f} "+ f"{names.get(addr, addr)[:90]}")++ if h.get("access"):+ # The access lane's verdicts per allocation site. Library-owned blocks (a FILE+ # buffer) are not tracked; bytes a library wrote without a store callback are+ # recognised through the fill pattern.+ acc_cols = [("untouched", "acc_untouched_bytes", "never read or written"),+ ("dead", "acc_dead_bytes", "written, never read"),+ ("idle", "acc_idle_bytes", "idle for a phase before free")]+ for sort_key, field, label in acc_cols:+ rows = [(a, r) for a, r in p["sites"].items() if r.get(field, 0) > 0]+ if not rows:+ continue+ print()+ print(f" ACCESS — top {args.top} sites by bytes {label}")+ print(f" {'MB':>9} {'blocks':>11} {'allocs':>12} {'uninit rd':>10} site")+ blocks_field = field.replace("_bytes", "_blocks")+ for a, r in sorted(rows, key=lambda t: (-t[1][field], t[0]))[:args.top]:+ print(f" {r[field] / MB:>9,.1f} {r.get(blocks_field, 0):>11,} {r['allocs']:>12,} "+ f"{r.get('acc_uninit_reads', 0):>10,} {names.get(a, a)[:90]}")+ rows = [(a, r) for a, r in p["sites"].items() if r.get("acc_uninit_reads", 0) > 0]+ if rows:+ print()+ print(f" ACCESS — top {args.top} sites by blocks read before any write (not calloc)")+ print(f" {'blocks':>11} {'allocs':>12} site")+ for a, r in sorted(rows, key=lambda t: (-t[1]["acc_uninit_reads"], t[0]))[:args.top]:+ print(f" {r['acc_uninit_reads']:>11,} {r['allocs']:>12,} {names.get(a, a)[:90]}")++ ct = [(k, a, w) for (k, a), w in p["work"].items() if k in CONTAINER_KINDS]+ if ct:+ print()+ # A container reports at every reset: calls are USES, cap/used sum over uses.+ # "peak idle" is the largest capacity a single instance left unused.+ print(f" CONTAINERS — top {args.top} by largest unused capacity of one instance")+ print(f" {'kind':<10} {'uses':>9} {'avg cap MB':>10} {'avg used MB':>11} {'unused%':>7} "+ f"{'peak idle MB':>12} {'growths':>9} created at")+ for k, a, w in sorted(ct, key=lambda t: (-t[2].get("peak", 0), t[1]))[:args.top]:+ uses = max(w["calls"], 1)+ print(f" {k[3:]:<10} {w['calls']:>9,} {w['bytes'] / uses / MB:>10,.2f} {w['aux1'] / uses / MB:>11,.2f} "+ f"{pct(w['bytes'] - w['aux1'], w['bytes']):>7.1f} {w.get('peak', 0) / MB:>12,.1f} "+ f"{w['aux2']:>9,} {names.get(a, a)[:90]}")++ def section(title, kinds, sort_key, cols):+ rows = [(k, a, w) for (k, a), w in p["work"].items() if k in kinds]+ if not rows:+ return+ print()+ print(f" CPU — {title}")+ print(" " + " ".join(f"{c[0]:>{c[1]}}" for c in cols) + " site")+ for k, a, w in sorted(rows, key=lambda t: (-sort_key(t[2]), t[1]))[:args.top]:+ cells = []+ for c in cols:+ v = c[2](k, w)+ cells.append(f"{v:>{c[1]}}" if isinstance(v, str) else f"{v:>{c[1]},.0f}")+ print(" " + " ".join(cells) + f" {names.get(a, a)[:90]}")++ section("byte work (mem*/str*) by bytes", BYTE_KINDS, lambda w: w["bytes"],+ [("kind", 8, lambda k, w: k), ("calls", 12, lambda k, w: w["calls"]),+ ("MB", 10, lambda k, w: w["bytes"] / MB), ("repeats", 10, lambda k, w: w["repeats"]),+ ("max call", 10, lambda k, w: w.get("peak", 0))])+ section("I/O by calls", IO_KINDS, lambda w: w["calls"],+ [("kind", 8, lambda k, w: k), ("calls", 12, lambda k, w: w["calls"]),+ ("MB", 10, lambda k, w: w["bytes"] / MB), ("tiny", 10, lambda k, w: w["aux1"])])+ section("paths opened/statted again", PATH_KINDS, lambda w: w["repeats"],+ [("kind", 8, lambda k, w: k), ("calls", 12, lambda k, w: w["calls"]),+ ("repeats", 10, lambda k, w: w["repeats"]), ("failed", 10, lambda k, w: w["aux1"])])+ section("mutex contention", {"mutex"}, lambda w: w["aux1"],+ [("calls", 12, lambda k, w: w["calls"]), ("contended", 10, lambda k, w: w["aux1"])])+ section("parallel_for imbalance", {"parallel_for"}, lambda w: w["aux1"],+ [("calls", 8, lambda k, w: w["calls"]), ("items", 10, lambda k, w: w["bytes"]),+ ("imbalance", 10, lambda k, w: w["aux1"])])+ # imbalance / (work + imbalance): the share of the pool's capacity that sat idle+ # while the busiest worker finished (0% = perfectly even).+ section("pool balance by work done (per-worker events)", {"pool_ops"}, lambda w: w["aux1"],+ [("calls", 8, lambda k, w: w["calls"]), ("events", 14, lambda k, w: w["bytes"]),+ ("idle%", 6, lambda k, w: f"{pct(w['aux1'], w['bytes'] + w['aux1']):.1f}"),+ ("workers", 8, lambda k, w: w["aux2"] / max(w["calls"], 1))])+ section("hash-table lookups", {"ht_get"}, lambda w: w["repeats"],+ [("gets", 12, lambda k, w: w["calls"]), ("misses", 12, lambda k, w: w["aux1"]),+ ("repeats", 12, lambda k, w: w["repeats"])])+ section("dead writes (access lane)", {"dead_write"}, lambda w: w["bytes"],+ [("blocks", 10, lambda k, w: w["aux1"]), ("MB", 10, lambda k, w: w["bytes"] / MB)])+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/memwaste-scaling.py b/scripts/memwaste-scaling.pynew file mode 100644index 000000000..ea8f1b060--- /dev/null+++ b/scripts/memwaste-scaling.py@@ -0,0 +1,262 @@+#!/usr/bin/env python3+"""+memwaste-scaling.py — the CPU scaling lane of the waste sanitizer.++Work that grows faster than the input is the costliest CPU waste there is, and a+single run cannot see it: a function that burns 5 % of the time on the Go corpus+may burn 60 % on a corpus four times larger. This lane indexes the SAME real+source tree replicated K times and 2K times, with clang's execution counters in+every object (vendored ones included), and compares, function by function, how+much work each leg did. Linear work doubles; a function whose counters grow by+more than --threshold (default 2.3, i.e. 15 % above linear) while doing at least+--min-excess extra counter increments is reported as super-linear.++The counters are COUNTS, not time: the verdict is a function of the code and the+input, so it can gate (O9). Wall time is printed as information only.++Replicas are hard links (copies when the file system refuses) under+<out>/leg-<n>/r<i>/, so every replica is a separate directory -- a separate Go+package, a separate module -- and the registry, namespace and resolution paths+see N times the same names: exactly the coupling behind the #1669 class of bugs.++Usage:+ scripts/memwaste-scaling.py --corpus ~/perf-bench/go/src/net --replicas 2+ scripts/memwaste-scaling.py --corpus DIR --replicas 3 --no-build --json++Build: MEMWASTE_PROFILE=1 into build/memwaste-profile with clang (the driver does+it unless --no-build). llvm-profdata is taken from the directory of the clang+that built the binary, so the profile format always matches.+"""+from __future__ import annotations++import argparse+import json+import os+import re+import shutil+import subprocess+import sys+import tempfile+import time++ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+BUILD_DIR = "build/memwaste-profile"+++def find_profdata(cc: str) -> str:+ cc_path = shutil.which(cc)+ if cc_path:+ cand = os.path.join(os.path.dirname(os.path.realpath(cc_path)), "llvm-profdata")+ if os.access(cand, os.X_OK):+ return cand+ for name in ("llvm-profdata",) + tuple(f"llvm-profdata-{v}" for v in range(30, 13, -1)):+ p = shutil.which(name)+ if p:+ return p+ if shutil.which("xcrun"):+ res = subprocess.run(["xcrun", "--find", "llvm-profdata"], capture_output=True, text=True, check=False)+ if res.returncode == 0 and res.stdout.strip():+ return res.stdout.strip()+ sys.exit("memwaste-scaling: llvm-profdata not found next to the compiler or on PATH")+++def replicate(src: str, dst: str, copies: int) -> int:+ """Hard-link `copies` replicas of src under dst/r<i>; returns the file count."""+ files = 0+ for i in range(copies):+ base = os.path.join(dst, f"r{i}")+ for dirpath, dirnames, filenames in os.walk(src):+ dirnames[:] = [d for d in dirnames if d not in (".git", ".hg", ".svn")]+ rel = os.path.relpath(dirpath, src)+ out_dir = os.path.join(base, rel)+ os.makedirs(out_dir, exist_ok=True)+ for name in filenames:+ s = os.path.join(dirpath, name)+ if os.path.islink(s) or not os.path.isfile(s):+ continue+ d = os.path.join(out_dir, name)+ try:+ os.link(s, d)+ except OSError:+ shutil.copy2(s, d)+ files += 1+ return files+++def run_leg(binary: str, profdata: str, corpus: str, work: str, copies: int) -> tuple[dict, dict]:+ leg = os.path.join(work, f"leg-{copies}")+ tree = os.path.join(leg, "corpus")+ os.makedirs(tree, exist_ok=True)+ nfiles = replicate(corpus, tree, copies)+ raw_dir = os.path.join(leg, "profraw")+ os.makedirs(raw_dir, exist_ok=True)+ runtime = tempfile.mkdtemp(prefix="cbm-sc.", dir="/tmp")+ os.chmod(runtime, 0o700)+ env = dict(os.environ)+ env.update({+ "LLVM_PROFILE_FILE": os.path.join(raw_dir, "cbm-%p.profraw"),+ "CBM_PROFILE_DIR": raw_dir, # the index worker names its own: worker-<pid>.profraw+ "CBM_CACHE_DIR": os.path.join(leg, "cache"),+ "CBM_RUNTIME_DIR": runtime,+ "CBM_LOG_LEVEL": "error",+ })+ env.pop("CBM_MEMWASTE", None) # production code paths: the event layer stays dormant+ t0 = time.monotonic()+ try:+ res = subprocess.run([binary, "cli", "index_repository", "--repo-path", tree, "--mode", "full"],+ env=env, capture_output=True, text=True, check=False)+ finally:+ shutil.rmtree(runtime, ignore_errors=True)+ wall = time.monotonic() - t0+ if res.returncode != 0:+ sys.exit(f"memwaste-scaling: index of leg {copies} failed (rc={res.returncode}):\n{res.stderr[-2000:]}")+ raws = sorted(os.path.join(raw_dir, f) for f in os.listdir(raw_dir) if f.endswith(".profraw"))+ workers = [r for r in raws if os.path.basename(r).startswith("worker-")]+ if not workers:+ sys.exit("memwaste-scaling: the index worker wrote no profile -- is the binary built with "+ "MEMWASTE_PROFILE=1, and does it call cbm_memev_process_exit before _Exit?")+ # The comparison is the WORKER's work: the daemon and CLI processes differ in+ # number between legs (and hash the binary at start-up), which is not scaling.+ info = {"copies": copies, "files": nfiles, "wall_s": round(wall, 1), "profiles": len(raws)}+ return merge_counts(profdata, workers, os.path.join(leg, "worker.profdata")), \+ dict(info, other=merge_counts(profdata, [r for r in raws if r not in workers],+ os.path.join(leg, "other.profdata")))+++def merge_counts(profdata: str, raws: list[str], merged: str) -> dict[str, int]:+ if not raws:+ return {}+ subprocess.run([profdata, "merge", "-o", merged] + raws, check=True)+ shown = subprocess.run([profdata, "show", "--all-functions", "--counts", merged],+ capture_output=True, text=True, check=True).stdout+ return parse_counts(shown)+++FUNC_RE = re.compile(r"^ (\S.*):$")+++def parse_counts(text: str) -> dict[str, int]:+ """Total counter increments per function from `llvm-profdata show --counts`."""+ out: dict[str, int] = {}+ name = None+ for line in text.splitlines():+ m = FUNC_RE.match(line)+ if m:+ name = m.group(1)+ continue+ if name is None:+ continue+ s = line.strip()+ if s.startswith("Function count:"):+ out[name] = out.get(name, 0) + int(s.split(":", 1)[1])+ elif s.startswith("Block counts:"):+ vals = s.split(":", 1)[1].strip().strip("[]")+ if vals:+ out[name] = out.get(name, 0) + sum(int(v) for v in vals.split(",") if v.strip())+ return out+++def display_name(raw: str) -> str:+ # static functions are "file.c;name" (or "file.c:name" in older formats)+ for sep in (";", ":"):+ if sep in raw:+ file_part, fn = raw.rsplit(sep, 1)+ return f"{fn} ({os.path.basename(file_part)})"+ return raw+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--corpus", required=True, help="a real source tree to replicate")+ ap.add_argument("--replicas", type=int, default=2, help="K: legs index K and 2K replicas")+ ap.add_argument("--threshold", type=float, default=2.3, help="2K/K counter ratio that is super-linear")+ ap.add_argument("--min-excess", type=int, default=10_000_000,+ help="ignore functions whose work beyond linear is below this many counter increments")+ ap.add_argument("--top", type=int, default=30)+ ap.add_argument("--out", help="work directory (default: a new temp dir)")+ ap.add_argument("--no-build", action="store_true")+ ap.add_argument("--cc", default="clang")+ ap.add_argument("--json", action="store_true")+ args = ap.parse_args()++ corpus = os.path.abspath(os.path.expanduser(args.corpus))+ if not os.path.isdir(corpus):+ sys.exit(f"memwaste-scaling: {corpus} is not a directory")+ binary = os.path.join(ROOT, BUILD_DIR, "codebase-memory-mcp")+ if not args.no_build:+ cxx = "clang++" if args.cc == "clang" else args.cc.replace("clang", "clang++")+ subprocess.run([os.path.join(ROOT, "scripts", "build.sh"), f"BUILD_DIR={BUILD_DIR}",+ "MEMWASTE_PROFILE=1", f"CC={args.cc}", f"CXX={cxx}"],+ check=True, stdout=subprocess.DEVNULL)+ if not os.access(binary, os.X_OK):+ sys.exit(f"memwaste-scaling: {binary} is missing (build it, or drop --no-build)")+ profdata = find_profdata(args.cc)+ work = os.path.abspath(args.out) if args.out else tempfile.mkdtemp(prefix="cbm-scaling.")+ os.makedirs(work, exist_ok=True)+ for sub in os.listdir(work):+ if sub.startswith("leg-"):+ shutil.rmtree(os.path.join(work, sub), ignore_errors=True)++ k = args.replicas+ small, info_k = run_leg(binary, profdata, corpus, work, k)+ large, info_2k = run_leg(binary, profdata, corpus, work, 2 * k)++ total_k = sum(small.values())+ total_2k = sum(large.values())+ rows = []+ for name, c2 in large.items():+ c1 = small.get(name, 0)+ excess = c2 - 2 * c1+ ratio = (c2 / c1) if c1 else float("inf")+ rows.append({"function": name, "k": c1, "2k": c2, "ratio": ratio, "excess": excess})+ flagged = [r for r in rows if r["ratio"] > args.threshold and r["excess"] >= args.min_excess]+ flagged.sort(key=lambda r: -r["excess"])+ heaviest = sorted(rows, key=lambda r: -r["2k"])[: args.top]++ for info in (info_k, info_2k):+ other = info.pop("other", {}) or {}+ info["other_total"] = sum(other.values())+ info["other_top"] = sorted(other.items(), key=lambda kv: -kv[1])[:10]+ report = {+ "corpus": corpus, "legs": [info_k, info_2k], "threshold": args.threshold,+ "min_excess": args.min_excess, "total_counts": {"k": total_k, "2k": total_2k},+ "total_ratio": (total_2k / total_k) if total_k else None,+ "superlinear": flagged,+ "work_dir": work,+ }+ if args.json:+ json.dump(report, sys.stdout, indent=1, default=lambda o: None if o == float("inf") else o)+ print()+ return 1 if flagged else 0++ print(f"corpus {corpus}")+ for info in (info_k, info_2k):+ print(f" leg {info['copies']:>2} replicas: {info['files']:,} files, {info['profiles']} process profile(s), "+ f"{info['wall_s']} s wall (information only)")+ print(f" worker counter increments: k={total_k:,} 2k={total_2k:,} ratio={report['total_ratio']:.3f}")+ for info in (info_k, info_2k):+ other = info.get("other") or {}+ top = sorted(other.items(), key=lambda kv: -kv[1])[:3]+ print(f" other processes at {info['copies']} replicas: {sum(other.values()):,} increments; heaviest: "+ + ", ".join(f"{display_name(n)} {c:,}" for n, c in top))+ print()+ print(f" SUPER-LINEAR — ratio > {args.threshold} and >= {args.min_excess:,} increments beyond linear")+ if not flagged:+ print(" none")+ else:+ print(f" {'ratio':>7} {'excess':>16} {'k':>16} {'2k':>16} function")+ for r in flagged[: args.top]:+ ratio = "new" if r["ratio"] == float("inf") else f"{r['ratio']:.2f}"+ print(f" {ratio:>7} {r['excess']:>16,} {r['k']:>16,} {r['2k']:>16,} {display_name(r['function'])[:100]}")+ print()+ print(f" HEAVIEST at 2k — top {args.top} by counter increments (share of all work, scaling ratio)")+ for r in heaviest:+ ratio = "new" if r["ratio"] == float("inf") else f"{r['ratio']:.2f}"+ print(f" {100.0 * r['2k'] / total_2k:>6.2f}% x{ratio:>5} {display_name(r['function'])[:100]}")+ print()+ print(f"work dir: {work}")+ return 1 if flagged else 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/memwaste.sh b/scripts/memwaste.shnew file mode 100644index 000000000..8d1935d7e--- /dev/null+++ b/scripts/memwaste.sh@@ -0,0 +1,182 @@+#!/usr/bin/env bash+# memwaste.sh — run the waste sanitizer on a repository and print the ranked report.+#+# The sanitizer (src/foundation/mem_events.h) measures memory that is CORRECT but+# unnecessary: allocator slack, churn, realloc copying, memory that never reaches+# the memory core. This driver is the one entry point for every venue: a local+# run on a bench corpus, the PR-CI fixture run, the dry-run and release runs.+#+# Usage:+# scripts/memwaste.sh <repo-path> [--lane event|access] [--top N] [--sort METRIC]+# [--out DIR] [--no-build] [--json] [--binary PATH]+#+# --binary runs a kept copy of a flavour binary (a baseline, a replay) instead of+# the one in the build directory; it implies --no-build.+#+# What it does+# 1. builds the `memwaste` flavour (production flags + the event layer) into+# build/memwaste, unless --no-build;+# 2. indexes <repo-path> with CBM_MEMWASTE=1 into a private cache and a private,+# SHORT runtime directory (the coordination socket has a 104-byte path limit);+# 3. prints the symbolised report and writes the raw dump + JSON next to it;+# 4. FAILS when the measurement itself is unsound: a table overflowed, or more+# than 0.1 % of frees were of blocks the layer never saw. A sanitizer that+# silently loses records produces false confidence.+#+# Nothing here reads a clock to decide anything: wall time is printed as+# information only. Gates are counts.+set -euo pipefail++ROOT="$(cd "$(dirname "$0")/.." && pwd)"+REPO=""+TOP=25+SORT=bytes+OUT=""+BUILD=1+JSON=0+LANE=event+BINARY_OVERRIDE=""++while [ $# -gt 0 ]; do+ case "$1" in+ --binary) BINARY_OVERRIDE="$2"; BUILD=0; shift 2 ;;+ --top) TOP="$2"; shift 2 ;;+ --sort) SORT="$2"; shift 2 ;;+ --out) OUT="$2"; shift 2 ;;+ --no-build) BUILD=0; shift ;;+ --json) JSON=1; shift ;;+ --lane) LANE="$2"; shift 2 ;;+ -h | --help) sed -n '2,24p' "$0"; exit 0 ;;+ # A typo'd flag must never be read as the repository path — it would index+ # nothing and report an empty lane as a clean one.+ -*) echo "memwaste.sh: unknown argument $1. Please consult --help." >&2; exit 2 ;;+ *) REPO="$1"; shift ;;+ esac+done++if [ -z "$REPO" ] || [ ! -d "$REPO" ]; then+ echo "memwaste.sh: a repository directory is required. Please consult --help." >&2+ exit 2+fi++case "$LANE" in+event) BUILD_ARGS=(BUILD_DIR=build/memwaste MEMWASTE=1 ${CC:+CC=$CC} ${CXX:+CXX=$CXX})+ BIN="$ROOT/build/memwaste/codebase-memory-mcp" ;;+access) BUILD_ARGS=(BUILD_DIR=build/memwaste-access MEMWASTE_ACCESS=1 CC=clang CXX=clang++)+ BIN="$ROOT/build/memwaste-access/codebase-memory-mcp" ;;+*) echo "memwaste.sh: --lane must be event or access" >&2; exit 2 ;;+esac+if [ "$BUILD" -eq 1 ]; then+ "$ROOT/scripts/build.sh" "${BUILD_ARGS[@]}" >/dev/null+fi+if [ -n "$BINARY_OVERRIDE" ]; then+ BIN="$BINARY_OVERRIDE"+fi+if [ ! -x "$BIN" ]; then+ echo "memwaste.sh: $BIN is missing (build failed, or --no-build without a build)" >&2+ exit 2+fi++if [ -z "$OUT" ]; then+ OUT="$(mktemp -d "${TMPDIR:-/tmp}/cbm-memwaste.XXXXXX")"+fi+mkdir -p "$OUT"+DUMP="$OUT/memwaste.jsonl"+rm -f "$DUMP"++RUN_BIN="$BIN"+if command -v cygpath >/dev/null 2>&1 && ! command -v winepath >/dev/null 2>&1; then+ # Native Windows: the server walks the ancestor chain of its binary, cache and+ # runtime directories and refuses any that grant mutation to untrusted SIDs.+ # The checkout and msys /tmp both do, so the run happens from a root under+ # USERPROFILE stamped to the current user, SYSTEM and Administrators -- by+ # SID, the way scripts/memlab.sh does it (#1532).+ PROFILE_DIR="$(cygpath -u "$USERPROFILE")"+ WORK="$(mktemp -d "$PROFILE_DIR/cbm-mw.XXXXXX")"+ WORK_W="$(cygpath -w "$WORK")"+ SID_QUERY='[System.Security.Principal.WindowsIdentity]::GetCurrent().User.Value'+ ME="$(powershell.exe -NoProfile -NonInteractive -Command "$SID_QUERY" 2>/dev/null | tr -d '\r\n')"+ case "$ME" in+ S-1-*) ME="*$ME" ;;+ *) echo "memwaste.sh: could not resolve the current user's SID" >&2; exit 2 ;;+ esac+ if ! MSYS2_ARG_CONV_EXCL='*' icacls "$WORK_W" /inheritance:r \+ /grant:r "${ME}:(OI)(CI)F" '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' /Q >/dev/null 2>&1; then+ echo "memwaste.sh: could not stamp $WORK" >&2+ exit 2+ fi+ mkdir -p "$WORK/cache" "$WORK/rt"+ cp "$BIN.exe" "$WORK/codebase-memory-mcp.exe" 2>/dev/null || cp "$BIN" "$WORK/codebase-memory-mcp.exe"+ RUN_BIN="$WORK/codebase-memory-mcp.exe"+ CACHE_DIR="$WORK_W\\cache"+ RUNTIME="$WORK_W\\rt"+ DUMP_ENV="$(cygpath -w "$DUMP")"+else+ WORK="$(mktemp -d /tmp/cbm-mw.XXXXXX)"+ chmod 700 "$WORK"+ # A fresh cache every run: an existing index turns --mode full into an+ # incremental pass that measures hashing instead of indexing.+ rm -rf "$OUT/cache"+ mkdir -p "$OUT/cache"+ CACHE_DIR="$OUT/cache"+ RUNTIME="$WORK"+ DUMP_ENV="$DUMP"+fi++# CBM_MEMWASTE_KEEP=1 keeps the private root (binary copy, cache, logs) for inspection.+cleanup() {+ if [ "${CBM_MEMWASTE_KEEP:-0}" != 1 ]; then+ rm -rf "$WORK"+ fi+}+trap cleanup EXIT++INDEX_RC=0+CBM_MEMWASTE=1 CBM_MEMWASTE_OUT="$DUMP_ENV" CBM_CACHE_DIR="$CACHE_DIR" CBM_RUNTIME_DIR="$RUNTIME" \+ CBM_LOG_LEVEL=error "$RUN_BIN" cli index_repository --repo-path "$REPO" --mode full \+ >"$OUT/index.json" || INDEX_RC=$?+if [ "$INDEX_RC" -ne 0 ]; then+ # A failed index is a finding about the flavour, never a silent partial report.+ echo "memwaste.sh: the index failed (rc=$INDEX_RC); its logs follow" >&2+ for log in "$WORK"/cache/logs/* "$WORK"/cache/logs/.worker-log-* "$OUT"/cache/logs/* "$OUT"/cache/logs/.worker-log-*; do+ if [ -f "$log" ]; then+ echo "--- $log" >&2+ tail -n 60 "$log" >&2+ fi+ done+ exit 1+fi++if [ ! -s "$DUMP" ]; then+ echo "memwaste.sh: the sanitizer wrote no dump -- is the flavour built with MEMWASTE=1?" >&2+ exit 1+fi++python3 "$ROOT/scripts/memwaste-report.py" "$DUMP" --binary "$BIN" --json >"$OUT/memwaste.json"+if [ "$JSON" -eq 1 ]; then+ cat "$OUT/memwaste.json"+else+ python3 "$ROOT/scripts/memwaste-report.py" "$DUMP" --binary "$BIN" --top "$TOP" --sort "$SORT"+ echo+ echo "raw dump: $DUMP"+ echo "json: $OUT/memwaste.json"+fi++# Soundness of the measurement itself.+python3 - "$OUT/memwaste.json" <<'PY'+import json, sys+procs = json.load(open(sys.argv[1]))["processes"]+bad = []+for pid, proc in procs.items():+ h = proc["header"]+ if h.get("site_table_full", 0) or h.get("pointer_table_full", 0):+ bad.append(f"process {pid}: a table overflowed (sites={h.get('site_table_full')}, pointers={h.get('pointer_table_full')})")+ frees = h.get("frees", 0) + h.get("untracked_frees", 0)+ if frees and h.get("untracked_frees", 0) * 1000 > frees:+ bad.append(f"process {pid}: {h['untracked_frees']:,} of {frees:,} frees were of blocks the layer never saw (> 0.1 %)")+if bad:+ print("memwaste.sh: UNSOUND MEASUREMENT", file=sys.stderr)+ for b in bad:+ print(" " + b, file=sys.stderr)+ sys.exit(1)+PYdiff --git a/scripts/sanitizer-coverage-map.py b/scripts/sanitizer-coverage-map.pynew file mode 100644index 000000000..d961ec976--- /dev/null+++ b/scripts/sanitizer-coverage-map.py@@ -0,0 +1,306 @@+#!/usr/bin/env python3+"""+sanitizer-coverage-map.py — what our sanitizer lanes instrument, what they+actually execute, and every hole, vendored code included.++A sanitizer only checks code that runs under it. A function no suite reaches is+unchecked in every lane, and until now nobody could name those functions. This+tool answers three questions and keeps the answers from regressing:++ static Which translation units does each lane COMPILE with its sanitizer,+ per the build's own compile commands (make -n)? Lists every TU no+ dynamic lane instruments and every TU the static lanes filter out.+ run Build one lane with clang source coverage on top of its sanitizer+ flags, run its test runner (suites optional), and export per-function+ execution counts (lcov) -- vendored objects included, because the+ flags ride on the lane's own sanitizer variable, which the vendored+ rules already use.+ union Merge the per-lane exports: the functions that ran under NO lane,+ grouped by file, vendored code flagged.+ ratchet Compare a union against the checked-in baseline: a function that+ joins the never-executed list fails; one that leaves it is reported+ so the baseline can be lowered.+ map Validate and print scripts/sanitizer-map.json: every defect class x+ platform cell must be covered, partial, or a named limit with a reason.++`run` executes test suites, so it belongs to the lane's own venue (local CI, the+CI job); `static`, `union`, `ratchet` and `map` run anywhere in seconds.++Usage:+ scripts/sanitizer-coverage-map.py static+ scripts/sanitizer-coverage-map.py run --lane asan --out build/covmap [SUITE...]+ scripts/sanitizer-coverage-map.py union --out build/covmap+ scripts/sanitizer-coverage-map.py ratchet --out build/covmap+ scripts/sanitizer-coverage-map.py map+"""+from __future__ import annotations++import argparse+import json+import os+import shlex+import shutil+import subprocess+import sys+import tempfile+from collections import defaultdict++ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+BASELINE = os.path.join(ROOT, "scripts", "sanitizer-coverage-baseline.txt")+MAP_FILE = os.path.join(ROOT, "scripts", "sanitizer-map.json")+COVERAGE_FLAGS = "-fprofile-instr-generate -fcoverage-mapping"+SRC_EXT = (".c", ".cc", ".cpp")++# lane -> how the lane builds and runs. `var` is the make variable carrying the+# lane's sanitizer flags; the coverage flags are appended to it.+LANES = {+ "asan": {"target": "test-runner", "var": "SANITIZE",+ "value": "-fsanitize=address,undefined -fno-omit-frame-pointer", "env": {}},+ "tsan": {"target": "test-runner-tsan", "var": "TSAN_SANITIZE",+ "value": "-fsanitize=thread -fno-omit-frame-pointer",+ "env": {"TSAN_OPTIONS": "halt_on_error=1:report_thread_leaks=0"}},+ "lsan": {"target": "test-runner", "var": "SANITIZE",+ "value": "-fsanitize=address,undefined -fno-omit-frame-pointer",+ "env": {"ASAN_OPTIONS": "detect_leaks=1:halt_on_error=1"}},+}+DYNAMIC_FLAGS = ("-fsanitize=address", "-fsanitize=thread", "-fsanitize=memory", "-fsanitize=undefined",+ "-fsanitize=address,undefined")+++def is_vendored(path: str) -> bool:+ # grammar_<lang>.c and the runtime/unity wrappers only #include vendored sources+ base = os.path.basename(path)+ return "vendored/" in path or base.startswith(("grammar_", "ts_runtime")) or base in (+ "sqlite3.c", "mimalloc.c", "unixcoder_blob.c")+++def dry_commands(target: str, extra: list[str]) -> list[list[str]]:+ res = subprocess.run(["make", "-f", "Makefile.cbm", "-n", "-B", target] + extra,+ cwd=ROOT, capture_output=True, text=True, check=False)+ cmds = []+ for line in res.stdout.replace("\\\n", " ").splitlines():+ try:+ argv = shlex.split(line)+ except ValueError:+ continue+ if argv and any(a.endswith(SRC_EXT) for a in argv) and ("-c" in argv or "-o" in argv):+ cmds.append(argv)+ return cmds+++def make_var(name: str) -> list[str]:+ """The fully expanded value of a Makefile.cbm variable (works with make 3.81: no --eval)."""+ with tempfile.NamedTemporaryFile("w", suffix=".mk", delete=False) as mk:+ mk.write("print-cbm-var-%:\n\t@echo $($*)\n")+ helper = mk.name+ try:+ res = subprocess.run(["make", "-s", "-f", "Makefile.cbm", "-f", helper, f"print-cbm-var-{name}"],+ cwd=ROOT, capture_output=True, text=True, check=False)+ finally:+ os.unlink(helper)+ return res.stdout.split() if res.returncode == 0 else []+++def cmd_static(args) -> int:+ instrumented: dict[str, set[str]] = defaultdict(set)+ compiled: set[str] = set()+ for lane, spec in {"asan": LANES["asan"], "tsan": LANES["tsan"]}.items():+ bdir = f"build/covmap-static-{lane}"+ for argv in dry_commands(f"{bdir}/{spec['target']}", [f"BUILD_DIR={bdir}"]):+ flags = " ".join(argv)+ for a in argv:+ if a.endswith(SRC_EXT) and os.path.isfile(os.path.join(ROOT, a)):+ compiled.add(a)+ if any(f in flags for f in ("-fsanitize=address", "-fsanitize=thread")):+ instrumented[a].add(lane)+ uninstrumented = sorted(s for s in compiled if not instrumented.get(s))+ lint_srcs = set(make_var("LINT_SRCS"))+ static_skipped = sorted(s for s in compiled if s not in lint_srcs)+ out = {+ "translation_units": len(compiled),+ "not_instrumented_by_any_dynamic_lane": uninstrumented,+ "not_in_static_lanes": static_skipped,+ "per_tu": {s: sorted(instrumented.get(s, ())) for s in sorted(compiled)},+ }+ if args.json:+ json.dump(out, sys.stdout, indent=1)+ print()+ return 0+ print(f"translation units in the test builds: {len(compiled)}")+ print(f" instrumented by ASan+UBSan: {sum(1 for v in instrumented.values() if 'asan' in v)}"+ f" by TSan: {sum(1 for v in instrumented.values() if 'tsan' in v)}")+ print()+ print(f" NOT instrumented by any dynamic lane ({len(uninstrumented)}):")+ for s in uninstrumented:+ print(f" {s}{' [vendored]' if is_vendored(s) else ''}")+ print()+ print(f" outside the static lanes' source list (cppcheck, clang-analyzer) ({len(static_skipped)}):")+ vend = [s for s in static_skipped if is_vendored(s)]+ own = [s for s in static_skipped if not is_vendored(s) and not s.startswith("tests/")]+ print(f" vendored: {len(vend)} project: {len(own)}")+ for s in own:+ print(f" {s}")+ return 0+++def find_llvm_tool(name: str, cc: str) -> str:+ cc_path = shutil.which(cc)+ if cc_path:+ cand = os.path.join(os.path.dirname(os.path.realpath(cc_path)), name)+ if os.access(cand, os.X_OK):+ return cand+ p = shutil.which(name)+ if p:+ return p+ if shutil.which("xcrun"):+ res = subprocess.run(["xcrun", "--find", name], capture_output=True, text=True, check=False)+ if res.returncode == 0:+ return res.stdout.strip()+ sys.exit(f"sanitizer-coverage-map: {name} not found next to {cc} or on PATH")+++def cmd_run(args) -> int:+ spec = LANES[args.lane]+ out = os.path.abspath(args.out)+ os.makedirs(out, exist_ok=True)+ bdir = f"build/covmap-{args.lane}"+ cxx = "clang++" if args.cc == "clang" else args.cc.replace("clang", "clang++")+ flags = f"{spec['value']} {COVERAGE_FLAGS}"+ subprocess.run(["make", "-f", "Makefile.cbm", f"-j{os.cpu_count() or 4}", f"{bdir}/{spec['target']}",+ f"BUILD_DIR={bdir}", f"CC={args.cc}", f"CXX={cxx}", f"{spec['var']}={flags}"],+ cwd=ROOT, check=True)+ raw_dir = os.path.join(out, f"{args.lane}-profraw")+ shutil.rmtree(raw_dir, ignore_errors=True)+ os.makedirs(raw_dir)+ env = dict(os.environ)+ env.update(spec["env"])+ env["LLVM_PROFILE_FILE"] = os.path.join(raw_dir, "%p-%m.profraw")+ runner = os.path.join(ROOT, bdir, spec["target"])+ rc = subprocess.run([runner] + args.suites, cwd=ROOT, env=env, check=False).returncode+ raws = [os.path.join(raw_dir, f) for f in os.listdir(raw_dir) if f.endswith(".profraw")]+ if not raws:+ sys.exit("sanitizer-coverage-map: the runner wrote no profile")+ profdata = find_llvm_tool("llvm-profdata", args.cc)+ cov = find_llvm_tool("llvm-cov", args.cc)+ merged = os.path.join(out, f"{args.lane}.profdata")+ subprocess.run([profdata, "merge", "-sparse", "-o", merged] + raws, check=True)+ with open(os.path.join(out, f"{args.lane}.lcov"), "w") as fh:+ subprocess.run([cov, "export", "-format=lcov", "-skip-expansions", f"-instr-profile={merged}", runner],+ cwd=ROOT, stdout=fh, check=True)+ print(f"{args.lane}: runner rc={rc}, {len(raws)} profile(s) -> {out}/{args.lane}.lcov")+ return 0 if rc == 0 else rc+++def read_lcov(path: str) -> dict[tuple[str, str], int]:+ counts: dict[tuple[str, str], int] = {}+ src = None+ for line in open(path, encoding="utf-8", errors="replace"):+ if line.startswith("SF:"):+ src = os.path.relpath(line[3:].strip(), ROOT)+ elif line.startswith("FNDA:") and src:+ count, name = line[5:].strip().split(",", 1)+ key = (src, name)+ counts[key] = max(counts.get(key, 0), int(count))+ return counts+++def cmd_union(args) -> int:+ out = os.path.abspath(args.out)+ lanes = [f[:-5] for f in sorted(os.listdir(out)) if f.endswith(".lcov")]+ if not lanes:+ sys.exit(f"sanitizer-coverage-map: no <lane>.lcov under {out} (run `run` first)")+ total: dict[tuple[str, str], int] = {}+ for lane in lanes:+ for key, c in read_lcov(os.path.join(out, f"{lane}.lcov")).items():+ total[key] = max(total.get(key, 0), c)+ never = sorted(k for k, c in total.items() if c == 0 and not k[0].startswith(("tests/", "..")))+ with open(os.path.join(out, "never-executed.txt"), "w") as fh:+ for src, fn in never:+ fh.write(f"{src}\t{fn}\n")+ by_file = defaultdict(int)+ for src, _ in never:+ by_file[src] += 1+ functions = sum(1 for k in total if not k[0].startswith(("tests/", "..")))+ print(f"lanes merged: {', '.join(lanes)}")+ print(f"functions: {functions:,} never executed under any lane: {len(never):,} "+ f"(vendored: {sum(1 for s, _ in never if is_vendored(s)):,})")+ for src, n in sorted(by_file.items(), key=lambda kv: (-kv[1], kv[0]))[: args.top]:+ print(f" {n:>6} {src}{' [vendored]' if is_vendored(src) else ''}")+ print(f"list: {out}/never-executed.txt")+ return 0+++def cmd_ratchet(args) -> int:+ current_path = os.path.join(os.path.abspath(args.out), "never-executed.txt")+ current = set(open(current_path).read().splitlines())+ if not os.path.exists(BASELINE):+ print(f"no baseline yet: copy {current_path} to {BASELINE} to arm the ratchet")+ return 0+ base = set(l for l in open(BASELINE).read().splitlines() if l and not l.startswith("#"))+ new = sorted(current - base)+ gone = sorted(base - current)+ for line in new:+ print(f"NEW never-executed: {line}")+ if gone:+ print(f"{len(gone)} function(s) now executed: lower the baseline (remove them from {BASELINE})")+ return 1 if new else 0+++def cmd_map(args) -> int:+ data = json.load(open(MAP_FILE))+ platforms = data["platforms"]+ bad = []+ for cls in data["classes"]:+ cells = cls["cells"]+ for plat in platforms:+ cell = cells.get(plat)+ if not cell:+ bad.append(f"{cls['name']} / {plat}: missing")+ elif cell.get("status") not in ("covered", "partial", "limit"):+ bad.append(f"{cls['name']} / {plat}: status must be covered, partial or limit")+ elif cell["status"] != "covered" and not cell.get("why"):+ bad.append(f"{cls['name']} / {plat}: {cell['status']} needs a why")+ if not args.quiet:+ width = max(len(c["name"]) for c in data["classes"])+ print(f"{'defect class':<{width}} " + " ".join(f"{p:<14}" for p in platforms))+ for cls in data["classes"]:+ row = []+ for plat in platforms:+ cell = cls["cells"].get(plat, {})+ row.append(f"{cell.get('status', '?') + ':' + cell.get('lane', ''):<14}"[:14])+ print(f"{cls['name']:<{width}} " + " ".join(row))+ print()+ for cls in data["classes"]:+ for plat in platforms:+ cell = cls["cells"].get(plat, {})+ if cell.get("status") in ("partial", "limit"):+ print(f" {cell['status']:<7} {cls['name']} / {plat}: {cell.get('why', '')}")+ for b in bad:+ print(f"MAP DEFECT: {b}", file=sys.stderr)+ return 1 if bad else 0+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ sub = ap.add_subparsers(dest="cmd", required=True)+ s = sub.add_parser("static")+ s.add_argument("--json", action="store_true")+ r = sub.add_parser("run")+ r.add_argument("--lane", choices=sorted(LANES), required=True)+ r.add_argument("--out", default="build/covmap")+ r.add_argument("--cc", default="clang")+ r.add_argument("suites", nargs="*")+ u = sub.add_parser("union")+ u.add_argument("--out", default="build/covmap")+ u.add_argument("--top", type=int, default=40)+ t = sub.add_parser("ratchet")+ t.add_argument("--out", default="build/covmap")+ m = sub.add_parser("map")+ m.add_argument("--quiet", action="store_true")+ args = ap.parse_args()+ return {"static": cmd_static, "run": cmd_run, "union": cmd_union,+ "ratchet": cmd_ratchet, "map": cmd_map}[args.cmd](args)+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/sanitizer-map.json b/scripts/sanitizer-map.jsonnew file mode 100644index 000000000..d2ee15fb6--- /dev/null+++ b/scripts/sanitizer-map.json@@ -0,0 +1,126 @@+{+ "about": "Defect class x platform coverage of the sanitizer lanes. Every cell is covered, partial (with why), or a named limit (with why). Validated by scripts/sanitizer-coverage-map.py map. Which functions each lane actually EXECUTES is the run/union/ratchet part of the same tool.",+ "platforms": ["linux-x64", "linux-arm64", "macos-arm64", "windows-x64", "windows-arm64"],+ "classes": [+ {+ "name": "heap/stack memory errors",+ "cells": {+ "linux-x64": {"status": "covered", "lane": "asan", "venue": "PR CI"},+ "linux-arm64": {"status": "covered", "lane": "asan", "venue": "local container"},+ "macos-arm64": {"status": "partial", "lane": "asan+diag", "why": "use-after-return and use-after-scope only on the Homebrew-LLVM diag lane, which is local-only"},+ "windows-x64": {"status": "covered", "lane": "asan", "venue": "PR CI"},+ "windows-arm64": {"status": "limit", "lane": "none", "why": "no ASan runtime for Windows ARM64; the class is covered for Windows by the x64 leg"}+ }+ },+ {+ "name": "allocator-path memory errors",+ "cells": {+ "linux-x64": {"status": "partial", "lane": "asan", "why": "test builds use the C runtime allocator: mimalloc bindings, the --wrap shims and the slab never run sanitized; mimalloc itself is built without ASan"},+ "linux-arm64": {"status": "partial", "lane": "asan", "why": "same as linux-x64"},+ "macos-arm64": {"status": "partial", "lane": "asan", "why": "same as linux-x64; the zone-observer path runs only in the memwaste flavours"},+ "windows-x64": {"status": "partial", "lane": "asan", "why": "same as linux-x64"},+ "windows-arm64": {"status": "limit", "lane": "none", "why": "no ASan runtime for Windows ARM64"}+ }+ },+ {+ "name": "undefined behaviour",+ "cells": {+ "linux-x64": {"status": "partial", "lane": "ubsan", "why": "tre (regex) is built with -fno-sanitize=alignment"},+ "linux-arm64": {"status": "partial", "lane": "ubsan", "why": "tre (regex) is built with -fno-sanitize=alignment"},+ "macos-arm64": {"status": "partial", "lane": "ubsan", "why": "tre (regex) is built with -fno-sanitize=alignment"},+ "windows-x64": {"status": "partial", "lane": "ubsan", "why": "tre (regex) is built with -fno-sanitize=alignment"},+ "windows-arm64": {"status": "partial", "lane": "ubsan-trap", "why": "trap mode: a violation stops the run without a message"}+ }+ },+ {+ "name": "leaks",+ "cells": {+ "linux-x64": {"status": "covered", "lane": "lsan", "venue": "PR CI (on by default under ASan)"},+ "linux-arm64": {"status": "covered", "lane": "lsan", "venue": "local container"},+ "macos-arm64": {"status": "covered", "lane": "test-lsan", "venue": "local (Homebrew LLVM; Apple clang refuses detect_leaks)"},+ "windows-x64": {"status": "partial", "lane": "memwaste-event", "why": "no leak sanitizer on Windows; the waste lane's live-per-site report at the last phase and the soak slope stand in"},+ "windows-arm64": {"status": "partial", "lane": "memwaste-event", "why": "no leak sanitizer on Windows; the waste lane's live-per-site report at the last phase and the soak slope stand in"}+ }+ },+ {+ "name": "uninitialised reads",+ "cells": {+ "linux-x64": {"status": "partial", "lane": "msan", "why": "CI only; deep-recursion suites need larger thread stacks under MSan"},+ "linux-arm64": {"status": "partial", "lane": "msan", "why": "locally red: three deep-recursion suites overflow their thread stacks"},+ "macos-arm64": {"status": "partial", "lane": "memwaste-access", "why": "no MSan on macOS; the access lane's read-before-write verdict on a real corpus is the second net, per block, not per byte"},+ "windows-x64": {"status": "partial", "lane": "memwaste-access", "why": "no MSan on Windows; the access lane (clang) is the second net"},+ "windows-arm64": {"status": "partial", "lane": "memwaste-access", "why": "no MSan on Windows; the access lane (clang) is the second net"}+ }+ },+ {+ "name": "data races",+ "cells": {+ "linux-x64": {"status": "partial", "lane": "tsan", "why": "a subset of suites runs under TSan"},+ "linux-arm64": {"status": "partial", "lane": "tsan", "why": "a subset of suites runs under TSan"},+ "macos-arm64": {"status": "partial", "lane": "tsan", "why": "a subset of suites runs under TSan"},+ "windows-x64": {"status": "limit", "lane": "none", "why": "no TSan on Windows"},+ "windows-arm64": {"status": "limit", "lane": "none", "why": "no TSan on Windows"}+ }+ },+ {+ "name": "static analysis",+ "cells": {+ "linux-x64": {"status": "partial", "lane": "cppcheck+lint-mem+codeql", "why": "vendored trees are filtered out of cppcheck and the analyzer gate"},+ "linux-arm64": {"status": "partial", "lane": "cppcheck+lint-mem", "why": "vendored trees are filtered out"},+ "macos-arm64": {"status": "partial", "lane": "lint-mem", "why": "vendored trees are filtered out"},+ "windows-x64": {"status": "limit", "lane": "none", "why": "static lanes are platform-independent source passes; Windows-only code is analysed by CodeQL only"},+ "windows-arm64": {"status": "limit", "lane": "none", "why": "static lanes are platform-independent source passes"}+ }+ },+ {+ "name": "hostile input",+ "cells": {+ "linux-x64": {"status": "partial", "lane": "security-fuzz+libfuzzer", "why": "fixed MCP payloads in CI; libFuzzer targets for extractors, Cypher and config editors run in dry-run/nightly"},+ "linux-arm64": {"status": "partial", "lane": "libfuzzer", "why": "local only"},+ "macos-arm64": {"status": "partial", "lane": "libfuzzer", "why": "local only (Homebrew LLVM ships libFuzzer; Apple clang does not)"},+ "windows-x64": {"status": "limit", "lane": "none", "why": "libFuzzer is not built for the MinGW toolchain"},+ "windows-arm64": {"status": "limit", "lane": "none", "why": "libFuzzer is not built for the MinGW toolchain"}+ }+ },+ {+ "name": "algorithmic blow-up",+ "cells": {+ "linux-x64": {"status": "covered", "lane": "complexity+memwaste-scaling"},+ "linux-arm64": {"status": "covered", "lane": "complexity+memwaste-scaling"},+ "macos-arm64": {"status": "covered", "lane": "complexity+memwaste-scaling"},+ "windows-x64": {"status": "partial", "lane": "complexity", "why": "the per-function scaling lane needs clang's profile runtime; whole-pipeline counters only"},+ "windows-arm64": {"status": "partial", "lane": "complexity", "why": "the per-function scaling lane needs clang's profile runtime; whole-pipeline counters only"}+ }+ },+ {+ "name": "memory waste",+ "cells": {+ "linux-x64": {"status": "covered", "lane": "memwaste-event+access"},+ "linux-arm64": {"status": "covered", "lane": "memwaste-event+access"},+ "macos-arm64": {"status": "covered", "lane": "memwaste-event+access"},+ "windows-x64": {"status": "partial", "lane": "memwaste-event", "why": "libc wrappers cover mem*/str* and FILE I/O only; the access lane needs a clang build"},+ "windows-arm64": {"status": "partial", "lane": "memwaste-event", "why": "libc wrappers cover mem*/str* and FILE I/O only"}+ }+ },+ {+ "name": "CPU waste",+ "cells": {+ "linux-x64": {"status": "covered", "lane": "memwaste-cpu+scaling"},+ "linux-arm64": {"status": "covered", "lane": "memwaste-cpu+scaling"},+ "macos-arm64": {"status": "covered", "lane": "memwaste-cpu+scaling"},+ "windows-x64": {"status": "partial", "lane": "memwaste-cpu", "why": "no system-call or pthread lock wrappers on Windows: Win32 file and lock APIs are imports --wrap cannot redirect"},+ "windows-arm64": {"status": "partial", "lane": "memwaste-cpu", "why": "no system-call or pthread lock wrappers on Windows: Win32 file and lock APIs are imports --wrap cannot redirect"}+ }+ },+ {+ "name": "executed-code coverage of the lanes",+ "cells": {+ "linux-x64": {"status": "partial", "lane": "sanitizer-coverage-map", "why": "needs a clang job; the gcc legs are not measured"},+ "linux-arm64": {"status": "covered", "lane": "sanitizer-coverage-map"},+ "macos-arm64": {"status": "covered", "lane": "sanitizer-coverage-map"},+ "windows-x64": {"status": "limit", "lane": "none", "why": "no sanitizer lane on Windows executes under clang source coverage yet"},+ "windows-arm64": {"status": "limit", "lane": "none", "why": "no sanitizer lane on Windows executes under clang source coverage yet"}+ }+ }+ ]+}diff --git a/src/foundation/mem_core.c b/src/foundation/mem_core.cindex dee1fb759..425e617fa 100644--- a/src/foundation/mem_core.c+++ b/src/foundation/mem_core.c@@ -3,6 +3,7 @@*/#include "foundation/mem_core.h"#include "foundation/mem.h"+#include "foundation/mem_events.h"/* Ownership check for blocks handed back to the core (defined with cbm_free). */static void check_owned(const void *block, const char *op);@@ -152,6 +153,7 @@ static void class_flush_one(cbm_mem_class_t cls) {}void cbm_mem_class_flush_thread(void) {+ cbm_memev_flush_thread(); /* same seam: work-item end, thread end, every reader */for (int i = 0; i < CBM_MEM_CLASS_COUNT; i++) {class_flush_one((cbm_mem_class_t)i);}@@ -205,8 +207,62 @@ size_t cbm_mem_usable_size(const void *block) {}#endif+/* -- Waste-sanitizer hooks (mem_events.h) ---------------------------------+ * Compiled to nothing outside the `memwaste` flavour. The core names the+ * caller's site and the class; who REPORTS the event depends on whether an+ * observer sees the backing allocator:+ * - backing is mimalloc called directly (CBM_BIND_TS_ALLOCATOR): no observer+ * ever sees it, the core reports everything itself;+ * - backing is plain malloc and an observer is installed: the observer+ * reports, the hint hands it the site and the class;+ * - backing is plain malloc, no observer: the core reports.+ * Frees are reported BEFORE the block goes back (see mem_events.h). */+#if defined(CBM_MEMWASTE) && CBM_MEMWASTE+static bool core_reports_frees(void) {+#if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR+ return true;+#else+ return !cbm_memev_observer_installed();+#endif+}+#define MEMEV_HINT(cls) \+ do { \+ if (cbm_memev_enabled() && !cbm_memev_hint_pending()) { \+ cbm_memev_hint(__builtin_return_address(0), (int)(cls)); \+ } \+ } while (0)+#define MEMEV_ALLOCATED(block, bytes, flags) \+ do { \+ if (cbm_memev_hint_pending()) { \+ cbm_memev_alloc_ex((block), (bytes), \+ (block) ? charge_size((block), (bytes)) : 0, NULL, \+ (flags)); \+ } \+ } while (0)+#define MEMEV_REALLOCATED(old_block, grown, bytes) \+ do { \+ if (cbm_memev_hint_pending()) { \+ cbm_memev_realloc((old_block), (grown), (bytes), \+ (grown) ? charge_size((grown), (bytes)) : 0, NULL); \+ } \+ } while (0)+#define MEMEV_FREEING(block) \+ do { \+ if (cbm_memev_enabled() && core_reports_frees()) { \+ cbm_memev_free(block); \+ } \+ } while (0)+#else+#define MEMEV_HINT(cls) ((void)0)+#define MEMEV_ALLOCATED(block, bytes, flags) ((void)0)+#define MEMEV_REALLOCATED(old_block, grown, bytes) ((void)0)+#define MEMEV_FREEING(block) ((void)0)+#endif+void *cbm_alloc(cbm_mem_class_t cls, size_t bytes) {+ MEMEV_HINT(cls);void *block = CBM_BACKING_MALLOC(bytes ? bytes : CBM_ALLOC_ONE);+ MEMEV_ALLOCATED(block, bytes, 0);if (!block) {return NULL;}@@ -215,7 +271,9 @@ void *cbm_alloc(cbm_mem_class_t cls, size_t bytes) {}void *cbm_calloc(cbm_mem_class_t cls, size_t bytes) {+ MEMEV_HINT(cls);void *block = CBM_BACKING_CALLOC(bytes ? bytes : CBM_ALLOC_ONE);+ MEMEV_ALLOCATED(block, bytes, CBM_MEMEV_ZEROED);if (!block) {return NULL;}@@ -231,7 +289,11 @@ void *cbm_realloc(cbm_mem_class_t cls, void *block, size_t bytes) {/* Measure BEFORE: after realloc the old block is gone and its size is* unknowable, so the decrement has to be computed first. */size_t old = charge_size(block, 0);+ MEMEV_HINT(cls);+ CBM_MEMEV_BACKING(1);void *next = CBM_BACKING_REALLOC(block, bytes ? bytes : CBM_ALLOC_ONE);+ CBM_MEMEV_BACKING(-1);+ MEMEV_REALLOCATED(block, next, bytes);if (!next) {return NULL; /* original intact and still charged - correct */}@@ -245,6 +307,7 @@ char *cbm_mem_strdup(cbm_mem_class_t cls, const char *s) {return NULL;}size_t len = strlen(s) + CBM_ALLOC_ONE;+ MEMEV_HINT(cls); /* the site is OUR caller, not cbm_alloc's */char *copy = (char *)cbm_alloc(cls, len);if (!copy) {return NULL;@@ -279,6 +342,7 @@ void cbm_free(cbm_mem_class_t cls, void *block) {}check_owned(block, "free");class_sub(cls, charge_size(block, 0), CBM_ALLOC_ONE);+ MEMEV_FREEING(block);CBM_BACKING_FREE(block);}diff --git a/src/foundation/mem_events.c b/src/foundation/mem_events.cnew file mode 100644index 000000000..80b4ee19f--- /dev/null+++ b/src/foundation/mem_events.c@@ -0,0 +1,2178 @@+/*+ * mem_events.c — see mem_events.h.+ *+ * Layout:+ * sites[] global, open-addressed, keyed by return address. Keys are+ * claimed with one CAS; counters are touched only under the+ * merge lock, from per-thread deltas.+ * shards[] the pointer table, 256 independently locked open-addressed+ * maps. A block is routinely freed by a thread that did not+ * allocate it, so the pointer table cannot be per-thread.+ * work[] CPU work counters keyed by (return address, kind), merged+ * the same way as the sites.+ * tl_state per-thread deltas for every site and work row the thread+ * touched, dirty lists so a flush walks only those, the core+ * handshake, the re-entrancy guard and the repeat caches.+ * access lane (CBM_MEMWASTE_ACCESS) a shadow map from every 16-byte slot of+ * a tracked block to a per-block record the compiler's load and+ * store callbacks update.+ *+ * Everything the layer owns comes from raw virtual memory: it runs INSIDE the+ * allocator it observes and must never call it.+ */+#include "foundation/mem_events.h"++#if (defined(CBM_MEMWASTE) && CBM_MEMWASTE) || \+ (defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS)++#include "foundation/mem.h"++#include <stdatomic.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>++#ifdef _WIN32+#ifndef WIN32_LEAN_AND_MEAN+#define WIN32_LEAN_AND_MEAN+#endif+#include <fcntl.h> /* _O_WRONLY, _O_CREAT, _O_APPEND, _O_BINARY */+#include <io.h>+#include <windows.h>+#else+#include <fcntl.h>+#include <pthread.h>+#include <sys/mman.h>+#include <unistd.h>+#if defined(__linux__)+#include <link.h> /* dl_iterate_phdr: the load bias of the main program */+#endif+#endif++/* The layer's own code is never instrumented by the access lane: its tables+ * are not program data, and a hook that reached instrumented code would pay+ * for itself on every table probe. */+#if defined(__clang__)+#pragma clang attribute push(__attribute__((no_sanitize("coverage"))), apply_to = function)+#endif++enum {+ MEMEV_SITE_BITS = 16,+ MEMEV_SITE_CAP = 1 << MEMEV_SITE_BITS,+ MEMEV_WORK_BITS = 17,+ MEMEV_WORK_CAP = 1 << MEMEV_WORK_BITS,+ MEMEV_SHARD_BITS = 8,+ MEMEV_SHARDS = 1 << MEMEV_SHARD_BITS,+ MEMEV_SHARD_INIT_BITS = 10,+ MEMEV_SHARD_MAX_BITS = 26,+ MEMEV_LOAD_NUM = 7, /* grow at 70 % occupancy (live + tombstones) */+ MEMEV_LOAD_DEN = 10,+ MEMEV_SEQ_BITS = 24,+ MEMEV_SEQ_MASK = (1 << MEMEV_SEQ_BITS) - 1,+ MEMEV_PHASE_CAP = 63, /* stored in 6 bits */+ MEMEV_PHASE_LABEL = 48,+ MEMEV_LINE = 2048, /* a site line with every field at 20 digits is ~1,150 bytes */+ MEMEV_PTR_SHIFT = 4, /* allocators align to 16: the low bits carry no entropy */+ MEMEV_REPEAT_SLOTS = 256,+ MEMEV_PATH_BITS = 20,+ MEMEV_PATH_CAP = 1 << MEMEV_PATH_BITS,+ MEMEV_RUN_MIN = 8, /* a run of the fill byte this long was never written */+ MEMEV_FILL_BYTE = 0xA5,+};++#define MEMEV_NO_INDEX UINT32_MAX+#define MEMEV_TOMBSTONE ((uintptr_t)1)+#define MEMEV_FILL_CAP ((size_t)256 * 1024 * 1024) /* never fill (or scan) beyond this */+#define MEMEV_SITE_MASK ((uintptr_t)0x00FFFFFFFFFFFFFFULL)++/* ── raw virtual memory ───────────────────────────────────────────────*/+/* The layer's own footprint, reported in every dump header: a sanitizer that+ * cannot say what it costs cannot be trusted to measure anybody else's waste.+ * On Windows every byte here is committed; on POSIX it is lazily backed. */+static _Atomic uint64_t g_layer_vm_bytes;+static _Atomic int64_t g_layer_threads;++static void *vm_get(size_t bytes) {+#ifdef _WIN32+ void *p = VirtualAlloc(NULL, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);+#else+ void *p = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);+ if (p == MAP_FAILED) {+ p = NULL;+ }+#endif+ if (p) {+ atomic_fetch_add_explicit(&g_layer_vm_bytes, bytes, memory_order_relaxed);+ }+ return p;+}++static void vm_put(void *p, size_t bytes) {+ if (!p) {+ return;+ }+ atomic_fetch_sub_explicit(&g_layer_vm_bytes, bytes, memory_order_relaxed);+#ifdef _WIN32+ (void)bytes;+ VirtualFree(p, 0, MEM_RELEASE);+#else+ munmap(p, bytes);+#endif+}++/* ── spin lock ────────────────────────────────────────────────────────+ * Held for a handful of instructions around one table probe. A mutex would+ * have to be initialised, and initialisation may allocate. */+typedef atomic_flag memev_lock_t;++static void lock_take(memev_lock_t *l) {+ while (atomic_flag_test_and_set_explicit(l, memory_order_acquire)) {+ }+}++static void lock_drop(memev_lock_t *l) {+ atomic_flag_clear_explicit(l, memory_order_release);+}++/* ── tables ───────────────────────────────────────────────────────────*/+typedef struct {+ uint64_t allocs;+ uint64_t frees;+ uint64_t reallocs;+ uint64_t requested_bytes;+ uint64_t usable_bytes;+ uint64_t realloc_copy_bytes;+ uint64_t short_lived;+ uint64_t raw_bytes;+ uint64_t scanned_blocks;+ uint64_t never_written_bytes;+ uint64_t over_requested_bytes;+ uint64_t zero_untouched_bytes;+ uint64_t acc_untouched_blocks;+ uint64_t acc_untouched_bytes;+ uint64_t acc_dead_blocks;+ uint64_t acc_dead_bytes;+ uint64_t acc_uninit_reads;+ uint64_t acc_idle_bytes;+ uint64_t acc_opaque_blocks;+ int64_t live_bytes;+ int64_t live_blocks;+} site_counters_t;++typedef struct {+ uint64_t calls;+ uint64_t bytes;+ uint64_t aux1;+ uint64_t aux2;+ uint64_t repeats;+ uint64_t peak;+} work_counters_t;++enum {+ META_PHASE_MASK = 0x3F,+ META_FILLED = 0x40,+ META_ZEROED = 0x80,+};++typedef struct {+ uintptr_t block; /* 0 empty, MEMEV_TOMBSTONE deleted */+ uint64_t usable;+ uint64_t requested;+ uint32_t seq; /* thread slot << MEMEV_SEQ_BITS | that thread's alloc counter */+ uint16_t site;+ int8_t mem_class;+ uint8_t meta; /* phase (6 bits) | FILLED | ZEROED */+} ptr_entry_t;++typedef struct {+ memev_lock_t lock;+ ptr_entry_t *entries;+ uint32_t bits;+ uint32_t used; /* live + tombstones */+ uint32_t live;+} shard_t;++typedef struct {+ const void *table;+ uint64_t key_hash;+ uint64_t generation;+} ht_pair_t;++typedef struct {+ const char *ptr;+ size_t len;+ uint64_t content_hash;+} strlen_pair_t;++typedef struct {+ site_counters_t *delta; /* MEMEV_SITE_CAP entries */+ uint32_t *dirty; /* site ids touched since the last flush */+ uint8_t *is_dirty;+ uint32_t dirty_count;+ work_counters_t *work_delta; /* MEMEV_WORK_CAP entries */+ uint32_t *work_dirty;+ uint8_t *work_is_dirty;+ uint32_t work_dirty_count;+ uint32_t slot; /* thread slot, 8 bits */+ uint32_t alloc_seq; /* this thread's allocation counter */+ uint64_t untracked_frees;+ uint64_t pointer_table_full;+ uint64_t site_table_full;+ uint64_t work_table_full;+ strlen_pair_t strlen_cache[MEMEV_REPEAT_SLOTS];+ ht_pair_t ht_cache[MEMEV_REPEAT_SLOTS];+ /* The handshake and the re-entrancy guard live HERE, not in _Thread_local+ * variables. On macOS the first touch of a _Thread_local in a thread+ * allocates its storage THROUGH MALLOC: inside an allocation hook that+ * re-enters the hook, which touches the variable again, until the stack is+ * gone (SIGSEGV at start-up, 2026-09-17; MinGW's emulated TLS allocates the+ * same way). A pthread key / FLS slot never allocates, and the state it+ * points at comes from raw virtual memory. */+ void *hint_site;+ int hint_class;+ bool hint_set;+ bool busy; /* the layer is invisible to itself */+ uint32_t backing; /* depth of backing allocator calls (access lane) */+ uint64_t ops; /* allocations + work notes by this thread */+ uint32_t library_depth;+ void *library_site;+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS+ uint32_t acc_ids[512];+ uint32_t acc_id_count;+#endif+} tl_state_t;++static _Atomic uintptr_t g_site_keys[MEMEV_SITE_CAP];+static site_counters_t g_site_counters[MEMEV_SITE_CAP];+static uint64_t g_snap_retained[MEMEV_SITE_CAP];+static uint64_t g_snap_dup_bytes[MEMEV_SITE_CAP];+static uint64_t g_snap_dup_blocks[MEMEV_SITE_CAP];+static _Atomic uintptr_t g_work_keys[MEMEV_WORK_CAP];+static work_counters_t g_work_counters[MEMEV_WORK_CAP];+static _Atomic uint64_t g_path_hashes[MEMEV_PATH_CAP];+static memev_lock_t g_merge_lock = ATOMIC_FLAG_INIT;+static shard_t g_shards[MEMEV_SHARDS];+static _Atomic uint64_t g_untracked_frees;+static _Atomic uint64_t g_foreign_frees;+static _Atomic uint64_t g_pointer_table_full;+static _Atomic uint64_t g_site_table_full;+static _Atomic uint64_t g_work_table_full;+static _Atomic uint32_t g_next_slot;+static _Atomic uint32_t g_phase;+static char g_phase_label[MEMEV_PHASE_LABEL] = "start";+static _Atomic int g_enabled = -1;+static _Atomic int g_fill = -1;+static _Atomic int g_dupes = -1;+static _Atomic bool g_observer_installed;+static _Atomic uintptr_t g_image_base;++static const char *const g_work_names[CBM_WORK_KIND_COUNT] = {+ "memcpy", "memmove", "memset", "memcmp", "strlen", "strcmp", "strncmp",+ "read", "write", "pread", "pwrite", "open", "stat", "fopen",+ "fread", "fwrite", "opendir", "readdir", "mutex", "parallel_for", "pool_ops",+ "ht_get", "ht_set", "dead_write", "ct_arena", "ct_dyn_array", "ct_hash_table",+};++const char *cbm_work_kind_name(cbm_work_kind_t kind) {+ if ((int)kind < 0 || kind >= CBM_WORK_KIND_COUNT) {+ return "invalid";+ }+ return g_work_names[kind];+}++static int env_flag(const char *name, int dflt) {+ const char *v = getenv(name);+ if (!v || !v[0]) {+ return dflt;+ }+ return (v[0] == '1' && v[1] == '\0') ? 1 : 0;+}++#if defined(__linux__)+extern char **environ;+#endif++bool cbm_memev_enabled(void) {+ int state = atomic_load_explicit(&g_enabled, memory_order_acquire);+ if (state >= 0) {+ return state == 1;+ }+#if defined(__linux__)+ /* Code can run before libc has set up the environment: the loader's+ * pre-init hooks (the UBSan runtime that -fsanitize-coverage links, which+ * allocates through dlsym) reach this through the malloc and mutex wrappers.+ * getenv answers NULL there, and caching that answer switched the layer+ * off for the whole process (Linux access lane, no dump, 2026-09-17).+ * Decide only once the environment exists. */+ if (!environ) {+ return false;+ }+#endif+ int on = env_flag("CBM_MEMWASTE", 0);+ atomic_store_explicit(&g_enabled, on, memory_order_release);+ return on == 1;+}++static bool fill_enabled(void) {+ int state = atomic_load_explicit(&g_fill, memory_order_relaxed);+ if (state < 0) {+ state = env_flag("CBM_MEMWASTE_FILL", 1);+ atomic_store_explicit(&g_fill, state, memory_order_relaxed);+ }+ return state == 1;+}++static bool dupes_enabled(void) {+ int state = atomic_load_explicit(&g_dupes, memory_order_relaxed);+ if (state < 0) {+ state = env_flag("CBM_MEMWASTE_DUPES", 1);+ atomic_store_explicit(&g_dupes, state, memory_order_relaxed);+ }+ return state == 1;+}++void cbm_memev_scan_for_tests(bool fill, bool dupes) {+ atomic_store_explicit(&g_fill, fill ? 1 : 0, memory_order_relaxed);+ atomic_store_explicit(&g_dupes, dupes ? 1 : 0, memory_order_relaxed);+}++void cbm_memev_force_for_tests(bool on) {+ atomic_store_explicit(&g_enabled, on ? 1 : 0, memory_order_release);+}++void cbm_memev_set_image_base(uintptr_t base) {+ atomic_store_explicit(&g_image_base, base, memory_order_release);+}++void cbm_memev_set_observer_installed(bool installed) {+ atomic_store_explicit(&g_observer_installed, installed, memory_order_release);+}++bool cbm_memev_observer_installed(void) {+ return atomic_load_explicit(&g_observer_installed, memory_order_acquire);+}++/* ── per-thread slot: a key, never _Thread_local (see tl_state_t) ───────*/+static _Atomic int g_key_state; /* 0 none, 1 being created, 2 ready */+#ifdef _WIN32+static DWORD g_key;+#else+static pthread_key_t g_key;+#endif++static size_t tl_bytes(void);+static void tl_release(void *raw);+#ifdef _WIN32+#endif++static bool key_ready(void) {+ int state = atomic_load_explicit(&g_key_state, memory_order_acquire);+ while (state != 2) {+ int expect = 0;+ if (state == 0 && atomic_compare_exchange_strong_explicit(&g_key_state, &expect, 1,+ memory_order_acq_rel,+ memory_order_acquire)) {+#ifdef _WIN32+ /* A plain TLS slot, released from the loader's thread-detach+ * callback (cbm_memev_thread_end, compat_thread.c). An FLS+ * destructor never ran in the static MinGW link, and by the time+ * the detach callback runs the FLS value is already gone -- the+ * state leaked 22 MB per thread (71 threads by the similarity pass,+ * then commit exhaustion, 2026-09-17). A TLS slot is still readable+ * there. */+ g_key = TlsAlloc();+ bool ok = g_key != TLS_OUT_OF_INDEXES;+#else+ bool ok = pthread_key_create(&g_key, tl_release) == 0;+#endif+ atomic_store_explicit(&g_key_state, ok ? 2 : 0, memory_order_release);+ if (!ok) {+ return false;+ }+ }+ state = atomic_load_explicit(&g_key_state, memory_order_acquire);+ }+ return true;+}++/* A thread whose state was already released keeps this marker in its slot. Its+ * teardown still frees (and on Windows allocates): without the marker every such+ * call would create a fresh 22 MB state that no destructor ever releases --+ * Windows runs an FLS callback once per thread, it does not re-run it the way+ * POSIX re-runs key destructors. Observed as commit exhaustion in the Windows+ * index worker, 2026-09-17. */+static char g_tl_exited_marker;+#define TL_EXITED ((void *)&g_tl_exited_marker)++static void *tl_slot_value(void) {+#ifdef _WIN32+ return TlsGetValue(g_key);+#else+ return pthread_getspecific(g_key);+#endif+}++static tl_state_t *tl_peek(void) {+ if (!key_ready()) {+ return NULL;+ }+ void *v = tl_slot_value();+ return v == TL_EXITED ? NULL : (tl_state_t *)v;+}++static size_t tl_bytes(void) {+ return sizeof(tl_state_t) + ((size_t)MEMEV_SITE_CAP * sizeof(site_counters_t)) ++ ((size_t)MEMEV_SITE_CAP * sizeof(uint32_t)) + (size_t)MEMEV_SITE_CAP ++ ((size_t)MEMEV_WORK_CAP * sizeof(work_counters_t)) ++ ((size_t)MEMEV_WORK_CAP * sizeof(uint32_t)) + (size_t)MEMEV_WORK_CAP;+}++static tl_state_t *tl_get(void) {+ if (!key_ready()) {+ return NULL;+ }+ void *v = tl_slot_value();+ if (v == TL_EXITED) {+ return NULL; /* this thread is ending: observe nothing more */+ }+ if (v) {+ return (tl_state_t *)v;+ }+ char *raw = (char *)vm_get(tl_bytes());+ if (!raw) {+ return NULL;+ }+ tl_state_t *st = (tl_state_t *)raw;+ raw += sizeof(tl_state_t);+ st->delta = (site_counters_t *)raw;+ raw += (size_t)MEMEV_SITE_CAP * sizeof(site_counters_t);+ st->dirty = (uint32_t *)raw;+ raw += (size_t)MEMEV_SITE_CAP * sizeof(uint32_t);+ st->is_dirty = (uint8_t *)raw;+ raw += (size_t)MEMEV_SITE_CAP;+ st->work_delta = (work_counters_t *)raw;+ raw += (size_t)MEMEV_WORK_CAP * sizeof(work_counters_t);+ st->work_dirty = (uint32_t *)raw;+ raw += (size_t)MEMEV_WORK_CAP * sizeof(uint32_t);+ st->work_is_dirty = (uint8_t *)raw;+ atomic_fetch_add_explicit(&g_layer_threads, 1, memory_order_relaxed);+ st->slot = atomic_fetch_add_explicit(&g_next_slot, 1, memory_order_relaxed) &+ ((1U << MEMEV_SHARD_BITS) - 1U);+#ifdef _WIN32+ if (!TlsSetValue(g_key, st)) {+ vm_put(st, tl_bytes());+ return NULL;+ }+#else+ if (pthread_setspecific(g_key, st) != 0) {+ vm_put(st, tl_bytes());+ return NULL;+ }+#endif+ return st;+}++static site_counters_t *tl_site(tl_state_t *st, uint32_t site) {+ if (!st->is_dirty[site]) {+ st->is_dirty[site] = 1;+ st->dirty[st->dirty_count++] = site;+ }+ return &st->delta[site];+}++static work_counters_t *tl_work(tl_state_t *st, uint32_t idx) {+ if (!st->work_is_dirty[idx]) {+ st->work_is_dirty[idx] = 1;+ st->work_dirty[st->work_dirty_count++] = idx;+ }+ return &st->work_delta[idx];+}++/* ── keyed tables ─────────────────────────────────────────────────────*/+static uint64_t mix(uint64_t x) {+ x ^= x >> 33;+ x *= 0xff51afd7ed558ccdULL;+ x ^= x >> 33;+ x *= 0xc4ceb9fe1a85ec53ULL;+ x ^= x >> 33;+ return x;+}++/* Index of `key` in a CAS-claimed open-addressed key table, claiming a slot on+ * first sight. MEMEV_NO_INDEX when the table is full — counted by the caller,+ * never silently dropped. */+static uint32_t key_index(_Atomic uintptr_t *keys, uint32_t cap, uintptr_t key) {+ uint32_t idx = (uint32_t)(mix((uint64_t)key) & (cap - 1));+ for (uint32_t probe = 0; probe < cap; probe++) {+ uintptr_t seen = atomic_load_explicit(&keys[idx], memory_order_acquire);+ if (seen == key) {+ return idx;+ }+ if (seen == 0) {+ uintptr_t expect = 0;+ if (atomic_compare_exchange_strong_explicit(&keys[idx], &expect, key,+ memory_order_acq_rel,+ memory_order_acquire) ||+ expect == key) {+ return idx;+ }+ }+ idx = (idx + 1) & (cap - 1);+ }+ return MEMEV_NO_INDEX;+}++static uint32_t site_index(uintptr_t site) {+ site &= MEMEV_SITE_MASK;+ if (site == 0) {+ site = (uintptr_t)1; /* "unknown caller" is a site like any other */+ }+ return key_index(g_site_keys, MEMEV_SITE_CAP, site);+}++static uint32_t work_index(cbm_work_kind_t kind, uintptr_t site) {+ uintptr_t key = (site & MEMEV_SITE_MASK) | ((uintptr_t)((unsigned)kind + 1U) << 56);+ return key_index(g_work_keys, MEMEV_WORK_CAP, key);+}++/* ── pointer shards ───────────────────────────────────────────────────*/+static shard_t *shard_of(uintptr_t block) {+ return &g_shards[mix((uint64_t)(block >> MEMEV_PTR_SHIFT)) >> (64 - MEMEV_SHARD_BITS)];+}++static uint32_t slot_of(uintptr_t block, uint32_t bits) {+ return (uint32_t)(mix((uint64_t)(block >> MEMEV_PTR_SHIFT)) & ((1U << bits) - 1U));+}++static bool shard_resize(shard_t *sh, uint32_t bits) {+ if (bits > MEMEV_SHARD_MAX_BITS) {+ return false;+ }+ size_t bytes = ((size_t)1 << bits) * sizeof(ptr_entry_t);+ ptr_entry_t *next = (ptr_entry_t *)vm_get(bytes);+ if (!next) {+ return false;+ }+ uint32_t live = 0;+ if (sh->entries) {+ uint32_t old_cap = 1U << sh->bits;+ for (uint32_t i = 0; i < old_cap; i++) {+ ptr_entry_t *e = &sh->entries[i];+ if (e->block == 0 || e->block == MEMEV_TOMBSTONE) {+ continue;+ }+ uint32_t s = slot_of(e->block, bits);+ while (next[s].block != 0) {+ s = (s + 1) & ((1U << bits) - 1U);+ }+ next[s] = *e;+ live++;+ }+ vm_put(sh->entries, ((size_t)1 << sh->bits) * sizeof(ptr_entry_t));+ }+ sh->entries = next;+ sh->bits = bits;+ sh->used = live;+ sh->live = live;+ return true;+}++/* Caller holds the shard lock. */+static ptr_entry_t *shard_insert(shard_t *sh, uintptr_t block) {+ if (!sh->entries && !shard_resize(sh, MEMEV_SHARD_INIT_BITS)) {+ return NULL;+ }+ uint32_t cap = 1U << sh->bits;+ if ((uint64_t)(sh->used + 1) * MEMEV_LOAD_DEN > (uint64_t)cap * MEMEV_LOAD_NUM) {+ /* Mostly tombstones: rehash in place at the same size; otherwise double. */+ uint32_t want = ((uint64_t)sh->live * 2 * MEMEV_LOAD_DEN > (uint64_t)cap * MEMEV_LOAD_NUM)+ ? sh->bits + 1+ : sh->bits;+ if (!shard_resize(sh, want)) {+ return NULL;+ }+ cap = 1U << sh->bits;+ }+ uint32_t s = slot_of(block, sh->bits);+ ptr_entry_t *grave = NULL;+ for (uint32_t probe = 0; probe < cap; probe++) {+ ptr_entry_t *e = &sh->entries[s];+ if (e->block == block) {+ return e; /* address reuse without a free we saw: overwrite */+ }+ if (e->block == MEMEV_TOMBSTONE && !grave) {+ grave = e;+ } else if (e->block == 0) {+ ptr_entry_t *dst = grave ? grave : e;+ if (!grave) {+ sh->used++;+ }+ sh->live++;+ dst->block = block;+ return dst;+ }+ s = (s + 1) & (cap - 1);+ }+ return NULL;+}++/* Caller holds the shard lock. */+static ptr_entry_t *shard_find(shard_t *sh, uintptr_t block) {+ if (!sh->entries) {+ return NULL;+ }+ uint32_t cap = 1U << sh->bits;+ uint32_t s = slot_of(block, sh->bits);+ for (uint32_t probe = 0; probe < cap; probe++) {+ ptr_entry_t *e = &sh->entries[s];+ if (e->block == block) {+ return e;+ }+ if (e->block == 0) {+ return NULL;+ }+ s = (s + 1) & (cap - 1);+ }+ return NULL;+}++/* ── fill pattern and scan ────────────────────────────────────────────+ * A new block is filled with MEMEV_FILL_BYTE. At free, a run of at least+ * MEMEV_RUN_MIN fill bytes is memory nobody ever wrote, and the last byte that+ * is NOT the fill byte is the high-water mark of what was written. Runs are+ * counted at word granularity (aligned 8-byte words), which is within 7 bytes+ * per run edge and fast enough for hundreds of gigabytes. A calloc block is+ * scanned the same way for zero bytes, which is an upper bound: a written zero+ * is indistinguishable from an untouched one. */+typedef struct {+ uint64_t run_bytes;+ uint64_t high_water;+} scan_t;++static scan_t scan_block(const uint8_t *p, size_t n, uint8_t pat) {+ scan_t out = {0, 0};+ uint64_t patw = 0x0101010101010101ULL * pat;+ size_t hw = n;+ while (hw > 0 && (hw & 7U) != 0 && p[hw - 1] == pat) {+ hw--;+ }+ while (hw >= 8) {+ uint64_t w;+ memcpy(&w, p + hw - 8, 8);+ if (w != patw) {+ break;+ }+ hw -= 8;+ }+ while (hw > 0 && p[hw - 1] == pat) {+ hw--;+ }+ size_t tail = n - hw;+ if (tail < MEMEV_RUN_MIN) {+ hw = n; /* a short tail of the fill byte is as likely written as not */+ tail = 0;+ }+ uint64_t runs = tail;+ size_t i = 0;+ while (i + 8 <= hw) {+ uint64_t w;+ memcpy(&w, p + i, 8);+ if (w == patw) {+ runs += 8;+ }+ i += 8;+ }+ out.run_bytes = runs;+ out.high_water = hw;+ return out;+}++/* ── access lane ──────────────────────────────────────────────────────*/+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS+enum {+ /* 48-bit user address space: arm64 Linux maps the program and its heap+ * above 2^47 (0xaaaa... and 0xffff...), which a 47-bit map silently ignored. */+ ACC_L1_BITS = 16,+ ACC_L1_CAP = 1 << ACC_L1_BITS,+ ACC_SLOT_SHIFT = 4,+ ACC_REGION_SLOTS = 1 << 28, /* one 4 GB region in 16-byte slots */+ ACC_RECS_BITS = 27,+ ACC_WRITTEN = 1,+ ACC_READ = 2,+ ACC_READ_BEFORE_WRITE = 4,+ ACC_ID_BATCH = 256,+};++typedef struct {+ _Atomic uint8_t flags;+ uint8_t last_phase;+ uint8_t zeroed;+ uint8_t filled; /* holds the fill pattern: a byte that differs was written by someone */+ uint32_t site;+ uint64_t usable;+ _Atomic uintptr_t writer;+} acc_rec_t;++static _Atomic(uint32_t *) g_acc_l2[ACC_L1_CAP];+static acc_rec_t *g_acc_recs;+static uint32_t *g_acc_free;+static uint32_t g_acc_free_count;+static uint32_t g_acc_next = 1;+static memev_lock_t g_acc_lock = ATOMIC_FLAG_INIT;+static _Atomic int g_acc_ready;++static bool acc_init(void) {+ if (atomic_load_explicit(&g_acc_ready, memory_order_acquire) == 2) {+ return true;+ }+ lock_take(&g_acc_lock);+ if (atomic_load_explicit(&g_acc_ready, memory_order_relaxed) != 2) {+ g_acc_recs = (acc_rec_t *)vm_get(((size_t)1 << ACC_RECS_BITS) * sizeof(acc_rec_t));+ g_acc_free = (uint32_t *)vm_get(((size_t)1 << ACC_RECS_BITS) * sizeof(uint32_t));+ atomic_store_explicit(&g_acc_ready, (g_acc_recs && g_acc_free) ? 2 : 1,+ memory_order_release);+ }+ lock_drop(&g_acc_lock);+ return atomic_load_explicit(&g_acc_ready, memory_order_acquire) == 2;+}++static uint32_t *acc_region(uint32_t l1, bool create) {+ uint32_t *l2 = atomic_load_explicit(&g_acc_l2[l1], memory_order_acquire);+ if (l2 || !create) {+ return l2;+ }+ uint32_t *fresh = (uint32_t *)vm_get((size_t)ACC_REGION_SLOTS * sizeof(uint32_t));+ if (!fresh) {+ return NULL;+ }+ uint32_t *expect = NULL;+ if (!atomic_compare_exchange_strong_explicit(&g_acc_l2[l1], &expect, fresh,+ memory_order_acq_rel, memory_order_acquire)) {+ vm_put(fresh, (size_t)ACC_REGION_SLOTS * sizeof(uint32_t));+ return expect;+ }+ return fresh;+}++/* Write `id` into every slot of [a, a + n). */+static void acc_map(uintptr_t a, uint64_t n, uint32_t id) {+ uintptr_t end = a + (uintptr_t)(n ? n : 1);+ while (a < end) {+ uint32_t l1 = (uint32_t)(a >> 32);+ if (l1 >= ACC_L1_CAP) {+ return;+ }+ uint32_t *l2 = acc_region(l1, id != 0);+ uintptr_t region_end = ((uintptr_t)l1 + 1) << 32;+ uintptr_t stop = end < region_end ? end : region_end;+ if (l2) {+ uint64_t first = ((uint32_t)a) >> ACC_SLOT_SHIFT;+ uint64_t last = (stop == region_end) ? (uint64_t)ACC_REGION_SLOTS+ : ((((uint32_t)(stop - 1)) >> ACC_SLOT_SHIFT) + 1);+ for (uint64_t s = first; s < last; s++) {+ l2[s] = id;+ }+ }+ a = stop;+ }+}++static uint32_t acc_lookup(uintptr_t a) {+ uint32_t l1 = (uint32_t)(a >> 32);+ if (l1 >= ACC_L1_CAP) {+ return 0;+ }+ uint32_t *l2 = atomic_load_explicit(&g_acc_l2[l1], memory_order_relaxed);+ return l2 ? l2[((uint32_t)a) >> ACC_SLOT_SHIFT] : 0;+}++static uint32_t acc_take_id(tl_state_t *st) {+ if (st->acc_id_count == 0) {+ lock_take(&g_acc_lock);+ while (st->acc_id_count < ACC_ID_BATCH) {+ uint32_t id;+ if (g_acc_free_count > 0) {+ id = g_acc_free[--g_acc_free_count];+ } else if (g_acc_next < (1U << ACC_RECS_BITS)) {+ id = g_acc_next++;+ } else {+ break;+ }+ st->acc_ids[st->acc_id_count++] = id;+ }+ lock_drop(&g_acc_lock);+ }+ return st->acc_id_count ? st->acc_ids[--st->acc_id_count] : 0;+}++static void acc_give_id(tl_state_t *st, uint32_t id) {+ if (st->acc_id_count < (uint32_t)(sizeof(st->acc_ids) / sizeof(st->acc_ids[0]))) {+ st->acc_ids[st->acc_id_count++] = id;+ return;+ }+ lock_take(&g_acc_lock);+ g_acc_free[g_acc_free_count++] = id;+ while (st->acc_id_count > ACC_ID_BATCH) {+ g_acc_free[g_acc_free_count++] = st->acc_ids[--st->acc_id_count];+ }+ lock_drop(&g_acc_lock);+}++static void acc_track(tl_state_t *st, uintptr_t block, uint64_t usable, uint32_t site,+ bool zeroed, bool filled, bool written) {+ if (!acc_init()) {+ return;+ }+ uint32_t id = acc_take_id(st);+ if (!id) {+ return;+ }+ acc_rec_t *r = &g_acc_recs[id];+ /* A block that already holds content when the layer hears of it (the+ * Linux/Windows strdup wrappers report AFTER the copy) was written by the+ * allocation itself: without this every such block read a first time was a+ * read-before-write -- 16.5 M false verdicts on Linux, 12.6 M of them the+ * reach-cache key copies (2026-09-17). */+ atomic_store_explicit(&r->flags, written ? ACC_WRITTEN : 0, memory_order_relaxed);+ atomic_store_explicit(&r->writer, 0, memory_order_relaxed);+ r->last_phase = (uint8_t)atomic_load_explicit(&g_phase, memory_order_relaxed);+ r->zeroed = zeroed ? 1 : 0;+ r->filled = filled ? 1 : 0;+ r->site = site;+ r->usable = usable;+ acc_map(block, usable, id);+}++static void work_add(tl_state_t *st, cbm_work_kind_t kind, uintptr_t site, uint64_t bytes,+ uint64_t aux1, uint64_t aux2, uint64_t repeats);++/* The verdict on a block that is going away: what its life was spent on.+ * `scan_written` is the fill scan's word on it (1 written, 0 not, -1 no scan):+ * a library call the compiler did not instrument (snprintf, strcpy, qsort)+ * writes without a store callback, and the pattern still shows it. */+static void acc_untrack(tl_state_t *st, uintptr_t block, uint64_t usable, int scan_written) {+ if (atomic_load_explicit(&g_acc_ready, memory_order_relaxed) != 2) {+ return;+ }+ uint32_t id = acc_lookup(block);+ if (!id) {+ return;+ }+ acc_rec_t *r = &g_acc_recs[id];+ uint8_t f = atomic_load_explicit(&r->flags, memory_order_relaxed);+ site_counters_t *c = tl_site(st, r->site);+ /* OPAQUE: the fill pattern shows writes, yet no callback ever fired. The+ * block was used only by code the lane cannot see -- an uninstrumented+ * library, or an aggregate copy (`*dst = *src` of a struct) that the+ * compiler expands inline after instrumentation, emitting no load or store+ * callback. Its reads are invisible too, so neither "untouched" nor+ * "written, never read" nor "idle" can be claimed about it. */+ bool opaque = scan_written == 1 && !(f & (ACC_WRITTEN | ACC_READ));+ if (opaque) {+ c->acc_opaque_blocks++;+ } else if (!(f & (ACC_WRITTEN | ACC_READ))) {+ c->acc_untouched_blocks++;+ c->acc_untouched_bytes += r->usable;+ } else if ((f & ACC_WRITTEN) && !(f & ACC_READ)) {+ /* Seen written, never seen read. A read through an inline struct copy is+ * invisible, so this is "no instrumented read", named as such in reports. */+ c->acc_dead_blocks++;+ c->acc_dead_bytes += r->usable;+ uintptr_t writer = atomic_load_explicit(&r->writer, memory_order_relaxed);+ if (writer) { /* 0: only a library wrote it -- counted at the site, no writer to name */+ work_add(st, CBM_WORK_DEAD_WRITE, writer, r->usable, 1, 0, 0);+ }+ }+ if ((f & ACC_READ_BEFORE_WRITE) && !r->zeroed) {+ c->acc_uninit_reads++;+ }+ if (!opaque && r->last_phase < (uint8_t)atomic_load_explicit(&g_phase, memory_order_relaxed)) {+ c->acc_idle_bytes += r->usable;+ }+ acc_map(block, usable, 0);+ acc_give_id(st, id);+}++static uint32_t acc_move(uintptr_t old_block, uint64_t old_usable, uintptr_t grown,+ uint64_t usable) {+ if (atomic_load_explicit(&g_acc_ready, memory_order_relaxed) != 2) {+ return 0;+ }+ uint32_t id = acc_lookup(old_block);+ if (!id) {+ return 0;+ }+ acc_map(old_block, old_usable, 0);+ g_acc_recs[id].usable = usable;+ acc_map(grown, usable, id);+ return id;+}++static inline void acc_touch(uintptr_t a, bool store, uintptr_t pc) {+ uint32_t id = acc_lookup(a);+ if (!id) {+ return;+ }+ acc_rec_t *r = &g_acc_recs[id];+ uint8_t f = atomic_load_explicit(&r->flags, memory_order_relaxed);+ if (store) {+ if (!(f & ACC_WRITTEN)) {+ atomic_fetch_or_explicit(&r->flags, ACC_WRITTEN, memory_order_relaxed);+ atomic_store_explicit(&r->writer, pc, memory_order_relaxed);+ }+ } else if (!(f & ACC_READ)) {+ uint8_t mark = ACC_READ;+ if (!(f & ACC_WRITTEN)) {+ /* The program is about to read this byte, so looking at it is safe. */+ bool library_wrote = r->filled && *(const uint8_t *)a != MEMEV_FILL_BYTE;+ mark |= library_wrote ? ACC_WRITTEN : ACC_READ_BEFORE_WRITE;+ }+ atomic_fetch_or_explicit(&r->flags, mark, memory_order_relaxed);+ }+ uint8_t ph = (uint8_t)atomic_load_explicit(&g_phase, memory_order_relaxed);+ if (r->last_phase != ph) {+ r->last_phase = ph;+ }+}++#define ACC_PC() ((uintptr_t)__builtin_return_address(0))+void __sanitizer_cov_load1(uint8_t *a) {+ acc_touch((uintptr_t)a, false, 0);+}+void __sanitizer_cov_load2(uint16_t *a) {+ acc_touch((uintptr_t)a, false, 0);+}+void __sanitizer_cov_load4(uint32_t *a) {+ acc_touch((uintptr_t)a, false, 0);+}+void __sanitizer_cov_load8(uint64_t *a) {+ acc_touch((uintptr_t)a, false, 0);+}+void __sanitizer_cov_load16(__int128 *a) {+ acc_touch((uintptr_t)a, false, 0);+}+void __sanitizer_cov_store1(uint8_t *a) {+ acc_touch((uintptr_t)a, true, ACC_PC());+}+void __sanitizer_cov_store2(uint16_t *a) {+ acc_touch((uintptr_t)a, true, ACC_PC());+}+void __sanitizer_cov_store4(uint32_t *a) {+ acc_touch((uintptr_t)a, true, ACC_PC());+}+void __sanitizer_cov_store8(uint64_t *a) {+ acc_touch((uintptr_t)a, true, ACC_PC());+}+void __sanitizer_cov_store16(__int128 *a) {+ acc_touch((uintptr_t)a, true, ACC_PC());+}+void __sanitizer_cov_bool_flag_init(uint8_t *start, uint8_t *stop) {+ (void)start;+ (void)stop;+}++/* Verdicts are per block, so touching both ends of the range marks every block+ * it can belong to (a range never spans two allocations in a correct program). */+void cbm_memev_access(const void *p, size_t n, bool store, void *pc) {+ if (!p || n == 0 || atomic_load_explicit(&g_acc_ready, memory_order_relaxed) != 2) {+ return;+ }+ tl_state_t *st = tl_get();+ if (!st || st->busy || st->backing) {+ return;+ }+ uintptr_t a = (uintptr_t)p;+ acc_touch(a, store, (uintptr_t)pc);+ if (n > 1) {+ acc_touch(a + (uintptr_t)(n - 1), store, (uintptr_t)pc);+ }+}++void cbm_memev_backing(int delta) {+ tl_state_t *st = tl_get();+ if (st) {+ st->backing = (uint32_t)((int)st->backing + delta);+ }+}+#else+static void work_add(tl_state_t *st, cbm_work_kind_t kind, uintptr_t site, uint64_t bytes,+ uint64_t aux1, uint64_t aux2, uint64_t repeats);+#endif /* CBM_MEMWASTE_ACCESS */++/* ── the handshake ────────────────────────────────────────────────────*/+void cbm_memev_hint(void *site, int mem_class) {+ if (!cbm_memev_enabled()) {+ return;+ }+ tl_state_t *st = tl_get();+ if (st) {+ st->hint_site = site;+ st->hint_class = mem_class;+ st->hint_set = true;+ }+}++bool cbm_memev_hint_pending(void) {+ if (!cbm_memev_enabled()) {+ return false;+ }+ const tl_state_t *st = tl_peek();+ return st && st->hint_set;+}++/* ── events ───────────────────────────────────────────────────────────*/+static void record_alloc(tl_state_t *st, uintptr_t block, size_t requested, size_t usable,+ uintptr_t site, int mem_class, unsigned flags) {+ st->ops++;+ uint32_t sid = site_index(site);+ if (sid == MEMEV_NO_INDEX) {+ st->site_table_full++;+ return;+ }+ uint64_t charged = usable ? usable : requested;+ st->alloc_seq = (st->alloc_seq + 1) & (uint32_t)MEMEV_SEQ_MASK;++ bool zeroed = (flags & CBM_MEMEV_ZEROED) != 0;+ bool filled = false;+ if (!zeroed && !(flags & (CBM_MEMEV_FOREIGN | CBM_MEMEV_WRITTEN)) && charged <= MEMEV_FILL_CAP &&+ fill_enabled()) {+ memset((void *)block, MEMEV_FILL_BYTE, (size_t)charged);+ filled = true;+ }++ shard_t *sh = shard_of(block);+ lock_take(&sh->lock);+ ptr_entry_t *e = shard_insert(sh, block);+ if (e) {+ e->usable = charged;+ e->requested = requested;+ e->seq = (st->slot << MEMEV_SEQ_BITS) | st->alloc_seq;+ e->site = (uint16_t)sid;+ e->mem_class = (int8_t)mem_class;+ e->meta = (uint8_t)((atomic_load_explicit(&g_phase, memory_order_relaxed) &+ META_PHASE_MASK) |+ (filled ? META_FILLED : 0) | (zeroed ? META_ZEROED : 0));+ }+ lock_drop(&sh->lock);+ if (!e) {+ st->pointer_table_full++;+ }++ site_counters_t *c = tl_site(st, sid);+ c->allocs++;+ c->requested_bytes += requested;+ c->usable_bytes += charged;+ if (mem_class == CBM_MEMEV_CLASS_RAW) {+ c->raw_bytes += charged;+ }+ if (e) {+ c->live_bytes += (int64_t)charged;+ c->live_blocks++;+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS+ if (!(flags & (CBM_MEMEV_FOREIGN | CBM_MEMEV_LIBRARY))) {+ acc_track(st, block, charged, sid, zeroed, filled,+ (flags & CBM_MEMEV_WRITTEN) != 0);+ }+#endif+ }+}++void cbm_memev_alloc_ex(void *block, size_t requested, size_t usable, void *site,+ unsigned flags) {+ if (!cbm_memev_enabled()) {+ return;+ }+ tl_state_t *st = tl_get();+ if (!st || st->busy) {+ return;+ }+ bool hinted = st->hint_set;+ st->hint_set = false; /* consumed even when the allocation failed */+ if (!block) {+ return;+ }+ st->busy = true;+ record_alloc(st, (uintptr_t)block, requested, usable,+ (uintptr_t)(hinted ? st->hint_site : site),+ hinted ? st->hint_class : CBM_MEMEV_CLASS_RAW,+ hinted ? (flags & ~(unsigned)CBM_MEMEV_FOREIGN) : flags);+ st->busy = false;+}++void cbm_memev_alloc(void *block, size_t requested, size_t usable, void *site) {+ cbm_memev_alloc_ex(block, requested, usable, site, 0);+}++/* Removes the record of `block`. Returns false when the layer never saw it. */+static bool take_entry(uintptr_t block, ptr_entry_t *out) {+ shard_t *sh = shard_of(block);+ lock_take(&sh->lock);+ ptr_entry_t *e = shard_find(sh, block);+ bool found = e != NULL;+ if (found) {+ *out = *e;+ e->block = MEMEV_TOMBSTONE;+ sh->live--;+ }+ lock_drop(&sh->lock);+ return found;+}++void cbm_memev_free(void *block) {+ if (!block || !cbm_memev_enabled()) {+ return;+ }+ tl_state_t *st = tl_get();+ if (!st || st->busy) {+ return;+ }+ st->busy = true;+ ptr_entry_t e;+ if (take_entry((uintptr_t)block, &e)) {+ site_counters_t *c = tl_site(st, e.site);+ c->frees++;+ c->live_bytes -= (int64_t)e.usable;+ c->live_blocks--;+ uint32_t slot = e.seq >> MEMEV_SEQ_BITS;+ uint32_t age =+ (st->alloc_seq - (e.seq & (uint32_t)MEMEV_SEQ_MASK)) & (uint32_t)MEMEV_SEQ_MASK;+ if (slot == st->slot && age <= CBM_MEMEV_CHURN_WINDOW) {+ c->short_lived++;+ }+ int scan_written = -1;+ if ((e.meta & (META_FILLED | META_ZEROED)) && e.usable <= MEMEV_FILL_CAP) {+ bool zeroed = (e.meta & META_ZEROED) != 0;+ scan_t sc = scan_block((const uint8_t *)block, (size_t)e.usable,+ zeroed ? 0 : MEMEV_FILL_BYTE);+ scan_written = sc.high_water > 0 ? 1 : 0;+ c->scanned_blocks++;+ if (zeroed) {+ c->zero_untouched_bytes += e.usable - sc.high_water;+ } else {+ c->never_written_bytes += sc.run_bytes;+ if (e.requested > sc.high_water) {+ c->over_requested_bytes += e.requested - sc.high_water;+ }+ }+ }+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS+ acc_untrack(st, (uintptr_t)block, e.usable, scan_written);+#else+ (void)scan_written;+#endif+ } else {+ st->untracked_frees++;+ }+ st->busy = false;+}++void cbm_memev_free_foreign(void *block) {+ if (!block || !cbm_memev_enabled()) {+ return;+ }+ atomic_fetch_add_explicit(&g_foreign_frees, 1, memory_order_relaxed);+}++void cbm_memev_realloc(void *old_block, void *grown, size_t requested, size_t usable,+ void *site) {+ if (!cbm_memev_enabled()) {+ return;+ }+ tl_state_t *st = tl_get();+ if (!st || st->busy) {+ return;+ }+ bool hinted = st->hint_set;+ void *hint_site = st->hint_site;+ int hint_class = st->hint_class;+ st->hint_set = false;+ if (!grown) {+ return;+ }+ st->busy = true;+ ptr_entry_t e;+ bool had = old_block && take_entry((uintptr_t)old_block, &e);+ if (had) {+ /* The record keeps its ORIGINAL site: a realloc chain is one object+ * growing, and its copies are that object's cost. */+ site_counters_t *c = tl_site(st, e.site);+ c->reallocs++;+ c->live_bytes -= (int64_t)e.usable;+ c->live_blocks--;+ if (grown != old_block) {+ c->realloc_copy_bytes += e.usable < requested ? e.usable : requested;+ }+ uint64_t charged = usable ? usable : requested;+ uint8_t meta = e.meta;+ if ((meta & META_FILLED) && charged > e.usable && charged <= MEMEV_FILL_CAP) {+ /* the new tail is uninitialised: it gets the pattern too */+ memset((char *)grown + e.usable, MEMEV_FILL_BYTE, (size_t)(charged - e.usable));+ } else if (charged > e.usable) {+ meta &= (uint8_t)~(META_FILLED | META_ZEROED); /* mixed content: stop scanning */+ }+ shard_t *sh = shard_of((uintptr_t)grown);+ lock_take(&sh->lock);+ ptr_entry_t *n = shard_insert(sh, (uintptr_t)grown);+ if (n) {+ n->usable = charged;+ n->requested = requested;+ n->seq = e.seq;+ n->site = e.site;+ n->mem_class = e.mem_class;+ n->meta = meta;+ }+ lock_drop(&sh->lock);+ c->requested_bytes += requested;+ c->usable_bytes += charged;+ if (e.mem_class == CBM_MEMEV_CLASS_RAW) {+ c->raw_bytes += charged;+ }+ if (n) {+ c->live_bytes += (int64_t)charged;+ c->live_blocks++;+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS+ (void)acc_move((uintptr_t)old_block, e.usable, (uintptr_t)grown, charged);+#endif+ } else {+ st->pointer_table_full++;+ }+ } else {+ /* The old block was never seen (allocated before the observer, or by a+ * path it does not cover): the grown block already holds the program's+ * data, so it is recorded without the fill. realloc(NULL, n) is a fresh+ * allocation and gets the fill like any malloc. */+ record_alloc(st, (uintptr_t)grown, requested, usable,+ (uintptr_t)(hinted ? hint_site : site),+ hinted ? hint_class : CBM_MEMEV_CLASS_RAW,+ old_block ? CBM_MEMEV_WRITTEN : 0); /* realloc(NULL, n) is a malloc */+ }+ st->busy = false;+}++/* ── CPU work ─────────────────────────────────────────────────────────*/+static void work_add(tl_state_t *st, cbm_work_kind_t kind, uintptr_t site, uint64_t bytes,+ uint64_t aux1, uint64_t aux2, uint64_t repeats) {+ st->ops++;+ uint32_t idx = work_index(kind, site);+ if (idx == MEMEV_NO_INDEX) {+ st->work_table_full++;+ return;+ }+ work_counters_t *w = tl_work(st, idx);+ w->calls++;+ w->bytes += bytes;+ w->aux1 += aux1;+ w->aux2 += aux2;+ w->repeats += repeats;+ bool container = kind == CBM_WORK_CT_ARENA || kind == CBM_WORK_CT_DYN_ARRAY ||+ kind == CBM_WORK_CT_HASH_TABLE;+ uint64_t single = container ? (bytes > aux1 ? bytes - aux1 : 0) : bytes;+ if (single > w->peak) {+ w->peak = single;+ }+}++void cbm_memev_library_enter(void *site) {+ tl_state_t *st = cbm_memev_enabled() ? tl_get() : NULL;+ if (st && st->library_depth++ == 0) {+ st->library_site = site;+ }+}++void cbm_memev_library_leave(void) {+ tl_state_t *st = cbm_memev_enabled() ? tl_get() : NULL;+ if (st && st->library_depth > 0 && --st->library_depth == 0) {+ st->library_site = NULL;+ }+}++void *cbm_memev_library_site(void) {+ tl_state_t *st = cbm_memev_enabled() ? tl_get() : NULL;+ return st ? st->library_site : NULL;+}++#if defined(CBM_MEMWASTE_PROFILE) && CBM_MEMWASTE_PROFILE+void __llvm_profile_set_filename(const char *pattern);+int __llvm_profile_write_file(void);+#endif++void cbm_memev_process_role(const char *role) {+#if defined(CBM_MEMWASTE_PROFILE) && CBM_MEMWASTE_PROFILE+ const char *dir = getenv("CBM_PROFILE_DIR");+ if (!dir || !dir[0] || !role) {+ return;+ }+ /* The runtime keeps the pointer, not a copy: the pattern must outlive us. */+ static char pattern[1024];+ int n = snprintf(pattern, sizeof(pattern), "%s/%s-%%p.profraw", dir, role);+ if (n > 0 && (size_t)n < sizeof(pattern)) {+ __llvm_profile_set_filename(pattern);+ }+#else+ (void)role;+#endif+}++void cbm_memev_process_exit(void) {+ if (cbm_memev_enabled()) {+ const char *out = getenv("CBM_MEMWASTE_OUT");+ if (out && out[0]) {+ (void)cbm_memev_dump(out, "exit");+ }+ }+#if defined(CBM_MEMWASTE_PROFILE) && CBM_MEMWASTE_PROFILE+ (void)__llvm_profile_write_file();+#endif+}++uint64_t cbm_memev_thread_ops(void) {+ if (!cbm_memev_enabled()) {+ return 0;+ }+ tl_state_t *st = tl_get();+ return st ? st->ops : 0;+}++void cbm_work_note(cbm_work_kind_t kind, void *site, uint64_t bytes, uint64_t aux1,+ uint64_t aux2) {+ if ((int)kind < 0 || kind >= CBM_WORK_KIND_COUNT || !cbm_memev_enabled()) {+ return;+ }+ tl_state_t *st = tl_get();+ if (!st || st->busy) {+ return;+ }+ st->busy = true;+ work_add(st, kind, (uintptr_t)site, bytes, aux1, aux2, 0);+ st->busy = false;+}++__attribute__((noinline)) void cbm_memev_da_note(size_t cap_bytes, size_t used_bytes,+ size_t cap_slots) {+ if (!cbm_memev_enabled() || cap_bytes == 0) {+ return;+ }+ uint64_t growths = 0;+ for (size_t c = 8; c < cap_slots; c <<= 1) {+ growths++;+ }+ cbm_work_note(CBM_WORK_CT_DYN_ARRAY, __builtin_return_address(0), cap_bytes, used_bytes,+ growths);+}++/* FNV-1a over the first bytes of a string of known length; the length is+ * mixed in, so two strings that share a long prefix still differ. */+enum { MEMEV_CONTENT_HASH_MAX = 4096 };+static uint64_t bytes_hash(const char *s, size_t len) {+ uint64_t h = 0xcbf29ce484222325ULL ^ (uint64_t)len;+ size_t n = len < MEMEV_CONTENT_HASH_MAX ? len : MEMEV_CONTENT_HASH_MAX;+ for (size_t i = 0; s && i < n; i++) {+ h ^= (uint8_t)s[i];+ h *= 0x100000001b3ULL;+ }+ return h;+}++void cbm_work_note_strlen(void *site, const char *s, size_t len) {+ if (!cbm_memev_enabled()) {+ return;+ }+ tl_state_t *st = tl_get();+ if (!st || st->busy) {+ return;+ }+ st->busy = true;+ strlen_pair_t *slot =+ &st->strlen_cache[mix((uint64_t)(uintptr_t)s) & (MEMEV_REPEAT_SLOTS - 1)];+ uint64_t content = bytes_hash(s, len);+ uint64_t repeat =+ (slot->ptr == s && slot->len == len && slot->content_hash == content) ? 1 : 0;+ slot->ptr = s;+ slot->len = len;+ slot->content_hash = content;+ work_add(st, CBM_WORK_STRLEN, (uintptr_t)site, len, 0, 0, repeat);+ st->busy = false;+}++/* FNV-1a over a NUL-terminated path, without calling the strlen we wrap. */+static uint64_t path_hash(const char *p) {+ uint64_t h = 0xcbf29ce484222325ULL;+ for (; p && *p; p++) {+ h ^= (uint8_t)*p;+ h *= 0x100000001b3ULL;+ }+ return h;+}++void cbm_work_note_path(cbm_work_kind_t kind, void *site, const char *path, bool failed) {+ if (!cbm_memev_enabled()) {+ return;+ }+ tl_state_t *st = tl_get();+ if (!st || st->busy) {+ return;+ }+ st->busy = true;+ uint64_t h = mix(path_hash(path) ^ ((uint64_t)kind << 58)) | 1U;+ uint64_t repeat = 0;+ uint32_t idx = (uint32_t)(h & (MEMEV_PATH_CAP - 1));+ for (uint32_t probe = 0; probe < MEMEV_PATH_CAP; probe++) {+ uint64_t seen = atomic_load_explicit(&g_path_hashes[idx], memory_order_relaxed);+ if (seen == h) {+ repeat = 1;+ break;+ }+ if (seen == 0) {+ uint64_t expect = 0;+ if (atomic_compare_exchange_strong_explicit(&g_path_hashes[idx], &expect, h,+ memory_order_relaxed,+ memory_order_relaxed)) {+ break;+ }+ if (expect == h) {+ repeat = 1;+ break;+ }+ }+ idx = (idx + 1) & (MEMEV_PATH_CAP - 1);+ }+ work_add(st, kind, (uintptr_t)site, 0, failed ? 1 : 0, 0, repeat);+ st->busy = false;+}++void cbm_work_note_ht(cbm_work_kind_t kind, void *site, const void *table, const char *key,+ uint64_t generation, bool hit) {+ if (!cbm_memev_enabled()) {+ return;+ }+ tl_state_t *st = tl_get();+ if (!st || st->busy) {+ return;+ }+ st->busy = true;+ uint64_t repeat = 0;+ if (kind == CBM_WORK_HT_GET) {+ uint64_t key_hash = path_hash(key);+ ht_pair_t *slot =+ &st->ht_cache[mix((uint64_t)(uintptr_t)table ^ (key_hash * 0x9e3779b97f4a7c15ULL)) &+ (MEMEV_REPEAT_SLOTS - 1)];+ repeat = (slot->table == table && slot->key_hash == key_hash &&+ slot->generation == generation)+ ? 1+ : 0;+ slot->table = table;+ slot->key_hash = key_hash;+ slot->generation = generation;+ }+ work_add(st, kind, (uintptr_t)site, 0, (kind == CBM_WORK_HT_GET && !hit) ? 1 : 0, 0, repeat);+ st->busy = false;+}++/* ── merge ────────────────────────────────────────────────────────────*/+static void flush_state(tl_state_t *st) {+ lock_take(&g_merge_lock);+ for (uint32_t i = 0; i < st->dirty_count; i++) {+ uint32_t sid = st->dirty[i];+ site_counters_t *d = &st->delta[sid];+ site_counters_t *g = &g_site_counters[sid];+ g->allocs += d->allocs;+ g->frees += d->frees;+ g->reallocs += d->reallocs;+ g->requested_bytes += d->requested_bytes;+ g->usable_bytes += d->usable_bytes;+ g->realloc_copy_bytes += d->realloc_copy_bytes;+ g->short_lived += d->short_lived;+ g->raw_bytes += d->raw_bytes;+ g->scanned_blocks += d->scanned_blocks;+ g->never_written_bytes += d->never_written_bytes;+ g->over_requested_bytes += d->over_requested_bytes;+ g->zero_untouched_bytes += d->zero_untouched_bytes;+ g->acc_untouched_blocks += d->acc_untouched_blocks;+ g->acc_untouched_bytes += d->acc_untouched_bytes;+ g->acc_dead_blocks += d->acc_dead_blocks;+ g->acc_dead_bytes += d->acc_dead_bytes;+ g->acc_uninit_reads += d->acc_uninit_reads;+ g->acc_idle_bytes += d->acc_idle_bytes;+ g->acc_opaque_blocks += d->acc_opaque_blocks;+ g->live_bytes += d->live_bytes;+ g->live_blocks += d->live_blocks;+ memset(d, 0, sizeof(*d));+ st->is_dirty[sid] = 0;+ }+ st->dirty_count = 0;+ for (uint32_t i = 0; i < st->work_dirty_count; i++) {+ uint32_t idx = st->work_dirty[i];+ work_counters_t *d = &st->work_delta[idx];+ work_counters_t *g = &g_work_counters[idx];+ g->calls += d->calls;+ g->bytes += d->bytes;+ g->aux1 += d->aux1;+ g->aux2 += d->aux2;+ g->repeats += d->repeats;+ if (d->peak > g->peak) {+ g->peak = d->peak;+ }+ memset(d, 0, sizeof(*d));+ st->work_is_dirty[idx] = 0;+ }+ st->work_dirty_count = 0;+ lock_drop(&g_merge_lock);+ atomic_fetch_add_explicit(&g_untracked_frees, st->untracked_frees, memory_order_relaxed);+ atomic_fetch_add_explicit(&g_pointer_table_full, st->pointer_table_full,+ memory_order_relaxed);+ atomic_fetch_add_explicit(&g_site_table_full, st->site_table_full, memory_order_relaxed);+ atomic_fetch_add_explicit(&g_work_table_full, st->work_table_full, memory_order_relaxed);+ st->untracked_frees = 0;+ st->pointer_table_full = 0;+ st->site_table_full = 0;+ st->work_table_full = 0;+}++void cbm_memev_flush_thread(void) {+ if (!cbm_memev_enabled()) {+ return;+ }+ tl_state_t *st = tl_peek();+ if (!st || st->busy) {+ return;+ }+ st->busy = true;+ flush_state(st);+ st->busy = false;+}++/* Thread exit (pthread key / FLS destructor): hand over what is pending, then+ * give the raw memory back. */+static void tl_release(void *raw);++void cbm_memev_thread_end(void) {+ if (atomic_load_explicit(&g_key_state, memory_order_acquire) != 2) {+ return; /* no thread ever had state: nothing to release */+ }+ void *v = tl_slot_value();+ if (v && v != TL_EXITED) {+ tl_release(v);+ }+}++static void tl_release(void *raw) {+ tl_state_t *st = (tl_state_t *)raw;+ if (!st || raw == TL_EXITED) {+ return;+ }+ st->busy = true;+ flush_state(st);+ /* Mark the slot before the memory goes: whatever the rest of the thread's+ * teardown allocates or frees must find the marker, neither a freed pointer+ * nor an empty slot that would build a new state. */+#ifdef _WIN32+ (void)TlsSetValue(g_key, TL_EXITED);+#else+ (void)pthread_setspecific(g_key, TL_EXITED);+#endif+ atomic_fetch_sub_explicit(&g_layer_threads, 1, memory_order_relaxed);+ vm_put(st, tl_bytes());+}++/* ── phase snapshot ───────────────────────────────────────────────────+ * Live blocks allocated in an earlier phase are RETAINED: the question is+ * whether the phase that is running still needs them. Small live blocks with+ * identical content are DUPLICATES: one interned copy would do. Both are+ * snapshots of the live set taken after the workers joined. */+static uint64_t content_hash(const uint8_t *p, size_t n) {+ uint64_t h = 0xcbf29ce484222325ULL ^ ((uint64_t)n * 0x9e3779b97f4a7c15ULL);+ for (size_t i = 0; i < n; i++) {+ h ^= p[i];+ h *= 0x100000001b3ULL;+ }+ return h | 1U;+}++static void snapshot_phase(uint32_t phase_now) {+ memset(g_snap_retained, 0, sizeof(g_snap_retained));+ memset(g_snap_dup_bytes, 0, sizeof(g_snap_dup_bytes));+ memset(g_snap_dup_blocks, 0, sizeof(g_snap_dup_blocks));+ bool dupes = dupes_enabled();+ uint64_t live_total = 0;+ for (uint32_t i = 0; i < MEMEV_SHARDS; i++) {+ live_total += g_shards[i].live;+ }+ size_t set_cap = 1024;+ while (set_cap < live_total * 2 && set_cap < ((size_t)1 << 30)) {+ set_cap <<= 1;+ }+ uint64_t *set = dupes ? (uint64_t *)vm_get(set_cap * sizeof(uint64_t)) : NULL;+ for (uint32_t i = 0; i < MEMEV_SHARDS; i++) {+ shard_t *sh = &g_shards[i];+ lock_take(&sh->lock);+ uint32_t cap = sh->entries ? (1U << sh->bits) : 0;+ for (uint32_t j = 0; j < cap; j++) {+ const ptr_entry_t *e = &sh->entries[j];+ if (e->block == 0 || e->block == MEMEV_TOMBSTONE) {+ continue;+ }+ if ((uint32_t)(e->meta & META_PHASE_MASK) < phase_now) {+ g_snap_retained[e->site] += e->usable;+ }+ if (!set || e->usable > CBM_MEMEV_DUP_MAX || !(e->meta & (META_FILLED | META_ZEROED))) {+ continue;+ }+ const uint8_t *p = (const uint8_t *)e->block;+ scan_t sc = scan_block(p, (size_t)e->usable,+ (e->meta & META_ZEROED) ? 0 : MEMEV_FILL_BYTE);+ if (sc.high_water == 0) {+ continue; /* nothing written: that is never-written waste, not a duplicate */+ }+ uint64_t h = content_hash(p, (size_t)sc.high_water);+ size_t s = (size_t)(h & (set_cap - 1));+ for (;;) {+ if (set[s] == 0) {+ set[s] = h;+ break;+ }+ if (set[s] == h) {+ g_snap_dup_bytes[e->site] += e->usable;+ g_snap_dup_blocks[e->site]++;+ break;+ }+ s = (s + 1) & (set_cap - 1);+ }+ }+ lock_drop(&sh->lock);+ }+ vm_put(set, set_cap * sizeof(uint64_t));+}++void cbm_memev_phase(const char *label) {+ if (!cbm_memev_enabled()) {+ return;+ }+ tl_state_t *st = tl_get();+ if (!st || st->busy) {+ return;+ }+ st->busy = true;+ flush_state(st);+ uint32_t next = atomic_load_explicit(&g_phase, memory_order_relaxed) + 1;+ if (next <= MEMEV_PHASE_CAP) {+ atomic_store_explicit(&g_phase, next, memory_order_relaxed);+ }+ lock_take(&g_merge_lock);+ snprintf(g_phase_label, sizeof(g_phase_label), "%s", label ? label : "-");+ lock_drop(&g_merge_lock);+ snapshot_phase(atomic_load_explicit(&g_phase, memory_order_relaxed));+ st->busy = false;+ const char *out = getenv("CBM_MEMWASTE_OUT");+ if (out && out[0]) {+ char why[MEMEV_PHASE_LABEL + 8];+ snprintf(why, sizeof(why), "phase:%s", label ? label : "-");+ (void)cbm_memev_dump(out, why);+ }+}++/* ── report ───────────────────────────────────────────────────────────*/+void cbm_memev_totals(cbm_memev_totals_t *out) {+ if (!out) {+ return;+ }+ memset(out, 0, sizeof(*out));+ cbm_memev_flush_thread();+ lock_take(&g_merge_lock);+ for (uint32_t i = 0; i < MEMEV_SITE_CAP; i++) {+ if (atomic_load_explicit(&g_site_keys[i], memory_order_relaxed) == 0) {+ continue;+ }+ const site_counters_t *g = &g_site_counters[i];+ out->sites++;+ out->allocs += g->allocs;+ out->frees += g->frees;+ out->reallocs += g->reallocs;+ out->requested_bytes += g->requested_bytes;+ out->usable_bytes += g->usable_bytes;+ out->realloc_copy_bytes += g->realloc_copy_bytes;+ out->raw_bytes += g->raw_bytes;+ if (g->live_bytes > 0) {+ out->live_bytes += (uint64_t)g->live_bytes;+ }+ if (g->live_blocks > 0) {+ out->live_blocks += (uint64_t)g->live_blocks;+ }+ }+ lock_drop(&g_merge_lock);+ out->untracked_frees = atomic_load_explicit(&g_untracked_frees, memory_order_relaxed);+ out->foreign_frees = atomic_load_explicit(&g_foreign_frees, memory_order_relaxed);+ out->pointer_table_full = atomic_load_explicit(&g_pointer_table_full, memory_order_relaxed);+ out->site_table_full = atomic_load_explicit(&g_site_table_full, memory_order_relaxed);+ out->work_table_full = atomic_load_explicit(&g_work_table_full, memory_order_relaxed);+}++static void site_row_of(uint32_t i, uintptr_t key, cbm_memev_site_t *row) {+ const site_counters_t *g = &g_site_counters[i];+ *row = (cbm_memev_site_t){+ .site = key,+ .allocs = g->allocs,+ .frees = g->frees,+ .reallocs = g->reallocs,+ .requested_bytes = g->requested_bytes,+ .usable_bytes = g->usable_bytes,+ .realloc_copy_bytes = g->realloc_copy_bytes,+ .live_bytes = g->live_bytes > 0 ? (uint64_t)g->live_bytes : 0,+ .live_blocks = g->live_blocks > 0 ? (uint64_t)g->live_blocks : 0,+ .short_lived = g->short_lived,+ .raw_bytes = g->raw_bytes,+ .scanned_blocks = g->scanned_blocks,+ .never_written_bytes = g->never_written_bytes,+ .over_requested_bytes = g->over_requested_bytes,+ .zero_untouched_bytes = g->zero_untouched_bytes,+ .retained_bytes = g_snap_retained[i],+ .dup_bytes = g_snap_dup_bytes[i],+ .dup_blocks = g_snap_dup_blocks[i],+ .acc_untouched_blocks = g->acc_untouched_blocks,+ .acc_untouched_bytes = g->acc_untouched_bytes,+ .acc_dead_blocks = g->acc_dead_blocks,+ .acc_dead_bytes = g->acc_dead_bytes,+ .acc_uninit_reads = g->acc_uninit_reads,+ .acc_idle_bytes = g->acc_idle_bytes,+ .acc_opaque_blocks = g->acc_opaque_blocks,+ };+}++static int by_usable_desc(const void *a, const void *b) {+ const cbm_memev_site_t *x = (const cbm_memev_site_t *)a;+ const cbm_memev_site_t *y = (const cbm_memev_site_t *)b;+ if (x->usable_bytes != y->usable_bytes) {+ return x->usable_bytes < y->usable_bytes ? 1 : -1;+ }+ /* Equal bytes: order by address, so the report is a function of the data. */+ return x->site < y->site ? -1 : (x->site > y->site ? 1 : 0);+}++/* Top `max` sites by cumulative usable bytes, selected without allocating. */+size_t cbm_memev_sites(cbm_memev_site_t *out, size_t max) {+ if (!out || max == 0) {+ return 0;+ }+ cbm_memev_flush_thread();+ size_t n = 0;+ lock_take(&g_merge_lock);+ for (uint32_t i = 0; i < MEMEV_SITE_CAP; i++) {+ uintptr_t key = atomic_load_explicit(&g_site_keys[i], memory_order_relaxed);+ if (key == 0) {+ continue;+ }+ cbm_memev_site_t row;+ site_row_of(i, key, &row);+ if (n < max) {+ out[n++] = row;+ continue;+ }+ size_t smallest = 0;+ for (size_t k = 1; k < n; k++) {+ if (by_usable_desc(&out[k], &out[smallest]) > 0) {+ smallest = k;+ }+ }+ if (by_usable_desc(&row, &out[smallest]) < 0) {+ out[smallest] = row;+ }+ }+ lock_drop(&g_merge_lock);+ tl_state_t *st = tl_get();+ bool was_busy = st && st->busy;+ if (st) {+ st->busy = true; /* qsort is libc; keep anything it does out of the tables */+ }+ qsort(out, n, sizeof(out[0]), by_usable_desc);+ if (st) {+ st->busy = was_busy;+ }+ return n;+}++size_t cbm_work_rows(cbm_work_row_t *out, size_t max) {+ if (!out || max == 0) {+ return 0;+ }+ cbm_memev_flush_thread();+ size_t n = 0;+ lock_take(&g_merge_lock);+ for (uint32_t i = 0; i < MEMEV_WORK_CAP && n < max; i++) {+ uintptr_t key = atomic_load_explicit(&g_work_keys[i], memory_order_relaxed);+ if (key == 0) {+ continue;+ }+ const work_counters_t *g = &g_work_counters[i];+ out[n++] = (cbm_work_row_t){+ .site = key & MEMEV_SITE_MASK,+ .kind = (cbm_work_kind_t)((key >> 56) - 1),+ .calls = g->calls,+ .bytes = g->bytes,+ .aux1 = g->aux1,+ .aux2 = g->aux2,+ .repeats = g->repeats,+ .peak = g->peak,+ };+ }+ lock_drop(&g_merge_lock);+ return n;+}++static bool write_all(int fd, const char *buf, size_t len) {+ while (len > 0) {+#ifdef _WIN32+ int put = _write(fd, buf, (unsigned)len);+#else+ ssize_t put = write(fd, buf, len);+#endif+ if (put <= 0) {+ return false;+ }+ buf += put;+ len -= (size_t)put;+ }+ return true;+}++#if defined(__linux__)+static int first_object_bias(struct dl_phdr_info *info, size_t size, void *out) {+ (void)size;+ *(uintptr_t *)out = (uintptr_t)info->dlpi_addr;+ return 1; /* the first object is the main program */+}+#endif++/* Sites are raw return addresses; the report subtracts the image base to get+ * the address the symboliser expects. macOS records it when the zone observer+ * installs; Linux (the load bias: 0 for a non-PIE binary) and Windows (the+ * runtime module base) record it here, once, before the first dump. */+static void ensure_image_base(void) {+ if (atomic_load_explicit(&g_image_base, memory_order_relaxed) != 0) {+ return;+ }+#if defined(__linux__)+ uintptr_t bias = 0;+ dl_iterate_phdr(first_object_bias, &bias);+ atomic_store_explicit(&g_image_base, bias, memory_order_relaxed);+#elif defined(_WIN32)+ atomic_store_explicit(&g_image_base, (uintptr_t)GetModuleHandleW(NULL), memory_order_relaxed);+#endif+}++static bool dump_locked(const char *path, const char *why) {+ ensure_image_base();+ cbm_memev_totals_t t;+ cbm_memev_totals(&t);+#ifdef _WIN32+ int fd = _open(path, _O_WRONLY | _O_CREAT | _O_APPEND | _O_BINARY, 0600);+ int pid = (int)GetCurrentProcessId();+#else+ int fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0600);+ int pid = (int)getpid();+#endif+ if (fd < 0) {+ return false;+ }+ char label[MEMEV_PHASE_LABEL];+ lock_take(&g_merge_lock);+ snprintf(label, sizeof(label), "%s", g_phase_label);+ lock_drop(&g_merge_lock);+ char line[MEMEV_LINE];+ int len = snprintf(+ line, sizeof(line),+ "{\"memwaste\":2,\"why\":\"%s\",\"pid\":%d,\"image_base\":\"0x%llx\",\"phase\":%u,"+ "\"phase_label\":\"%s\",\"rss_bytes\":%llu,\"allocs\":%llu,\"frees\":%llu,"+ "\"untracked_frees\":%llu,\"foreign_frees\":%llu,\"reallocs\":%llu,\"requested_bytes\":%llu,\"usable_bytes\":%llu,"+ "\"realloc_copy_bytes\":%llu,\"live_bytes\":%llu,\"live_blocks\":%llu,\"raw_bytes\":%llu,"+ "\"sites\":%llu,\"site_table_full\":%llu,\"pointer_table_full\":%llu,"+ "\"work_table_full\":%llu,\"layer_vm_bytes\":%llu,\"layer_threads\":%lld,"+ "\"fill\":%d,\"access\":%d}\n",+ why ? why : "-", pid,+ (unsigned long long)atomic_load_explicit(&g_image_base, memory_order_relaxed),+ atomic_load_explicit(&g_phase, memory_order_relaxed), label,+ (unsigned long long)cbm_mem_rss(), (unsigned long long)t.allocs,+ (unsigned long long)t.frees, (unsigned long long)t.untracked_frees,+ (unsigned long long)t.foreign_frees,+ (unsigned long long)t.reallocs, (unsigned long long)t.requested_bytes,+ (unsigned long long)t.usable_bytes, (unsigned long long)t.realloc_copy_bytes,+ (unsigned long long)t.live_bytes, (unsigned long long)t.live_blocks,+ (unsigned long long)t.raw_bytes, (unsigned long long)t.sites,+ (unsigned long long)t.site_table_full, (unsigned long long)t.pointer_table_full,+ (unsigned long long)t.work_table_full,+ (unsigned long long)atomic_load_explicit(&g_layer_vm_bytes, memory_order_relaxed),+ (long long)atomic_load_explicit(&g_layer_threads, memory_order_relaxed),+ fill_enabled() ? 1 : 0,+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS+ 1+#else+ 0+#endif+ );+ bool ok = len > 0 && (size_t)len < sizeof(line) && write_all(fd, line, (size_t)len);++ lock_take(&g_merge_lock);+ for (uint32_t i = 0; ok && i < MEMEV_SITE_CAP; i++) {+ uintptr_t key = atomic_load_explicit(&g_site_keys[i], memory_order_relaxed);+ if (key == 0) {+ continue;+ }+ cbm_memev_site_t r;+ site_row_of(i, key, &r);+ len = snprintf(+ line, sizeof(line),+ "{\"site\":\"0x%llx\",\"allocs\":%llu,\"frees\":%llu,\"reallocs\":%llu,"+ "\"requested_bytes\":%llu,\"usable_bytes\":%llu,\"realloc_copy_bytes\":%llu,"+ "\"live_bytes\":%llu,\"live_blocks\":%llu,\"short_lived\":%llu,\"raw_bytes\":%llu,"+ "\"scanned_blocks\":%llu,\"never_written_bytes\":%llu,\"over_requested_bytes\":%llu,"+ "\"zero_untouched_bytes\":%llu,\"retained_bytes\":%llu,\"dup_bytes\":%llu,"+ "\"dup_blocks\":%llu,\"acc_untouched_blocks\":%llu,\"acc_untouched_bytes\":%llu,"+ "\"acc_dead_blocks\":%llu,\"acc_dead_bytes\":%llu,\"acc_uninit_reads\":%llu,"+ "\"acc_idle_bytes\":%llu,\"acc_opaque_blocks\":%llu}\n",+ (unsigned long long)r.site, (unsigned long long)r.allocs, (unsigned long long)r.frees,+ (unsigned long long)r.reallocs, (unsigned long long)r.requested_bytes,+ (unsigned long long)r.usable_bytes, (unsigned long long)r.realloc_copy_bytes,+ (unsigned long long)r.live_bytes, (unsigned long long)r.live_blocks,+ (unsigned long long)r.short_lived, (unsigned long long)r.raw_bytes,+ (unsigned long long)r.scanned_blocks, (unsigned long long)r.never_written_bytes,+ (unsigned long long)r.over_requested_bytes,+ (unsigned long long)r.zero_untouched_bytes, (unsigned long long)r.retained_bytes,+ (unsigned long long)r.dup_bytes, (unsigned long long)r.dup_blocks,+ (unsigned long long)r.acc_untouched_blocks, (unsigned long long)r.acc_untouched_bytes,+ (unsigned long long)r.acc_dead_blocks, (unsigned long long)r.acc_dead_bytes,+ (unsigned long long)r.acc_uninit_reads, (unsigned long long)r.acc_idle_bytes,+ (unsigned long long)r.acc_opaque_blocks);+ ok = len > 0 && (size_t)len < sizeof(line) && write_all(fd, line, (size_t)len);+ }+ for (uint32_t i = 0; ok && i < MEMEV_WORK_CAP; i++) {+ uintptr_t key = atomic_load_explicit(&g_work_keys[i], memory_order_relaxed);+ if (key == 0) {+ continue;+ }+ const work_counters_t *g = &g_work_counters[i];+ if (g->calls == 0) {+ continue;+ }+ len = snprintf(line, sizeof(line),+ "{\"work\":\"%s\",\"site\":\"0x%llx\",\"calls\":%llu,\"bytes\":%llu,"+ "\"aux1\":%llu,\"aux2\":%llu,\"repeats\":%llu,\"peak\":%llu}\n",+ cbm_work_kind_name((cbm_work_kind_t)((key >> 56) - 1)),+ (unsigned long long)(key & MEMEV_SITE_MASK), (unsigned long long)g->calls,+ (unsigned long long)g->bytes, (unsigned long long)g->aux1,+ (unsigned long long)g->aux2, (unsigned long long)g->repeats,+ (unsigned long long)g->peak);+ ok = len > 0 && (size_t)len < sizeof(line) && write_all(fd, line, (size_t)len);+ }+ lock_drop(&g_merge_lock);+#ifdef _WIN32+ _close(fd);+#else+ close(fd);+#endif+ return ok;+}++bool cbm_memev_dump(const char *path, const char *why) {+ if (!path || !path[0]) {+ return false;+ }+ static memev_lock_t dump_lock = ATOMIC_FLAG_INIT;+ tl_state_t *st = tl_get();+ bool was_busy = st && st->busy;+ if (st && !was_busy) {+ flush_state(st); /* this thread's pending events belong in the dump */+ }+ if (st) {+ st->busy = true; /* the dump's own writes and allocations are not the program's work */+ }+ lock_take(&dump_lock);+ bool ok = dump_locked(path, why);+ lock_drop(&dump_lock);+ if (st) {+ st->busy = was_busy;+ }+ return ok;+}++void cbm_memev_reset_for_tests(void) {+ cbm_memev_flush_thread();+ lock_take(&g_merge_lock);+ for (uint32_t i = 0; i < MEMEV_SITE_CAP; i++) {+ atomic_store_explicit(&g_site_keys[i], 0, memory_order_relaxed);+ }+ for (uint32_t i = 0; i < MEMEV_WORK_CAP; i++) {+ atomic_store_explicit(&g_work_keys[i], 0, memory_order_relaxed);+ }+ for (uint32_t i = 0; i < MEMEV_PATH_CAP; i++) {+ atomic_store_explicit(&g_path_hashes[i], 0, memory_order_relaxed);+ }+ memset(g_site_counters, 0, sizeof(g_site_counters));+ memset(g_work_counters, 0, sizeof(g_work_counters));+ memset(g_snap_retained, 0, sizeof(g_snap_retained));+ memset(g_snap_dup_bytes, 0, sizeof(g_snap_dup_bytes));+ memset(g_snap_dup_blocks, 0, sizeof(g_snap_dup_blocks));+ lock_drop(&g_merge_lock);+ for (uint32_t i = 0; i < MEMEV_SHARDS; i++) {+ shard_t *sh = &g_shards[i];+ lock_take(&sh->lock);+ vm_put(sh->entries, sh->entries ? ((size_t)1 << sh->bits) * sizeof(ptr_entry_t) : 0);+ sh->entries = NULL;+ sh->bits = 0;+ sh->used = 0;+ sh->live = 0;+ lock_drop(&sh->lock);+ }+ atomic_store_explicit(&g_untracked_frees, 0, memory_order_relaxed);+ atomic_store_explicit(&g_foreign_frees, 0, memory_order_relaxed);+ atomic_store_explicit(&g_pointer_table_full, 0, memory_order_relaxed);+ atomic_store_explicit(&g_site_table_full, 0, memory_order_relaxed);+ atomic_store_explicit(&g_work_table_full, 0, memory_order_relaxed);+ atomic_store_explicit(&g_phase, 0, memory_order_relaxed);+ atomic_store_explicit(&g_fill, 0, memory_order_relaxed); /* tests use synthetic addresses */+ tl_state_t *st = tl_peek();+ if (st) {+ st->hint_set = false;+ memset(st->strlen_cache, 0, sizeof(st->strlen_cache));+ memset(st->ht_cache, 0, sizeof(st->ht_cache));+ }+}++#if defined(__clang__)+#pragma clang attribute pop+#endif++#else /* layer compiled out: every entry point is an empty stub */++bool cbm_memev_enabled(void) {+ return false;+}+void cbm_memev_alloc(void *block, size_t requested, size_t usable, void *site) {+ (void)block;+ (void)requested;+ (void)usable;+ (void)site;+}+void cbm_memev_alloc_ex(void *block, size_t requested, size_t usable, void *site,+ unsigned flags) {+ (void)block;+ (void)requested;+ (void)usable;+ (void)site;+ (void)flags;+}+void cbm_memev_free(void *block) {+ (void)block;+}+void cbm_memev_free_foreign(void *block) {+ (void)block;+}+void cbm_memev_realloc(void *old_block, void *grown, size_t requested, size_t usable,+ void *site) {+ (void)old_block;+ (void)grown;+ (void)requested;+ (void)usable;+ (void)site;+}+void cbm_memev_hint(void *site, int mem_class) {+ (void)site;+ (void)mem_class;+}+bool cbm_memev_hint_pending(void) {+ return false;+}+void cbm_memev_set_image_base(uintptr_t base) {+ (void)base;+}+void cbm_memev_set_observer_installed(bool installed) {+ (void)installed;+}+bool cbm_memev_observer_installed(void) {+ return false;+}+void cbm_memev_flush_thread(void) {+}+void cbm_memev_phase(const char *label) {+ (void)label;+}+const char *cbm_work_kind_name(cbm_work_kind_t kind) {+ (void)kind;+ return "off";+}+uint64_t cbm_memev_thread_ops(void) {+ return 0;+}+void cbm_memev_process_role(const char *role) {+ (void)role;+}+void cbm_memev_thread_end(void) {}+void cbm_memev_process_exit(void) {}+void cbm_memev_library_enter(void *site) {+ (void)site;+}+void cbm_memev_library_leave(void) {}+void *cbm_memev_library_site(void) {+ return NULL;+}+void cbm_work_note(cbm_work_kind_t kind, void *site, uint64_t bytes, uint64_t aux1,+ uint64_t aux2) {+ (void)kind;+ (void)site;+ (void)bytes;+ (void)aux1;+ (void)aux2;+}+void cbm_memev_da_note(size_t cap_bytes, size_t used_bytes, size_t cap_slots) {+ (void)cap_bytes;+ (void)used_bytes;+ (void)cap_slots;+}+void cbm_work_note_strlen(void *site, const char *s, size_t len) {+ (void)site;+ (void)s;+ (void)len;+}+void cbm_work_note_path(cbm_work_kind_t kind, void *site, const char *path, bool failed) {+ (void)kind;+ (void)site;+ (void)path;+ (void)failed;+}+void cbm_work_note_ht(cbm_work_kind_t kind, void *site, const void *table, const char *key,+ uint64_t generation, bool hit) {+ (void)kind;+ (void)site;+ (void)table;+ (void)key;+ (void)generation;+ (void)hit;+}+void cbm_memev_totals(cbm_memev_totals_t *out) {+ if (out) {+ *out = (cbm_memev_totals_t){0};+ }+}+size_t cbm_memev_sites(cbm_memev_site_t *out, size_t max) {+ (void)out;+ (void)max;+ return 0;+}+size_t cbm_work_rows(cbm_work_row_t *out, size_t max) {+ (void)out;+ (void)max;+ return 0;+}+bool cbm_memev_dump(const char *path, const char *why) {+ (void)path;+ (void)why;+ return false;+}+void cbm_memev_reset_for_tests(void) {+}+void cbm_memev_scan_for_tests(bool fill, bool dupes) {+ (void)fill;+ (void)dupes;+}+void cbm_memev_force_for_tests(bool on) {+ (void)on;+}++#endifdiff --git a/src/foundation/mem_events.h b/src/foundation/mem_events.hnew file mode 100644index 000000000..30eb05088--- /dev/null+++ b/src/foundation/mem_events.h@@ -0,0 +1,359 @@+/*+ * mem_events.h — the waste sanitizer: memory and CPU work that is correct but+ * unnecessary, attributed to the code that caused it.+ *+ * ASan, MSan, TSan and LSan find memory that is WRONG. Nothing in the tree+ * measured memory or work that is merely WASTED — allocator slack, blocks+ * bigger than what is written into them, churn an arena would have served,+ * realloc chains copying the same bytes again, duplicate strings, memory held+ * across phases that never touch it, the same string measured twice, the same+ * file opened twice. The memory-core rounds found such waste BY HAND; this+ * layer turns it into a ranked report.+ *+ * On 2026-09-17 a forwarding malloc-zone observer showed the Go indexing worker+ * making 92.3 million raw allocations OUTSIDE the memory core on macOS — 128 GB+ * requested cumulatively in a 41 s run — that no instrument in the tree saw.+ *+ * WHERE IT LISTENS+ * - the memory core (mem_core.c) announces its own calls with a site + class;+ * - platform observers see the malloc family underneath: the Linux and+ * Windows --wrap shims, the macOS malloc-zone observer;+ * - bound allocators that bypass both (SQLite, tree-sitter -> mimalloc) report+ * directly;+ * - containers (arena, dynamic array, hash table) report capacity vs use;+ * - libc wrappers (mem_override_libc.c) report byte and system-call work;+ * - the ACCESS lane (-fsanitize-coverage=trace-loads,trace-stores) reports+ * reads and writes of every tracked block.+ *+ * COSTS NOTHING UNLESS ASKED FOR. Compiled in only under -DCBM_MEMWASTE=1 (the+ * `memwaste` build flavour, `make MEMWASTE=1`) and dormant unless CBM_MEMWASTE=1+ * is set. The access lane additionally needs -DCBM_MEMWASTE_ACCESS=1+ * (`make MEMWASTE_ACCESS=1`). Release binaries contain none of it.+ *+ * NO PER-BLOCK HEADER (see mem_core.h: 16 bytes x ~100 M kernel blocks is+ * 1.6 GB). Block state lives in side tables backed by raw virtual memory, so+ * the layer never calls the allocator it is observing.+ *+ * DETERMINISM (O9). Every quantity a gate may read is a SUM over events —+ * per-site totals — or a snapshot taken at a phase mark after the workers+ * joined. Nothing here reads a clock.+ *+ * ENVIRONMENT+ * CBM_MEMWASTE=1 wake the layer+ * CBM_MEMWASTE_OUT=path append JSON-lines dumps at every phase mark+ * CBM_MEMWASTE_FILL=0 do not fill new blocks with the never-written pattern+ * CBM_MEMWASTE_DUPES=0 skip the duplicate-content scan at phase marks+ */+#ifndef CBM_MEM_EVENTS_H+#define CBM_MEM_EVENTS_H++#include <stdbool.h>+#include <stddef.h>+#include <stdint.h>++/* Class value for blocks that never passed through the memory core. */+#define CBM_MEMEV_CLASS_RAW (-1)++/* Allocation flags. */+enum {+ CBM_MEMEV_ZEROED = 1u, /* calloc: the allocator zeroed it, never fill it */+ CBM_MEMEV_FOREIGN = 2u, /* a system library allocating for itself: observe, never fill */+ CBM_MEMEV_LIBRARY = 4u, /* allocated inside a wrapped library call, charged to its caller */+ /* The block already holds its content when it is reported (a strdup copy, a+ * realloc whose old block the layer never saw): observe, NEVER fill -- the+ * fill pattern would overwrite live data. */+ CBM_MEMEV_WRITTEN = 8u,+};++/* Blocks at or below this size take part in the duplicate-content scan. */+enum { CBM_MEMEV_DUP_MAX = 256 };++/* Freed by the allocating thread within this many of its own allocations:+ * churn an arena or a stack buffer would have served. */+enum { CBM_MEMEV_CHURN_WINDOW = 64 };++/* True when the layer is compiled in AND CBM_MEMWASTE=1. Cached after the+ * first call; safe from inside an allocator hook (it never allocates). */+bool cbm_memev_enabled(void);++/* ── Allocation events ────────────────────────────────────────────────+ * `site` is a return address inside the code that asked for the memory.+ * `usable` is what the allocator actually handed out (0 when unknown: the+ * layer then charges `requested`). All return immediately when dormant. */+void cbm_memev_alloc(void *block, size_t requested, size_t usable, void *site);+void cbm_memev_alloc_ex(void *block, size_t requested, size_t usable, void *site, unsigned flags);+void cbm_memev_free(void *block);+/* A free of a block another allocator made (Windows: the C runtime or a system+ * DLL allocating where --wrap cannot reach). Counted apart from untracked+ * frees: its allocation was never observable, so it says what the layer cannot+ * see, not that the layer missed something it should have seen. */+void cbm_memev_free_foreign(void *block);+/* One event for a realloc: the old block's record moves to `grown`, the copy+ * the allocator had to make is charged to the ORIGINAL allocation site. */+void cbm_memev_realloc(void *old_block, void *grown, size_t requested, size_t usable, void *site);++/* ── Core <-> observer handshake ──────────────────────────────────────+ * The memory core and a platform observer can both see one allocation. The+ * core announces itself first; the observer consumes the hint and the event+ * carries the CORE's caller as site and the class attached. When nothing+ * consumed the hint, the core reports the event itself:+ *+ * cbm_memev_hint(site, cls);+ * block = backing_malloc(n);+ * if (cbm_memev_hint_pending()) {+ * cbm_memev_alloc(block, n, usable, NULL);+ * }+ *+ * FREES HAVE NO HANDSHAKE, on purpose: a free event must be recorded BEFORE+ * the block goes back to the allocator, or another thread can be handed the+ * same address, record its allocation, and have it erased by the late free.+ * Where an observer sees the backing allocator it reports frees; everywhere+ * else the core reports them itself, first. */+void cbm_memev_hint(void *site, int mem_class);+bool cbm_memev_hint_pending(void);+void cbm_memev_set_observer_installed(bool installed);+bool cbm_memev_observer_installed(void);++/* Load address of the main image, written into every report header: sites are+ * raw addresses in a randomised process, and offline symbolisation needs the+ * base they are relative to. 0 = unknown. */+void cbm_memev_set_image_base(uintptr_t base);++/* ── Merge points ─────────────────────────────────────────────────────+ * Per-thread counters are pushed to the shared tables here: work-item end,+ * thread end, before any reader. */+void cbm_memev_flush_thread(void);++/* This thread is ending: merge its counters and release its state now. POSIX+ * does this from a pthread key destructor. A static MinGW link runs no FLS or+ * key destructors at thread exit (the same reason compat_thread.c releases the+ * mimalloc thread heap from a loader TLS callback, #581), so Windows threads+ * left their 22 MB state behind -- 71 of them by the similarity pass on the+ * Go corpus, then commit exhaustion (2026-09-17). Safe to call twice. */+void cbm_memev_thread_end(void);++/* A pipeline phase boundary. Blocks allocated from now on carry the new phase;+ * the live set is snapshotted (retained-across-phases, duplicate content) and+ * a dump is appended to CBM_MEMWASTE_OUT. Called after the workers joined. */+void cbm_memev_phase(const char *label);++/* ── Process role and exit ────────────────────────────────────────────+ * The index worker leaves with _Exit (no exit handlers: multi-GB teardown is+ * skipped), which also skips every at-exit writer -- the waste dump and clang's+ * profile runtime. cbm_memev_process_exit writes both first; it is a no-op when+ * neither lane is compiled in or active.+ * cbm_memev_process_role names this process's execution-count profile+ * <CBM_PROFILE_DIR>/<role>-<pid>.profraw, so the scaling lane can compare the+ * worker's work without the daemon and CLI processes mixed in. */+void cbm_memev_process_role(const char *role);+void cbm_memev_process_exit(void);++/* ── Access lane: what the compiler cannot see ─────────────────────────+ * The access lane learns about reads and writes from compiler callbacks, and a+ * library call is not compiled with them: a block filled by memcpy or read(2)+ * would look never written. The library wrappers report those ranges here.+ * While the allocator itself runs (a realloc copying the old block), its reads+ * and writes are not the program's: the shims bracket every backing call that+ * can copy. Both vanish outside the access lane. */+#if defined(CBM_MEMWASTE_ACCESS) && CBM_MEMWASTE_ACCESS+void cbm_memev_access(const void *p, size_t n, bool store, void *pc);+void cbm_memev_backing(int delta);+#define CBM_MEMEV_ACCESS(p, n, store, pc) cbm_memev_access((p), (n), (store), (pc))+#define CBM_MEMEV_BACKING(delta) cbm_memev_backing(delta)+#else+#define CBM_MEMEV_ACCESS(p, n, store, pc) ((void)0)+#define CBM_MEMEV_BACKING(delta) ((void)0)+#endif++/* A wrapped library call (fopen, fread, opendir...) is running on this thread.+ * Memory the library allocates for itself meanwhile -- a FILE buffer, a DIR+ * stream -- is charged to `site` (the wrapper's caller) with CBM_MEMEV_LIBRARY:+ * it is counted like any memory, but the access lane leaves it alone, because+ * only the uninstrumented library ever reads it. Nestable; the outermost site+ * wins. */+void cbm_memev_library_enter(void *site);+void cbm_memev_library_leave(void);+void *cbm_memev_library_site(void);++/* ── CPU work ─────────────────────────────────────────────────────────+ * Work is COUNTED, never timed: a count is a function of the code and the+ * input, so it can gate. Semantics of the counters per kind:+ *+ * kind bytes aux1 aux2+ * memcpy..strncmp bytes processed - -+ * read, write,+ * pread, pwrite,+ * fread, fwrite bytes transferred tiny calls (<512 B) -+ * open, stat,+ * fopen, opendir - failures -+ * readdir - - -+ * mutex - contended -+ * parallel_for items imbalance items workers+ * pool_ops events by workers imbalance events workers+ * ht_get - misses -+ * ht_set - table growths -+ * dead_write bytes never read blocks -+ * ct_arena,+ * ct_dyn_array,+ * ct_hash_table capacity bytes used bytes growths+ *+ * `repeats` counts the same work done again, where the answer cannot have+ * changed: strlen of the same pointer holding the same bytes (recent, same+ * thread), open/stat of the same path (process-wide), a hash-table lookup of+ * the same key TEXT in the same table with no write to that table in between+ * (recent, same thread). A reused buffer holding a different string is not a+ * repeat.+ *+ * `peak` is the largest single note: the biggest call for byte work and I/O,+ * and for containers the largest capacity one instance left unused. Containers+ * report at every reset, so `bytes` and `aux1` sum over USES (a per-thread arena+ * rewound for 28,000 files reports 28,000 times); `peak` is the memory figure. */+typedef enum {+ CBM_WORK_MEMCPY = 0,+ CBM_WORK_MEMMOVE,+ CBM_WORK_MEMSET,+ CBM_WORK_MEMCMP,+ CBM_WORK_STRLEN,+ CBM_WORK_STRCMP,+ CBM_WORK_STRNCMP,+ CBM_WORK_READ,+ CBM_WORK_WRITE,+ CBM_WORK_PREAD,+ CBM_WORK_PWRITE,+ CBM_WORK_OPEN,+ CBM_WORK_STAT,+ CBM_WORK_FOPEN,+ CBM_WORK_FREAD,+ CBM_WORK_FWRITE,+ CBM_WORK_OPENDIR,+ CBM_WORK_READDIR,+ CBM_WORK_MUTEX,+ CBM_WORK_PARALLEL_FOR,+ CBM_WORK_POOL_OPS,+ CBM_WORK_HT_GET,+ CBM_WORK_HT_SET,+ CBM_WORK_DEAD_WRITE,+ CBM_WORK_CT_ARENA,+ CBM_WORK_CT_DYN_ARRAY,+ CBM_WORK_CT_HASH_TABLE,+ CBM_WORK_KIND_COUNT+} cbm_work_kind_t;++enum { CBM_WORK_TINY_IO = 512 };++const char *cbm_work_kind_name(cbm_work_kind_t kind);+void cbm_work_note(cbm_work_kind_t kind, void *site, uint64_t bytes, uint64_t aux1,+ uint64_t aux2);+void cbm_work_note_strlen(void *site, const char *s, size_t len);++/* Events (allocations and work notes) this thread has produced so far. A pool+ * compares its workers by this: a pool over N work SLOTS whose workers each pull+ * their own items shows its real imbalance only in the work done. */+uint64_t cbm_memev_thread_ops(void);++#if defined(CBM_MEMWASTE) && CBM_MEMWASTE && !defined(_WIN32)+/* The one counting lock behind every pthread_mutex_lock in the flavour+ * (mem_override_libc.c): try first, count a contended acquisition at `site`,+ * then block. cbm_mutex_lock passes its own caller as the site. */+int cbm_memev_mutex_lock(void *pthread_mutex, void *site);+#endif+void cbm_work_note_path(cbm_work_kind_t kind, void *site, const char *path, bool failed);+/* `generation` is the table's write counter: a lookup repeats only when the+ * table has not been written since the same key was last looked up. */+void cbm_work_note_ht(cbm_work_kind_t kind, void *site, const void *table, const char *key,+ uint64_t generation, bool hit);++/* Containers report capacity against use when they are freed or reset.+ * `site` is where the container was created. */+static inline void cbm_memev_container(cbm_work_kind_t kind, void *site, uint64_t capacity,+ uint64_t used, uint64_t growths) {+ cbm_work_note(kind, site, capacity, used, growths);+}++/* A dynamic array being freed: capacity against count, in bytes. The site is+ * the function that freed it (the helper is never inlined, so its return+ * address lands there). Growths are the doublings from 8 slots to `cap`. */+void cbm_memev_da_note(size_t cap_bytes, size_t used_bytes, size_t cap_slots);++/* ── Report ───────────────────────────────────────────────────────────*/+typedef struct {+ uint64_t allocs; /* alloc events seen */+ uint64_t frees; /* free events matched to a tracked block */+ uint64_t untracked_frees; /* frees of blocks the layer never saw */+ uint64_t foreign_frees; /* frees of blocks another allocator made */+ uint64_t reallocs;+ uint64_t requested_bytes; /* cumulative */+ uint64_t usable_bytes; /* cumulative */+ uint64_t realloc_copy_bytes; /* bytes the allocator had to move */+ uint64_t live_bytes; /* usable bytes outstanding now */+ uint64_t live_blocks;+ uint64_t raw_bytes; /* cumulative usable bytes never tagged by the core */+ uint64_t sites; /* distinct allocation sites */+ /* Records the fixed tables could not hold. A profiler that silently loses+ * records produces false confidence; these make the blind spot a number. */+ uint64_t site_table_full;+ uint64_t pointer_table_full;+ uint64_t work_table_full;+} cbm_memev_totals_t;++void cbm_memev_totals(cbm_memev_totals_t *out);++typedef struct {+ uintptr_t site;+ uint64_t allocs;+ uint64_t frees;+ uint64_t reallocs;+ uint64_t requested_bytes;+ uint64_t usable_bytes;+ uint64_t realloc_copy_bytes;+ uint64_t live_bytes;+ uint64_t live_blocks;+ uint64_t short_lived;+ uint64_t raw_bytes;+ /* fill-pattern scan at free */+ uint64_t scanned_blocks;+ uint64_t never_written_bytes; /* bytes still holding the fill pattern */+ uint64_t over_requested_bytes; /* requested beyond the last written byte */+ uint64_t zero_untouched_bytes; /* calloc: trailing zero bytes (upper bound) */+ /* phase snapshot (the most recent phase mark) */+ uint64_t retained_bytes; /* live, allocated in an EARLIER phase */+ uint64_t dup_bytes; /* live small blocks identical to another */+ uint64_t dup_blocks;+ /* access lane */+ uint64_t acc_untouched_blocks;+ uint64_t acc_untouched_bytes;+ uint64_t acc_dead_blocks; /* written, never read */+ uint64_t acc_dead_bytes;+ uint64_t acc_uninit_reads; /* read before any write, not calloc */+ uint64_t acc_idle_bytes; /* last accessed in an earlier phase than freed */+ uint64_t acc_opaque_blocks; /* written (fill scan) with no callback: no verdict possible */+} cbm_memev_site_t;++size_t cbm_memev_sites(cbm_memev_site_t *out, size_t max);++typedef struct {+ uintptr_t site;+ cbm_work_kind_t kind;+ uint64_t calls;+ uint64_t bytes;+ uint64_t aux1;+ uint64_t aux2;+ uint64_t repeats;+ uint64_t peak;+} cbm_work_row_t;++size_t cbm_work_rows(cbm_work_row_t *out, size_t max);++/* Append the report as JSON lines to `path`: one header object, one object per+ * allocation site, one per work row. Raw addresses; symbolisation is offline+ * (scripts/memwaste-report.py). `why` labels the dump. */+bool cbm_memev_dump(const char *path, const char *why);++/* Test seams. Never called in production. */+void cbm_memev_reset_for_tests(void);+void cbm_memev_force_for_tests(bool on);+/* Tests that hand the layer REAL memory turn the fill pattern and the duplicate+ * scan on (reset turns fill off: most tests use synthetic addresses). */+void cbm_memev_scan_for_tests(bool fill, bool dupes);++#endif /* CBM_MEM_EVENTS_H */diff --git a/src/foundation/mem_override_darwin.c b/src/foundation/mem_override_darwin.cnew file mode 100644index 000000000..370e6428a--- /dev/null+++ b/src/foundation/mem_override_darwin.c@@ -0,0 +1,267 @@+/*+ * mem_override_darwin.c — the macOS allocation observer of the waste sanitizer.+ *+ * WHY THIS FILE EXISTS+ *+ * Linux and Windows observe the malloc family through the linker's --wrap+ * (mem_override_posix.c, mem_override_win.c). macOS ld has no --wrap, and the+ * mimalloc global override is off there permanently: the two-level namespace+ * splits this binary's free from the system libraries' malloc, and a pointer+ * crossing that boundary aborts. So on macOS nothing saw raw malloc at all —+ * the platform the project is developed on was the one platform that could not+ * be measured.+ *+ * HOW IT WORKS+ *+ * A malloc ZONE that forwards every entry to the original default zone and+ * reports the event. Forwarding is the whole safety argument: a block never+ * changes allocator, so there is no boundary for it to cross. The zone claims+ * ownership through size(), which is what routes free() back through it.+ *+ * Proven on 2026-09-17 inside the real indexer (Go corpus): no abort, a block+ * allocated BEFORE installation frees cleanly after it, wall time 41.2 s ->+ * 41.3 s, and 92.3 million raw allocations that nothing had ever counted.+ *+ * THREE TRAPS, each one hit during that proof:+ * 1. malloc_default_zone() returns a VIRTUAL wrapper that dispatches to+ * whatever is default at call time. Forwarding to it recurses forever once+ * this zone is the default. The real slot 0 comes from+ * malloc_get_all_zones().+ * 2. Registration appends. The default is slot 0, and the only way in is the+ * rotation jemalloc uses: unregister slot 0 (the last zone moves into it),+ * re-register it (appended), until this zone is first.+ * 3. free() arrives through free_definite_size, not through free.+ *+ * Compiled only into the `memwaste` flavour; dormant unless CBM_MEMWASTE=1.+ */+#if defined(__APPLE__) && defined(CBM_MEMWASTE) && CBM_MEMWASTE++#include "foundation/mem_events.h"++#include <mach-o/dyld.h>+#include <mach-o/getsect.h>+#include <mach/mach.h>+#include <malloc/malloc.h>+#include <pthread.h>+#include <stdint.h>+#include <string.h>++enum { OBS_ZONE_VERSION = 9, OBS_MAX_ROTATIONS = 16, OBS_MAX_FRAMES = 24 };++static malloc_zone_t *g_orig;+static malloc_zone_t g_obs;+static uintptr_t g_text_lo;+static uintptr_t g_text_hi;++/* The site of a raw allocation is the first return address inside OUR image:+ * above the zone entry sit libmalloc's own frames, and often libc's (strdup,+ * getpwuid), before the code of ours that caused the call. Frames are walked+ * through the frame-pointer chain, bounded by this thread's stack, so a+ * foreign or corrupt frame ends the walk instead of faulting. */+static void *caller_site(void *frame, unsigned *flags) {+ *flags = 0;+ if (cbm_memev_hint_pending()) {+ return NULL; /* the memory core already named the site */+ }+ void *library_caller = cbm_memev_library_site();+ if (library_caller) {+ *flags = CBM_MEMEV_LIBRARY; /* a FILE buffer, a DIR stream: the library's own */+ return library_caller;+ }+ pthread_t self = pthread_self();+ uintptr_t hi = (uintptr_t)pthread_get_stackaddr_np(self);+ uintptr_t lo = hi - pthread_get_stacksize_np(self);+ uintptr_t *fp = (uintptr_t *)frame;+ void *first = NULL;+ for (int depth = 0; depth < OBS_MAX_FRAMES; depth++) {+ if ((uintptr_t)fp < lo || (uintptr_t)fp + (2 * sizeof(uintptr_t)) > hi ||+ ((uintptr_t)fp & (sizeof(uintptr_t) - 1)) != 0) {+ break;+ }+ uintptr_t ret = fp[1];+ uintptr_t *next = (uintptr_t *)fp[0];+ if (ret == 0) {+ break;+ }+ if (!first) {+ first = (void *)ret;+ }+ if (ret >= g_text_lo && ret < g_text_hi) {+ return (void *)ret;+ }+ if (next <= fp) {+ break; /* the chain must climb */+ }+ fp = next;+ }+ *flags = CBM_MEMEV_FOREIGN;+ return first; /* a system library allocating for itself: observe, never fill */+}++static size_t obs_size(malloc_zone_t *z, const void *p) {+ (void)z;+ return g_orig->size(g_orig, p);+}++static void *obs_malloc(malloc_zone_t *z, size_t n) {+ (void)z;+ void *p = g_orig->malloc(g_orig, n);+ if (p && cbm_memev_enabled()) {+ unsigned flags;+ void *site = caller_site(__builtin_frame_address(0), &flags);+ cbm_memev_alloc_ex(p, n, g_orig->size(g_orig, p), site, flags);+ }+ return p;+}++static void *obs_calloc(malloc_zone_t *z, size_t count, size_t n) {+ (void)z;+ void *p = g_orig->calloc(g_orig, count, n);+ if (p && cbm_memev_enabled()) {+ unsigned flags;+ void *site = caller_site(__builtin_frame_address(0), &flags);+ cbm_memev_alloc_ex(p, count * n, g_orig->size(g_orig, p), site,+ flags | CBM_MEMEV_ZEROED);+ }+ return p;+}++static void *obs_valloc(malloc_zone_t *z, size_t n) {+ (void)z;+ void *p = g_orig->valloc(g_orig, n);+ if (p && cbm_memev_enabled()) {+ unsigned flags;+ void *site = caller_site(__builtin_frame_address(0), &flags);+ cbm_memev_alloc_ex(p, n, g_orig->size(g_orig, p), site, flags);+ }+ return p;+}++static void *obs_memalign(malloc_zone_t *z, size_t align, size_t n) {+ (void)z;+ void *p = g_orig->memalign(g_orig, align, n);+ if (p && cbm_memev_enabled()) {+ unsigned flags;+ void *site = caller_site(__builtin_frame_address(0), &flags);+ cbm_memev_alloc_ex(p, n, g_orig->size(g_orig, p), site, flags);+ }+ return p;+}++static void *obs_realloc(malloc_zone_t *z, void *p, size_t n) {+ (void)z;+ unsigned flags = 0;+ void *site = cbm_memev_enabled() ? caller_site(__builtin_frame_address(0), &flags) : NULL;+ CBM_MEMEV_BACKING(1);+ void *q = g_orig->realloc(g_orig, p, n);+ CBM_MEMEV_BACKING(-1);+ if (q && cbm_memev_enabled()) {+ cbm_memev_realloc(p, q, n, g_orig->size(g_orig, q), site);+ }+ return q;+}++static void obs_free(malloc_zone_t *z, void *p) {+ (void)z;+ cbm_memev_free(p);+ g_orig->free(g_orig, p);+}++static void obs_free_definite_size(malloc_zone_t *z, void *p, size_t n) {+ (void)z;+ cbm_memev_free(p);+ if (g_orig->free_definite_size) {+ g_orig->free_definite_size(g_orig, p, n);+ } else {+ g_orig->free(g_orig, p);+ }+}++static unsigned obs_batch_malloc(malloc_zone_t *z, size_t size, void **results, unsigned count) {+ (void)z;+ unsigned got = g_orig->batch_malloc ? g_orig->batch_malloc(g_orig, size, results, count) : 0;+ if (cbm_memev_enabled()) {+ unsigned flags;+ void *site = caller_site(__builtin_frame_address(0), &flags);+ for (unsigned i = 0; i < got; i++) {+ cbm_memev_alloc_ex(results[i], size, g_orig->size(g_orig, results[i]), site, flags);+ }+ }+ return got;+}++static void obs_batch_free(malloc_zone_t *z, void **blocks, unsigned count) {+ (void)z;+ for (unsigned i = 0; i < count; i++) {+ cbm_memev_free(blocks[i]);+ }+ if (g_orig->batch_free) {+ g_orig->batch_free(g_orig, blocks, count);+ }+}++static void obs_destroy(malloc_zone_t *z) {+ (void)z;+}++static size_t obs_pressure_relief(malloc_zone_t *z, size_t goal) {+ (void)z;+ return g_orig->pressure_relief ? g_orig->pressure_relief(g_orig, goal) : 0;+}++static malloc_zone_t *slot_zero(void) {+ vm_address_t *zones = NULL;+ unsigned count = 0;+ if (malloc_get_all_zones(mach_task_self(), NULL, &zones, &count) != KERN_SUCCESS ||+ count == 0) {+ return NULL;+ }+ return (malloc_zone_t *)zones[0];+}++__attribute__((constructor)) static void cbm_mem_observe_install(void) {+ if (!cbm_memev_enabled()) {+ return;+ }+ const struct mach_header_64 *hdr = (const struct mach_header_64 *)_dyld_get_image_header(0);+ unsigned long text_size = 0;+ uint8_t *text = hdr ? getsegmentdata(hdr, "__TEXT", &text_size) : NULL;+ g_text_lo = (uintptr_t)text;+ g_text_hi = g_text_lo + text_size;+ cbm_memev_set_image_base((uintptr_t)hdr);++ g_orig = slot_zero();+ if (!g_orig) {+ return;+ }+ memset(&g_obs, 0, sizeof(g_obs));+ g_obs.size = obs_size;+ g_obs.malloc = obs_malloc;+ g_obs.calloc = obs_calloc;+ g_obs.valloc = obs_valloc;+ g_obs.free = obs_free;+ g_obs.realloc = obs_realloc;+ g_obs.destroy = obs_destroy;+ g_obs.zone_name = "cbm_memwaste_observer";+ g_obs.batch_malloc = obs_batch_malloc;+ g_obs.batch_free = obs_batch_free;+ g_obs.introspect = g_orig->introspect; /* leaks/heap keep working */+ g_obs.version = OBS_ZONE_VERSION;+ g_obs.memalign = obs_memalign;+ g_obs.free_definite_size = obs_free_definite_size;+ g_obs.pressure_relief = obs_pressure_relief;+ malloc_zone_register(&g_obs);+ for (int spin = 0; spin < OBS_MAX_ROTATIONS; spin++) {+ malloc_zone_t *first = slot_zero();+ if (!first || first == &g_obs) {+ break;+ }+ malloc_zone_unregister(first);+ malloc_zone_register(first);+ }+ cbm_memev_set_observer_installed(slot_zero() == &g_obs);+}++#else+typedef int cbm_mem_override_darwin_unused_t;+#endifdiff --git a/src/foundation/mem_override_libc.c b/src/foundation/mem_override_libc.cnew file mode 100644index 000000000..b86b8e0d1--- /dev/null+++ b/src/foundation/mem_override_libc.c@@ -0,0 +1,738 @@+/*+ * mem_override_libc.c — the CPU lane of the waste sanitizer: byte work and+ * system calls, counted per call site.+ *+ * WHAT IT COUNTS. memcpy, memmove, memset, memcmp, strlen, strcmp, strncmp+ * (calls and bytes), read/write/pread/pwrite/fread/fwrite (bytes and tiny+ * calls), open/stat/fopen/opendir (failures and the same path again), readdir,+ * and pthread_mutex_lock (contended acquisitions; POSIX only -- Windows locks+ * are counted where cbm_mutex_lock takes them).+ * Calls the compiler inlines (small constant-size copies) are not calls and are+ * not counted: what is counted is exactly the work that reaches the library.+ *+ * HOW IT LISTENS, per platform+ * Linux, Windows the linker's --wrap redirects every reference in OUR+ * objects to __wrap_<name>; __real_<name> is the original.+ * The flavour adds the flags (Makefile.cbm, MEMWASTE=1).+ * macOS ld has no --wrap and dyld ignores __interpose tuples in the+ * main executable (proven 2026-09-17). Instead this file+ * DEFINES the symbols: references in our objects bind to the+ * definitions in our own image, and each definition reaches+ * libSystem's through dlsym(RTLD_NEXT). Calls made inside+ * libSystem keep libSystem's own functions. Fortified builds+ * call __memcpy_chk and friends, so those are defined too.+ *+ * The CALLER's return address is the site: this file is the observation point.+ * Compiled only into the `memwaste` flavour; dormant unless CBM_MEMWASTE=1.+ * Everything here forwards unconditionally: counting can only ever be skipped,+ * never the work.+ */+#if defined(CBM_MEMWASTE) && CBM_MEMWASTE++#include "foundation/mem_events.h"++#include <errno.h>+#include <stdarg.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#if !defined(_WIN32)+#include <pthread.h>+#endif++#if defined(__clang__)+#pragma clang attribute push(__attribute__((no_sanitize("coverage"))), apply_to = function)+#endif++#define SITE() __builtin_return_address(0)++static inline void note_io(cbm_work_kind_t kind, void *site, long long got, size_t asked) {+ if (!cbm_memev_enabled()) {+ return;+ }+ uint64_t bytes = got > 0 ? (uint64_t)got : 0;+ cbm_work_note(kind, site, bytes, (asked > 0 && asked < CBM_WORK_TINY_IO) ? 1 : 0, 0);+}++/* Access lane: a library call's reads and writes, charged to its caller. */+#define ACC_LOAD(p, n) CBM_MEMEV_ACCESS((p), (n), false, SITE())+#define ACC_STORE(p, n) CBM_MEMEV_ACCESS((p), (n), true, SITE())++static inline void note_bytes(cbm_work_kind_t kind, void *site, size_t n) {+ if (cbm_memev_enabled()) {+ cbm_work_note(kind, site, n, 0, 0);+ }+}++/* ══════════════════════════════════════════════════════════════════════ */+#if defined(__APPLE__)++#include <dirent.h>+#include <dlfcn.h>+#include <fcntl.h>+#include <sys/stat.h>+#include <unistd.h>++#undef memcpy+#undef memmove+#undef memset+#undef strlen+#undef strcmp+#undef strncmp+#undef memcmp++/* Resolved in a constructor. Until then the byte functions run a plain loop+ * (dlsym and the dynamic loader call them before any constructor), marked+ * no_builtin so the optimiser cannot turn the loop back into a call to itself. */+static void *(*r_memcpy)(void *, const void *, size_t);+static void *(*r_memmove)(void *, const void *, size_t);+static void *(*r_memset)(void *, int, size_t);+static int (*r_memcmp)(const void *, const void *, size_t);+static size_t (*r_strlen)(const char *);+static int (*r_strcmp)(const char *, const char *);+static int (*r_strncmp)(const char *, const char *, size_t);+static void *(*r_memcpy_chk)(void *, const void *, size_t, size_t);+static void *(*r_memmove_chk)(void *, const void *, size_t, size_t);+static void *(*r_memset_chk)(void *, int, size_t, size_t);+static ssize_t (*r_read)(int, void *, size_t);+static ssize_t (*r_write)(int, const void *, size_t);+static ssize_t (*r_pread)(int, void *, size_t, off_t);+static ssize_t (*r_pwrite)(int, const void *, size_t, off_t);+static int (*r_open)(const char *, int, ...);+static int (*r_stat)(const char *, struct stat *);+static int (*r_lstat)(const char *, struct stat *);+static FILE *(*r_fopen)(const char *, const char *);+static size_t (*r_fread)(void *, size_t, size_t, FILE *);+static size_t (*r_fwrite)(const void *, size_t, size_t, FILE *);+static DIR *(*r_opendir)(const char *);+static struct dirent *(*r_readdir)(DIR *);+static int (*r_mutex_lock)(pthread_mutex_t *);+static int (*r_mutex_trylock)(pthread_mutex_t *);++__attribute__((constructor(101))) static void libc_resolve(void) {+ r_memcpy = dlsym(RTLD_NEXT, "memcpy");+ r_memmove = dlsym(RTLD_NEXT, "memmove");+ r_memset = dlsym(RTLD_NEXT, "memset");+ r_memcmp = dlsym(RTLD_NEXT, "memcmp");+ r_strlen = dlsym(RTLD_NEXT, "strlen");+ r_strcmp = dlsym(RTLD_NEXT, "strcmp");+ r_strncmp = dlsym(RTLD_NEXT, "strncmp");+ r_memcpy_chk = dlsym(RTLD_NEXT, "__memcpy_chk");+ r_memmove_chk = dlsym(RTLD_NEXT, "__memmove_chk");+ r_memset_chk = dlsym(RTLD_NEXT, "__memset_chk");+ r_read = dlsym(RTLD_NEXT, "read");+ r_write = dlsym(RTLD_NEXT, "write");+ r_pread = dlsym(RTLD_NEXT, "pread");+ r_pwrite = dlsym(RTLD_NEXT, "pwrite");+ r_open = dlsym(RTLD_NEXT, "open");+ r_stat = dlsym(RTLD_NEXT, "stat");+ r_lstat = dlsym(RTLD_NEXT, "lstat");+ r_fopen = dlsym(RTLD_NEXT, "fopen");+ r_fread = dlsym(RTLD_NEXT, "fread");+ r_fwrite = dlsym(RTLD_NEXT, "fwrite");+ r_opendir = dlsym(RTLD_NEXT, "opendir");+ r_readdir = dlsym(RTLD_NEXT, "readdir");+ r_mutex_lock = dlsym(RTLD_NEXT, "pthread_mutex_lock");+ r_mutex_trylock = dlsym(RTLD_NEXT, "pthread_mutex_trylock");+}++/* System calls are only reachable after libc_resolve; a call from an earlier+ * constructor resolves its own target once. */+#define RESOLVED(ptr, name) ((ptr) ? (ptr) : ((ptr) = dlsym(RTLD_NEXT, name)))++__attribute__((no_builtin("memcpy", "memmove", "memset")))+static void *slow_move(void *d, const void *s, size_t n) {+ unsigned char *dd = (unsigned char *)d;+ const unsigned char *ss = (const unsigned char *)s;+ if (dd < ss) {+ for (size_t i = 0; i < n; i++) {+ dd[i] = ss[i];+ }+ } else {+ for (size_t i = n; i > 0; i--) {+ dd[i - 1] = ss[i - 1];+ }+ }+ return d;+}++__attribute__((no_builtin("memset")))+static void *slow_set(void *d, int c, size_t n) {+ unsigned char *dd = (unsigned char *)d;+ for (size_t i = 0; i < n; i++) {+ dd[i] = (unsigned char)c;+ }+ return d;+}++__attribute__((no_builtin("strlen")))+static size_t slow_strlen(const char *s) {+ size_t i = 0;+ while (s[i]) {+ i++;+ }+ return i;+}++__attribute__((no_builtin("memcmp", "strcmp", "strncmp")))+static int slow_ncmp(const char *a, const char *b, size_t n, bool stop_at_nul) {+ for (size_t i = 0; i < n; i++) {+ unsigned char x = (unsigned char)a[i];+ unsigned char y = (unsigned char)b[i];+ if (x != y) {+ return x < y ? -1 : 1;+ }+ if (stop_at_nul && x == 0) {+ return 0;+ }+ }+ return 0;+}++void *(memcpy)(void *restrict d, const void *restrict s, size_t n) {+ note_bytes(CBM_WORK_MEMCPY, SITE(), n);+ ACC_LOAD(s, n);+ ACC_STORE(d, n);+ return r_memcpy ? r_memcpy(d, s, n) : slow_move(d, s, n);+}++void *(memmove)(void *d, const void *s, size_t n) {+ note_bytes(CBM_WORK_MEMMOVE, SITE(), n);+ ACC_LOAD(s, n);+ ACC_STORE(d, n);+ return r_memmove ? r_memmove(d, s, n) : slow_move(d, s, n);+}++void *(memset)(void *d, int c, size_t n) {+ note_bytes(CBM_WORK_MEMSET, SITE(), n);+ ACC_STORE(d, n);+ return r_memset ? r_memset(d, c, n) : slow_set(d, c, n);+}++int (memcmp)(const void *a, const void *b, size_t n) {+ note_bytes(CBM_WORK_MEMCMP, SITE(), n);+ ACC_LOAD(a, n);+ ACC_LOAD(b, n);+ return r_memcmp ? r_memcmp(a, b, n) : slow_ncmp((const char *)a, (const char *)b, n, false);+}++size_t (strlen)(const char *s) {+ size_t n = r_strlen ? r_strlen(s) : slow_strlen(s);+ if (cbm_memev_enabled()) {+ cbm_work_note_strlen(SITE(), s, n);+ ACC_LOAD(s, n + 1);+ }+ return n;+}++int (strcmp)(const char *a, const char *b) {+ note_bytes(CBM_WORK_STRCMP, SITE(), 0);+ ACC_LOAD(a, 1);+ ACC_LOAD(b, 1);+ return r_strcmp ? r_strcmp(a, b) : slow_ncmp(a, b, (size_t)-1, true);+}++int (strncmp)(const char *a, const char *b, size_t n) {+ note_bytes(CBM_WORK_STRNCMP, SITE(), n);+ ACC_LOAD(a, 1);+ ACC_LOAD(b, 1);+ return r_strncmp ? r_strncmp(a, b, n) : slow_ncmp(a, b, n, true);+}++void *__memcpy_chk(void *d, const void *s, size_t n, size_t dn) {+ note_bytes(CBM_WORK_MEMCPY, SITE(), n);+ ACC_LOAD(s, n);+ ACC_STORE(d, n);+ return RESOLVED(r_memcpy_chk, "__memcpy_chk")(d, s, n, dn);+}++void *__memmove_chk(void *d, const void *s, size_t n, size_t dn) {+ note_bytes(CBM_WORK_MEMMOVE, SITE(), n);+ ACC_LOAD(s, n);+ ACC_STORE(d, n);+ return RESOLVED(r_memmove_chk, "__memmove_chk")(d, s, n, dn);+}++void *__memset_chk(void *d, int c, size_t n, size_t dn) {+ note_bytes(CBM_WORK_MEMSET, SITE(), n);+ ACC_STORE(d, n);+ return RESOLVED(r_memset_chk, "__memset_chk")(d, c, n, dn);+}++ssize_t read(int fd, void *buf, size_t n) {+ ssize_t got = RESOLVED(r_read, "read")(fd, buf, n);+ note_io(CBM_WORK_READ, SITE(), (long long)got, n);+ ACC_STORE(buf, got > 0 ? (size_t)got : 0);+ return got;+}++ssize_t write(int fd, const void *buf, size_t n) {+ ssize_t put = RESOLVED(r_write, "write")(fd, buf, n);+ note_io(CBM_WORK_WRITE, SITE(), (long long)put, n);+ ACC_LOAD(buf, put > 0 ? (size_t)put : 0);+ return put;+}++ssize_t pread(int fd, void *buf, size_t n, off_t off) {+ ssize_t got = RESOLVED(r_pread, "pread")(fd, buf, n, off);+ note_io(CBM_WORK_PREAD, SITE(), (long long)got, n);+ ACC_STORE(buf, got > 0 ? (size_t)got : 0);+ return got;+}++ssize_t pwrite(int fd, const void *buf, size_t n, off_t off) {+ ssize_t put = RESOLVED(r_pwrite, "pwrite")(fd, buf, n, off);+ note_io(CBM_WORK_PWRITE, SITE(), (long long)put, n);+ ACC_LOAD(buf, put > 0 ? (size_t)put : 0);+ return put;+}++int open(const char *path, int flags, ...) {+ int mode = 0;+ if (flags & O_CREAT) {+ va_list ap;+ va_start(ap, flags);+ mode = va_arg(ap, int);+ va_end(ap);+ }+ int fd = RESOLVED(r_open, "open")(path, flags, mode);+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_OPEN, SITE(), path, fd < 0);+ }+ return fd;+}++int stat(const char *path, struct stat *st) {+ int rc = RESOLVED(r_stat, "stat")(path, st);+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_STAT, SITE(), path, rc != 0);+ }+ return rc;+}++int lstat(const char *path, struct stat *st) {+ int rc = RESOLVED(r_lstat, "lstat")(path, st);+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_STAT, SITE(), path, rc != 0);+ }+ return rc;+}++FILE *fopen(const char *restrict path, const char *restrict mode) {+ cbm_memev_library_enter(SITE());+ FILE *f = RESOLVED(r_fopen, "fopen")(path, mode);+ cbm_memev_library_leave();+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_FOPEN, SITE(), path, f == NULL);+ }+ return f;+}++size_t fread(void *restrict buf, size_t size, size_t count, FILE *restrict f) {+ cbm_memev_library_enter(SITE());+ size_t got = RESOLVED(r_fread, "fread")(buf, size, count, f);+ cbm_memev_library_leave();+ note_io(CBM_WORK_FREAD, SITE(), (long long)(got * size), size * count);+ ACC_STORE(buf, got * size);+ return got;+}++size_t fwrite(const void *restrict buf, size_t size, size_t count, FILE *restrict f) {+ cbm_memev_library_enter(SITE());+ size_t put = RESOLVED(r_fwrite, "fwrite")(buf, size, count, f);+ cbm_memev_library_leave();+ note_io(CBM_WORK_FWRITE, SITE(), (long long)(put * size), size * count);+ ACC_LOAD(buf, put * size);+ return put;+}++DIR *opendir(const char *path) {+ cbm_memev_library_enter(SITE());+ DIR *d = RESOLVED(r_opendir, "opendir")(path);+ cbm_memev_library_leave();+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_OPENDIR, SITE(), path, d == NULL);+ }+ return d;+}++struct dirent *readdir(DIR *d) {+ note_bytes(CBM_WORK_READDIR, SITE(), 0);+ cbm_memev_library_enter(SITE());+ struct dirent *ent = RESOLVED(r_readdir, "readdir")(d);+ cbm_memev_library_leave();+ return ent;+}++int cbm_memev_mutex_lock(void *pthread_mutex, void *site) {+ pthread_mutex_t *m = (pthread_mutex_t *)pthread_mutex;+ int (*lock)(pthread_mutex_t *) = RESOLVED(r_mutex_lock, "pthread_mutex_lock");+ if (!cbm_memev_enabled()) {+ return lock(m);+ }+ int rc = RESOLVED(r_mutex_trylock, "pthread_mutex_trylock")(m);+ bool contended = rc == EBUSY;+ if (contended) {+ rc = lock(m);+ }+ cbm_work_note(CBM_WORK_MUTEX, site, 0, contended ? 1 : 0, 0);+ return rc;+}++int pthread_mutex_lock(pthread_mutex_t *m) {+ return cbm_memev_mutex_lock(m, SITE());+}++/* ══════════════════════════════════════════════════════════════════════ */+#elif defined(__linux__)++#include <dirent.h>+#include <fcntl.h>+#include <sys/stat.h>+#include <unistd.h>++void *__real_memcpy(void *, const void *, size_t);+void *__real_memmove(void *, const void *, size_t);+void *__real_memset(void *, int, size_t);+int __real_memcmp(const void *, const void *, size_t);+size_t __real_strlen(const char *);+int __real_strcmp(const char *, const char *);+int __real_strncmp(const char *, const char *, size_t);+void *__real___memcpy_chk(void *, const void *, size_t, size_t);+void *__real___memmove_chk(void *, const void *, size_t, size_t);+void *__real___memset_chk(void *, int, size_t, size_t);+ssize_t __real_read(int, void *, size_t);+ssize_t __real___read_chk(int, void *, size_t, size_t);+ssize_t __real_write(int, const void *, size_t);+ssize_t __real_pread(int, void *, size_t, off_t);+ssize_t __real_pwrite(int, const void *, size_t, off_t);+ssize_t __real_pread64(int, void *, size_t, off_t);+ssize_t __real_pwrite64(int, const void *, size_t, off_t);+int __real_open(const char *, int, ...);+int __real_open64(const char *, int, ...);+int __real___open_2(const char *, int);+int __real___open64_2(const char *, int);+int __real_stat(const char *, struct stat *);+int __real_lstat(const char *, struct stat *);+FILE *__real_fopen(const char *, const char *);+FILE *__real_fopen64(const char *, const char *);+size_t __real_fread(void *, size_t, size_t, FILE *);+size_t __real___fread_chk(void *, size_t, size_t, size_t, FILE *);+size_t __real_fwrite(const void *, size_t, size_t, FILE *);+DIR *__real_opendir(const char *);+struct dirent *__real_readdir(DIR *);+int __real_pthread_mutex_lock(pthread_mutex_t *);++void *__wrap_memcpy(void *d, const void *s, size_t n) {+ note_bytes(CBM_WORK_MEMCPY, SITE(), n);+ ACC_LOAD(s, n);+ ACC_STORE(d, n);+ return __real_memcpy(d, s, n);+}+void *__wrap_memmove(void *d, const void *s, size_t n) {+ note_bytes(CBM_WORK_MEMMOVE, SITE(), n);+ ACC_LOAD(s, n);+ ACC_STORE(d, n);+ return __real_memmove(d, s, n);+}+void *__wrap_memset(void *d, int c, size_t n) {+ note_bytes(CBM_WORK_MEMSET, SITE(), n);+ ACC_STORE(d, n);+ return __real_memset(d, c, n);+}+int __wrap_memcmp(const void *a, const void *b, size_t n) {+ note_bytes(CBM_WORK_MEMCMP, SITE(), n);+ ACC_LOAD(a, n);+ ACC_LOAD(b, n);+ return __real_memcmp(a, b, n);+}+size_t __wrap_strlen(const char *s) {+ size_t n = __real_strlen(s);+ if (cbm_memev_enabled()) {+ cbm_work_note_strlen(SITE(), s, n);+ ACC_LOAD(s, n + 1);+ }+ return n;+}+int __wrap_strcmp(const char *a, const char *b) {+ note_bytes(CBM_WORK_STRCMP, SITE(), 0);+ ACC_LOAD(a, 1);+ ACC_LOAD(b, 1);+ return __real_strcmp(a, b);+}+int __wrap_strncmp(const char *a, const char *b, size_t n) {+ note_bytes(CBM_WORK_STRNCMP, SITE(), n);+ ACC_LOAD(a, 1);+ ACC_LOAD(b, 1);+ return __real_strncmp(a, b, n);+}+void *__wrap___memcpy_chk(void *d, const void *s, size_t n, size_t dn) {+ note_bytes(CBM_WORK_MEMCPY, SITE(), n);+ ACC_LOAD(s, n);+ ACC_STORE(d, n);+ return __real___memcpy_chk(d, s, n, dn);+}+void *__wrap___memmove_chk(void *d, const void *s, size_t n, size_t dn) {+ note_bytes(CBM_WORK_MEMMOVE, SITE(), n);+ ACC_LOAD(s, n);+ ACC_STORE(d, n);+ return __real___memmove_chk(d, s, n, dn);+}+void *__wrap___memset_chk(void *d, int c, size_t n, size_t dn) {+ note_bytes(CBM_WORK_MEMSET, SITE(), n);+ ACC_STORE(d, n);+ return __real___memset_chk(d, c, n, dn);+}+ssize_t __wrap_read(int fd, void *buf, size_t n) {+ ssize_t got = __real_read(fd, buf, n);+ note_io(CBM_WORK_READ, SITE(), (long long)got, n);+ ACC_STORE(buf, got > 0 ? (size_t)got : 0);+ return got;+}+ssize_t __wrap___read_chk(int fd, void *buf, size_t n, size_t bn) {+ ssize_t got = __real___read_chk(fd, buf, n, bn);+ note_io(CBM_WORK_READ, SITE(), (long long)got, n);+ ACC_STORE(buf, got > 0 ? (size_t)got : 0);+ return got;+}+ssize_t __wrap_write(int fd, const void *buf, size_t n) {+ ssize_t put = __real_write(fd, buf, n);+ note_io(CBM_WORK_WRITE, SITE(), (long long)put, n);+ ACC_LOAD(buf, put > 0 ? (size_t)put : 0);+ return put;+}+ssize_t __wrap_pread(int fd, void *buf, size_t n, off_t off) {+ ssize_t got = __real_pread(fd, buf, n, off);+ note_io(CBM_WORK_PREAD, SITE(), (long long)got, n);+ ACC_STORE(buf, got > 0 ? (size_t)got : 0);+ return got;+}+ssize_t __wrap_pwrite(int fd, const void *buf, size_t n, off_t off) {+ ssize_t put = __real_pwrite(fd, buf, n, off);+ note_io(CBM_WORK_PWRITE, SITE(), (long long)put, n);+ ACC_LOAD(buf, put > 0 ? (size_t)put : 0);+ return put;+}+ssize_t __wrap_pread64(int fd, void *buf, size_t n, off_t off) {+ ssize_t got = __real_pread64(fd, buf, n, off);+ note_io(CBM_WORK_PREAD, SITE(), (long long)got, n);+ ACC_STORE(buf, got > 0 ? (size_t)got : 0);+ return got;+}+ssize_t __wrap_pwrite64(int fd, const void *buf, size_t n, off_t off) {+ ssize_t put = __real_pwrite64(fd, buf, n, off);+ note_io(CBM_WORK_PWRITE, SITE(), (long long)put, n);+ ACC_LOAD(buf, put > 0 ? (size_t)put : 0);+ return put;+}+static int open_mode(int flags, va_list ap) {+ return (flags & O_CREAT) ? va_arg(ap, int) : 0;+}+int __wrap_open(const char *path, int flags, ...) {+ va_list ap;+ va_start(ap, flags);+ int mode = open_mode(flags, ap);+ va_end(ap);+ int fd = __real_open(path, flags, mode);+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_OPEN, SITE(), path, fd < 0);+ }+ return fd;+}+int __wrap_open64(const char *path, int flags, ...) {+ va_list ap;+ va_start(ap, flags);+ int mode = open_mode(flags, ap);+ va_end(ap);+ int fd = __real_open64(path, flags, mode);+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_OPEN, SITE(), path, fd < 0);+ }+ return fd;+}+int __wrap___open_2(const char *path, int flags) {+ int fd = __real___open_2(path, flags);+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_OPEN, SITE(), path, fd < 0);+ }+ return fd;+}+int __wrap___open64_2(const char *path, int flags) {+ int fd = __real___open64_2(path, flags);+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_OPEN, SITE(), path, fd < 0);+ }+ return fd;+}+int __wrap_stat(const char *path, struct stat *st) {+ int rc = __real_stat(path, st);+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_STAT, SITE(), path, rc != 0);+ }+ return rc;+}+int __wrap_lstat(const char *path, struct stat *st) {+ int rc = __real_lstat(path, st);+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_STAT, SITE(), path, rc != 0);+ }+ return rc;+}+FILE *__wrap_fopen(const char *path, const char *mode) {+ FILE *f = __real_fopen(path, mode);+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_FOPEN, SITE(), path, f == NULL);+ }+ return f;+}+FILE *__wrap_fopen64(const char *path, const char *mode) {+ FILE *f = __real_fopen64(path, mode);+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_FOPEN, SITE(), path, f == NULL);+ }+ return f;+}+size_t __wrap_fread(void *buf, size_t size, size_t count, FILE *f) {+ size_t got = __real_fread(buf, size, count, f);+ note_io(CBM_WORK_FREAD, SITE(), (long long)(got * size), size * count);+ ACC_STORE(buf, got * size);+ return got;+}+size_t __wrap___fread_chk(void *buf, size_t bn, size_t size, size_t count, FILE *f) {+ size_t got = __real___fread_chk(buf, bn, size, count, f);+ note_io(CBM_WORK_FREAD, SITE(), (long long)(got * size), size * count);+ ACC_STORE(buf, got * size);+ return got;+}+size_t __wrap_fwrite(const void *buf, size_t size, size_t count, FILE *f) {+ size_t put = __real_fwrite(buf, size, count, f);+ note_io(CBM_WORK_FWRITE, SITE(), (long long)(put * size), size * count);+ ACC_LOAD(buf, put * size);+ return put;+}+DIR *__wrap_opendir(const char *path) {+ DIR *d = __real_opendir(path);+ if (cbm_memev_enabled()) {+ cbm_work_note_path(CBM_WORK_OPENDIR, SITE(), path, d == NULL);+ }+ return d;+}+struct dirent *__wrap_readdir(DIR *d) {+ note_bytes(CBM_WORK_READDIR, SITE(), 0);+ return __real_readdir(d);+}++int cbm_memev_mutex_lock(void *pthread_mutex, void *site) {+ pthread_mutex_t *m = (pthread_mutex_t *)pthread_mutex;+ if (!cbm_memev_enabled()) {+ return __real_pthread_mutex_lock(m);+ }+ int rc = pthread_mutex_trylock(m);+ bool contended = rc == EBUSY;+ if (contended) {+ rc = __real_pthread_mutex_lock(m);+ }+ cbm_work_note(CBM_WORK_MUTEX, site, 0, contended ? 1 : 0, 0);+ return rc;+}++int __wrap_pthread_mutex_lock(pthread_mutex_t *m) {+ return cbm_memev_mutex_lock(m, SITE());+}++/* ══════════════════════════════════════════════════════════════════════ */+#elif defined(_WIN32)++#include <wchar.h>++void *__real_memcpy(void *, const void *, size_t);+void *__real_memmove(void *, const void *, size_t);+void *__real_memset(void *, int, size_t);+int __real_memcmp(const void *, const void *, size_t);+size_t __real_strlen(const char *);+int __real_strcmp(const char *, const char *);+int __real_strncmp(const char *, const char *, size_t);+FILE *__real__wfopen(const wchar_t *, const wchar_t *);+size_t __real_fread(void *, size_t, size_t, FILE *);+size_t __real_fwrite(const void *, size_t, size_t, FILE *);++void *__wrap_memcpy(void *d, const void *s, size_t n) {+ note_bytes(CBM_WORK_MEMCPY, SITE(), n);+ ACC_LOAD(s, n);+ ACC_STORE(d, n);+ return __real_memcpy(d, s, n);+}+void *__wrap_memmove(void *d, const void *s, size_t n) {+ note_bytes(CBM_WORK_MEMMOVE, SITE(), n);+ ACC_LOAD(s, n);+ ACC_STORE(d, n);+ return __real_memmove(d, s, n);+}+void *__wrap_memset(void *d, int c, size_t n) {+ note_bytes(CBM_WORK_MEMSET, SITE(), n);+ ACC_STORE(d, n);+ return __real_memset(d, c, n);+}+int __wrap_memcmp(const void *a, const void *b, size_t n) {+ note_bytes(CBM_WORK_MEMCMP, SITE(), n);+ ACC_LOAD(a, n);+ ACC_LOAD(b, n);+ return __real_memcmp(a, b, n);+}+size_t __wrap_strlen(const char *s) {+ size_t n = __real_strlen(s);+ if (cbm_memev_enabled()) {+ cbm_work_note_strlen(SITE(), s, n);+ ACC_LOAD(s, n + 1);+ }+ return n;+}+int __wrap_strcmp(const char *a, const char *b) {+ note_bytes(CBM_WORK_STRCMP, SITE(), 0);+ ACC_LOAD(a, 1);+ ACC_LOAD(b, 1);+ return __real_strcmp(a, b);+}+int __wrap_strncmp(const char *a, const char *b, size_t n) {+ note_bytes(CBM_WORK_STRNCMP, SITE(), n);+ ACC_LOAD(a, 1);+ ACC_LOAD(b, 1);+ return __real_strncmp(a, b, n);+}+/* A wide path's hash is taken over its UTF-16 units squeezed to bytes: equal+ * paths hash equal, which is all the repeat detector needs. */+FILE *__wrap__wfopen(const wchar_t *path, const wchar_t *mode) {+ FILE *f = __real__wfopen(path, mode);+ if (cbm_memev_enabled()) {+ char narrow[1024];+ size_t i = 0;+ for (; path && path[i] && i + 1 < sizeof(narrow); i++) {+ narrow[i] = (char)(path[i] & 0x7F ? path[i] : '?');+ }+ narrow[i] = '\0';+ cbm_work_note_path(CBM_WORK_FOPEN, SITE(), narrow, f == NULL);+ }+ return f;+}+size_t __wrap_fread(void *buf, size_t size, size_t count, FILE *f) {+ size_t got = __real_fread(buf, size, count, f);+ note_io(CBM_WORK_FREAD, SITE(), (long long)(got * size), size * count);+ ACC_STORE(buf, got * size);+ return got;+}+size_t __wrap_fwrite(const void *buf, size_t size, size_t count, FILE *f) {+ size_t put = __real_fwrite(buf, size, count, f);+ note_io(CBM_WORK_FWRITE, SITE(), (long long)(put * size), size * count);+ ACC_LOAD(buf, put * size);+ return put;+}++#endif /* platform */++#if defined(__clang__)+#pragma clang attribute pop+#endif++#else+typedef int cbm_mem_override_libc_unused_t;+#endif /* CBM_MEMWASTE */diff --git a/src/foundation/mem_override_posix.c b/src/foundation/mem_override_posix.cindex 008d8010d..af340fd23 100644--- a/src/foundation/mem_override_posix.c+++ b/src/foundation/mem_override_posix.c@@ -32,36 +32,61 @@* and a difference in the output is a difference in behaviour rather than an* artifact of two different instrumentation schemes.*- * Linux only: --wrap is a GNU ld/lld feature. macOS ld has no equivalent, so- * macOS runs census-only and the profiler reports nothing there. That gap is- * stated in the harness spec rather than hidden behind zeros that look like- * "no allocations".+ * Linux only: --wrap is a GNU ld/lld feature. macOS ld has no equivalent; there+ * the waste layer observes through a forwarding malloc zone instead+ * (mem_override_darwin.c), so all three platforms report the same site tables.*/-#include "foundation/mem_profile.h"-#include <mimalloc.h>#include <stddef.h>+#include <string.h>#if defined(__linux__)+/* Waste sanitizer (mem_events.h). The wrapper IS the observation point, so the+ * caller's return address is the allocation site. Compiled to nothing outside+ * the `memwaste` flavour. Frees are reported BEFORE the block goes back. */+#if defined(CBM_MEMWASTE) && CBM_MEMWASTE+#include "foundation/mem_events.h"+#define OBS_ALLOC_F(block, requested, flags) \+ do { \+ if ((block) && cbm_memev_enabled()) { \+ cbm_memev_alloc_ex((block), (requested), mi_usable_size(block), \+ __builtin_return_address(0), (unsigned)(flags)); \+ } \+ } while (0)+#define OBS_ALLOC(block, requested) OBS_ALLOC_F((block), (requested), 0)+#define OBS_REALLOC(old_block, grown, requested) \+ do { \+ if ((grown) && cbm_memev_enabled()) { \+ cbm_memev_realloc((old_block), (grown), (requested), mi_usable_size(grown), \+ __builtin_return_address(0)); \+ } \+ } while (0)+#define OBS_FREE(block) cbm_memev_free(block)+#else+#define OBS_ALLOC(block, requested) ((void)0)+#define OBS_ALLOC_F(block, requested, flags) ((void)0)+#define OBS_REALLOC(old_block, grown, requested) ((void)0)+#define OBS_FREE(block) ((void)0)+#endif+void *__wrap_malloc(size_t size) {void *block = mi_malloc(size);- cbm_mem_profile_alloc(block, size);+ OBS_ALLOC(block, size);return block;}void *__wrap_calloc(size_t count, size_t size) {void *block = mi_calloc(count, size);- cbm_mem_profile_alloc(block, count * size);+ OBS_ALLOC_F(block, count * size, CBM_MEMEV_ZEROED);return block;}void *__wrap_realloc(void *block, size_t size) {- if (block) {- cbm_mem_profile_free(block);- }+ CBM_MEMEV_BACKING(1);void *grown = mi_realloc(block, size);- cbm_mem_profile_alloc(grown, size);+ CBM_MEMEV_BACKING(-1);+ OBS_REALLOC(block, grown, size);return grown;}@@ -71,16 +96,43 @@ void __wrap_free(void *block) {}/* Unlike Windows there is no foreign-allocator hazard to route around:* every malloc in this image is already mimalloc's. */- cbm_mem_profile_free(block);+ OBS_FREE(block);mi_free(block);}char *__wrap_strdup(const char *text) {char *copy = mi_strdup(text);- if (copy) {- cbm_mem_profile_alloc(copy, mi_usable_size(copy));- }+ OBS_ALLOC_F(copy, copy ? strlen(copy) + 1 : 0, CBM_MEMEV_WRITTEN);return copy;}+/* The aligned family and strndup reach mimalloc too; without these a block+ * from posix_memalign (the slab allocator's pages) is invisible, and its+ * free reads as untracked. */+char *__wrap_strndup(const char *text, size_t length) {+ char *copy = mi_strndup(text, length);+ OBS_ALLOC_F(copy, copy ? strlen(copy) + 1 : 0, CBM_MEMEV_WRITTEN);+ return copy;+}++int __wrap_posix_memalign(void **out, size_t alignment, size_t size) {+ int rc = mi_posix_memalign(out, alignment, size);+ if (rc == 0 && out) {+ OBS_ALLOC(*out, size);+ }+ return rc;+}++void *__wrap_aligned_alloc(size_t alignment, size_t size) {+ void *block = mi_aligned_alloc(alignment, size);+ OBS_ALLOC(block, size);+ return block;+}++void *__wrap_memalign(size_t alignment, size_t size) {+ void *block = mi_memalign(alignment, size);+ OBS_ALLOC(block, size);+ return block;+}+#endif /* __linux__ */diff --git a/src/foundation/mem_override_win.c b/src/foundation/mem_override_win.cindex 8b51cb690..cf7e1a790 100644--- a/src/foundation/mem_override_win.c+++ b/src/foundation/mem_override_win.c@@ -42,6 +42,7 @@* mimalloc is always safe.*/#include "foundation/mem.h"+#include "foundation/mem_events.h" /* CBM_MEMEV_BACKING: empty outside the access lane */#include <mimalloc.h>#include <stddef.h>@@ -68,32 +69,73 @@ static inline bool mem_override_is_ours(const void *block) {return mi_is_in_heap_region(block);}+/* Waste sanitizer (mem_events.h). The wrapper IS the observation point, so the+ * caller's return address is the allocation site. Compiled to nothing outside+ * the `memwaste` flavour. Frees are reported BEFORE the block goes back. */+#if defined(CBM_MEMWASTE) && CBM_MEMWASTE+#define OBS_ALLOC_F(block, requested, flags) \+ do { \+ if ((block) && cbm_memev_enabled()) { \+ cbm_memev_alloc_ex((block), (requested), mi_usable_size(block), \+ __builtin_return_address(0), (unsigned)(flags)); \+ } \+ } while (0)+#define OBS_ALLOC(block, requested) OBS_ALLOC_F((block), (requested), 0)+#define OBS_REALLOC(old_block, grown, requested) \+ do { \+ if ((grown) && cbm_memev_enabled()) { \+ cbm_memev_realloc((old_block), (grown), (requested), mi_usable_size(grown), \+ __builtin_return_address(0)); \+ } \+ } while (0)+/* A block mimalloc did not make came from the CRT or a system DLL: the layer+ * could never have seen its allocation, so it is not an untracked free. */+#define OBS_FREE(block) \+ (mem_override_is_ours(block) ? cbm_memev_free(block) : cbm_memev_free_foreign(block))+#else+#define OBS_ALLOC(block, requested) ((void)0)+#define OBS_ALLOC_F(block, requested, flags) ((void)0)+#define OBS_REALLOC(old_block, grown, requested) ((void)0)+#define OBS_FREE(block) ((void)0)+#endif+void *__wrap_malloc(size_t size) {- return mi_malloc(size);+ void *block = mi_malloc(size);+ OBS_ALLOC(block, size);+ return block;}void *__wrap_calloc(size_t count, size_t size) {- return mi_calloc(count, size);+ void *block = mi_calloc(count, size);+ OBS_ALLOC_F(block, count * size, CBM_MEMEV_ZEROED);+ return block;}char *__wrap_strdup(const char *text) {- return mi_strdup(text);+ char *copy = mi_strdup(text);+ OBS_ALLOC_F(copy, copy ? strlen(copy) + 1 : 0, CBM_MEMEV_WRITTEN);+ return copy;}char *__wrap_strndup(const char *text, size_t length) {- return mi_strndup(text, length);+ char *copy = mi_strndup(text, length);+ OBS_ALLOC_F(copy, copy ? strlen(copy) + 1 : 0, CBM_MEMEV_WRITTEN);+ return copy;}void *__wrap__aligned_malloc(size_t size, size_t alignment) {/* Windows spells aligned allocation _aligned_malloc(size, alignment) —* note the argument order is the reverse of mi_malloc_aligned's. */- return mi_malloc_aligned(size, alignment);+ void *block = mi_malloc_aligned(size, alignment);+ OBS_ALLOC(block, size);+ return block;}void __wrap_free(void *block) {if (!block) {return;}+ OBS_FREE(block);if (mem_override_is_ours(block)) {mi_free(block);return;@@ -108,6 +150,7 @@ void __wrap__aligned_free(void *block) {if (!block) {return;}+ OBS_FREE(block);if (mem_override_is_ours(block)) {mi_free(block);return;@@ -117,10 +160,15 @@ void __wrap__aligned_free(void *block) {void *__wrap_realloc(void *block, size_t size) {if (!block) {- return mi_malloc(size);+ void *fresh = mi_malloc(size);+ OBS_ALLOC(fresh, size);+ return fresh;}if (mem_override_is_ours(block)) {+ CBM_MEMEV_BACKING(1);void *grown = mi_realloc(block, size);+ CBM_MEMEV_BACKING(-1);+ OBS_REALLOC(block, grown, size);return grown;}/* A CRT block cannot be handed to mi_realloc, and the CRT's realloc must@@ -134,6 +182,7 @@ void *__wrap_realloc(void *block, size_t size) {return NULL; /* original stays valid, as realloc requires */}memcpy(moved, block, old_size < size ? old_size : size);+ OBS_REALLOC(block, moved, size);__real_free(block);return moved;}diff --git a/src/foundation/mem_profile.c b/src/foundation/mem_profile.cdeleted file mode 100644index f0c43b745..000000000--- a/src/foundation/mem_profile.c+++ /dev/null@@ -1,325 +0,0 @@-/*- * mem_profile.c — see mem_profile.h.- *- * Two fixed tables, both open-addressed with linear probing:- * sites[] stack hash -> {frames, live bytes, live blocks, totals}- * pointers[] block address -> {site index, size}- *- * A single mutex guards both. That is heavy for a hot path and entirely- * acceptable here: profiling is opt-in, and a diagnostic that is fast but- * racy would produce numbers nobody can trust — which is the failure mode- * this whole harness exists to correct.- */-#include "mem_profile.h"--#include "compat_fs.h"-#include "compat_thread.h"-#include "foundation/constants.h"-#include "platform.h"--#include <stdatomic.h>-#include <stdint.h>-#include <stdio.h>-#include <stdlib.h>-#include <string.h>--#ifdef _WIN32-#ifndef WIN32_LEAN_AND_MEAN-#define WIN32_LEAN_AND_MEAN-#endif-#include <windows.h>-#else-#include <execinfo.h>-#endif--enum {- PROFILE_FRAMES = 8, /* stack depth per site */- PROFILE_SKIP_FRAMES = 2, /* this file's own frames */- PROFILE_SITES = 4096, /* distinct call sites */- PROFILE_POINTERS = 262144, /* tracked live blocks */- PROFILE_DEFAULT_MIN = 32768,-};--typedef struct {- uint64_t hash; /* 0 = empty slot */- void *frames[PROFILE_FRAMES];- int frame_count;- size_t live_bytes;- size_t live_blocks;- size_t total_bytes;- size_t total_blocks;- size_t peak_live_bytes;-} profile_site_t;--typedef struct {- void *block; /* NULL = empty slot */- size_t size;- size_t site;-} profile_pointer_t;--/* The profiler runs INSIDE the allocator: every hook here is reached from- * malloc/free. Anything it calls that itself allocates — reading an env var,- * the logger, even a probe — re-enters the hook, which recurses until the stack- * dies (silently: the process produces no stdout, no stderr and no log) and- * would deadlock the non-recursive mutex on the way. This guard makes the- * profiler invisible to itself. */-static _Thread_local bool g_profile_reentrant;--static cbm_mutex_t g_profile_mutex;-static profile_site_t g_sites[PROFILE_SITES];-static profile_pointer_t g_pointers[PROFILE_POINTERS];-static cbm_mem_profile_totals_t g_totals;-static atomic_int g_profile_enabled = -1;-static size_t g_profile_min = PROFILE_DEFAULT_MIN;--size_t cbm_mem_profile_threshold(void) {- (void)cbm_mem_profile_enabled();- return g_profile_min;-}--bool cbm_mem_profile_enabled(void) {- int state = atomic_load_explicit(&g_profile_enabled, memory_order_acquire);- if (state >= 0) {- return state == 1;- }- char buf[CBM_SZ_32];- bool on = cbm_safe_getenv("CBM_MEM_PROFILE", buf, sizeof(buf), NULL) != NULL && buf[0] == '1';- if (on) {- char min_buf[CBM_SZ_32];- if (cbm_safe_getenv("CBM_MEM_PROFILE_MIN", min_buf, sizeof(min_buf), NULL) != NULL) {- long parsed = strtol(min_buf, NULL, 10);- if (parsed > 0) {- g_profile_min = (size_t)parsed;- }- }- cbm_mutex_init(&g_profile_mutex);- }- atomic_store_explicit(&g_profile_enabled, on ? 1 : 0, memory_order_release);- return on;-}--/* FNV-1a over the return addresses: cheap, and collisions merely merge two- * sites rather than corrupting counts. */-static uint64_t profile_hash(void *const *frames, int count) {- uint64_t hash = 1469598103934665603ULL;- for (int i = 0; i < count; i++) {- uintptr_t value = (uintptr_t)frames[i];- for (size_t byte = 0; byte < sizeof(value); byte++) {- hash ^= (value >> (byte * 8)) & 0xFFU;- hash *= 1099511628211ULL;- }- }- return hash == 0 ? 1 : hash; /* 0 marks an empty slot */-}--static int profile_capture(void **frames, void *caller) {-#ifdef _WIN32- int captured = (int)CaptureStackBackTrace(PROFILE_SKIP_FRAMES, PROFILE_FRAMES, frames, NULL);- if (captured > 0) {- return captured;- }- /* Fall back to the caller the hook supplied. Using a return address from- * THIS frame would name the profiler, not the allocating code. */- if (caller) {- frames[0] = caller;- return 1;- }- frames[0] = __builtin_return_address(0);- return frames[0] != NULL ? 1 : 0;-#else- (void)caller;- void *raw[PROFILE_FRAMES + PROFILE_SKIP_FRAMES];- int got = backtrace(raw, PROFILE_FRAMES + PROFILE_SKIP_FRAMES);- int count = got - PROFILE_SKIP_FRAMES;- if (count <= 0) {- return 0;- }- if (count > PROFILE_FRAMES) {- count = PROFILE_FRAMES;- }- memcpy(frames, raw + PROFILE_SKIP_FRAMES, (size_t)count * sizeof(*frames));- return count;-#endif-}--/* Returns SIZE_MAX when the table is full, so the caller can count the loss. */-static size_t profile_intern_site_locked(void **frames, int count) {- uint64_t hash = profile_hash(frames, count);- size_t index = (size_t)(hash % PROFILE_SITES);- for (size_t probe = 0; probe < PROFILE_SITES; probe++) {- profile_site_t *site = &g_sites[index];- if (site->hash == hash) {- return index;- }- if (site->hash == 0) {- site->hash = hash;- site->frame_count = count;- memcpy(site->frames, frames, (size_t)count * sizeof(*frames));- g_totals.sites++;- return index;- }- index = (index + 1) % PROFILE_SITES;- }- return SIZE_MAX;-}--static size_t profile_pointer_slot(const void *block) {- uintptr_t value = (uintptr_t)block;- /* Block addresses are page-aligned at these sizes, so mix the high bits- * down before masking or every entry lands in the same few buckets. */- value ^= value >> 20;- value *= 2654435761U;- return (size_t)(value % PROFILE_POINTERS);-}--void cbm_mem_profile_alloc(void *block, size_t size) {- cbm_mem_profile_alloc_at(block, size, NULL);-}--void cbm_mem_profile_alloc_at(void *block, size_t size, void *caller) {- if (!block || g_profile_reentrant) {- return;- }- g_profile_reentrant = true;- if (!cbm_mem_profile_enabled() || size < g_profile_min) {- g_profile_reentrant = false;- return;- }- void *frames[PROFILE_FRAMES];- int count = profile_capture(frames, caller);- if (count <= 0) {- cbm_mutex_lock(&g_profile_mutex);- g_totals.capture_failed++;- cbm_mutex_unlock(&g_profile_mutex);- g_profile_reentrant = false;- return;- }- cbm_mutex_lock(&g_profile_mutex);- size_t site_index = profile_intern_site_locked(frames, count);- if (site_index == SIZE_MAX) {- g_totals.site_table_full++;- cbm_mutex_unlock(&g_profile_mutex);- g_profile_reentrant = false;- return;- }- size_t slot = profile_pointer_slot(block);- bool stored = false;- for (size_t probe = 0; probe < PROFILE_POINTERS; probe++) {- if (g_pointers[slot].block == NULL) {- g_pointers[slot].block = block;- g_pointers[slot].size = size;- g_pointers[slot].site = site_index;- stored = true;- break;- }- slot = (slot + 1) % PROFILE_POINTERS;- }- if (!stored) {- g_totals.pointer_table_full++;- cbm_mutex_unlock(&g_profile_mutex);- g_profile_reentrant = false;- return;- }- profile_site_t *site = &g_sites[site_index];- site->live_bytes += size;- site->live_blocks++;- site->total_bytes += size;- site->total_blocks++;- if (site->live_bytes > site->peak_live_bytes) {- site->peak_live_bytes = site->live_bytes;- }- g_totals.live_bytes += size;- g_totals.live_blocks++;- g_totals.total_bytes += size;- cbm_mutex_unlock(&g_profile_mutex);- g_profile_reentrant = false;-}--void cbm_mem_profile_free(void *block) {- if (!block || g_profile_reentrant) {- return;- }- g_profile_reentrant = true;- if (!cbm_mem_profile_enabled()) {- g_profile_reentrant = false;- return;- }- cbm_mutex_lock(&g_profile_mutex);- size_t slot = profile_pointer_slot(block);- for (size_t probe = 0; probe < PROFILE_POINTERS; probe++) {- profile_pointer_t *entry = &g_pointers[slot];- if (entry->block == block) {- profile_site_t *site = &g_sites[entry->site];- site->live_bytes -= entry->size;- site->live_blocks--;- g_totals.live_bytes -= entry->size;- g_totals.live_blocks--;- entry->block = NULL;- entry->size = 0;- entry->site = 0;- cbm_mutex_unlock(&g_profile_mutex);- g_profile_reentrant = false;- return;- }- if (entry->block == NULL) {- break; /* linear probe ended: never tracked (below threshold) */- }- slot = (slot + 1) % PROFILE_POINTERS;- }- /* Expected for every sub-threshold allocation, so this is a scale check,- * not an error: it should track the small-allocation rate, not grow with- * retained bytes. */- g_totals.untracked_frees++;- cbm_mutex_unlock(&g_profile_mutex);- g_profile_reentrant = false;-}--void cbm_mem_profile_totals(cbm_mem_profile_totals_t *out) {- if (!out) {- return;- }- memset(out, 0, sizeof(*out));- if (!cbm_mem_profile_enabled()) {- return;- }- cbm_mutex_lock(&g_profile_mutex);- *out = g_totals;- cbm_mutex_unlock(&g_profile_mutex);-}--bool cbm_mem_profile_dump(const char *path, const char *label) {- if (!path || !cbm_mem_profile_enabled()) {- return false;- }- FILE *out = cbm_fopen(path, "ab");- if (!out) {- return false;- }- cbm_mutex_lock(&g_profile_mutex);- (void)fprintf(out,- "{\"label\":\"%s\",\"threshold\":%zu,\"sites\":%zu,\"live_bytes\":%zu,"- "\"live_blocks\":%zu,\"total_bytes\":%zu,\"untracked_frees\":%zu,"- "\"site_table_full\":%zu,\"pointer_table_full\":%zu}\n",- label ? label : "", g_profile_min, g_totals.sites, g_totals.live_bytes,- g_totals.live_blocks, g_totals.total_bytes, g_totals.untracked_frees,- g_totals.site_table_full, g_totals.pointer_table_full);- for (size_t i = 0; i < PROFILE_SITES; i++) {- profile_site_t *site = &g_sites[i];- if (site->hash == 0 || site->total_blocks == 0) {- continue;- }- (void)fprintf(out,- "{\"label\":\"%s\",\"site\":%llu,\"live_bytes\":%zu,\"live_blocks\":%zu,"- "\"total_bytes\":%zu,\"total_blocks\":%zu,\"peak_live\":%zu,\"frames\":[",- label ? label : "", (unsigned long long)site->hash, site->live_bytes,- site->live_blocks, site->total_bytes, site->total_blocks,- site->peak_live_bytes);- for (int f = 0; f < site->frame_count; f++) {- (void)fprintf(out, "%s\"%p\"", f == 0 ? "" : ",", site->frames[f]);- }- (void)fprintf(out, "]}\n");- }- cbm_mutex_unlock(&g_profile_mutex);- (void)fclose(out);- return true;-}diff --git a/src/foundation/mem_profile.h b/src/foundation/mem_profile.hdeleted file mode 100644index 44613a45b..000000000--- a/src/foundation/mem_profile.h+++ /dev/null@@ -1,66 +0,0 @@-/*- * mem_profile.h — Which allocation sites retain memory?- *- * The census answers "which POOL holds the memory"; it cannot answer "which- * allocation put it there". Without that, every hypothesis about #581 needed a- * code change plus a soak to test, so the investigation could only ask one- * yes/no question per eight minutes — and two mechanisms that looked confirmed- * were later falsified. This records the missing half.- *- * Only allocations at or above a size threshold are tracked (default 32 KiB).- * That is deliberate: arena fragmentation is driven by large transient runs,- * and tracking every small allocation would cost more than the signal is worth.- * The threshold is reported alongside the data so the blind spot is explicit.- *- * Off unless CBM_MEM_PROFILE=1. All tables are fixed-capacity and allocated- * once; when a table fills, the overflow is COUNTED and reported rather than- * silently dropped, because a profiler that quietly loses records produces- * exactly the false confidence this is meant to end.- */-#ifndef CBM_MEM_PROFILE_H-#define CBM_MEM_PROFILE_H--#include <stdbool.h>-#include <stddef.h>--/* Gate: CBM_MEM_PROFILE=1. Read once, cached. */-bool cbm_mem_profile_enabled(void);--/* Minimum allocation size recorded, from CBM_MEM_PROFILE_MIN (bytes). */-size_t cbm_mem_profile_threshold(void);--/* Record an allocation / release. Safe to call unconditionally: both return- * immediately when profiling is off or the size is below the threshold. */-void cbm_mem_profile_alloc(void *block, size_t size);--/* Same, but with the caller's return address supplied by the hook itself.- * Required on Windows ARM64, where CaptureStackBackTrace returns nothing: a- * return address captured inside the capture helper names the helper, so every- * allocation collapses onto a handful of hook addresses and the attribution- * column becomes useless. Each hook must evaluate __builtin_return_address(0)- * in its OWN frame and pass it here. */-void cbm_mem_profile_alloc_at(void *block, size_t size, void *caller);-void cbm_mem_profile_free(void *block);--typedef struct {- size_t sites; /* distinct call sites seen */- size_t live_bytes; /* currently retained, attributed */- size_t live_blocks;- size_t total_bytes; /* cumulative allocated at/above threshold */- size_t untracked_frees; /* frees whose pointer was not in the table */- size_t site_table_full; /* records lost: site table exhausted */- size_t pointer_table_full; /* records lost: pointer table exhausted */- /* Stack capture returned nothing. On Windows ARM64 CaptureStackBackTrace- * can legitimately fail, and dropping those samples silently makes a- * working profiler look like an empty one. */- size_t capture_failed;-} cbm_mem_profile_totals_t;--void cbm_mem_profile_totals(cbm_mem_profile_totals_t *out);--/* Append one JSON object per site to `path`, biggest live first, with the raw- * return addresses of each site's stack. Symbolisation is deliberately left to- * the offline analyser so the hot path never loads a symbol handler. */-bool cbm_mem_profile_dump(const char *path, const char *label);--#endif /* CBM_MEM_PROFILE_H */diff --git a/src/main.c b/src/main.cindex 202aa6a18..52113bdcc 100644--- a/src/main.c+++ b/src/main.c@@ -71,6 +71,7 @@ enum {#include "foundation/compat_fs.h"#include "foundation/compat_thread.h"#include "foundation/mem.h"+#include "foundation/mem_events.h"#include "foundation/profile.h"#include "foundation/sha256.h"#include "foundation/secure_random.h"@@ -858,6 +859,7 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo/* The graph lives on this process's heaps: SQLite gets heaps of its* own here, and only here (cbm_sqlite_dedicated_heap). */cbm_sqlite_dedicated_heap(true);+ cbm_memev_process_role("worker");}cbm_index_set_worker_role_options(index_worker, response_out, worker_single_thread,worker_marker, worker_quarantine,@@ -1024,6 +1026,12 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo/* Supervised worker: hand the full result string to the parent via the* response file before printing (parent reads it back on a clean exit). */const char *ro = cbm_index_worker_response_out();+ if (cbm_index_worker_active()) {+ /* Waste dump + execution counts BEFORE the response: the parent may+ * end the worker as soon as the response file is complete, and the+ * fast exit below skips every at-exit writer anyway. */+ cbm_memev_process_exit();+ }bool worker_response_written = false;if (ro) {FILE *rf = cbm_fopen(ro, "wb");diff --git a/tests/test_mem_events.c b/tests/test_mem_events.cnew file mode 100644index 000000000..a3ddfd05c--- /dev/null+++ b/tests/test_mem_events.c@@ -0,0 +1,510 @@+/*+ * test_mem_events.c — the allocation-event layer of the waste sanitizer.+ *+ * The layer never dereferences a block, so these tests feed it synthetic+ * addresses: every assertion is about ACCOUNTING, and the verdicts are pure+ * functions of the event sequence (O9) — no allocator, no clock, no timing.+ */+#include "test_framework.h"+#include "../src/foundation/compat.h"+#include "../src/foundation/compat_fs.h"+#include "../src/foundation/compat_thread.h"+#include "../src/foundation/mem_events.h"++#include <stdint.h>+#include <stdio.h>+#include <string.h>+#ifndef _WIN32+#include <unistd.h>+#else+#include <process.h>+#define getpid _getpid+#endif++#define BLK(n) ((void *)(uintptr_t)(0x100000u + ((uintptr_t)(n) << 4)))+#define SITE_A ((void *)(uintptr_t)0xA000)+#define SITE_B ((void *)(uintptr_t)0xB000)+#define SITE_C ((void *)(uintptr_t)0xC000)++static void fresh(void) {+ cbm_memev_force_for_tests(true);+ cbm_memev_reset_for_tests();+}++static bool site_row(void *site, cbm_memev_site_t *out) {+ static cbm_memev_site_t rows[64];+ size_t n = cbm_memev_sites(rows, 64);+ for (size_t i = 0; i < n; i++) {+ if (rows[i].site == (uintptr_t)site) {+ *out = rows[i];+ return true;+ }+ }+ return false;+}++/* Requested vs usable is the allocator's slack, and it belongs to the site+ * that asked: 3 x (100 requested, 112 handed out) = 36 bytes nobody can use. */+TEST(memev_slack_is_charged_to_the_asking_site) {+ fresh();+ for (int i = 0; i < 3; i++) {+ cbm_memev_alloc(BLK(i), 100, 112, SITE_A);+ }+ cbm_memev_alloc(BLK(9), 64, 64, SITE_B);+ cbm_memev_site_t a;+ ASSERT_TRUE(site_row(SITE_A, &a));+ ASSERT_EQ(a.allocs, 3);+ ASSERT_EQ(a.requested_bytes, 300);+ ASSERT_EQ(a.usable_bytes, 336);+ ASSERT_EQ(a.live_bytes, 336);+ ASSERT_EQ(a.live_blocks, 3);+ cbm_memev_free(BLK(1));+ ASSERT_TRUE(site_row(SITE_A, &a));+ ASSERT_EQ(a.frees, 1);+ ASSERT_EQ(a.live_bytes, 224);+ ASSERT_EQ(a.live_blocks, 2);+ cbm_memev_totals_t t;+ cbm_memev_totals(&t);+ ASSERT_EQ(t.allocs, 4);+ ASSERT_EQ(t.sites, 2);+ ASSERT_EQ(t.live_bytes, 288);+ ASSERT_EQ(t.untracked_frees, 0);+ PASS();+}++/* A block freed by its own thread within a few allocations is churn: an+ * arena or a stack buffer would have served. One that outlives the window+ * is not. */+TEST(memev_short_lived_blocks_are_counted_as_churn) {+ fresh();+ for (int i = 0; i < 10; i++) {+ cbm_memev_alloc(BLK(i), 32, 32, SITE_A);+ cbm_memev_free(BLK(i));+ }+ cbm_memev_alloc(BLK(100), 32, 32, SITE_B); /* lives across the window */+ for (int i = 0; i < CBM_MEMEV_CHURN_WINDOW + 8; i++) {+ cbm_memev_alloc(BLK(200 + i), 8, 8, SITE_C);+ }+ cbm_memev_free(BLK(100));+ cbm_memev_site_t a;+ cbm_memev_site_t b;+ ASSERT_TRUE(site_row(SITE_A, &a));+ ASSERT_TRUE(site_row(SITE_B, &b));+ ASSERT_EQ(a.short_lived, 10);+ ASSERT_EQ(b.frees, 1);+ ASSERT_EQ(b.short_lived, 0);+ PASS();+}++/* A realloc chain is ONE object growing. The bytes the allocator had to move+ * are that object's cost, charged to the site that created it — not to the+ * realloc call. Growth in place moves nothing. */+TEST(memev_realloc_copy_bytes_go_to_the_original_site) {+ fresh();+ cbm_memev_alloc(BLK(1), 64, 64, SITE_A);+ cbm_memev_realloc(BLK(1), BLK(2), 128, 128, SITE_B); /* moved: 64 bytes copied */+ cbm_memev_realloc(BLK(2), BLK(2), 200, 256, SITE_B); /* in place: nothing copied */+ cbm_memev_realloc(BLK(2), BLK(3), 512, 512, SITE_B); /* moved: 256 bytes copied */+ cbm_memev_site_t a;+ cbm_memev_site_t b;+ ASSERT_TRUE(site_row(SITE_A, &a));+ ASSERT_FALSE(site_row(SITE_B, &b)); /* the realloc call site owns nothing */+ ASSERT_EQ(a.reallocs, 3);+ ASSERT_EQ(a.realloc_copy_bytes, 64 + 256);+ ASSERT_EQ(a.live_blocks, 1);+ ASSERT_EQ(a.live_bytes, 512);+ cbm_memev_free(BLK(3));+ ASSERT_TRUE(site_row(SITE_A, &a));+ ASSERT_EQ(a.live_bytes, 0);+ cbm_memev_totals_t t;+ cbm_memev_totals(&t);+ ASSERT_EQ(t.untracked_frees, 0);+ PASS();+}++/* The memory core announces itself before calling its backing allocator. An+ * observer underneath consumes the hint: one event, the CORE's caller as the+ * site, the class attached. Without an observer the hint stays pending and the+ * core emits the event itself. Either way the block is counted once. */+TEST(memev_hint_gives_one_event_with_the_cores_site_and_class) {+ fresh();+ /* observed path: hint, then the observer's event (which passes its own frame) */+ cbm_memev_hint(SITE_A, 5);+ ASSERT_TRUE(cbm_memev_hint_pending());+ cbm_memev_alloc(BLK(1), 40, 48, SITE_C);+ ASSERT_FALSE(cbm_memev_hint_pending());+ /* unobserved path: nobody consumed it, the core emits */+ cbm_memev_hint(SITE_A, 5);+ if (cbm_memev_hint_pending()) {+ cbm_memev_alloc(BLK(2), 40, 48, NULL);+ }+ /* an allocation that never touched the core */+ cbm_memev_alloc(BLK(3), 40, 48, SITE_B);++ cbm_memev_site_t a;+ cbm_memev_site_t b;+ cbm_memev_site_t c;+ ASSERT_TRUE(site_row(SITE_A, &a));+ ASSERT_TRUE(site_row(SITE_B, &b));+ ASSERT_FALSE(site_row(SITE_C, &c)); /* the observer's frame never becomes a site */+ ASSERT_EQ(a.allocs, 2);+ ASSERT_EQ(a.raw_bytes, 0); /* tagged by the core */+ ASSERT_EQ(b.raw_bytes, 48); /* the unmigrated surface, as a number */+ cbm_memev_totals_t t;+ cbm_memev_totals(&t);+ ASSERT_EQ(t.raw_bytes, 48);++ /* frees carry no handshake: whoever is responsible reports BEFORE the block goes back */+ cbm_memev_free(BLK(1));+ cbm_memev_free(BLK(2));+ cbm_memev_totals(&t);+ ASSERT_EQ(t.frees, 2);+ ASSERT_EQ(t.untracked_frees, 0);+ PASS();+}++/* A free of a block the layer never saw is a NUMBER, not a silent no-op: it+ * measures how much traffic predates or bypasses the observers. */+TEST(memev_untracked_free_is_counted_not_dropped) {+ fresh();+ cbm_memev_free(BLK(77));+ cbm_memev_free(NULL);+ cbm_memev_totals_t t;+ cbm_memev_totals(&t);+ ASSERT_EQ(t.untracked_frees, 1);+ ASSERT_EQ(t.frees, 0);+ PASS();+}++/* A block another allocator made (the Windows C runtime, a system DLL) is not a+ * block the layer missed: it is counted apart, so the soundness gate that reads+ * untracked frees sees only real misses (the Windows daemon's 14 CRT blocks+ * tripped it, 2026-09-17). */+TEST(memev_foreign_free_is_counted_apart_from_untracked) {+ fresh();+ cbm_memev_free_foreign(BLK(78));+ cbm_memev_free_foreign(NULL);+ cbm_memev_totals_t t;+ cbm_memev_totals(&t);+ ASSERT_EQ(t.foreign_frees, 1);+ ASSERT_EQ(t.untracked_frees, 0);+ ASSERT_EQ(t.frees, 0);+ PASS();+}++typedef struct {+ int first;+ int count;+ void *site;+} span_t;++static void *alloc_span(void *arg) {+ const span_t *s = (const span_t *)arg;+ for (int i = 0; i < s->count; i++) {+ cbm_memev_alloc(BLK(s->first + i), 48, 64, s->site);+ }+ cbm_memev_flush_thread();+ return NULL;+}++static void *free_span(void *arg) {+ const span_t *s = (const span_t *)arg;+ for (int i = 0; i < s->count; i++) {+ cbm_memev_free(BLK(s->first + i));+ }+ cbm_memev_flush_thread();+ return NULL;+}++enum { MEMEV_T = 4, MEMEV_PER_T = 50000 };++/* Blocks are routinely freed by a thread that did not allocate them, and the+ * per-site totals are SUMS — so they cannot depend on which thread did what or+ * in which order the threads ran. 200,000 live blocks also walks every shard+ * through several growth steps. */+TEST(memev_cross_thread_frees_balance_and_totals_are_order_independent) {+ fresh();+ cbm_thread_t th[MEMEV_T];+ span_t spans[MEMEV_T];+ for (int i = 0; i < MEMEV_T; i++) {+ spans[i] = (span_t){+ .first = i * MEMEV_PER_T, .count = MEMEV_PER_T, .site = (i % 2) ? SITE_A : SITE_B};+ ASSERT_EQ(cbm_thread_create(&th[i], 0, alloc_span, &spans[i]), 0);+ }+ for (int i = 0; i < MEMEV_T; i++) {+ cbm_thread_join(&th[i]);+ }+ cbm_memev_totals_t t;+ cbm_memev_totals(&t);+ ASSERT_EQ(t.allocs, (uint64_t)MEMEV_T * MEMEV_PER_T);+ ASSERT_EQ(t.live_blocks, (uint64_t)MEMEV_T * MEMEV_PER_T);+ ASSERT_EQ(t.live_bytes, (uint64_t)MEMEV_T * MEMEV_PER_T * 64);+ ASSERT_EQ(t.pointer_table_full, 0);++ /* every span is freed by a DIFFERENT thread than the one that allocated it */+ span_t rotated[MEMEV_T];+ for (int i = 0; i < MEMEV_T; i++) {+ rotated[i] = spans[(i + 1) % MEMEV_T];+ ASSERT_EQ(cbm_thread_create(&th[i], 0, free_span, &rotated[i]), 0);+ }+ for (int i = 0; i < MEMEV_T; i++) {+ cbm_thread_join(&th[i]);+ }+ cbm_memev_totals(&t);+ ASSERT_EQ(t.frees, (uint64_t)MEMEV_T * MEMEV_PER_T);+ ASSERT_EQ(t.live_blocks, 0);+ ASSERT_EQ(t.live_bytes, 0);+ ASSERT_EQ(t.untracked_frees, 0);+ cbm_memev_site_t a;+ cbm_memev_site_t b;+ ASSERT_TRUE(site_row(SITE_A, &a));+ ASSERT_TRUE(site_row(SITE_B, &b));+ ASSERT_EQ(a.allocs, (uint64_t)(MEMEV_T / 2) * MEMEV_PER_T);+ ASSERT_EQ(b.usable_bytes, (uint64_t)(MEMEV_T / 2) * MEMEV_PER_T * 64);+ ASSERT_EQ(a.short_lived, 0); /* freed by another thread: never churn */+ PASS();+}++/* Dormant means dormant: with the layer off nothing is recorded and a hint+ * never lingers to mis-attribute a later event. */+TEST(memev_dormant_layer_records_nothing) {+ fresh();+ cbm_memev_force_for_tests(false);+ cbm_memev_hint(SITE_A, 3);+ cbm_memev_alloc(BLK(1), 10, 16, SITE_A);+ ASSERT_FALSE(cbm_memev_hint_pending());+ cbm_memev_free(BLK(1));+ cbm_memev_force_for_tests(true);+ cbm_memev_totals_t t;+ cbm_memev_totals(&t);+ ASSERT_EQ(t.allocs, 0);+ ASSERT_EQ(t.untracked_frees, 0);+ ASSERT_EQ(t.sites, 0);+ PASS();+}++/* The report is JSON lines: one header, then one line per site and one per+ * work row. Ordering is the report tool's job; the dump only has to be+ * complete. */+TEST(memev_dump_writes_header_sites_and_work) {+ fresh();+ cbm_memev_alloc(BLK(1), 10, 16, SITE_A);+ cbm_memev_alloc(BLK(2), 900, 1024, SITE_B);+ cbm_work_note(CBM_WORK_MEMCPY, SITE_C, 4096, 0, 0);+ char path[512];+ snprintf(path, sizeof(path), "%s/cbm_memev_dump_%d.jsonl", cbm_tmpdir(), (int)getpid());+ (void)remove(path);+ ASSERT_TRUE(cbm_memev_dump(path, "test"));+ FILE *f = cbm_fopen(path, "r");+ ASSERT_NOT_NULL(f);+ char line[4096];+ int n = 0;+ bool header = false;+ bool site_a = false;+ bool site_b = false;+ bool work = false;+ while (fgets(line, sizeof(line), f)) {+ if (n++ == 0) {+ header = strstr(line, "\"memwaste\":2") && strstr(line, "\"why\":\"test\"") &&+ strstr(line, "\"allocs\":2");+ }+ site_a = site_a || strstr(line, "\"site\":\"0xa000\",\"allocs\":1");+ site_b = site_b || (strstr(line, "\"site\":\"0xb000\"") && strstr(line, "\"usable_bytes\":1024"));+ work = work || (strstr(line, "\"work\":\"memcpy\"") && strstr(line, "\"site\":\"0xc000\"") &&+ strstr(line, "\"bytes\":4096") && strstr(line, "\"peak\":4096"));+ }+ fclose(f);+ (void)remove(path);+ ASSERT_TRUE(header);+ ASSERT_TRUE(site_a);+ ASSERT_TRUE(site_b);+ ASSERT_TRUE(work);+ ASSERT_EQ(n, 4);+ PASS();+}++static bool work_row(cbm_work_kind_t kind, void *site, cbm_work_row_t *out) {+ static cbm_work_row_t rows[128];+ size_t n = cbm_work_rows(rows, 128);+ for (size_t i = 0; i < n; i++) {+ if (rows[i].kind == kind && rows[i].site == (uintptr_t)site) {+ *out = rows[i];+ return true;+ }+ }+ return false;+}++/* The fill pattern is how the layer sees memory nobody wrote. A 4 KB block+ * that got 100 bytes: 3,996 bytes never written, and of the 1,000 requested+ * 900 lie beyond the last write. A hole in the middle is never-written memory+ * too, but it is not over-requested: the bytes after it were used. */+TEST(memev_fill_scan_measures_never_written_and_over_requested) {+ fresh();+ cbm_memev_scan_for_tests(true, false);+ static uint8_t big[4096];+ static uint8_t holed[128];+ cbm_memev_alloc(big, 1000, sizeof(big), SITE_A);+ ASSERT_EQ(big[4095], 0xA5); /* the layer filled it */+ memset(big, 1, 100);+ cbm_memev_free(big);++ cbm_memev_alloc(holed, 80, sizeof(holed), SITE_B);+ memset(holed, 2, 16); /* [0, 16) written */+ memset(holed + 64, 3, 16); /* [64, 80) written: [16, 64) is a 48-byte hole */+ cbm_memev_free(holed);++ cbm_memev_site_t a;+ cbm_memev_site_t b;+ ASSERT_TRUE(site_row(SITE_A, &a));+ ASSERT_TRUE(site_row(SITE_B, &b));+ ASSERT_EQ(a.scanned_blocks, 1);+ ASSERT_EQ(a.never_written_bytes, 3996);+ ASSERT_EQ(a.over_requested_bytes, 900);+ ASSERT_EQ(b.never_written_bytes, 48 + 48); /* the hole + the unwritten tail */+ ASSERT_EQ(b.over_requested_bytes, 0);+ PASS();+}++/* A calloc'd block is not filled -- its zeros are the contract. Its zero tail+ * at free is an UPPER bound on untouched memory (a written zero looks the same). */+TEST(memev_calloc_tail_is_scanned_for_zeros) {+ fresh();+ cbm_memev_scan_for_tests(true, false);+ static uint8_t block[256];+ memset(block, 0, sizeof(block)); /* what calloc hands out */+ cbm_memev_alloc_ex(block, 256, sizeof(block), SITE_A, CBM_MEMEV_ZEROED);+ ASSERT_EQ(block[200], 0); /* never filled */+ memset(block, 7, 32);+ cbm_memev_free(block);+ cbm_memev_site_t a;+ ASSERT_TRUE(site_row(SITE_A, &a));+ ASSERT_EQ(a.zero_untouched_bytes, 224);+ ASSERT_EQ(a.never_written_bytes, 0);+ PASS();+}++/* A phase boundary is a snapshot: what each site still holds, and which small+ * live blocks carry the same bytes as another -- memory an interning table or+ * a shared constant would have served once. */+TEST(memev_phase_snapshot_finds_retained_memory_and_duplicates) {+ fresh();+ cbm_memev_scan_for_tests(true, true);+ static uint8_t blocks[4][64];+ for (int i = 0; i < 3; i++) {+ cbm_memev_alloc(blocks[i], 16, sizeof(blocks[i]), SITE_A);+ memcpy(blocks[i], "same bytes here!", 16);+ }+ cbm_memev_alloc(blocks[3], 16, sizeof(blocks[3]), SITE_B);+ memcpy(blocks[3], "different bytes!", 16);+ cbm_memev_phase("snapshot");+ cbm_memev_site_t a;+ cbm_memev_site_t b;+ ASSERT_TRUE(site_row(SITE_A, &a));+ ASSERT_TRUE(site_row(SITE_B, &b));+ ASSERT_EQ(a.retained_bytes, 3 * 64);+ ASSERT_EQ(a.dup_blocks, 2); /* the first copy is the original */+ ASSERT_EQ(a.dup_bytes, 2 * 64);+ ASSERT_EQ(b.retained_bytes, 64);+ ASSERT_EQ(b.dup_blocks, 0);+ for (int i = 0; i < 4; i++) {+ cbm_memev_free(blocks[i]);+ }+ PASS();+}++/* Work rows sum per (kind, site); `peak` keeps the largest single note. For a+ * container that is the biggest capacity one instance left unused -- the sums+ * run over USES, the peak is the memory figure. */+TEST(memev_work_rows_sum_and_keep_the_peak) {+ fresh();+ cbm_work_note(CBM_WORK_MEMCPY, SITE_A, 100, 0, 0);+ cbm_work_note(CBM_WORK_MEMCPY, SITE_A, 5000, 0, 0);+ cbm_memev_container(CBM_WORK_CT_ARENA, SITE_B, 1000, 100, 2);+ cbm_memev_container(CBM_WORK_CT_ARENA, SITE_B, 500, 450, 0);+ cbm_work_row_t w;+ ASSERT_TRUE(work_row(CBM_WORK_MEMCPY, SITE_A, &w));+ ASSERT_EQ(w.calls, 2);+ ASSERT_EQ(w.bytes, 5100);+ ASSERT_EQ(w.peak, 5000);+ ASSERT_TRUE(work_row(CBM_WORK_CT_ARENA, SITE_B, &w));+ ASSERT_EQ(w.calls, 2);+ ASSERT_EQ(w.bytes, 1500);+ ASSERT_EQ(w.aux1, 550);+ ASSERT_EQ(w.aux2, 2);+ ASSERT_EQ(w.peak, 900);+ PASS();+}++/* A repeat is work whose answer could not have changed. strlen of the same+ * buffer holding a DIFFERENT string of the same length is new work; a lookup+ * of the same key text repeats even from another buffer, unless the table was+ * written in between. */+TEST(memev_repeats_require_unchanged_input) {+ fresh();+ char buf[16];+ memcpy(buf, "alpha", 6);+ cbm_work_note_strlen(SITE_A, buf, 5);+ cbm_work_note_strlen(SITE_A, buf, 5); /* repeat */+ memcpy(buf, "omega", 6);+ cbm_work_note_strlen(SITE_A, buf, 5); /* same pointer, same length, new text */+ cbm_work_row_t w;+ ASSERT_TRUE(work_row(CBM_WORK_STRLEN, SITE_A, &w));+ ASSERT_EQ(w.calls, 3);+ ASSERT_EQ(w.repeats, 1);++ static const int table = 0;+ char k1[8];+ char k2[8];+ memcpy(k1, "key", 4);+ memcpy(k2, "key", 4);+ cbm_work_note_ht(CBM_WORK_HT_GET, SITE_B, &table, k1, 0, true);+ cbm_work_note_ht(CBM_WORK_HT_GET, SITE_B, &table, k2, 0, true); /* same text: repeat */+ cbm_work_note_ht(CBM_WORK_HT_GET, SITE_B, &table, k2, 1, true); /* table written since */+ cbm_work_note_ht(CBM_WORK_HT_GET, SITE_B, &table, "other", 1, false);+ ASSERT_TRUE(work_row(CBM_WORK_HT_GET, SITE_B, &w));+ ASSERT_EQ(w.calls, 4);+ ASSERT_EQ(w.repeats, 1);+ ASSERT_EQ(w.aux1, 1); /* one miss */++ cbm_work_note_path(CBM_WORK_STAT, SITE_A, "/tmp/cbm-memev-path", false);+ cbm_work_note_path(CBM_WORK_STAT, SITE_C, "/tmp/cbm-memev-path", false); /* process-wide */+ cbm_work_note_path(CBM_WORK_OPEN, SITE_C, "/tmp/cbm-memev-path", true); /* other kind */+ ASSERT_TRUE(work_row(CBM_WORK_STAT, SITE_C, &w));+ ASSERT_EQ(w.repeats, 1);+ ASSERT_TRUE(work_row(CBM_WORK_OPEN, SITE_C, &w));+ ASSERT_EQ(w.repeats, 0);+ ASSERT_EQ(w.aux1, 1); /* a failure */+ PASS();+}++/* Pools compare their workers by the events each produced. */+TEST(memev_thread_ops_count_this_threads_events) {+ fresh();+ uint64_t before = cbm_memev_thread_ops();+ cbm_memev_alloc(BLK(1), 8, 8, SITE_A);+ cbm_memev_alloc(BLK(2), 8, 8, SITE_A);+ cbm_work_note(CBM_WORK_MEMSET, SITE_B, 8, 0, 0);+ ASSERT_EQ(cbm_memev_thread_ops() - before, 3);+ cbm_memev_free(BLK(1));+ cbm_memev_free(BLK(2));+ PASS();+}++SUITE(mem_events) {+ RUN_TEST(memev_slack_is_charged_to_the_asking_site);+ RUN_TEST(memev_short_lived_blocks_are_counted_as_churn);+ RUN_TEST(memev_realloc_copy_bytes_go_to_the_original_site);+ RUN_TEST(memev_hint_gives_one_event_with_the_cores_site_and_class);+ RUN_TEST(memev_untracked_free_is_counted_not_dropped);+ RUN_TEST(memev_foreign_free_is_counted_apart_from_untracked);+ RUN_TEST(memev_cross_thread_frees_balance_and_totals_are_order_independent);+ RUN_TEST(memev_dormant_layer_records_nothing);+ RUN_TEST(memev_dump_writes_header_sites_and_work);+ RUN_TEST(memev_fill_scan_measures_never_written_and_over_requested);+ RUN_TEST(memev_calloc_tail_is_scanned_for_zeros);+ RUN_TEST(memev_phase_snapshot_finds_retained_memory_and_duplicates);+ RUN_TEST(memev_work_rows_sum_and_keep_the_peak);+ RUN_TEST(memev_repeats_require_unchanged_input);+ RUN_TEST(memev_thread_ops_count_this_threads_events);+}