Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(mem): vendor mimalloc 3.5.3 so free(NULL) before init is safe
Martin Vogelf9f44b586996
diff --git a/Makefile.cbm b/Makefile.cbmindex 3b93146f1..32124ab12 100644--- a/Makefile.cbm+++ b/Makefile.cbm@@ -45,8 +45,9 @@ endif# -Wdate-time: with -Werror, any use of __DATE__/__TIME__/__TIMESTAMP__ fails the# build. Build-time entropy is how one identical source tree produced a new hash-# on every compile (see the local patch in vendored/mimalloc/src/options.c) —-# once removed, it must not creep back through a new call site.+# on every compile (mimalloc's version banner, patched locally until upstream+# dropped the clock in v3.4.5; see vendored/mimalloc/PATCHES.md) — once+# removed, it must not creep back through a new call site.CFLAGS_COMMON = -std=c11 -D_DEFAULT_SOURCE -D_GNU_SOURCE -Wall -Wextra -Werror \-Wno-unused-parameter -Wno-sign-compare -Wdate-time \$(GCC_ONLY_FLAGS) \@@ -182,8 +183,9 @@ endififeq ($(IS_MINGW),yes)# --no-insert-timestamp: the PE header otherwise carries the link wall clock, so# two Windows builds of identical source are never byte-identical and every-# release is a brand-new file to a reputation system. Same motivation as the-# mimalloc __DATE__ patch; ELF and Mach-O have no equivalent field to clear.+# release is a brand-new file to a reputation system. Same motivation as+# dropping mimalloc's __DATE__ banner; ELF and Mach-O have no equivalent field+# to clear.WIN32_LIBS := -lws2_32 -lpsapi -lshell32 -ladvapi32 -lbcrypt -Wl,--allow-multiple-definition -Wl,--stack,8388608 -Wl,--no-insert-timestamp -staticendif@@ -653,10 +655,14 @@ endif# work inside the allocator (2026-09-17). Committed-bytes accounting, which the# memory budget reads, is kept in every build (subproc stats, not MI_STAT).# The TEST build below keeps mimalloc's debug checks.+# MI_PROFILE=0: mimalloc 3.5.2+ compiles its allocation-sampling profiler in by+# default, which puts a sample countdown on the allocation path. The product+# never uses it, so the production allocator leaves it out (the contract in+# tests/test_prod_allocator_contract.sh checks the banner has no profiling line).MIMALLOC_CFLAGS = -std=c11 -O2 -w \-Ivendored/mimalloc/include \-Ivendored/mimalloc/src \- -DMI_OVERRIDE=1 -DMI_DEBUG=0 \+ -DMI_OVERRIDE=1 -DMI_DEBUG=0 -DMI_PROFILE=0 \$(MIMALLOC_OVERRIDE_DEFINE)MIMALLOC_CFLAGS_TEST = -std=c11 -g -O1 -w \-Ivendored/mimalloc/include \diff --git a/THIRD_PARTY.md b/THIRD_PARTY.mdindex 4b76c28e8..17e88a695 100644--- a/THIRD_PARTY.md+++ b/THIRD_PARTY.md@@ -121,10 +121,12 @@ License summary:| wyhash | `internal/cbm/vendored/wyhash/` | Unlicense (public domain) | [wangyi-fudan/wyhash](https://github.com/wangyi-fudan/wyhash) |Local modifications to these libraries are documented next to the-vendored sources (currently only SQLite: `vendored/sqlite3/PATCHES.md`,-raising the Unix VFS `MAX_PATHNAME` ceiling from 512 to 4096 to match-CBM's 4 KiB path support). Patches must be reapplied on every upstream-refresh and are covered by `scripts/vendored-checksums.txt`.+vendored sources: `vendored/sqlite3/PATCHES.md` (raising the Unix VFS+`MAX_PATHNAME` ceiling from 512 to 4096 to match CBM's 4 KiB path support)+and `vendored/mimalloc/PATCHES.md` (the upstream source pin, mimalloc v3.5.3+at commit `d4881d33`, plus the macOS decommit-recommit change and one comment+rewording). Patches must be reapplied on every upstream refresh and are+covered by `scripts/vendored-checksums.txt`.The graph-UI HTTP server is a first-party implementation(`src/ui/httpd.c` + `src/ui/http_server.c`) — no third-party HTTP librarydiff --git a/scripts/ci/generate-sbom.py b/scripts/ci/generate-sbom.pyindex ff2e97efd..1a7d8eacb 100644--- a/scripts/ci/generate-sbom.py+++ b/scripts/ci/generate-sbom.py@@ -40,7 +40,7 @@ sbom = {"packages": [{"SPDXID": "SPDXRef-Package-sqlite3", "name": "sqlite3", "versionInfo": "3.51.3", "licenseDeclared": "blessing", "downloadLocation": "https://sqlite.org", "filesAnalyzed": False},{"SPDXID": "SPDXRef-Package-yyjson", "name": "yyjson", "versionInfo": "0.12.0", "licenseDeclared": "MIT", "downloadLocation": "https://github.com/ibireme/yyjson", "filesAnalyzed": False},- {"SPDXID": "SPDXRef-Package-mimalloc", "name": "mimalloc", "versionInfo": "3.3.2", "licenseDeclared": "MIT", "downloadLocation": "https://github.com/microsoft/mimalloc", "filesAnalyzed": False},+ {"SPDXID": "SPDXRef-Package-mimalloc", "name": "mimalloc", "versionInfo": "3.5.3", "licenseDeclared": "MIT", "downloadLocation": "https://github.com/microsoft/mimalloc", "filesAnalyzed": False},{"SPDXID": "SPDXRef-Package-xxhash", "name": "xxhash", "versionInfo": "0.8.3", "licenseDeclared": "BSD-2-Clause", "downloadLocation": "https://github.com/Cyan4973/xxHash", "filesAnalyzed": False},{"SPDXID": "SPDXRef-Package-tre", "name": "tre", "versionInfo": "0.8.0", "licenseDeclared": "BSD-2-Clause", "downloadLocation": "https://github.com/laurikari/tre", "filesAnalyzed": False},{"SPDXID": "SPDXRef-Package-tree-sitter", "name": "tree-sitter", "versionInfo": "0.24.4", "licenseDeclared": "MIT", "downloadLocation": "https://github.com/tree-sitter/tree-sitter", "filesAnalyzed": False},diff --git a/scripts/vendored-checksums.txt b/scripts/vendored-checksums.txtindex 7762ba82f..845f125a7 100644--- a/scripts/vendored-checksums.txt+++ b/scripts/vendored-checksums.txt@@ -992,50 +992,56 @@ a86b7735823b0408a4f3b922be86d4d25b6f4ab0dab32e03fb77246ca439dcf2 internal/cbm/v9b4bc8245565c98ccfc61c07749928b57e7c0f6fddb0530c4f6aa1971893d88b internal/cbm/vendored/zstd/zstd.h66a8c3f71d12ea6e797e4f622f31f3f8f81c41b36f48cad4f5de7d8bfb6aac0a internal/cbm/vendored/zstd/zstd_errors.h82ade5d7d9b029044b5fbbc326207fc47c2d44ee4dd71e8a559c36d728217de9 vendored/mimalloc/LICENSE-dd4f25cae53209d45d73f8e6a2b9c219e8fc7434d97f20eba9af1a8b850030fd vendored/mimalloc/include/mimalloc-new-delete.h+edb95ac4f912bbcde6c1b658e147e757896c881eae249dbb5bbbae66ebdf6deb vendored/mimalloc/PATCHES.md+0a9c7e6daea2814876eab49697922f98b630c27ab9b089fbe4bd74f6e56edd68 vendored/mimalloc/include/mimalloc-new-delete.h21fcf61c4443341ac6bf6ea528af31dc7267e8e3456fc64bfd07704503032175 vendored/mimalloc/include/mimalloc-override.h-5260df227ea1612b8678a06b063cc8f48e739b551554541cdeb4d430a7e257b3 vendored/mimalloc/include/mimalloc-stats.h-869161433ab2d807a75752d0762accf5b68b6b24ce67e417af46bf181b9b6c72 vendored/mimalloc/include/mimalloc.h-e9512b521a82cc0cbc262f6060dd70bb4243ac4a7d69d6ab8b1ef8e56f181a21 vendored/mimalloc/include/mimalloc/atomic.h-57a7297e18ae464a7bfd3fc25a46229008602385697305c69ed289b89ece7631 vendored/mimalloc/include/mimalloc/bits.h-5aeeff21d667daf4aadcc66f634aa957ecd688d15b277828f367c8c2274251a8 vendored/mimalloc/include/mimalloc/internal.h-555302d2fcd05e974441d0c9c9208c26e860db4b3f17b9025a08f42e0f4807ba vendored/mimalloc/include/mimalloc/prim.h-1f378b4b6063c078d2aeb5d916c1d30b5db787a44ab9118e7daf673b086882c6 vendored/mimalloc/include/mimalloc/track.h-b54e760da2a8d9d8a9cb708e280268c2d194c99bc518b21f72db70100578ad6a vendored/mimalloc/include/mimalloc/types.h-70420b4430da4bb6b5197fd27c6c0df485249c53a986bc9536c9fb643fc7613b vendored/mimalloc/src/alloc-aligned.c-6c8e9b0d042481c55791edb7330daec64bedaf26f0e70c9ca40de3cc17a690fe vendored/mimalloc/src/alloc-override.c-98d0eb877d7b9cdf7e9f857aaf3c739b69f746b91221e0e427e82bdd7cec90b3 vendored/mimalloc/src/alloc-posix.c-a92d58e47220e5b6d138b83d92498771be4195ee59136a134e4461f6f726b9b8 vendored/mimalloc/src/alloc.c-86fde644ede2e8d582e53499568a1713516a20326e0350eaa06a28029cece91b vendored/mimalloc/src/arena-meta.c-a1d6900d2b9a17461aa2a72a7791dfb22f6167b86f7ac6f15f981d9735aa2adb vendored/mimalloc/src/arena.c-de16033e029488e9ade4152ea1274e1a5ecbdc2c8a8d8ee50b745fd037ee6f6c vendored/mimalloc/src/bitmap.c-2e7a54ff44592a1fe8fe929e691322622f8aa1e0b1e80954781f332a88dd6cd1 vendored/mimalloc/src/bitmap.h-c9a845dd7b9020f26d73761d235fb75e7bc4a3682576de750050ac4098211b43 vendored/mimalloc/src/free.c-ae8e88737c9861e8e071fe5f1e7acc52093bcffd136a359a6b473101e968c14f vendored/mimalloc/src/heap.c-eca1fe77d72ab2aaccc447c8119056e9c4e8851bda9a95472b573539f76107ad vendored/mimalloc/src/init.c-ebd95b475f15d5387e6ef3c88a9cb47b88e64d7e55998ae1dc656614676a443b vendored/mimalloc/src/libc.c-192ce06ac9076445d223205fb340d8a1fc818dbf2b4c4a801dfd2c10bc6c1162 vendored/mimalloc/src/options.c-011e965f28c0e2c885a6834d51bd756a458dd37ebfed6fd4d9ad6607ce0390ed vendored/mimalloc/src/os.c-1be3a208c678fc34187c2a8ffb75f3fe1eecc674bbf8b2af4d1f329506b1fad5 vendored/mimalloc/src/page-map.c-7ec275805c9f00eb66710ffa99817b3ed1dba9826435d08f4866f7935152f6ae vendored/mimalloc/src/page-queue.c-d3614a59f2407a32e75b051deb1aff31b233f587d325dcc5420ba508551b687e vendored/mimalloc/src/page.c-7488e9e562ca85d2d757ec48e4d334935263a2333aa05b6061d8520f427c5890 vendored/mimalloc/src/prim/emscripten/prim.c-ce14eb6dfd55f241095b55f70983931a2d92f7a6a622a9b360cdd70ab3b53208 vendored/mimalloc/src/prim/osx/alloc-override-zone.c+c0aabbf2fbe54ce55e7dee23cb1891c2d65e0e1f5b18d9fe1a4e733c0c91d996 vendored/mimalloc/include/mimalloc-profile.h+57aaf0b0db23521a32b0c6d7e0969e37ce69466fcfb0cb9b2d0114c2b56be182 vendored/mimalloc/include/mimalloc-stats.h+8ec1bff9902bab6682e3ecbf7613ccc915e082ca743fdd0c396d3954323fe8bf vendored/mimalloc/include/mimalloc.h+7be611200328a99b052eda81abb8ad78b7e03446a0f6d302c1eb31cc9f096398 vendored/mimalloc/include/mimalloc/atomic.h+1ecfb1bd1a7b6f73c335d21f3411c9ce0de3cbf6710e2d209e6cb6df2baf1ed3 vendored/mimalloc/include/mimalloc/bits.h+26c10f7b990c51629618c74c64e3263a1fe00da787c2e72c56a79f87393f602c vendored/mimalloc/include/mimalloc/internal.h+a701ed0c182b4d0039bf683b4ac197011b8b794971a616c76ca64f036bc30d8b vendored/mimalloc/include/mimalloc/prim-tls.h+1987e8e2eedc07bb181bf2a11a27bec80a5309c32cfa66a56900fb4cbb64b172 vendored/mimalloc/include/mimalloc/prim.h+31d94e981640bc4c340f21887610a9cd7242af4c5adc578f722ad03cb852db2e vendored/mimalloc/include/mimalloc/track.h+01abbaf1cc5572675b97f4115f205335e319b93249ee5163fb3bf2ad0fbbceae vendored/mimalloc/include/mimalloc/types.h+8390ff8d0e6c2954db1d95bfc1a4388b302605a1c5102c0495e5163838186e7f vendored/mimalloc/src/alloc-aligned.c+6576949f30978f3b7544d9696af12dac9615074a5c2fc05e3681475575bc6ca3 vendored/mimalloc/src/alloc-override.c+a6dcdef4694964c972e6cf37f197f93c95f321566a3b5ccc1761129257255e93 vendored/mimalloc/src/alloc-posix.c+e074a8a622f6317d42eb43e48db8cde3345696c96d6749523ef4fd58aeb55bb7 vendored/mimalloc/src/alloc.c+27fce7003a5108fb9d0c61e92e0cd7870f687f7e3cf12ad19c7fb3954b2a0bc2 vendored/mimalloc/src/arena.c+b33f7b220358667fb0eddcd6569ec7b79f7ef61daf08448c8448c8d1a8741b09 vendored/mimalloc/src/bitmap.c+f3a9375ec0b56d515bea4b23732bf9a04964efecc516eb8aaac8b58ebf4512d9 vendored/mimalloc/src/bitmap.h+56a1c57bd08c49359e19e8cdcd2a11b50c30bb12a53063375374fb3f5c8d6189 vendored/mimalloc/src/free.c+a33d339568e5a7d16d6bae5650a79b1aef649e9b4ad0261be982580c2971eb64 vendored/mimalloc/src/heap.c+1452ca684eca6d959eec9eae91750162f7738055f995bd4863f79b87aab88a8c vendored/mimalloc/src/init.c+b774d5750409006b690ab6262f037d78362ca978729314493636b358249e7fbd vendored/mimalloc/src/libc.c+5bfe80a0c473371d299840e8ed20831601d4998e4164e2e0bf71303697395824 vendored/mimalloc/src/options.c+06b2d6d8a4cf3bfcc1822bb09aeaa4e7b741e7a5241f70b4f693b8038f455b00 vendored/mimalloc/src/os.c+ff3509ae3d4185e9cb2a95e1f35daba3329e998bac917ac7938f1bf06dba0f78 vendored/mimalloc/src/page-map.c+4128ca8b8d4df1116f73bf9a397903ca9941701c635345b1241295c3a31ff47d vendored/mimalloc/src/page-queue.c+ad0df1217e96669ef63539c345ea7dc4d637c3269be2d4de36ac3a5a42fb0c12 vendored/mimalloc/src/page.c+896fc1016a26e10c6cf96259d0524c57e123ac9dcef1ba04f23d0add445286dd vendored/mimalloc/src/prim/emscripten/prim.c+502efbc8616f18be83713759438a0e5b8017ed215947ddfd48e74b85fd899114 vendored/mimalloc/src/prim/osx/alloc-override-zone.c247a9952465eb105be03a9962e922b085ce5d52034775551a61cd8594275be73 vendored/mimalloc/src/prim/osx/prim.c-cc771788e5ba591efb681829a8724f8662a1f50f3db39cf5ebe7738b8ed57e44 vendored/mimalloc/src/prim/prim.c-792531421c247fca0766ff7630057ec491173578ca4d1919ae34f05eb289ccf4 vendored/mimalloc/src/prim/readme.md-20594d38ef4fc0c641f20a8664d17ee03ce195ca21b44e036f43cdb7bcd0bd0d vendored/mimalloc/src/prim/unix/prim.c-bee9c655a17e1e93bf4908e0a902fd3b566628447b1446fcbfa92bca2e76ca15 vendored/mimalloc/src/prim/wasi/prim.c+884443f66808143d645dc6254c38e27f91b540a667ea23fe0ba1ef1caaf2b114 vendored/mimalloc/src/prim/prim-tls.c+44419a450e742812d8f3c881d957bdd6cf18f40b90dfe1153084d2b64666f12d vendored/mimalloc/src/prim/prim.c+28510fce6710d7d08224135e2e986e85995d7907232d3d6e278a82213ca78935 vendored/mimalloc/src/prim/readme.md+a5ce291e50e965fbeedc632f54bc9f10b78c0548cf7739cd477e21c6e2751b3a vendored/mimalloc/src/prim/unix/prim.c+2bc862cddb522314d58b3c6ef041e101c25cf04dfdda35cf78f91dbb6140cf46 vendored/mimalloc/src/prim/wasi/prim.cf70e61a1f2fe63da2f51c581895cf4a0918a9bffb0cf8600171f297141446962 vendored/mimalloc/src/prim/windows/etw-mimalloc.wprp4f3110ef2054c95cb275be96cb279224d3d46a728c5117bd1435425f382de778 vendored/mimalloc/src/prim/windows/etw.h445e6a70f9728fcc76a5ba52079336064d5e2257c2969023f1c64bf943776f54 vendored/mimalloc/src/prim/windows/etw.man-5cde60fcef94cd79ab816030be00f8c1bdf18fab343563508c681ec6e762d1f1 vendored/mimalloc/src/prim/windows/prim.c+41d96c081cbda6bcad0e8270eab7c3d69f94dfe3d3dcd8a9567569c595984cb0 vendored/mimalloc/src/prim/windows/prim.c1bcb43eef7fcddf6d8f18c6d5ee9bd350090ed183a1009d4f1c9b4d81a205bb9 vendored/mimalloc/src/prim/windows/readme.md-a96a1acc75fd6595811b28420cf0495b87fac98c4dbc73aa0d742b6824f607c8 vendored/mimalloc/src/random.c-4059be2a6676693be3433f5aedc711f56301772e65f3573654c236d6d1cfd074 vendored/mimalloc/src/static.c-16825bbc252de422982b5728df7eda181d952824f5e3ce958c7fc85feae64b8e vendored/mimalloc/src/stats.c-7f76bcace4b0578f2063477752138adabd99f7b10a276998e296a312c47a7cfe vendored/mimalloc/src/theap.c-49be69a4b674e09690a63870830f3ffb05698ed4210949ef5add11957a08325b vendored/mimalloc/src/threadlocal.c+2bd12844f9de0392fdb00ea70f5ad15649ffd8e49f9a4d2388e15d6469a7b703 vendored/mimalloc/src/random.c+4edc7700c2dd911cd03919d1098e973d887900ab08a5f41fd9648a79517fca57 vendored/mimalloc/src/sample-guarded.c+3df74fee931244a302293f7d215dc87e3b5c120ab7df207edd92d0ff19eb3729 vendored/mimalloc/src/sample-profile.c+fadfb75216098793f9f81e57d6ff3b63914084976526ed7867c9ca212825d784 vendored/mimalloc/src/static.c+d78e0467008946b40d5356315a9ba76e04354ead65b1aafabbb7bd091c3826a6 vendored/mimalloc/src/stats.c+41c147e8eb6e960687b79a201dd4498fb80f58923a0a4d73e04e80ef315b8eb6 vendored/mimalloc/src/subproc.c+91271bc76502c743d109f32201c02271f63a9f6ec51ef207f8b533f6de39df27 vendored/mimalloc/src/theap.c+01dbaacd5ac6685dc59c40d82a3f8932e5290d4d05f9a441424e4ea53e1ba973 vendored/mimalloc/src/threadlocal.ccfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30 vendored/nomic/LICENSE893a74d9e3f0a74358344edc999e165739df343b36297967f3fd9cd462022f39 vendored/nomic/NOTICEdab3009c0d76b0c5e05ad8abc7c9f8f6effca547fea3c0394be96418a85c1081 vendored/nomic/code_tokens.hdiff --git a/vendored/mimalloc/PATCHES.md b/vendored/mimalloc/PATCHES.mdnew file mode 100644index 000000000..fb97f9ace--- /dev/null+++ b/vendored/mimalloc/PATCHES.md@@ -0,0 +1,95 @@+# Vendored mimalloc: source pin and local patches++## Source pin++- Upstream: <https://github.com/microsoft/mimalloc>+- Tag: `v3.5.3` (annotated tag object `deac3dcf5f0614884013d411a5c07e16bec96c9f`)+- Commit: `d4881d338125e1cb7c47ba4cfb398d6f7c0c8d45`+ (tree `fbc4e8f72287d127c079ada5ede870b7d63e55f5`)+- Vendored: `LICENSE`, every file under `include/` and every file under `src/`.+ Nothing else (no build files, tests or docs).++v3.5.3 is upstream's latest published release at the time of the pin+(GitHub release of 2026-09-17, on the `main3` branch).++v3.5.3 contains upstream commit `b2226c195fe884223f6d6f68ea52df476e57e5c2`+(mimalloc #1341, first released in v3.4.4): `free(NULL)` must work before the+allocator is initialized. With 3.3.2's release build, glibc 2.44's+`newlocale()` freed NULL during libstdc++ start-up, before mimalloc's+constructor, and every Linux binary died with SIGSEGV (reported with a+reproduction by Felix Rath in #2562). The preinit fixture in+`tests/test_prod_allocator_contract.sh` pins it.++## Verifying the vendored bytes++Every vendored file is byte-identical to the tag except the two hunks below.+Re-check from a fresh bare clone (nothing in it is built or run):++```sh+git clone --bare https://github.com/microsoft/mimalloc.git /tmp/mimalloc.git+git --git-dir=/tmp/mimalloc.git rev-parse 'v3.5.3^{commit}' # d4881d33...+for f in $(git --git-dir=/tmp/mimalloc.git ls-tree -r --name-only v3.5.3 -- LICENSE include src); do+ [ "$(git --git-dir=/tmp/mimalloc.git rev-parse "v3.5.3:$f")" = \+ "$(git hash-object "vendored/mimalloc/$f")" ] || echo "differs: $f"+done+```++The expected output is exactly the two files named below, and+`vendored/mimalloc/include` plus `vendored/mimalloc/src` must hold no file the+tag lacks.++## Local patches (reapply on every refresh)++Each patch is pinned by a check that fails when the patch is dropped.++### 1. macOS decommit always recommits: `src/prim/unix/prim.c`++In `_mi_prim_decommit`, macOS takes the recommit path that upstream reserves for+`MI_DEBUG` / `MI_SECURE>2` builds, so a purged range is not re-accounted as a+whole by `MADV_FREE_REUSE` when the arena reuses it. This cut the indexer's peak+`phys_footprint` substantially (measurements in the inline comment and in commit+63fb7b3d; re-measured on v3.5.3 when this pin was taken). Linux and Windows keep+upstream's behaviour.++Pinned by: the decommit fixture in `tests/test_prod_allocator_contract.sh`+(`needs_recommit` must be true on macOS and upstream's false elsewhere).++### 2. Comment wording: `src/prim/osx/alloc-override-zone.c`++Comment only, no code change: "crash in fork()." reads "crash while forking.".+`scripts/security-vendored.sh` rejects call spellings such as `fork(` anywhere+in vendored C, comments included, so the upstream sentence fails that gate.+(Wording first proposed in #2562.)++Pinned by: `scripts/security-vendored.sh` itself, which reports the pristine+upstream line as a blocked subprocess call.++## Build configuration++`Makefile.cbm` compiles the production object with `-DMI_PROFILE=0`, which+leaves out mimalloc's allocation-sampling profiler (on by default since 3.5.2)+and its sample countdown on the allocation path. Pinned by: the banner fixture+in `tests/test_prod_allocator_contract.sh` (a profiling build prints a+`profiling` line). The test build keeps upstream's default.++## Dropped patches++- Reproducible version banner (`src/options.c`, carried from 1f674c80 up to+ 3.4.4): upstream removed `__DATE__`/`__TIME__` from the banner in v3.4.5+ (mimalloc #1355). The banner fixture in+ `tests/test_prod_allocator_contract.sh` still asserts that the first line is+ exactly `vX.Y.Z` with no build clock, so a future refresh cannot bring it+ back unnoticed.++## Refresh procedure++1. Replace `LICENSE`, `include/` and `src/` with the new tag's files, deleting+ files the tag no longer has, and verify the staged tree against the tag+ before patching.+2. Reapply the patches above, or drop the ones upstream has absorbed (and move+ them to "Dropped patches" with the upstream reference).+3. Update the pin in this file and `versionInfo` in+ `scripts/ci/generate-sbom.py`, then run `scripts/security-vendored.sh --update`.+4. After `make -f Makefile.cbm cbm`, run+ `bash tests/test_prod_allocator_contract.sh build/c` on Linux and macOS+ (`scripts/test.sh` runs it on every CI leg), and `scripts/security-vendored.sh`.diff --git a/vendored/mimalloc/include/mimalloc-new-delete.h b/vendored/mimalloc/include/mimalloc-new-delete.hindex c16f4a665..bb1bc5075 100644--- a/vendored/mimalloc/include/mimalloc-new-delete.h+++ b/vendored/mimalloc/include/mimalloc-new-delete.h@@ -37,7 +37,7 @@ terms of the MIT license. A copy of the license can be found in the filevoid operator delete (void* p, const std::nothrow_t&) noexcept { mi_free(p); }void operator delete[](void* p, const std::nothrow_t&) noexcept { mi_free(p); }- mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); }+ mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); }mi_decl_new(n) void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); }mi_decl_new_nothrow(n) void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }@@ -48,7 +48,7 @@ terms of the MIT license. A copy of the license can be found in the filevoid operator delete[](void* p, std::size_t n) noexcept { mi_free_size(p,n); };#endif- #if (__cplusplus > 201402L || defined(__cpp_aligned_new))+ #if (__cplusplus > 201402L && defined(__cpp_aligned_new))void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };diff --git a/vendored/mimalloc/include/mimalloc-profile.h b/vendored/mimalloc/include/mimalloc-profile.hnew file mode 100644index 000000000..5a1327ff7--- /dev/null+++ b/vendored/mimalloc/include/mimalloc-profile.h@@ -0,0 +1,79 @@+/* ----------------------------------------------------------------------------+Copyright (c) 2024-2026, Microsoft Research, Daan Leijen+This is free software; you can redistribute it and/or modify it under the+terms of the MIT license. A copy of the license can be found in the file+"LICENSE" at the root of this distribution.+-----------------------------------------------------------------------------*/+#pragma once+#ifndef MIMALLOC_PROFILE_H+#define MIMALLOC_PROFILE_H++#include <mimalloc.h>+#include <stdbool.h> // bool+#include <stdint.h> // uint64_t++//---------------------------------------------------------------------------+// Initial support for profiling hooks; still experimental+//---------------------------------------------------------------------------++#define MI_PROFILE_SAMPLE_DATA_MAX_SIZE (1024)++// User defined profiler; see `test_profile.c` for an example.+typedef struct mi_profiler_s mi_profiler_t;++// Profiler data is stored together with each sampled allocation (unless the `on_free` field in the profiler is NULL.)+typedef struct mi_profiler_sample_data_s {+ size_t user_data_size; // size of the user_data (should be the `mi_profiler_t.sample_data_size`)+ void* user_data[1]; // default, but can be less or more (up to 1KiB), depending on `sample_data_size`+} mi_profiler_sample_data_t;++// Profiling callback invoked on each sampled allocation.+// If `profiler_data!=NULL` (i.e. when `on_free` is not NULL), then `profiler_data->requested_size == requested_size`.+typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t requested_size, size_t bytes_sample_rate, uint64_t bytes_since_last_sample, const mi_heap_t* heap);++// Profiling callback invoked on each sampled in-place re-allocation.+typedef size_t (mi_cdecl mi_profiler_on_realloc_inplace_fun)(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t old_size, const mi_heap_t* heap);++// Profiling callback invoked on a previously sampled allocation.+typedef void (mi_cdecl mi_profiler_on_free_fun )(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, const mi_heap_t* heap);++// A profiler+// All fields are considered immutable such that they can be copied and accessed concurrently. All fields can be NULL/0.+struct mi_profiler_s {+ void* reserved; // opaque; reserved by mimalloc+ size_t sample_data_size; // size of required profiler data for each sampled allocation (or zero for no data)+ size_t initial_sample_rate;// initial sample rate in bytes (set to at least 1 or higher) (can be adjusted by `on_alloc`)+ mi_profiler_on_alloc_fun* on_alloc; // called on a sampled allocation (may be called concurrently)+ mi_profiler_on_free_fun* on_free; // called on when previous sampled allocation is freed (may be called concurrently)+ mi_profiler_on_realloc_inplace_fun* on_realloc_inplace; // (as yet unused) called on in-place reallocation of a previous sampled allocation (may be called concurrently)+ // ... more user fields allowed+};++// Exported definitions+#ifdef __cplusplus+extern "C" {+#endif++// attach a profiler to a particular heap only.+mi_decl_export bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler);++// disable profiling for a particular heap; useful for a heap that the profiler uses itself for metadata.+mi_decl_export void mi_heap_profile_disable(mi_heap_t* heap);++// attach a profiler to any (current and future) heaps in a sub-process (unless those heaps disabled profiling)+mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler);++// attach a profiler to any heaps in the main sub-process+mi_decl_export bool mi_profile(mi_profiler_t* profiler);++// start sampling+mi_decl_export bool mi_profiler_start(mi_profiler_t* profiler);++// end sampling+mi_decl_export bool mi_profiler_stop(mi_profiler_t* profiler);++#ifdef __cplusplus+}+#endif++#endif // MIMALLOC_PROFILE_Hdiff --git a/vendored/mimalloc/include/mimalloc-stats.h b/vendored/mimalloc/include/mimalloc-stats.hindex 1c620fd1c..ab45a0715 100644--- a/vendored/mimalloc/include/mimalloc-stats.h+++ b/vendored/mimalloc/include/mimalloc-stats.h@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2024-2025, Microsoft Research, Daan Leijen+Copyright (c) 2024-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -22,7 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file#elif __cplusplus >= 201103L#define mi_decl_align(a) alignas(a)#else-#define mi_decl_align(a)+#define mi_decl_align(a) _Alignas(a)#endif@@ -49,7 +49,7 @@ typedef struct mi_stat_counter_s {MI_STAT_COUNT(threads) /* number of threads */ \MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \- MI_STAT_COUNT(malloc_requested) /* malloc requested bytes */ \+ MI_STAT_COUNTER(malloc_requested) /* malloc requested bytes */ \\MI_STAT_COUNTER(mmap_calls) \MI_STAT_COUNTER(commit_calls) \@@ -75,11 +75,16 @@ typedef struct mi_stat_counter_s {/* only on v3 */ \MI_STAT_COUNT(heaps) \MI_STAT_COUNT(theaps) \+ MI_STAT_COUNT(pages_os_abandoned) /* pages in the os abandoned list (bad) */ \+ MI_STAT_COUNT(pages_os_allocated) /* pages allocated outside arenas (bad) */ \MI_STAT_COUNTER(pages_reclaim_on_alloc) \MI_STAT_COUNTER(pages_reclaim_on_free) \MI_STAT_COUNTER(pages_reabandon_full) \MI_STAT_COUNTER(pages_unabandon_busy_wait) \- MI_STAT_COUNTER(heaps_delete_wait)+ MI_STAT_COUNTER(heaps_delete_wait) \+ MI_STAT_COUNTER(pages_stat_updates) /* calls to successful page_stat_update */ \+ MI_STAT_COUNTER(pages_stat_update_count) /* total free/allocs */ \+ MI_STAT_COUNTER(profile_samples) /* total sampled profiled allocations */ \// Size bins for chunkstypedef enum mi_chunkbin_e {@@ -161,6 +166,12 @@ mi_decl_export bool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_idmi_decl_export char* mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULLmi_decl_export size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept;+// get theap stats (only thread safe on theaps belonging to the calling thread)+mi_decl_export bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept;++// add the theap stats to the stats of the parent heap and clear the theap stats+mi_decl_export void mi_theap_stats_merge_to_heap(mi_theap_t* theap) mi_attr_noexcept;+#ifdef __cplusplus}#endifdiff --git a/vendored/mimalloc/include/mimalloc.h b/vendored/mimalloc/include/mimalloc.hindex f373f5dc8..2985edd94 100644--- a/vendored/mimalloc/include/mimalloc.h+++ b/vendored/mimalloc/include/mimalloc.h@@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file#ifndef MIMALLOC_H#define MIMALLOC_H-#define MI_MALLOC_VERSION 30302 // major + 2 digits minor + 2 digits patch+#define MI_MALLOC_VERSION 30503 // major + 2 digits minor + 2 digits patch// ------------------------------------------------------// Compiler specific attributes@@ -95,8 +95,9 @@ terms of the MIT license. A copy of the license can be found in the file// Includes// -------------------------------------------------------#include <stddef.h> // size_t+#include <stddef.h> // size_t, wchar_t#include <stdbool.h> // bool+#include <assert.h> // inline assertions#ifdef __cplusplusextern "C" {@@ -119,8 +120,8 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char*// ------------------------------------------------------// Extended allocation functions// -------------------------------------------------------#define MI_SMALL_WSIZE_MAX (128)-#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*))+#define MI_SMALL_WSIZE_MAX 128+#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(size_t)) // mimalloc considers `sizeof(size_t)` as the machine word sizemi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);@@ -133,10 +134,17 @@ mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize)mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept;mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept;+// `mi_free_size` can be more efficient (as it calls mi_free_small internally).+mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept;+// `mi_free_small` is for special applications like language runtimes.-// it should only be used to free objects from `mi_(heap_)(m|z)alloc_small` and is potentially a tiny bit faster than `mi_free`+// it should only be used to free objects from `mi_*alloc_small` and is potentially a tiny bit faster than `mi_free`mi_decl_export void mi_free_small(void* p) mi_attr_noexcept;+// As `mi_free_small` but `p` should not be a NULL pointer.+mi_decl_export void mi_free_small_nonnull(void* p) mi_attr_noexcept;++// -------------------------------------------------------------------------------------// Aligned allocation// Note that `alignment` always follows `size` for consistency with unaligned@@ -157,12 +165,13 @@ mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t new// Typed allocation, the type is always the first parameter// -------------------------------------------------------#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp)))-#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp)))+#define mi_malloc_tp(tp) ((tp*)mi_malloc_csize(sizeof(tp)))+#define mi_zalloc_tp(tp) ((tp*)mi_zalloc_csize(sizeof(tp)))#define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp)))#define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp)))#define mi_reallocn_tp(tp,p,n) ((tp*)mi_reallocn(p,n,sizeof(tp)))#define mi_recalloc_tp(tp,p,n) ((tp*)mi_recalloc(p,n,sizeof(tp)))+#define mi_free_tp(tp,p) (mi_free_csize(p,sizeof(tp)))#define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))#define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))@@ -187,7 +196,7 @@ typedef void (mi_cdecl mi_error_fun)(int err, void* arg);mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg);mi_decl_export void mi_collect(bool force) mi_attr_noexcept;-mi_decl_export int mi_version(void) mi_attr_noexcept;+mi_decl_export int mi_version(void);mi_decl_export void mi_options_print(void) mi_attr_noexcept;mi_decl_export void mi_process_info_print(void) mi_attr_noexcept;mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;@@ -329,6 +338,7 @@ mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept;mi_decl_export void mi_arenas_print(void) mi_attr_noexcept;mi_decl_export size_t mi_arena_min_alignment(void);mi_decl_export size_t mi_arena_min_size(void);+mi_decl_export size_t mi_arena_max_object_size(void);typedef void* mi_arena_id_t;mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);@@ -382,8 +392,9 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_calloc(mi_theapmi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);mi_decl_nodiscard mi_decl_export void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);-+mi_decl_nodiscard mi_decl_export void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);// ------------------------------------------------------// Experimental@@ -465,25 +476,26 @@ typedef enum mi_option_e {mi_option_deprecated_purge_extend_delay,mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's)mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)- mi_option_visit_abandoned, // allow visiting theap blocks from abandoned threads (=0)+ mi_option_deprecated_visit_abandoned, // allow visiting theap blocks from abandoned threads (=0)mi_option_guarded_min, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)mi_option_guarded_max, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)mi_option_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000)mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)mi_option_generic_collect, // collect theaps every N (=10000) generic allocation calls- mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from the current theap, 1 allow always+ mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallow always, 0 allows if the page originated from the current theap, 1 allow alwaysmi_option_page_full_retain, // retain N full (small) pages per size class (=2)mi_option_page_max_candidates, // max candidate pages to consider for allocation (=4)mi_option_max_vabits, // max user space virtual address bits to consider (=48)mi_option_pagemap_commit, // commit the full pagemap (to always catch invalid pointer uses) (=0)mi_option_page_commit_on_demand, // commit page memory on-demandmi_option_page_max_reclaim, // don't reclaim pages of the same originating theap if we already own N pages (in that size class) (=-1 (unlimited))- mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=16)+ mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=32)mi_option_allow_thp, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.mi_option_minimal_purge_size, // set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled.mi_option_arena_max_object_size, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).- mi_option_arena_is_numa_local, // experimental+ mi_option_arena_is_numa_local, // experimental: associate local numa node with an initial arena allocation+ mi_option_collect_merges_stats, // on each theap collection, the theap stats are merged automatically with the parent heap_mi_option_last,// legacy option namesmi_option_large_os_pages = mi_option_allow_large_os_pages,@@ -513,13 +525,13 @@ mi_decl_export void mi_option_set_default(mi_option_t option, long value);// note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our theap before free-ing.// --------------------------------------------------------------------------------------------------------mi_decl_export void mi_cfree(void* p) mi_attr_noexcept;+mi_decl_export bool mi_cfree(void* p) mi_attr_noexcept;mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept;mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept;mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept;mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept;-mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept;+mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size); // mi_attr_noexcept;mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);@@ -530,19 +542,22 @@ mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* ptrp, size_t count, smi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept;mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept;-mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept;mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept;mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept;mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept;// wide characters-#include <wchar.h> // wchar_tmi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept;mi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept mi_attr_malloc;mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc;-// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.-// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).+// --------------------------------------------------------+// C++ wrappers+// The `mi_new` wrappers implement C++ semantics on out-of-memory+// instead of directly returning `NULL`. (and call `std::get_new_handler`+// and potentially raise a `std::bad_alloc` exception).+// --------------------------------------------------------+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1);mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);@@ -554,6 +569,63 @@ mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount,mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2);mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) mi_attr_malloc mi_attr_alloc_size(2);+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_alloc_new_nothrow(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);+++// --------------------------------------------------------------------------+// Inlined constant size allocations.+// These are meant for runtime systems, or overrides where we need the best+// performance for small, constant-size allocations.+// These are only better than the regular functions if the size or alignment+// are indeed constant at the call site.+// --------------------------------------------------------------------------++// Internal machine word size allocation. `wsize` is the allocation size in machine words (`sizeof(size_t)`)+mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept mi_attr_malloc;+mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept mi_attr_malloc;+mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept mi_attr_malloc;+mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept mi_attr_malloc;++static inline size_t mi_wsize_from_size(size_t size) mi_attr_noexcept {+ return (size + sizeof(size_t) - 1) / sizeof(size_t);+}++static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexcept {+ if (size <= MI_SMALL_SIZE_MAX) { return mi_wmalloc_small(mi_wsize_from_size(size)); } else { return mi_malloc(size); }+}+static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept {+ if (size <= MI_SMALL_SIZE_MAX) { return mi_wzalloc_small(mi_wsize_from_size(size)); } else { return mi_zalloc(size); }+}+static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept {+ assert(theap!=NULL);+ if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wmalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); }+}+static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept {+ assert(theap!=NULL);+ if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wzalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); }+}++static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept {+ if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); }+}+static inline void mi_free_csize_nonnull(void* p, size_t size) mi_attr_noexcept {+ assert(p!=NULL);+ if (size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); }+}+static inline void mi_free_csize_aligned(void* p, size_t size, size_t aligned) mi_attr_noexcept {+ if (aligned <= size && size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); }+}+static inline void mi_free_csize_aligned_nonnull(void* p, size_t size, size_t aligned) mi_attr_noexcept {+ assert(p!=NULL);+ if (aligned <= size && size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); }+}++// ------------------------------------------------------+// C++ standard library allocator interface.+// ------------------------------------------------------+#ifdef __cplusplus}#endifdiff --git a/vendored/mimalloc/include/mimalloc/atomic.h b/vendored/mimalloc/include/mimalloc/atomic.hindex e4abde6b0..9c2339802 100644--- a/vendored/mimalloc/include/mimalloc/atomic.h+++ b/vendored/mimalloc/include/mimalloc/atomic.h@@ -9,13 +9,14 @@ terms of the MIT license. A copy of the license can be found in the file#define MI_ATOMIC_H// include windows.h or pthreads.h-#if defined(_WIN32)+#if defined(_WIN32) || defined(__CYGWIN__) // we use windows locks on cygwin, but otherwise treat it at unix#ifndef WIN32_LEAN_AND_MEAN#define WIN32_LEAN_AND_MEAN#endif#include <windows.h>-#elif !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))-#define MI_USE_PTHREADS+#endif+#if MI_TLS_MODEL_PTHREADS || (!defined(_WIN32) && !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)))+#define MI_USE_PTHREADS 1#include <pthread.h>#endif@@ -215,24 +216,22 @@ static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintpt(void)(mo);return (uintptr_t)MI_MSC_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);}-static inline void mi_atomic_thread_fence(mi_memory_order mo) {- (void)(mo);- _Atomic(uintptr_t) x = 0;- mi_atomic_exchange_explicit(&x, 1, mo);-}static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) {(void)(mo);// assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc#if defined(_M_IX86) || defined(_M_X64)+ // on x86/x64 we have a strong memory model so any load is acquirereturn (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);#elif defined(_M_ARM) || defined(_M_ARM64)if (mo == mi_memory_order_relaxed) {return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);}+ #if !defined(__clang__) // work around __ldar missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689else if (mo <= mi_memory_order_acquire) {return MI_MSC_XX(__ldar)((volatile const uintptr_t*)p);}+ #endifelse {const uintptr_t u = (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);__dmb(15); // _ARM(64)_BARRIER_SY@@ -252,9 +251,11 @@ static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mif (mo == mi_memory_order_relaxed) {MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);}+ #if !defined(__clang__) // work around __stlr missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689else if (mo <= mi_memory_order_release) {MI_MSC_XX(__stlr)((volatile uintptr_t*)p,x);}+ #endifelse {mi_atomic_exchange_explicit(p, x, mo);}@@ -272,7 +273,7 @@ static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_oif (mo == mi_memory_order_relaxed) {return __iso_volatile_load64((volatile const int64_t*)p);}- #if defined(_M_ARM64)+ #if defined(_M_ARM64) && !defined(__clang__) // work around __ldar64 missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689else if (mo <= mi_memory_order_acquire) {return __ldar64((volatile const uintptr_t*)p);}@@ -297,7 +298,7 @@ static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, miif (mo == mi_memory_order_relaxed) {__iso_volatile_store64((volatile int64_t*)p,x);}- #if defined(_M_ARM64)+ #if defined(_M_ARM64) && !defined(__clang__) // work around __stlr64 missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689else if (mo == mi_memory_order_release) {__stlr64((volatile uint64_t*)p, (uint64_t)x);}@@ -335,18 +336,18 @@ static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const vola}}-static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) {+static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)* p, int64_t x) {int64_t current;do {current = *p;} while (current < x && _InterlockedCompareExchange64(p, x, current) != current);}-static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) {+static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t)* p, int64_t i) {mi_atomic_addi64_relaxed(p, i);}-static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) {+static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t)* p, int64_t* exp, int64_t des) {const int64_t read = _InterlockedCompareExchange64(p, des, *exp);if (read == *exp) {return true;@@ -358,17 +359,17 @@ static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p,}// The pointer macros cast to `uintptr_t`.-#define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p))-#define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p))+#define mi_atomic_load_ptr_acquire(tp,p) ((tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p)))+#define mi_atomic_load_ptr_relaxed(tp,p) ((tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p)))#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)-#define mi_atomic_exchange_ptr_relaxed(tp,p,x) (tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x)-#define mi_atomic_exchange_ptr_release(tp,p,x) (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x)-#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x)+#define mi_atomic_exchange_ptr_relaxed(tp,p,x) ((tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x))+#define mi_atomic_exchange_ptr_release(tp,p,x) ((tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x))+#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) ((tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x))#define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire))#define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed))@@ -417,7 +418,7 @@ typedef _Atomic(uintptr_t) mi_atomic_guard_t;#define mi_lock_maybe(lock,acquire) for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) )-#if defined(_WIN32)+#if defined(_WIN32) || defined(__CYGWIN__)typedef struct mi_lock_s {SRWLOCK mutex; // slim reader-writer lock@@ -496,10 +497,10 @@ static inline void mi_lock_release(mi_lock_t* lock) {lock->mutex.unlock();}static inline void mi_lock_init(mi_lock_t* lock) {- new(&lock->mutex) std::mutex();+ new(&lock->mutex) std::mutex(); // in-place constructor}static inline void mi_lock_done(mi_lock_t* lock) {- (void)(lock);+ lock->mutex.~mutex(); // in-place destructor}#else@@ -524,6 +525,7 @@ static inline bool mi_lock_try_acquire(mi_lock_t* lock) {return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (uintptr_t)1);}static inline void mi_lock_acquire(mi_lock_t* lock) {+ size_t ticks = 0;for (int i = 0; i < 10000; i++) { // for at most 10000 tries?if (mi_lock_try_acquire(lock)) return;_mi_prim_thread_yield();diff --git a/vendored/mimalloc/include/mimalloc/bits.h b/vendored/mimalloc/include/mimalloc/bits.hindex 617877283..5aa0d8375 100644--- a/vendored/mimalloc/include/mimalloc/bits.h+++ b/vendored/mimalloc/include/mimalloc/bits.h@@ -16,6 +16,15 @@ terms of the MIT license. A copy of the license can be found in the file#include <stddef.h> // size_t#include <stdint.h> // int64_t etc#include <stdbool.h> // bool+#include <limits.h> // LONG_MAX++#if defined(__cplusplus)+#define mi_decl_externc extern "C"+#define mi_init_struct_zero { }+#else+#define mi_decl_externc+#define mi_init_struct_zero { 0 }+#endif// ------------------------------------------------------// Size of a pointer.@@ -102,13 +111,13 @@ typedef int32_t mi_ssize_t;#include <intrin.h>#endif-#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // msvc+#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // avx2 implies bmi2#define __BMI2__ 1#endif-#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__)) && !defined(__BMI1__) // msvc+#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__) || defined(__BMI__)) && !defined(__BMI1__) // bmi2 implies bmi1#define __BMI1__ 1#endif-#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // msvc+#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // avx2 implies lzcnt#define __LZCNT__ 1#endif@@ -126,9 +135,18 @@ typedef int32_t mi_ssize_t;#define MI_MAX_VABITS (32)#endif+// the MI_MIN_VABITS determine how many bits of the address space are always committed in the page_map+#if MI_MAX_VABITS <= 32+#define MI_MIN_VABITS (32)+#elif MI_MAX_VABITS <= 42+#define MI_MIN_VABITS MI_MAX_VABITS+#else+#define MI_MIN_VABITS (43) /* 8 TiB */+#endif+// use a flat page-map or a 2-level one#ifndef MI_PAGE_MAP_FLAT-#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && MI_SECURE==0 && !MI_PAGE_META_IS_SEPARATED+#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && !MI_SECURE && !MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP#define MI_PAGE_MAP_FLAT 1#else#define MI_PAGE_MAP_FLAT 0@@ -177,12 +195,16 @@ typedef int32_t mi_ssize_t;-------------------------------------------------------------------------------- */size_t _mi_popcount_generic(size_t x);+extern bool _mi_cpu_has_popcnt;static inline size_t mi_popcount(size_t x) {#if mi_has_builtinz(popcount)return mi_builtinz(popcount)(x);- #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)- return mi_msc_builtinz(__popcnt)(x);+ #elif defined(_MSC_VER) && (MI_ARCH_ARM64 || MI_ARCH_ARM32)+ return mi_msc_builtinz(_CountOneBits)(x);+ #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86)+ if (_mi_cpu_has_popcnt) { return mi_msc_builtinz(__popcnt)(x); }+ else { return _mi_popcount_generic(x); } // see issue #1291#elif MI_ARCH_X64 && defined(__BMI1__)return (size_t)_mm_popcnt_u64(x);#elsediff --git a/vendored/mimalloc/include/mimalloc/internal.h b/vendored/mimalloc/include/mimalloc/internal.hindex febf2559e..b79649c72 100644--- a/vendored/mimalloc/include/mimalloc/internal.h+++ b/vendored/mimalloc/include/mimalloc/internal.h@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -40,6 +40,7 @@ terms of the MIT license. A copy of the license can be found in the file#define mi_decl_weak#define mi_decl_hidden#define mi_decl_cold+#define mi_assume_aligned(p,sz) (p)#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc#if !MI_TRACK_ASAN#define mi_decl_forceinline __attribute__((always_inline)) inline@@ -50,11 +51,17 @@ terms of the MIT license. A copy of the license can be found in the file#define mi_decl_thread __thread#define mi_decl_noreturn __attribute__((noreturn))#define mi_decl_weak __attribute__((weak))+#if defined(__MINGW32__) || defined(__CYGWIN__)+#define mi_decl_hidden+#else#define mi_decl_hidden __attribute__((visibility("hidden")))-#if (__GNUC__ >= 4) || defined(__clang__)+#endif+#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)#define mi_decl_cold __attribute__((cold))+#define mi_assume_aligned(p,sz) __builtin_assume_aligned(p,sz)#else#define mi_decl_cold+#define mi_assume_aligned(p,sz) (p)#endif#elif __cplusplus >= 201103L // c++11#define mi_decl_forceinline inline@@ -64,6 +71,7 @@ terms of the MIT license. A copy of the license can be found in the file#define mi_decl_weak#define mi_decl_hidden#define mi_decl_cold+#define mi_assume_aligned(p,sz) (p)#else#define mi_decl_forceinline inline#define mi_decl_noinline@@ -72,6 +80,7 @@ terms of the MIT license. A copy of the license can be found in the file#define mi_decl_weak#define mi_decl_hidden#define mi_decl_cold+#define mi_assume_aligned(p,sz) (p)#endif#if defined(__GNUC__) || defined(__clang__)@@ -85,17 +94,7 @@ terms of the MIT license. A copy of the license can be found in the file#define mi_likely(x) (x)#endif-#ifndef __has_builtin-#define __has_builtin(x) 0-#endif--#if defined(__cplusplus)-#define mi_decl_externc extern "C"-#else-#define mi_decl_externc-#endif--#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) // includes clang and icc+#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__)#define mi_decl_maybe_unused __attribute__((unused))#elif __cplusplus >= 201703L // c++17#define mi_decl_maybe_unused [[maybe_unused]]@@ -103,13 +102,10 @@ terms of the MIT license. A copy of the license can be found in the file#define mi_decl_maybe_unused#endif-#if defined(__cplusplus)-#define mi_decl_externc extern "C"-#else-#define mi_decl_externc+#ifndef __has_builtin+#define __has_builtin(x) 0#endif-#if defined(__EMSCRIPTEN__) && !defined(__wasi__)#define __wasi__#endif@@ -126,19 +122,19 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_listint _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...);char _mi_toupper(char c);int _mi_strnicmp(const char* s, const char* t, size_t n);-void _mi_strlcpy(char* dest, const char* src, size_t dest_size);-void _mi_strlcat(char* dest, const char* src, size_t dest_size);+bool _mi_strlcpy(char* dest, const char* src, size_t dest_size); // returns true if the entire src was copied+bool _mi_strlcat(char* dest, const char* src, size_t dest_size); // returns true if the entire src was appendedsize_t _mi_strlen(const char* s);size_t _mi_strnlen(const char* s, size_t max_len);char* _mi_strnstr(char* s, size_t max_len, const char* pat);bool _mi_streq(const char* s, const char* t);-bool _mi_getenv(const char* name, char* result, size_t result_size);+int _mi_getenv(const char* name, char* result, size_t result_size);+void _mi_detect_cpu_features(void);// "options.c"void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);void _mi_raw_message(const char* fmt, ...);-void _mi_message(const char* fmt, ...);void _mi_warning_message(const char* fmt, ...);void _mi_verbose_message(const char* fmt, ...);void _mi_trace_message(const char* fmt, ...);@@ -152,79 +148,93 @@ void _mi_random_init(mi_random_ctx_t* ctx);void _mi_random_init_weak(mi_random_ctx_t* ctx);void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx);void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx);-uintptr_t _mi_random_next(mi_random_ctx_t* ctx);-uintptr_t _mi_theap_random_next(mi_theap_t* theap);-uintptr_t _mi_os_random_weak(uintptr_t extra_seed);-static inline uintptr_t _mi_random_shuffle(uintptr_t x);+size_t _mi_random_next(mi_random_ctx_t* ctx);+size_t _mi_theap_random_next(mi_theap_t* theap);+size_t _mi_os_random_weak(size_t extra_seed);+static inline size_t _mi_random_shuffle(size_t x);++// prim-tls.c+void _mi_tls_slots_init(void);+void _mi_tls_slots_done(void);+mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;+void _mi_theap_default_set(mi_theap_t* theap);+void _mi_theap_cached_set(mi_theap_t* theap);++// subproc.c+mi_subproc_t* _mi_subproc_main_init(void);+void _mi_subproc_main_done(void);+mi_subproc_t* _mi_subproc_main(void);+bool _mi_subproc_is_main(mi_subproc_t* subproc);+mi_subproc_t* _mi_subproc(void); // current subproc of this thread+mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc);+mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id);+void _mi_subprocs_unsafe_destroy_all(void);++void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid );+void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* p, size_t newsize, mi_memid_t* memid );+void* _mi_meta_zalloc_aligned( mi_subproc_t* subproc, size_t size, size_t alignment, mi_memid_t* memid );+void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid);+bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* p);+// init.c-extern mi_decl_hidden mi_decl_cache_align const mi_page_t _mi_page_empty;+mi_page_t* _mi_page_empty_get(void);void _mi_auto_process_init(void);void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept;bool _mi_is_redirected(void);bool _mi_allocator_init(const char** message);void _mi_allocator_done(void);-bool _mi_is_main_thread(void);bool _mi_preloading(void); // true while the C runtime is not initialized yetvoid _mi_thread_done(mi_theap_t* theap);+mi_theap_t* _mi_thread_init(void);+mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap);+bool _mi_is_empty_theap(const mi_theap_t* theap);-mi_subproc_t* _mi_subproc(void);-mi_subproc_t* _mi_subproc_main(void);-mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc);-mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id);--mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;-size_t _mi_thread_seq_id(void) mi_attr_noexcept;-bool _mi_is_heap_main(const mi_heap_t* heap);-bool _mi_is_theap_main(const mi_theap_t* theap);-void _mi_theap_guarded_init(mi_theap_t* theap);-void _mi_theap_options_init(mi_theap_t* theap);-mi_theap_t* _mi_theap_default_safe(void); // ensure the returned theap is initialized-mi_theap_t* _mi_theap_main_safe(void);-// os.cvoid _mi_os_init(void); // called from process init-void* _mi_os_alloc(size_t size, mi_memid_t* memid);-void* _mi_os_zalloc(size_t size, mi_memid_t* memid);-void _mi_os_free(void* p, size_t size, mi_memid_t memid);-void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc );+void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid);+void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid);+void _mi_os_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid);+void _mi_os_free_ex(mi_subproc_t* subproc, void* p, size_t size, bool still_committed, mi_memid_t memid );size_t _mi_os_page_size(void);size_t _mi_os_guard_page_size(void);size_t _mi_os_good_alloc_size(size_t size);bool _mi_os_has_overcommit(void);bool _mi_os_has_virtual_reserve(void);+bool _mi_os_canuse_thp(void);size_t _mi_os_virtual_address_bits(void);size_t _mi_os_minimal_purge_size(void);-bool _mi_os_reset(void* addr, size_t size);-bool _mi_os_decommit(void* addr, size_t size);-void _mi_os_reuse(void* p, size_t size);-mi_decl_nodiscard bool _mi_os_commit(void* p, size_t size, bool* is_zero);-mi_decl_nodiscard bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size);+bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size);+bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size);+void _mi_os_reuse(mi_subproc_t* subproc, void* p, size_t size);+mi_decl_nodiscard bool _mi_os_commit(mi_subproc_t* subproc, void* p, size_t size, bool* is_zero);+mi_decl_nodiscard bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed);mi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size);bool _mi_os_unprotect(void* addr, size_t size);-bool _mi_os_purge(void* p, size_t size);-bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stats_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg);+bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size);+bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stats_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg);size_t _mi_os_secure_guard_page_size(void);-bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid);-bool _mi_os_secure_guard_page_set_before(void* addr, mi_memid_t memid);-bool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid);-bool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid);+bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid);+bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid);+bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid);+bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid);int _mi_os_numa_node(void);int _mi_os_numa_node_count(void);-void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid);-void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid);+void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid);+void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid);void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size);bool _mi_os_canuse_large_page(size_t size, size_t alignment);size_t _mi_os_large_page_size(void);-void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid);+void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid);// threadlocal.c+#define mi_thread_local_key_fast ((mi_thread_local_t)1)mi_thread_local_t _mi_thread_local_create(void);void _mi_thread_local_free( mi_thread_local_t key );@@ -241,8 +251,7 @@ bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid);void* _mi_arenas_alloc_aligned(mi_heap_t* heap, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid);-void _mi_arenas_free(void* p, size_t size, mi_memid_t memid);-bool _mi_arenas_contain(const void* p);+void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid);void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld);void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc);@@ -252,59 +261,57 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapvoid _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */);bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page);-// arena-meta.c-void* _mi_meta_zalloc( size_t size, mi_memid_t* memid );-void _mi_meta_free(void* p, size_t size, mi_memid_t memid);-bool _mi_meta_is_meta_page(void* p);-// "page-map.c"bool _mi_page_map_init(void);mi_decl_nodiscard bool _mi_page_map_register(mi_page_t* page);void _mi_page_map_unregister(mi_page_t* page);void _mi_page_map_unregister_range(void* start, size_t size);mi_page_t* _mi_safe_ptr_page(const void* p);-void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc);+void _mi_page_map_unsafe_destroy(void);// "page.c"-void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable) mi_attr_noexcept mi_attr_malloc;+void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc;+void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc;void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocksvoid _mi_page_unfull(mi_page_t* page);void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq); // free the pagevoid _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread...--size_t _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append);void _mi_deferred_free(mi_theap_t* theap, bool force);-void _mi_page_free_collect(mi_page_t* page, bool force);-void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head);+mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head);mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page);bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq);+void _mi_page_update_stats(mi_page_t* page);size_t _mi_page_stats_bin(const mi_page_t* page); // for statssize_t _mi_bin_size(size_t bin); // for statssize_t _mi_bin(size_t size); // for stats// "theap.c"+void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld);+mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld);mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld);-void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock);-void _mi_theap_default_set(mi_theap_t* theap);-void _mi_theap_cached_set(mi_theap_t* theap);void _mi_theap_collect_retired(mi_theap_t* theap, bool force);void _mi_theap_collect_abandon(mi_theap_t* theap);bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg);void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page);-bool _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock);++void _mi_heap_detach_theaps( mi_heap_t* heap );+void _mi_tld_detach_theaps( mi_tld_t* tld );void _mi_theap_incref(mi_theap_t* theap);void _mi_theap_decref(mi_theap_t* theap);+void _mi_theap_merge_stats(mi_theap_t* theap);// "heap.c"+void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id);void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page);mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap); // get (and possible create) the theap belonging to a heap-mi_decl_cold mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap); // get the theap for a heap without initializing (and return NULL in that case)void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to); // in "arena.c"void _mi_heap_destroy_pages(mi_heap_t* heap_from); // in "arena.c"-void _mi_heap_force_destroy(mi_heap_t* heap); // allow destroying the main heap+void _mi_heap_force_destroy(mi_heap_t* heap, bool acquire_heaps_lock); // allow destroying the main heap+mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_heap_main);+bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap);// "stats.c"void _mi_stats_init(void);@@ -316,16 +323,35 @@ mi_msecs_t _mi_clock_start(void);// "alloc.c"void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned`-void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept;-void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned`-void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept;+void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned`+void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept;mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p);void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size);++// "free.c"+void _mi_free_subproc_safe(void* p) mi_attr_noexcept;+void _mi_page_unguard_all(mi_page_t* page);+size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept;+#if MI_DEBUG>1bool _mi_page_is_valid(mi_page_t* page);#endif+// "sample-guarded.c"+mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept;+void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p);+void _mi_page_unguard_all(mi_page_t* page);+void _mi_theap_guarded_init(mi_theap_t* theap);++// "sample-profile.c"+mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept;+size_t _mi_theap_update_sample_rate(mi_theap_t* theap);++mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept;+void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p);+size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate);+// ------------------------------------------------------// Assertions@@ -357,20 +383,32 @@ mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const----------------------------------------------------------- */// add to stat keeping track of the peak-void __mi_stat_increase(mi_stat_count_t* stat, size_t amount);-void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount);-void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount);-void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount);+void __mi_stat_increase(mi_stat_count_t* stat, uint64_t amount);+void __mi_stat_decrease(mi_stat_count_t* stat, uint64_t amount);+void __mi_stat_increase_mt(mi_stat_count_t* stat, uint64_t amount);+void __mi_stat_decrease_mt(mi_stat_count_t* stat, uint64_t amount);// adjust stat in special cases to compensate for double counting (and does not adjust peak values and can decrease the total)-void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount);-void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount);-void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount);-void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount);+void __mi_stat_adjust_increase(mi_stat_count_t* stat, uint64_t amount);+void __mi_stat_adjust_decrease(mi_stat_count_t* stat, uint64_t amount);+void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, uint64_t amount);+void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, uint64_t amount);// counters can just be increased-void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);-void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount);+static inline void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, uint64_t amount) {+ mi_assert_internal(amount<=INT64_MAX);+ mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount);+}++static inline void __mi_stat_counter_increase(mi_stat_counter_t* stat, uint64_t amount) {+ mi_assert_internal(amount<=INT64_MAX);+ stat->total += (int64_t)amount;+}++static inline void __mi_stat_counter_decrease(mi_stat_counter_t* stat, uint64_t amount) {+ mi_assert_internal(amount<=INT64_MAX);+ stat->total -= (int64_t)amount;+}#define mi_heap_stat_counter_increase(heap,stat,amount) __mi_stat_counter_increase_mt( &(heap)->stats.stat, amount)#define mi_heap_stat_increase(heap,stat,amount) __mi_stat_increase_mt( &(heap)->stats.stat, amount)@@ -384,16 +422,60 @@ void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount);#define mi_subproc_stat_adjust_increase(subproc,stat,amount) __mi_stat_adjust_increase_mt( &(subproc)->stats.stat, amount)#define mi_subproc_stat_adjust_decrease(subproc,stat,amount) __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amount)-#define mi_os_stat_counter_increase(stat,amount) mi_subproc_stat_counter_increase(_mi_subproc(),stat,amount)-#define mi_os_stat_increase(stat,amount) mi_subproc_stat_increase(_mi_subproc(),stat,amount)-#define mi_os_stat_decrease(stat,amount) mi_subproc_stat_decrease(_mi_subproc(),stat,amount)-#define mi_theap_stat_counter_increase(theap,stat,amount) __mi_stat_counter_increase( &(theap)->stats.stat, amount)+#define mi_theap_stat_counter_decrease(theap,stat,amount) __mi_stat_counter_decrease( &(theap)->stats.stat, amount)#define mi_theap_stat_increase(theap,stat,amount) __mi_stat_increase( &(theap)->stats.stat, amount)#define mi_theap_stat_decrease(theap,stat,amount) __mi_stat_decrease( &(theap)->stats.stat, amount)#define mi_theap_stat_adjust_increase(theap,stat,amnt) __mi_stat_adjust_increase( &(theap)->stats.stat, amnt)#define mi_theap_stat_adjust_decrease(theap,stat,amnt) __mi_stat_adjust_decrease( &(theap)->stats.stat, amnt)+#define mi_theapx_stat_counter_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_counter_increase(theap,stat,amount); } else { mi_heap_stat_counter_increase(heap,stat,amount); }+#define mi_theapx_stat_adjust_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_adjust_decrease(theap,stat,amount); } else { mi_heap_stat_adjust_decrease(heap,stat,amount); }+#define mi_theapx_stat_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_increase(theap,stat,amount); } else { mi_heap_stat_increase(heap,stat,amount); }+#define mi_theapx_stat_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_decrease(theap,stat,amount); } else { mi_heap_stat_decrease(heap,stat,amount); }+++/* -----------------------------------------------------------+ pthread thread locals+----------------------------------------------------------- */++#if MI_USE_PTHREADS++#if defined(__APPLE__) && defined(__aarch64__)+#define MI_PTHREAD_KEY_INVALID ((pthread_key_t)(0)) // nicer codegen+#else+#define MI_PTHREAD_KEY_INVALID ((pthread_key_t)(-1))+#endif++#if defined(__linux__) && defined(__GLIBC__)+// pthread_getspecific returns NULL for invalid keys. <https://man7.org/linux/man-pages/man3/pthread_getspecific.3p.html>+// see also: <https://github.com/lattera/glibc/blob/master/nptl/pthread_getspecific.c>+#define MI_PTHREADS_GET_INVALID_KEY_IS_NULL 1+#endif++mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruct)(void*), void* init);++static inline void* mi_pthread_key_get(pthread_key_t key) {+ #if !MI_PTHREADS_GET_INVALID_KEY_IS_NULL+ if mi_unlikely(key==MI_PTHREAD_KEY_INVALID) return NULL;+ #endif+ return pthread_getspecific(key);+}++static inline bool mi_pthread_key_set(pthread_key_t* pkey, void* val) {+ if mi_likely(*pkey!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(*pkey,val); return true; }+ else if (val!=NULL) { return _mi_pthread_key_create(pkey,NULL,val); }+ else return true;+}++static inline void mi_pthread_key_delete(pthread_key_t* pkey) {+ const pthread_key_t key = *pkey;+ if (key!=MI_PTHREAD_KEY_INVALID) {+ *pkey = MI_PTHREAD_KEY_INVALID;+ pthread_key_delete(key);+ }+}+#endif/* -----------------------------------------------------------Options (exposed for the debugger)@@ -440,11 +522,22 @@ typedef struct mi_option_desc_s {// initialize a local variable to zero; use memset as compilers optimize constant sized memset's#define _mi_memzero_var(x) memset(&x,0,sizeof(x))+// minimum+static inline size_t mi_min(size_t x, size_t y) { return (x <= y ? x : y); }++// maximum+static inline size_t mi_max(size_t x, size_t y) { return (x >= y ? x : y); }+// Is `x` a power of two? (0 is considered a power of two)static inline bool _mi_is_power_of_two(uintptr_t x) {return ((x & (x - 1)) == 0);}+// valid alignment values are as posix memalign: <https://en.cppreference.com/c/memory/aligned_alloc#Notes>+static inline bool mi_alignment_is_valid(size_t alignment) {+ return ((alignment!=0) && _mi_is_power_of_two(alignment));+}+// Is a pointer aligned?static inline bool _mi_is_aligned(const void* p, size_t alignment) {return (alignment==0 || ((uintptr_t)p % alignment) == 0);@@ -453,7 +546,7 @@ static inline bool _mi_is_aligned(const void* p, size_t alignment) {// Align upwardsstatic inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {mi_assert_internal(alignment != 0);- uintptr_t mask = alignment - 1;+ const uintptr_t mask = alignment - 1;if ((alignment & mask) == 0) { // power of two?return ((sz + mask) & ~mask);}@@ -467,18 +560,19 @@ static inline void* _mi_align_up_ptr(const void* p, size_t alignment) {return (void*)_mi_align_up((uintptr_t)p, alignment);}+// Align downstatic inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {mi_assert_internal(alignment != 0);- uintptr_t mask = alignment - 1;- if ((alignment & mask) == 0) { // power of two?+ const uintptr_t mask = alignment - 1;+ if ((alignment & mask) == 0) { // power of two?return (sz & ~mask);}else {- return ((sz / alignment) * alignment);+ return ((sz/alignment)*alignment);}}-// align a pointer downwards+// Align a pointer downwardsstatic inline void* _mi_align_down_ptr(const void* p, size_t alignment) {return (void*)_mi_align_down((uintptr_t)p, alignment);}@@ -505,13 +599,6 @@ static inline bool mi_mem_is_zero(const void* p, size_t size) {return true;}-// Align a byte size to a size in _machine words_,-// i.e. byte size == `wsize*sizeof(void*)`.-static inline size_t _mi_wsize_from_size(size_t size) {- mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t));- return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t);-}-// Overflow detecting multiply#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5))#include <limits.h> // UINT_MAX, ULONG_MAX@@ -519,10 +606,10 @@ static inline size_t _mi_wsize_from_size(size_t size) {#undef _CLOCK_T#endifstatic inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {- #if (SIZE_MAX == ULONG_MAX)- return __builtin_umull_overflow(count, size, (unsigned long *)total);- #elif (SIZE_MAX == UINT_MAX)+ #if (SIZE_MAX == UINT_MAX)return __builtin_umul_overflow(count, size, (unsigned int *)total);+ #elif (SIZE_MAX == ULONG_MAX)+ return __builtin_umull_overflow(count, size, (unsigned long *)total);#elsereturn __builtin_umulll_overflow(count, size, (unsigned long long *)total);#endif@@ -562,32 +649,46 @@ static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* totHeap functions------------------------------------------------------------------------------------------- */-extern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_THREAD_LOCAL)-extern mi_decl_hidden const mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated+extern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_LOCAL)+extern mi_decl_hidden mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated++static inline mi_heap_t* _mi_theap_heap_peek(const mi_theap_t* theap) {+ mi_assert_internal(theap!=NULL);+ return mi_atomic_load_ptr_relaxed(mi_heap_t,&theap->heap);+}static inline mi_heap_t* _mi_theap_heap(const mi_theap_t* theap) {- return mi_atomic_load_ptr_acquire(mi_heap_t,&theap->heap);+ mi_heap_t* const heap = _mi_theap_heap_peek(theap);+ mi_assert_internal(heap!=NULL);+ return heap;}static inline bool mi_theap_is_initialized(const mi_theap_t* theap) {- return (theap != NULL && _mi_theap_heap(theap) != NULL);+ return (theap != NULL && _mi_theap_heap_peek(theap) != NULL);+}++static inline mi_subproc_t* _mi_theap_subproc(const mi_theap_t* theap) {+ mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc);+ mi_assert_internal(!mi_theap_is_initialized(theap) || _mi_theap_heap(theap)->subproc == subproc);+ return subproc;}-static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t size) {- mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE));- const size_t idx = _mi_wsize_from_size(size);+static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t xsize, bool is_wsize) {+ mi_assert_internal(is_wsize ? xsize <= (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE) : xsize <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE));+ const size_t idx = (is_wsize ? xsize : mi_wsize_from_size(xsize));mi_assert_internal(idx < MI_PAGES_DIRECT);return theap->pages_free_direct[idx];}+static inline bool mi_theap_is_detached(mi_theap_t* theap) {+ return (theap!=NULL && theap->tld->thread_id == MI_THREADID_DETACHED);+}-//static inline uintptr_t _mi_ptr_cookie(const void* p) {-// extern mi_theap_t _mi_theap_main;-// mi_assert_internal(_mi_theap_main.cookie != 0);-// return ((uintptr_t)p ^ _mi_theap_main.cookie);-//}-+static inline bool mi_theap_matches_thread(mi_theap_t* theap) {+ const mi_threadid_t tid = _mi_thread_id();+ return (theap==NULL || theap->tld==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap));+}/* -----------------------------------------------------------The page map maps addresses to `mi_page_t` pointers@@ -598,20 +699,30 @@ static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t// flat page-map committed on demand, using one byte per slice (64 KiB).// single indirection and low commit, but large initial virtual reserve (4 GiB with 48 bit virtual addresses)// used by default on <= 40 bit virtual address spaces.-extern mi_decl_hidden uint8_t* _mi_page_map;+extern mi_decl_hidden _Atomic(uint8_t*) _mi_page_map;+extern mi_decl_hidden _Atomic(void*) _mi_page_map_max_address;static inline size_t _mi_page_map_index(const void* p) {return (size_t)((uintptr_t)p >> MI_ARENA_SLICE_SHIFT);}+static inline uint8_t _mi_page_map_at(size_t idx) {+ return mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map)[idx];+}+static inline mi_page_t* _mi_ptr_page_ex(const void* p, bool* valid) {const size_t idx = _mi_page_map_index(p);- const size_t ofs = _mi_page_map[idx];+ const size_t ofs = _mi_page_map_at(idx);if (valid != NULL) { *valid = (ofs != 0); }return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) + 1 - ofs) << MI_ARENA_SLICE_SHIFT);}static inline mi_page_t* _mi_checked_ptr_page(const void* p) {+ #if MI_MIN_VABITS < MI_INTPTR_BITS+ if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) {+ if (p > mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL;+ }+ #endifbool valid;mi_page_t* const page = _mi_ptr_page_ex(p, &valid);return (valid ? page : NULL);@@ -633,10 +744,17 @@ static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) {#define MI_PAGE_MAP_SUB_SHIFT (13)#define MI_PAGE_MAP_SUB_COUNT (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT)#define MI_PAGE_MAP_SHIFT (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)-#define MI_PAGE_MAP_COUNT (MI_ZU(1) << MI_PAGE_MAP_SHIFT)typedef mi_page_t** mi_submap_t;-extern mi_decl_hidden _Atomic(mi_submap_t)* _mi_page_map;+typedef struct mi_page_map_s {+ _Atomic(size_t) committed_count; // currently committed entries+ size_t reserved_size; // full reserved size (mi_page_map_t + submaps)+ mi_memid_t memid; // provenance+ mi_lock_t lock; // used when allocating new submaps+ _Atomic(mi_submap_t) submaps[1];+} mi_page_map_t;++extern mi_decl_hidden _Atomic(mi_page_map_t*) __mi_page_map;static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) {const size_t u = (size_t)((uintptr_t)p / MI_ARENA_SLICE_SIZE);@@ -644,33 +762,81 @@ static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) {return (u / MI_PAGE_MAP_SUB_COUNT);}-static inline mi_submap_t _mi_page_map_at(size_t idx) {- return mi_atomic_load_ptr_relaxed(mi_page_t*, &_mi_page_map[idx]);+static inline mi_page_map_t* _mi_page_map(void) {+ return mi_atomic_load_ptr_relaxed(mi_page_map_t,&__mi_page_map);+}++static inline mi_submap_t _mi_page_map_at(const mi_page_map_t* pmap, size_t idx) {+ return mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]);}static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) {+ const mi_page_map_t* pmap = _mi_page_map();size_t sub_idx;const size_t idx = _mi_page_map_index(p, &sub_idx);- return (_mi_page_map_at(idx))[sub_idx]; // NULL if p==NULL+ return _mi_page_map_at(pmap,idx)[sub_idx]; // NULL if p==NULL}static inline mi_page_t* _mi_checked_ptr_page(const void* p) {+ const mi_page_map_t* pmap = _mi_page_map();size_t sub_idx;const size_t idx = _mi_page_map_index(p, &sub_idx);- mi_submap_t const sub = _mi_page_map_at(idx);+ const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count);+ if mi_unlikely(idx >= committed_count) return NULL;+ // #if MI_MIN_VABITS < MI_INTPTR_BITS // is still invalid if free is called before the pagemap is initialized+ // if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) {+ // const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count);+ // if mi_unlikely(idx >= committed_count) return NULL;+ // }+ // #endif+ mi_submap_t const sub = _mi_page_map_at(pmap,idx);if mi_unlikely(sub == NULL) return NULL;return sub[sub_idx];}#endif+#if MI_PAGE_META_IS_ALIGNED+// if the page meta data is aligned in front of pages we can find it efficiently+// without needing to go through the page map (for valid pointers).+static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) {+ mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT);+ // const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE;+ const uintptr_t page_idx = ((uintptr_t)p / MI_ARENA_SLICE_SIZE) % (MI_PAGE_META_ALIGNMENT / MI_ARENA_SLICE_SIZE);+ mi_assert_internal(page_idx <= MI_PAGE_META_ALIGNED_COUNT);+ #if MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_RISCV // better code on x64/x86/riscv64+ mi_page_t* const page = (mi_page_t*)((uintptr_t)page_metas | (page_idx * sizeof(mi_page_t)));+ #else+ mi_page_t* const page = &page_metas[page_idx];+ #endif+ return page;++}++static inline mi_page_t* _mi_aligned_ptr_page(const void* p) {+ mi_page_t* const page = _mi_aligned_ptr_page0(p);+ if mi_unlikely(page==NULL) return NULL;+ #if MI_DEBUG+ mi_page_t* const cpage = _mi_checked_ptr_page(p);+ if mi_unlikely(cpage==NULL) {+ _mi_error_message(EINVAL, "_mi_aligned_ptr_page: invalid pointer: %p\n", p);+ return NULL;+ }+ #endif+ return mi_atomic_load_ptr_acquire(mi_page_t, &page->self);+}+#endifstatic inline mi_page_t* _mi_ptr_page(const void* p) {mi_assert_internal(p==NULL || mi_is_in_heap_region(p));- #if MI_DEBUG || MI_SECURE || defined(__APPLE__)- return _mi_checked_ptr_page(p);- #else- return _mi_unchecked_ptr_page(p);+ #if MI_SECURE || MI_FREE_IS_CHECKED+ return _mi_checked_ptr_page(p);+ #elif MI_PAGE_META_IS_ALIGNED+ return _mi_aligned_ptr_page(p);+ #elif MI_DEBUG+ return _mi_checked_ptr_page(p);+ #else+ return _mi_unchecked_ptr_page(p);#endif}@@ -683,7 +849,8 @@ static inline size_t mi_page_block_size(const mi_page_t* page) {// Page startstatic inline uint8_t* mi_page_start(const mi_page_t* page) {- return page->page_start;+ // multiplication must be done in `size_t`+ return (uint8_t*)page + page->page_offset;}static inline size_t mi_page_size(const mi_page_t* page) {@@ -720,19 +887,27 @@ static inline size_t mi_page_usable_block_size(const mi_page_t* page) {}static inline bool mi_page_meta_is_separated(const mi_page_t* page) {- #if MI_PAGE_META_IS_SEPARATED- // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_ALIGNED_FREE_SMALL- return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(page->page_start, MI_ARENA_SLICE_ALIGN));+ #if MI_PAGE_META_IS_ALIGNED+ #if MI_PAGE_META_SMALL_IS_ALIGNED+ return (page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN));+ #else+ MI_UNUSED_RELEASE(page);+ mi_assert_internal(page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN));+ return true;+ #endif+ #elif MI_PAGE_META_IS_SEPARATED+ // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_SMALL_IS_ALIGNED+ return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN));#elseMI_UNUSED(page);- return false;+ return false;#endif}static inline uint8_t* mi_page_slice_start(const mi_page_t* page) {- if (mi_page_meta_is_separated(page)) {+ if (mi_page_meta_is_separated(page)) {// page meta info is at a separate location (at `arena->pages`)- return (uint8_t*)_mi_align_down_ptr(page->page_start, MI_ARENA_SLICE_ALIGN);+ return (uint8_t*)_mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN);}else {// page meta info is at the start of the page slices@@ -740,21 +915,57 @@ static inline uint8_t* mi_page_slice_start(const mi_page_t* page) {}}-// This gives the offset relative to the start slice of a page.+// This gives the offset relative to the start slice of a page.static inline size_t mi_page_slice_offset_of(const mi_page_t* page, size_t offset_relative_to_page_start) {- return (page->page_start - mi_page_slice_start(page)) + offset_relative_to_page_start;+ return (mi_page_start(page) - mi_page_slice_start(page)) + offset_relative_to_page_start;+}++// How much of the page is committed relative to the slice start? (or 0 if fully committed already)+static inline size_t mi_page_slice_committed(const mi_page_t* page) {+ return ((size_t)page->slice_pcommitted * _mi_os_page_size());}// Currently committed part of a pagestatic inline size_t mi_page_committed(const mi_page_t* page) {- return (page->slice_committed == 0 ? mi_page_size(page) : page->slice_committed - mi_page_slice_offset_of(page,0));+ const size_t slice_committed = mi_page_slice_committed(page);+ return (slice_committed == 0 ? mi_page_size(page) : slice_committed - mi_page_slice_offset_of(page,0));+}++static inline size_t mi_page_used(const mi_page_t* page) {+ mi_assert_internal(page != NULL);+ return mi_xused_used_count(page->xused);+}++static inline void mi_page_used_reset(mi_page_t* page) {+ page->xused = mi_xused_used_reset(page->xused);+}++static inline size_t mi_page_alloc_count(const mi_page_t* page) {+ return mi_xused_alloc_count(page->xused);+}++static inline size_t mi_page_last_used(const mi_page_t* page) {+ #if MI_SIZE_SIZE >= 8+ return (page->xused.used_alloc >> 32) & 0xFFFF;+ #else+ return page->xlast_used;+ #endif}+static inline size_t mi_page_last_alloc(const mi_page_t* page) {+ #if MI_SIZE_SIZE >= 8+ return (page->xused.used_alloc >> 48) & 0xFFFF;+ #else+ return page->xlast_alloc;+ #endif+}++// are all blocks in a page freed?// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`.static inline bool mi_page_all_free(const mi_page_t* page) {mi_assert_internal(page != NULL);- return (page->used == 0);+ return (mi_page_used(page)==0);}// are there immediately available blocks, i.e. blocks available on the free list.@@ -773,7 +984,7 @@ static inline bool mi_page_is_expandable(const mi_page_t* page) {static inline bool mi_page_is_full(const mi_page_t* page) {- const bool full = (page->reserved == page->used);+ const bool full = (page->reserved == mi_page_used(page));mi_assert_internal(!full || page->free == NULL);return full;}@@ -782,14 +993,14 @@ static inline bool mi_page_is_full(const mi_page_t* page) {static inline bool mi_page_is_mostly_used(const mi_page_t* page) {if (page==NULL) return true;uint16_t frac = page->reserved / 8U;- return (page->reserved - page->used <= frac);+ return (page->reserved - mi_page_used(page) <= frac);}// is more than (n-1)/n'th of a page in use?static inline bool mi_page_is_used_at_frac(const mi_page_t* page, uint16_t n) {if (page==NULL) return true;uint16_t frac = page->reserved / n;- return (page->reserved - page->used <= frac);+ return (page->reserved - mi_page_used(page) <= frac);}@@ -805,6 +1016,10 @@ static inline mi_page_queue_t* mi_page_queue(const mi_theap_t* theap, size_t sizreturn pq;}+static inline size_t mi_page_min_commit_size(void) {+ const size_t psize = _mi_os_page_size();+ return (MI_PAGE_MIN_COMMIT_SIZE >= psize ? MI_PAGE_MIN_COMMIT_SIZE : psize);+}//-----------------------------------------------------------// Page thread id and flags@@ -842,7 +1057,8 @@ static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) {mi_theap_t* const theap = page->theap;mi_assert_internal(theap!=NULL);if (theap != NULL) {- const size_t size = page->capacity * mi_page_block_size(page);+ mi_assert_internal(page->capacity==page->reserved);+ const size_t size = page->reserved * mi_page_block_size(page);if (in_full) { theap->pages_full_size += size; }else { mi_assert_internal(size <= theap->pages_full_size); theap->pages_full_size -= size; }}@@ -893,7 +1109,7 @@ static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) {static inline mi_theap_t* mi_page_theap(const mi_page_t* page) {mi_assert_internal(!mi_page_is_abandoned(page));- mi_assert_internal(page->theap != NULL);+ mi_assert_internal(page->theap != NULL && page->theap != &_mi_theap_empty);return page->theap;}@@ -906,25 +1122,53 @@ static inline mi_tld_t* mi_page_tld(const mi_page_t* page) {static inline mi_heap_t* mi_page_heap(const mi_page_t* page) {mi_heap_t* heap = page->heap;- // we use NULL for the main heap to make `_mi_page_get_associated_theap` fast in `free.c:mi_abandoned_page_try_reclaim`.- if mi_likely(heap==NULL) heap = mi_heap_main();mi_assert_internal(heap != NULL);return heap;}+static inline mi_subproc_t* mi_page_subproc(const mi_page_t* page) {+ mi_heap_t* const heap = mi_page_heap(page);+ return heap->subproc;+}++static inline mi_heap_t* mi_arena_heap_main(const mi_arena_t* arena) {+ return _mi_subproc_heap_main(arena->subproc);+}++static inline mi_heap_t* mi_heap_get_heap_main(const mi_heap_t* heap) {+ return _mi_subproc_heap_main(heap->subproc);+}++static inline bool _mi_is_heap_main(const mi_heap_t* heap) {+ mi_assert_internal(heap!=NULL);+ return (mi_heap_get_heap_main(heap) == heap);+}++static inline bool _mi_is_process_heap_main(const mi_heap_t* heap) {+ mi_assert_internal(heap!=NULL);+ return (_mi_subproc_main()->heap_main == heap);+}+//-----------------------------------------------------------// Thread free list and ownership//-----------------------------------------------------------// Thread free flag helpersstatic inline mi_block_t* mi_tf_block(mi_thread_free_t tf) {- return (mi_block_t*)(tf & ~1);+ return (mi_block_t*)(tf & ~1);}+static inline bool mi_tf_is_owned(mi_thread_free_t tf) {return ((tf & 1) == 1);}+static inline mi_thread_free_t mi_tf_create(mi_block_t* block, bool owned) {- return (mi_thread_free_t)((uintptr_t)block | (owned ? 1 : 0));+ const uintptr_t base = (uintptr_t)block | (owned ? 1 : 0);+ return (mi_thread_free_t)base;+}++static inline mi_thread_free_t mi_tf_set_owned(mi_thread_free_t tf, bool owned) {+ return mi_tf_create(mi_tf_block(tf), owned);}// Thread free access@@ -935,7 +1179,7 @@ static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) {// are there any available blocks?static inline bool mi_page_has_any_available(const mi_page_t* page) {mi_assert_internal(page != NULL && page->reserved > 0);- return (page->used < page->reserved || (mi_page_thread_free(page) != NULL));+ return (mi_page_used(page) < page->reserved || (mi_page_thread_free(page) != NULL));}// Owned?@@ -951,14 +1195,26 @@ static inline bool mi_page_claim_ownership(mi_page_t* page) {/* -------------------------------------------------------------------- Guarded objects+ Guarded and profiled objects------------------------------------------------------------------- */-#if MI_GUARDED++#define MI_SAMPLE_RATE_MAX (SIZE_MAX/4)+#define MI_SAMPLE_COUNTDOWN_MAX (MI_MAX_ALLOC_SIZE)++static inline bool mi_theap_should_sample(mi_theap_t* theap, size_t req_size) {+ // note: this should return `true` on an empty theap so we initialize it's countdown to `-1`.+ mi_assert_internal(req_size <= SIZE_MAX/2);+ // const size_t sample_countdown = theap->sample_countdown - req_size;+ // return ((mi_ssize_t)sample_countdown < 0);+ return ((mi_ssize_t)theap->sample_countdown < (mi_ssize_t)req_size);+}+// we always align guarded pointers in a block at an offset// the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones#define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0))+#define MI_BLOCK_TAG_PROFILED ((mi_encoded_t)(1))#define MI_BLOCK_TAG_GUARDED (~MI_BLOCK_TAG_ALIGNED)-#endif+static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) {#if MI_GUARDED@@ -967,33 +1223,28 @@ static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void*#elseMI_UNUSED(block); MI_UNUSED(p);return false;-#endif+#endif}-#if MI_GUARDED-static inline bool mi_theap_malloc_use_guarded(mi_theap_t* theap, size_t size) {- // this code is written to result in fast assembly as it is on the hot path for allocation- const size_t count = theap->guarded_sample_count - 1; // if the rate was 0, this will underflow and count for a long time..- if mi_likely(count != 0) {- // no sample- theap->guarded_sample_count = count;- return false;- }- else if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) {- // use guarded allocation- theap->guarded_sample_count = theap->guarded_sample_rate; // reset- return (theap->guarded_sample_rate != 0);- }- else {- // failed size criteria, rewind count (but don't write to an empty theap)- if (theap->guarded_sample_rate != 0) { theap->guarded_sample_count = 1; }- return false;- }+static inline bool mi_block_ptr_is_sampled(const mi_block_t* block, const void* p) {+#if MI_GUARDED || MI_PROFILE+ const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block;+ return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next != MI_BLOCK_TAG_ALIGNED);+#else+ MI_UNUSED(block); MI_UNUSED(p);+ return false;+#endif}-mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept;+static inline bool mi_profiler_is_enabled(const mi_profiler_t* prof) {+ _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved;+ return (mi_atomic_load_acquire(penabled) != 0);+}-#endif+static inline bool mi_profiler_set_enabled(mi_profiler_t* prof, bool enable) {+ _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved;+ return (mi_atomic_exchange_release(penabled, (enable ? 1 : 0)) != 0);+}/* -------------------------------------------------------------------@@ -1028,25 +1279,46 @@ static inline bool mi_is_in_same_page(const void* p, const void* q) {}static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) {- void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]);+ const uintptr_t k1 = keys[0];+ #if MI_PAGE_KEY_COUNT==2+ const uintptr_t k2 = keys[1];+ #else+ const uintptr_t k2 = mi_rotr(k1,13);+ #endif+ void* p = (void*)(mi_rotr(x - k1, k1) ^ k2);return (p==null ? NULL : p);}static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) {- uintptr_t x = (uintptr_t)(p==NULL ? null : p);- return mi_rotl(x ^ keys[1], keys[0]) + keys[0];+ const uintptr_t k1 = keys[0];+ #if MI_PAGE_KEY_COUNT==2+ const uintptr_t k2 = keys[1];+ #else+ const uintptr_t k2 = mi_rotr(k1,13);+ #endif+ const uintptr_t x = (uintptr_t)(p==NULL ? null : p);+ return mi_rotl(x ^ k2, k1) + k1;}static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const uintptr_t* keys) {const uint32_t x = (uint32_t)(mi_ptr_encode(null,p,keys));// make the lowest byte 0 to prevent spurious read overflows which could be a security issue (issue #951)+ // also clear bit 9 which we set only when a block is freed.#if MI_BIG_ENDIAN- return (x & 0x00FFFFFF);+ return (x & 0x00FFFEFF);#else- return (x & 0xFFFFFF00);+ return (x & 0xFFFFFE00);#endif}+static inline uint32_t mi_ptr_encode_canary_freed(void) {+ return (0x00DEAD00); // set bit 9 so it is different from any valid canary+}++static inline bool mi_ptr_decode_canary_is_freed(uint32_t canary) {+ return (canary == mi_ptr_encode_canary_freed());+}+static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) {mi_track_mem_defined(block,sizeof(mi_block_t));mi_block_t* next;@@ -1071,14 +1343,15 @@ static inline void mi_block_set_nextx(const void* null, mi_block_t* block, constmi_track_mem_noaccess(block,sizeof(mi_block_t));}+mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next); // in options.c+static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) {#if MI_ENCODE_FREELISTmi_block_t* next = mi_block_nextx(page,block,page->keys);// check for free list corruption: is `next` at least in the same page?- // TODO: check if `next` is `page->block_size` aligned?- if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) {- _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next);- next = NULL;+ // todo: check if `next` is `page->block_size` aligned?+ if mi_unlikely(next!=NULL && !mi_page_contains_address(page,next)) {+ return _mi_block_next_is_corrupted(page,block,next); // returns NULL}return next;#else@@ -1096,6 +1369,8 @@ static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, c#endif}++/* -----------------------------------------------------------arena blocks----------------------------------------------------------- */@@ -1136,32 +1411,55 @@ static inline mi_memid_t _mi_memid_create_os(void* base, size_t size, bool commireturn memid;}-static inline mi_memid_t _mi_memid_create_meta(void* mpage, size_t block_idx, size_t block_count) {- mi_memid_t memid = _mi_memid_create(MI_MEM_META);- memid.mem.meta.meta_page = mpage;- memid.mem.meta.block_index = (uint32_t)block_idx;- memid.mem.meta.block_count = (uint32_t)block_count;+static inline mi_memid_t _mi_memid_create_static(void* p, size_t size) {+ mi_memid_t memid = _mi_memid_create(MI_MEM_STATIC);+ memid.mem.malloc.base = p;+ memid.mem.malloc.size = size;memid.initially_committed = true;- memid.initially_zero = true;memid.is_pinned = true;return memid;}+static inline mi_memid_t _mi_memid_create_malloc(void* p, size_t size, bool iszero) {+ mi_memid_t memid = _mi_memid_create(MI_MEM_MALLOC);+ memid.mem.malloc.base = p;+ memid.mem.malloc.size = size;+ memid.initially_committed = true;+ memid.initially_zero = iszero;+ memid.is_pinned = true;+ return memid;+}++static inline size_t _mi_memid_size(mi_memid_t memid) {+ if (mi_memid_is_os(memid)) {+ return memid.mem.os.size;+ }+ else if (memid.memkind == MI_MEM_ARENA) {+ return mi_size_of_slices(memid.mem.arena.slice_count);+ }+ else if (memid.memkind == MI_MEM_MALLOC) {+ return memid.mem.malloc.size;+ }+ else {+ mi_assert_internal(mi_memid_needs_no_free(memid));+ return 0;+ }+}// -------------------------------------------------------------------// Fast "random" shuffle// --------------------------------------------------------------------static inline uintptr_t _mi_random_shuffle(uintptr_t x) {+static inline size_t _mi_random_shuffle(size_t x) {if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros-#if (MI_INTPTR_SIZE>=8)+#if (MI_SIZE_SIZE>=8)// by Sebastiano Vigna, see: <http://xoshiro.di.unimi.it/splitmix64.c>x ^= x >> 30;x *= 0xbf58476d1ce4e5b9UL;x ^= x >> 27;x *= 0x94d049bb133111ebUL;x ^= x >> 31;-#elif (MI_INTPTR_SIZE==4)+#elif (MI_SIZE_SIZE==4)// by Chris Wellons, see: <https://nullprogram.com/blog/2018/07/31/>x ^= x >> 16;x *= 0x7feb352dUL;@@ -1175,108 +1473,83 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x) {// ---------------------------------------------------------------------------------// Provide our own `_mi_memcpy/set` for potential performance optimizations.-//-// For now, only on x64/x86 we optimize to `rep movsb/stosb`.-// Generally, we check for "fast short rep movsb/stosb" (FSRM/FSRS) or "fast enhanced rep movsb" (ERMS) support-// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253.-// Todo: we see improvements on win32 but less with glibc; we might want to only enable this on windows.// ----------------------------------------------------------------------------------#if !MI_TRACK_ENABLED && (MI_ARCH_X64 || MI_ARCH_X86) && (defined(_WIN32) || defined(__GNUC__))--extern mi_decl_hidden size_t _mi_cpu_movsb_max; // in init.c-extern mi_decl_hidden size_t _mi_cpu_stosb_max;--static inline void mi_rep_movsb(void* dst, const void* src, size_t n) {- #if defined(__GNUC__)- __asm volatile("rep movsb" : "+D"(dst), "+c"(n), "+S"(src) : : "memory");- #else- __movsb((unsigned char*)dst, (const unsigned char*)src, n);- #endif-}--static inline void mi_rep_stosb(void* dst, uint8_t val, size_t n) {- #if defined(__GNUC__)- __asm volatile("rep stosb" : "+D"(dst), "+c"(n) : "a"(val) : "memory");- #else- __stosb((unsigned char*)dst, val, n);- #endif-}--static inline void _mi_memcpy(void* dst, const void* src, size_t n) {- if mi_likely(n <= _mi_cpu_movsb_max) { // has fsrm && n <= 127 (todo: and maybe has erms?)- mi_rep_movsb(dst, src, n);- }- else {- memcpy(dst, src, n);- }+static inline void* _mi_memcpy(void* dst, const void* src, size_t n) {+ return memcpy(dst, src, n);}-static inline void _mi_memset(void* dst, int val, size_t n) {- if mi_likely(n <= _mi_cpu_stosb_max) { // has fsrs && n <= 127- mi_rep_stosb(dst, (uint8_t)val, n);- }- else {- memset(dst, val, n);- }+static inline void* _mi_memset(void* dst, int val, size_t n) {+ return memset(dst, val, n);}-#else--static inline void _mi_memcpy(void* dst, const void* src, size_t n) {- memcpy(dst, src, n);+static inline void* _mi_memset_backward(void* dst, int val, size_t n) {+ memset((uint8_t*)dst - n, val, n);+ return dst;}-static inline void _mi_memset(void* dst, int val, size_t n) {- memset(dst, val, n);+static inline void* _mi_memzero(void* dst, size_t n) {+ return _mi_memset(dst, 0, n);}-#endif--// --------------------------------------------------------------------------------// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned-// This is used for example in `mi_realloc`.-// ---------------------------------------------------------------------------------#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)--// On GCC/CLang we provide a hint that the pointers are word aligned.-static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {- mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));- void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);- const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE);- _mi_memcpy(adst, asrc, n);+static inline void* _mi_memzero_backward(void* dst, size_t n) {+ return _mi_memset_backward(dst, 0, n);}-static inline void _mi_memset_aligned(void* dst, int val, size_t n) {- mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);- void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);- _mi_memset(adst, val, n);+static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) {+ // on gcc/clang we can provide a hint that the pointers are word aligned.+ mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE) && _mi_is_aligned(src,MI_SIZE_SIZE));+ void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE);+ const void* asrc = mi_assume_aligned(src, MI_SIZE_SIZE);+ return _mi_memcpy(adst, asrc, n);}-#else--// Default fallback on `_mi_memcpy`-static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {- mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));- _mi_memcpy(dst, src, n);+static inline void* _mi_memset_aligned(void* dst, int val, size_t n) {+ mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE));+ void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE);+ return _mi_memset(adst, val, n);}-static inline void _mi_memset_aligned(void* dst, int val, size_t n) {- mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);- _mi_memset(dst, val, n);-}--#endif--static inline void _mi_memzero(void* dst, size_t n) {- _mi_memset(dst, 0, n);+static inline void* _mi_memzero_aligned(void* dst, size_t n) {+ return _mi_memset_aligned(dst, 0, n);}-static inline void _mi_memzero_aligned(void* dst, size_t n) {- _mi_memset_aligned(dst, 0, n);+// Zero a block: blocks are always aligned with a positive bsize in machine-word bytes.+static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize) {+ mi_assert_internal(bsize%MI_SIZE_SIZE == 0);+ mi_assert_internal(bsize > 0);+ mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE));+ mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || _mi_is_aligned(dst,MI_MAX_ALIGN_SIZE));++ // fast memzero for small sizes based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned)+ // assumes constant memset(p,0,N) gets optimized to fast simd stores by the compiler+ // note: disabled on riscv for now as a constant memset is not always replaced correctly by current compilers.+ // (compile with -DMI_USE_MEMZERO16X=0 to disable this)+ #if (!defined(MI_USE_MEMZERO16X) && !MI_ARCH_RISCV) || (MI_USE_MEMZERO16X != 0) // 16x MI_SIZE_SIZE (128 bytes on 64-bit)+ if mi_unlikely(bsize < 2*MI_SIZE_SIZE) { // bsize < 16 (8)+ *((size_t*)dst) = 0;+ return dst;+ }+ mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE));+ uint8_t* const start = (uint8_t*)mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE);+ uint8_t* const end = start + bsize; // note: if bsize is always a multiple of 16 then end is always aligned as well (but due to padding this does not hold)+ if mi_likely(bsize < 8*MI_SIZE_SIZE) { // bsize < 64 (32)+ const size_t ofs = (bsize>>1)&(2*MI_SIZE_SIZE); mi_assert_internal(bsize < 4*MI_SIZE_SIZE ? ofs==0 : ofs==2*MI_SIZE_SIZE); // ofs == 16 (8)+ _mi_memzero(start, 2*MI_SIZE_SIZE);+ _mi_memzero(start+ofs, 2*MI_SIZE_SIZE);+ _mi_memzero_backward(end-ofs, 2*MI_SIZE_SIZE);+ _mi_memzero_backward(end, 2*MI_SIZE_SIZE);+ return dst;+ }+ if mi_likely(bsize <= 16*MI_SIZE_SIZE) { // bsize < 128 (64)+ _mi_memzero(start, 8*MI_SIZE_SIZE);+ _mi_memzero_backward(end, 8*MI_SIZE_SIZE);+ return dst;+ }+ #endif+ // fallback to regular memset for larger sizes+ void* const wdst = mi_assume_aligned(dst,MI_SIZE_SIZE);+ return _mi_memzero_aligned(wdst, bsize);}--#endif // MI_INTERNAL_Hdiff --git a/vendored/mimalloc/include/mimalloc/prim-tls.h b/vendored/mimalloc/include/mimalloc/prim-tls.hnew file mode 100644index 000000000..da78d0193--- /dev/null+++ b/vendored/mimalloc/include/mimalloc/prim-tls.h@@ -0,0 +1,434 @@+/* ----------------------------------------------------------------------------+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen+This is free software; you can redistribute it and/or modify it under the+terms of the MIT license. A copy of the license can be found in the file+"LICENSE" at the root of this distribution.+-----------------------------------------------------------------------------*/+#pragma once+#ifndef MIMALLOC_PRIM_TLS_H+#define MIMALLOC_PRIM_TLS_H++#include "types.h"+#include "internal.h" // mi_decl_hidden++// --------------------------------------------------------------------------+// We need fast access to both a unique thread id (in `free.c:mi_free`) and+// to a thread-local theap pointer (in `alloc.c:mi_malloc`).+//+// For performance, we tend to use specialized code for various platforms.+// This leads to quite a few ifdefs but it is just for performance and there+// is always a portable fallback (based on regular thread local variables).+//+// Windows : use NtCurrentTeB and TlsAlloc (MI_TLS_MODEL_WIN32)+// Linux,FreeBSD : use thread locals with the initial-exec model (MI_TLS_MODEL_LOCAL)+// macOS : use pthread locals with assembly for the thread-id (MI_TLS_MODEL_PTHREADS)+// Android,OpenBSD : use pthread locals (MI_TLS_MODEL_PTHREADS). todo: maybe on Android MI_TLS_MODEL_LOCAL is better?+// --------------------------------------------------------------------------++// static inline void** mi_prim_thread_pointer(void); // get a pointer to the thread local storage+// static inline void* mi_prim_tls_slot(size_t slot); // directly read an entry from the thread local storage block+// static inline void mi_prim_tls_slot_set(size_t slot, void* value);++static inline mi_threadid_t _mi_prim_thread_id(void); // get a unique id for a thread+static inline mi_theap_t* _mi_theap_default(void); // the default thread local theap+static inline mi_theap_t* _mi_theap_cached(void); // last used thread local theap using the _heap_ api+static inline bool _mi_thread_is_initialized(void); // a thread is initialized if it has a default theap+static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap); // get the thread local theap belonging to a heap+static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap); // get the theap but don't update _mi_theap_cached+static inline mi_theap_t* _mi_page_associated_theap_peek(const mi_page_t* page); // get the theap associated with a page (used in `mi_free_collect_mt`)+++// Default TLS model+#if !defined(MI_TLS_MODEL_LOCAL) && !defined(MI_TLS_MODEL_PTHREADS) && !defined(MI_TLS_MODEL_FIXED) && !defined(MI_TLS_MODEL_WIN32)+#if defined(_WIN32)+#define MI_TLS_MODEL_WIN32 1+#elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__) // and FreeBSD?+#define MI_TLS_MODEL_PTHREADS 1+#else+#define MI_TLS_MODEL_LOCAL 1+#endif+#endif+++//-------------------------------------------------------------------+// Access to the thread pointer+//-------------------------------------------------------------------++// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id+// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)+// Nevertheless, it seems needed on older graviton platforms (see issue #851).+// For now, we only enable this for specific platforms.+#if !defined(MI_USE_BUILTIN_THREAD_POINTER) /* allow user override */+ #if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \+ && !defined(__CYGWIN__) \+ && !defined(MI_LIBC_MUSL) \+ && (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */+ #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \+ || (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__riscv)) \+ || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \+ || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__) || defined(__riscv)))+ #define MI_USE_BUILTIN_THREAD_POINTER 1+ #endif+ #endif+#endif++#if defined(MI_PRIM_THREAD_POINTER) // potential user override+static inline void** mi_prim_thread_pointer(void) {+ return MI_PRIM_THREAD_POINTER();+}+#elif defined(_WIN32)+static inline void** mi_prim_thread_pointer(void) {+ return (void**)NtCurrentTeb();+}+#elif MI_USE_BUILTIN_THREAD_POINTER+static inline void** mi_prim_thread_pointer(void) {+ return (void**)__builtin_thread_pointer();+}+#elif defined(__GNUC__) && !defined(__CYGWIN__)+ #if defined(__aarch64__)+ static inline void** mi_prim_thread_pointer(void) {+ void** tcb;+ #if defined(__APPLE__) // M1, issue rgb(62, 76, 62)+ __asm__ ("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r" (tcb));+ #else+ __asm__ ("mrs %0, tpidr_el0" : "=r" (tcb));+ #endif+ return tcb;+ }+ #elif defined(__riscv)+ static inline void** mi_prim_thread_pointer(void) {+ void** tcb;+ __asm__ ("mv %0, tp" : "=r" (tcb));+ return tcb;+ }+ #elif defined(__arm__)+ static inline void** mi_prim_thread_pointer(void) {+ void** tcb;+ __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\n\tbic %0, %0, #3" : "=r" (tcb));+ return tcb;+ }+ #elif defined(__i386__)+ static inline void** mi_prim_thread_pointer(void) {+ void** tcb;+ __asm__ ("movl %%gs:0, %0" : "=r" (tcb) : : ); // x86 32-bit always uses GS+ return tcb;+ }+ #elif defined(__x86_64__)+ static inline void** mi_prim_thread_pointer(void) {+ void** tcb;+ #if defined(__APPLE__)+ __asm__ ("movq %%gs:0, %0" : "=r" (tcb) : : ); // x86_64 macOSX uses GS+ #elif (MI_SIZE_SIZE==4)+ __asm__ ("movl %%fs:0, %0" : "=r" (tcb) : : ); // x32 ABI+ #else+ __asm__ ("movq %%fs:0, %0" : "=r" (tcb) : : ); // x86_64 Linux, BSD uses FS+ #endif+ return tcb;+ }+ #else+ #define MI_NO_THREAD_POINTER (1)+ #endif+#elif MI_USE_PTHREADS && defined(__APPLE__)+static inline void** mi_prim_thread_pointer(void) {+ return (void**)pthread_self();+}+#else+#define MI_NO_THREAD_POINTER (1)+#endif++#if !MI_NO_THREAD_POINTER+#define MI_HAS_TLS_SLOT (1)+static inline void* mi_prim_tls_slot(size_t slot) {+ #if defined(_WIN32)+ #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC)+ return (void*)__readgsqword((unsigned long)(slot*sizeof(void*))); // direct load at offset from gs+ #elif _M_IX86 && !defined(_M_ARM64EC)+ return (void*)__readfsdword((unsigned long)(slot*sizeof(void*))); // direct load at offset from fs+ #else+ return mi_prim_thread_pointer()[slot];+ #endif+ #else+ return mi_prim_thread_pointer()[slot];+ #endif+}++static inline void mi_prim_tls_slot_set(size_t slot, void* value) {+ mi_prim_thread_pointer()[slot] = value;+}+#endif++/* ----------------------------------------------------------------------------------------+ Get a unique thread-id for a thread+---------------------------------------------------------------------------------------- */++// Get a unique id for the current thread.+#if defined(MI_PRIM_THREAD_ID)+static inline mi_threadid_t __mi_prim_thread_id(void) {+ return MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488)+}+#elif !MI_NO_THREAD_POINTER+static inline mi_threadid_t __mi_prim_thread_id(void) {+ #if defined(__BIONIC__)+ return (mi_threadid_t)mi_prim_tls_slot(1);+ #else+ return (mi_threadid_t)mi_prim_thread_pointer();+ #endif+}+#else+// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).+extern mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper;+static inline mi_threadid_t __mi_prim_thread_id(void) {+ return (uintptr_t)&__mi_thread_id_helper;+}+#endif++static inline mi_threadid_t _mi_prim_thread_id(void) {+ const mi_threadid_t tid = __mi_prim_thread_id();+ mi_assert_internal(tid > MI_THREADID_DETACHED);+ mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear?+ return tid;+}+++/* ----------------------------------------------------------------------------------------+Get the thread local default theap: `_mi_theap_default()` (and the cached heap `_mi_theap_cached`).++This is inlined here as it is on the fast path for allocation functions.+We have 4 models:++- MI_TLS_MODEL_LOCAL: use regular thread local (default on Linux, FreeBSD, etc)+ On most platforms (Linux, FreeBSD, NetBSD, etc), this just returns a+ thread local variable (`__mi_theap_default`). With the initial-exec TLS model this ensures+ that the storage will always be available and properly initialized (with an empty theap).++ On some platforms the underlying TLS implementation (or the loader) will call itself `malloc`+ on a first access to a thread local and recurse in the MI_TLS_MODEL_LOCAL.+ A way around this is to define MI_TLS_RECURSE_GUARD which adds an extra check if the process+ is initialized before accessing the thread-local. This is a check in the fast path though+ so this should be avoided.++- MI_TLS_MODEL_PTHREADS: use `pthread_getspecific`. (default on macOS and OpenBSD, maybe good for Android as well?)+ Use pthread local storage. Can be as fast as thread locals on many platforms (like recent macOS).++- MI_TLS_MODEL_FIXED: use a fixed slot in the TLS block.+ This reserves an unused and fixed TLS slot. This is fast and avoids the problem+ where the underlying TLS implementation (or the loader) will call itself `malloc`+ on a first access to a thread local (and recurse in the MI_TLS_MODEL_LOCAL).+ This goes wrong though if the OS or a library uses the same fixed slot, and also+ prevents multiple instances of mimalloc in the same process.++- MI_TLS_MODEL_WIN32: use a dynamically allocated slot with TlsAlloc. (default on Windows)+ We use TlsAlloc'd slot. First tries to use one of the "direct" first 64 slots which+ are the fastest, but falls back to using "expansion" slots when needed (up to 1088 slots).+ (If the allocated slot happens to always be under 64 for a particular program,+ one might use cmake with `-DMI_WIN_DIRECT_TLS=ON` to skip the expansion slot test in the fast path.)++Each model should define `MI_THEAP_INITASNULL` to signify that the initial value+returned from `_mi_theap_default()` can be `NULL` (instead of the address of the empty heap).+This incurs an extra check in the fast path (but can often be combined in an existing check).+------------------------------------------------------------------------------------------- */++#if !defined(MI_TLS_RECURSE_GUARD) && MI_TLS_MODEL_LOCAL && defined(__APPLE__)+#define MI_TLS_RECURSE_GUARD 1 // macOS can allocate on thread-local initialization+#endif++// Declared this way to optimize register spills and branches+mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void);++static inline mi_theap_t* __mi_theap_empty(void) {+ #if __GNUC__+ __asm(""); // prevent conditional load+ return (mi_theap_t*)&_mi_theap_empty;+ #else+ return _mi_theap_empty_get();+ #endif+}++#if MI_TLS_MODEL_LOCAL+// Thread local with an initial value (default on Linux). Very efficient.+extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default; // default theap to allocate from+extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached; // theap from the last used heap++// defined in `init.c`; do not use these directly+extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called?++static inline mi_theap_t* _mi_theap_default(void) {+ #if defined(MI_TLS_RECURSE_GUARD)+ if mi_unlikely(!_mi_process_is_initialized) return _mi_theap_empty_get();+ #endif+ return __mi_theap_default;+}++static inline mi_theap_t* _mi_theap_cached(void) {+ return __mi_theap_cached;+}++#elif MI_TLS_MODEL_PTHREADS+// Dynamic pthread slots. This can be fast depending on the platform (default for macOS and OpenBSD)+// On some platforms (like macOS), the loader might allocate on thread local declarations which+// can be avoided with pthreads.+#define MI_THEAP_INITASNULL 1++extern mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_default_key;+extern mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_cached_key;++static inline mi_theap_t* _mi_theap_default(void) {+ pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_default_key);+ #if defined(__APPLE__) && defined(__aarch64__) && MI_HAS_TLS_SLOT+ // on apple arm64, the pthread specific slots are direct slots; inline it to avoid a stack frame setup in `mi_malloc`+ // todo: this is probably also the case on x64 and power pc?+ if (key == MI_PTHREAD_KEY_INVALID) return NULL;+ return (mi_theap_t*)mi_prim_tls_slot(key);+ #else+ return (mi_theap_t*)mi_pthread_key_get(key);+ #endif+}++static inline mi_theap_t* _mi_theap_cached(void) {+ pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_cached_key);+ #if defined(__APPLE__) && defined(__aarch64__) && MI_HAS_TLS_SLOT+ if (key == MI_PTHREAD_KEY_INVALID) return NULL;+ return (mi_theap_t*)mi_prim_tls_slot(key);+ #else+ return (mi_theap_t*)mi_pthread_key_get(key);+ #endif+}++#elif MI_TLS_MODEL_WIN32+// Dynamic TLS slots -- this is the default on Windows.+#define MI_THEAP_INITASNULL 1++// We try to use direct slots (64 available), but can also use the expansion slots (upto 1024 extra available)+// See <https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/pebteb/teb/index.htm> for the offsets.+#if MI_SIZE_SIZE==4+#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_SIZE_SIZE)+#else+#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_SIZE_SIZE)+#endif++extern mi_decl_hidden _Atomic(size_t) _mi_theap_default_slot;+extern mi_decl_hidden _Atomic(size_t) _mi_theap_cached_slot;+extern mi_decl_hidden _Atomic(size_t) _mi_theap_default_expansion_slot;+extern mi_decl_hidden _Atomic(size_t) _mi_theap_cached_expansion_slot;++static inline mi_theap_t* _mi_theap_default(void) {+ const size_t slot = mi_atomic_load_relaxed(&_mi_theap_default_slot);+ mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);+ #if !MI_WIN_DIRECT_TLS+ if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?+ mi_theap_t** const eslots = (mi_theap_t**)theap; // theap is actually the expansion slot entry+ if mi_likely(eslots!=NULL) { // is it initialized? (on this thread)+ theap = eslots[mi_atomic_load_relaxed(&_mi_theap_default_expansion_slot)];+ }+ }+ #endif+ return theap;+}++static inline mi_theap_t* _mi_theap_cached(void) {+ const size_t slot = mi_atomic_load_relaxed(&_mi_theap_cached_slot);+ mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);+ #if !MI_WIN_DIRECT_TLS+ if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?+ mi_theap_t** const eslots = (mi_theap_t**)theap; // theap is the expansion slot entry+ if mi_likely(eslots!=NULL) { // is it initialized? (on this thread)+ theap = eslots[mi_atomic_load_relaxed(&_mi_theap_cached_expansion_slot)];+ }+ }+ #endif+ return theap;+}++#elif MI_TLS_MODEL_FIXED+// Fixed TLS slot. Can be the fastest approach, but does not work if there are multiple instances of+// mimalloc in the same process. Most OS's do not have official user reserved fixed slots so this cannot be+// guaranteed to work in general.+#define MI_THEAP_INITASNULL 1++#if !MI_HAS_TLS_SLOT+#error this platform cannot support MI_TLS_MODEL_FIXED without defining mi_prim_tls_slot+#endif++#if !defined(MI_TLS_MODEL_FIXED_DEFAULT)+ #if defined(__APPLE__) && !defined(__POWERPC__) // macOS on arm64 or x64+ // we use the last two swift framework slots which seem unused.+ // we may want to use slot 6 and 11 instead which are only used by Windows emulation.+ // see <https://github.com/apple/darwin-libpthread/blob/main/private/pthread/tsd_private.h#L99> for assigned slots+ #define MI_TLS_MODEL_FIXED_DEFAULT 108+ #define MI_TLS_MODEL_FIXED_CACHED 109+ #elif defined(_WIN32)+ // we use two seemingly unused fields in the Windows TEB.+ // see <https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/pebteb/teb/index.htm>+ #define MI_TLS_MODEL_FIXED_DEFAULT 5 // arbitrary user pointer+ #define MI_TLS_MODEL_FIXED_CACHED 7 // environment pointer (used by OS2)+ #else+ #error define the TLS model fixed slots (or change the TLS model away from MI_TLS_MODEL_FIXED)+ #endif+#endif++static inline mi_theap_t* _mi_theap_default(void) {+ return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_DEFAULT);+}++static inline mi_theap_t* _mi_theap_cached(void) {+ return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_CACHED);+}++#else+#error "no TLS model is defined for this platform?"+#endif+++// Check if a thread is initialized (without using a thread-local if using fixed slots)+static inline bool _mi_thread_is_initialized(void) {+ return mi_theap_is_initialized(_mi_theap_default());+}++// Get (and possible create) the theap belonging to a heap+// We cache the last accessed theap in `_mi_theap_cached` for better performance.+static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap) {+ mi_theap_t* theap = _mi_theap_cached();+ #if MI_THEAP_INITASNULL+ if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap;+ #else+ if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap;+ #endif+ return _mi_heap_theap_get_or_init(heap);+}++static inline mi_theap_t* _mi_heap_theap_cached(mi_heap_t* heap) {+ mi_theap_t* theap = _mi_theap_cached();+ #if MI_THEAP_INITASNULL+ if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap;+ #else+ if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap;+ #endif+ return NULL;+}++// Get the theap belonging to a heap without creating it if it is not yet initialized.+static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) {+ mi_theap_t* theap = _mi_theap_cached();+ #if MI_THEAP_INITASNULL+ if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap;+ #else+ if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap;+ #endif+ theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); // don't update the cache on a query+ mi_assert_internal(theap==NULL || (!_mi_is_empty_theap(theap) && theap->heap==heap));+ return theap;+}++// Find the associated theap or NULL if it does not exist (during shutdown)+// Should be fast as it is called in `free.c:mi_free_try_collect`.+static inline mi_theap_t* _mi_page_associated_theap_peek(const mi_page_t* page) {+ mi_heap_t* const heap = mi_page_heap(page);+ mi_theap_t* const theap = (mi_theap_t*)_mi_thread_local_get(heap->theap);+ if (theap==NULL || theap->tld==NULL /* heap destroy */) return NULL;+ if (theap->heap != heap) return NULL; // should never happen, but can happen for a free across subprocesses, which can happen during pthread tls storage deallocation+ mi_assert_internal(!_mi_is_empty_theap(theap) && mi_theap_matches_thread(theap));+ // note: for pages allocated by a detached theap, the returned theap may not be detached+ return theap;+}++#endif // MI_PRIM_TLS_Hdiff --git a/vendored/mimalloc/include/mimalloc/prim.h b/vendored/mimalloc/include/mimalloc/prim.hindex aecfc8074..82b2d9c3d 100644--- a/vendored/mimalloc/include/mimalloc/prim.h+++ b/vendored/mimalloc/include/mimalloc/prim.h@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -7,7 +7,8 @@ terms of the MIT license. A copy of the license can be found in the file#pragma once#ifndef MIMALLOC_PRIM_H#define MIMALLOC_PRIM_H-#include "internal.h" // mi_decl_hidden++#include "types.h"// --------------------------------------------------------------------------// This file specifies the primitive portability API.@@ -23,7 +24,7 @@ terms of the MIT license. A copy of the license can be found in the file// OS memory configurationtypedef struct mi_os_mem_config_s {size_t page_size; // default to 4KiB- size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows)+ size_t large_page_size; // 0 if not supported, usually 2MiBsize_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB)size_t physical_memory_in_kib; // physical memory size in KiBsize_t virtual_address_bits; // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization)@@ -107,7 +108,9 @@ void _mi_prim_out_stderr( const char* msg );// Get an environment variable. (only for options)// name != NULL, result != NULL, result_size >= 64-bool _mi_prim_getenv(const char* name, char* result, size_t result_size);+// Return 1 for success, 0 if not found,+// and -1 on error (for example, if `getenv` cannot be called yet during preloading).+int _mi_prim_getenv(const char* name, char* result, size_t result_size);// Fill a buffer with strong randomness; return `false` on error or if@@ -126,417 +129,8 @@ void _mi_prim_thread_associate_default_theap(mi_theap_t* theap);// Is this thread part of a thread pool?bool _mi_prim_thread_is_in_threadpool(void);-// Yield to other threads. Should be similar to `sleep(0)`.+// Yield to other threads. Should be similar to `sleep(0)`.// Is called only in rare situations and does not have to be lightning fast.void _mi_prim_thread_yield(void);-//--------------------------------------------------------------------// Access to TLS (thread local storage) slots.-// We need fast access to both a unique thread id (in `free.c:mi_free`) and-// to a thread-local theap pointer (in `alloc.c:mi_malloc`).-// To achieve this we use specialized code for various platforms.-//---------------------------------------------------------------------// On some libc + platform combinations we can directly access a thread-local storage (TLS) slot.-// The TLS layout depends on both the OS and libc implementation so we use specific tests for each main platform.-// If you test on another platform and it works please send a PR :-)-// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register.-//-// Note: we would like to prefer `__builtin_thread_pointer()` nowadays instead of using assembly,-// but unfortunately we can not detect support reliably (see issue #883)-// We also use it on Apple OS as we use a TLS slot for the default theap there.-#if (defined(_WIN32)) || \- (defined(__GNUC__) && ( \- (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \- || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \- || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \- || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \- || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \- ))--static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept {- void* res;- const size_t ofs = (slot*sizeof(void*));- #if defined(_WIN32)- #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC)- res = (void*)__readgsqword((unsigned long)ofs); // direct load at offset from gs- #elif _M_IX86 && !defined(_M_ARM64EC)- res = (void*)__readfsdword((unsigned long)ofs); // direct load at offset from fs- #else- res = ((void**)NtCurrentTeb())[slot]; MI_UNUSED(ofs);- #endif- #elif defined(__i386__)- __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS- #elif defined(__APPLE__) && defined(__x86_64__)- __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS- #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)- __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI- #elif defined(__x86_64__)- __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS- #elif defined(__arm__)- void** tcb; MI_UNUSED(ofs);- __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));- res = tcb[slot];- #elif defined(__aarch64__)- void** tcb; MI_UNUSED(ofs);- #if defined(__APPLE__) // M1, issue #343- __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));- #else- __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));- #endif- res = tcb[slot];- #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781- MI_UNUSED(ofs);- res = pthread_getspecific(slot);- #else- #define MI_HAS_TLS_SLOT 0- MI_UNUSED(ofs);- res = NULL;- #endif- return res;-}--#ifndef MI_HAS_TLS_SLOT-#define MI_HAS_TLS_SLOT 1-#endif--// setting a tls slot is only used on macOS for now-static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {- const size_t ofs = (slot*sizeof(void*));- #if defined(_WIN32)- ((void**)NtCurrentTeb())[slot] = value; MI_UNUSED(ofs);- #elif defined(__i386__)- __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS- #elif defined(__APPLE__) && defined(__x86_64__)- __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS- #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)- __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI- #elif defined(__x86_64__)- __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS- #elif defined(__arm__)- void** tcb; MI_UNUSED(ofs);- __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));- tcb[slot] = value;- #elif defined(__aarch64__)- void** tcb; MI_UNUSED(ofs);- #if defined(__APPLE__) // M1, issue #343- __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));- #else- __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));- #endif- tcb[slot] = value;- #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781- MI_UNUSED(ofs);- pthread_setspecific(slot, value);- #else- MI_UNUSED(ofs); MI_UNUSED(value);- #endif-}--#endif---// defined in `init.c`; do not use these directly-extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main; // theap belonging to the main heap-extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called?---//--------------------------------------------------------------------// Get a fast unique thread id.-//-// Getting the thread id should be performant as it is called in the-// fast path of `_mi_free` and we specialize for various platforms as-// inlined definitions. Regular code should call `init.c:_mi_thread_id()`.-// We only require _mi_prim_thread_id() to return a unique id-// for each thread (unequal to zero).-//----------------------------------------------------------------------// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id-// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)-// Nevertheless, it seems needed on older graviton platforms (see issue #851).-// For now, we only enable this for specific platforms.-#if !defined(MI_USE_BUILTIN_THREAD_POINTER) /* allow user override */- #if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \- && !defined(__CYGWIN__) \- && !defined(MI_LIBC_MUSL) \- && (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */- #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \- || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \- || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))- #define MI_USE_BUILTIN_THREAD_POINTER 1- #endif- #endif-#endif--static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept;--static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {- const mi_threadid_t tid = __mi_prim_thread_id();- mi_assert_internal(tid > 1);- mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear?- return tid;-}--// Get a unique id for the current thread.-#if defined(MI_PRIM_THREAD_ID)--static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {- const mi_threadid_t tid = MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488)- mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits- return tid;-}--#elif defined(_WIN32)--static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {- // Windows: works on Intel and ARM in both 32- and 64-bit- return (uintptr_t)NtCurrentTeb();-}--#elif MI_USE_BUILTIN_THREAD_POINTER--static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {- // Works on most Unix based platforms with recent compilers- return (uintptr_t)__builtin_thread_pointer();-}--#elif MI_HAS_TLS_SLOT--static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {- #if defined(__BIONIC__)- // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id- // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86- return (uintptr_t)mi_prim_tls_slot(1);- #else- // in all our other targets, slot 0 is the thread id- // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h- // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36- return (uintptr_t)mi_prim_tls_slot(0);- #endif-}--#else--// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).-static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {- return (uintptr_t)&__mi_theap_main;-}--#endif----/* -----------------------------------------------------------------------------------------Get the thread local default theap: `_mi_theap_default()` (and the cached heap `_mi_theap_cached`).--This is inlined here as it is on the fast path for allocation functions.-We have 4 models:--- MI_TLS_MODEL_THREAD_LOCAL: use regular thread local (default on Linux, FreeBSD, etc)- On most platforms (Linux, FreeBSD, NetBSD, etc), this just returns a- thread local variable (`__mi_theap_default`). With the initial-exec TLS model this ensures- that the storage will always be available and properly initialized (with an empty theap).-- On some platforms the underlying TLS implementation (or the loader) will call itself `malloc`- on a first access to a thread local and recurse in the MI_TLS_MODEL_THREAD_LOCAL.- A way around this is to define MI_TLS_RECURSE_GUARD which adds an extra check if the process- is initialized before accessing the thread-local. This is a check in the fast path though- so this should be avoided.--- MI_TLS_MODEL_FIXED_SLOT: use a fixed slot in the TLS block (default on macOS)- This reserves an unused and fixed TLS slot. This is fast and avoids the problem- where the underlying TLS implementation (or the loader) will call itself `malloc`- on a first access to a thread local (and recurse in the MI_TLS_MODEL_THREAD_LOCAL).- This goes wrong though if the OS or a library uses the same fixed slot.--- MI_TLS_MODEL_DYNAMIC_WIN32: use a dynamically allocated slot with TlsAlloc. (default on Windows)- Windows has somewhat slow thread locals so by default we use TlsAlloc'd slots which- can be more efficient. First tries to use one of the "direct" first 64 slots which- are the fastest, but falls back to using "expansion" slots when needed (up to 1088 slots).- (If the allocated slot happens to always be under 64 for a particular program,- one might use cmake with `-DMI_WIN_DIRECT_TLS=ON` to skip the expansion slot test in the fast path.)--- MI_TLS_MODEL_DYNAMIC_PTHREADS: use `pthread_getspecific`. (default on OpenBSD, maybe good for Android as well?)- Use pthread local storage. Somewhat slow but can work well depending on the platform.--Each model should define `MI_THEAP_INITASNULL` to signify that the initial value-returned from `_mi_theap_default()` can be `NULL` (instead of the address of the empty heap).-This incurs an extra check in the fast path (but can often be combined in an existing check).-------------------------------------------------------------------------------------------- */--static inline mi_theap_t* _mi_theap_default(void);-static inline mi_theap_t* _mi_theap_cached(void);--#if defined(_WIN32)- #define MI_TLS_MODEL_DYNAMIC_WIN32 1-#elif defined(__APPLE__) && MI_HAS_TLS_SLOT && !defined(__POWERPC__) // macOS on arm64 or x64- // #define MI_TLS_MODEL_DYNAMIC_PTHREADS 1 // also works but a bit slower- #define MI_TLS_MODEL_FIXED_SLOT 1- #define MI_TLS_MODEL_FIXED_SLOT_DEFAULT 108 // seems unused. @apple: it would be great to get 2 official slots for custom allocators :-)- #define MI_TLS_MODEL_FIXED_SLOT_CACHED 109- // we used before __PTK_FRAMEWORK_OLDGC_KEY9 (89) but that seems used now.- // see <https://github.com/rweichler/substrate/blob/master/include/pthread_machdep.h>-#elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__)- #define MI_TLS_MODEL_DYNAMIC_PTHREADS 1- // #define MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL 1-#else- #define MI_TLS_MODEL_THREAD_LOCAL 1-#endif--// Declared this way to optimize register spills and branches-mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void);--static inline mi_theap_t* __mi_theap_empty(void) {- #if __GNUC__- __asm(""); // prevent conditional load- return (mi_theap_t*)&_mi_theap_empty;- #else- return _mi_theap_empty_get();- #endif-}--#if MI_TLS_MODEL_THREAD_LOCAL-// Thread local with an initial value (default on Linux). Very efficient.--extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default; // default theap to allocate from-extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached; // theap from the last used heap--static inline mi_theap_t* _mi_theap_default(void) {- #if defined(MI_TLS_RECURSE_GUARD)- if (mi_unlikely(!_mi_process_is_initialized)) return _mi_theap_empty_get();- #endif- return __mi_theap_default;-}--static inline mi_theap_t* _mi_theap_cached(void) {- return __mi_theap_cached;-}--#elif MI_TLS_MODEL_FIXED_SLOT-// Fixed TLS slot (default on macOS).-#define MI_THEAP_INITASNULL 1--static inline mi_theap_t* _mi_theap_default(void) {- return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_DEFAULT);-}--static inline mi_theap_t* _mi_theap_cached(void) {- return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_CACHED);-}--#elif MI_TLS_MODEL_DYNAMIC_WIN32-// Dynamic TLS slot (default on Windows)-#define MI_THEAP_INITASNULL 1--// We try to use direct slots (64), but can also use the expansion slots (upto 1024 extra available)-// See <https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/pebteb/teb/index.htm> for the offsets.-#if MI_SIZE_SIZE==4-#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_SIZE_SIZE)-#else-#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_SIZE_SIZE)-#endif--extern mi_decl_hidden size_t _mi_theap_default_slot;-extern mi_decl_hidden size_t _mi_theap_cached_slot;-extern mi_decl_hidden size_t _mi_theap_default_expansion_slot;-extern mi_decl_hidden size_t _mi_theap_cached_expansion_slot;--static inline mi_theap_t* _mi_theap_default(void) {- const size_t slot = _mi_theap_default_slot;- mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);- #if !MI_WIN_DIRECT_TLS- if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?- if mi_likely(theap!=NULL) { // initialized (on this thread)?- theap = ((mi_theap_t**)theap)[_mi_theap_default_expansion_slot];- }- }- #endif- return theap;-}--static inline mi_theap_t* _mi_theap_cached(void) {- const size_t slot = _mi_theap_cached_slot;- mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);- #if !MI_WIN_DIRECT_TLS- if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?- if mi_likely(theap!=NULL) { // initialized (on this thread)?- theap = ((mi_theap_t**)theap)[_mi_theap_cached_expansion_slot];- }- }- #endif- return theap;-}--#elif MI_TLS_MODEL_DYNAMIC_PTHREADS-// Dynamic pthread slot on less common platforms. This is not too bad. (default on OpenBSD)-#define MI_THEAP_INITASNULL 1--extern mi_decl_hidden pthread_key_t _mi_theap_default_key;-extern mi_decl_hidden pthread_key_t _mi_theap_cached_key;--static inline mi_theap_t* _mi_theap_default(void) {- #if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL- // we can skip this check if using the initial key will return NULL from pthread_getspecific- if mi_unlikely(_mi_theap_default_key==0) { return NULL; }- #endif- return (mi_theap_t*)pthread_getspecific(_mi_theap_default_key);-}--static inline mi_theap_t* _mi_theap_cached(void) {- #if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL- // we can skip this check if using the initial key will return NULL from pthread_getspecific- if mi_unlikely(_mi_theap_cached_key==0) { return NULL; }- #endif- return (mi_theap_t*)pthread_getspecific(_mi_theap_cached_key);-}--#else-#error "no TLS model is defined for this platform?"-#endif---// Check if a thread is initialized (without using a thread-local if using fixed slots)-static inline bool _mi_thread_is_initialized(void) {- return (mi_theap_is_initialized(_mi_theap_default()));-}--// Get (and possible create) the theap belonging to a heap-// We cache the last accessed theap in `_mi_theap_cached` for better performance.-static inline mi_theap_t* _mi_heap_theap(const mi_heap_t* heap) {- mi_theap_t* theap = _mi_theap_cached();- #if MI_THEAP_INITASNULL- if mi_likely(theap!=NULL && _mi_theap_heap(theap)==heap) return theap;- #else- if mi_likely(_mi_theap_heap(theap)==heap) return theap;- #endif- return _mi_heap_theap_get_or_init(heap);-}--// Get the theap belonging to a heap without creating in if it is not yet initialized.-static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) {- mi_theap_t* theap = _mi_theap_cached();- #if MI_THEAP_INITASNULL- if mi_unlikely(theap==NULL || _mi_theap_heap(theap)!=heap)- #else- if mi_unlikely(_mi_theap_heap(theap)!=heap)- #endif- {- theap = _mi_heap_theap_get_peek(heap); // don't update the cache on a query (?)- }- mi_assert(theap==NULL || _mi_theap_heap(theap)==heap);- return theap;-}--// Find the associated theap or NULL if it does not exist (during shutdown)-// Should be fast as it is called in `free.c:mi_free_try_collect`.-static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) {- mi_heap_t* const heap = page->heap;- mi_theap_t* theap;- if mi_likely(heap==NULL) { theap = __mi_theap_main; } // note: on macOS accessing the thread_local can cause allocation during thread shutdown (and reinitialize the thread)!- else { theap = _mi_heap_theap_peek(heap); }- mi_assert_internal(theap==NULL || _mi_thread_id()==theap->tld->thread_id);- return theap;-}-#endif // MI_PRIM_Hdiff --git a/vendored/mimalloc/include/mimalloc/track.h b/vendored/mimalloc/include/mimalloc/track.hindex 1f4021297..e6a57c8ec 100644--- a/vendored/mimalloc/include/mimalloc/track.h+++ b/vendored/mimalloc/include/mimalloc/track.h@@ -133,18 +133,18 @@ defined, undefined, or not accessible at all:#endif-#if MI_PADDING+#if MI_PADDING_CHECK_BYTES && !MI_GUARDED#define mi_track_malloc(p,reqsize,zero) \- if ((p)!=NULL) { \+ do { if ((p)!=NULL) { \mi_assert_internal(mi_usable_size(p)==(reqsize)); \mi_track_malloc_size(p,reqsize,reqsize,zero); \- }+ } } while(0)#else#define mi_track_malloc(p,reqsize,zero) \- if ((p)!=NULL) { \+ do { if ((p)!=NULL) { \mi_assert_internal(mi_usable_size(p)>=(reqsize)); \mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \- }+ } } while(0)#endif#endif // MI_TRACK_Hdiff --git a/vendored/mimalloc/include/mimalloc/types.h b/vendored/mimalloc/include/mimalloc/types.hindex 31fdace9c..530c464a3 100644--- a/vendored/mimalloc/include/mimalloc/types.h+++ b/vendored/mimalloc/include/mimalloc/types.h@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -24,12 +24,13 @@ terms of the MIT license. A copy of the license can be found in the file#include <mimalloc-stats.h>+#include <mimalloc-profile.h>#include <stddef.h> // ptrdiff_t#include <stdint.h> // uintptr_t, uint16_t, etc#include <stdbool.h> // bool#include <limits.h> // SIZE_MAX etc.#include <errno.h> // error codes-#include "bits.h" // size defines (MI_INTPTR_SIZE etc), bit operations+#include "bits.h" // size defines (MI_SIZE_SIZE etc), bit operations#include "atomic.h" // _Atomic primitives// Minimal alignment necessary. On most platforms 16 bytes are needed@@ -38,6 +39,9 @@ terms of the MIT license. A copy of the license can be found in the file#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t)#endif+#if MI_MAX_ALIGN_SIZE < MI_INTPTR_SIZE+#error MI_MAX_ALIGN_SIZE must be at least MI_INTPTR_SIZE+#endif// ------------------------------------------------------// Variants@@ -51,15 +55,15 @@ terms of the MIT license. A copy of the license can be found in the file// #define MI_TRACK_ASAN 1// #define MI_TRACK_ETW 1-// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).-// #define MI_STAT 1+// Define MI_STATS as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).+// #define MI_STATS 1// Define MI_SECURE to enable security mitigations-// #define MI_SECURE 1 // guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption-// #define MI_SECURE 2 // randomize relative allocation addresses (within mimalloc pages)-// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)-// #define MI_SECURE 4 // checks for double free (may be more expensive) (`-DMI_SECURE=ON`)-// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`)+// #define MI_SECURE 1 // check invalid pointer free, guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption+// #define MI_SECURE 2 // randomize relative allocation addresses within mimalloc pages+// #define MI_SECURE 3 // check buffer overflow, check double free, encode free lists (and detect corrupted free lists)+// #define MI_SECURE 4 // same as level 3 for now (`-DMI_SECURE=ON`)+// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`), and byte-precise buffer overflow checks.#if !defined(MI_SECURE)#define MI_SECURE 0@@ -77,29 +81,49 @@ terms of the MIT license. A copy of the license can be found in the file#endif#endif-// Statistics (0=only essential, 1=normal, 2=more fine-grained (expensive) tracking)-#ifndef MI_STAT+// Statistics (0=only essential, 1=detailed (fast, can be enabled always), 2=slightly more expensive tracking of precise requested bytes)+#ifndef MI_STATS#if (MI_DEBUG>0)-#define MI_STAT 2+#define MI_STATS 2#else-#define MI_STAT 0+#define MI_STATS 1+#endif#endif++// Enable profiling support (0=off, 1=fast, can be enabled always, 2=allow fine-grained sample rates(<64 KiB), a tad more expensive)+#ifndef MI_PROFILE+#define MI_PROFILE 1#endif// Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options)-#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_PAGE_META_ALIGNED_FREE_SMALL+#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_OPT_FREE_SMALL#define MI_GUARDED 1#endif+// For profiling or guarded pages we need to sample every once in a while. (0=no sampling, 1=fast, 2=allow fine grained sample rates(<64 KiB), a tad more expensive)+#ifndef MI_SAMPLE+#if MI_PROFILE>1 || MI_GUARDED>1+ #define MI_SAMPLE 2+#elif MI_PROFILE || MI_GUARDED+ #define MI_SAMPLE 1+#else+ #define MI_SAMPLE 0+#endif+#endif+// Reserve extra padding at the end of each block to be more resilient against theap block overflows.-// The padding can detect buffer overflow on free.-#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW))+// The padding can detect heap-block overflow on free, and provides byte-precise `mi_usable_size`.+#if !defined(MI_PADDING)+#if (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW))#define MI_PADDING 1+#else+#define MI_PADDING 0+#endif#endif-// Check padding bytes; allows byte-precise buffer overflow detection-#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1)-#define MI_PADDING_CHECK 1+// Check for byte-precise buffer overflow?+#if !defined(MI_PADDING_CHECK_BYTES) && MI_PADDING && (MI_SECURE>=5 || MI_DEBUG>=1)+#define MI_PADDING_CHECK_BYTES 1#endif@@ -109,6 +133,17 @@ terms of the MIT license. A copy of the license can be found in the file#define MI_ENCODE_FREELIST 1#endif+// Deprecated (double-free is now checked with padding)+// #if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0))+// #define MI_CHECK_DOUBLE_FREE 1+// #endif++#if MI_SECURE>=4 || MI_PADDING || (MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP)+#define MI_PAGE_KEY_COUNT 2+#else+#define MI_PAGE_KEY_COUNT 1+#endif+// Enable large pages for objects between 64KiB and 512KiB.// This should perhaps be disabled by default as for many workloads the block sizes above 64 KiB// are quite random which can lead to too many partially used large pages (but see issue #1104).@@ -116,10 +151,9 @@ terms of the MIT license. A copy of the license can be found in the file#define MI_ENABLE_LARGE_PAGES 1#endif-// Place page meta info at the start of the page area or keep it separate?-// Separate keeps the page info at the arena start (default) which is more secure-// and reduces wasted space due to alignment and block sizes.-// (but also reserves more memory up front (about 2MiB per GiB))+// Place page meta info at the start of the page area or keep it separate?+// Separate keeps the page info at the arena start (default) which is more secure+// and reduces wasted space due to alignment and block sizes.#if !defined(MI_PAGE_META_IS_SEPARATED)#if MI_PAGE_MAP_FLAT#define MI_PAGE_META_IS_SEPARATED 0@@ -128,19 +162,38 @@ terms of the MIT license. A copy of the license can be found in the file#endif#endif+// We can choose to page meta info aligned at the start of every MI_PAGE_META_ALIGNED_CHUNKS+// This can be used to have a faster `mi_free(_small)` as we can avoid a page_map lookup.+// This only works if valid pointers are passed to `mi_free` though. However, checked+// free `mi_cfree` can still use the page map to validate pointers.+#if !MI_FREE_IS_CHECKED && !MI_FREE_USE_PAGEMAP+#if MI_PAGE_META_IS_SEPARATED+#define MI_PAGE_META_IS_ALIGNED 1+#define MI_PAGE_META_ALIGNED_CHUNKS MI_SIZE_SIZE+#else+#warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)"+#endif+#endif+// We can choose to only put page info of small pages at the start of the page area.// This can be used to have a slightly faster `mi_free_small` function for specialized// cases (like language runtime systems).-#if !defined(MI_PAGE_META_ALIGNED_FREE_SMALL)-#define MI_PAGE_META_ALIGNED_FREE_SMALL 0+#if !defined(MI_PAGE_META_SMALL_IS_ALIGNED)+#if defined(MI_OPT_FREE_SMALL) && MI_OPT_FREE_SMALL==0+#define MI_PAGE_META_SMALL_IS_ALIGNED 0+#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE && !MI_GUARDED // guarded can put small allocations into big blocks in a medium page (which would make `mi_free_small` fail)+#define MI_PAGE_META_SMALL_IS_ALIGNED 1+#else+#define MI_PAGE_META_SMALL_IS_ALIGNED 0+#endif#endif// Configuration checks#if !MI_PAGE_META_IS_SEPARATED && MI_SECURE#error "secure mode should use separated page infos"#endif-#if MI_PAGE_META_ALIGNED_FREE_SMALL && MI_SECURE-#error "secure mode cannot use MI_PAGE_META_ALIGNED_FREE_SMALL"+#if MI_PAGE_META_SMALL_IS_ALIGNED && MI_SECURE+#error "secure mode cannot use MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED)"#endif#if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT#error "cannot have a flat page map with separated page infos"@@ -149,7 +202,6 @@ terms of the MIT license. A copy of the license can be found in the file#warning "mimalloc assertions enabled in a release build"#endif-// --------------------------------------------------------------// Sizes of internal data-structures// (comments specify sizes on 64-bit, usually 32-bit is halved)@@ -159,14 +211,14 @@ terms of the MIT license. A copy of the license can be found in the file#ifndef MI_ARENA_SLICE_SHIFT#ifdef MI_SMALL_PAGE_SHIFT // backward compatibility#define MI_ARENA_SLICE_SHIFT MI_SMALL_PAGE_SHIFT- #elif MI_SECURE>=5 && __APPLE__ && MI_ARCH_ARM64+ #elif MI_SECURE>=5 && ((__APPLE__ && MI_ARCH_ARM64) || (defined(PAGE_SIZE) && PAGE_SIZE >= 16*MI_KiB))#define MI_ARENA_SLICE_SHIFT (17) // 128 KiB to not waste too much due to 16 KiB guard pages#else#define MI_ARENA_SLICE_SHIFT (13 + MI_SIZE_SHIFT) // 64 KiB (32 KiB on 32-bit)#endif#endif-#if MI_ARENA_SLICE_SHIFT < 12-#error Arena slices should be at least 4KiB+#if MI_ARENA_SLICE_SHIFT < 13+#error Arena slices should be at least 8KiB#endif#ifndef MI_BCHUNK_BITS_SHIFT@@ -182,6 +234,7 @@ terms of the MIT license. A copy of the license can be found in the file#define MI_BCHUNK_BITS (1 << MI_BCHUNK_BITS_SHIFT) // sub-bitmaps in arena's are "bchunks" of 512 bits#define MI_ARENA_SLICE_SIZE (MI_ZU(1) << MI_ARENA_SLICE_SHIFT) // arena's allocate in slices of 64 KiB#define MI_ARENA_SLICE_ALIGN (MI_ARENA_SLICE_SIZE)+#define MI_ARENA_CHUNK_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE)#define MI_ARENA_MIN_OBJ_SLICES (1)#define MI_ARENA_MAX_CHUNK_OBJ_SLICES (MI_BCHUNK_BITS) // 32 MiB (or 8 MiB on 32-bit)@@ -206,11 +259,18 @@ terms of the MIT license. A copy of the license can be found in the file#define MI_BIN_COUNT (MI_BIN_FULL+1)// We never allocate more than PTRDIFF_MAX (see also <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)-#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX+#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX-// Minimal commit for a page on-demand commit (should be >= OS page size)-#define MI_PAGE_MIN_COMMIT_SIZE MI_ARENA_SLICE_SIZE+// Minimal commit for a page on-demand commit+#define MI_PAGE_MIN_COMMIT_SIZE (16*MI_KiB) /* MI_ARENA_SLICE_SIZE */+#if MI_PAGE_META_IS_ALIGNED+#define MI_PAGE_META_ALIGNED_COUNT (MI_PAGE_META_ALIGNED_CHUNKS * MI_BCHUNK_BITS)+#define MI_PAGE_META_ALIGNMENT (MI_PAGE_META_ALIGNED_COUNT * MI_ARENA_SLICE_SIZE) // 256 MiB (32 MiB on 32-bit)+#define MI_ARENA_ALIGNMENT MI_PAGE_META_ALIGNMENT+#else+#define MI_ARENA_ALIGNMENT MI_ARENA_SLICE_ALIGN+#endif// ------------------------------------------------------// Arena's are large reserved areas of memory allocated from@@ -248,15 +308,14 @@ typedef struct mi_subproc_s mi_subproc_t;// Memory can reside in arena's, direct OS allocated, meta-data pages, or statically allocated.// The memid keeps track of this.typedef enum mi_memkind_e {- MI_MEM_NONE, // not allocated+ MI_MEM_NONE, // not allocated (or static)MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example)MI_MEM_STATIC, // allocated in a static area and should not be freed (the initial main theap data for example (`init.c`))- MI_MEM_META, // allocated with the meta data allocator (`arena-meta.c`)MI_MEM_OS, // allocated from the OSMI_MEM_OS_HUGE, // allocated as huge OS pages (usually 1GiB, pinned to physical memory)MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`)MI_MEM_ARENA, // allocated from an arena (the usual case) (`arena.c`)- MI_MEM_HEAP_MAIN // allocated in the main heap (for theaps)+ MI_MEM_MALLOC // allocated with mi_malloc} mi_memkind_t;static inline bool mi_memkind_is_os(mi_memkind_t memkind) {@@ -267,6 +326,7 @@ static inline bool mi_memkind_needs_no_free(mi_memkind_t memkind) {return (memkind <= MI_MEM_STATIC);}+typedef struct mi_meta_page_s mi_meta_page_t;typedef struct mi_memid_os_info {void* base; // actual base address of the block (used for offset aligned allocations)@@ -280,17 +340,16 @@ typedef struct mi_memid_arena_info {uint32_t slice_count; // allocated slices} mi_memid_arena_info_t;-typedef struct mi_memid_meta_info {- void* meta_page; // meta-page that contains the block- uint32_t block_index; // block index in the meta-data page- uint32_t block_count; // allocated blocks-} mi_memid_meta_info_t;+typedef struct mi_memid_malloc_info {+ void* base; // returned pointer+ size_t size; // allocated size+} mi_memid_malloc_info_t;typedef struct mi_memid_s {union {- mi_memid_os_info_t os; // only used for MI_MEM_OS- mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA- mi_memid_meta_info_t meta; // only used for MI_MEM_META+ mi_memid_os_info_t os; // only used for MI_MEM_OS(_HUGE/_REMAP)+ mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA+ mi_memid_malloc_info_t malloc; // only used for MI_MEM_MALLOC} mem;mi_memkind_t memkind;bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large (2Mib) or huge (1GiB) OS pages)@@ -298,6 +357,8 @@ typedef struct mi_memid_s {bool initially_zero; // `true` if the memory was originally zero initialized} mi_memid_t;+#define MI_MEMID_INIT(kind) {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ }+#define MI_MEMID_STATIC MI_MEMID_INIT(MI_MEM_STATIC)static inline bool mi_memid_is_os(mi_memid_t memid) {return mi_memkind_is_os(memid.memkind);@@ -335,6 +396,7 @@ typedef struct mi_block_s {#define MI_PAGE_IN_FULL_QUEUE MI_ZU(0x01)#define MI_PAGE_HAS_INTERIOR_POINTERS MI_ZU(0x02)#define MI_PAGE_FLAG_MASK MI_ZU(0x03)+#define MI_PAGE_FLAG_BITS (2)typedef size_t mi_page_flags_t;// There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue),@@ -342,7 +404,8 @@ typedef size_t mi_page_flags_t;// in an arena (in `mi_heap_t.arena_pages.pages_abandoned`) so these can be quickly found for reuse.// Abandoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked)#define MI_THREADID_ABANDONED MI_ZU(0)-#define MI_THREADID_ABANDONED_MAPPED (MI_PAGE_FLAG_MASK + 1)+#define MI_THREADID_ABANDONED_MAPPED (MI_ZU(1) << MI_PAGE_FLAG_BITS)+#define MI_THREADID_DETACHED (MI_ZU(2) << MI_PAGE_FLAG_BITS)// Thread free list.// Points to a list of blocks that are freed by other threads.@@ -351,6 +414,29 @@ typedef size_t mi_page_flags_t;// This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`.typedef uintptr_t mi_thread_free_t;+// We store the currently used block count together with the total malloc call count as 16-bit numbers.+// This is done for better codegen `mi_malloc/mi_free` (where we can increment both at once as `used_alloc += 0x10001` for example).+// The `last_used` is the `used` count since the statistics are last updated; on 32-bit platforms this+// is a separate field in `mi_page_t` but on 64-bit we use the upper 32-bits to store it.+// We need the `alloc_count` and `last_used` to efficiently calculate allocation and free statistics even+// in a release build; this way we can update the stats in the slow path (`_mi_page_update_stats`).+typedef union mi_used_s {+ size_t used_alloc; // used + alloc_count+ // the following struct is unused but nice for debugging+ struct {+ uint16_t used_count;+ uint16_t alloc_count;+ #if MI_SIZE_SIZE >= 8+ uint16_t last_used;+ uint16_t last_alloc;+ #endif+ } debug_le;+} mi_used_t;++static inline size_t mi_xused_used_count(mi_used_t xused) { return (xused.used_alloc & 0xFFFF); }+static inline size_t mi_xused_alloc_count(mi_used_t xused) { return ((xused.used_alloc>>16) & 0xFFFF); }+static inline mi_used_t mi_xused_used_reset(mi_used_t xused) { xused.used_alloc = xused.used_alloc & ~0xFFFF; return xused; }+// A page contains blocks of one specific size (`block_size`).// Each page has three list of free blocks:// `free` for blocks that can be allocated,@@ -376,44 +462,48 @@ typedef uintptr_t mi_thread_free_t;// that case the `xthreadid` is 0 or 4 (4 is for abandoned pages that// are in the `pages_abandoned` lists of an arena, these are called "mapped" abandoned pages).// - page flags are in the bottom 3 bits of `xthread_id` for the fast path in `mi_free`.-// - The layout is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc`-// - Using `uint16_t` does not seem to slow things down+// - The layout below is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc`-typedef struct mi_page_s {+typedef struct mi_page_s {+ #if (MI_PAGE_META_IS_ALIGNED)+ _Atomic(struct mi_page_s*) self; // points to the actual page info (for pages that span multiple slices)+ #endif_Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`)-mi_block_t* free; // list of available free blocks (`malloc` allocates from this list)- uint16_t used; // number of blocks in use (including blocks in `thread_free`)+ mi_used_t xused; // number of blocks in use (including blocks in `thread_free`) (and the allocated count for statistics)+ #if MI_SIZE_SIZE < 8+ uint16_t xlast_used; // for statistics; on 64-bit platforms it is in bits 32..47 of xused.+ uint16_t xlast_alloc; // for sampling; on 64-bit platforms it is in bits 48..63 of xused.+ #endif+ mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)++ size_t block_size; // const: size available in each block (always `>0`)+ size_t page_offset; // const: relative offset from the page to the start of the blocksuint16_t capacity; // number of blocks committeduint16_t reserved; // number of blocks reserved in memory+ uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already)uint8_t retire_expire; // expiration count for retired blocksbool free_is_zero; // `true` if the blocks in the free list are zero initialized-- mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)++ // next cache line_Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`)-- size_t block_size; // const: size available in each block (always `>0`)- uint8_t* page_start; // const: start of the blocks-- #if (MI_ENCODE_FREELIST || MI_PADDING)- uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary- #endif-mi_theap_t* theap; // the theap owning this page (may not be valid or NULL for abandoned pages)mi_heap_t* heap; // const: the heap owning this pagestruct mi_page_s* next; // next page owned by the theap with the same `block_size`struct mi_page_s* prev; // previous page owned by the theap with the same `block_size`- size_t slice_committed; // committed size relative to the first arena slice of the page data (or 0 if the page is fully committed already)mi_memid_t memid; // const: provenance of the page memory++ #if (MI_ENCODE_FREELIST || MI_PADDING)+ uintptr_t keys[MI_PAGE_KEY_COUNT]; // const: one or two random keys to encode the free lists (see `_mi_block_next`) or padding canary+ #endif} mi_page_t;-// ------------------------------------------------------// Object sizes// -------------------------------------------------------#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // pages must be aligned on this for the page map.+#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // page area's must be aligned on this for the page map.#define MI_PAGE_MIN_START_BLOCK_ALIGN MI_MAX_ALIGN_SIZE // minimal block alignment for the first block in a page (16b)#define MI_PAGE_MAX_START_BLOCK_ALIGN2 (4*MI_KiB) // maximal block alignment for "power of 2"-sized blocks (such that we guarantee natural alignment)#define MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4*MI_KiB) // also aligns any multiple of this size to avoid TLB misses.@@ -423,7 +513,7 @@ typedef struct mi_page_s {#define MI_SMALL_MAX_OBJ_SIZE ((MI_SMALL_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // = 10 KiB#if MI_ENABLE_LARGE_PAGES#define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // ~ 84 KiB-#define MI_LARGE_MAX_OBJ_SIZE (MI_LARGE_PAGE_SIZE/8) // <= 512 KiB // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`+#define MI_LARGE_MAX_OBJ_SIZE (MI_LARGE_PAGE_SIZE/8) // <= 512 KiB. note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`#else#define MI_MEDIUM_MAX_OBJ_SIZE (MI_MEDIUM_PAGE_SIZE/8) // <= 64 KiB#define MI_LARGE_MAX_OBJ_SIZE MI_MEDIUM_MAX_OBJ_SIZE // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`@@ -434,6 +524,23 @@ typedef struct mi_page_s {#error "mimalloc internal: define more bins"#endif+// static invariant: MI_MAX_SINGLETON_BIN >= _mi_bin(MI_LARGE_MAX_OBJ_SIZE) (See init.c for the size bins)+#if (MI_LARGE_MAX_OBJ_WSIZE <= 8192) // 64 KiB+#define MI_MAX_SINGLETON_BIN (48)+#elif (MI_LARGE_MAX_OBJ_WSIZE <= 32768) // 256KiB+#define MI_MAX_SINGLETON_BIN (56)+#elif (MI_LARGE_MAX_OBJ_WSIZE <= 65536) // 512KiB+#define MI_MAX_SINGLETON_BIN (60)+#else+#define MI_MAX_SINGLETON_BIN MI_BIN_HUGE+#endif++// The small object max size must be larger than twice the small size max minus one,+// such that any aligned allocation with `alignment <= size <= MI_SMALL_SIZE_MAX` will+// be allocated in a small page (so `mi_free_csize` can delegate correctly to `mi_free_small`.+#if MI_SMALL_MAX_OBJ_SIZE <= (2*(MI_SMALL_WSIZE_MAX * MI_SIZE_SIZE)-1)+#error "mimalloc internal: the small object max size is too small"+#endif// ------------------------------------------------------// Page kinds@@ -488,10 +595,10 @@ typedef struct mi_random_cxt_s {#if MI_PADDINGtypedef struct mi_padding_s {uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)- uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes)+ uint32_t delta; // padding bytes before the block. (mi_full_usable_size(p) - delta == exact allocated bytes)} mi_padding_t;#define MI_PADDING_SIZE (sizeof(mi_padding_t))-#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE)+#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_SIZE_SIZE - 1) / MI_SIZE_SIZE)#else#define MI_PADDING_SIZE 0#define MI_PADDING_WSIZE 0@@ -502,11 +609,34 @@ typedef struct mi_padding_s {// A thread-local heap ("theap") owns a set of thread-local pages.struct mi_theap_s {+ // put in front for fast small allocations+ mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.++ // less frequently accessed fieldsmi_tld_t* tld; // thread-local data_Atomic(mi_heap_t*) heap; // the heap this theap belongs to.- _Atomic(size_t) refcount; // reference count+ _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction)+ _Atomic(size_t) refcount; // reference count (needed for safe heap destroy)++ // config+ long page_full_retain; // how many full pages can be retained per queue (before abandoning them)+ bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages+ bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint+ bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED`++ // sampling+ size_t sample_countdown; // sample countdown in requested bytes (don't change the field order; see `internal.h:_mi_theap_get_free_small_page`)+ size_t sample_rate; // current sampling rate in requested bytes (or 0 to disable) (for profiling and guarded mode)+ uint64_t sample_requested; // total allocated/requested bytes since the last sample+ size_t profile_sample_rate; // sampling rate in requested bytes for profiling+ size_t profile_sample_countdown; // countdown in requested bytes for profiling+ size_t guarded_sample_rate; // sampling rate in requested bytes for guarded objects+ size_t guarded_sample_countdown; // countdown in requested bytes for guarded objects+ size_t guarded_size_min; // minimal size for guarded objects+ size_t guarded_size_max; // maximal size for guarded objects++ // statsunsigned long long heartbeat; // monotonic heartbeat count- uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`)mi_random_ctx_t random; // random number context used for secure allocationsize_t page_count; // total number of pages in the `pages` queues.size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues)@@ -515,21 +645,13 @@ struct mi_theap_s {long generic_count; // how often is `_mi_malloc_generic` called?long generic_collect_count; // how often is `_mi_malloc_generic` called without collecting?+ // theaps belong to heaps and threadsmi_theap_t* tnext; // list of theaps in this thread- mi_theap_t* tprev;+ mi_theap_t* tprev;mi_theap_t* hnext; // list of theaps of the owning `heap`mi_theap_t* hprev;-- long page_full_retain; // how many full pages can be retained per queue (before abandoning them)- bool allow_page_reclaim; // `true` if this theap should not reclaim abandoned pages- bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint- #if MI_GUARDED- size_t guarded_size_min; // minimal size for guarded objects- size_t guarded_size_max; // maximal size for guarded objects- size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages)- size_t guarded_sample_count; // current sample count (counting down to 0)- #endif- mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.++ // page queuesmi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or "bin")mi_memid_t memid; // provenance of the theap struct itself (meta or os)mi_stats_t stats; // thread-local statistics@@ -566,13 +688,15 @@ typedef struct mi_heap_s {mi_theap_t* theaps; // list of all thread-local theaps belonging to this heap (using the `hnext`/`hprev` fields)mi_lock_t theaps_lock; // lock for the theaps list operations+ _Atomic(mi_profiler_t*) profiler;+_Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // total count of abandoned pages in this heapmi_page_t* os_abandoned_pages; // list of pages that are OS allocated and not in an arenami_lock_t os_abandoned_pages_lock; // lock for the os abandoned pages list (this lock protects list operations)_Atomic(mi_arena_pages_t*) arena_pages[MI_MAX_ARENAS]; // track owned and abandoned pages in the arenas (entries can be NULL)mi_lock_t arena_pages_lock; // lock to update the arena_pages array-+ mi_memid_t memid; // provenance of the heap memorymi_stats_t stats; // statistics for this heap; periodically updated by merging from each theap} mi_heap_t;@@ -590,6 +714,7 @@ struct mi_subproc_s {size_t subproc_seq; // unique id for sub-processesmi_subproc_t* next; // list of all sub-processesmi_subproc_t* prev;+ _Atomic(mi_meta_page_t*) meta_pages; // meta data pages_Atomic(size_t) arena_count; // current count of arena's_Atomic(mi_arena_t*) arenas[MI_MAX_ARENAS]; // arena's of this sub-process@@ -597,16 +722,22 @@ struct mi_subproc_s {mi_decl_align(8) // needed on some 32-bit platforms_Atomic(int64_t) purge_expire; // expiration is set if any arenas can be purged- _Atomic(mi_heap_t*) heap_main; // main heap for this sub process+ _Atomic(mi_heap_t*) heap_main; // main heap for this sub processmi_heap_t* heaps; // heaps belonging to this sub-processmi_lock_t heaps_lock;+ mi_theap_t* theap_meta; // detached theap for allocating meta-data+ mi_lock_t theap_meta_lock; // all allocations in theap_meta need a lock+_Atomic(size_t) thread_count; // current threads associated with this sub-process_Atomic(size_t) thread_total_count; // total created threads associated with this sub-process_Atomic(size_t) heap_count; // current heaps in this sub-process (== |heaps|)_Atomic(size_t) heap_total_count; // total created heaps in this sub-process+ _Atomic(mi_profiler_t*) profiler;+mi_memid_t memid; // provenance of this memory block (meta or static)+ mi_subproc_t* parent; // subproc in which this one was allocatedmi_decl_align(8) // needed on some 32-bit platformsmi_stats_t stats; // subprocess statistics; updated for arena/OS stats like committed,// and otherwise merged with heap stats when those are deleted@@ -646,18 +777,18 @@ struct mi_tld_s {to reserve large arenas upfront and be able to reuse the memory more effectively.-----------------------------------------------------------------------------*/-#define MI_ARENA_BIN_COUNT (MI_BIN_COUNT)+#define MI_ARENA_BIN_COUNT (MI_MAX_SINGLETON_BIN+1)#define MI_ARENA_MIN_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) // 32 MiB (or 8 MiB on 32-bit)-#define MI_ARENA_MAX_SIZE (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE)+#define MI_ARENA_MAX_SIZE (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE) // 16 GiBtypedef struct mi_bitmap_s mi_bitmap_t; // atomic bitmap (defined in `src/bitmap.h`)typedef struct mi_bbitmap_s mi_bbitmap_t; // atomic binned bitmap (defined in `src/bitmap.h`)-typedef struct mi_arena_pages_s {+struct mi_arena_pages_s {mi_bitmap_t* pages; // all registered pages (abandoned and owned)mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT]; // abandoned pages per size bin (a set bit means the start of the page)// followed by the bitmaps (whose siz`es depend on the arena size)-} mi_arena_pages_t;+};// A memory arena@@ -665,6 +796,7 @@ typedef struct mi_arena_s {mi_memid_t memid; // provenance of the memory areami_subproc_t* subproc; // subprocess this arena belongs to (`this 'element-of' this->subproc->arenas`)size_t arena_idx; // index in the arenas array+ void* start; // actual start of the arena area (the arena_s info may come later due to guard pages etc.)size_t slice_count; // total size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`)size_t info_slices; // initial slices reserved for the arena bitmaps@@ -713,6 +845,10 @@ typedef struct mi_arena_s {#ifndef EOVERFLOW // count*size overflow#define EOVERFLOW (75)#endif+#ifndef ENOENT // environment variable not found+#define ENOENT (2)+#endif+/* -----------------------------------------------------------Debug constantsdiff --git a/vendored/mimalloc/src/alloc-aligned.c b/vendored/mimalloc/src/alloc-aligned.cindex 97dc6045b..51f29732c 100644--- a/vendored/mimalloc/src/alloc-aligned.c+++ b/vendored/mimalloc/src/alloc-aligned.c@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file#include "mimalloc.h"#include "mimalloc/internal.h"-#include "mimalloc/prim.h" // _mi_theap_default+#include "mimalloc/prim-tls.h" // _mi_theap_default#include <string.h> // memset@@ -18,7 +18,7 @@ terms of the MIT license. A copy of the license can be found in the filestatic bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) {// certain blocks are always allocated at a certain natural alignment.// (see also `arena.c:mi_arenas_page_alloc_fresh`).- mi_assert_internal(_mi_is_power_of_two(alignment) && (alignment > 0));+ mi_assert_internal(mi_alignment_is_valid(alignment));if (alignment > size) return false;const size_t bsize = mi_good_size(size);const bool ok = (bsize <= MI_PAGE_MAX_START_BLOCK_ALIGN2 && _mi_is_power_of_two(bsize)) || // power-of-two under N@@ -27,8 +27,8 @@ static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) {return ok;}-#if MI_GUARDED-static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero) mi_attr_noexcept {+#if 0+static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero, mi_page_t** ppage) mi_attr_noexcept {// use over allocation for guarded blocksl#if MI_THEAP_INITASNULLif mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); }@@ -39,7 +39,7 @@ static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap,return NULL;}const size_t oversize = size + alignment - 1;- void* const base = _mi_theap_malloc_guarded(theap, oversize, zero);+ void* const base = _mi_theap_malloc_guarded(theap, oversize, zero, ppage);if (base==NULL) return NULL;void* const p = _mi_align_up_ptr(base, alignment);mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size);@@ -47,32 +47,18 @@ static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap,mi_assert_internal(_mi_is_aligned(p, alignment));return p;}--static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) {- #if MI_THEAP_INITASNULL- if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); }- #endif- const size_t rate = theap->guarded_sample_rate;- // only write if `rate!=0` so we don't write to the constant `_mi_theap_empty`- if (rate != 0) { theap->guarded_sample_rate = 0; }- void* p = _mi_theap_malloc_zero(theap, size, zero, usable);- if (rate != 0) { theap->guarded_sample_rate = rate; }- return p;-}-#else-static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) {- return _mi_theap_malloc_zero(theap, size, zero, usable);-}#endif+// Fallback aligned allocation that over-allocates -- split out for better codegen-static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept+static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept{mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE));- mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));+ mi_assert_internal(mi_alignment_is_valid(alignment));void* p;size_t oversize;+ mi_page_t* page;if mi_unlikely(alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {// use OS allocation for large alignments and allocate inside a singleton page (not in an arena)// This can support alignments >= MI_PAGE_ALIGN by ensuring the object can be aligned@@ -84,7 +70,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap}oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size);// note: no guarded as alignment > 0- p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, usable); // the page block size should be large enough to align in the single huge page block+ p = _mi_theap_malloc_zero(theap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page blockif (p == NULL) return NULL;}else {@@ -92,9 +78,11 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theapmi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) && alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN);mi_assert_internal(size < SIZE_MAX - alignment); // `oversize` cannot overflowoversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and alignment 64k, we would allocate a 64k block and pointing just beyond that.- p = mi_theap_malloc_zero_no_guarded(theap, oversize, zero, usable);+ p = _mi_theap_malloc_zero(theap, oversize, zero, 0, &page);if (p == NULL) return NULL;}+ mi_assert_internal(page == _mi_ptr_page(p));+ if (ppage!=NULL) { *ppage = page; }// .. and align within the allocationconst uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)`@@ -107,17 +95,20 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap// and we no longer own it. We should be careful to only read constant fields in the page,// or use safe atomic access as in `mi_page_set_has_interior_pointers`.// (we can access the page though since the just allocated pointer keeps it alive)- mi_page_t* page = _mi_ptr_page(p);if (aligned_p != p) {mi_page_set_has_interior_pointers(page, true);- #if MI_GUARDED++ #if MI_GUARDED || MI_PROFILE// set tag to aligned so mi_usable_size works with guard pages+ // note: we could have gotten a profiled or guarded block but that is still ok+ // and in such case the MI_BLOCK_TAG_ALIGNED is just not at the start of the real heap blockif (adjust >= sizeof(mi_block_t)) {mi_block_t* const block = (mi_block_t*)p;block->next = MI_BLOCK_TAG_ALIGNED;}#endif- _mi_padding_shrink(page, (mi_block_t*)p, adjust + size);+ mi_block_t* const base = _mi_page_ptr_unalign(page, p);+ _mi_padding_shrink(page, base, adjust + size);}// todo: expand padding if overallocated ?@@ -127,22 +118,13 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theapmi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust);#if MI_DEBUG > 1mi_page_t* const apage = _mi_ptr_page(aligned_p);- void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p);- mi_assert_internal(p == unalign_p);+ mi_block_t* unalign_p = _mi_page_ptr_unalign(apage, aligned_p);+ mi_assert_internal(p == (void*)unalign_p || mi_block_ptr_is_sampled(unalign_p, aligned_p));#endif- // now zero the block if needed- //if (alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {- // // for the tracker, on huge aligned allocations only from the start of the large block is defined- // mi_track_mem_undefined(aligned_p, size);- // if (zero) {- // _mi_memzero_aligned(aligned_p, mi_usable_size(aligned_p));- // }- //}-if (p != aligned_p) {mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p));- #if MI_GUARDED+ #if MI_GUARDED || MI_PROFILEmi_track_mem_defined(p, sizeof(mi_block_t));#endif}@@ -150,85 +132,97 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap}// Generic primitive aligned allocation -- split out for better codegen-static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept+static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept{- mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));+ mi_assert_internal(mi_alignment_is_valid(alignment));// we don't allocate more than MI_MAX_ALLOC_SIZE (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) {- _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);+ _mi_error_message(EINVAL, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);return NULL;}// use regular allocation if it is guaranteed to fit the alignment constraints.// this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist// a page with the right block size, and if we always use the over-alloc fallback that would never happen.- if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) {- void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, usable);- mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);- const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0;- if mi_likely(is_aligned_or_null) {- return p;- }- else {- // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct..- mi_assert(false);- mi_free(p);+ #if MI_THEAP_INITASNULL+ if mi_likely(theap!=NULL)+ #endif+ {+ #if MI_SAMPLE // only try if we would not take a sample+ if mi_likely(!mi_theap_should_sample(theap,size))+ #endif+ {+ if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment))+ {+ mi_page_t* page = NULL;+ void* p = _mi_theap_malloc_zero(theap, size, zero, 0,&page);+ if (ppage!=NULL) { *ppage = page; }+ const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0;+ if mi_likely(is_aligned_or_null) {+ return p;+ }+ else {+ // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct..+ // (but it can happen if this allocation just so happens to be sampled but we checked for that)+ mi_assert_internal(false); // p==NULL || mi_block_ptr_is_profiled_or_guarded(_mi_page_ptr_unalign(page,p),p));+ mi_free(p);+ }+ }}}// fall back to over-allocation- return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,usable);+ return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,ppage);}+static mi_decl_cold mi_decl_noinline void* mi_error_bad_alignment(size_t size, size_t alignment, size_t offset) {+ _mi_error_message(EINVAL, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset);+ return NULL;+}+// Primitive aligned allocation-static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept+static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept{// note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size.- if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)- #if MI_DEBUG > 0- _mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\n", size, alignment);- #endif- return NULL;- }-- #if MI_GUARDED- #if MI_THEAP_INITASNULL- if mi_likely(theap!=NULL)- #endif- if (offset==0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN && mi_theap_malloc_use_guarded(theap,size)) {- return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero);+ if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two and multiple of void* (see <https://en.cppreference.com/w/c/memory/aligned_alloc#Notes>)+ return mi_error_bad_alignment(size, alignment, offset);}- #endif// try first if there happens to be a small block available with just the right alignment// since most small power-of-2 blocks (under MI_PAGE_MAX_BLOCK_START_ALIGN2) are already// naturally aligned this can be often the case.- #if MI_THEAP_INITASNULL- if mi_likely(theap!=NULL)- #endif+ if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size){- if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) {- const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`- const size_t padsize = size + MI_PADDING_SIZE;- mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize);- if mi_likely(page->free != NULL) {- const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0;- if mi_likely(is_aligned)- {- if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }- void* p = _mi_page_malloc_zero(theap, page, padsize, zero);- mi_assert_internal(p != NULL);- mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);- mi_track_malloc(p, size, zero);- return p;+ #if MI_THEAP_INITASNULL+ if mi_likely(theap!=NULL)+ #endif+ {+ #if MI_SAMPLE // check if we shouldn't take a sample+ if mi_likely(!mi_theap_should_sample(theap,size))+ #endif+ {+ const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`+ const size_t padsize = size + MI_PADDING_SIZE;+ mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize, false);+ if mi_likely(page->free != NULL) {+ const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0;+ if mi_likely(is_aligned)+ {+ if (ppage!=NULL) { *ppage = page; }+ void* p = _mi_page_malloc_zero(theap, page, padsize, zero); // theap!=NULL+ mi_assert_internal(p != NULL);+ mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);+ mi_track_malloc(p, size, zero);+ return p;+ }}}}}// fallback to generic aligned allocation- return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, usable);+ return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, ppage);}@@ -248,7 +242,7 @@ static mi_decl_restrict void* mi_theap_zalloc_aligned_at(mi_theap_t* theap, sizereturn mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, true, NULL);}-static mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept {+mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept {return mi_theap_zalloc_aligned_at(theap, size, alignment, 0);}@@ -275,8 +269,11 @@ mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t areturn mi_theap_malloc_aligned(_mi_theap_default(), size, alignment);}-mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {- return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, block_size);+mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept {+ mi_page_t* page;+ void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, &page);+ if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); }+ return p;}mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {@@ -287,8 +284,11 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t areturn mi_theap_zalloc_aligned(_mi_theap_default(), size, alignment);}-mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {- return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, block_size);+mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept {+ mi_page_t* page;+ void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, &page);+ if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); }+ return p;}mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {@@ -330,24 +330,30 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap,// ------------------------------------------------------static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept {- mi_assert(alignment > 0);- if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);+ mi_assert(mi_alignment_is_valid(alignment));+ if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)+ return mi_error_bad_alignment(newsize,alignment,offset);+ }+ if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero);if (p == NULL) return mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,zero,NULL);- size_t size = mi_usable_size(p);- if (newsize <= size && newsize >= (size - (size / 2))- && (((uintptr_t)p + offset) % alignment) == 0) {- return p; // reallocation still fits, is aligned and not more than 25% waste+ const size_t size = mi_usable_size(p);+ if (newsize <= size && newsize >= (size - (size / 2)) && (((uintptr_t)p + offset) & (alignment-1)) == 0) {+ return p; // reallocation still fits, is aligned and not more than 50% waste}else {- // note: we don't zero allocate upfront so we only zero initialize the expanded part- void* newp = mi_theap_malloc_aligned_at(theap,newsize,alignment,offset);+ // note: we don't zero allocate upfront so we only zero initialize the expanded part (at the cost of calling mi_usable_size)+ mi_page_t* newpage;+ void* const newp = mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,false/*zero?*/,&newpage);if (newp != NULL) {- if (zero && newsize > size) {+ const size_t copy_size = (newsize > size ? size : newsize);+ const size_t zero_start = (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized+ const size_t usable = _mi_page_usable_size(newpage,newp);+ mi_assert_internal(usable >= newsize); // use usable for zero'ing, issue #763+ if (zero && usable > zero_start) {// also set last word in the previous allocation to zero to ensure any padding is zero-initialized- size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);- _mi_memzero((uint8_t*)newp + start, newsize - start);+ _mi_memzero((uint8_t*)newp + zero_start, usable - zero_start);}- _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize));+ _mi_memcpy(newp, p, copy_size); // cannot be aligned due to abitrary offset... (todo: require offset to be a multiple of sizeof(void*)?)mi_free(p); // only free if successful}return newp;@@ -356,7 +362,7 @@ static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_tstatic void* mi_theap_realloc_zero_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept {mi_assert(alignment > 0);- if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);+ if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero);return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,0,zero);}diff --git a/vendored/mimalloc/src/alloc-override.c b/vendored/mimalloc/src/alloc-override.cindex 93d066dc1..bcb96b52d 100644--- a/vendored/mimalloc/src/alloc-override.c+++ b/vendored/mimalloc/src/alloc-override.c@@ -127,7 +127,7 @@ typedef void* mi_nothrow_t;#elif defined(_MSC_VER)_Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_HYBRIDPATCHABLEvoid* __cdecl _expand(_Pre_notnull_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) {- return mi_expand(_Block, _Size);+ return mi__expand(_Block, _Size);}_Check_return_ _ACRTIMPsize_t __cdecl _msize_base(_Pre_notnull_ void* _Block) _CRT_NOEXCEPT {@@ -293,7 +293,7 @@ typedef void* mi_nothrow_t;void _ZdlPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete(void*, std::align_val_t, std::nothrow_t const&)void _ZdaPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete[](void*, std::align_val_t, std::nothrow_t const&)- #if (MI_INTPTR_SIZE==8) || (MI_INTPTR_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257+ #if (MI_SIZE_SIZE==8) || (MI_SIZE_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bitvoid* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bitvoid* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }@@ -302,7 +302,7 @@ typedef void* mi_nothrow_t;void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }- #elif (MI_INTPTR_SIZE==4)+ #elif (MI_SIZE_SIZE==4)void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bitvoid* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bitvoid* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }@@ -324,14 +324,25 @@ typedef void* mi_nothrow_t;extern "C" {#endif+// defined here instead alloc-posix so we can alias it+mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept {+ if (!mi_is_in_heap_region(p)) return 0;+ return mi_usable_size(p);+}++mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {+ if (!mi_is_in_heap_region(p)) return 0;+ return mi_usable_size(p);+}+#ifndef MI_OSX_IS_INTERPOSED// Forward Posix/Unix calls as wellvoid* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize)- size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p)+ size_t malloc_size(const void* p) MI_FORWARD1(mi_malloc_size,p)#if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__)- size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p)+ size_t malloc_usable_size(void *p) MI_FORWARD1(mi_malloc_usable_size,p)#else- size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p)+ size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_malloc_usable_size,p)#endif// No forwarding here due to aliasing/name mangling issuesdiff --git a/vendored/mimalloc/src/alloc-posix.c b/vendored/mimalloc/src/alloc-posix.cindex 60639ff49..a22524942 100644--- a/vendored/mimalloc/src/alloc-posix.c+++ b/vendored/mimalloc/src/alloc-posix.c@@ -31,34 +31,16 @@ terms of the MIT license. A copy of the license can be found in the file#define ENOMEM 12#endif--mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept {- // if (!mi_is_in_heap_region(p)) return 0;- return mi_usable_size(p);-}--mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {- // if (!mi_is_in_heap_region(p)) return 0;- return mi_usable_size(p);-}-mi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept {return mi_good_size(size);}-void mi_cfree(void* p) mi_attr_noexcept {- if (mi_is_in_heap_region(p)) {- mi_free(p);- }-}--int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept {+int mi_posix_memalign(void** p, size_t alignment, size_t size) { // mi_attr_noexcept (issue #794)// Note: The spec dictates we should not modify `*p` on an error. (issue#27)// <http://man7.org/linux/man-pages/man3/posix_memalign.3.html>if (p == NULL) return EINVAL;- if ((alignment % sizeof(void*)) != 0) return EINVAL; // natural alignment- // it is also required that alignment is a power of 2 and > 0; this is checked in `mi_malloc_aligned`- if (alignment==0 || !_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2+ // it is required that alignment is a power of 2 and a multiple of sizeof(void*)+ if (alignment<sizeof(void*) || !_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2void* q = mi_malloc_aligned(size, alignment);if (q==NULL && size != 0) return ENOMEM;mi_assert_internal(_mi_is_aligned(q,alignment));@@ -122,7 +104,7 @@ mi_decl_nodiscard int mi_reallocarr( void* ptrp, size_t count, size_t size ) mi_}void** op = (void**)ptrp;if (total == 0) {- free(*op);+ mi_free(*op);*op = NULL;return 0;}diff --git a/vendored/mimalloc/src/alloc.c b/vendored/mimalloc/src/alloc.cindex aa556bd05..a176db77f 100644--- a/vendored/mimalloc/src/alloc.c+++ b/vendored/mimalloc/src/alloc.c@@ -1,6 +1,5 @@-/* -----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -12,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file#include "mimalloc.h"#include "mimalloc/internal.h"#include "mimalloc/atomic.h"-#include "mimalloc/prim.h" // _mi_prim_thread_id()+#include "mimalloc/prim-tls.h" // _mi_prim_thread_id()#include <string.h> // memset, strlen (for mi_strdup)#include <stdlib.h> // malloc, abort@@ -22,35 +21,83 @@ terms of the MIT license. A copy of the license can be found in the file#include "free.c"#undef MI_IN_ALLOC_C+// -------------------------------------------------------// Allocation+// Core allocation functions.+// Every allocation goes through `mi_page_malloc_zero`// ------------------------------------------------------+#if MI_PADDING+static mi_decl_noinline void mi_page_block_setup_padding(mi_page_t* page, mi_block_t* block, size_t size) mi_attr_noexcept {+ const size_t bsize = mi_page_usable_block_size(page);+ mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize);+ const ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE));+ mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta));+ mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess+ padding->canary = mi_ptr_encode_canary(page,block,page->keys);+ padding->delta = (uint32_t)(delta);+ #if MI_PADDING_CHECK_BYTES+ if (!mi_page_is_huge(page)) {+ uint8_t* fill = (uint8_t*)padding - delta;+ const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes+ for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; }+ }+ #endif+}+#endif+// Fast allocation in a page: just pop from the free list.// Fall back to generic allocation only if the list is empty.-// Note: in release mode the (inlined) routine is about 7 instructions with a single test.-static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero, size_t* usable) mi_attr_noexcept+// Note: even though there is a lot of checks etc in the source,+// in release mode the (inlined) routine is about 7 to 10 instructions with a single test.+static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, size_t sample_countdown, bool zero, mi_page_t** ppage) mi_attr_noexcept{if (page->block_size != 0) { // not the empty theapmi_assert_internal(mi_page_block_size(page) >= size);mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));}// check the free listmi_block_t* const block = page->free;- if mi_unlikely(block == NULL) {- return _mi_malloc_generic(theap, size, (zero ? 1 : 0), usable);+ mi_used_t xused = page->xused;+ xused.used_alloc += 0x10001; // increment both (16-bit) used count and alloc count+ #if defined(__GNUC__)+ __asm("" : : : "memory" ); // always load the `used` field before the test+ #endif+ if (block == NULL) {+ return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage);}mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);- if (usable != NULL) { *usable = mi_page_usable_block_size(page); };+ if (ppage != NULL) { *ppage = page; };// pop from the free list- page->free = mi_block_next(page, block);- page->used++;+ mi_block_t* next = mi_block_next(page,block);+ mi_track_mem_undefined(block,sizeof(*block));++ #if MI_SECURE+ if (!zero) block->next = 0; // don't leak internal data+ #endif++ page->free = next;+ page->xused = xused;mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page);mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE));+ #if MI_SAMPLE==2+ const size_t req_size = size - MI_PADDING_SIZE;+ mi_assert_internal(theap->sample_countdown == sample_countdown); // we pass it to improve codegen+ mi_assert_internal(theap->sample_countdown >= req_size);+ theap->sample_countdown = sample_countdown - req_size;+ #else+ MI_UNUSED(sample_countdown);+ #endif++ #if MI_STATS>=2+ mi_theap_stat_counter_increase(theap,malloc_requested,size - MI_PADDING_SIZE);+ #endif+#if MI_DEBUG>3if (page->free_is_zero && size > sizeof(*block)) {mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block)));@@ -63,90 +110,68 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_const size_t bsize = mi_page_usable_block_size(page);mi_track_mem_undefined(block, bsize);- #if (MI_STAT>0)- if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {- mi_theap_stat_increase(theap, malloc_normal, bsize);- #if (MI_STAT>1)- mi_theap_stat_counter_increase(theap, malloc_normal_count, 1);- const size_t bin = _mi_bin(bsize);- mi_theap_stat_increase(theap, malloc_bins[bin], 1);- mi_theap_stat_increase(theap, malloc_requested, size - MI_PADDING_SIZE);- #endif- }+ // track per-block statistics+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));++ // in debug mode initialize with 0xD0+ #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN+ if mi_likely(!zero && !mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); }#endif// zero the block? note: we need to zero the full block size (issue #63)- if mi_likely(!zero) {- // #if MI_SECURE- block->next = 0; // don't leak internal data- // #endif- #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN- if (!mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); }- #endif- }- else {+ if mi_unlikely(zero) {if (!page->free_is_zero) {- _mi_memzero_aligned(block, bsize);+ mi_assert_internal(bsize%MI_SIZE_SIZE == 0);+ #if !MI_PADDING+ return // use tail-call+ #endif+ _mi_memzero_block(block,bsize);}else {- block->next = 0;- mi_track_mem_defined(block, bsize);- }+ block->next = 0;+ mi_track_mem_defined(block, bsize);+ }}-++ // setup padding (must come after zero'ing)#if MI_PADDING // && !MI_TRACK_ENABLED- mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize);- ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE));- #if (MI_DEBUG>=2)- mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta));- #endif- mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess- padding->canary = mi_ptr_encode_canary(page,block,page->keys);- padding->delta = (uint32_t)(delta);- #if MI_PADDING_CHECK- if (!mi_page_is_huge(page)) {- uint8_t* fill = (uint8_t*)padding - delta;- const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes- for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; }- }- #endif+ mi_page_block_setup_padding(page,block,size);#endif- return block;+ return block;}// extra entries for improved efficiency in `alloc-aligned.c` (and in `page.c:mi_malloc_generic`.extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept {- return mi_page_malloc_zero(theap, page, size, zero, NULL);+ return mi_page_malloc_zero(theap, page, size, theap->sample_countdown, zero, NULL);}-#if MI_GUARDED-mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept;-#endif--// main allocation primitives for small and generic allocation-// internal small size allocation-static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept+// Internal small size allocation; assumes the `theap` is non-NULL.+// This looks up the page directly from a the direct page entries in the theap.+// The size can be in words (as a wsize); we assume this will be inlined for best code for `mi_wmalloc_small` and similar functions.+static mi_decl_forceinline mi_decl_restrict void* mi_theap_nonnull_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept{+ size_t size = (is_wsize ? xsize * MI_SIZE_SIZE : xsize);mi_assert(theap != NULL);mi_assert(size <= MI_SMALL_SIZE_MAX);#if MI_DEBUG- const uintptr_t tid = _mi_thread_id();- mi_assert(theap->tld->thread_id == 0 || theap->tld->thread_id == tid); // theaps are thread local+ mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local#endif- #if (MI_PADDING || MI_GUARDED)- if mi_unlikely(size == 0) { size = sizeof(void*); }+ #if (MI_PADDING) // || MI_GUARDED)+ if mi_unlikely(xsize == 0) { xsize = (is_wsize ? 1 : MI_SIZE_SIZE); size += MI_SIZE_SIZE; }#endif- #if MI_GUARDED- if mi_unlikely(mi_theap_malloc_use_guarded(theap,size)) {- return _mi_theap_malloc_guarded(theap, size, zero);- }++ // we only sample if fine-grained sampling is enabled (otherwise we sample in mi_malloc_generic)+ #if MI_SAMPLE==2+ if mi_unlikely(mi_theap_should_sample(theap,size)) { return _mi_theap_malloc_sampled(theap,size,zero,ppage); }#endif++ // get page in constant time+ mi_page_t* page = _mi_theap_get_free_small_page(theap, xsize + (is_wsize ? MI_PADDING_WSIZE : MI_PADDING_SIZE), is_wsize);- // get page in constant time, and allocate from it- mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE);- void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, usable);+ // and allocate+ void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, theap->sample_countdown, zero, ppage);mi_track_malloc(p,size,zero);#if MI_DEBUG>3@@ -157,23 +182,17 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonreturn p;}-// internal generic allocation-static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept+// Internal generic allocation.+// Goes to `page.c:_mi_malloc_generic` which handles large and aligned allocations.+// Every small allocation eventually goes through this generic allocation path if it cannot be satisfied from the small page cache.+static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept{- #if MI_GUARDED- #if MI_THEAP_INITASNULL- if (theap!=NULL)- #endif- if (huge_alignment==0 && mi_theap_malloc_use_guarded(theap, size)) {- return _mi_theap_malloc_guarded(theap, size, zero);- }- #endif#if !MI_THEAP_INITASNULLmi_assert(theap!=NULL);#endif- mi_assert(theap==NULL || theap->tld->thread_id == 0 || theap->tld->thread_id == _mi_thread_id()); // theaps are thread local+ mi_assert(mi_theap_matches_thread(theap)); // theaps are thread localmi_assert((huge_alignment & 1)==0);- void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, usable); // note: size can overflow but it is detected in malloc_generic+ void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, ppage); // note: size can overflow but it is detected in malloc_genericmi_track_malloc(p, size, zero);#if MI_DEBUG>3@@ -184,47 +203,102 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, sizereturn p;}-// internal small allocation-static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept {- #if MI_THEAP_INITASNULL+// ------------------------------------------------------------------------+// Small allocations: mi_malloc_small and variants (zalloc, wmalloc, wzalloc)+// ------------------------------------------------------------------------++// internal small allocation where the theap can be NULL on some platforms (Windows and macOS)+static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept {+ #if !MI_THEAP_INITASNULL+ return mi_theap_nonnull_xmalloc_small_zero(theap, xsize, is_wsize, zero, ppage);+ #elseif (theap!=NULL) {- return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);+ return mi_theap_nonnull_xmalloc_small_zero(theap, xsize, is_wsize, zero, ppage);}else {- return mi_theap_malloc_generic(theap, size, zero, 0, usable); // tailcall+ return mi_theap_malloc_generic(theap, (is_wsize ? xsize * MI_SIZE_SIZE : xsize), zero, 0, ppage); // tailcall}- #else- return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);#endif}--// allocate a small block+// Allocate a small block from a theap; these routines assume a non-NULL theap.mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {- return mi_theap_malloc_small_zero(theap, size, false, NULL);+ mi_assert(theap!=NULL);+ return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, false, NULL);+}++mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept {+ mi_assert(theap!=NULL);+ return mi_theap_nonnull_xmalloc_small_zero(theap, wsize, true, false, NULL);}+mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {+ mi_assert(theap!=NULL);+ return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, true, NULL);+}++mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept {+ mi_assert(theap!=NULL);+ return mi_theap_nonnull_xmalloc_small_zero(theap, wsize, true, true, NULL);+}++// Regular small allocation functions that use the default theap.mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept {- return mi_theap_malloc_small(_mi_theap_default(), size);+ return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, NULL);+}++mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept {+ return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, NULL);+}++mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept {+ return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, false, NULL );}+mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept {+ return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, true, NULL);+}+++mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {+ return mi_theap_malloc_small(_mi_heap_theap_get_or_init(heap), size);+}mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {- return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, false, NULL);+ // we could also use: mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(theap), size, false, NULL); }+ // but the following prevents using a stack frame. We use this to optimize some select functions only.+ mi_theap_t* const theap = _mi_heap_theap_cached(heap);+ if mi_likely(theap!=NULL) { return mi_theap_malloc_small(theap, size); }+ else { return mi_heap_init_malloc_small(heap, size); }}-// The main internal allocation functions-static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept {+mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {+ return mi_theap_zalloc_small(_mi_heap_theap_get_or_init(heap), size);+}+mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {+ // optimize: return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL);+ mi_theap_t* const theap = _mi_heap_theap_cached(heap);+ if mi_likely(theap!=NULL) { return mi_theap_zalloc_small(theap, size); }+ else { return mi_heap_init_zalloc_small(heap, size); }+}+++// ------------------------------------------------------------------------------+// Main allocation functions: mi_malloc and variants (zalloc, wmalloc, wzalloc)+// ------------------------------------------------------------------------------++mi_decl_nodiscard static mi_decl_forceinline mi_decl_restrict void* mi_theap_nonnull_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept {+ mi_assert(theap!=NULL);// fast path for small objectsif mi_likely(size <= MI_SMALL_SIZE_MAX) {mi_assert_internal(huge_alignment == 0);- return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);+ return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, zero, ppage);}else {- return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable);+ return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage);}}-extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept {+mi_decl_nodiscard extern mi_decl_forceinline mi_decl_restrict void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept {// fast path for small objects#if MI_THEAP_INITASNULLif mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX)@@ -233,59 +307,67 @@ extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, siz#endif{mi_assert_internal(huge_alignment == 0);- return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);+ return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, zero, ppage);}else {- return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable);+ return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage);}}-void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept {- return _mi_theap_malloc_zero_ex(theap, size, zero, 0, usable);+mi_decl_nodiscard static inline mi_decl_restrict void* mi_theap_nonnull_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {+ mi_assert(theap!=NULL);+ return mi_theap_nonnull_malloc_zero(theap, size, false, 0, NULL);}+mi_decl_nodiscard static inline mi_decl_restrict void* mi_theap_nonnull_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {+ mi_assert(theap!=NULL);+ return mi_theap_nonnull_malloc_zero(theap, size, true, 0, NULL);+}// Main allocation functions+// (note: to reduce variants, we only use nonnull theap variants for small allocations)mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {- return _mi_theap_malloc_zero(theap, size, false, NULL);-}--mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept {- return mi_theap_malloc(_mi_theap_default(), size);+ return _mi_theap_malloc_zero(theap, size, false, 0, NULL);}-mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {- return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL);+mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {+ return _mi_theap_malloc_zero(theap, size, true, 0, NULL);}--// zero initialized small block-mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept {- return mi_theap_malloc_small_zero(_mi_theap_default(), size, true, NULL);+mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept {+ return mi_theap_malloc(_mi_theap_default(), size);}-mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {- return mi_theap_malloc_small_zero(theap, size, true, NULL);+mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept {+ return mi_theap_zalloc(_mi_theap_default(), size);}-mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {- return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL);+mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {+ return mi_theap_nonnull_malloc(_mi_heap_theap_get_or_init(heap), size);}+mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {+ // return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL);+ mi_theap_t* const theap = _mi_heap_theap_cached(heap);+ if mi_likely(theap!=NULL) { return mi_theap_nonnull_malloc(theap, size); }+ else { return mi_heap_init_malloc(heap, size); }--mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {- return _mi_theap_malloc_zero(theap, size, true, NULL);}-mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept {- return _mi_theap_malloc_zero(_mi_theap_default(), size, true, NULL);+mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {+ return mi_theap_nonnull_zalloc(_mi_heap_theap_get_or_init(heap), size);}-mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {- return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL);+ // optimize: return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL);+ mi_theap_t* const theap = _mi_heap_theap_cached(heap);+ if mi_likely(theap!=NULL) { return mi_theap_nonnull_zalloc(theap, size); }+ else { return mi_heap_init_zalloc(heap, size); }}+// ----------------------------------------------------------+// mi_calloc variants+// ----------------------------------------------------------+mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept {size_t total;if (mi_count_size_overflow(count,size,&total)) return NULL;@@ -293,47 +375,64 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_}mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept {- return mi_theap_calloc(_mi_theap_default(),count,size);+ return mi_theap_calloc(_mi_theap_default(), count, size);}mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {- size_t total;- if (mi_count_size_overflow(count, size, &total)) return NULL;- return mi_heap_zalloc(heap, total);+ return mi_theap_calloc(_mi_heap_theap(heap), count, size);}+// ----------------------------------------------------------+// Usable block sizes+// ----------------------------------------------------------+// Return usable size-mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* usable) mi_attr_noexcept {- return mi_theap_malloc_small_zero(_mi_theap_default(), size, false, usable);+static void* mi_ublock_size( void* p, mi_page_t* page, size_t* pblock_size ) {+ mi_assert_internal(page == _mi_ptr_page(p));+ if (pblock_size!=NULL) {+ if (p!=NULL) { *pblock_size = mi_page_block_size(page); }+ }+ return p;+}++mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept {+ mi_page_t* page;+ void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, &page);+ return mi_ublock_size(p,page,pblock_size);}-mi_decl_nodiscard mi_decl_restrict void* mi_theap_umalloc(mi_theap_t* theap, size_t size, size_t* usable) mi_attr_noexcept {- return _mi_theap_malloc_zero_ex(theap, size, false, 0, usable);+mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept {+ mi_page_t* page;+ void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, &page);+ return mi_ublock_size(p,page,pblock_size);}-mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* usable) mi_attr_noexcept {- return mi_theap_umalloc(_mi_theap_default(), size, usable);+mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* pblock_size) mi_attr_noexcept {+ mi_page_t* page;+ void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, false, 0, &page);+ return mi_ublock_size(p,page,pblock_size);}-mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* usable) mi_attr_noexcept {- return _mi_theap_malloc_zero_ex(_mi_theap_default(), size, true, 0, usable);+mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* pblock_size) mi_attr_noexcept {+ mi_page_t* page;+ void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, true, 0, &page);+ return mi_ublock_size(p,page,pblock_size);}-mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* usable) mi_attr_noexcept {+mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* pblock_size) mi_attr_noexcept {size_t total;if (mi_count_size_overflow(count,size,&total)) return NULL;- return mi_uzalloc(total, usable);+ return mi_uzalloc(total, pblock_size);}+// ----------------------------------------------------------// Uninitialized `calloc`-static mi_decl_restrict void* mi_theap_mallocn(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept {- size_t total;- if (mi_count_size_overflow(count, size, &total)) return NULL;- return mi_theap_malloc(theap, total);-}+// ----------------------------------------------------------mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept {- return mi_theap_mallocn(_mi_theap_default(),count,size);+ size_t total;+ if (mi_count_size_overflow(count, size, &total)) return NULL;+ return mi_malloc(total);}mi_decl_nodiscard mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {@@ -342,6 +441,9 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_treturn mi_heap_malloc(heap, total);}+// ----------------------------------------------------------+// Reallocation+// ----------------------------------------------------------// Expand (or shrink) in place (or fail)void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {@@ -351,14 +453,14 @@ void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {return NULL;#elseif (p == NULL) return NULL;- const mi_page_t* const page = mi_validate_ptr_page(p,"mi_expand");- const size_t size = _mi_usable_size(p,page);+ const mi_page_t* const page = mi_ptr_page_validate(p,"mi_expand");+ const size_t size = _mi_page_usable_size(page,p);if (newsize > size) return NULL;return p; // it fits#endif}-void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {+static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept {// if p == NULL then behave as malloc.// else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)).// (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.)@@ -367,63 +469,80 @@ void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zeif (p==NULL) {page = NULL;size = 0;- if (usable_pre!=NULL) { *usable_pre = 0; }+ if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; }}- else {- page = mi_validate_ptr_page(p,"mi_realloc");- size = _mi_usable_size(p,page);- if (usable_pre!=NULL) { *usable_pre = mi_page_usable_block_size(page); }+ else {+ page = mi_ptr_page_validate(p,"mi_realloc");+ if mi_unlikely(page==NULL) { // invalid pointer+ if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; }+ if (pblock_size_post!=NULL) { *pblock_size_post = 0; }+ return NULL;+ }+ size = _mi_page_usable_size(page,p);+ if (pblock_size_pre!=NULL) { *pblock_size_pre = mi_page_block_size(page); }+ }+ // check if we can reuse the existing block+ if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)+ mi_assert_internal(page!=NULL); // note: page!=NULL (since if p==NULL, we have size=0 and size>=newsize>0+ #if MI_THEAP_INITASNULL+ if (theap!=NULL)+ #endif+ {+ if (mi_page_heap(page)==_mi_theap_heap_peek(theap)) { // and within the same heap+ mi_assert_internal(p!=NULL);+ // todo: do not track as the usable size is still the same in the free; adjust potential padding?+ // mi_track_resize(p,size,newsize)+ // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); }+ if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(page); }+ return p; // reallocation still fits and not more than 50% waste+ }+ }}- if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0 // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)- && mi_page_heap(page)==_mi_theap_heap(theap)) // and within the same heap- {- mi_assert_internal(p!=NULL);- // todo: do not track as the usable size is still the same in the free; adjust potential padding?- // mi_track_resize(p,size,newsize)- // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); }- if (usable_post!=NULL) { *usable_post = mi_page_usable_block_size(page); }- return p; // reallocation still fits and not more than 50% waste- }- void* newp = mi_theap_umalloc(theap,newsize,usable_post);+ // note: we don't zero allocate upfront so we only zero initialize the expanded part+ mi_page_t* newpage; // use block_size for zero-ing, issue #763+ void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,0,&newpage);if mi_likely(newp != NULL) {- if (zero && newsize > size) {- // also set last word in the previous allocation to zero to ensure any padding is zero-initialized- const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);- _mi_memzero((uint8_t*)newp + start, newsize - start);+ if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(newpage); }+ const size_t copy_size = (newsize > size ? size : newsize);+ const size_t zero_start = _mi_align_down( (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0), sizeof(intptr_t)); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized+ const size_t usable = _mi_page_usable_size(newpage,newp);+ mi_assert_internal(usable >= newsize);+ if (zero && usable > zero_start) {+ _mi_memzero_aligned((uint8_t*)newp + zero_start, usable - zero_start);}else if (newsize == 0) {((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725)}if mi_likely(p != NULL) {- const size_t copysize = (newsize > size ? size : newsize);- mi_track_mem_defined(p,copysize); // _mi_useable_size may be too large for byte precise memory tracking..- _mi_memcpy(newp, p, copysize);- mi_free(p); // only free the original pointer if successful // todo: optimize since page is known?+ mi_track_mem_defined(p,copy_size); // _mi_useable_size may be too large for byte precise memory tracking..+ _mi_memcpy_aligned(newp, p, copy_size);+ mi_free(p); // only free the original pointer if successful}}return newp;}-mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {- return _mi_theap_realloc_zero(theap, p, newsize, false, NULL, NULL);+void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept {+ return mi_theap_realloc_zero_ex(theap,p,newsize,zero,NULL,NULL);}-static void* mi_theap_reallocn(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept {- size_t total;- if (mi_count_size_overflow(count, size, &total)) return NULL;- return mi_theap_realloc(theap, p, total);-}---// Reallocate but free `p` on errors-static void* mi_theap_reallocf(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {- void* newp = mi_theap_realloc(theap, p, newsize);- if (newp==NULL && p!=NULL) mi_free(p);- return newp;+// for theap_realloc/rezalloc we allow theap==NULL to reduce variants+mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {+ if (p==NULL) {+ return mi_theap_malloc(theap,newsize);+ }+ else {+ return _mi_theap_realloc_zero(theap, p, newsize, false);+ }}-static void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {- return _mi_theap_realloc_zero(theap, p, newsize, true, NULL, NULL);+mi_decl_nodiscard void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {+ if (p==NULL) {+ return mi_theap_zalloc(theap,newsize);+ }+ else {+ return _mi_theap_realloc_zero(theap, p, newsize, true);+ }}static void* mi_theap_recalloc(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept {@@ -432,20 +551,27 @@ static void* mi_theap_recalloc(mi_theap_t* theap, void* p, size_t count, size_treturn mi_theap_rezalloc(theap, p, total);}-mi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept {return mi_theap_realloc(_mi_theap_default(),p,newsize);}mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept {- return mi_theap_reallocn(_mi_theap_default(),p,count,size);+ size_t total;+ if (mi_count_size_overflow(count, size, &total)) return NULL;+ return mi_theap_realloc(_mi_theap_default(),p,total);}-mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {- return _mi_theap_realloc_zero(_mi_theap_default(),p,newsize, false, usable_pre, usable_post);+mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept {+ return mi_theap_realloc_zero_ex(_mi_theap_default(),p,newsize, false, pblock_size_pre, pblock_size_post);}// Reallocate but free `p` on errors+static void* mi_theap_reallocf(mi_theap_t* xtheap, void* p, size_t newsize) mi_attr_noexcept {+ void* newp = mi_theap_realloc(xtheap, p, newsize);+ if (newp==NULL && p!=NULL) mi_free(p);+ return newp;+}+mi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept {return mi_theap_reallocf(_mi_theap_default(),p,newsize);}@@ -458,13 +584,14 @@ mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_return mi_theap_recalloc(_mi_theap_default(), p, count, size);}-mi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {return mi_theap_realloc(_mi_heap_theap(heap), p, newsize);}mi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {- return mi_theap_reallocn(_mi_heap_theap(heap), p, count, size);+ size_t total;+ if (mi_count_size_overflow(count, size, &total)) return NULL;+ return mi_theap_realloc(_mi_heap_theap(heap), p, total);}// Reallocate but free `p` on errors@@ -481,17 +608,16 @@ mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count,}-// ------------------------------------------------------// strdup, strndup, and realpath// ------------------------------------------------------// `strdup` using mi_malloc-mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* theap, const char* s) mi_attr_noexcept {+mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* xtheap, const char* s) mi_attr_noexcept {if (s == NULL) return NULL;size_t len = _mi_strlen(s);if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1- char* t = (char*)mi_theap_malloc(theap,len+1);+ char* t = (char*)mi_theap_malloc(xtheap,len+1);if (t == NULL) return NULL;_mi_memcpy(t, s, len);t[len] = 0;@@ -507,11 +633,11 @@ mi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const c}// `strndup` using mi_malloc-mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* theap, const char* s, size_t n) mi_attr_noexcept {+mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* xtheap, const char* s, size_t n) mi_attr_noexcept {if (s == NULL) return NULL;const size_t len = _mi_strnlen(s,n); // len <= nif (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1- char* t = (char*)mi_theap_malloc(theap, len+1);+ char* t = (char*)mi_theap_malloc(xtheap, len+1);if (t == NULL) return NULL;_mi_memcpy(t, s, len);t[len] = 0;@@ -533,21 +659,38 @@ mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const#define PATH_MAX MAX_PATH#endif-mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept {+mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* xtheap, const char* fname, char* resolved_name) mi_attr_noexcept {// todo: use GetFullPathNameW to allow longer file names+ if (fname==NULL || *fname==0) {+ errno = EINVAL;+ return NULL;+ }char buf[PATH_MAX];DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL);if (res == 0) {- errno = GetLastError(); return NULL;+ DWORD err = GetLastError();+ switch (err) {+ case ERROR_LOCK_VIOLATION:+ case ERROR_SHARING_VIOLATION:+ case ERROR_INVALID_ACCESS: errno = EACCES; break;+ case ERROR_INVALID_HANDLE:+ case ERROR_INVALID_FUNCTION: errno = EINVAL; break;+ case ERROR_PATH_NOT_FOUND: errno = ENOTDIR; break;+ case ERROR_FILE_NOT_FOUND: errno = ENOENT; break;+ case ERROR_NOT_ENOUGH_MEMORY: errno = ENOMEM; break;+ default: errno = EIO;+ }+ return NULL;}else if (res > PATH_MAX) {- errno = EINVAL; return NULL;+ errno = ENAMETOOLONG;+ return NULL;}else if (resolved_name != NULL) {return resolved_name;}else {- return mi_theap_strndup(theap, buf, PATH_MAX);+ return mi_theap_strndup(xtheap, buf, PATH_MAX);}}@@ -573,7 +716,7 @@ static size_t mi_path_max(void) {return pmax;}-char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept {+char* mi_theap_realpath(mi_theap_t* xtheap, const char* fname, char* resolved_name) mi_attr_noexcept {if (resolved_name != NULL) {return realpath(fname,resolved_name);}@@ -587,13 +730,13 @@ char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_namreturn result;*/const size_t n = mi_path_max();- char* const buf = (char*)mi_zalloc(n+1);+ char* const buf = (char*)mi_theap_zalloc(xtheap,n+1);if (buf == NULL) {errno = ENOMEM;return NULL;}char* rname = realpath(fname,buf);- char* result = mi_theap_strndup(theap,rname,n); // ok if `rname==NULL`+ char* result = mi_theap_strndup(xtheap,rname,n); // ok if `rname==NULL`mi_free(buf);return result;}@@ -615,8 +758,11 @@ The standard requires calling into `get_new_handler` andthrowing the bad_alloc exception on failure. If we compilewith a C++ compiler we can implement this precisely. If weuse a C compiler we cannot throw a `bad_alloc` exception-but we call `exit` instead (i.e. not returning).+but we call `abort` instead (i.e. not returning).+Also, the standard requires calling the new handler until+it returns false, but we limit the total calls.-------------------------------------------------------*/+#define MI_TRY_NEW_MAX (4)#ifdef __cplusplus#include <new>@@ -638,10 +784,19 @@ static bool mi_try_new_handler(bool nothrow) {#endifreturn false;}- else {+ else if (!nothrow) {h();return true;}+ else {+ try {+ h();+ }+ catch(...) { // swallow std::bad_alloc+ return false; // stop trying+ }+ return true;+ }}#elsetypedef void (*std_new_handler_t)(void);@@ -678,22 +833,18 @@ static bool mi_try_new_handler(bool nothrow) {static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, bool nothrow ) {void* p = NULL;- while(p == NULL && mi_try_new_handler(nothrow)) {+ for(int i = 0; i < MI_TRY_NEW_MAX && p == NULL && mi_try_new_handler(nothrow); i++) {+ if (size > MI_MAX_ALLOC_SIZE) return NULL; // call try_new_handler at least oncep = mi_theap_malloc(theap,size);}return p;}-static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) {- return mi_theap_try_new(_mi_theap_default(), size, nothrow);-}-static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow) {return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow);}--mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) {+mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) {void* p = mi_theap_malloc(theap,size);if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false);return p;@@ -709,8 +860,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, sizereturn p;}--mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) {+mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) {size_t total;if mi_unlikely(mi_count_size_overflow(count, size, &total)) {mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc@@ -730,42 +880,56 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, si}+mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_nothrow(mi_theap_t* theap, size_t size) mi_attr_noexcept {+ void* p = mi_theap_malloc(theap,size);+ if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, true);+ return p;+}+mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept {- void* p = mi_malloc(size);- if mi_unlikely(p == NULL) return mi_try_new(size, true);+ return mi_theap_alloc_new_nothrow(_mi_theap_default(), size);+}++static mi_decl_noinline void* mi_try_new_aligned(size_t size, size_t alignment, bool nothrow) {+ void* p = NULL;+ for(int i = 0; i < MI_TRY_NEW_MAX && p==NULL && mi_try_new_handler(nothrow); i++) {+ if (!mi_alignment_is_valid(alignment)) return NULL;+ p = mi_malloc_aligned(size,alignment);+ }return p;}mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) {- void* p;- do {- p = mi_malloc_aligned(size, alignment);- }- while(p == NULL && mi_try_new_handler(false));+ void* p = mi_malloc_aligned(size, alignment);+ if mi_unlikely(p==NULL) return mi_try_new_aligned(size,alignment,false);return p;}mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept {- void* p;- do {- p = mi_malloc_aligned(size, alignment);- }- while(p == NULL && mi_try_new_handler(true));+ void* p = mi_malloc_aligned(size, alignment);+ if mi_unlikely(p==NULL) return mi_try_new_aligned(size,alignment,true);return p;}+static mi_decl_noinline void* mi_try_new_realloc(void* p, size_t newsize) {+ void* q = NULL;+ for(int i = 0; i < MI_TRY_NEW_MAX && q==NULL && mi_try_new_handler(false); i++) {+ if (newsize > MI_MAX_ALLOC_SIZE) return NULL;+ q = mi_realloc(p,newsize);+ }+ return q;+}+mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) {- void* q;- do {- q = mi_realloc(p, newsize);- } while (q == NULL && mi_try_new_handler(false));+ void* q = mi_realloc(p, newsize);+ if (q == NULL) return mi_try_new_realloc(p,newsize);return q;}mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) {size_t total;if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) {- mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc+ mi_try_new_handler(false);return NULL;}else {@@ -773,93 +937,6 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) {}}-#if MI_GUARDED-// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true).-// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`)-// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size`--static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) {- // TODO: we can still make padding work by moving it out of the guard page area- mi_page_t* const page = _mi_ptr_page(block);- mi_page_set_has_interior_pointers(page, true);- block->next = MI_BLOCK_TAG_GUARDED;-- // set guard page at the end of the block- const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local`- const size_t os_page_size = _mi_os_page_size();- mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t));- if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) {- // should never happen- mi_free(block);- return NULL;- }- uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size;- // note: the alignment of the guard page relies on blocks being os_page_size aligned which- // is ensured in `mi_arena_page_alloc_fresh`.- mi_assert_internal(_mi_is_aligned(block, os_page_size));- mi_assert_internal(_mi_is_aligned(guard_page, os_page_size));- if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) {- const bool ok = _mi_os_protect(guard_page, os_page_size);- if mi_unlikely(!ok) {- _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size);- }- }- else {- _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size);- }-- // align pointer just in front of the guard page- size_t offset = block_size - os_page_size - obj_size;- mi_assert_internal(offset > sizeof(mi_block_t));- if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) {- // give up to place it right in front of the guard page if the offset is too large for unalignment- offset = MI_PAGE_MAX_OVERALLOC_ALIGN;- }- uint8_t* const p = (uint8_t*)block + offset;- mi_assert_internal(p == guard_page - obj_size);- mi_track_align(block, p, offset, obj_size);- mi_track_mem_defined(block, sizeof(mi_block_t));- return p;-}--mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept-{- // allocate multiple of page size ending in a guard page- // ensure minimal alignment requirement?- if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow- _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size);- return NULL;- }- const size_t os_page_size = _mi_os_page_size();- const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE));- const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE);- const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size);- mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, NULL);- if (block==NULL) return NULL;- void* const p = mi_block_ptr_set_guarded(block, obj_size);- if (p == NULL) return p;- if (zero) {- _mi_memzero_aligned(p,obj_size); // we have to zero here as padding might have written here (if the blocksize > reqsize + os_page_size)- }-- // stats- mi_track_malloc(p, obj_size, zero);- if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); }- mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1);- #if MI_STAT>1- // adjust stats to only count the allocated size of the block (and not the guard page)- mi_theap_stat_adjust_decrease(theap, malloc_requested, req_size);- mi_theap_stat_increase(theap, malloc_requested, size);- #endif- #if MI_DEBUG>3- if (zero) {- mi_assert_expensive(mi_mem_is_zero(p, size));- }- #endif- return p;-}-#endif-// ------------------------------------------------------// ensure explicit external inline definitions are emitted!// ------------------------------------------------------@@ -868,10 +945,11 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size,void* _mi_externs[] = {(void*)&_mi_page_malloc_zero,(void*)&_mi_theap_malloc_zero,- (void*)&_mi_theap_malloc_zero_ex,(void*)&mi_theap_malloc,(void*)&mi_theap_zalloc,(void*)&mi_theap_malloc_small,+ (void*)&mi_theap_zalloc_small,+ (void*)&mi_theap_calloc,(void*)&mi_malloc,(void*)&mi_malloc_small,(void*)&mi_zalloc,diff --git a/vendored/mimalloc/src/arena-meta.c b/vendored/mimalloc/src/arena-meta.cdeleted file mode 100644index cee38caa1..000000000--- a/vendored/mimalloc/src/arena-meta.c+++ /dev/null@@ -1,179 +0,0 @@-/* -----------------------------------------------------------------------------Copyright (c) 2019-2024, Microsoft Research, Daan Leijen-This is free software; you can redistribute it and/or modify it under the-terms of the MIT license. A copy of the license can be found in the file-"LICENSE" at the root of this distribution.------------------------------------------------------------------------------*/--/* ----------------------------------------------------------------------------- We have a special "mini" allocator just for allocation of meta-data like- the theap (`mi_theap_t`) or thread-local data (`mi_tld_t`).-- We reuse the bitmap of the arena's for allocation of 64b blocks inside- an arena slice (64KiB).- We always ensure that meta data is zero'd (we zero on `free`)------------------------------------------------------------------------------*/--#include "mimalloc.h"-#include "mimalloc/internal.h"-#include "bitmap.h"--/* ------------------------------------------------------------ Meta data allocation------------------------------------------------------------ */--#define MI_META_PAGE_SIZE MI_ARENA_SLICE_SIZE-#define MI_META_PAGE_ALIGN MI_ARENA_SLICE_ALIGN--// large enough such that META_MAX_SIZE > 4k (even on 32-bit)-#define MI_META_BLOCK_SIZE (1 << (16 - MI_BCHUNK_BITS_SHIFT)) // 128 on 64-bit-#define MI_META_BLOCK_ALIGN MI_META_BLOCK_SIZE-#define MI_META_BLOCKS_PER_PAGE (MI_META_PAGE_SIZE / MI_META_BLOCK_SIZE) // 512-#define MI_META_MAX_SIZE (MI_BCHUNK_SIZE * MI_META_BLOCK_SIZE)--#if MI_META_MAX_SIZE <= 4096-#error "max meta object size should be at least 4KiB"-#endif--typedef struct mi_meta_page_s {- _Atomic(struct mi_meta_page_s*) next; // a linked list of meta-data pages (never released)- mi_memid_t memid; // provenance of the meta-page memory itself- mi_bbitmap_t blocks_free; // a small bitmap with 1 bit per block.-} mi_meta_page_t;--static mi_decl_cache_align _Atomic(mi_meta_page_t*) mi_meta_pages = MI_ATOMIC_VAR_INIT(NULL);---#if MI_DEBUG > 1-static mi_meta_page_t* mi_meta_page_of_ptr(void* p, size_t* block_idx) {- mi_meta_page_t* mpage = (mi_meta_page_t*)((uint8_t*)_mi_align_down_ptr(p,MI_META_PAGE_ALIGN) + _mi_os_secure_guard_page_size());- if (block_idx != NULL) {- *block_idx = ((uint8_t*)p - (uint8_t*)mpage) / MI_META_BLOCK_SIZE;- }- return mpage;-}-#endif--static mi_meta_page_t* mi_meta_page_next( mi_meta_page_t* mpage ) {- return mi_atomic_load_ptr_acquire(mi_meta_page_t, &mpage->next);-}--static void* mi_meta_block_start( mi_meta_page_t* mpage, size_t block_idx ) {- mi_assert_internal(_mi_is_aligned((uint8_t*)mpage - _mi_os_secure_guard_page_size(), MI_META_PAGE_ALIGN));- mi_assert_internal(block_idx < MI_META_BLOCKS_PER_PAGE);- void* p = ((uint8_t*)mpage - _mi_os_secure_guard_page_size() + (block_idx * MI_META_BLOCK_SIZE));- mi_assert_internal(mpage == mi_meta_page_of_ptr(p,NULL));- return p;-}--// allocate a fresh meta page and add it to the global list.-static mi_meta_page_t* mi_meta_page_zalloc(void) {- // allocate a fresh arena slice- // note: careful with _mi_subproc as it may recurse into mi_tld and meta_page_zalloc again.. (same with _mi_os_numa_node()...)- mi_memid_t memid;- uint8_t* base = (uint8_t*)_mi_arenas_alloc_aligned(mi_heap_main(), MI_META_PAGE_SIZE, MI_META_PAGE_ALIGN, 0,- true /* commit*/, (MI_SECURE==0) /* allow large? */,- NULL /* req arena */, 0 /* thread_seq */, -1 /* numa node */, &memid);- if (base == NULL) return NULL;- mi_assert_internal(_mi_is_aligned(base,MI_META_PAGE_ALIGN));- if (!memid.initially_zero) {- _mi_memzero_aligned(base, MI_ARENA_SLICE_SIZE);- }-- // guard pages- #if MI_SECURE >= 1- _mi_os_secure_guard_page_set_at(base, memid);- _mi_os_secure_guard_page_set_before(base + MI_META_PAGE_SIZE, memid);- #endif-- // initialize the page and free block bitmap- mi_meta_page_t* mpage = (mi_meta_page_t*)(base + _mi_os_secure_guard_page_size());- mpage->memid = memid;- mi_bbitmap_init(&mpage->blocks_free, MI_META_BLOCKS_PER_PAGE, true /* already_zero */);- const size_t mpage_size = offsetof(mi_meta_page_t,blocks_free) + mi_bbitmap_size(MI_META_BLOCKS_PER_PAGE, NULL);- const size_t info_blocks = _mi_divide_up(mpage_size,MI_META_BLOCK_SIZE);- const size_t guard_blocks = _mi_divide_up(_mi_os_secure_guard_page_size(), MI_META_BLOCK_SIZE);- mi_assert_internal(info_blocks + 2*guard_blocks < MI_META_BLOCKS_PER_PAGE);- mi_bbitmap_unsafe_setN(&mpage->blocks_free, info_blocks + guard_blocks, MI_META_BLOCKS_PER_PAGE - info_blocks - 2*guard_blocks);-- // push atomically in front of the meta page list- // (note: there is no ABA issue since we never free meta-pages)- mi_meta_page_t* old = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages);- do {- mi_atomic_store_ptr_release(mi_meta_page_t, &mpage->next, old);- } while(!mi_atomic_cas_ptr_weak_acq_rel(mi_meta_page_t,&mi_meta_pages,&old,mpage));- return mpage;-}---// allocate meta-data-mi_decl_noinline void* _mi_meta_zalloc( size_t size, mi_memid_t* pmemid )-{- mi_assert_internal(pmemid != NULL);- size = _mi_align_up(size,MI_META_BLOCK_SIZE);- if (size == 0 || size > MI_META_MAX_SIZE) return NULL;- const size_t block_count = _mi_divide_up(size,MI_META_BLOCK_SIZE);- mi_assert_internal(block_count > 0 && block_count < MI_BCHUNK_BITS);- mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages);- mi_meta_page_t* mpage = mpage0;- while (mpage != NULL) {- size_t block_idx;- if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {- // found and claimed `block_count` blocks- *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);- return mi_meta_block_start(mpage,block_idx);- }- else {- mpage = mi_meta_page_next(mpage);- }- }- // failed to find space in existing pages- if (mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages) != mpage0) {- // the page list was updated by another thread in the meantime, retry- return _mi_meta_zalloc(size,pmemid);- }- // otherwise, allocate a fresh metapage and try once more- mpage = mi_meta_page_zalloc();- if (mpage != NULL) {- size_t block_idx;- if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {- // found and claimed `block_count` blocks- *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);- return mi_meta_block_start(mpage,block_idx);- }- }- // if all this failed, allocate from the OS- return _mi_os_alloc(size, pmemid);-}--// free meta-data-mi_decl_noinline void _mi_meta_free(void* p, size_t size, mi_memid_t memid) {- if (p==NULL) return;- if (memid.memkind == MI_MEM_META) {- mi_assert_internal(_mi_divide_up(size, MI_META_BLOCK_SIZE) == memid.mem.meta.block_count);- const size_t block_count = memid.mem.meta.block_count;- const size_t block_idx = memid.mem.meta.block_index;- mi_meta_page_t* mpage = (mi_meta_page_t*)memid.mem.meta.meta_page;- mi_assert_internal(mi_meta_page_of_ptr(p,NULL) == mpage);- mi_assert_internal(block_idx + block_count <= MI_META_BLOCKS_PER_PAGE);- mi_assert_internal(mi_bbitmap_is_clearN(&mpage->blocks_free, block_idx, block_count));- // we zero on free (and on the initial page allocation) so we don't need a "dirty" map- _mi_memzero_aligned(mi_meta_block_start(mpage, block_idx), block_count*MI_META_BLOCK_SIZE);- mi_bbitmap_setN(&mpage->blocks_free, block_idx, block_count);- }- else {- _mi_arenas_free(p,size,memid);- }-}--// used for debug output-bool _mi_meta_is_meta_page(void* p)-{- mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t, &mi_meta_pages);- mi_meta_page_t* mpage = mpage0;- while (mpage != NULL) {- if ((void*)mpage == p) return true;- mpage = mi_meta_page_next(mpage);- }- return false;-}diff --git a/vendored/mimalloc/src/arena.c b/vendored/mimalloc/src/arena.cindex a5686d554..1b7c8cb33 100644--- a/vendored/mimalloc/src/arena.c+++ b/vendored/mimalloc/src/arena.c@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2019-2025, Microsoft Research, Daan Leijen+Copyright (c) 2019-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -21,9 +21,10 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo#include "mimalloc.h"#include "mimalloc/internal.h"-#include "mimalloc/prim.h"+#include "mimalloc/prim-tls.h"#include "bitmap.h"+/* -----------------------------------------------------------Arena id's----------------------------------------------------------- */@@ -86,36 +87,56 @@ static bool mi_heap_has_page(mi_heap_t* heap, mi_arena_t* arena, mi_page_t* page#endifsize_t mi_arena_min_alignment(void) {- return MI_ARENA_SLICE_ALIGN;+ return MI_ARENA_ALIGNMENT;}size_t mi_arena_min_size(void) {return MI_ARENA_MIN_SIZE;}-static size_t mi_arena_max_object_size(void) {+// slices reserved for page meta info at the start of aligned chunks+static size_t mi_arena_page_meta_aligned_slice_count(void) {+ #if MI_PAGE_META_IS_ALIGNED+ return _mi_divide_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE);+ #else+ return 0;+ #endif+}++// fixed limit for the maximum object size in an arena+static size_t mi_arena_max_fixed_object_size(void) {+ #if MI_PAGE_META_IS_ALIGNED+ return (MI_PAGE_META_ALIGNMENT - _mi_align_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE));+ #else+ return (MI_ARENA_MAX_SIZE - MI_ARENA_CHUNK_SIZE); // minus an initial chunk to accommodate meta info+ #endif+}++// Maximum object size allowed to be allocated in an arena+size_t mi_arena_max_object_size(void) {size_t max_size = mi_option_get_size(mi_option_arena_max_object_size);max_size = _mi_align_up(max_size, MI_ARENA_SLICE_SIZE);if (max_size <= MI_ARENA_MIN_OBJ_SIZE) {return MI_ARENA_MIN_OBJ_SIZE;}- else if (max_size >= MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE) { // minus a bchunk to accommodate meta info- return (MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE);+ else if (max_size >= mi_arena_max_fixed_object_size()) {+ return mi_arena_max_fixed_object_size();}else {return max_size;}}-mi_decl_nodiscard static bool mi_arena_commit(mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t already_committed) {+mi_decl_nodiscard static bool mi_arena_commit(mi_subproc_t* subproc, mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t stat_already_committed) {+ mi_assert_internal(subproc!=NULL);if (arena != NULL && arena->commit_fun != NULL) {return (*arena->commit_fun)(true, start, size, is_zero, arena->commit_fun_arg);}- else if (already_committed > 0) {- return _mi_os_commit_ex(start, size, is_zero, already_committed);+ else if (stat_already_committed > 0) {+ return _mi_os_commit_ex(subproc, start, size, is_zero, stat_already_committed);}else {- return _mi_os_commit(start, size, is_zero);+ return _mi_os_commit(subproc, start, size, is_zero);}}@@ -133,7 +154,7 @@ static size_t mi_arena_size(mi_arena_t* arena) {// Start of the arena memory areastatic uint8_t* mi_arena_start(mi_arena_t* arena) {- return ((uint8_t*)arena);+ return ((uint8_t*)arena->start);}// Start of a slice@@ -144,17 +165,22 @@ uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) {mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) {mi_assert_internal(slice_index < arena->slice_count);+ #if MI_PAGE_META_IS_ALIGNED+ mi_page_t* const page = _mi_aligned_ptr_page(mi_arena_slice_start(arena,slice_index)); // todo: optimize?+ return page;+ #elseif (arena->pages_meta != NULL) {mi_page_t* const page = &arena->pages_meta[slice_index];- #if MI_PAGE_META_ALIGNED_FREE_SMALL+ #if MI_PAGE_META_SMALL_IS_ALIGNED// pages with small blocks still have the page at the start of the slice (and set the `block_size` in pages_meta to 0)if (page->block_size>0) return page;#elsereturn page;#endif- }- // fall through (for MI_PAGE_META_ALIGNED_FREE_SMALL)+ }+ // fall through (for MI_PAGE_META_SMALL_IS_ALIGNED)return (mi_page_t*)mi_arena_slice_start(arena,slice_index);+ #endif}// Arena area@@ -207,7 +233,6 @@ static size_t mi_page_full_size(mi_page_t* page) {}}-/* -----------------------------------------------------------Arena Allocation----------------------------------------------------------- */@@ -215,6 +240,8 @@ static size_t mi_page_full_size(mi_page_t* page) {static mi_decl_noinline void* mi_arena_try_alloc_at(mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid){+ mi_assert_internal(arena!=NULL);+ mi_assert_internal(slice_count>0);size_t slice_index;if (!mi_bbitmap_try_find_and_clearN(arena->slices_free, tseq, slice_count, &slice_index)) return NULL;@@ -231,6 +258,10 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(mi_assert_internal(already_dirty <= touched_slices);touched_slices -= already_dirty;}+ else {+ // todo: properly count touched pages with a separate bitmap?+ touched_slices = 0;+ }// set commit stateif (commit) {@@ -239,7 +270,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(if (already_committed < slice_count) {// not all committed, try to commit nowbool commit_zero = false;- if (!mi_arena_commit(arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed))) {+ if (!mi_arena_commit(arena->subproc, arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(already_committed))) {// if the commit fails, release ownership, and return NULL;// note: this does not roll back dirty bits but that is ok.mi_bbitmap_setN(arena->slices_free, slice_index, slice_count);@@ -264,7 +295,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(}else {// already fully committed.- _mi_os_reuse(p, mi_size_of_slices(slice_count));+ _mi_os_reuse(arena->subproc, p, mi_size_of_slices(slice_count));// if the OS has overcommit, and this is the first time we access these pages, then// count the commit now (as at arena reserve we didn't count those commits as these are on-demand)if (_mi_os_has_overcommit() && touched_slices > 0 && !arena->memid.is_pinned /* huge pages, issue #1236 */) {@@ -424,24 +455,21 @@ static size_t mi_arena_start_idx(mi_heap_t* heap, size_t tseq, size_t arena_cyclconst size_t _arena_count = mi_arenas_get_count(heap->subproc); \const size_t _arena_cycle = (_arena_count == 0 ? 0 : _arena_count - 1); /* first search the arenas below the last one */ \/* always start searching in the arena's below the max */ \- const size_t _start = mi_arena_start_idx(heap,tseq,_arena_cycle); \+ const size_t _start = (req_arena==NULL ? mi_arena_start_idx(heap,tseq,_arena_cycle) : ((mi_arena_t*)req_arena)->arena_idx); \+ mi_assert_internal(_start <= _arena_count); \for (size_t _i = 0; _i < _arena_count; _i++) { \mi_arena_t* name_arena; \- if (req_arena != NULL) { \- name_arena = req_arena; /* if there is a specific req_arena, only search that one */\- if (_i > 0) break; /* only once */ \+ size_t _idx; \+ if (_i < _arena_cycle) { \+ _idx = _i + _start; \+ if (_idx >= _arena_cycle) { _idx -= _arena_cycle; } /* adjust so we rotate through the cycle */ \} \else { \- size_t _idx; \- if (_i < _arena_cycle) { \- _idx = _i + _start; \- if (_idx >= _arena_cycle) { _idx -= _arena_cycle; } /* adjust so we rotate through the cycle */ \- } \- else { \- _idx = _i; /* remaining arena's after the cycle */ \- } \- name_arena = mi_arena_from_index(heap->subproc,_idx); \+ _idx = _i; /* remaining arena's after the cycle */ \} \+ name_arena = mi_arena_from_index(heap->subproc,_idx); \+ if (req_arena != NULL && name_arena != req_arena && \+ (name_arena == NULL || name_arena->parent != req_arena)) continue; /* only the requested arena or its children */ \if (name_arena != NULL) \{@@ -512,7 +540,7 @@ static mi_decl_noinline void* mi_arenas_try_alloc(if (req_arena != NULL) return NULL;// don't create arena's while preloading (todo: or should we?)- if (_mi_preloading()) return NULL;+ // if (_mi_preloading()) return NULL;// don't create arena's if OS allocation is disallowedif (mi_option_is_enabled(mi_option_disallow_os_alloc)) return NULL;@@ -540,6 +568,7 @@ static mi_decl_noinline void* mi_arenas_try_alloc(// Allocate from the OS (if allowed)static void* mi_arena_os_alloc_aligned(+ mi_subproc_t* subproc,size_t size, size_t alignment, size_t align_offset,bool commit, bool allow_large,mi_arena_id_t req_arena_id, mi_memid_t* memid)@@ -551,10 +580,10 @@ static void* mi_arena_os_alloc_aligned(}if (align_offset > 0) {- return _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, memid);+ return _mi_os_alloc_aligned_at_offset(subproc, size, alignment, align_offset, commit, allow_large, memid);}else {- return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);+ return _mi_os_alloc_aligned(subproc, size, alignment, commit, allow_large, memid);}}@@ -579,7 +608,7 @@ void* _mi_arenas_alloc_aligned( mi_heap_t* heap,}// fall back to the OS- void* p = mi_arena_os_alloc_aligned(size, alignment, align_offset, commit, allow_large, req_arena, memid);+ void* p = mi_arena_os_alloc_aligned(heap->subproc, size, alignment, align_offset, commit, allow_large, req_arena, memid);return p;}@@ -596,10 +625,10 @@ void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_lar// release ownership of a page. This may free the page if all blocks were concurrently// freed in the meantime. Returns true if the page was freed.-static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theap) {+static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theapx) {mi_assert_internal(mi_page_is_owned(page));mi_assert_internal(mi_page_is_abandoned(page));- mi_assert_internal(_mi_thread_id()==current_theap->tld->thread_id);+ mi_assert_internal(mi_theap_matches_thread(current_theapx));mi_thread_free_t tf_new;mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free);do {@@ -607,14 +636,14 @@ static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theap)while mi_unlikely(mi_tf_block(tf_old) != NULL) {_mi_page_free_collect(page, false); // update usedif (mi_page_all_free(page)) { // it may become free just before unowning it- _mi_arenas_page_unabandon(page, current_theap);- _mi_arenas_page_free(page, current_theap);+ _mi_arenas_page_unabandon(page, current_theapx);+ _mi_arenas_page_free(page, current_theapx);return true;}tf_old = mi_atomic_load_relaxed(&page->xthread_free);}mi_assert_internal(mi_tf_block(tf_old)==NULL);- tf_new = mi_tf_create(NULL, false);+ tf_new = mi_tf_create(NULL, false);} while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new));return false;}@@ -655,7 +684,8 @@ static mi_arena_t* mi_page_arena_pages(mi_page_t* page, size_t* slice_index, sizmi_arena_t* const arena = mi_arena_from_memid(page->memid, slice_index, slice_count);mi_assert_internal(arena != NULL);if (parena_pages != NULL) {- mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(mi_page_heap(page), arena);+ mi_heap_t* heap = mi_page_heap(page);+ mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena);mi_assert_internal(arena_pages != NULL);mi_assert_internal(slice_index==NULL || mi_bitmap_is_set(arena_pages->pages, *slice_index));*parena_pages = arena_pages;@@ -697,7 +727,9 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t slMI_UNUSED(slice_count);const size_t bin = _mi_bin(block_size);- mi_assert_internal(bin < MI_BIN_COUNT);+ if (bin >= MI_ARENA_BIN_COUNT) {+ return NULL; // singleton page size+ }// any abandoned in our size class?mi_assert_internal(heap != NULL);@@ -729,7 +761,7 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl_mi_page_free_collect(page, false); // update `used` countmi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));- mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));+ mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);@@ -743,8 +775,11 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t slreturn NULL;}-static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid) {- MI_UNUSED_RELEASE(block_size);+static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t max_page_meta_count, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid, mi_arena_pages_t** parena_pages ) {+ MI_UNUSED(max_page_meta_count);+ mi_assert_internal(parena_pages!=NULL);++ *parena_pages = NULL;const bool allow_large = (MI_SECURE < 5); // 5 = guard page at end of each arena pageconst size_t page_alignment = MI_ARENA_SLICE_ALIGN;@@ -764,82 +799,173 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_start = (uint8_t*)mi_arenas_try_alloc(heap, slice_count, page_alignment, commit, allow_large, req_arena, tld->thread_seq, numa_node, memid);if (start != NULL) {mi_arena_pages_t* const arena_pages = mi_heap_ensure_arena_pages(heap, memid->mem.arena.arena);+ *parena_pages = arena_pages;if (arena_pages==NULL) {- _mi_arenas_free(start, mi_size_of_slices(slice_count), *memid); // roll back+ _mi_arenas_free(heap->subproc, start, mi_size_of_slices(slice_count), *memid); // roll backstart = NULL;}else {// note: the following assert should hold if we could check it atomically, but in a concurrent setting we may already allocate in slice_count// mi_assert_internal(mi_bitmap_is_clearN(arena_pages->pages, memid->mem.arena.slice_index, memid->mem.arena.slice_count));mi_assert_internal(mi_bitmap_is_clear(arena_pages->pages, memid->mem.arena.slice_index));- mi_bitmap_set(arena_pages->pages, memid->mem.arena.slice_index);+ // don't set yet: mi_bitmap_set(arena_pages->pages, memid->mem.arena.slice_index);}}}// otherwise fall back to the OSif (start == NULL) {+ #if MI_PAGE_META_IS_ALIGNED+ size_t page_offset; // offset in the block for the page area+ uint8_t* os_start;+ if (block_alignment < MI_PAGE_META_ALIGNMENT) {+ page_offset = (block_alignment < MI_PAGE_ALIGN ? MI_PAGE_ALIGN : block_alignment);+ os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, MI_PAGE_META_ALIGNMENT, 0 /* align offset */, false /* commit */, false /* allow large */, req_arena, memid);+ }+ else {+ // if we allow alignment >= MI_PAGE_META_ALIGNMENT we need to substract 1 from a pointer+ // in _mi_aligned_ptr_page (and test for (intptr_t)p < 0 instead of NULL). We avoid this by limiting the max alignment.+ _mi_warning_message("requested alignment is too large (%zu KiB)\n", block_alignment / MI_KiB);+ errno = EINVAL;+ return NULL;+ // page_offset = MI_PAGE_META_ALIGNMENT;+ // os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, block_alignment, MI_PAGE_META_ALIGNMENT /* align offset */, false /* commit */, false /* allow large */, req_arena, memid);+ }+ if (os_start==NULL) return NULL;+ // commit page info and page+ const size_t min_page_count = _mi_divide_up(page_offset + MI_ARENA_SLICE_SIZE,MI_ARENA_SLICE_SIZE) + max_page_meta_count;+ const size_t min_page_meta = min_page_count * sizeof(mi_page_t);+ mi_assert_internal(min_page_meta < page_offset);+ bool is_zero;+ bool ok = mi_arena_commit(heap->subproc,req_arena,os_start,min_page_meta,&is_zero,min_page_meta /* don't count in stats? */);+ if (ok && commit) {+ ok = mi_arena_commit(heap->subproc,req_arena,os_start + page_offset,alloc_size,NULL,0);+ }+ if (!ok) { _mi_os_free(heap->subproc,os_start,alloc_size+page_offset,*memid); return NULL; }+ if (!is_zero && !memid->initially_zero) {+ _mi_memzero_aligned(os_start,min_page_meta);+ }+ start = os_start + page_offset;+ mi_assert_internal(_mi_is_aligned(start,block_alignment));+ mi_assert_internal(_mi_is_aligned(os_start,MI_PAGE_META_ALIGNMENT));+ mi_assert_internal(_mi_align_down_ptr(start-1,MI_PAGE_META_ALIGNMENT) == os_start);+ mi_assert_internal((uint8_t*)_mi_aligned_ptr_page0(start) < os_start + min_page_meta);+ memid->initially_committed = true; // so we don't commit again+ #elseif (os_align) {// note: slice_count already includes the page- mi_assert_internal(slice_count >= mi_slice_count_of_size(block_size) + mi_slice_count_of_size(page_alignment));- start = (uint8_t*)mi_arena_os_alloc_aligned(alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid);+ start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid);+ mi_assert_internal(_mi_is_aligned(start + page_alignment, block_alignment));}else {- start = (uint8_t*)mi_arena_os_alloc_aligned(alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid);+ start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid);}+ #endif+ if (start!=NULL) { mi_heap_stat_increase(heap,pages_os_allocated,1); }}if (start == NULL) return NULL;mi_assert_internal(_mi_is_aligned(start, MI_PAGE_ALIGN));- mi_assert_internal(!os_align || _mi_is_aligned(start + page_alignment, block_alignment));return start;}-static size_t mi_page_block_start(size_t block_size, bool os_align)+// Only used for non-separate pages+mi_decl_maybe_unused static size_t mi_page_block_start(size_t block_size, bool os_align){+ size_t offset;#if MI_GUARDED// in a guarded build, we align pages with blocks a multiple of an OS page size, to the OS page size// this ensures that all blocks in such pages are OS page size aligned (which is needed for the guard pages)const size_t os_page_size = _mi_os_page_size();mi_assert_internal(MI_PAGE_ALIGN >= os_page_size);if (!os_align && block_size % os_page_size == 0 && block_size > os_page_size /* at least 2 or more */ ) {- return _mi_align_up(mi_page_info_size(), os_page_size);+ offset = _mi_align_up(mi_page_info_size(), os_page_size);}else#endifif (os_align) {- return MI_PAGE_ALIGN;+ offset = MI_PAGE_ALIGN;}- else if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) {+ else if (block_size != 0 && _mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) {// naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB)- return _mi_align_up(mi_page_info_size(), block_size);+ offset = _mi_align_up(mi_page_info_size(), block_size);+ if (block_size < 64) { offset += 3*block_size; }+ }+ else if (block_size != 0 && block_size <= MI_SMALL_SIZE_MAX) {+ // align small blocks to their size+ offset = _mi_align_up(mi_page_info_size(), block_size);}else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) {// also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB)- return _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2);+ offset = _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2);}else {// otherwise start after the info- return mi_page_info_size();+ offset = mi_page_info_size();+ }+ return _mi_align_up(offset,MI_MAX_ALIGN_SIZE);+}+++// Free a page without modifying page_bin stats+static void mi_arenas_page_free_prim(mi_page_t* page);++static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_start) {+ #if MI_PAGE_META_IS_ALIGNED+ if (memid_slice.memkind == MI_MEM_ARENA || mi_memid_is_os(memid_slice)) {+ // ensure the meta data is committed+ if (memid_slice.memkind == MI_MEM_ARENA) {+ mi_arena_t* const arena = memid_slice.mem.arena.arena;+ uint8_t* const meta_slices = (uint8_t*)_mi_align_down_ptr(slice_start,MI_PAGE_META_ALIGNMENT);+ mi_assert_internal(meta_slices >= mi_arena_start(arena));+ const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE;+ if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) {+ // try to commit all page meta slices now+ const size_t meta_slice_count = mi_arena_page_meta_aligned_slice_count();+ // the following assertion does not hold in a concurrent setting..+ // mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, meta_slice_index, meta_slice_count));+ const size_t commit_size = meta_slice_count * MI_ARENA_SLICE_SIZE;+ if (!mi_arena_commit(arena->subproc, arena, meta_slices, commit_size, NULL, 0)) {+ // if the commit fails return NULL+ return NULL;+ }+ // set the commit bits+ mi_bitmap_setN(arena->slices_committed, meta_slice_index, meta_slice_count, NULL);+ }+ }+ mi_page_t* const page_meta = _mi_aligned_ptr_page0(slice_start);+ return page_meta;+ }+ #else+ if (memid_slice.memkind == MI_MEM_ARENA) {+ MI_UNUSED(slice_start);+ mi_arena_t* const arena = memid_slice.mem.arena.arena;+ if (arena->pages_meta != NULL) {+ mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0);+ return &arena->pages_meta[memid_slice.mem.arena.slice_index];+ }}+ #endif+ return NULL;}// Allocate a fresh pagestatic mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit){- const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);- const size_t alloc_size = mi_size_of_slices(slice_count);- mi_memid_t memid = _mi_memid_none();- uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,block_size,block_alignment,os_align,commit,&memid);+ const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);+ const bool singleton = (os_align || block_size > MI_LARGE_MAX_OBJ_SIZE);+ const size_t max_page_meta_count = (singleton && slice_count > 2 ? 2 : slice_count);++ const size_t alloc_size = mi_size_of_slices(slice_count);+ mi_memid_t memid = _mi_memid_none();+ mi_arena_pages_t* arena_pages = NULL;+ uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,max_page_meta_count,block_alignment,os_align,commit,&memid,&arena_pages);if (!slice_start) return NULL;// guard page at the end of mimalloc page?#if MI_SECURE>=5mi_assert(alloc_size > _mi_os_secure_guard_page_size());const size_t page_noguard_size = alloc_size - _mi_os_secure_guard_page_size();- if (memid.initially_committed) {- _mi_os_secure_guard_page_set_at(slice_start + page_noguard_size, memid);- }#elseconst size_t page_noguard_size = alloc_size;#endif@@ -850,53 +976,77 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cousize_t block_start = 0;// allocate page meta info at the arena start?- if (memid.memkind == MI_MEM_ARENA) {- mi_arena_t* const arena = memid.mem.arena.arena;- if (arena->pages_meta != NULL) {- mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0);- mi_page_t* const page_meta = &arena->pages_meta[memid.mem.arena.slice_index];- mi_assert_internal(page_meta->block_size == 0);- #if MI_PAGE_META_ALIGNED_FREE_SMALL- // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice,- // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`)- if (block_size > MI_SMALL_SIZE_MAX)+ mi_page_t* const page_meta = mi_arena_page_meta(memid,slice_start);+ if (page_meta!=NULL) {+ mi_assert_internal(page_meta->block_size == 0);+ #if MI_PAGE_META_SMALL_IS_ALIGNED+ // if `block_size <= MI_SMALL_MAX_OBJ_SIZE` we put the page info in front of the slice,+ // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`)+ if (!os_align && block_size <= MI_SMALL_MAX_OBJ_SIZE) {+ // put page info in front of the slice+ page = (mi_page_t*)slice_start;+ block_start = mi_page_block_start(block_size, os_align);+ }+ else+ #endif+ {+ page_meta_is_separate = true;+ page = page_meta;+ block_start = 0;+ #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET)+ #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */#endif+ if (block_size >= MI_SIZE_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET &&+ _mi_is_power_of_two(block_size)){- page = page_meta;- page_meta_is_separate = true;- block_start = 0;- #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET)- #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */- #endif- if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) {- block_start += block_size;- }- mi_assert_internal(page->block_size == 0);- _mi_memzero_aligned(page, sizeof(*page));+ block_start = _mi_align_up(mi_page_info_size(), block_size); // to maintain natural alignment+ if (block_size < 64) { block_start += 3*block_size; }}+ mi_assert_internal(page->block_size == 0);+ _mi_memzero_aligned(page, sizeof(*page));}}if (page == NULL) {+ #if MI_PAGE_META_IS_ALIGNED+ // can only happen on failing to commit the page meta info+ _mi_arenas_free(_mi_theap_subproc(theap),slice_start,alloc_size,memid);+ return NULL;+ #else// put page meta info in front of the slicepage = (mi_page_t*)slice_start;block_start = mi_page_block_start(block_size, os_align);+ #endif+ }+ mi_assert_internal(block_size < MI_MAX_ALIGN_SIZE || block_start % MI_MAX_ALIGN_SIZE == 0);+ if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) {+ mi_assert_internal(block_start % block_size == 0); // natural alignment (see also alloc_aligned.c)}// commit first block?size_t commit_size = 0;if (!memid.initially_committed) {- commit_size = _mi_align_up(block_start + block_size, MI_PAGE_MIN_COMMIT_SIZE);+ commit_size = _mi_align_up(block_start + block_size, mi_page_min_commit_size());if (commit_size > page_noguard_size) { commit_size = page_noguard_size; }bool is_zero = false;- if mi_unlikely(!mi_arena_commit( mi_memid_arena(memid), slice_start, commit_size, &is_zero, 0)) {- _mi_arenas_free(slice_start, alloc_size, memid);+ if mi_unlikely(!mi_arena_commit( _mi_theap_subproc(theap), mi_memid_arena(memid), slice_start, commit_size, &is_zero, 0)) {+ _mi_arenas_free(_mi_theap_subproc(theap), slice_start, alloc_size, memid);return NULL;}}+ // now we can finish initalization and use `mi_arenas_free_page_prim` on error++ // zero initialize the page meta dataif (!memid.initially_zero && !page_meta_is_separate) {_mi_memzero_aligned(page, sizeof(*page));}+ // set the guard page+ #if MI_SECURE>=5+ if (memid.initially_committed) {+ _mi_os_secure_guard_page_set_at(_mi_theap_subproc(theap), slice_start + page_noguard_size, memid);+ }+ #endif+// claimed free slices: initialize the page partlyif (!memid.initially_zero && memid.initially_committed) {mi_track_mem_undefined(slice_start, slice_count * MI_ARENA_SLICE_SIZE);@@ -916,23 +1066,62 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_couconst size_t reserved = (os_align ? 1 : (page_noguard_size - block_start) / block_size);mi_assert_internal(reserved > 0 && reserved <= UINT16_MAX);- // initialize- page->reserved = (uint16_t)reserved;- page->page_start = slice_start + block_start;+ // initialize the page start+ uint8_t* const start = slice_start + block_start;+ mi_assert_internal(start > (uint8_t*)page);+ page->page_offset = start - (uint8_t*)page;++ // initialize page meta-data+ page->reserved = (uint16_t)reserved;page->block_size = block_size;- page->slice_committed = commit_size;page->memid = memid;page->free_is_zero = memid.initially_zero;++ mi_assert_internal((commit && commit_size==0) || (!commit && (commit_size <= UINT16_MAX * _mi_os_page_size())));+ page->slice_pcommitted = (uint16_t)(commit_size / _mi_os_page_size());++ page->heap = _mi_theap_heap(theap);+ mi_page_set_theap(page,theap);+ // mi_assert_internal(mi_page_theap(page) == _mi_heap_theap_peek(page->heap))++ #if MI_PAGE_META_IS_ALIGNED+ mi_assert_internal(page_meta!=NULL);+ mi_atomic_store_ptr_release(mi_page_t,&page_meta->self,page);+ if (slice_count > 1) {+ // at least two for large singleton blocks as guard pages can have a large offset beyond a single slice+ for(size_t i = 1; i < max_page_meta_count; i++) {+ mi_assert_internal(page_meta[i].block_size == 0);+ mi_atomic_store_ptr_release(mi_page_t,&page_meta[i].self,page);+ }+ }+ #if MI_DEBUG>1+ mi_page_t* pstart = _mi_aligned_ptr_page0(slice_start);+ mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pstart->self)==page);+ if (reserved>1) {+ mi_page_t* pend = _mi_aligned_ptr_page0(slice_start + (slice_count*MI_ARENA_SLICE_SIZE) - 1);+ mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pend->self)==page);+ }+ #endif+ #endif+mi_assert_internal(page->free==NULL);mi_assert_internal(page_meta_is_separate == mi_page_meta_is_separated(page));mi_assert_internal(mi_page_slice_start(page) == slice_start);+ mi_assert_internal(mi_page_size(page) <= page_noguard_size);+++ // now register in the arena_pages+ if (arena_pages!=NULL) {+ mi_assert_internal(memid.memkind == MI_MEM_ARENA);+ mi_bitmap_set(arena_pages->pages, memid.mem.arena.slice_index);+ }// and own itmi_page_claim_ownership(page);// register in the page mapif mi_unlikely(!_mi_page_map_register(page)) {- _mi_arenas_free( slice_start, alloc_size, memid );+ mi_arenas_page_free_prim(page);return NULL;}@@ -943,7 +1132,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_coumi_assert_internal(_mi_is_aligned(mi_page_slice_start(page),MI_PAGE_ALIGN));mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);mi_assert_internal(mi_page_block_size(page) == block_size);- mi_assert_internal(mi_page_is_abandoned(page));+ // mi_assert_internal(mi_page_is_abandoned(page));mi_assert_internal(mi_page_is_owned(page));return page;@@ -960,14 +1149,15 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c// 2. find a free block, potentially allocating a new arenaconst long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand);- const bool commit = (slice_count <= mi_slice_count_of_size(MI_PAGE_MIN_COMMIT_SIZE) || // always commit small pages- (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0));+ const bool commit = (mi_page_min_commit_size() >= slice_count * MI_ARENA_SLICE_SIZE || // always commit small pages+ (slice_count >= mi_slice_count_of_size(UINT16_MAX * _mi_os_page_size())) || // always commit pages too large to hold a 32-bit slice_committed+ (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0));page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit);if (page == NULL) return NULL;mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count);if (!_mi_page_init(theap, page)) {- _mi_arenas_free( page, mi_page_full_size(page), page->memid);+ _mi_arenas_page_free(page,theap);return NULL;}@@ -977,8 +1167,12 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c// Allocate a page containing one block (very large, or with large alignment)static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment){+ #if MI_PAGE_META_IS_ALIGNED+ const size_t info_size = 0;+ #elseconst bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);- const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size());+ const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size());+ #endif#if MI_SECURE < 2const size_t slice_count = mi_slice_count_of_size(info_size + block_size);#else@@ -990,7 +1184,7 @@ static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t blockmi_assert(page->reserved == 1);if (!_mi_page_init(theap, page)) {- _mi_arenas_free( page, mi_page_full_size(page), page->memid);+ _mi_arenas_page_free(page,theap);return NULL;}@@ -1000,6 +1194,8 @@ static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t blockmi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) {mi_page_t* page;+ // semi static assert: ensure that all non-singleton block size bins are covered.+ mi_assert(_mi_bin(MI_LARGE_MAX_OBJ_SIZE) < MI_ARENA_BIN_COUNT);if mi_unlikely(block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {mi_assert_internal(_mi_is_power_of_two(block_alignment));page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment);@@ -1029,25 +1225,13 @@ mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t blreturn page;}-void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {+static void mi_arenas_page_free_prim(mi_page_t* page) {mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);mi_assert_internal(mi_page_is_owned(page));mi_assert_internal(mi_page_all_free(page));- mi_assert_internal(mi_page_is_abandoned(page));mi_assert_internal(page->next==NULL && page->prev==NULL);- mi_assert_internal(current_theapx == NULL || _mi_thread_id()==current_theapx->tld->thread_id);-- if (current_theapx != NULL) {- mi_theap_stat_decrease(current_theapx, page_bins[_mi_page_stats_bin(page)], 1);- mi_theap_stat_decrease(current_theapx, pages, 1);- }- else {- mi_heap_t* const heap = mi_page_heap(page);- mi_heap_stat_decrease(heap, page_bins[_mi_page_stats_bin(page)], 1);- mi_heap_stat_decrease(heap, pages, 1);- }-+#if MI_DEBUG>1if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {size_t bin = _mi_bin(mi_page_block_size(page));@@ -1056,8 +1240,8 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {mi_arena_pages_t* arena_pages = NULL;mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));- mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));- mi_assert_internal(mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1));+ mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));+ mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1));mi_assert_internal(mi_bitmap_is_setN(arena_pages->pages, slice_index, 1));// note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may// be (temporarily) not true if the free happens while trying to reclaim@@ -1065,34 +1249,38 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {}#endif+ // unregister page+ _mi_page_map_unregister(page);+// recommit guard page at the end?// we must do this since we may later allocate large spans over this page and cannot have a guard page in between#if MI_SECURE >= 5if (!page->memid.is_pinned) {- _mi_os_secure_guard_page_reset_before(mi_page_slice_start(page) + mi_page_full_size(page), page->memid);+ _mi_os_secure_guard_page_reset_before(mi_page_subproc(page), mi_page_slice_start(page) + mi_page_full_size(page), page->memid);}#endif- // unregister page- _mi_page_map_unregister(page);+ // and freeif (page->memid.memkind == MI_MEM_ARENA) {mi_arena_pages_t* arena_pages;size_t slice_index;size_t slice_count; MI_UNUSED(slice_count);mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);mi_assert_internal(arena_pages!=NULL);+ mi_assert_internal(arena->subproc == mi_page_subproc(page));mi_bitmap_clear(arena_pages->pages, slice_index);- if (page->slice_committed > 0) {+ const size_t slice_committed = mi_page_slice_committed(page);+ if (slice_committed > 0) {// if committed on-demand, set the commit bits to account commit properly- mi_assert_internal(mi_page_full_size(page) >= page->slice_committed);- const size_t total_slices = page->slice_committed / MI_ARENA_SLICE_SIZE; // conservative+ mi_assert_internal(mi_page_full_size(page) >= slice_committed);+ const size_t total_slices = slice_committed / MI_ARENA_SLICE_SIZE; // conservative//mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, slice_index, total_slices));mi_assert_internal(slice_count >= total_slices);if (total_slices > 0) {mi_bitmap_setN(arena->slices_committed, slice_index, total_slices, NULL);}// any left over?- const size_t extra = page->slice_committed % MI_ARENA_SLICE_SIZE;+ const size_t extra = slice_committed % MI_ARENA_SLICE_SIZE;if (extra > 0) {// pretend it was decommitted alreadymi_subproc_stat_decrease(arena->subproc, committed, extra);@@ -1102,60 +1290,91 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));}}+ else {+ mi_heap_stat_decrease(page->heap, pages_os_allocated, 1);+ }if (mi_page_meta_is_separated(page)) { page->block_size = 0; } // for assertion checking- _mi_arenas_free( mi_page_slice_start(page), mi_page_full_size(page), page->memid);+ _mi_arenas_free( mi_page_subproc(page), mi_page_slice_start(page), mi_page_full_size(page), page->memid);+}++void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {+ mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));+ mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);+ mi_assert_internal(mi_page_is_owned(page));+ mi_assert_internal(mi_page_all_free(page));+ mi_assert_internal(mi_page_is_abandoned(page));+ mi_assert_internal(page->next==NULL && page->prev==NULL);+ mi_assert_internal(mi_theap_matches_thread(current_theapx));++ mi_heap_t* const heap = mi_page_heap(page);+ mi_theapx_stat_decrease(heap, current_theapx, page_bins[_mi_page_stats_bin(page)], 1);+ mi_theapx_stat_decrease(heap, current_theapx, pages, 1);+ _mi_page_free_collect(page,false); // update used count for cross-thread free's+ _mi_page_update_stats(page); // and update the stats+ mi_arenas_page_free_prim(page);}/* -----------------------------------------------------------Arena abandon----------------------------------------------------------- */-void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) {+void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) {mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);mi_assert_internal(mi_page_is_owned(page));mi_assert_internal(mi_page_is_abandoned(page));mi_assert_internal(!mi_page_all_free(page));mi_assert_internal(page->next==NULL && page->prev == NULL);- mi_assert_internal(_mi_thread_id()==current_theap->tld->thread_id);+ mi_assert_internal(mi_theap_matches_thread(current_theapx));// mi_assert_internal(current_theap == _mi_page_associated_theap(page));- mi_heap_t* heap = mi_page_heap(page); mi_assert_internal(heap==_mi_theap_heap(current_theap));+ // note: somewhat expensive to update here, but might be good as then we attribute+ // the current allocations/frees to the current thread/theap. Otherwise it might be+ // reclaimed later in another thread/theap and those allocations/frees get attributed there...+ _mi_page_update_stats(page);++ // add to abandoned?+ mi_heap_t* heap = mi_page_heap(page);if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {// make available for allocationssize_t bin = _mi_bin(mi_page_block_size(page));- size_t slice_index;- size_t slice_count;- mi_arena_pages_t* arena_pages = NULL;- mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena);-- mi_assert_internal(!mi_page_is_singleton(page));- mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));- mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));- mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));-- mi_page_set_abandoned_mapped(page);- const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index);- MI_UNUSED(was_clear); mi_assert_internal(was_clear);- mi_atomic_increment_relaxed(&heap->abandoned_count[bin]);- mi_theap_stat_increase(current_theap, pages_abandoned, 1);+ mi_assert_internal(bin < MI_ARENA_BIN_COUNT);+ if (bin < MI_ARENA_BIN_COUNT) { // paranoia+ size_t slice_index;+ size_t slice_count;+ mi_arena_pages_t* arena_pages = NULL;+ mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena);++ mi_assert_internal(!mi_page_is_singleton(page));+ mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));+ mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));+ mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));++ mi_page_set_abandoned_mapped(page);+ const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index);+ MI_UNUSED(was_clear); mi_assert_internal(was_clear);+ mi_atomic_increment_relaxed(&heap->abandoned_count[bin]);+ mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1);+ mi_abandoned_page_unown(page, current_theapx);+ return;+ }}- else {- // page is full (or a singleton), or the page is OS/externally allocated- // leave as is; it will be reclaimed when an object is free'd in the page- // but for non-arena pages, add to the subproc list so these can be visited- if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) {- mi_lock(&heap->os_abandoned_pages_lock) {- // push in front- page->prev = NULL;- page->next = heap->os_abandoned_pages;- if (page->next != NULL) { page->next->prev = page; }- heap->os_abandoned_pages = page;- }+ // otherwise,+ // page is full (or a singleton), or the page is OS/externally allocated+ // leave as is; it will be reclaimed when an object is free'd in the page+ // but for non-arena pages, add to the subproc list so these can be visited+ if (page->memid.memkind != MI_MEM_ARENA) {+ mi_lock(&heap->os_abandoned_pages_lock) {+ // push in front+ page->prev = NULL;+ page->next = heap->os_abandoned_pages;+ if (page->next != NULL) { page->next->prev = page; }+ heap->os_abandoned_pages = page;}- mi_theap_stat_increase(current_theap, pages_abandoned, 1);+ mi_theapx_stat_increase(heap, current_theapx, pages_os_abandoned, 1);}- mi_abandoned_page_unown(page, current_theap);+ mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1);+ mi_abandoned_page_unown(page, current_theapx);}@@ -1174,16 +1393,13 @@ bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) {}else {// do not use _mi_heap_theap as we may call this during shutdown of threads and don't want to reinitialize the theap- mi_theap_t* const theap = _mi_page_associated_theap_peek(page);- if (theap == NULL) {- return false;- }- else {- mi_theap_stat_counter_increase(theap, pages_reabandon_full, 1);- mi_theap_stat_adjust_decrease(theap, pages_abandoned, 1); // adjust as we are not abandoning fresh- _mi_arenas_page_abandon(page, theap);- return true;- }+ mi_theap_t* const theapx = _mi_page_associated_theap_peek(page); // can be NULL+ mi_heap_t* const heap = mi_page_heap(page);+ // if (theapx==NULL) return false;+ mi_theapx_stat_counter_increase(heap, theapx, pages_reabandon_full, 1);+ mi_theapx_stat_adjust_decrease(heap, theapx, pages_abandoned, 1); // adjust as we are not abandoning fresh+ _mi_arenas_page_abandon(page, theapx);+ return true;}}@@ -1193,30 +1409,31 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) {mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);mi_assert_internal(mi_page_is_owned(page));mi_assert_internal(mi_page_is_abandoned(page));- mi_assert_internal(current_theapx==NULL || _mi_thread_id()==current_theapx->tld->thread_id);+ mi_assert_internal(mi_theap_matches_thread(current_theapx));mi_heap_t* const heap = mi_page_heap(page);if (mi_page_is_abandoned_mapped(page)) {mi_assert_internal(page->memid.memkind==MI_MEM_ARENA);// remove from the abandoned map- size_t bin = _mi_bin(mi_page_block_size(page));+ const size_t bin = _mi_bin(mi_page_block_size(page));+ mi_assert_internal(bin < MI_ARENA_BIN_COUNT);size_t slice_index;size_t slice_count;mi_arena_pages_t* arena_pages;mi_arena_t* arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena);mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));- mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));+ mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));// this busy waits until a concurrent reader (from alloc_abandoned) is done- mi_bitmap_clear_once_set(arena_pages->pages_abandoned[bin], slice_index);+ mi_bitmap_clear_once_set(arena->subproc, arena_pages->pages_abandoned[bin], slice_index);mi_page_clear_abandoned_mapped(page);mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]);}else {// page is full (or a singleton), page is OS allocated// if not an arena page, remove from the subproc os pages list- if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) {+ if (page->memid.memkind != MI_MEM_ARENA) {mi_lock(&heap->os_abandoned_pages_lock) {if (page->prev != NULL) { page->prev->next = page->next; }if (page->next != NULL) { page->next->prev = page->prev; }@@ -1224,14 +1441,10 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) {page->next = NULL;page->prev = NULL;}+ mi_theapx_stat_decrease(heap, current_theapx, pages_os_abandoned, 1);}}- if (current_theapx!=NULL) {- mi_theap_stat_decrease(current_theapx, pages_abandoned, 1);- }- else {- mi_heap_stat_decrease(heap, pages_abandoned, 1);- }+ mi_theapx_stat_decrease(heap, current_theapx, pages_abandoned, 1);}@@ -1241,7 +1454,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) {static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slices);static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq);-void _mi_arenas_free(void* p, size_t size, mi_memid_t memid) {+void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) {if (p==NULL) return;if (size==0) return;@@ -1250,26 +1463,28 @@ void _mi_arenas_free(void* p, size_t size, mi_memid_t memid) {if (mi_memkind_is_os(memid.memkind)) {// was a direct OS allocation, pass through- _mi_os_free(p, size, memid);+ _mi_os_free(subproc, p, size, memid);}else if (memid.memkind == MI_MEM_ARENA) {// allocated in an arenasize_t slice_count;size_t slice_index;mi_arena_t* arena = mi_arena_from_memid(memid, &slice_index, &slice_count);+ mi_assert_internal(arena!=NULL);+ mi_assert_internal(arena->subproc == subproc);mi_assert_internal((size%MI_ARENA_SLICE_SIZE)==0);mi_assert_internal((slice_count*MI_ARENA_SLICE_SIZE)==size);mi_assert_internal(mi_arena_slice_start(arena,slice_index) <= (uint8_t*)p);mi_assert_internal(mi_arena_slice_start(arena,slice_index) + mi_size_of_slices(slice_count) > (uint8_t*)p);// checksif (arena == NULL) {- _mi_error_message(EINVAL, "trying to free from an invalid arena: %p, size %zu, memid: 0x%zx\n", p, size, memid);+ _mi_error_message(EINVAL, "trying to free from an invalid arena: %p, size %zu, memkind: 0x%x\n", p, size, memid.memkind);return;}mi_assert_internal(slice_index < arena->slice_count);mi_assert_internal(slice_index >= mi_arena_info_slices(arena));if (slice_index < mi_arena_info_slices(arena) || slice_index >= arena->slice_count) {- _mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memid: 0x%zx\n", p, size, memid);+ _mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memkind: 0x%x\n", p, size, memid.memkind);return;}@@ -1286,8 +1501,8 @@ void _mi_arenas_free(void* p, size_t size, mi_memid_t memid) {return;};}- else if (memid.memkind == MI_MEM_META) {- _mi_meta_free(p, size, memid);+ else if (memid.memkind == MI_MEM_MALLOC) {+ _mi_free_subproc_safe(p);}else {// arena was none, external, or static; nothing to do@@ -1328,10 +1543,10 @@ static bool mi_arenas_contain_ex(const void* p, mi_arena_t* parent) {return false;}-// Is a pointer inside any of our arenas?-bool _mi_arenas_contain(const void* p) {- return mi_arenas_contain_ex(p, NULL);-}+// // Is a pointer inside any of our arenas?+// bool _mi_arenas_contain(const void* p) {+// return mi_arenas_contain_ex(p, NULL);+// }// Is a pointer contained in the given arena area?bool mi_arena_contains(mi_arena_id_t arena_id, const void* p) {@@ -1357,7 +1572,7 @@ static void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) {// mi_lock_done(&arena->abandoned_visit_lock);mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL);if (mi_memkind_is_os(arena->memid.memkind)) {- _mi_os_free_ex(mi_arena_start(arena), mi_arena_size(arena), true, arena->memid, subproc); // pass `subproc` to avoid accessing the theap pointer (in `_mi_subproc()`)+ _mi_os_free_ex(subproc, mi_arena_start(arena), mi_arena_size(arena), true, arena->memid);}}}@@ -1430,13 +1645,17 @@ static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) {return size;}+static mi_arena_t* mi_arena_info(void* area) {+ return (mi_arena_t*)((uint8_t*)area + mi_size_of_slices(mi_arena_page_meta_aligned_slice_count()));+}+static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base) {if (slice_count == 0) slice_count = MI_BCHUNK_BITS;mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0);- const size_t base_size = _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE);+ const size_t base_size = mi_size_of_slices(mi_arena_page_meta_aligned_slice_count()) + _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE);const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandonedconst size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free- #if MI_PAGE_META_IS_SEPARATED+ #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNEDconst size_t pages_size = slice_count * sizeof(mi_page_t);#elseconst size_t pages_size = 0;@@ -1457,9 +1676,9 @@ static mi_bitmap_t* mi_arena_bitmap_init(size_t slice_count, uint8_t** base) {return bitmap;}-static mi_bbitmap_t* mi_arena_bbitmap_init(size_t slice_count, uint8_t** base) {+static mi_bbitmap_t* mi_arena_bbitmap_init(mi_subproc_t* subproc, size_t slice_count, uint8_t** base) {mi_bbitmap_t* bbitmap = (mi_bbitmap_t*)(*base);- *base = (*base) + mi_bbitmap_init(bbitmap, slice_count, true /* already zero */);+ *base = (*base) + mi_bbitmap_init(subproc, bbitmap, slice_count, true /* already zero */);return bbitmap;}@@ -1467,7 +1686,7 @@ static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) {const size_t slice_count = arena->slice_count;size_t bitmap_base = 0;const size_t size = mi_arena_pages_size(slice_count, &bitmap_base);- mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)mi_heap_zalloc_aligned(mi_heap_main(), size, MI_BCHUNK_SIZE);+ mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)mi_heap_zalloc_aligned(arena->subproc->heap_main, size, MI_BCHUNK_SIZE);if (arena_pages==NULL) return NULL;uint8_t* base = (uint8_t*)arena_pages + bitmap_base;mi_assert_internal(_mi_is_aligned(base, MI_BCHUNK_SIZE));@@ -1483,7 +1702,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,int numa_node, bool exclusive,mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id){- mi_assert_internal(_mi_is_aligned(start,MI_ARENA_SLICE_ALIGN));+ mi_assert_internal(_mi_is_aligned(start,MI_ARENA_ALIGNMENT));mi_assert_internal(mi_size_of_slices(slice_count)>=MI_ARENA_MIN_SIZE);if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now@@ -1498,13 +1717,11 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,_mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", mi_size_of_slices(slice_count)/MI_KiB, mi_size_of_slices(info_slices+1)/MI_KiB);return NULL;}- else if (info_slices >= MI_ARENA_MAX_CHUNK_OBJ_SLICES) {- _mi_warning_message("cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)", mi_size_of_slices(slice_count)/MI_MiB, info_slices, MI_ARENA_MAX_CHUNK_OBJ_SLICES);- return NULL;- }-- mi_arena_t* arena = (mi_arena_t*)start;-+ // else if (info_slices >= MI_ARENA_MAX_CHUNK_OBJ_SLICES) {+ // _mi_warning_message("cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)", mi_size_of_slices(slice_count)/MI_MiB, info_slices, MI_ARENA_MAX_CHUNK_OBJ_SLICES);+ // return NULL;+ // }+// commit & zero if neededif (!memid.initially_committed) {size_t commit_size = mi_size_of_slices(info_slices);@@ -1512,10 +1729,10 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,if (!memid.is_pinned) { commit_size -= _mi_os_secure_guard_page_size(); }bool ok = false;if (commit_fun != NULL) {- ok = (*commit_fun)(true /* commit */, arena, commit_size, NULL, commit_fun_arg);+ ok = (*commit_fun)(true /* commit */, start, commit_size, NULL, commit_fun_arg);}else {- ok = _mi_os_commit(arena, commit_size, NULL);+ ok = _mi_os_commit(subproc, start, commit_size, NULL);}if (!ok) {_mi_warning_message("unable to commit meta-data for OS memory");@@ -1525,13 +1742,15 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,else if (!memid.is_pinned) {// if MI_SECURE, set a guard page at the end of the arena info// todo: this does not respect the commit_fun as the memid is of external memory- _mi_os_secure_guard_page_set_before((uint8_t*)arena + mi_size_of_slices(info_slices), memid);+ _mi_os_secure_guard_page_set_before(subproc, (uint8_t*)start + mi_size_of_slices(info_slices), memid);}if (!memid.initially_zero) {- _mi_memzero(arena, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size());+ _mi_memzero(start, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size());}// init+ mi_arena_t* arena = mi_arena_info(start);+ arena->start = start;arena->subproc = subproc;arena->memid = memid;arena->is_exclusive = exclusive;@@ -1551,7 +1770,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,// init bitmapsuint8_t* base = mi_arena_start(arena) + bitmap_base;- arena->slices_free = mi_arena_bbitmap_init(slice_count, &base);+ arena->slices_free = mi_arena_bbitmap_init(subproc, slice_count, &base);arena->slices_committed = mi_arena_bitmap_init(slice_count, &base);arena->slices_dirty = mi_arena_bitmap_init(slice_count, &base);arena->slices_purge = mi_arena_bitmap_init(slice_count, &base);@@ -1559,7 +1778,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base);}- #if MI_PAGE_META_IS_SEPARATED+ #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNEDarena->pages_meta = (mi_page_t*)base;base += (slice_count * sizeof(mi_page_t));#else@@ -1568,7 +1787,23 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,mi_assert_internal(mi_size_of_slices(info_slices) >= (size_t)(base - mi_arena_start(arena)));// reserve our meta info (and reserve slices outside the memory area)+ #if MI_PAGE_META_IS_ALIGNED+ for(size_t i = 0; i < arena->slice_count; i += MI_PAGE_META_ALIGNED_COUNT) {+ // set all free slices (and skip the slices reserved for the page meta info)+ const size_t meta_slices = (i==0 ? info_slices : mi_arena_page_meta_aligned_slice_count());+ const size_t start_idx = (i + meta_slices);+ size_t count = MI_PAGE_META_ALIGNED_COUNT - meta_slices;+ if (start_idx < arena->slice_count) {+ if (count + start_idx > arena->slice_count) {+ count = arena->slice_count - start_idx;+ mi_assert_internal(count > 0);+ }+ mi_bbitmap_unsafe_setN(arena->slices_free, start_idx, count);+ }+ }+ #elsemi_bbitmap_unsafe_setN(arena->slices_free, info_slices /* start */, arena->slice_count - info_slices);+ #endifif (memid.initially_committed) {mi_bitmap_unsafe_setN(arena->slices_committed, 0, arena->slice_count);}@@ -1584,15 +1819,14 @@ static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t smi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept{// checks- mi_assert(_mi_is_aligned(start, MI_ARENA_SLICE_SIZE));mi_assert(start!=NULL);if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); }if (start==NULL) return false;- if (!_mi_is_aligned(start, MI_ARENA_SLICE_SIZE)) {+ if (!_mi_is_aligned(start, MI_ARENA_ALIGNMENT)) {// we can align the start since the memid tracks the real base of the memory.- void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_SLICE_SIZE);+ void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_ALIGNMENT);const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start;- if (diff >= size || (size - diff) < MI_ARENA_SLICE_SIZE) {+ if (diff >= size || (size - diff) < MI_ARENA_ALIGNMENT) {_mi_warning_message("after alignment, the size of the arena becomes too small (memory at %p with size %zu)\n", start, size);return false;}@@ -1675,12 +1909,18 @@ bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinne// Reserve a range of regular OS memorystatic int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) {if (arena_id != NULL) *arena_id = _mi_arena_id_none();- size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice+ if (size <= MI_MAX_ALLOC_SIZE) {+ size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice+ }+ if (size > MI_MAX_ALLOC_SIZE) {+ _mi_error_message(EOVERFLOW, "memory reservation request is too large (size %zu)\n", size);+ return ENOMEM;+ }mi_memid_t memid;- void* start = _mi_os_alloc_aligned(size, MI_ARENA_SLICE_ALIGN, commit, allow_large, &memid);- if (start == NULL) return ENOMEM;+ void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_ALIGNMENT, commit, allow_large, &memid);+ if (start == NULL) return ENOMEM;if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) {- _mi_os_free_ex(start, size, commit, memid, NULL);+ _mi_os_free_ex(subproc, start, size, commit, memid);_mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024));return ENOMEM;}@@ -1756,9 +1996,8 @@ static void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) {}static int mi_page_commit_usage(mi_page_t* page) {- // if (mi_page_size(page) <= MI_PAGE_MIN_COMMIT_SIZE) return 100;const size_t committed_size = mi_page_committed(page);- const size_t used_size = page->used * mi_page_block_size(page);+ const size_t used_size = mi_page_used(page) * mi_page_block_size(page);return (int)(used_size * 100 / committed_size);}@@ -1777,7 +2016,8 @@ static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena,bit_set_count++;c = 'p';color = MI_GRAY;- if (mi_page_is_singleton(page)) { c = 's'; }+ if (_mi_meta_is_meta_page(arena->subproc,page)) { c = 'm'; }+ else if (mi_page_is_singleton(page)) { c = 's'; }else if (mi_page_is_full(page)) { c = 'f'; }if (!mi_page_is_abandoned(page)) { c = _mi_toupper(c); }int commit_usage = mi_page_commit_usage(page);@@ -1790,8 +2030,13 @@ static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena,else {c = '?';if (bit_of_page > 0) { c = '-'; }- else if (_mi_meta_is_meta_page(start)) { c = 'm'; color = MI_GRAY; }+ // else if (_mi_meta_is_meta_page(arena->subproc,start)) { c = 'm'; color = MI_GRAY; }else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; }+ #if MI_PAGE_META_IS_ALIGNED+ else if ((slice_index % MI_PAGE_META_ALIGNED_COUNT) == 0 && (size_t)bit <= mi_arena_page_meta_aligned_slice_count()) {+ { c = 'i'; color = MI_GRAY; }+ }+ #endif// else if (mi_bitmap_is_setN(arena->pages_purge, slice_index + bit, NULL)) { c = '*'; }else if (mi_bbitmap_is_setN(arena->slices_free, slice_index+bit,1)) {if (mi_bitmap_is_set(arena->slices_purge, slice_index + bit)) { c = '~'; color = MI_ORANGE; }@@ -1898,13 +2143,13 @@ static void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narrosize_t page_total = 0;for (size_t i = 0; i < max_arenas; i++) {mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);- if (arena == NULL) break;+ if (arena == NULL) continue;mi_assert(arena->subproc == subproc);// slice_total += arena->slice_count;- _mi_raw_message("%sarena %zu at %p: %zu slices (%zu MiB)%s%s, subproc: %p, numa: %i\n",+ _mi_raw_message("%sarena %zu at %p: %zu slices (%zu MiB)%s%s, subproc: %zu, numa: %i\n",(arena->parent==NULL ? "" : "(sub)"), i, arena, arena->slice_count, (size_t)(mi_size_of_slices(arena->slice_count)/MI_MiB),(arena->memid.is_pinned ? ", pinned" : ""), (arena->is_exclusive ? ", exclusive" : ""),- arena->subproc, arena->numa_node);+ arena->subproc->subproc_seq, arena->numa_node);//if (show_inuse) {// free_total += mi_debug_show_bbitmap("in-use slices", arena->slice_count, arena->slices_free, true, NULL);//}@@ -1919,7 +2164,7 @@ static void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narro// mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena);// if (arena_pages != NULL){- const char* header1 = "chunks (p:page, f:full, s:singleton, P,F,S:not abandoned, i:arena-info, m:meta-data, ~:free-purgable, _:free-committed, .:free-reserved)";+ const char* header1 = "chunks (p:page, f:full, s:single, m:meta-data, i:arena-info, P,F,S,M:not abandoned, ~:free-purgable, _:free-committed, .:free-reserved)";const char* header2 = (narrow ? "\n " : " ");const char* header3 = "(chunk bin: S:small, M : medium, L : large, X : other)";page_total += mi_debug_show_chunks(header1, header2, header3, arena->slice_count,@@ -1951,18 +2196,19 @@ int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_mif (pages==0) return 0;if (numa_node < -1) numa_node = -1;if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count();+ mi_subproc_t* subproc = _mi_subproc();size_t hsize = 0;size_t pages_reserved = 0;mi_memid_t memid;- void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid);+ void* p = _mi_os_alloc_huge_os_pages(subproc, pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid);if (p==NULL || pages_reserved==0) {_mi_warning_message("failed to reserve %zu GiB huge pages\n", pages);return ENOMEM;}_mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages);- if (!mi_manage_os_memory_ex2(_mi_subproc(), p, hsize, numa_node, exclusive, memid, NULL, NULL, arena_id)) {- _mi_os_free(p, hsize, memid);+ if (!mi_manage_os_memory_ex2(subproc, p, hsize, numa_node, exclusive, memid, NULL, NULL, arena_id)) {+ _mi_os_free(subproc, p, hsize, memid);return ENOMEM;}return 0;@@ -2019,7 +2265,14 @@ int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reservstatic long mi_arena_purge_delay(void) {// <0 = no purging allowed, 0=immediate purging, >0=milli-second delay- return (mi_option_get(mi_option_purge_delay) * mi_option_get(mi_option_arena_purge_mult));+ const long delay = mi_option_get(mi_option_purge_delay);+ const long mult = mi_option_get(mi_option_arena_purge_mult);+ if (delay<0 || mult<0) { return -1; }+ if (delay==0 || mult==0) { return 0; }+ size_t total;+ if (mi_mul_overflow((size_t)delay, (size_t)mult, &total)) { return delay; }+ if (total > LONG_MAX) { return delay; }+ return (long)total;}// reset or decommit in an arena and update the commit bitmap@@ -2035,7 +2288,7 @@ static bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_csize_t already_committed;mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed); // pretend all committed.. (as we lack a clearN call that counts the already set bits..)const bool all_committed = (already_committed == slice_count);- const bool needs_recommit = _mi_os_purge_ex(p, size, all_committed /* allow reset? */, mi_size_of_slices(already_committed), arena->commit_fun, arena->commit_fun_arg);+ const bool needs_recommit = _mi_os_purge_ex(arena->subproc, p, size, all_committed /* allow reset? */, mi_size_of_slices(already_committed), arena->commit_fun, arena->commit_fun_arg);if (needs_recommit) {// no longer committed@@ -2137,10 +2390,8 @@ static int mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force)// expired yet?mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire);- if (!force) {- if (expire==0) return -1;- if (expire > now) return 0;- }+ if (expire==0) return -1;+ if (!force && expire > now) return 0;// reset expiremi_atomic_storei64_release(&arena->purge_expire, (mi_msecs_t)0);@@ -2254,7 +2505,7 @@ bool _mi_heap_visit_blocks(mi_heap_t* heap, bool abandoned_only, bool visit_blocmi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena);if (ok && arena_pages != NULL) {if (abandoned_only) {- for (size_t bin = 0; ok && bin < MI_BIN_COUNT; bin++) {+ for (size_t bin = 0; ok && bin < MI_ARENA_BIN_COUNT; bin++) {// todo: if we had a single abandoned page map as well, this can be faster.if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) > 0) {ok = _mi_bitmap_forall_set(arena_pages->pages_abandoned[bin], &mi_heap_visit_page_at, arena, &visit_info);@@ -2317,25 +2568,35 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* aremi_assert_internal(mi_page_is_abandoned(page));mi_assert_internal(mi_page_is_owned(page));- if (page->used==0) {+ if (mi_page_used(page)==0) {// free the page_mi_arenas_page_free(page, theap);}else if (heap_target==NULL) {+ #if MI_GUARDED+ _mi_page_unguard_all(page); // remove potential interior guard pages+ #endif// destroy the page- page->used=0; // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case+ mi_page_used_reset(page); // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case_mi_arenas_page_free(page, theap);}else {// move the page to `heap_target` as an abandoned page// first remove it from the current heapconst size_t sbin = _mi_page_stats_bin(page);- size_t slice_index;- size_t slice_count;- mi_arena_pages_t* arena_pages = NULL;- mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);- mi_assert_internal(mi_bitmap_is_set(arena_pages->pages, slice_index));- mi_bitmap_clear(arena_pages->pages, slice_index);+ mi_arena_t* arena = NULL;+ size_t slice_index = 0;+ if (page->memid.memkind == MI_MEM_ARENA) {+ size_t slice_count;+ mi_arena_pages_t* arena_pages = NULL;+ arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);+ mi_assert_internal(mi_bitmap_is_set(arena_pages->pages, slice_index));+ mi_bitmap_clear(arena_pages->pages, slice_index);+ }+ else {+ // os allocated+ mi_assert_internal(mi_memid_is_os(page->memid) && page->next == NULL);+ }if (theap != NULL) {mi_theap_stat_decrease(theap, page_bins[sbin], 1);mi_theap_stat_decrease(theap, pages, 1);@@ -2347,16 +2608,18 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* aremi_theap_t* theap_target = info->theap_target;// and then add it to the new target heap- mi_arena_pages_t* arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);- if mi_unlikely(arena_pages_target==NULL) {- // if we cannot allocate this, we move it to the main heap instead (which does not require allocation)- heap_target = mi_heap_main();- theap_target = mi_heap_theap(heap_target);- arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);- mi_assert_internal(arena_pages_target!=NULL);- }- mi_assert_internal(mi_bitmap_is_clear(arena_pages_target->pages, slice_index));- mi_bitmap_set(arena_pages_target->pages, slice_index);+ if (arena != NULL) {+ mi_arena_pages_t* arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);+ if mi_unlikely(arena_pages_target==NULL) {+ // if we cannot allocate this, we move it to the main heap instead (which does not require allocation)+ heap_target = mi_arena_heap_main(arena);+ theap_target = mi_heap_theap(heap_target); // todo: find through theap_target tld?+ arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);+ mi_assert_internal(arena_pages_target!=NULL);+ }+ mi_assert_internal(mi_bitmap_is_clear(arena_pages_target->pages, slice_index));+ mi_bitmap_set(arena_pages_target->pages, slice_index);+ }page->heap = heap_target;mi_theap_stat_increase(theap_target, page_bins[sbin], 1);mi_theap_stat_increase(theap_target, pages, 1);@@ -2374,7 +2637,7 @@ static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) {_mi_heap_visit_blocks(heap, false, false, &mi_heap_delete_page, &info);#if MI_DEBUG>1// no more arena pages?- for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {+ for (size_t i = 0; i < MI_MAX_ARENAS; i++) {mi_arena_pages_t* const arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);if (arena_pages!=NULL) {mi_assert_internal(mi_bitmap_is_all_clear(arena_pages->pages));@@ -2386,7 +2649,7 @@ static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) {mi_assert_internal(heap->os_abandoned_pages == NULL);}// nor arena abandoned pages?- for (size_t i = 0; i < MI_BIN_COUNT; i++) {+ for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {mi_assert_internal(mi_atomic_load_relaxed(&heap->abandoned_count[i])==0);}#endif@@ -2394,7 +2657,7 @@ static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) {void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to) {if (_mi_is_heap_main(heap_from)) return;- if (heap_to==NULL) { heap_to = mi_heap_main(); }+ if (heap_to==NULL) { heap_to = heap_from->subproc->heap_main; }mi_heap_delete_pages(heap_from, heap_to);}@@ -2443,7 +2706,7 @@ mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t*// adjust abandoned page countmi_subproc_t* const subproc = arena->subproc;- for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {+ for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) {const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]);if (count > 0) { mi_atomic_decrement_acq_rel(&subproc->abandoned_count[bin]); }}@@ -2503,7 +2766,7 @@ mi_decl_export bool mi_arena_reload(void* start, size_t size, mi_commit_fun_t* c}// adjust abandoned page count- for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {+ for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) {const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]);if (count > 0) { mi_atomic_decrement_acq_rel(&arena->subproc->abandoned_count[bin]); }}diff --git a/vendored/mimalloc/src/bitmap.c b/vendored/mimalloc/src/bitmap.cindex 67269e8bf..238c4d4c4 100644--- a/vendored/mimalloc/src/bitmap.c+++ b/vendored/mimalloc/src/bitmap.c@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2019-2024 Microsoft Research, Daan Leijen+Copyright (c) 2019-2026 Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -107,9 +107,9 @@ static inline bool mi_bfield_atomic_clear(_Atomic(mi_bfield_t)*b, size_t idx, bo}// Clear a bit but only when/once it is set. This is used by concurrent free's while-// the page is abandoned and mapped. This can incure a busy wait :-( but it should-// happen almost never (and is accounted for in the stats)-static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_t idx) {+// the page is abandoned and mapped. This can incur a busy wait :-( but it should+// be quite rare (and is accounted for in the stats)+static inline void mi_bfield_atomic_clear_once_set(mi_subproc_t* subproc, _Atomic(mi_bfield_t)*b, size_t idx) {mi_assert_internal(idx < MI_BFIELD_BITS);const mi_bfield_t mask = mi_bfield_mask(1, idx);;mi_bfield_t old = mi_atomic_load_relaxed(b);@@ -117,7 +117,7 @@ static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_if mi_unlikely((old&mask) == 0) {old = mi_atomic_load_acquire(b);if ((old&mask)==0) {- mi_subproc_stat_counter_increase(_mi_subproc(), pages_unabandon_busy_wait, 1);+ mi_subproc_stat_counter_increase(subproc, pages_unabandon_busy_wait, 1);}while ((old&mask)==0) { // busy wait_mi_prim_thread_yield();@@ -133,8 +133,7 @@ static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_// statistics correctly).static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, size_t* already_set) {mi_assert_internal(mask != 0);- mi_bfield_t old = mi_atomic_load_relaxed(b);- while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)) {}; // try to atomically set the mask bits until success+ const mi_bfield_t old = mi_atomic_or_acq_rel(b,mask);if (already_set!=NULL) { *already_set = mi_bfield_popcount(old&mask); }return ((old&mask) == 0);}@@ -143,8 +142,7 @@ static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t// `all_clear` is set to `true` if the new bfield became zero.static inline bool mi_bfield_atomic_clear_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, bool* all_clear) {mi_assert_internal(mask != 0);- mi_bfield_t old = mi_atomic_load_relaxed(b);- while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)) {}; // try to atomically clear the mask bits until success+ const mi_bfield_t old = mi_atomic_and_acq_rel(b,~mask);if (all_clear != NULL) { *all_clear = ((old&~mask)==0); }return ((old&mask) == mask);}@@ -155,65 +153,79 @@ static inline bool mi_bfield_atomic_setX(_Atomic(mi_bfield_t)*b, size_t* alreadyreturn (old==0);}-// static inline bool mi_bfield_atomic_clearX(_Atomic(mi_bfield_t)*b, bool* all_clear) {-// const mi_bfield_t old = mi_atomic_exchange_release(b, mi_bfield_zero());-// if (all_clear!=NULL) { *all_clear = true; }-// return (~old==0);-// }// ------- mi_bfield_atomic_try_clear ----------------------------------------// Tries to clear a mask atomically, and returns true if the mask bits atomically transitioned from mask to 0-// and false otherwise (leaving the bit field as is).+// and false otherwise (leaving the bit field as is). Returns the acquired value back in `expect` (regardless of success).// `all_clear` is set to `true` if the new bfield became zero.-static inline bool mi_bfield_atomic_try_clear_mask_of(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, mi_bfield_t expect, bool* all_clear) {+// `did_temp_clear_bits` is set to `true` some bits were temporarily cleared.+static inline bool mi_bfield_atomic_try_clear_mask_optimistic(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, mi_bfield_t* previous, bool* did_temp_clear_bits) {mi_assert_internal(mask != 0);+ // note: we could also use a strong cas but generally an optimistic atomic and/or is more efficient (at least on arm64)// try to atomically clear the mask bits- do {- if ((expect & mask) != mask) { // are all bits still set?- if (all_clear != NULL) { *all_clear = (expect == 0); }- return false;+ mi_bfield_t old = mi_atomic_and_acq_rel(b,~mask);+ if (previous!=NULL) { *previous = old; }+ if mi_likely((old&mask)==mask) {+ // all bits in the mask transitioned from 1 to 0 atomically+ return true;+ }+ else {+ // failed to transition all+ if ((old&mask)!=0) {+ // restore accidentally cleared ones+ mi_atomic_or_acq_rel(b, old&mask);+ if (did_temp_clear_bits!=NULL) { *did_temp_clear_bits = true; }}- } while (!mi_atomic_cas_weak_acq_rel(b, &expect, expect & ~mask));- if (all_clear != NULL) { *all_clear = ((expect & ~mask) == 0); }- return true;-}--static inline bool mi_bfield_atomic_try_clear_mask(_Atomic(mi_bfield_t)* b, mi_bfield_t mask, bool* all_clear) {- mi_assert_internal(mask != 0);- const mi_bfield_t expect = mi_atomic_load_relaxed(b);- return mi_bfield_atomic_try_clear_mask_of(b, mask, expect, all_clear);+ return false;+ }}-// Tries to clear a bit atomically. Returns `true` if the bit transitioned from 1 to 0-// and `false` otherwise leaving the bfield `b` as-is.+// Tries to clear a bit atomically. For performance, it assumes there is a good chance of success.+// Returns `true` if the bit transitioned from 1 to 0+// and `false` otherwise (leaving the bfield `b` as-is).// `all_clear` is set to true if the new bfield became zero (and false otherwise)-mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear(_Atomic(mi_bfield_t)* b, size_t idx, bool* all_clear) {+mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear_optimistic(_Atomic(mi_bfield_t)* b, size_t idx, mi_bfield_t* previous) {mi_assert_internal(idx < MI_BFIELD_BITS);- const mi_bfield_t mask = mi_bfield_one()<<idx;- return mi_bfield_atomic_try_clear_mask(b, mask, all_clear);+ const mi_bfield_t mask = ((mi_bfield_t)1<<idx);+ return mi_bfield_atomic_try_clear_mask_optimistic(b, mask, previous, NULL); // single bit never clears temporarily}-// Tries to clear a byte atomically, and returns true if the byte atomically transitioned from 0xFF to 0+// Tries to clear a byte atomically. For performance, it assumes there is a good chance of success.+// Returns true if the byte atomically transitioned from 0xFF to 0// `all_clear` is set to true if the new bfield became zero (and false otherwise)-mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear8(_Atomic(mi_bfield_t)*b, size_t idx, bool* all_clear) {+mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear8_optimistic(_Atomic(mi_bfield_t)*b, size_t idx, mi_bfield_t* previous, bool* did_temp_clear_bits) {mi_assert_internal(idx < MI_BFIELD_BITS);mi_assert_internal((idx%8)==0);const mi_bfield_t mask = ((mi_bfield_t)0xFF)<<idx;- return mi_bfield_atomic_try_clear_mask(b, mask, all_clear);+ return mi_bfield_atomic_try_clear_mask_optimistic(b, mask, previous, did_temp_clear_bits);+}++// Try to clear a mask of bits atomically. For performance, it assumes these is a good chance of success.+// Returns true all bits transitioned from all 1's to 0's.+// and false otherwise leaving the bit field as-is. `all_clear` is set to true if the new bfield became zero.+static inline bool mi_bfield_atomic_try_clear_mask(_Atomic(mi_bfield_t)* b, mi_bfield_t mask, bool* all_clear, bool* did_temp_clear_bits) {+ mi_assert_internal(mask != 0);+ mi_bfield_t old = mi_atomic_load_relaxed(b); // check first if there is a good chance the mask bits are all set+ bool cleared = false;+ if ((old&mask)==mask) {+ cleared = mi_bfield_atomic_try_clear_mask_optimistic(b, mask, &old, did_temp_clear_bits);+ }+ if (all_clear != NULL) { *all_clear = ((cleared ? old&~mask : old) == 0); }+ return cleared;}// Try to clear a full field of bits atomically, and return true all bits transitioned from all 1's to 0's.// and false otherwise leaving the bit field as-is.// `all_clear` is set to true if the new bfield became zero (which is always the case if successful).static inline bool mi_bfield_atomic_try_clearX(_Atomic(mi_bfield_t)*b, bool* all_clear) {- mi_bfield_t old = mi_bfield_all_set();- if (mi_atomic_cas_strong_acq_rel(b, &old, mi_bfield_zero())) {- if (all_clear != NULL) { *all_clear = true; }- return true;+ mi_bfield_t old = mi_atomic_load_relaxed(b); // check first if there is a good chance the bits are all set+ bool cleared = false;+ if (old==mi_bfield_all_set()) {+ cleared = mi_atomic_cas_strong_acq_rel(b,&old,mi_bfield_zero());}- else return false;+ if (all_clear != NULL) { *all_clear = (cleared ? true : (old==0)); }+ return cleared;}@@ -331,7 +343,7 @@ mi_decl_noinline static bool mi_bchunk_xsetNC(mi_xset_t set, mi_bchunk_t* chunk,bool all_clear = false;const bool transition = (set ? mi_bfield_atomic_set_mask(&chunk->bfields[field], mask, &already_set): mi_bfield_atomic_clear_mask(&chunk->bfields[field], mask, &all_clear));- mi_assert_internal((transition && already_set == 0) || (!transition && already_set > 0));+ mi_assert_internal(!set || ((transition && already_set == 0) || (!transition && already_set > 0)));all_transition = all_transition && transition;total_already_set += already_set;maybe_all_clear = maybe_all_clear && all_clear;@@ -369,7 +381,7 @@ static inline bool mi_bchunk_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bif (n==1) return mi_bchunk_clear(chunk, cidx, maybe_all_clear);// if (n==8) return mi_bchunk_clear8(chunk, cidx, maybe_all_clear);// if (n==MI_BFIELD_BITS) return mi_bchunk_clearX(chunk, cidx, maybe_all_clear);- // TODO: implement mi_bchunk_xsetNX instead of setNX+ // todo: implement mi_bchunk_xsetNX instead of setNXreturn mi_bchunk_xsetNC(MI_BIT_CLEAR, chunk, cidx, n, NULL, maybe_all_clear);}@@ -445,14 +457,14 @@ static inline bool mi_bchunk_is_xsetN(mi_xset_t set, const mi_bchunk_t* chunk, s// ------- mi_bchunk_try_clear ---------------------------------------// Clear `0 < n <= MI_BITFIELD_BITS`. Can cross over a bfield boundary.-static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) {+static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear, bool* did_temp_clear_bits) {mi_assert_internal(cidx < MI_BCHUNK_BITS);mi_assert_internal(n <= MI_BFIELD_BITS);const size_t i = cidx / MI_BFIELD_BITS;const size_t idx = cidx % MI_BFIELD_BITS;if mi_likely(idx + n <= MI_BFIELD_BITS) {// within one field- return mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(n, idx), pmaybe_all_clear);+ return mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(n, idx), pmaybe_all_clear, did_temp_clear_bits);}else {// spanning two fields (todo: use double-word atomic ops?)@@ -460,14 +472,15 @@ static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_tmi_assert_internal(m < n);mi_assert_internal(i < MI_BCHUNK_FIELDS - 1);bool field1_is_clear;- if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &field1_is_clear)) return false;+ if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &field1_is_clear, did_temp_clear_bits)) return false;// try the second field as wellmi_assert_internal(n - m > 0);mi_assert_internal(n - m < MI_BFIELD_BITS);bool field2_is_clear;- if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &field2_is_clear)) {+ if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &field2_is_clear, did_temp_clear_bits)) {// we failed to clear the second field, restore the first onemi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), NULL);+ if (did_temp_clear_bits != NULL) { *did_temp_clear_bits = true; }return false;}if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = field1_is_clear && field2_is_clear; }@@ -488,7 +501,7 @@ static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t// and false otherwise leaving all bit fields as is.// Note: this is the complex one as we need to unwind partial atomic operations if we fail halfway..// `maybe_all_clear` is set to `true` if all the bfields involved become zero.-mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) {+mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear, bool* did_temp_clear_bits) {mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);mi_assert_internal(n>0);if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = true; }@@ -504,7 +517,7 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cimi_assert_internal(start_field < MI_BCHUNK_FIELDS);const mi_bfield_t mask_start = mi_bfield_mask(m, start_idx);bool maybe_all_clear;- if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_start, &maybe_all_clear)) return false;+ if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_start, &maybe_all_clear, did_temp_clear_bits)) return false;// done?mi_assert_internal(m <= n);@@ -528,7 +541,7 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cimi_assert_internal(field < MI_BCHUNK_FIELDS);const mi_bfield_t mask_end = mi_bfield_mask(n, 0);bool field_is_clear;- if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_end, &field_is_clear)) goto restore;+ if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_end, &field_is_clear, did_temp_clear_bits)) goto restore;maybe_all_clear = maybe_all_clear && field_is_clear;}@@ -538,6 +551,7 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cirestore:// `field` is the index of the field that failed to set atomically; we need to restore all previous fieldsmi_assert_internal(field > start_field);+ if (did_temp_clear_bits != NULL) { *did_temp_clear_bits = true; }while( field > start_field) {field--;if (field == start_field) {@@ -551,11 +565,11 @@ restore:}-static inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear) {+static inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear, bool* did_temp_clear_bits) {mi_assert_internal(n>0);// if (n==MI_BFIELD_BITS) return mi_bchunk_try_clearX(chunk, cidx, maybe_all_clear);- if (n<=MI_BFIELD_BITS) return mi_bchunk_try_clearNX(chunk, cidx, n, maybe_all_clear);- return mi_bchunk_try_clearNC(chunk, cidx, n, maybe_all_clear);+ if (n<=MI_BFIELD_BITS) return mi_bchunk_try_clearNX(chunk, cidx, n, maybe_all_clear, did_temp_clear_bits);+ return mi_bchunk_try_clearNC(chunk, cidx, n, maybe_all_clear, did_temp_clear_bits);}@@ -576,19 +590,24 @@ mi_decl_maybe_unused static inline bool mi_mm256_is_zero( __m256i vec) {}#endif+static inline bool mi_bchunk_try_find_and_clear_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) {mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS);- // note: this must be acquire (and not relaxed), or otherwise the AVX code below can loop forever- // as the compiler won't reload the registers vec1 and vec2 from memory again.- const mi_bfield_t b = mi_atomic_load_acquire(&chunk->bfields[chunk_idx]);- size_t idx;- if (mi_bfield_find_least_bit(b, &idx)) { // find the least bit- if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], mi_bfield_mask(1,idx), b, NULL)) { // clear it atomically- *pidx = (chunk_idx*MI_BFIELD_BITS) + idx;+ _Atomic(mi_bfield_t)* const bfield = &chunk->bfields[chunk_idx];+ mi_bfield_t b = mi_atomic_load_relaxed(bfield);+ if (b==0) return false;+ int tries = 0;+ do {+ const mi_bfield_t mask = (b & (~b+1)); // == (b & -b) but avoids a compiler warning -- clear all bits except the least-significant one+ b = mi_atomic_and_acq_rel(bfield,~mask); // clear the bit and set `b` to the previous value+ if mi_likely((b&mask)==mask) { // if we transitioned from 1 to 0, we actually cleared it+ size_t bitidx = 0;+ mi_bfield_find_least_bit(mask,&bitidx);+ *pidx = (chunk_idx*MI_BFIELD_BITS) + bitidx;mi_assert_internal(*pidx < MI_BCHUNK_BITS);return true;- }- }+ }+ } while (b!=0 && ++tries <= 4); // limit tries to reduce possible contentionreturn false;}@@ -686,28 +705,36 @@ static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidxreturn false;}-static inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx) {- mi_assert_internal(n==1); MI_UNUSED(n);+static inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) {+ mi_assert_internal(n==1); MI_UNUSED(n); MI_UNUSED(did_temp_clear_bits);return mi_bchunk_try_find_and_clear(chunk, pidx);}-mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) {- const mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[chunk_idx]);- // has_set8 has low bit in each byte set if the byte in x == 0xFF- const mi_bfield_t has_set8 =- ((~b - MI_BFIELD_LO_BIT8) & // high bit set if byte in x is 0xFF or < 0x7F- (b & MI_BFIELD_HI_BIT8)) // high bit set if byte in x is >= 0x80- >> 7; // shift high bit to low bit- size_t idx;- if (mi_bfield_find_least_bit(has_set8, &idx)) { // find least 1-bit- mi_assert_internal(idx <= (MI_BFIELD_BITS - 8));- mi_assert_internal((idx%8)==0);- if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], (mi_bfield_t)0xFF << idx, b, NULL)) { // unset the byte atomically- *pidx = (chunk_idx*MI_BFIELD_BITS) + idx;- mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS);- return true;+mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx, bool* did_temp_clear_bits) {+ _Atomic(mi_bfield_t)* const bfield = &chunk->bfields[chunk_idx];+ mi_bfield_t b = mi_atomic_load_relaxed(bfield);+ if (b==0) return false;+ int tries = 0;+ do {+ // has_set8 has low bit in each byte set if the byte in x == 0xFF+ const mi_bfield_t has_set8 =+ ((~b - MI_BFIELD_LO_BIT8) & // high bit set if byte in x is 0xFF or < 0x7F+ (b & MI_BFIELD_HI_BIT8)) // high bit set if byte in x is >= 0x80+ >> 7; // shift high bit to low bit+ size_t bitidx;+ if (mi_bfield_find_least_bit(has_set8, &bitidx)) { // find least 1-bit+ mi_assert_internal(bitidx <= (MI_BFIELD_BITS - 8));+ mi_assert_internal((bitidx%8)==0);+ if mi_likely(mi_bfield_atomic_try_clear8_optimistic(bfield,bitidx,&b,did_temp_clear_bits)) { // try to clear atomically (and update b)+ *pidx = (chunk_idx*MI_BFIELD_BITS) + bitidx;+ mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS);+ return true;+ }}- }+ else {+ return false;+ }+ } while(b!=0 && ++tries <= 4); // limit tries to reduce possible contentionreturn false;}@@ -715,9 +742,9 @@ mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchu// set `*pidx` to its bit index (0 <= *pidx < MI_BCHUNK_BITS) on success.// Used to find medium size pages in the free blocks.// todo: try neon version-static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, size_t* pidx) {+static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, size_t* pidx, bool* did_temp_clear_bits) {#if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)- while (true) {+ for(int tries=0; tries<4; tries++) { // paranoia: at most 4 tries// since a cache-line is 64b, load all at onceconst __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);const __m256i vec2 = _mm256_load_si256((const __m256i*)chunk->bfields+1);@@ -733,25 +760,29 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, sconst size_t chunk_idx = bidx / 8;const size_t idx = (bidx % 8)*8;mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS);- if mi_likely(mi_bfield_atomic_try_clear8(&chunk->bfields[chunk_idx], idx, NULL)) { // clear it atomically+ if mi_likely(mi_bfield_atomic_try_clear8_optimistic(&chunk->bfields[chunk_idx], idx, NULL, did_temp_clear_bits)) { // clear it atomically*pidx = (chunk_idx*MI_BFIELD_BITS) + idx;mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS);return true;}// try again- // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded }+ // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded+ // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206)+ #if defined(__GNUC__)+ __asm __volatile ("" : : "g"(chunk) : "memory");+ #endif}#elsefor (int i = 0; i < MI_BCHUNK_FIELDS; i++) {- if (mi_bchunk_try_find_and_clear8_at(chunk, i, pidx)) return true;+ if (mi_bchunk_try_find_and_clear8_at(chunk, i, pidx, did_temp_clear_bits)) return true;}- return false;#endif+ return false;}-static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx) {- mi_assert_internal(n==8); MI_UNUSED(n);- return mi_bchunk_try_find_and_clear8(chunk, pidx);+static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) {+ mi_assert_internal(n==8); MI_UNUSED(n); MI_UNUSED(did_temp_clear_bits);+ return mi_bchunk_try_find_and_clear8(chunk, pidx, did_temp_clear_bits);}@@ -759,7 +790,7 @@ static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n,// and try to clear them atomically.// set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success.// will cross bfield boundaries.-mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx) {+mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) {if (n == 0 || n > MI_BFIELD_BITS) return false;const mi_bfield_t mask = mi_bfield_mask(n, 0);// for all fields in the chunk@@ -775,7 +806,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk,const size_t bmask = mask<<idx;mi_assert_internal(bmask>>idx == mask);if ((b&bmask) == bmask) { // found a match with all bits set, try clearing atomically- if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[i], bmask, b0, NULL)) {+ if mi_likely(mi_bfield_atomic_try_clear_mask_optimistic(&chunk->bfields[i], bmask, &b0, did_temp_clear_bits)) { // updates b0 with the previous value*pidx = (i*MI_BFIELD_BITS) + idx;mi_assert_internal(*pidx < MI_BCHUNK_BITS);mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS);@@ -783,7 +814,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk,}else {// if we failed to atomically commit, reload b and try again from the start- b = b0 = mi_atomic_load_acquire(&chunk->bfields[i]);+ b = b0; // = mi_atomic_load_acquire(&chunk->bfields[i]);}}else {@@ -803,7 +834,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk,if (post + pre >= n) {// it fits -- try to claim it atomicallyconst size_t cidx = (i*MI_BFIELD_BITS) + (MI_BFIELD_BITS - post);- if (mi_bchunk_try_clearNX(chunk, cidx, n, NULL)) {+ if (mi_bchunk_try_clearNX(chunk, cidx, n, NULL, did_temp_clear_bits)) {// we cleared all atomically*pidx = cidx;mi_assert_internal(*pidx < MI_BCHUNK_BITS);@@ -821,7 +852,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk,// and try to clear them atomically.// set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success.// This can cross bfield boundaries.-static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx) {+static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) {if (n == 0 || n > MI_BCHUNK_BITS) return false; // cannot be more than a chunk// we first scan ahead to see if there is a range of `n` set bits, and only then try to clear atomically@@ -870,7 +901,7 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk,// did we find a range?if (m==0) {- if (mi_bchunk_try_clearN(chunk, cidx, n, NULL)) {+ if (mi_bchunk_try_clearN(chunk, cidx, n, NULL, did_temp_clear_bits)) {// we cleared all atomically*pidx = cidx;mi_assert_internal(*pidx < MI_BCHUNK_BITS);@@ -888,11 +919,11 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk,// ------- mi_bchunk_clear_once_set ----------------------------------------static inline void mi_bchunk_clear_once_set(mi_bchunk_t* chunk, size_t cidx) {+static inline void mi_bchunk_clear_once_set(mi_subproc_t* subproc, mi_bchunk_t* chunk, size_t cidx) {mi_assert_internal(cidx < MI_BCHUNK_BITS);const size_t i = cidx / MI_BFIELD_BITS;const size_t idx = cidx % MI_BFIELD_BITS;- mi_bfield_atomic_clear_once_set(&chunk->bfields[i], idx);+ mi_bfield_atomic_clear_once_set(subproc, &chunk->bfields[i], idx);}@@ -963,7 +994,7 @@ static bool mi_bchunk_bsr(mi_bchunk_t* chunk, size_t* pidx) {return false;}-static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) {+static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) {for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) {i--;mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);@@ -1050,7 +1081,7 @@ static void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, si// start chunk and indexsize_t chunk_idx = idx / MI_BCHUNK_BITS;const size_t cidx = idx % MI_BCHUNK_BITS;- const size_t ccount = _mi_divide_up(n, MI_BCHUNK_BITS);+ const size_t ccount = ((cidx + n - 1) / MI_BCHUNK_BITS) + 1;// first update the chunkmapmi_bchunk_setN(cmap, chunk_idx, ccount, NULL);@@ -1323,9 +1354,10 @@ static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunkreturn true;}else {- // failed to claim it, set abandoned mapping again (unless the page was freed)+ // failed to claim it, set abandoned mapping again (unless the page was freed and keep_set will be false)if (keep_set) {const bool wasclear = mi_bchunk_set(&bitmap->chunks[chunk_idx], cidx, NULL);+ mi_bitmap_chunkmap_set(bitmap, chunk_idx);mi_assert_internal(wasclear); MI_UNUSED(wasclear);}}@@ -1355,12 +1387,15 @@ bool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx) {mi_bfield_t cmap = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);size_t cmap_idx;if (mi_bsr(cmap,&cmap_idx)) {- // highest chunk- const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;- size_t cidx;- if (mi_bchunk_bsr(&bitmap->chunks[chunk_idx], &cidx)) {- *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;- return true;+ // from highest chunk to lowest (scan all in case the cmap entry was stale)+ for (size_t j = cmap_idx+1; j>0; ) {+ j--;+ const size_t chunk_idx = (i*MI_BFIELD_BITS) + j;+ size_t cidx;+ if (mi_bchunk_bsr(&bitmap->chunks[chunk_idx], &cidx)) {+ *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;+ return true;+ }}}}@@ -1388,12 +1423,12 @@ size_t mi_bitmap_popcount(mi_bitmap_t* bitmap) {// Clear a bit once it is set.-void mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx) {+void mi_bitmap_clear_once_set(mi_subproc_t* subproc, mi_bitmap_t* bitmap, size_t idx) {mi_assert_internal(idx < mi_bitmap_max_bits(bitmap));const size_t chunk_idx = idx / MI_BCHUNK_BITS;const size_t cidx = idx % MI_BCHUNK_BITS;mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap));- mi_bchunk_clear_once_set(&bitmap->chunks[chunk_idx], cidx);+ mi_bchunk_clear_once_set(subproc, &bitmap->chunks[chunk_idx], cidx);}@@ -1456,6 +1491,8 @@ bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* vismi_assert_internal(rng>=1 && rng<=MI_BFIELD_BITS);mi_assert_internal((idx % MI_BFIELD_BITS) + rng <= MI_BFIELD_BITS);mi_assert_internal((idx / MI_BCHUNK_BITS) < mi_bitmap_chunk_count(bitmap));+ // clear rng bits in b+ b = b & ~mi_bfield_mask(rng, bidx);if (!visit(idx, rng, arena, arg)) {// break early: reset the non-visited bitsif (b!=0) {@@ -1463,8 +1500,6 @@ bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* vis}return false;}- // clear rng bits in b- b = b & ~mi_bfield_mask(rng, bidx);}mi_assert_internal(rngcount == bpopcount);}@@ -1502,14 +1537,15 @@ bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_foconst size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0); // atomic clearmi_bfield_t skipped = 0; // but track which bits we skip so we can restore them- for(size_t shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) { // per `rngslices` to keep alignment+ size_t shift;+ for(shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) { // per `rngslices` to keep alignmentconst mi_bfield_t rngmask = mi_bfield_mask(rngslices, shift);if ((b & rngmask) == rngmask) {const size_t idx = base_idx + shift;if (!visit(idx, rngslices, arena, arg)) {// break early: restore non-visited entriesmi_bfield_t notyet_visited = 0;- if (shift + rngslices < MI_BFIELD_BITS) {+ if (rngslices + shift < MI_BFIELD_BITS) {notyet_visited = (b & (~(mi_bfield_t)0 << (shift + rngslices)));}mi_assert_internal((notyet_visited & skipped) == 0);@@ -1523,8 +1559,13 @@ bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_foskipped = skipped | (b & rngmask);}}-+ if (shift < MI_BFIELD_BITS) {+ // there are some non-visited top bits when `MI_BFIELD_BITS % rngslices != 0`.+ mi_assert_internal(MI_BFIELD_BITS % rngslices != 0);+ skipped = skipped | (b & (~(mi_bfield_t)0 << shift));+ }if (skipped != 0) {+ // restore non-visited entriesmi_atomic_or_relaxed(&chunk->bfields[j], skipped);}}@@ -1554,7 +1595,7 @@ size_t mi_bbitmap_size(size_t bit_count, size_t* pchunk_count) {// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true// returns the size of the bitmap-size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero) {+size_t mi_bbitmap_init(mi_subproc_t* subproc, mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero) {size_t chunk_count;const size_t size = mi_bbitmap_size(bit_count, &chunk_count);if (!already_zero) {@@ -1562,6 +1603,7 @@ size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zer}mi_atomic_store_release(&bbitmap->chunk_count, chunk_count);mi_assert_internal(mi_atomic_load_relaxed(&bbitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT);+ bbitmap->subproc = subproc;return size;}@@ -1572,27 +1614,16 @@ void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {}bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx) {+ // scan for highest zero bit in the bitmap+ // note: we cannot use the chunkmap since that only conservatively denotes if there might be a set bit in a chuck+ // todo: bbitmap_init rounds up the bitcount to BCHUNK_BITS and we should skip the top-padding!const size_t chunk_count = mi_bbitmap_chunk_count(bbitmap);- const size_t chunkmap_max = _mi_divide_up(chunk_count, MI_BFIELD_BITS);- size_t skip_at_top = chunk_count % MI_BFIELD_BITS;- for (size_t i = chunkmap_max; i > 0; ) {+ for(size_t i = chunk_count; i > 0; ) {i--;- mi_bfield_t cmap = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[i]);- size_t cmap_idx;- // don't consider top 0 bits; set those to 1 here- if (skip_at_top > 0) {- const size_t mask_top = (~mi_bfield_zero()) << (MI_BFIELD_BITS - skip_at_top);- skip_at_top = 0; // only for the first iteration- cmap |= mask_top;- }- if (mi_bsr(~cmap, &cmap_idx)) {- // highest chunk- const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;- size_t cidx;- if (mi_bchunk_bsr_inv(&bbitmap->chunks[chunk_idx], &cidx)) {- *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;- return true;- }+ size_t cidx;+ if (mi_bchunk_bsr_inv(&bbitmap->chunks[i], &cidx)) {+ *idx = (i * MI_BCHUNK_BITS) + cidx;+ return true;}}return false;@@ -1609,11 +1640,11 @@ static void mi_bbitmap_set_chunk_bin(mi_bbitmap_t* bbitmap, size_t chunk_idx, mifor (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) {if (ibin == bin) {const bool was_clear = mi_bchunk_set(& bbitmap->chunkmap_bins[ibin], chunk_idx, NULL);- if (was_clear) { mi_os_stat_increase(chunk_bins[ibin],1); }+ if (was_clear) { mi_subproc_stat_increase(bbitmap->subproc, chunk_bins[ibin],1); }}else {const bool was_set = mi_bchunk_clear(&bbitmap->chunkmap_bins[ibin], chunk_idx, NULL);- if (was_set) { mi_os_stat_decrease(chunk_bins[ibin],1); }+ if (was_set) { mi_subproc_stat_decrease(bbitmap->subproc,chunk_bins[ibin],1); }}}}@@ -1710,8 +1741,15 @@ bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap));if (cidx + n > MI_BCHUNK_BITS) return false;bool maybe_all_clear = false;- const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear);- if (cleared && maybe_all_clear) { mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); }+ bool did_temp_clear_bits = false;+ const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear, &did_temp_clear_bits);+ if (cleared && maybe_all_clear) {+ mi_assert_internal(!did_temp_clear_bits);+ mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx);+ } else if (did_temp_clear_bits) {+ // may have raced with a clearer (in on_find) so set the chunkmap bit conservatively+ mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false);+ }// note: we don't set the size class for an explicit try_clearN (only used by purging)return cleared;}@@ -1753,14 +1791,14 @@ bool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_(used to find free pages)-------------------------------------------------------------------------------- */-typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx);+typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx, bool* did_temp_clear_bits);// Go through the bbitmap and for every sequence of `n` set bits, call the visitor function.// If it returns `true` stop the search.//// This is used for finding free blocks and it is important to be efficient (with 2-level bitscan)// but also reduce fragmentation (through size bins).-static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find)+static bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find){// we space out threads to reduce contentionconst size_t cmap_max_count = _mi_divide_up(mi_bbitmap_chunk_count(bbitmap),MI_BFIELD_BITS);@@ -1814,7 +1852,8 @@ static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap,mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx];size_t cidx;- if ((*on_find)(chunk, n, &cidx)) {+ bool did_temp_clear_bits = false;+ if ((*on_find)(chunk, n, &cidx, &did_temp_clear_bits)) {if (cidx==0 && ibin == MI_CBIN_NONE) { // only the first block determines the size bin// this chunk is now reserved for the `bbin` size classmi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, bbin);@@ -1824,9 +1863,17 @@ static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap,return true;}else {+ // note: we may find that all are cleared only on a second iteration (when we fail to clear any bits)+ // but that is ok as the chunkmap is a conservative approximation.// todo: should _on_find_ return a boolean if there is a chance all are clear to avoid calling `try_clear?`- // we may find that all are cleared only on a second iteration but that is ok as the chunkmap is a conservative approximation.- mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx);+ // probably not as we already only call `try_clear` once we fail to clear any bits.+ if (did_temp_clear_bits) {+ // a concurrent find_and_claim may have cleared the chunkmap bit, restore it now+ mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false);+ }+ else {+ mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx);+ }}}mi_bfield_cycle_iterate_end(Y);@@ -1871,12 +1918,11 @@ bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t// Try to atomically clear `n` bits starting at `chunk_idx` where `n` can span over multiple chunksstatic bool mi_bchunk_try_clearN_(mi_bbitmap_t* bbitmap, size_t chunk_idx, size_t n) {mi_assert_internal((chunk_idx * MI_BCHUNK_BITS) + n <= mi_bbitmap_max_bits(bbitmap));-size_t m = n; // bits to gosize_t count = 0; // chunk countwhile (m > 0) {mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count];- if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL)) {+ if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL, NULL)) {goto rollback;}m = (m <= MI_BCHUNK_BITS ? 0 : m - MI_BCHUNK_BITS);@@ -1890,6 +1936,8 @@ rollback:count--;mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count];mi_bchunk_setN(chunk, 0, MI_BCHUNK_BITS, NULL);+ // since we may race with clearing, we need to set the chunkmap conservatively+ mi_bbitmap_chunkmap_set(bbitmap, chunk_idx + count, false);}return false;}@@ -1926,7 +1974,7 @@ bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t// did we find a suitable range?if (count == chunk_req) {- // now try to claim it!+ // now try to claim it!if (mi_bchunk_try_clearN_(bbitmap, chunk_idx, n)) {*pidx = (chunk_idx * MI_BCHUNK_BITS);for (size_t i = 0; i < count; i++) {@@ -1935,6 +1983,11 @@ bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_tmi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap));return true;}+ else {+ // contended: we reset count to retry from the first+ // (we still skip the first chunk to guarantee progress)+ count = 0;+ }}// keep searching but skip the scanned rangediff --git a/vendored/mimalloc/src/bitmap.h b/vendored/mimalloc/src/bitmap.hindex ca51a4e04..79eda77fa 100644--- a/vendored/mimalloc/src/bitmap.h+++ b/vendored/mimalloc/src/bitmap.h@@ -71,7 +71,7 @@ typedef size_t mi_bfield_t;#define MI_BCHUNK_FIELDS (MI_BCHUNK_BITS / MI_BFIELD_BITS) // 8 on both 64- and 32-bit-// some compiler (msvc in C mode) cannot have expressions in the alignment attribute+// some compilers (msvc in C mode) cannot have expressions in the alignment attribute#if MI_BCHUNK_SIZE==64#define mi_decl_bchunk_align mi_decl_align(64)#elif MI_BCHUNK_SIZE==32@@ -95,11 +95,11 @@ typedef mi_bchunk_t mi_bchunkmap_t;#define MI_BITMAP_MAX_CHUNK_COUNT (MI_BCHUNKMAP_BITS)#define MI_BITMAP_MIN_CHUNK_COUNT (1)-#if MI_SIZE_BITS > 32-#define MI_BITMAP_DEFAULT_CHUNK_COUNT (64) // 2 GiB on 64-bit -- this is for the page map-#else+// #if MI_SIZE_BITS > 32+// #define MI_BITMAP_DEFAULT_CHUNK_COUNT (128) // 4 GiB on 64-bit -- this is for the page map+// #else#define MI_BITMAP_DEFAULT_CHUNK_COUNT (1)-#endif+// #endif#define MI_BITMAP_MAX_BIT_COUNT (MI_BITMAP_MAX_CHUNK_COUNT * MI_BCHUNK_BITS) // 16 GiB arena#define MI_BITMAP_MIN_BIT_COUNT (MI_BITMAP_MIN_CHUNK_COUNT * MI_BCHUNK_BITS) // 32 MiB arena#define MI_BITMAP_DEFAULT_BIT_COUNT (MI_BITMAP_DEFAULT_CHUNK_COUNT * MI_BCHUNK_BITS) // 2 GiB arena@@ -199,7 +199,7 @@ mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t// Atomically clear a bit but only if it is set. Will block otherwise until the bit is set.// This is used to delay free-ing a page that it at the same time being considered to be// allocated from `mi_arena_try_abandoned` (and is in the `claim` function of `mi_bitmap_try_find_and_claim`).-void mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx);+void mi_bitmap_clear_once_set(mi_subproc_t* subproc, mi_bitmap_t* bitmap, size_t idx);// If a bit is set in the bitmap, return `true` and set `idx` to the index of the highest bit.@@ -260,8 +260,9 @@ static inline mi_chunkbin_t mi_chunkbin_of(size_t slice_count) {typedef mi_decl_bchunk_align struct mi_bbitmap_s {_Atomic(size_t) chunk_count; // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS)_Atomic(size_t) chunk_max_accessed; // max chunk index that was once cleared or set- #if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 2- size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 2]; // suppress warning on msvc by aligning manually+ mi_subproc_t* subproc; // constant, for stats+ #if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 3+ size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 3]; // suppress warning on msvc by aligning manually#endifmi_bchunkmap_t chunkmap;mi_bchunkmap_t chunkmap_bins[MI_CBIN_COUNT - 1]; // chunkmaps with bit set if the chunk is in that size class (excluding MI_CBIN_NONE)@@ -288,7 +289,7 @@ bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx);// Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true// returns the size of the bitmap.-size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero);+size_t mi_bbitmap_init(mi_subproc_t* subproc, mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero);// Set/clear a sequence of `n` bits in the bitmap (and can cross chunks).// Not atomic so only use if still local to a thread.diff --git a/vendored/mimalloc/src/free.c b/vendored/mimalloc/src/free.cindex 6f33340d0..722ee7bf6 100644--- a/vendored/mimalloc/src/free.c+++ b/vendored/mimalloc/src/free.c@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -9,15 +9,14 @@ terms of the MIT license. A copy of the license can be found in the file// add includes help an IDE#include "mimalloc.h"#include "mimalloc/internal.h"-#include "mimalloc/prim.h" // _mi_prim_thread_id()+#include "mimalloc/prim-tls.h" // _mi_prim_thread_id()#endif+// forward declarations-static void mi_check_padding(const mi_page_t* page, const mi_block_t* block);-static bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block);+mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size);+mi_decl_nodiscard static bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block);static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded);-static void mi_stat_free(const mi_page_t* page, const mi_block_t* block);-// ------------------------------------------------------// Free@@ -25,22 +24,29 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block);// regular free of a (thread local) block pointer// fast path written carefully to prevent spilling on the stack-static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full)+static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool check_full){- MI_UNUSED(was_guarded);// checks- if mi_unlikely(mi_check_is_double_free(page, block)) return;- if (!was_guarded) { mi_check_padding(page, block); }- if (track_stats) { mi_stat_free(page, block); }+ size_t usable_size;+ if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return;+ if mi_unlikely(!mi_check_double_free(page,block)) return; // usually checked with padding++ // mi_stat_free(page, block);+ mi_track_free_size(block, usable_size);+#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN- memset(block, MI_DEBUG_FREED, mi_page_block_size(page));+ const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size);+ _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize);#endif- if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); } // faster then mi_usable_size as we already know the page and that p is unaligned-+// actual free: push on the local free list+ mi_used_t xused = page->xused;+ xused.used_alloc--; // decrement used countmi_block_set_next(page, block, page->local_free);+ page->xused = xused;page->local_free = block;- if mi_unlikely(--page->used == 0) {+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));+ if mi_unlikely(mi_xused_used_count(xused) == 0) { // is used count zero ?if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page)_mi_page_retire(page);}@@ -51,75 +57,67 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool}// Forward declaration for multi-threaded collect-static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept;+static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept;// Free a block multi-threaded-static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded) mi_attr_noexcept+static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_reclaim) mi_attr_noexcept{- MI_UNUSED(was_guarded);- // adjust stats (after padding check and potentially recursive `mi_free` above)- mi_stat_free(page, block); // stat_free may access the padding- mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded));+ size_t usable_size;+ if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; // checking padding is safe for mt++ // adjust stats (after padding check )+ // mi_stat_free(page, block); // stat_free may access the padding+ mi_track_free_size(block, usable_size);// _mi_padding_shrink(page, block, sizeof(mi_block_t));-#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading+ #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threadingif (!was_guarded) {- size_t dbgsize = mi_usable_size(block);- if (dbgsize > MI_MiB) { dbgsize = MI_MiB; }+ const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size);_mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize);}-#endif+ #endif// push atomically on the page thread free list+ // mi_theap_t* theap = _mi_page_associated_theap_peek(page);mi_thread_free_t tf_new;mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free);do {mi_block_set_next(page, block, mi_tf_block(tf_old));- tf_new = mi_tf_create(block, true /* always use owned: try to claim it if the page is abandoned */);- } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // todo: release is enough?+ tf_new = mi_tf_create(block, true);+ } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new));// and atomically try to collect the page if it was abandoned- const bool is_owned_now = !mi_tf_is_owned(tf_old);- if (is_owned_now) {+ const bool is_newly_owned = mi_tf_is_owned(tf_new) && !mi_tf_is_owned(tf_old);+ if (is_newly_owned) {mi_assert_internal(mi_page_is_abandoned(page));- mi_free_try_collect_mt(page,block);+ // mi_assert_internal(!mi_page_is_abandoned_mapped(page));+ mi_free_try_collect_mt(page, block, allow_reclaim);}}// Adjust a block that was allocated aligned, to the actual start of the block in the page.// note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the-// `page_start` and `block_size` fields; however these are constant and the page won't be+// `page_woffset` and `block_size` fields; however these are constant and the page won't be// deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently.-mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {+static inline mi_block_t* mi_page_ptr_unalign_ex(const mi_page_t* page, const void* p, size_t* poffset) {mi_assert_internal(page!=NULL && p!=NULL);const size_t diff = (uint8_t*)p - mi_page_start(page);const size_t block_size = mi_page_block_size(page);- const size_t adjust = (_mi_is_power_of_two(block_size) ? diff & (block_size - 1) : diff % block_size);+ size_t adjust = diff & (block_size - 1);+ if mi_unlikely(!_mi_is_power_of_two(block_size)) {+ adjust = diff % block_size;+ }+ if (poffset!=NULL) { *poffset = adjust; }return (mi_block_t*)((uintptr_t)p - adjust);}-// forward declaration for a MI_GUARDED build-#if MI_GUARDED-static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration-static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {- if (mi_block_ptr_is_guarded(block, p)) {- mi_block_unguard(page, block, p);- return true;- }- else {- return false;- }-}-#else-static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {- MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p);- return false;+mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {+ return mi_page_ptr_unalign_ex(page,p,NULL);}-#endif-static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, void* p ) {+static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, const void* p ) {mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); // should never be an interior pointer#if MI_SECURE > 0// in secure mode we always unalign to guard against free-ing interior pointers@@ -130,63 +128,147 @@ static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, voi#endif}+static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool* was_guarded) mi_attr_noexcept {+ MI_UNUSED(was_guarded);+ if mi_likely(!mi_page_has_interior_pointers(page)) {+ return mi_validate_block_from_ptr(page,p);+ }+ else {+ size_t offset;+ mi_block_t* const block = mi_page_ptr_unalign_ex(page,p,&offset);+ #if MI_GUARDED || MI_PROFILE+ if (offset >= sizeof(mi_block_t)) {+ #if MI_PROFILE+ if (block->next == MI_BLOCK_TAG_PROFILED) {+ _mi_page_profile_on_free(page,block,p);+ }+ else+ #endif+ #if MI_GUARDED+ if (block->next == MI_BLOCK_TAG_GUARDED) {+ _mi_page_block_unguard(page, block, p);+ *was_guarded = true;+ }+ else+ #endif+ { /* nothing */ }+ }+ #endif+ return block;+ }+}// free a local pointer (page parameter comes first for better codegen)static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept {mi_assert_internal(p!=NULL && page != NULL);- mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));- const bool was_guarded = mi_block_check_unguard(page, block, p);- mi_free_block_local(page, block, was_guarded, true /* track stats */, true /* check for a full page */);+ bool was_guarded = false;+ mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded);+ // mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));+ // mi_block_check_profiled(page,block,p);+ // const bool was_guarded = mi_block_check_unguard(page, block, p);+ mi_free_block_local(page, block, was_guarded, true /* check for a full page */);}// free a pointer owned by another thread (page parameter comes first for better codegen)-static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p) mi_attr_noexcept {+static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p, bool allow_reclaim) mi_attr_noexcept {mi_assert_internal(p!=NULL && page != NULL);- mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));- const bool was_guarded = mi_block_check_unguard(page, block, p);- mi_free_block_mt(page, block, was_guarded);+ bool was_guarded = false;+ mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded);+ // mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));+ // mi_block_check_profiled(page,block,p);+ // const bool was_guarded = mi_block_check_unguard(page, block, p);+ mi_free_block_mt(page, block, was_guarded, allow_reclaim);}// generic free (for runtime integration)void mi_decl_noinline _mi_free_generic(mi_page_t* page, bool is_local, void* p) mi_attr_noexcept {if (is_local) mi_free_generic_local(page,p);- else mi_free_generic_mt(page,p);+ else mi_free_generic_mt(page,p,true);}// Get the page belonging to a pointer// Does further checks in debug mode to see if this was a valid pointer.-static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg)+static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const char* msg, bool free_small, bool check_p_for_null, mi_page_t** ppage){- MI_UNUSED_RELEASE(msg);+ MI_UNUSED_RELEASE(msg); MI_UNUSED(free_small);#if MI_DEBUG- if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) {+ if mi_unlikely(!_mi_is_aligned(p,MI_SIZE_SIZE) && !mi_option_is_enabled(mi_option_guarded_precise)) {_mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p);- return NULL;- }- mi_page_t* page = _mi_safe_ptr_page(p);- if (p != NULL && page == NULL) {- _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p);+ return false;}- return page;- #else- return _mi_ptr_page(p);#endif++ mi_page_t* page;+ #if MI_PAGE_META_SMALL_IS_ALIGNED+ if (free_small) {+ const uintptr_t up = _mi_align_down((uintptr_t)p,MI_SMALL_PAGE_SIZE); // like this for codegen on gcc+ if mi_unlikely(up==0 && check_p_for_null) { return false; }+ page = (mi_page_t*)up;+ }+ else+ #endif+ {+ #if MI_PAGE_META_IS_ALIGNED+ { page = _mi_aligned_ptr_page0(p); }+ #else+ { page = _mi_ptr_page(p); }+ #endif+ if mi_unlikely(page==NULL && check_p_for_null) {+ #if MI_DEBUG+ if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); }+ #endif+ return false;+ }+ }+ #if MI_DEBUG+ mi_page_t* const cpage = _mi_checked_ptr_page(p);+ if mi_unlikely(cpage==NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); }+ #endif++ #if MI_PAGE_META_IS_ALIGNED+ #if MI_PAGE_META_SMALL_IS_ALIGNED+ if (free_small) { mi_assert_internal(NULL == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); }+ else+ #elif MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE && !MI_GUARDED+ // for mi_free_small we can avoid a load-acquire (but not when guarded as that may still allocate large blocks)+ if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); }+ else+ #endif+ { page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); }+ #endif++ mi_assert_internal(page!=NULL);+ mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ );+ #if !MI_GUARDED+ if (free_small) { mi_assert_internal(page->block_size <= MI_SMALL_MAX_OBJ_SIZE); /* free small should only be called on small pages */ }+ #endif+ *ppage = page;+ return true;+}++static mi_decl_forceinline bool mi_ptr_page_is_valid(const void* p, const char* msg, mi_page_t** ppage) {+ return mi_ptr_page_is_valid_ex(p,msg,false /* free_small? */, true /* check_p_for_null */, ppage );+}++static mi_decl_forceinline mi_page_t* mi_ptr_page_validate(const void* p, const char* msg) {+ mi_page_t* page;+ return (mi_ptr_page_is_valid(p,msg,&page) ? page : NULL);}// Free a block// Fast path written carefully to prevent register spilling on the stack-static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable, mi_page_t* page)+static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t* pblock_size, bool allow_reclaim){- if mi_unlikely(page==NULL) return; // page will be NULL if p==NULLmi_assert_internal(p!=NULL && page!=NULL);- if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }+ if (pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); }- const mi_threadid_t xtid = (_mi_prim_thread_id() ^ mi_page_xthread_id(page));+ const mi_threadid_t ptid = mi_page_xthread_id(page);+ const mi_threadid_t xtid = (_mi_prim_thread_id() ^ ptid);if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0`// thread-local, aligned, and not a full pagemi_block_t* const block = mi_validate_block_from_ptr(page,p);- mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */);+ mi_free_block_local(page, block, false /* was guarded */, false /* no need to check if the page is full */);}else if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0`// page is local, but is full or contains (inner) aligned blocks; use generic path@@ -196,45 +278,125 @@ static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable, mi_page_t* pelse if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0`// blocks are aligned (and not a full page); push on the thread_free listmi_block_t* const block = mi_validate_block_from_ptr(page,p);- mi_free_block_mt(page,block,false /* was_guarded */);+ mi_free_block_mt(page,block,false /* was_guarded */, allow_reclaim);}else {// page is full or contains (inner) aligned blocks; use generic multi-thread path- mi_free_generic_mt(page, p);+ mi_free_generic_mt(page, p, allow_reclaim);}}-void mi_free(void* p) mi_attr_noexcept {- mi_page_t* const page = mi_validate_ptr_page(p,"mi_free");- mi_free_ex(p, NULL, page);+void mi_free(void* p) mi_attr_noexcept {+ mi_page_t* page;+ if mi_likely(mi_ptr_page_is_valid(p,"mi_free",&page)) {+ mi_free_nonnull(p, page, NULL, true /* allow reclaim? */);+ }}-void mi_ufree(void* p, size_t* usable) mi_attr_noexcept {- mi_page_t* const page = mi_validate_ptr_page(p,"mi_ufree");- mi_free_ex(p, usable, page);+void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept {+ mi_page_t* page;+ if mi_likely(mi_ptr_page_is_valid(p,"mi_ufree",&page)) {+ mi_free_nonnull(p, page, pblock_size, true /* allow reclaim? */);+ }+ else {+ if (pblock_size!=NULL) { *pblock_size = 0; }+ }}void mi_free_small(void* p) mi_attr_noexcept {- // We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`.- // If we keep page info in front of the page area for small objects, we can find the info- // just by aligning down the pointer instead of looking it up in the page map.- #if MI_PAGE_META_ALIGNED_FREE_SMALL- #if MI_GUARDED- #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as MI_GUARDED is defined"+ mi_page_t* page;+ if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small",true /* is_small? */,true /*check p for null*/, &page)) {+ mi_free_nonnull(p, page, NULL, true /* allow reclaim? */);+ }+}++void mi_free_small_nonnull(void* p) mi_attr_noexcept {+ mi_assert(p!=NULL);+ mi_page_t* page;+ if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small_nonnull",true /* is_small? */,false /*check p for null*/, &page)) {+ mi_free_nonnull(p, page, NULL, true /* allow reclaim? */);+ }+}++// Free a pointer that is potentially allocated in a different sub-process+void _mi_free_subproc_safe(void* p) mi_attr_noexcept {+ mi_page_t* page;+ if mi_likely(mi_ptr_page_is_valid(p,"_mi_free_subproc_safe",&page)) {+ // const bool allow_reclaim = (_mi_subproc() == mi_page_subproc(page));+ mi_free_nonnull(p, page, NULL, false /* allow reclaim */);+ }+}++// ------------------------------------------------------+// Free variants+// ------------------------------------------------------++void mi_free_size(void* p, size_t size) mi_attr_noexcept {+ MI_UNUSED_RELEASE(size);+ #if MI_DEBUG+ const mi_page_t* const page = mi_ptr_page_validate(p,"mi_free_size");+ if (page==NULL) return;+ mi_assert(p!=NULL);+ const mi_block_t* block = _mi_page_ptr_unalign(page, p);+ const size_t usable = _mi_page_usable_size(page,p);+ if mi_unlikely(size > usable) {+ const bool is_guarded = mi_block_ptr_is_guarded(block,p);+ if (!is_guarded) {+ _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable);+ mi_free(p);+ return;+ }+ }+ // const size_t is_aligned = ((void*)block != p);+ // if mi_unlikely(size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size((is_aligned ? 2 : 1)*MI_SMALL_SIZE_MAX)) { // note: we check *2 in case it was over-aligned+ // const bool is_guarded = mi_block_ptr_is_guarded(block,p);+ // if (!is_guarded) {+ // _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n (maybe a `new[]` was matched with `delete` instead of `delete[]`?)\n", p, size, mi_page_block_size(page));+ // mi_free(p);+ // return;+ // }+ // }+ #endif+ #if MI_PAGE_META_SMALL_IS_ALIGNED || MI_PAGE_META_IS_ALIGNED+ if mi_likely(size <= MI_SMALL_SIZE_MAX) {+ mi_free_small(p);+ }+ else+ #endif+ {mi_free(p);- #elif MI_ARENA_SLICE_ALIGN < MI_SMALL_PAGE_SIZE- #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size"+ }+}++void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept {+ mi_assert(((uintptr_t)p % alignment) == 0);+ // If the alignment is smaller than the `size`, then for `size <= MI_SMALL_SIZE_MAX`+ // the block must be allocated within a small page and can thus be handled safely by `mi_free_size`.+ // (since even with over-allocation the block size will be less than 2*MI_SMALL_SIZE_MAX <= MI_SMALL_OBJ_SIZE_MAX)+ if mi_likely(alignment <= size) {+ mi_free_size(p,size);+ }+ else {mi_free(p);- #else- mi_page_t* const page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE);- mi_assert(page == mi_validate_ptr_page(p,"mi_free_small"));- mi_assert((void*)page == _mi_align_down_ptr(page->page_start,MI_SMALL_PAGE_SIZE));- mi_assert(page->block_size <= MI_SMALL_SIZE_MAX); // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small`- mi_free_ex(p, NULL, page);- #endif- #else+ }+}++void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {+ MI_UNUSED_RELEASE(alignment);+ mi_assert(((uintptr_t)p % alignment) == 0);mi_free(p);- #endif+}++// checked free+bool mi_cfree(void* p) mi_attr_noexcept {+ mi_page_t* const page = _mi_checked_ptr_page(p);+ if mi_likely(page!=NULL) {+ mi_free_nonnull(p, page, NULL, true /* allow reclaim? */);+ return true;+ }+ else {+ return false;+ }}@@ -270,14 +432,14 @@ static bool mi_abandoned_page_try_reabandon_to_mapped(mi_page_t* page)// Release ownership of a page. This may free or reabandoned the page if other blocks are concurrently// freed in the meantime. Returns `true` if the page was freed.// By passing the captured `expected_thread_free`, we can often avoid calling `mi_page_free_collect`.-static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expected_thread_free) {+static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* page_thread_free) {mi_assert_internal(mi_page_is_owned(page));mi_assert_internal(mi_page_is_abandoned(page));mi_assert_internal(!mi_page_all_free(page));// try to cas atomically the original free list (`mt_free`) back with the ownership cleared.- mi_thread_free_t tf_expect = mi_tf_create(expected_thread_free, true);- mi_thread_free_t tf_new = mi_tf_create(expected_thread_free, false);- while mi_unlikely(!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_expect, tf_new)) {+ mi_thread_free_t tf_expect = mi_tf_create(page_thread_free,true);+ mi_thread_free_t tf_new = mi_tf_set_owned(tf_expect,false);+ while mi_unlikely(!mi_atomic_cas_strong_acq_rel(&page->xthread_free, &tf_expect, tf_new)) {mi_assert_internal(mi_tf_is_owned(tf_expect));// while the xthread_free list is not empty..while (mi_tf_block(tf_expect) != NULL) {@@ -286,9 +448,10 @@ static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expecif (mi_abandoned_page_try_free(page)) return;if (mi_abandoned_page_try_reabandon_to_mapped(page)) return;// otherwise continue un-owning- tf_expect = mi_atomic_load_relaxed(&page->xthread_free);+ tf_expect = mi_atomic_load_acquire(&page->xthread_free);}// and try again to release ownership+ mi_subproc_stat_counter_increase(mi_page_subproc(page), pages_unabandon_busy_wait, 1);mi_assert_internal(mi_tf_block(tf_expect)==NULL);tf_new = mi_tf_create(NULL, false);}@@ -315,14 +478,14 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, longmi_assert_internal(!mi_page_all_free(page));mi_assert_internal(page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE);mi_assert_internal(reclaim_on_free >= 0);-+// dont reclaim if we just have terminated this thread and we should// not reinitialize the theap for this thread. (can happen due to thread-local destructors for example -- issue #944)if (!_mi_thread_is_initialized()) return false;// get our theapmi_theap_t* const theap = _mi_page_associated_theap_peek(page);- if (theap==NULL || !theap->allow_page_reclaim) return false;+ if (theap==NULL || theap->tld==NULL || !theap->allow_page_reclaim) return false; // see issue #1289// todo: cache `is_in_threadpool` and `exclusive_arena` directly in the theap for performance?// set max_reclaim limit@@ -354,25 +517,27 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long// We freed a block in an abandoned page (that was not owned). Try to collect-static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept+static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept{mi_assert_internal(mi_page_is_owned(page));mi_assert_internal(mi_page_is_abandoned(page));mi_assert_internal(mt_free != NULL);+ // mi_assert_internal(!allow_reclaim || _mi_subproc() == mi_page_subproc(page)); // never reclaim across subprocesses+// we own the page now, and it is safe to collect the thread atomic free listif (page->block_size <= MI_SMALL_SIZE_MAX) {// use the `_partly` version to avoid atomic operations since we already have the `mt_free` pointing into the thread free list// (after this the `used` count might be too high (as some blocks may have been concurrently added to the thread free list and are yet uncounted).// however, if the page became completely free, the used count is guaranteed to be 0.)mi_assert_internal(page->reserved>=16); // below this even one freed block goes from full to no longer mostly used.- _mi_page_free_collect_partly(page, mt_free);+ mt_free = _mi_page_free_collect_partly(page, mt_free);}else {// for larger blocks we use the regular collect_mi_page_free_collect(page,false /* no force */);mt_free = NULL; // expected page->xthread_free value after collection}- const long reclaim_on_free = _mi_option_get_fast(mi_option_page_reclaim_on_free);+ const long reclaim_on_free = allow_reclaim && _mi_option_get_fast(mi_option_page_reclaim_on_free);#if MI_DEBUG > 1if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); }if (mi_page_is_full(page)) { mi_assert(mi_page_is_mostly_used(page)); }@@ -391,112 +556,98 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t*// -------------------------------------------------------// Usable size+// Usable size// ------------------------------------------------------// Bytes available in a block-static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p) mi_attr_noexcept {- const mi_block_t* block = _mi_page_ptr_unalign(page, p);+static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p, const mi_block_t* block) mi_attr_noexcept {+ // const mi_block_t* block = _mi_page_ptr_unalign(page, p);const bool is_guarded = mi_block_ptr_is_guarded(block,p);const size_t size = mi_page_usable_size_of(page, block, is_guarded);- const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block;- mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);- const size_t aligned_size = (size - adjust);+ mi_assert_internal((void*)p >= (void*)block);+ const size_t adjust = (uint8_t*)p - (uint8_t*)block;+ mi_assert_internal(adjust <= size);+ const size_t aligned_size = (adjust <= size ? size - adjust : 0); // size can be zero if the padding is corruptedreturn aligned_size;}-static inline size_t _mi_usable_size(const void* p, const mi_page_t* page) mi_attr_noexcept {+size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept {if mi_unlikely(page==NULL) return 0;- if mi_likely(!mi_page_has_interior_pointers(page)) {- const mi_block_t* block = (const mi_block_t*)p;+ mi_assert_internal(mi_ptr_page_validate(p,"_mi_page_usable_size") == page);+ const mi_block_t* block = _mi_page_ptr_unalign(page, p); // for safety, always unalign+ if mi_likely((const void*)block==page) {return mi_page_usable_size_of(page, block, false /* is guarded */);}else {// split out to separate routine for improved code generation- return mi_page_usable_aligned_size_of(page, p);+ return mi_page_usable_aligned_size_of(page, p, block);}}mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept {- const mi_page_t* const page = mi_validate_ptr_page(p,"mi_usable_size");- return _mi_usable_size(p,page);-}---// -------------------------------------------------------// Free variants-// --------------------------------------------------------void mi_free_size(void* p, size_t size) mi_attr_noexcept {- MI_UNUSED_RELEASE(size);- #if MI_DEBUG- const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size");- const size_t available = _mi_usable_size(p,page);- mi_assert(p == NULL || size <= available || available == 0 /* invalid pointer */ );- #endif- mi_free(p);-}--void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept {- MI_UNUSED_RELEASE(alignment);- mi_assert(((uintptr_t)p % alignment) == 0);- mi_free_size(p,size);-}--void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {- MI_UNUSED_RELEASE(alignment);- mi_assert(((uintptr_t)p % alignment) == 0);- mi_free(p);+ const mi_page_t* const page = mi_ptr_page_validate(p,"mi_usable_size");+ return _mi_page_usable_size(page,p);}// ------------------------------------------------------+// Deprecated: double free is usually checked with padding now+// as that is faster and works better for cross-thread free'ing.// Check for double free in secure and debug mode// This is somewhat expensive so only enabled for secure mode 4// -------------------------------------------------------#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0))+#if MI_SECURE>=3 && !MI_PADDING// linear check if the free list contains a specific element-static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) {- while (list != NULL) {+static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem, const char* list_kind) {+ const size_t max_count = page->capacity; // can never hold more blocks than the capacity+ size_t count = 0;+ while (list != NULL && count <= max_count) { // double-free can create cycles so we limit the number of iterationsif (elem==list) return true;list = mi_block_next(page, list);+ count++;+ }+ if mi_unlikely(count > max_count) {+ _mi_error_message(EFAULT, "corrupted %s list (possibly due to a double free)\n", list_kind);}return false;}-static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) {+static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const mi_block_t* block) {// The decoded value is in the same page (or NULL).// Walk the free lists to verify positively if it is already freed- if (mi_list_contains(page, page->free, block) ||- mi_list_contains(page, page->local_free, block) ||- mi_list_contains(page, mi_page_thread_free(page), block))+ if (mi_list_contains(page, page->free, block, "free") ||+ mi_list_contains(page, page->local_free, block, "local free") ||+ mi_list_contains(page, mi_page_thread_free(page), block, "thread free")){_mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page));- return true;+ return false;}- return false;+ return true;}-#define mi_track_page(page,access) { size_t psize; void* pstart = _mi_page_start(_mi_page_segment(page),page,&psize); mi_track_mem_##access( pstart, psize); }+// Used for double free checking to avoid checking free lists too frequently+static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi_block_t* block) {+ mi_block_t* n = mi_block_nextx(page,block,page->keys);+ return (_mi_is_aligned(block,MI_SIZE_SIZE) && // quick check: aligned pointer?+ (n==NULL || mi_page_contains_address(page,n))); // quick check: in the same page or NULL?+}-static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {- bool is_double_free = false;- mi_block_t* n = mi_block_nextx(page, block, page->keys); // pretend it is freed, and get the decoded first field- if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer?- (n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL?+// check if `block` was free'd before+static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) {+ if mi_unlikely(mi_block_could_be_double_free(page,block)) // quick check: next field is aligned in the same page or NULL?{// Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free?// (continue in separate function to improve code generation)- is_double_free = mi_check_is_double_freex(page, block);+ return mi_check_double_freex(page, block);}- return is_double_free;+ else return true;}#else-static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {+static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) {MI_UNUSED(page);MI_UNUSED(block);- return false;+ return true;}#endif@@ -506,30 +657,31 @@ static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block// ---------------------------------------------------------------------------#if MI_PADDING // && !MI_TRACK_ENABLED-static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) {+static inline bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize, bool* double_free) {*bsize = mi_page_usable_block_size(page);- const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize);+ mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize);mi_track_mem_defined(padding,sizeof(mi_padding_t));*delta = padding->delta;- uint32_t canary = padding->canary;- uintptr_t keys[2];- keys[0] = page->keys[0];- keys[1] = page->keys[1];- bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize);+ const uint32_t canary = padding->canary;+ const bool ok = (mi_ptr_encode_canary(page,block,page->keys) == canary && *delta <= *bsize);+ if (double_free!=NULL) {+ if mi_unlikely(!ok) { *double_free = mi_ptr_decode_canary_is_freed(canary); } // double free?+ else { padding->canary = mi_ptr_encode_canary_freed(); } // mark as freed+ }mi_track_mem_noaccess(padding,sizeof(mi_padding_t));return ok;}// Return the exact usable size of a block.static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {- if (is_guarded) {+ if mi_unlikely(is_guarded) {const size_t bsize = mi_page_block_size(page);return (bsize - _mi_os_page_size());}else {size_t bsize;size_t delta;- bool ok = mi_page_decode_padding(page, block, &delta, &bsize);+ bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL);mi_assert_internal(ok); mi_assert_internal(delta <= bsize);return (ok ? bsize - delta : 0);}@@ -540,9 +692,10 @@ static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* bl// contain the pointer for the delayed list, then shrink the padding (by decreasing delta)// so it will later not trigger an overflow error in `mi_free_block`.void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {+ if (mi_block_ptr_is_guarded(block,block + 1)) return;size_t bsize;size_t delta;- bool ok = mi_page_decode_padding(page, block, &delta, &bsize);+ bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL);mi_assert_internal(ok);if (!ok || (bsize - delta) >= min_size) return; // usually already enough spacemi_assert_internal(bsize >= min_size);@@ -555,9 +708,15 @@ void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const simi_track_mem_noaccess(padding,sizeof(mi_padding_t));}#else-static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {- MI_UNUSED(is_guarded); MI_UNUSED(block);- return mi_page_usable_block_size(page);+static inline size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {+ MI_UNUSED(block);+ if mi_unlikely(is_guarded) {+ const size_t bsize = mi_page_block_size(page);+ return (bsize - _mi_os_page_size());+ }+ else {+ return mi_page_usable_block_size(page);+ }}void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {@@ -565,16 +724,17 @@ void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const si}#endif-#if MI_PADDING && MI_PADDING_CHECK+#if MI_PADDING-static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) {+static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong, bool* is_double_free) {size_t bsize;size_t delta;- bool ok = mi_page_decode_padding(page, block, &delta, &bsize);+ bool ok = mi_page_decode_padding(page, block, &delta, &bsize, is_double_free );*size = *wrong = bsize;if (!ok) return false;mi_assert_internal(bsize >= delta);*size = bsize - delta;+ #if MI_PADDING_CHECK_BYTESif (!mi_page_is_huge(page)) {uint8_t* fill = (uint8_t*)block + bsize - delta;const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes@@ -588,41 +748,65 @@ static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, si}mi_track_mem_noaccess(fill, maxpad);}+ #endifreturn ok;}-static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {- size_t size;- size_t wrong;- if (!mi_verify_padding(page,block,&size,&wrong)) {- _mi_error_message(EFAULT, "buffer overflow in theap block %p of size %zu: write after %zu bytes\n", block, size, wrong );+mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) {+ if mi_unlikely(is_guarded) {+ const size_t bsize = mi_page_block_size(page);+ *usable_size = (bsize - _mi_os_page_size());+ return true;+ }+ else {+ size_t wrong;+ bool is_double_free;+ if mi_unlikely(!mi_verify_padding(page,block,usable_size,&wrong,&is_double_free)) {+ if (is_double_free) {+ _mi_error_message(EAGAIN, "double free detected of heap block %p with size %zu\n", block, *usable_size);+ }+ else {+ _mi_error_message(EFAULT, "buffer overflow in heap block %p of size %zu: write after %zu bytes\n", block, *usable_size, wrong );+ }+ return false;+ }+ return true;}}#else-static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {- MI_UNUSED(page);- MI_UNUSED(block);+mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) {+ *usable_size = mi_page_usable_size_of(page,block,is_guarded);+ return true;}#endif// only maintain stats for smaller objects if requested-#if (MI_STAT>0)+#if 0 && (MI_STATS>0)static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {- MI_UNUSED(block);- mi_theap_t* const theap = _mi_theap_default();- if (!mi_theap_is_initialized(theap)) return; // (for now) skip statistics if free'd after thread_done was called (usually a thread cleanup call by the OS)+ MI_UNUSED(block);+ mi_theap_t* theap = _mi_theap_default();+ mi_lock_t* lock = NULL;+ mi_subproc_t* const subproc = mi_page_subproc(page);+ mi_theap_t* const theap_meta = subproc->theap_meta;+ if mi_unlikely(!mi_theap_is_initialized(theap) || // can happen if free'd after thread_done was called (usually a thread cleanup call by the OS)+ // page->theap == subproc->theap_meta .. but we cannot read `theap` if we don't own the page+ (theap_meta != NULL && mi_page_thread_id(page) == theap_meta->tld->thread_id)) {+ theap = theap_meta;+ lock = &subproc->theap_meta_lock;+ mi_lock_acquire(lock);+ }const size_t bsize = mi_page_usable_block_size(page);- // #if (MI_STAT>1)+ // #if (MI_STATS>=2)// const size_t usize = mi_page_usable_size_of(page, block);// mi_theap_stat_decrease(theap, malloc_requested, usize);// #endifif (bsize <= MI_LARGE_MAX_OBJ_SIZE) {mi_theap_stat_decrease(theap, malloc_normal, bsize);- #if (MI_STAT > 1)+ #if (MI_STATS > 1)mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], 1);#endif}@@ -630,29 +814,13 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {const size_t bpsize = mi_page_block_size(page); // match stat in page.c:mi_huge_page_allocmi_theap_stat_decrease(theap, malloc_huge, bpsize);}++ if mi_unlikely(lock!=NULL) {+ mi_lock_release(lock);+ }}#elsevoid mi_stat_free(const mi_page_t* page, const mi_block_t* block) {MI_UNUSED(page); MI_UNUSED(block);}#endif---// Remove guard page when building with MI_GUARDED-#if MI_GUARDED-static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) {- MI_UNUSED(p);- mi_assert_internal(mi_block_ptr_is_guarded(block, p));- mi_assert_internal(mi_page_has_interior_pointers(page));- mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t));- mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED);-- const size_t bsize = mi_page_block_size(page);- const size_t psize = _mi_os_page_size();- mi_assert_internal(bsize > psize);- mi_assert_internal(!page->memid.is_pinned);- void* gpage = (uint8_t*)block + bsize - psize;- mi_assert_internal(_mi_is_aligned(gpage, psize));- _mi_os_unprotect(gpage, psize);-}-#endifdiff --git a/vendored/mimalloc/src/heap.c b/vendored/mimalloc/src/heap.cindex f0a016527..d7f1db132 100644--- a/vendored/mimalloc/src/heap.c+++ b/vendored/mimalloc/src/heap.c@@ -1,5 +1,5 @@/*-----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -7,7 +7,8 @@ terms of the MIT license. A copy of the license can be found in the file#include "mimalloc.h"#include "mimalloc/internal.h"-#include "mimalloc/prim.h" // _mi_theap_default+#include "mimalloc/prim.h" // _mi_prim_thread_yield+#include "mimalloc/prim-tls.h" // _mi_heap_theap/* -----------------------------------------------------------@@ -15,7 +16,7 @@ terms of the MIT license. A copy of the license can be found in the file----------------------------------------------------------- */mi_theap_t* mi_heap_theap(mi_heap_t* heap) {- return _mi_heap_theap(heap);+ return _mi_heap_theap(heap); // in prim.h}void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node) {@@ -30,64 +31,66 @@ void mi_heap_stats_merge_to_subproc(mi_heap_t* heap) {void mi_heap_stats_merge_to_main(mi_heap_t* heap) {if (heap==NULL) return;- _mi_stats_merge_into(&mi_heap_main()->stats, &heap->stats);+ _mi_stats_merge_into(&mi_heap_get_heap_main(heap)->stats, &heap->stats);}+bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap) {+ mi_assert_internal((uintptr_t)theap == 1 || _mi_theap_heap(theap)==heap);+ mi_assert_internal(!_mi_is_empty_theap(theap));+ mi_assert_internal(heap->theap != 0);+ return _mi_thread_local_set(heap->theap,theap);+}++// mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap) {+// mi_theap_t* theap;+// mi_assert_internal(heap->theap != 0);+// if mi_likely(heap->theap!=0) { // paranoia+// theap = (mi_theap_t*)_mi_thread_local_get(heap->theap);+// }+// else {+// _mi_error_message(EFAULT, "no thread-local reserved for heap (%p)\n", heap);+// return NULL;+// }+// mi_assert_internal(!_mi_is_empty_theap(theap));+// mi_assert_internal(theap->heap == heap); // this goes wrong if using main heaps across subprocesses (as all share the same key)+// return theap;+// }++static mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_heap){mi_heap_t* heap = (mi_heap_t*)const_heap;mi_assert_internal(heap!=NULL);- if (_mi_is_heap_main(heap)) {- // this can be called if the (main) thread is not yet initialized (as no allocation happened)- // but `theap_main_init_get()` will call `mi_thread_init()`- mi_theap_t* const theap = _mi_theap_main_safe();- mi_assert_internal(theap!=NULL && _mi_is_heap_main(_mi_theap_heap(theap)));- return theap;+ // initialize thread first in case this is the main heap+ // (which may allocate the default theap already for the main heap)+ if (!_mi_thread_is_initialized()) {+ mi_thread_init();}- // otherwise initialize the theap for this heap- // get the thread local- mi_assert_internal(heap->theap != 0);- if (heap->theap==0) { // paranoia- _mi_error_message(EFAULT, "no thread-local reserved for heap (%p)\n", heap);- return NULL;- }+ // get the thread local theapmi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap);// create a fresh theap?if (theap==NULL) {- // set first an invalid value to ensure the thread local storage is allocated- if (!_mi_thread_local_set(heap->theap, (mi_theap_t*)1)) {- _mi_error_message(EFAULT, "unable to allocate memory for thread local storage\n");- return NULL;- }- // then allocate the theap- theap = _mi_theap_create(heap, _mi_theap_default_safe()->tld);- _mi_thread_local_set(heap->theap, theap); // Cannot fail now as it was set before. Always set so the local is valid or NULL (and not 1)+ // allocate a fresh theap+ theap = _mi_theap_create(heap, mi_theap_get_default()->tld); // sets the theap thread localif (theap==NULL) {_mi_error_message(EFAULT, "unable to allocate memory for a thread local heap\n");return NULL;- }+ }+ _mi_heap_theap_set(heap, theap);+ mi_assert_internal(theap == (mi_theap_t*)_mi_thread_local_get(heap->theap));}return theap;}-// get the theap for a heap without initializing (and return NULL in that case)-mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap) {- if (heap==NULL || _mi_is_heap_main(heap)) {- return _mi_theap_main_safe();- }- else {- return (mi_theap_t*)_mi_thread_local_get(heap->theap);- }-}-// get (and possibly create) the theap belonging to a heap-mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap)+mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap){- mi_theap_t* theap = _mi_heap_theap_peek(heap);+ mi_assert_internal(heap->theap != 0);+ mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap);if mi_unlikely(theap==NULL) {theap = mi_heap_init_theap(heap);if (theap==NULL) { return (mi_theap_t*)&_mi_theap_empty_wrong; } // this will return NULL from page.c:_mi_malloc_generic@@ -96,28 +99,15 @@ mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap)return theap;}--mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) {- // always allocate heap data in the (subprocess) main heap- mi_heap_t* const heap_main = mi_heap_main();- // todo: allocate heap data in the exclusive arena ?- mi_heap_t* const heap = (mi_heap_t*)mi_heap_zalloc( heap_main, sizeof(mi_heap_t) );- if (heap==NULL) return NULL;-- // reserve a thread local slot for this heap (see also issue #1230)- const mi_thread_local_t theap_slot = _mi_thread_local_create();- if (theap_slot == 0) {- _mi_error_message(EFAULT, "unable to dynamically create a thread local for a heap\n");- mi_free(heap);- return NULL;- }-+void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap_slot, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id)+{// init fieldsheap->theap = theap_slot;- heap->subproc = heap_main->subproc;- heap->heap_seq = mi_atomic_increment_relaxed(&heap_main->subproc->heap_total_count);+ heap->subproc = subproc;+ heap->heap_seq = mi_atomic_increment_relaxed(&subproc->heap_total_count);heap->exclusive_arena = _mi_arena_from_id(exclusive_arena_id);heap->numa_node = -1; // no initial affinity+ heap->profiler = mi_atomic_load_ptr_acquire(mi_profiler_t,&subproc->profiler);mi_stats_header_init(&heap->stats);mi_lock_init(&heap->theaps_lock);mi_lock_init(&heap->os_abandoned_pages_lock);@@ -131,94 +121,141 @@ mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) {if (head!=NULL) { head->prev = heap; }heap->subproc->heaps = heap;}- mi_atomic_increment_relaxed(&heap_main->subproc->heap_count);- mi_subproc_stat_increase(heap_main->subproc, heaps, 1);+ mi_atomic_increment_relaxed(&subproc->heap_count);+ mi_subproc_stat_increase(subproc, heaps, 1);+ mi_assert_internal(_mi_is_heap_main(heap) ? heap->theap == mi_thread_local_key_fast : heap->theap != 0);+}++mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_main_heap) {+ mi_assert_internal(is_main_heap ? (subproc->heap_main == NULL && subproc->parent != NULL) : subproc->heap_main != NULL);+ // heap data is allocated in the current subproc+ mi_heap_t* const heap_main = (is_main_heap ? subproc->parent->heap_main : subproc->heap_main);+ // todo: allocate heap data in the exclusive arena ?+ mi_heap_t* const heap = (mi_heap_t*)mi_heap_zalloc( heap_main, sizeof(mi_heap_t) );+ if (heap==NULL) return NULL;+ // reserve a thread local slot for this heap (see also issue #1230)+ mi_thread_local_t theap_slot = (is_main_heap ? mi_thread_local_key_fast : _mi_thread_local_create());+ if (theap_slot == 0) {+ _mi_error_message(EFAULT, "unable to dynamically create a thread local for a heap\n");+ mi_free(heap);+ return NULL;+ }+ if (is_main_heap) {+ mi_assert_internal(subproc->heap_main == NULL);+ subproc->heap_main = heap;+ }+ _mi_heap_init(heap, theap_slot, subproc, exclusive_arena_id);return heap;}+mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) {+ // `mi_heap_new` may be the very first mimalloc call in a process, in which case the+ // main heap does not exist yet and `_mi_heap_new_for_subproc` would allocate from a NULL `subproc->heap_main`.+ mi_thread_init();+ return _mi_heap_new_for_subproc(_mi_subproc(), exclusive_arena_id, false);+}+mi_heap_t* mi_heap_new(void) {return mi_heap_new_in_arena(0);}// free all theaps belonging to this heap (without deleting their pages as we do this arena wise for efficiency)static void mi_heap_free_theaps(mi_heap_t* heap) {- // This can run concurrently with a thread that terminates (see `init.c:mi_thread_theaps_done`),+ // This can run concurrently with a thread that terminates (see `init.c:mi_thread_theaps_done`),// and we need to ensure we free theaps atomically.- // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock- // potential concurrent thread termination which tries to remove the theap from our theaps list.- bool all_freed;- do {- all_freed = true;- mi_theap_t* theap = NULL;- mi_lock(&heap->theaps_lock) {- theap = heap->theaps;- while(theap != NULL) {- mi_theap_t* next = theap->hnext;- if (!_mi_theap_free(theap, false /* dont re-acquire the heap->theaps_lock */, true /* acquire the tld->theaps_lock though */ )) {- all_freed = false;- }- theap = next;- }- }- if (!all_freed) {- mi_heap_stat_counter_increase(heap,heaps_delete_wait,1);- _mi_prim_thread_yield();++ // We first detach our theaps list from any thread local lists+ _mi_heap_detach_theaps(heap);++ // Now we can safely free the theaps+ mi_lock(&heap->theaps_lock) { // paranoia+ mi_theap_t* theap = heap->theaps;+ heap->theaps = NULL;+ while(theap != NULL) {+ mi_theap_t* next = theap->hnext;+ theap->hnext = NULL;+ theap->hprev = NULL;+ mi_assert_internal(theap->tld==NULL);+ // merge stats into the owning heap stats+ _mi_stats_merge_into(&heap->stats, &theap->stats);+ // and free+ _mi_theap_decref(theap); // a cached entry can still point to the theap+ theap = next;}- else {- mi_assert_internal(heap->theaps==NULL);- }+ }++ // set the theap thread local to NULL (so _mi_page_associated_theap does not read from a freed theap (through delete pages -> page_update_stats))+ if (!_mi_is_process_heap_main(heap)) {+ _mi_thread_local_free(heap->theap);+ heap->theap = 0;}- while(!all_freed);}// free the heap resources (assuming the pages are already moved/destroyed, and all theaps have been freed)-static void mi_heap_free(mi_heap_t* heap) {- mi_assert_internal(heap!=NULL && !_mi_is_heap_main(heap));+static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) {+ mi_assert_internal(heap!=NULL); // && !_mi_is_process_heap_main(heap));// free all arena pages infos- mi_lock(&heap->arena_pages_lock) {- for (size_t i = 0; i < MI_MAX_ARENAS; i++) {- mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);- if (arena_pages!=NULL) {- mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL);- mi_free(arena_pages);+ const bool is_main = _mi_is_heap_main(heap);+ if (!is_main) { // pages for the main heap are pre-allocated in the arenas+ mi_lock(&heap->arena_pages_lock) {+ for (size_t i = 0; i < MI_MAX_ARENAS; i++) {+ mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);+ if (arena_pages!=NULL) {+ mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL);+ _mi_free_subproc_safe(arena_pages);+ }}}}// remove the heap from the subproc- mi_heap_stats_merge_to_main(heap);+ if (!is_main) {+ mi_heap_stats_merge_to_main(heap);+ }+ else {+ _mi_stats_merge_into(&heap->subproc->stats,&heap->stats);+ }mi_atomic_decrement_relaxed(&heap->subproc->heap_count);mi_subproc_stat_decrease(heap->subproc, heaps, 1);- mi_lock(&heap->subproc->heaps_lock) {+ mi_lock_maybe(&heap->subproc->heaps_lock, acquire_heaps_lock) {if (heap->next!=NULL) { heap->next->prev = heap->prev; }if (heap->prev!=NULL) { heap->prev->next = heap->next; }else { heap->subproc->heaps = heap->next; }}- _mi_thread_local_free(heap->theap);mi_lock_done(&heap->theaps_lock);mi_lock_done(&heap->os_abandoned_pages_lock);mi_lock_done(&heap->arena_pages_lock);- mi_free(heap);+ if (!_mi_is_process_heap_main(heap)) {+ // _mi_thread_local_free(heap->theap);+ _mi_free_subproc_safe(heap);+ }}void mi_heap_delete(mi_heap_t* heap) {if (heap==NULL) return;- if (_mi_is_heap_main(heap)) {+ mi_heap_t* heap_main = mi_heap_get_heap_main(heap);+ if (heap == heap_main) {_mi_warning_message("cannot delete the main heap\n");return;}mi_heap_free_theaps(heap);- _mi_heap_move_pages(heap, mi_heap_main());- mi_heap_free(heap);+ _mi_heap_move_pages(heap, heap_main);+ mi_heap_free(heap,true /* acquire subproc->heaps_lock */);}-void _mi_heap_force_destroy(mi_heap_t* heap) {+void _mi_heap_force_destroy(mi_heap_t* heap, bool acquire_heaps_lock) {if (heap==NULL) return;mi_heap_free_theaps(heap);_mi_heap_destroy_pages(heap);- if (!_mi_is_heap_main(heap)) { mi_heap_free(heap); } // todo: release locks of the main heap?+ // if (_mi_subproc_main()->heap_main == heap) {+ // _mi_stats_merge_into(&heap->subproc->stats,&heap->stats);+ // }+ // else+ {+ mi_heap_free(heap, acquire_heaps_lock); // todo: release locks of the main heap?+ }}void mi_heap_destroy(mi_heap_t* heap) {@@ -227,17 +264,18 @@ void mi_heap_destroy(mi_heap_t* heap) {_mi_warning_message("cannot destroy the main heap\n");return;}- _mi_heap_force_destroy(heap);+ _mi_heap_force_destroy(heap,true /* acquire subproc->heaps_lock */);}mi_heap_t* mi_heap_of(const void* p) {- mi_page_t* page = _mi_safe_ptr_page(p);+ mi_page_t* const page = _mi_safe_ptr_page(p);if (page==NULL) return NULL;return mi_page_heap(page);}bool mi_any_heap_contains(const void* p) {- return (mi_heap_of(p)!=NULL);+ mi_page_t* const page = _mi_safe_ptr_page(p);+ return (page!=NULL);}bool mi_heap_contains(const mi_heap_t* heap, const void* p) {@@ -256,8 +294,8 @@ bool mi_check_owned(const void* p) {bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept {if (p==NULL) return false;const mi_page_t* const page = _mi_safe_ptr_page(p); // Get the page containing this pointer- if (page==NULL || page->used==page->capacity || page->capacity < page->reserved) return false;- // If the page is the head of the queue, it is currently being used for+ if (page==NULL || mi_page_used(page)==page->capacity || page->capacity < page->reserved) return false;+ // If the page is the head of the queue, it is currently being used for// allocations; we skip it to avoid immediate thrashing.if (page->prev == NULL) return false;@@ -265,9 +303,9 @@ bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t percconst mi_heap_t* const page_heap = mi_page_heap(page);if (page_heap==NULL) return false;if (heap!=NULL && page_heap!=heap) return false;-+// check utilizationif (page->capacity==0) return false;if (perc_threshold>=100) return true;- return (perc_threshold >= ((100UL*page->used) / page->capacity));+ return (perc_threshold >= ((100UL*mi_page_used(page)) / page->capacity));}diff --git a/vendored/mimalloc/src/init.c b/vendored/mimalloc/src/init.cindex 1fabd8918..9b0ded5b5 100644--- a/vendored/mimalloc/src/init.c+++ b/vendored/mimalloc/src/init.c@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -7,44 +7,61 @@ terms of the MIT license. A copy of the license can be found in the file#include "mimalloc.h"#include "mimalloc/internal.h"#include "mimalloc/prim.h"+#include "mimalloc/prim-tls.h"#include <string.h> // memcpy, memset#include <stdlib.h> // atexit-#define MI_MEMID_INIT(kind) {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ }-#define MI_MEMID_STATIC MI_MEMID_INIT(MI_MEM_STATIC)-// Empty page used to initialize the small free pages array-const mi_page_t _mi_page_empty = {+static const mi_page_t mi_page_empty = {+ #if MI_PAGE_META_IS_ALIGNED+ MI_ATOMIC_VAR_INIT(NULL), // self+ #endifMI_ATOMIC_VAR_INIT(0), // xthread_idNULL, // free- 0, // used+ {0}, // xused+ #if MI_SIZE_SIZE < 8+ 0, // xlast_used+ 0, // xlast_alloc+ #endif+ NULL, // local_free+ 0, // block_size+ 0, // page_offset0, // capacity0, // reserved capacity+ 0, // slice_pcommitted0, // retire_expirefalse, // is_zero- NULL, // local_freeMI_ATOMIC_VAR_INIT(0), // xthread_free- 0, // block_size- NULL, // page_start- #if (MI_PADDING || MI_ENCODE_FREELIST)- { 0, 0 }, // keys- #endifNULL, // theapNULL, // heapNULL, NULL, // next, prev- MI_ARENA_SLICE_SIZE, // page_committed- MI_MEMID_STATIC // memid+ MI_MEMID_STATIC, // memid+ #if (MI_PADDING || MI_ENCODE_FREELIST)+ #if MI_PAGE_KEY_COUNT==2+ { 0, 0 }, // keys+ #else+ { 0 }, // key+ #endif+ #endif};-#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty)+#define MI_PAGE_EMPTY() ((mi_page_t*)&mi_page_empty)++#if MI_SMALL_WSIZE_MAX == 128+#define MI_INIT_PAGES_DIRECT(p) MI_INIT128(p)+#elif MI_SMALL_WSIZE_MAX == 256+#define MI_INIT_PAGES_DIRECT(p) MI_INIT128(p), MI_INIT128(p)+#else+#error define initializer for direct pages+#endif-#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8)-#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }+#if (MI_PADDING>0) && (MI_SIZE_SIZE >= 8)+#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }#elif (MI_PADDING>0)-#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }+#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }#else-#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() }+#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY() }#endif@@ -70,15 +87,16 @@ const mi_page_t _mi_page_empty = {#define MI_STAT_COUNT(stat) {0,0,0},#define MI_STAT_COUNTER(stat) {0},-#define MI_STATS_NULL \- MI_STAT_FIELDS() \- \- { MI_INIT4(MI_STAT_COUNT_NULL) }, \- { { 0 }, { 0 }, { 0 }, { 0 } }, \- \- { MI_INIT74(MI_STAT_COUNT_NULL) }, \- { MI_INIT74(MI_STAT_COUNT_NULL) }, \- { MI_INIT5(MI_STAT_COUNT_NULL) }+#define MI_STATS_FIELDS_NULL \+ MI_STAT_FIELDS() /* regular stat fields */ \+ { MI_INIT4(MI_STAT_COUNT_NULL) }, /* stat reserved */ \+ { { 0 }, { 0 }, { 0 }, { 0 } }, /* stat counter reserved */ \+ { MI_INIT74(MI_STAT_COUNT_NULL) }, /* malloc_bins */ \+ { MI_INIT74(MI_STAT_COUNT_NULL) }, /* page bins */ \+ { MI_INIT5(MI_STAT_COUNT_NULL) } /* chunk bins */++#define MI_STATS_NULL \+ { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_FIELDS_NULL }// --------------------------------------------------------// Statically allocate an empty theap as the initial@@ -89,21 +107,11 @@ const mi_page_t _mi_page_empty = {// may lead to allocation itself on some platforms)// ---------------------------------------------------------static mi_decl_cache_align mi_subproc_t subproc_main-#if __cplusplus- = { }; // empty initializer to prevent running the constructor (with msvc)-#else- = { 0 }; // C zero initialize-#endif--static mi_subproc_t* subprocs = &subproc_main;-static mi_lock_t subprocs_lock;--static mi_decl_cache_align mi_tld_t tld_empty = {- 0, // thread_id+static mi_decl_cache_align mi_tld_t mi_tld_detached = {+ MI_THREADID_DETACHED, // thread_id0, // thread_seq0, // default numa node- &subproc_main, // subproc+ NULL, // subprocNULL, // theaps listMI_LOCK_INITIALIZER, // theaps lockfalse, // recurse@@ -111,37 +119,21 @@ static mi_decl_cache_align mi_tld_t tld_empty = {MI_MEMID_STATIC // memid};-mi_decl_cache_align const mi_theap_t _mi_theap_empty = {- &tld_empty, // tld+mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = {+ MI_SMALL_PAGES_EMPTY, // direct small pages+ &mi_tld_detached, // tldMI_ATOMIC_VAR_INIT(NULL), // heap+ MI_ATOMIC_VAR_INIT(NULL), // subprocMI_ATOMIC_VAR_INIT(1), // refcount- 0, // heartbeat- 0, // cookie- { {0}, {0}, 0, true }, // random- 0, // page count- MI_BIN_FULL, 0, // page retired min/max- 0, // pages_full_size- 0, 0, // generic count- NULL, NULL, // tnext, tprev- NULL, NULL, // hnext, hprev0, // full page retainfalse, // allow reclaimtrue, // allow abandon- #if MI_GUARDED- 0, 0, 0, 1, // sample count is 1 so we never write to it (see `internal.h:mi_theap_malloc_use_guarded`)- #endif- MI_SMALL_PAGES_EMPTY,- MI_PAGE_QUEUES_EMPTY,- MI_MEMID_STATIC,- { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats-};--mi_decl_cache_align const mi_theap_t _mi_theap_empty_wrong = {- &tld_empty, // tld- MI_ATOMIC_VAR_INIT(NULL), // heap- MI_ATOMIC_VAR_INIT(1), // refcount+ true, // is_detached+ ~MI_ZU(0), // sample countdown: "-1" (with a sample rate of 0, so we won't write to the empty theap with MI_SAMPLE==2)+ 0, 0, // sample rate, requested+ 0, 0, // profile rate, countdown+ 0, 0, 0, 0, // guarded rate, countdown, min, max0, // heartbeat- 0, // cookie{ {0}, {0}, 0, true }, // random0, // page countMI_BIN_FULL, 0, // page retired min/max@@ -149,128 +141,38 @@ mi_decl_cache_align const mi_theap_t _mi_theap_empty_wrong = {0, 0, // generic countNULL, NULL, // tnext, tprevNULL, NULL, // hnext, hprev- 0, // full page retain- false, // allow reclaim- true, // allow abandon- #if MI_GUARDED- 0, 0, 0, 1, // sample count is 1 so we never write to it (see `internal.h:mi_theap_malloc_use_guarded`)- #endif- MI_SMALL_PAGES_EMPTY,MI_PAGE_QUEUES_EMPTY,MI_MEMID_STATIC,- { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats+ MI_STATS_NULL, // stats};-// Heap for the main thread--extern mi_decl_hidden mi_decl_cache_align mi_theap_t theap_main;-extern mi_decl_hidden mi_decl_cache_align mi_heap_t heap_main;--static mi_decl_cache_align mi_tld_t tld_main = {- 0, // thread_id- 0, // thread_seq- 0, // numa node- &subproc_main, // subproc- &theap_main, // theaps list- MI_LOCK_INITIALIZER, // theaps lock- false, // recurse- false, // is_in_threadpool- MI_MEMID_STATIC // memid-};--mi_decl_cache_align mi_theap_t theap_main = {- &tld_main, // thread local data- MI_ATOMIC_VAR_INIT(&heap_main), // main heap- MI_ATOMIC_VAR_INIT(1), // refcount- 0, // heartbeat- 0, // initial cookie- { {0x846ca68b}, {0}, 0, true }, // random- 0, // page count- MI_BIN_FULL, 0, // page retired min/max- 0, // pages_full_size- 0, 0, // generic count- NULL, NULL, // tnext, tprev- NULL, NULL, // hnext, hprev- 2, // full page retain- true, // allow page reclaim- true, // allow page abandon- #if MI_GUARDED- 0, 0, 0, 0,- #endif- MI_SMALL_PAGES_EMPTY,- MI_PAGE_QUEUES_EMPTY,- MI_MEMID_STATIC,- { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats-};--mi_decl_cache_align mi_heap_t heap_main-#if __cplusplus- = { }; // empty initializer to prevent running the constructor (with msvc)-#else- = { 0 }; // C zero initialize-#endif--// the theap belonging to the main heap-mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main = NULL;+#undef MI_STAT_COUNT+#undef MI_STAT_COUNTER-mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {- mi_threadid_t tid = _mi_prim_thread_id();- mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits- return tid;-}-#if MI_TLS_MODEL_THREAD_LOCAL-// the thread-local main theap for allocation-mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default = (mi_theap_t*)&_mi_theap_empty;-// the last used non-main theap-mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached = (mi_theap_t*)&_mi_theap_empty;-#endif+// pre-allocate the process heap, and meta-data theap+static mi_decl_cache_align mi_heap_t mi_process_heap_main = mi_init_struct_zero;+static mi_decl_cache_align mi_theap_t mi_process_theap_meta = mi_init_struct_zero;-bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`.+// pre-allocate the initial tld and theap for the main thread (this is not strictly needed but nice for stats)+static mi_decl_cache_align mi_tld_t mi_process_tld_main = mi_init_struct_zero;+static mi_decl_cache_align mi_theap_t mi_process_theap_main = mi_init_struct_zero;-mi_stats_t _mi_stats_main = { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL };+mi_decl_hidden mi_decl_cache_align mi_theap_t _mi_theap_empty_wrong = mi_init_struct_zero; // used for error paths+mi_decl_hidden bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`.-#undef MI_STAT_COUNT-#undef MI_STAT_COUNTER---#if MI_GUARDED-mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) {- theap->guarded_sample_rate = sample_rate;- theap->guarded_sample_count = sample_rate; // count down samples- if (theap->guarded_sample_rate > 1) {- if (seed == 0) {- seed = _mi_theap_random_next(theap);- }- theap->guarded_sample_count = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate`- }-}-mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) {- theap->guarded_size_min = min;- theap->guarded_size_max = (min > max ? min : max);+mi_page_t* _mi_page_empty_get(void) {+ return (mi_page_t*)&mi_page_empty;}-void _mi_theap_guarded_init(mi_theap_t* theap) {- mi_theap_guarded_set_sample_rate(theap,- (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX),- (size_t)mi_option_get(mi_option_guarded_sample_seed));- mi_theap_guarded_set_size_bound(theap,- (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX),- (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) );-}-#else-mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) {- MI_UNUSED(theap); MI_UNUSED(sample_rate); MI_UNUSED(seed);+mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void) {+ return (mi_theap_t*)&_mi_theap_empty;}-mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) {- MI_UNUSED(theap); MI_UNUSED(min); MI_UNUSED(max);-}-void _mi_theap_guarded_init(mi_theap_t* theap) {- MI_UNUSED(theap);+bool _mi_is_empty_theap(const mi_theap_t* theap) {+ return (theap == &_mi_theap_empty);}-#endif/* -----------------------------------------------------------Initialization@@ -278,365 +180,212 @@ void _mi_theap_guarded_init(mi_theap_t* theap) {can cause allocation and induce recursion during initialization.----------------------------------------------------------- */+static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc);-// Initialize main subproc-static void mi_subproc_main_init(void) {- if (subproc_main.memid.memkind != MI_MEM_STATIC) {- subproc_main.memid = _mi_memid_create(MI_MEM_STATIC);- subproc_main.heaps = &heap_main;- subproc_main.heap_total_count = 1;- subproc_main.heap_count = 1;- mi_atomic_store_ptr_release(mi_heap_t, &subproc_main.heap_main, &heap_main);- __mi_stat_increase_mt(&subproc_main.stats.heaps, 1);- mi_stats_header_init(&subproc_main.stats);- mi_lock_init(&subproc_main.arena_reserve_lock);- mi_lock_init(&subproc_main.heaps_lock);- mi_lock_init(&subprocs_lock);- mi_lock_init(&tld_empty.theaps_lock);- }-}+// Initialize main heap+static void mi_heap_main_init_once(void) {+ mi_memid_t memid_static = _mi_memid_create(MI_MEM_STATIC);+ _mi_memcpy(&_mi_theap_empty_wrong,&_mi_theap_empty,sizeof(_mi_theap_empty_wrong));-// Initialize main tld-static void mi_tld_main_init(void) {- if (tld_main.thread_id == 0) {- tld_main.thread_id = _mi_prim_thread_id();- mi_lock_init(&tld_main.theaps_lock);- }-}+ // initialize the main subprocess+ mi_subproc_t* subproc_main = _mi_subproc_main_init();-void _mi_theap_options_init(mi_theap_t* theap) {- theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0);- theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0);- theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32);-}+ // detached tld for mi_theap_empty (and theap_meta)+ mi_tld_detached.memid = memid_static;+ mi_tld_init(&mi_tld_detached, 0, subproc_main);-// Initialization of the (statically allocated) main theap, and the main tld and subproc.-static void mi_theap_main_init(void) {- if mi_unlikely(theap_main.memid.memkind != MI_MEM_STATIC) {- // theap- theap_main.memid = _mi_memid_create(MI_MEM_STATIC);- #if defined(__APPLE__) || defined(_WIN32) && !defined(MI_SHARED_LIB)- _mi_random_init_weak(&theap_main.random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185)- #else- _mi_random_init(&theap_main.random);- #endif- theap_main.cookie = _mi_theap_random_next(&theap_main);- _mi_theap_options_init(&theap_main);- _mi_theap_guarded_init(&theap_main);- }-}+ // main process heap+ mi_process_heap_main.memid = memid_static;+ mi_atomic_store_ptr_release(mi_heap_t,&subproc_main->heap_main,&mi_process_heap_main);+ _mi_heap_init(&mi_process_heap_main,mi_thread_local_key_fast,subproc_main,0);-// Initialize main heap-static void mi_heap_main_init(void) {- if mi_unlikely(heap_main.subproc == NULL) {- heap_main.subproc = &subproc_main;- heap_main.theaps = &theap_main;+ // detached theap for allocating meta-data (we can allocate on this without having an initialized thread)+ mi_process_theap_meta.memid = memid_static;+ _mi_theap_init(&mi_process_theap_meta,&mi_process_heap_main,&mi_tld_detached);+ mi_process_theap_meta.allow_page_abandon = false; // for security, don't share with other threads+ mi_process_theap_meta.page_full_retain = 2;+ mi_process_theap_meta.sample_rate = 0; // no sampling for meta data+ mi_process_theap_meta.sample_countdown = 0;+ subproc_main->theap_meta = &mi_process_theap_meta;- mi_theap_main_init();- mi_subproc_main_init();- mi_tld_main_init();+ // mi_heap_theap_set(&mi_process_heap_main,&mi_process_theap_main); // set in `mi_thread_init(_theap_default)`+}- mi_lock_init(&heap_main.theaps_lock);- mi_lock_init(&heap_main.os_abandoned_pages_lock);- mi_lock_init(&heap_main.arena_pages_lock);+static void mi_heap_main_init(void) {+ mi_atomic_do_once {+ mi_heap_main_init_once();}}+mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) {+ mi_heap_t* heap = mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main);+ if mi_likely(heap!=NULL) {+ return heap;+ }+ else if (_mi_subproc_is_main(subproc)) {+ mi_heap_main_init();+ mi_assert_internal(mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main) != NULL);+ return mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main);+ }+ else {+ mi_assert_internal(false);+ return &mi_process_heap_main;+ }+}/* -----------------------------------------------------------Thread local data----------------------------------------------------------- */-// Allocate fresh tld-static mi_tld_t* mi_tld_alloc(void) {- if (_mi_is_main_thread()) {- mi_atomic_increment_relaxed(&tld_main.subproc->thread_count);- return &tld_main;+static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc) {+ tld->subproc = subproc;+ tld->theaps = NULL;+ mi_lock_init(&tld->theaps_lock);+ if (tld->thread_id == MI_THREADID_DETACHED) {+ tld->numa_node = -1;}else {- // allocate tld meta-data- // note: we need to be careful to not access the tld from `_mi_meta_zalloc`- // (and in turn from `_mi_arena_alloc_aligned` and `_mi_os_alloc_aligned`).- mi_memid_t memid;- mi_tld_t* tld = (mi_tld_t*)_mi_meta_zalloc(sizeof(mi_tld_t), &memid);- if (tld==NULL) {- _mi_error_message(ENOMEM, "unable to allocate memory for thread local data\n");- return NULL;- }- tld->memid = memid;- tld->theaps = NULL;- mi_lock_init(&tld->theaps_lock);- tld->subproc = &subproc_main;tld->numa_node = _mi_os_numa_node();tld->thread_id = _mi_prim_thread_id();- tld->thread_seq = mi_atomic_increment_relaxed(&tld->subproc->thread_total_count);tld->is_in_threadpool = _mi_prim_thread_is_in_threadpool();+ tld->thread_seq = tseq;mi_atomic_increment_relaxed(&tld->subproc->thread_count);- return tld;}+ return tld;}-#define MI_TLD_INVALID ((mi_tld_t*)1)--mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) {- mi_lock_done(&tld->theaps_lock);- if (tld != NULL && tld != MI_TLD_INVALID) {- mi_atomic_decrement_relaxed(&tld->subproc->thread_count);- _mi_meta_free(tld, sizeof(mi_tld_t), tld->memid);- }- #if 0- // do not read/write to `thread_tld` on older macOS <= 14 as that will re-initialize the thread local storage- // (since we are calling this during pthread shutdown)- // (and this could happen on other systems as well, so let's never do it)- thread_tld = MI_TLD_INVALID;- #endif-}--// return the thread local heap ensuring it is initialized (and not `NULL` or `&_mi_theap_empty`);-mi_theap_t* _mi_theap_default_safe(void) {- mi_theap_t* theap = _mi_theap_default();- if mi_likely(mi_theap_is_initialized(theap)) return theap;- mi_thread_init();- mi_assert_internal(mi_theap_is_initialized(_mi_theap_default()));- return _mi_theap_default();-}--// return the main theap ensuring it is initialized.-mi_theap_t* _mi_theap_main_safe(void) {- mi_theap_t* theap = __mi_theap_main;- if mi_unlikely(theap==NULL) { // if thread_init or default_set was never called- mi_thread_init(); // sets the default slot to the main theap- theap = _mi_theap_default();- mi_assert_internal(theap!=NULL);- mi_assert_internal(_mi_is_theap_main(theap));- if (_mi_is_theap_main(theap)) {- __mi_theap_main = theap;- }- }- mi_assert_internal(theap!=NULL && _mi_is_theap_main(theap));- return theap;-}---mi_subproc_t* _mi_subproc_main(void) {- return &subproc_main;-}+// Allocate fresh tld+static mi_tld_t* mi_tld_create(mi_subproc_t* subproc) {+ mi_assert_internal(subproc->theap_meta != NULL); // should be initialized on the main thread before other threads allocate+ const size_t tseq = mi_atomic_increment_relaxed(&subproc->thread_total_count);-mi_subproc_t* _mi_subproc(void) {- // should work without doing initialization (as it may be called from `_mi_tld -> mi_tld_alloc ... -> os_alloc -> _mi_subproc()`- // todo: this will still fail on OS systems where the first access to a thread-local causes allocation.- // on such systems we can check for this with the _mi_prim_get_default_theap as those are protected (by being- // stored in a TLS slot for example)- mi_theap_t* theap = _mi_theap_default();- if (theap == NULL) {- return _mi_subproc_main();+ mi_memid_t memid;+ mi_tld_t* tld;+ if (_mi_subproc_is_main(subproc) && tseq==0 /* first tld */) {+ tld = &mi_process_tld_main;+ memid = _mi_memid_create_static(tld,sizeof(*tld));}else {- return theap->tld->subproc; // avoid using thread local storage (`thread_tld`)- }-}--mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) {- mi_heap_t* heap = mi_atomic_load_ptr_relaxed(mi_heap_t,&subproc->heap_main);- if mi_likely(heap!=NULL) {- return heap;+ tld = (mi_tld_t*)_mi_meta_zalloc(subproc, sizeof(mi_tld_t), &memid);}- else {- mi_heap_main_init();- mi_assert_internal(mi_atomic_load_relaxed(&subproc->heap_main) != NULL);- return mi_atomic_load_ptr_relaxed(mi_heap_t,&subproc->heap_main);+ if (tld==NULL) {+ _mi_error_message(ENOMEM, "unable to allocate memory for thread local data\n");+ return NULL;}+ tld->memid = memid;+ return mi_tld_init(tld,tseq,subproc);}-mi_heap_t* mi_heap_main(void) {- return _mi_subproc_heap_main(_mi_subproc()); // don't use mi_theap_main_init_get() so this call works during process_init-}--bool _mi_is_heap_main(const mi_heap_t* heap) {- mi_assert_internal(heap!=NULL);- return (_mi_subproc_heap_main(heap->subproc) == heap);+mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) {+ if (tld==NULL) return;+ mi_atomic_decrement_relaxed(&tld->subproc->thread_count);+ tld->thread_id = (mi_threadid_t)(~0); // it is best to set an invalid tid for tld_main as sometimes the same thread-id+ // is reused by the OS after a thread has terminated. (see issue #1287)+ mi_lock_done(&tld->theaps_lock);+ _mi_meta_free(tld->subproc, tld, tld->memid); // note: safe for static tld}-bool _mi_is_theap_main(const mi_theap_t* theap) {- return (mi_theap_is_initialized(theap) && _mi_is_heap_main(_mi_theap_heap(theap)));-}/* ------------------------------------------------------------ Sub process+ Thread Init----------------------------------------------------------- */--mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) {- return (mi_subproc_t*)(subproc_id._mi_subproc_id);-}--mi_subproc_id_t _mi_subproc_to_id(mi_subproc_t* subproc) {- mi_subproc_id_t id = { subproc };- return id;-}--mi_subproc_id_t mi_subproc_main(void) {- return _mi_subproc_to_id(_mi_subproc_main());-}--mi_subproc_id_t mi_subproc_current(void) {- return _mi_subproc_to_id(_mi_subproc());-}--mi_subproc_id_t mi_subproc_new(void) {- static _Atomic(size_t) subproc_total_count;- mi_memid_t memid;- mi_subproc_t* subproc = (mi_subproc_t*)_mi_meta_zalloc(sizeof(mi_subproc_t),&memid);- if (subproc == NULL) return _mi_subproc_to_id(NULL);- subproc->memid = memid;- subproc->subproc_seq = mi_atomic_increment_relaxed(&subproc_total_count) + 1;- mi_stats_header_init(&subproc->stats);- mi_lock_init(&subproc->arena_reserve_lock);- mi_lock_init(&subproc->heaps_lock);- mi_lock(&subprocs_lock) {- // push on subproc list- subproc->next = subprocs;- if (subprocs!=NULL) { subprocs->prev = subproc; }- subprocs = subproc;+#if MI_DEBUG || defined(MI_TLS_RECURSE_GUARD)+static mi_theap_t* mi_heap_check_for_existing_theap(mi_heap_t* heap) {+ const mi_threadid_t tid = _mi_thread_id();+ mi_theap_t* thread_theap = NULL;+ mi_lock(&heap->theaps_lock) {+ for(mi_theap_t* theap = heap->theaps; theap != NULL; theap = theap->hnext ) {+ if (theap->tld->thread_id == tid) {+ thread_theap = theap;+ break;+ }+ }}- return _mi_subproc_to_id(subproc);+ return thread_theap;}+#endif-// destroy all subproc resources including arena's, heap's etc.-static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subprocs_lock)+// Initialize thread+mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap_main){- if (subproc==NULL) return;+ // ensure our process has started already+ mi_process_init();- // remove from the subproc list- mi_lock_maybe(&subprocs_lock, acquire_subprocs_lock) {- if (subproc->next!=NULL) { subproc->next->prev = subproc->prev; }- if (subproc->prev!=NULL) { subproc->prev->next = subproc->next; }- else { mi_assert_internal(subprocs==subproc); subprocs = subproc->next; }- }+ // if the theap_default is already set we have already initialized+ mi_theap_t* theap = _mi_theap_default();+ if (mi_theap_is_initialized(theap)) return theap;++ // initialize the default theap+ // note: we cannot access thread-locals yet as that can cause (recursive) allocation+ // (on macOS <= 14 for example where the loader allocates thread-local data on demand).+ if (heap_main==NULL) {+ heap_main = mi_heap_main();+ mi_assert_internal(heap_main == &mi_process_heap_main);+ }+ mi_assert_internal(heap_main!=NULL);++ #if MI_DEBUG || defined(MI_TLS_RECURSE_GUARD)+ theap = mi_heap_check_for_existing_theap(heap_main); // recursion check+ #if !defined(MI_TLS_RECURSE_GUARD)+ mi_assert_internal(theap==NULL);+ #endif+ #else+ theap = NULL;+ #endif- // destroy all subproc heaps- mi_lock(&subproc->heaps_lock) {- mi_heap_t* heap = subproc->heaps;- while (heap != NULL) {- mi_heap_t* next = heap->next;- if (heap!=subproc->heap_main) { mi_heap_destroy(heap); }- heap = next;+ if (theap==NULL) {+ // allocated the tld+ mi_tld_t* tld = mi_tld_create(heap_main->subproc);+ if (tld==NULL) return NULL; // out-of-memory on tld allocation+ // allocate and initialize the theap for the main heap+ if (tld==&mi_process_tld_main) {+ theap = &mi_process_theap_main; // initial theap is pre-allocated+ theap->memid = _mi_memid_create_static(theap,sizeof(*theap));+ }+ else {+ theap = _mi_theap_alloc(heap_main,tld); // otherwise meta allocate+ if (theap==NULL) { mi_tld_free(tld); return NULL; } // out-of-memory on theap allocation}- mi_assert_internal(subproc->heaps == subproc->heap_main);- _mi_heap_force_destroy(subproc->heap_main); // no warning if destroying the main heap+ _mi_theap_init(theap,heap_main,tld);}- // remove associated arenas- _mi_arenas_unsafe_destroy_all(subproc);+ // now initialize the thread+ _mi_theap_default_set(theap);+ // and only then set the heap_theap field as that accesses thread locals+ _mi_heap_theap_set(heap_main, theap); // todo: can fail!- // merge stats back into the main subproc?- if (subproc!=&subproc_main) {- _mi_stats_merge_into(&subproc_main.stats, &subproc->stats);- }+ mi_assert_internal(mi_theap_is_initialized(theap));+ mi_theap_t* const heap_theap = (heap_main==NULL ? NULL : (mi_theap_t*)_mi_thread_local_get(heap_main->theap));+ mi_assert_internal(heap_main==NULL || heap_theap == theap); MI_UNUSED_RELEASE(heap_theap);- // safe to release- // todo: should we refcount subprocesses?- mi_lock_done(&subproc->arena_reserve_lock);- mi_lock_done(&subproc->heaps_lock);- if (subproc!=&subproc_main) {- _mi_meta_free(subproc, sizeof(mi_subproc_t), subproc->memid);- }- else {- // for the main subproc, also release the global page map- _mi_page_map_unsafe_destroy(&subproc_main);- }-}--void mi_subproc_destroy(mi_subproc_id_t subproc_id) {- mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);- if (subproc==NULL || subproc==&subproc_main) return;- mi_subproc_unsafe_destroy(subproc, true /* take lock */);+ mi_subproc_stat_increase(_mi_theap_subproc(theap), threads, 1); // or theap stats and wait for merge?+ // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());+ return theap;}-static void mi_subprocs_unsafe_destroy_all(void) {- mi_lock(&subprocs_lock) {- mi_subproc_t* subproc = subprocs;- while (subproc!=NULL) {- mi_subproc_t* next = subproc->next;- if (subproc!=&subproc_main) {- mi_subproc_unsafe_destroy(subproc, false /* take subprocs lock */);- }- subproc = next;- }- }- mi_subproc_unsafe_destroy(&subproc_main, true /* take subprocs lock */);+mi_theap_t* _mi_thread_init(void) {+ return _mi_thread_init_with_heap(NULL);}--void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) {- mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);- mi_tld_t* const tld = _mi_theap_default_safe()->tld;- mi_assert(tld->subproc== &subproc_main);- if (tld->subproc != &subproc_main) {- _mi_warning_message("unable to add thread to the subprocess as it was already in another subprocess (id: %p)\n", subproc);- return;- }- tld->subproc = subproc;- tld->thread_seq = mi_atomic_increment_relaxed(&subproc->thread_total_count);- mi_atomic_decrement_relaxed(&subproc_main.thread_count);- mi_atomic_increment_relaxed(&subproc->thread_count);+void mi_decl_noinline mi_thread_init(void) mi_attr_noexcept {+ _mi_thread_init();}-bool mi_subproc_visit_heaps(mi_subproc_id_t subproc_id, mi_heap_visit_fun* visitor, void* arg) {- mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);- if (subproc==NULL) return false;- bool ok = true;- mi_lock(&subproc->heaps_lock) {- for (mi_heap_t* heap = subproc->heaps; heap!=NULL && ok; heap = heap->next) {- ok = (*visitor)(heap, arg);- }- }- return ok;-}-/* ------------------------------------------------------------ Allocate theap data+ Theaps done----------------------------------------------------------- */-// Initialize the thread local default theap, called from `mi_thread_init`-static mi_theap_t* _mi_thread_init_theap_default(void) {- mi_theap_t* theap = _mi_theap_default();- if (mi_theap_is_initialized(theap)) return theap;- if (_mi_is_main_thread()) {- mi_heap_main_init();- theap = &theap_main;- }- else {- // allocates tld data- // note: we cannot access thread-locals yet as that can cause (recursive) allocation- // (on macOS <= 14 for example where the loader allocates thread-local data on demand).- mi_tld_t* tld = mi_tld_alloc();- if (tld==NULL) return NULL; // things are very wrong if this fails (out of memory)- // allocate and initialize the theap for the main heap- theap = _mi_theap_create(mi_heap_main(), tld);- }- // associate the theap with this thread- // (this is safe, on macOS for example, the theap is set in a dedicated TLS slot and thus does not cause recursive allocation)- _mi_theap_default_set(theap);- return theap;-}--// Free the thread local theapsstatic void mi_thread_theaps_done(mi_tld_t* tld){- // reset the thread local theaps- _mi_theap_default_set((mi_theap_t*)&_mi_theap_empty);- _mi_theap_cached_set((mi_theap_t*)&_mi_theap_empty);- __mi_theap_main = NULL;-// abandon the pages of all theaps in this threadmi_lock(&tld->theaps_lock) {mi_theap_t* theap = tld->theaps;while (theap != NULL) {- mi_theap_t* next = theap->tnext;+ mi_theap_t* next = theap->tnext;// never destroy theaps; if a dll is linked statically with mimalloc,// there may still be delete/free calls after the mi_fls_done is called. Issue #207_mi_theap_collect_abandon(theap);@@ -645,39 +394,38 @@ static void mi_thread_theaps_done(mi_tld_t* tld)}}- // free the theaps of this thread.- // This can run concurrently with a `mi_heap_free_theaps` and we need to ensure we free theaps atomically.- // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock- // potential concurrent `mi_heap_free_theaps` which tries to remove the theap from our theaps list.- bool all_freed;- do {- all_freed = true;- mi_lock(&tld->theaps_lock) {- mi_theap_t* theap = tld->theaps;- while (theap != NULL) {- mi_theap_t* next = theap->tnext;- mi_assert_internal(theap->page_count==0);- if (!_mi_theap_free(theap, true /* acquire heap->theaps_lock */, false /* dont re-acquire the tld->theaps_lock*/ )) {- all_freed = false;- }- theap = next;- }- }- if (!all_freed) {- mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1);- _mi_prim_thread_yield();- }- else {- mi_assert_internal(tld->theaps==NULL);+ // reset the thread local theaps+ // note: do this after abandon as page->heap may be NULL and mi_heap_main should return the heap+ // belonging to the right subprocess+ _mi_theap_default_set((mi_theap_t*)&_mi_theap_empty);+ _mi_theap_cached_set((mi_theap_t*)&_mi_theap_empty);++ // We might run concurrently with a `mi_heap_free_theaps` and we need to ensure we free theaps atomically.+ // we first detach our theaps list from any heaps+ _mi_tld_detach_theaps(tld);++ // no heaps point to our theaps anymore, free them+ mi_lock(&tld->theaps_lock) { // paranoia+ mi_theap_t* theap = tld->theaps;+ tld->theaps = NULL;+ while (theap != NULL) {+ mi_theap_t* next = theap->tnext;+ mi_assert_internal(theap->page_count==0);+ mi_assert_internal(_mi_theap_heap_peek(theap)==NULL);+ theap->tld = NULL;+ theap->tnext = NULL;+ theap->tprev = NULL;+ mi_assert_internal(mi_atomic_load_relaxed(&theap->refcount) == 1); // as the cached entry is set to empty+ _mi_theap_decref(theap);+ theap = next;}- } while (!all_freed);+ }mi_assert(_mi_theap_default()==(mi_theap_t*)&_mi_theap_empty); // careful to not re-initialize the default theap during theap_deletemi_assert(!mi_theap_is_initialized(_mi_theap_default()));}-// --------------------------------------------------------// Try to run `mi_thread_done()` automatically so any memory// owned by the thread but not yet released can be abandoned@@ -694,48 +442,27 @@ static void mi_thread_theaps_done(mi_tld_t* tld)// to set up the thread local keys.// ---------------------------------------------------------// Set up handlers so `mi_thread_done` is called automatically+// Set up hooks so `mi_thread_done` is called automaticallystatic void mi_process_setup_auto_thread_done(void) {mi_atomic_do_once {_mi_prim_thread_init_auto_done();- _mi_theap_default_set(&theap_main);+ mi_theap_t* theap = _mi_theap_default();+ mi_assert_internal(mi_theap_is_initialized(theap));+ if (mi_theap_is_initialized(theap)) {+ _mi_theap_default_set(theap);+ }}}--bool _mi_is_main_thread(void) {- return (tld_main.thread_id==0 || tld_main.thread_id == _mi_thread_id());-}---// Initialize thread-void mi_thread_init(void) mi_attr_noexcept-{- // ensure our process has started already- mi_process_init();- // if the theap_default is already set we have already initialized- if (_mi_thread_is_initialized()) return;-- // initialize the default theap- if (_mi_thread_init_theap_default() == NULL) return; // out-of-memory on tld/theap allocation-- mi_heap_stat_increase(mi_heap_main(), threads, 1);- // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());-}-void mi_thread_done(void) mi_attr_noexcept {_mi_thread_done(NULL);}void _mi_thread_done(mi_theap_t* _theap_main)-{+{// NULL can be passed on some platformsif (_theap_main==NULL) {- _theap_main = __mi_theap_main; // don't call `mi_theap_main_safe` as that re-initializes the thread- if (_theap_main==NULL) { // can happen if `mi_theap_main_safe` is never called; but then the default is main- _theap_main = _mi_theap_default();- mi_assert_internal(_theap_main==NULL || _mi_is_theap_main(_theap_main));- }+ _theap_main = _mi_theap_default();}// prevent re-entrancy through theap_done/theap_set_default_direct (issue #699)@@ -743,14 +470,14 @@ void _mi_thread_done(mi_theap_t* _theap_main)return;}+ // get the current tld+ mi_tld_t* const tld = _theap_main->tld;+// release dynamic thread_local's_mi_thread_locals_thread_done();- // note: we store the tld as we should avoid reading `thread_tld` at this point (to avoid reinitializing the thread local storage)- mi_tld_t* const tld = _theap_main->tld;-// adjust stats- mi_heap_stat_decrease(_mi_subproc_heap_main(tld->subproc), threads, 1); // todo: or `_theap_main->heap`?+ mi_subproc_stat_decrease(tld->subproc, threads, 1); // todo: or `_theap_main->heap`?// check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local theaps...if (tld->thread_id != _mi_prim_thread_id()) return;@@ -762,225 +489,16 @@ void _mi_thread_done(mi_theap_t* _theap_main)mi_tld_free(tld);}--mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void) {- return (mi_theap_t*)&_mi_theap_empty;-}--#if MI_TLS_MODEL_DYNAMIC_WIN32--// If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed)-// See <https://en.wikipedia.org/wiki/Win32_Thread_Information_Block> for the offsets.-#if MI_SIZE_SIZE==4-#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_SIZE_SIZE)-#else-#define MI_TLS_DIRECT_FIRST (0x1480 / MI_SIZE_SIZE)-#endif-#define MI_TLS_DIRECT_SLOTS (64)-#define MI_TLS_EXPANSION_SLOTS (1024)--#if !MI_WIN_DIRECT_TLS-#define MI_TLS_INITIAL_SLOT MI_TLS_EXPANSION_SLOT-#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1)-#else-// with only direct entries, use the "arbitrary user data" field-// and assume it is NULL (see also <http://www.nynaeve.net/?p=98>)-#define MI_TLS_INITIAL_SLOT (5)-#define MI_TLS_INITIAL_EXPANSION_SLOT (0)-#endif--// we initially use the last of the expansion slots as the default NULL.-// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc (which is quite unlikely)-mi_decl_hidden mi_decl_cache_align size_t _mi_theap_default_slot = MI_TLS_INITIAL_SLOT;-mi_decl_hidden size_t _mi_theap_default_expansion_slot = MI_TLS_INITIAL_EXPANSION_SLOT;-mi_decl_hidden size_t _mi_theap_cached_slot = MI_TLS_INITIAL_SLOT;-mi_decl_hidden size_t _mi_theap_cached_expansion_slot = MI_TLS_INITIAL_EXPANSION_SLOT;--static DWORD mi_tls_raw_index_default = TLS_OUT_OF_INDEXES;-static DWORD mi_tls_raw_index_cached = TLS_OUT_OF_INDEXES;--static bool mi_win_tls_slot_alloc(size_t* slot, size_t* extended, DWORD* raw_index) {- const DWORD index = TlsAlloc();- *raw_index = index;- if (index==TLS_OUT_OF_INDEXES) {- *extended = 0;- *slot = 0;- return false;- }- else if (index<MI_TLS_DIRECT_SLOTS) {- *extended = 0;- *slot = index + MI_TLS_DIRECT_FIRST;- return true;- }- #if !MI_WIN_DIRECT_TLS- else if (index < MI_TLS_DIRECT_SLOTS + MI_TLS_EXPANSION_SLOTS - 1) { // check maximum number of expansion slots - 1 (as we use the last one as the default)- *extended = index - MI_TLS_DIRECT_SLOTS;- *slot = MI_TLS_EXPANSION_SLOT;- return true;- }- #endif- else {- // to high an index for us- _mi_error_message(EFAULT, "returned tls index was too high (%u)\n", index);- TlsFree(index);- *raw_index = TLS_OUT_OF_INDEXES;- *extended = 0;- *slot = 0;- return false;- }-}--static void mi_win_tls_slot_free(DWORD* raw_index) {- if (*raw_index != TLS_OUT_OF_INDEXES) {- TlsFree(*raw_index);- *raw_index = TLS_OUT_OF_INDEXES;- }-}--static void mi_tls_slots_init(void) {- mi_atomic_do_once {- bool ok = mi_win_tls_slot_alloc(&_mi_theap_default_slot, &_mi_theap_default_expansion_slot, &mi_tls_raw_index_default);- if (ok) {- ok = mi_win_tls_slot_alloc(&_mi_theap_cached_slot, &_mi_theap_cached_expansion_slot, &mi_tls_raw_index_cached);- }- if (!ok) {- _mi_error_message(EFAULT, "unable to allocate fast TLS user slot (0x%zx)\n", _mi_theap_cached_slot);- }- }-}--static void mi_tls_slots_done(void) {- mi_win_tls_slot_free(&mi_tls_raw_index_default);- mi_win_tls_slot_free(&mi_tls_raw_index_cached );-}--static void mi_win_tls_slot_set(size_t slot, size_t extended_slot, void* value) {- mi_assert_internal((slot >= MI_TLS_DIRECT_FIRST && slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) || slot == MI_TLS_EXPANSION_SLOT);- if (slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) {- mi_prim_tls_slot_set(slot, value);- }- else {- mi_assert_internal(extended_slot < MI_TLS_EXPANSION_SLOTS);- TlsSetValue((DWORD)(extended_slot + MI_TLS_DIRECT_SLOTS), value); // use TlsSetValue to initialize the TlsExpansion array if needed- }-}--#elif MI_TLS_MODEL_DYNAMIC_PTHREADS--// only for pthreads for now-mi_decl_hidden pthread_key_t _mi_theap_default_key = 0;-mi_decl_hidden pthread_key_t _mi_theap_cached_key = 0;--// create a non-zero pthread key-static int mi_pthread_key_create( pthread_key_t* pkey ) {- pthread_key_t key;- int err = pthread_key_create(&key, NULL);- if (err!=0) return err;- if (key==0) {- // if we get a zero key, create another one as we use 0 for an invalid key- pthread_key_t key2;- err = pthread_key_create(&key2, NULL);- pthread_key_delete(key); // delete the old key- if (err!=0) return err;- key = key2;- }- mi_assert_internal(key!=0);- *pkey = key;- return 0;-}--static void mi_tls_slots_init(void) {- mi_atomic_do_once {- int err = mi_pthread_key_create(&_mi_theap_default_key);- if (err==0) {- err = mi_pthread_key_create(&_mi_theap_cached_key);- }- if (err!=0) {- _mi_error_message(EFAULT, "unable to allocate pthread keys (error %d)\n", err);- }- }-}--static void mi_tls_slots_done(void) {- if (_mi_theap_default_key != 0) {- pthread_key_delete(_mi_theap_default_key);- _mi_theap_default_key = 0;- }- if (_mi_theap_cached_key != 0) {- pthread_key_delete(_mi_theap_cached_key);- _mi_theap_cached_key = 0;- }-}--#else--static void mi_tls_slots_init(void) {- // nothing-}--static void mi_tls_slots_done(void) {- // nothing-}--#endif--void _mi_theap_cached_set(mi_theap_t* theap) {- mi_theap_t* prev = _mi_theap_cached();- if (prev==theap) return;- // set- mi_tls_slots_init();- #if MI_TLS_MODEL_THREAD_LOCAL- __mi_theap_cached = theap;- #elif MI_TLS_MODEL_FIXED_SLOT- mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_SLOT_CACHED, theap);- #elif MI_TLS_MODEL_DYNAMIC_WIN32- mi_win_tls_slot_set(_mi_theap_cached_slot, _mi_theap_cached_expansion_slot, theap);- #elif MI_TLS_MODEL_DYNAMIC_PTHREADS- if (_mi_theap_cached_key!=0) pthread_setspecific(_mi_theap_cached_key, theap);- #endif- // update refcounts (so cached theap memory keeps available until no longer cached)- _mi_theap_incref(theap);- _mi_theap_decref(prev);-}--void _mi_theap_default_set(mi_theap_t* theap) {- mi_theap_t* const theap_old = _mi_theap_default();- mi_assert_internal(theap != NULL);- mi_assert_internal(theap->tld->thread_id==0 || theap->tld->thread_id==_mi_thread_id());- mi_tls_slots_init();- #if MI_TLS_MODEL_THREAD_LOCAL- __mi_theap_default = theap;- #elif MI_TLS_MODEL_FIXED_SLOT- mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_SLOT_DEFAULT, theap);- #elif MI_TLS_MODEL_DYNAMIC_WIN32- mi_win_tls_slot_set(_mi_theap_default_slot, _mi_theap_default_expansion_slot, theap);- #elif MI_TLS_MODEL_DYNAMIC_PTHREADS- if (_mi_theap_default_key!=0) pthread_setspecific(_mi_theap_default_key, theap);- #endif-- // set theap main if needed- if (mi_theap_is_initialized(theap)) {- // ensure the default theap is passed to `_mi_thread_done` as on some platforms we cannot access TLS at thread termination (as it would allocate again)- _mi_prim_thread_associate_default_theap(theap);- if (_mi_is_heap_main(_mi_theap_heap(theap))) {- __mi_theap_main = theap;- }- }-- // ensure either the default slot contains the main theap, or __mi_theap_main is initialized- if (mi_theap_is_initialized(theap_old) && _mi_is_heap_main(_mi_theap_heap(theap_old))) {- __mi_theap_main = theap_old;- }-}-void mi_thread_set_in_threadpool(void) mi_attr_noexcept {- mi_theap_t* theap = _mi_theap_default_safe();+ mi_theap_t* theap = mi_theap_get_default();theap->tld->is_in_threadpool = true;}+// ---------------------------------------------------------// Run functions on process init/done, and thread init/done+// Process init and done// --------------------------------------------------------+static bool os_preloading = true; // true until this module is initialized// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false.@@ -993,20 +511,12 @@ mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept {return _mi_is_redirected();}-// Called once by the process loader from `src/prim/prim.c`+// Called once by the process loader from `src/prim/prim.c` before `main` is called.void _mi_auto_process_init(void) {- // mi_heap_main_init();- // #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD)- // volatile mi_theap_t* dummy = __mi_theap_default; // access TLS to allocate it before setting tls_initialized to true;- // if (dummy == NULL) return; // use dummy or otherwise the access may get optimized away (issue #697)- // #endif-os_preloading = false;- mi_assert_internal(_mi_is_main_thread());mi_process_init();- mi_process_setup_auto_thread_done();- _mi_thread_locals_init();+_mi_options_post_init(); // now we can print to stderrif (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n");@@ -1018,98 +528,42 @@ void _mi_auto_process_init(void) {}// reseed random- _mi_random_reinit_if_weak(&theap_main.random);-}--// CPU features-mi_decl_cache_align size_t _mi_cpu_movsb_max = 0; // for size <= max, rep movsb is fast-mi_decl_cache_align size_t _mi_cpu_stosb_max = 0; // for size <= max, rep stosb is fast-mi_decl_cache_align bool _mi_cpu_has_popcnt = false;--#if (MI_ARCH_X64 || MI_ARCH_X86)-#if defined(__GNUC__)-// #include <cpuid.h>-static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {- // note: use explicit assembly instead of __get_cpuid as we need the sublevel (in ecx)- // (on Ubuntu 22 with WSL the __get_cpuid does not clear ecx for level 7 which is incorrect).- uint32_t eax, ebx, ecx, edx;- __asm __volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "a"(level), "c"(sublevel) : );- regs4[0] = eax;- regs4[1] = ebx;- regs4[2] = ecx;- regs4[3] = edx;- return true;-}--#elif defined(_MSC_VER)-static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {- __cpuidex((int32_t*)regs4, (int32_t)level, (int32_t)sublevel);- return true;-}-#else-static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {- MI_UNUSED(regs4); MI_UNUSED(level); MI_UNUSED(sublevel);- return false;-}-#endif--static void mi_detect_cpu_features(void) {- // FSRM for fast short rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017))- // EMRS for fast enhanced rep movsb/stosb support (not used at the moment, memcpy always seems faster?)- // FSRS for fast short rep stosb- bool amd = false;- bool fsrm = false;- // bool erms = false;- bool fsrs = false;- uint32_t cpu_info[4];- if (mi_cpuid(cpu_info, 0, 0)) {- amd = (cpu_info[2]==0x444d4163); // (Auth enti cAMD)- }- if (mi_cpuid(cpu_info, 7, 0)) {- fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>- // erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>- }- if (mi_cpuid(cpu_info, 7, 1)) {- fsrs = ((cpu_info[1] & (1 << 11)) != 0); // bit 11 of EBX: see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=1:_Extended_Features>- }- if (mi_cpuid(cpu_info, 1, 0)) {- _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see <https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits>- }-- if (fsrm) {- _mi_cpu_movsb_max = 127;- }- if (fsrs || (amd && fsrm)) { // fsrm on amd implies fsrs, see: https://marc.info/?l=git-commits-head&m=168186277717803- _mi_cpu_stosb_max = 127;+ mi_theap_t* theap = _mi_theap_default();+ if (theap != NULL) {+ _mi_random_reinit_if_weak(&theap->random);+ mi_subproc_t* subproc = _mi_theap_subproc(theap);+ if (subproc->theap_meta != NULL) {+ mi_lock(&subproc->theap_meta_lock) {+ _mi_random_reinit_if_weak(&subproc->theap_meta->random);+ }+ }}}-#else-static void mi_detect_cpu_features(void) {- #if MI_ARCH_ARM64- _mi_cpu_has_popcnt = true;++// Initialize the process; called by thread_init, the process loader, or an initial allocation (perhaps by the loader or a system library)+static void mi_process_init_once(void) {+ #if defined(__CYGWIN__) // we need to kickstart the cygwin runtime+ __mi_thread_id_helper = NULL;#endif-}-#endif+ _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id());+ _mi_detect_cpu_features();+ _mi_options_init(); // read environment (if possible)+ _mi_stats_init(); // start timer+ _mi_os_init(); // primitive dependent-// Initialize the process; called by thread_init or the process loader-static void mi_process_init_once(void) mi_attr_noexcept {- _mi_process_is_initialized = true;- _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id());+ mi_heap_main_init(); // before page_map_init so stats are working+ _mi_page_map_init(); // todo: this could fail.. should we abort in that case?+ mi_thread_init();- mi_detect_cpu_features();- _mi_options_init();- _mi_stats_init();- _mi_os_init();// the following can potentially allocate (on freeBSD for pthread keys)- // todo: do 2-phase so we can use stats at first, then later init the keys?- mi_heap_main_init(); // before page_map_init so stats are working- _mi_page_map_init(); // todo: this could fail.. should we abort in that case?- mi_thread_init();- _mi_process_is_initialized = true;+ _mi_tls_slots_init(); // pthread key create+ _mi_thread_locals_init(); // pthread key create+ _mi_process_is_initialized = true; // before `mi_process_setup_auto_thread_done` so `_mi_theap_default` returns the current theap with `MI_TLS_RECURSE_GUARD`+ mi_process_setup_auto_thread_done(); // after the above mi_thread_init so it can add the current theap- #if defined(_WIN32) && defined(MI_WIN_USE_FLS)+ #if defined(_WIN32) && defined(MI_WIN_INIT_USE_FLS)// On windows, when building as a static lib the FLS cleanup happens to early for the main thread.// To avoid this, set the FLS value for the main thread to NULL so the fls cleanup// will not call _mi_thread_done on the (still executing) main thread. See issue #508.@@ -1137,16 +591,14 @@ static void mi_process_init_once(void) mi_attr_noexcept {// Initialize the process; called by thread_init or the process loadervoid mi_process_init(void) mi_attr_noexcept {- // #if _MSC_VER < 1920- // mi_heap_main_init(); // vs2017 can dynamically re-initialize _mi_heap_main- // #endifmi_atomic_do_once {mi_process_init_once();}}-// Called when the process is done (cdecl as it is used with `at_exit` on some platforms)-void mi_cdecl mi_process_done(void) mi_attr_noexcept {++// Called when the process is done+static void mi_process_done_once(void) {// only shutdown if we were initializedif (!_mi_process_is_initialized) return;// ensure we are called once@@ -1154,8 +606,8 @@ void mi_cdecl mi_process_done(void) mi_attr_noexcept {if (process_done) return;process_done = true;- // free dynamic thread locals (if used at all)- _mi_thread_locals_done();+ // decref any cached theap+ _mi_theap_cached_set(_mi_theap_empty_get());// release any thread specific resources and ensure _mi_thread_done is called on all but the main thread_mi_prim_thread_done_auto_done();@@ -1170,30 +622,47 @@ void mi_cdecl mi_process_done(void) mi_attr_noexcept {#endif// done with tracking tools- mi_track_done()+ mi_track_done();// Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free// since after process_done there might still be other code running that calls `free` (like at_exit routines,// or C-runtime termination code.+ mi_subproc_t* subproc_main = _mi_subproc_main();if (mi_option_is_enabled(mi_option_destroy_on_exit)) {- mi_subprocs_unsafe_destroy_all(); // destroys all subprocs, arenas, and the page_map!+ _mi_subprocs_unsafe_destroy_all(); // destroys all mi_subprocs, arenas, thread locals, and the page_map!}else {- mi_heap_stats_merge_to_subproc(mi_heap_main());- }-- // careful now to no longer access any allocator functionality- if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {- mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL);+ // free dynamic thread locals (if used at all)+ _mi_thread_locals_thread_done();+ _mi_thread_locals_done();+ if (subproc_main->heap_main != NULL) {+ if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {+ mi_theap_collect(subproc_main->theap_meta, false /* force */); // update stats of all pages in theap_meta+ _mi_theap_merge_stats(subproc_main->theap_meta);+ _mi_theap_merge_stats(_mi_theap_default()); // _mi_thread_locals_done can free+ mi_heap_stats_merge_to_subproc(subproc_main->heap_main);+ mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); // note: can try to access (the now freed) thread_locals in mi_heap_theap_peek+ }+ }}- mi_lock_done(&subprocs_lock);- mi_tls_slots_done();++ _mi_tls_slots_done();+ _mi_subproc_main_done();_mi_allocator_done();- _mi_verbose_message("process done: 0x%zx\n", tld_main.thread_id);+ _mi_verbose_message("process done %zu\n", sizeof(mi_page_t)); // : 0x%zx\n", mi_process_tld_main.thread_id);os_preloading = true; // don't call the C runtime anymore}++// Call when the process is done (cdecl as it is used with `at_exit` on some platforms)+void mi_cdecl mi_process_done(void) mi_attr_noexcept {+ mi_atomic_do_once {+ mi_process_done_once();+ }+}++// Called automatically when the process is done (cdecl as it is used with `at_exit` on some platforms)void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept {- if (_mi_option_get_fast(mi_option_destroy_on_exit)>1) return;+ if (_mi_option_get_fast(mi_option_destroy_on_exit)>=2) return; // allow disabling auto process donemi_process_done();}diff --git a/vendored/mimalloc/src/libc.c b/vendored/mimalloc/src/libc.cindex c74003a14..8f211230d 100644--- a/vendored/mimalloc/src/libc.c+++ b/vendored/mimalloc/src/libc.c@@ -1,12 +1,12 @@/* -----------------------------------------------------------------------------Copyright (c) 2018-2024, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.-----------------------------------------------------------------------------*/// ---------------------------------------------------------// This module defines various std libc functions to reduce+// This module defines various standard libc functions to reduce// the dependency on libc, and also prevent errors caused// by some libc implementations when called before `main`// executes (due to malloc redirection)@@ -39,8 +39,8 @@ bool _mi_streq(const char* s, const char* t) {return (*s == *t);}-void _mi_strlcpy(char* dest, const char* src, size_t dest_size) {- if (dest==NULL || src==NULL || dest_size == 0) return;+bool _mi_strlcpy(char* dest, const char* src, size_t dest_size) {+ if (dest==NULL || src==NULL || dest_size == 0) return (src==NULL || *src==0);// copy until end of src, or when dest is (almost) fullwhile (*src != 0 && dest_size > 1) {*dest++ = *src++;@@ -48,23 +48,24 @@ void _mi_strlcpy(char* dest, const char* src, size_t dest_size) {}// always zero terminate*dest = 0;+ return (*src == 0);}-void _mi_strlcat(char* dest, const char* src, size_t dest_size) {- if (dest==NULL || src==NULL || dest_size == 0) return;+bool _mi_strlcat(char* dest, const char* src, size_t dest_size) {+ if (dest==NULL || src==NULL || dest_size == 0) return (src==NULL || *src==0);// find end of string in the dest bufferwhile (*dest != 0 && dest_size > 1) {dest++;dest_size--;}// and catenate- _mi_strlcpy(dest, src, dest_size);+ return _mi_strlcpy(dest, src, dest_size);}size_t _mi_strnlen(const char* s, size_t max_len) {if (s==NULL) return 0;size_t len = 0;- while(s[len] != 0 && len < max_len) { len++; }+ while(len < max_len && s[len] != 0) { len++; }return len;}@@ -76,7 +77,7 @@ char* _mi_strnstr(char* s, size_t max_len, const char* pat) {if (s==NULL) return NULL;if (pat==NULL) return s;const size_t m = _mi_strnlen(s, max_len);- const size_t n = _mi_strlen(pat);+ const size_t n = _mi_strlen(pat);for (size_t start = 0; start + n <= m; start++) {size_t i = 0;while (i<n && pat[i]==s[start+i]) {@@ -88,16 +89,18 @@ char* _mi_strnstr(char* s, size_t max_len, const char* pat) {}#ifdef MI_NO_GETENV-bool _mi_getenv(const char* name, char* result, size_t result_size) {+int _mi_getenv(const char* name, char* result, size_t result_size) {MI_UNUSED(name);MI_UNUSED(result);MI_UNUSED(result_size);- return false;+ return ENOENT;}#else-bool _mi_getenv(const char* name, char* result, size_t result_size) {- if (name==NULL || result == NULL || result_size < 64) return false;- return _mi_prim_getenv(name,result,result_size);+int _mi_getenv(const char* name, char* result, size_t result_size) {+ if (name==NULL || result == NULL || result_size < 64) return ENOENT;+ // change the result of _mi_prim_getenv to an errno result+ const int res = _mi_prim_getenv(name,result,result_size);+ return (res > 0 ? 0 : (res == 0 ? ENOENT : EAGAIN));}#endif@@ -116,14 +119,14 @@ bool _mi_atomic_once_enter(mi_atomic_once_t* once) {}const mi_threadid_t current_tid = _mi_thread_id();if (once_tid == current_tid) {- return false; // recursive invocation; we need this for process_init for example- }+ return false; // recursive invocation; don't block on ourselves+ }mi_lock_acquire(&once->lock);uintptr_t expected = 0;if (mi_atomic_cas_strong_acq_rel(&once->tid, &expected, current_tid)) { // could use atomic_load/store as wellreturn true; // should execute and release- }+ }else {mi_lock_release(&once->lock);return false; // already another thread entered and released@@ -134,8 +137,72 @@ void _mi_atomic_once_release(mi_atomic_once_t* once) {if (mi_atomic_load_acquire(&once->tid)>1) { // paranoiami_atomic_store_release(&once->tid,1); // done executingmi_lock_release(&once->lock);- }+ }+}++#if MI_USE_PTHREADS+mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruct)(void*), void* init) {+ int err = pthread_key_create(pkey,destruct);+ if mi_unlikely(err!=0) {+ *pkey = MI_PTHREAD_KEY_INVALID;+ _mi_error_message(ENOMEM,"unable to allocate a thread local variable (error %d)\n", err);+ return false;+ }+ mi_assert_internal(*pkey != MI_PTHREAD_KEY_INVALID);+ if (init!=NULL) {+ pthread_setspecific(*pkey,init);+ };+ mi_assert_internal(pthread_getspecific(*pkey)==init);+ return true;}+#endif++// --------------------------------------------------------+// Detect CPU features+// --------------------------------------------------------+mi_decl_cache_align bool _mi_cpu_has_popcnt = false;++#if (MI_ARCH_X64 || MI_ARCH_X86)+#if defined(__GNUC__)+// #include <cpuid.h>+static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {+ // note: use explicit assembly instead of __get_cpuid as we need the sublevel (in ecx)+ // (on Ubuntu 22 with WSL the __get_cpuid does not clear ecx for level 7 which is incorrect).+ uint32_t eax, ebx, ecx, edx;+ __asm __volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "a"(level), "c"(sublevel) : );+ regs4[0] = eax;+ regs4[1] = ebx;+ regs4[2] = ecx;+ regs4[3] = edx;+ return true;+}++#elif defined(_MSC_VER)+static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {+ __cpuidex((int32_t*)regs4, (int32_t)level, (int32_t)sublevel);+ return true;+}+#else+static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {+ MI_UNUSED(regs4); MI_UNUSED(level); MI_UNUSED(sublevel);+ return false;+}+#endif++void _mi_detect_cpu_features(void) {+ uint32_t cpu_info[4];+ if (mi_cpuid(cpu_info, 1, 0)) {+ _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see <https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits>+ }+}++#else+void _mi_detect_cpu_features(void) {+ #if MI_ARCH_ARM64 || defined(__riscv_zbb) || defined(__riscv_b)+ _mi_cpu_has_popcnt = true;+ #endif+}+#endif// --------------------------------------------------------@@ -196,22 +263,20 @@ static void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char*mi_outc('0',out,end);}else {- // output digits in reverse- char* start = *out;- while (x > 0) {+ #define MI_MAX_OUT_DIGITS (160) /* a 512 bit number has 155 digits */+ char num[MI_MAX_OUT_DIGITS];+ int dcount = 0;+ while(x>0 && dcount < MI_MAX_OUT_DIGITS) {char digit = (char)(x % base);- mi_outc((digit <= 9 ? '0' + digit : 'A' + digit - 10),out,end);+ num[dcount++] = (digit <= 9 ? '0' + digit : 'A' + digit - 10);x = x / base;}+ if (dcount>=MI_MAX_OUT_DIGITS) return; // don't output anything?if (prefix != 0) {mi_outc(prefix, out, end);}- size_t len = *out - start;- // and reverse in-place- for (size_t i = 0; i < (len / 2); i++) {- char c = start[len - i - 1];- start[len - i - 1] = start[i];- start[i] = c;+ while(dcount-- > 0) {+ mi_outc(num[dcount], out, end);}}}@@ -258,7 +323,10 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {if (c >= '1' && c <= '9') {width = (c - '0'); MI_NEXTC();while (c >= '0' && c <= '9') {- width = (10 * width) + (c - '0'); MI_NEXTC();+ if (width < SIZE_MAX/1024) { // no overflow+ width = (10 * width) + (c - '0');+ }+ MI_NEXTC();}if (c == 0) break; // extra check due to while}@@ -297,7 +365,7 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {if (width == 0 && (c == 'x' || c == 'p')) {if (c == 'p') { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(void*))); }if (width == 0) { width = 2; }- fill = '0';+ if (alignright) { fill = '0'; }}mi_out_num(x, (c == 'x' || c == 'p' ? 16 : 10), numplus, &out, end);}@@ -350,6 +418,7 @@ int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...) {return written;}+#undef MI_NEXTC// --------------------------------------------------------@@ -365,7 +434,7 @@ static size_t mi_ctz_generic32(uint32_t x) {31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};if (x==0) return 32;- return debruijn[(uint32_t)((x & -(int32_t)x) * (uint32_t)(0x077CB531U)) >> 27];+ return debruijn[(uint32_t)((x & (~x + 1U)) * (uint32_t)(0x077CB531U)) >> 27];}static size_t mi_clz_generic32(uint32_t x) {diff --git a/vendored/mimalloc/src/options.c b/vendored/mimalloc/src/options.cindex c2d8fa914..eaa2119de 100644--- a/vendored/mimalloc/src/options.c+++ b/vendored/mimalloc/src/options.c@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -17,7 +17,7 @@ static long mi_max_warning_count = 16; // stop outputting warnings after this (ustatic void mi_add_stderr_output(void);-int mi_version(void) mi_attr_noexcept {+int mi_version(void) {return MI_MALLOC_VERSION;}@@ -49,7 +49,7 @@ int mi_version(void) mi_attr_noexcept {// in KiB#ifndef MI_DEFAULT_ARENA_RESERVE- #if (MI_INTPTR_SIZE>4)+ #if (MI_SIZE_SIZE>4)#define MI_DEFAULT_ARENA_RESERVE 1024L*1024L#else#define MI_DEFAULT_ARENA_RESERVE 128L*1024L@@ -77,15 +77,15 @@ int mi_version(void) mi_attr_noexcept {#endif#ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE-#if MI_GUARDED && !MI_DEBUG-#define MI_DEFAULT_GUARDED_SAMPLE_RATE 4000+#if MI_GUARDED && MI_DEBUG+#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 /* MI_MiB */#else-#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0+#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0#endif#endif#ifndef MI_DEFAULT_PAGEMAP_COMMIT-#if defined(__APPLE__) // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access+#if defined(__APPLE__) && MI_PAGE_MAP_FLAT // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access#define MI_DEFAULT_PAGEMAP_COMMIT 1#else#define MI_DEFAULT_PAGEMAP_COMMIT 0@@ -104,10 +104,15 @@ int mi_version(void) mi_attr_noexcept {#if defined(__ANDROID__)#define MI_DEFAULT_ALLOW_THP 0#else-#define MI_DEFAULT_ALLOW_THP 1+// #define MI_DEFAULT_ALLOW_THP 1 // allow THP but purging may split up THP pages+#define MI_DEFAULT_ALLOW_THP 2 // allow THP and set the minimal purge size to 2MiB to avoid breaking them up#endif#endif+#ifndef MI_DEFAULT_COLLECT_MERGES_STATS+#define MI_DEFAULT_COLLECT_MERGES_STATS 1+#endif+// Static optionsstatic mi_option_desc_t mi_options[_mi_option_last] ={@@ -121,7 +126,7 @@ static mi_option_desc_t mi_options[_mi_option_last] ={ MI_DEFAULT_VERBOSE, MI_OPTION_UNINIT, MI_OPTION(verbose) },// some of the following options are experimental and not all combinations are allowed.- { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit) },+ { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit) },{ MI_DEFAULT_ARENA_EAGER_COMMIT,MI_OPTION_UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux){ 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit)@@ -134,9 +139,9 @@ static mi_option_desc_t mi_options[_mi_option_last] =MI_OPTION_UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`){ 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread{ 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free- { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_abandoned_page_purge) },+ { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_abandoned_page_purge) },{ 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_reset) }, // reset segment memory on free (needs eager commit)- { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit_delay) },+ { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit_delay) },{ 1000,MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) }, // purge delay in milli-seconds{ 0, MI_OPTION_UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes.{ 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas)@@ -146,15 +151,11 @@ static mi_option_desc_t mi_options[_mi_option_last] ={ 10, MI_OPTION_UNINIT, MI_OPTION(deprecated_max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try.{ 0, MI_OPTION_UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees!{ MI_DEFAULT_ARENA_RESERVE, MI_OPTION_UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`)- { 1, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's+ { 4, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's{ 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(deprecated_purge_extend_delay, decommit_extend_delay) },{ MI_DEFAULT_DISALLOW_ARENA_ALLOC, MI_OPTION_UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's){ 400, MI_OPTION_UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries.-#if defined(MI_VISIT_ABANDONED)- { 1, MI_OPTION_INITIALIZED, MI_OPTION(visit_abandoned) }, // allow visiting theap blocks in abandoned segments; requires taking locks during reclaim.-#else- { 0, MI_OPTION_UNINIT, MI_OPTION(visit_abandoned) },-#endif+ { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_visit_abandoned) },{ 0, MI_OPTION_UNINIT, MI_OPTION(guarded_min) }, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects{ MI_GiB, MI_OPTION_UNINIT, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects{ 0, MI_OPTION_UNINIT, MI_OPTION(guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)@@ -175,16 +176,18 @@ static mi_option_desc_t mi_options[_mi_option_last] =MI_OPTION_UNINIT, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class{ MI_DEFAULT_ALLOW_THP,MI_OPTION_UNINIT, MI_OPTION(allow_thp) }, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.- { 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if `mi_option_allow_thp==2`).- { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE,- MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).+ { 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if THP is enabled).+ { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE,+ MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).{ 0, MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) }, // associate local numa node with an initial arena allocation+ { MI_DEFAULT_COLLECT_MERGES_STATS,+ MI_OPTION_UNINIT, MI_OPTION(collect_merges_stats) }, // on each theap collect, stats are merged with the parent heap};static void mi_option_init(mi_option_desc_t* desc);static bool mi_option_has_size_in_kib(mi_option_t option) {- return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve ||+ return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve ||option == mi_option_minimal_purge_size || option == mi_option_arena_max_object_size);}@@ -203,7 +206,7 @@ void _mi_options_init(void) {_mi_warning_message("option 'allow_large_os_pages' is disabled to allow for guarded objects\n");}}- #endif+ #endif}// called at actual process load, it should be safe to print now@@ -221,15 +224,6 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_const int vermajor = MI_MALLOC_VERSION/10000;const int verminor = (MI_MALLOC_VERSION%10000)/100;const int verpatch = (MI_MALLOC_VERSION%100);- /* CBM LOCAL PATCH (keep across mimalloc refreshes): upstream appends- * "(built on __DATE__, __TIME__)" here. mimalloc is linked into every cbm- * binary, so those two macros made every build byte-unique — two compilations- * of identical source seconds apart could never produce the same hash. That- * defeats reproducible builds, and it also means a released artifact can never- * inherit an antivirus false-positive determination made about its- * predecessor, because every rebuild is a brand-new file to a reputation- * system. The banner keeps the version; the wall clock is not diagnostics.- * -Wdate-time in CFLAGS_COMMON makes any new use a build failure. */_mi_fprintf(out, arg, "v%i.%i.%i%s%s\n", vermajor, verminor, verpatch,#if defined(MI_CMAKE_BUILD_TYPE)", " mi_stringify(MI_CMAKE_BUILD_TYPE)@@ -252,16 +246,34 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr__mi_fprintf(out, arg, "option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : ""));}- // show build configuration- _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG );- _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE );- _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL);+ // show build configuration+ #if MI_PAGE_META_IS_ALIGNED && MI_PAGE_META_SMALL_IS_ALIGNED+ _mi_fprintf(out, arg, "free mode : (small) aligned, page size: %zu\n", sizeof(mi_page_t));+ #elif MI_PAGE_META_IS_ALIGNED+ _mi_fprintf(out, arg, "free mode : aligned, page size: %zu\n", sizeof(mi_page_t));+ #elif MI_PAGE_META_SMALL_IS_ALIGNED+ _mi_fprintf(out, arg, "free mode : small aligned + pagemap, page size: %zu\n", sizeof(mi_page_t));+ #elif MI_FREE_IS_CHECKED+ _mi_fprintf(out, arg, "free mode : checked, page size: %zu\n", sizeof(mi_page_t));+ #else+ _mi_fprintf(out, arg, "free mode : pagemap, page size: %zu\n", sizeof(mi_page_t));+ #endif+ #if MI_ENCODE_FREELIST+ _mi_fprintf(out, arg, "free lists : encoded with %d key(s)\n", MI_PAGE_KEY_COUNT);+ #endif#if MI_GUARDED- _mi_fprintf(out, arg, "guarded build: %s\n", mi_option_get(mi_option_guarded_sample_rate) != 0 ? "enabled" : "disabled");+ _mi_fprintf(out, arg, "guarded mode: %s, rate=%ld\n", MI_SAMPLE==2 ? "fine-grained" : "enabled", mi_option_get(mi_option_guarded_sample_rate));+ #endif+ #if MI_PROFILE+ _mi_fprintf(out, arg, "profiling : %s\n", MI_SAMPLE==2 ? "fine-grained" : "enabled");#endif#if MI_TSAN_mi_fprintf(out, arg, "thread santizer enabled\n");#endif+ _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL);+ _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG );+ _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE );+ _mi_fprintf(out, arg, "padding : %s\n", (MI_PADDING==0 ? "none" : (MI_PADDING==1 ? "enabled" : "byte precise")));}mi_decl_export void mi_options_print(void) mi_attr_noexcept {@@ -446,25 +458,12 @@ static void mi_add_stderr_output(void) {static _Atomic(size_t) error_count; // = 0; // when >= max_error_count stop emitting errorsstatic _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings-// When overriding malloc, we may recurse into mi_vfprintf if an allocation-// inside the C runtime causes another message.-// In some cases (like on macOS) the loader already allocates which-// calls into mimalloc; if we then access thread locals (like `recurse`)-// this may crash as the access may call _tlv_bootstrap that tries to-// (recursively) invoke malloc again to allocate space for the thread local-// variables on demand. This is why we use a _mi_preloading test on such-// platforms. However, C code generator may move the initial thread local address-// load before the `if` and we therefore split it out in a separate function.-static mi_decl_thread bool recurse = false;-static mi_decl_noinline bool mi_recurse_enter_prim(void) {- if (recurse) return false;- recurse = true;return true;}static mi_decl_noinline void mi_recurse_exit_prim(void) {- recurse = false;+ /* nothing */}static bool mi_recurse_enter(void) {@@ -482,7 +481,7 @@ static void mi_recurse_exit(void) {}void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) {- if (out==NULL || (void*)out==(void*)stdout || (void*)out==(void*)stderr) { // TODO: use mi_out_stderr for stderr?+ if (out==NULL || (void*)out==(void*)stdout || (void*)out==(void*)stderr) { // todo: use mi_out_stderr for stderr?if (!mi_recurse_enter()) return;out = mi_out_get_default(&arg);if (prefix != NULL) out(prefix, arg);@@ -514,7 +513,7 @@ void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) {}static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) {- if (prefix != NULL && _mi_strnlen(prefix,33) <= 32 && !_mi_is_main_thread()) {+ if (prefix != NULL && _mi_strnlen(prefix,33) <= 32) { // && !_mi_is_main_thread()) {char tprefix[64];_mi_snprintf(tprefix, sizeof(tprefix), "%sthread 0x%tx: ", prefix, (uintptr_t)_mi_thread_id());mi_vfprintf(out, arg, tprefix, fmt, args);@@ -531,13 +530,6 @@ void _mi_raw_message(const char* fmt, ...) {va_end(args);}-void _mi_message(const char* fmt, ...) {- va_list args;- va_start(args, fmt);- mi_vfprintf_thread(NULL, NULL, "mimalloc: ", fmt, args);- va_end(args);-}-void _mi_trace_message(const char* fmt, ...) {if (mi_option_get(mi_option_verbose) <= 1) return; // only with verbose level 2 or higherva_list args;@@ -590,24 +582,27 @@ static _Atomic(void*) mi_error_arg; // = NULLstatic void mi_error_default(int err) {MI_UNUSED(err);-#if (MI_DEBUG>0)- if (err==EFAULT) {- #ifdef _MSC_VER- __debugbreak();- #endif- abort();- }-#endif-#if (MI_SECURE>0)- if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data)- abort();- }-#endif-#if defined(MI_XMALLOC)- if (err==ENOMEM || err==EOVERFLOW) { // abort on memory allocation fails in xmalloc mode- abort();+ #if (MI_DEBUG>0)+ if (err==EFAULT) {+ #ifdef _MSC_VER+ __debugbreak();+ #endif+ abort();+ }+ #endif+ #if (MI_SECURE>0)+ if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data)+ abort();+ }+ #endif+ #if defined(MI_XMALLOC)+ if (err==ENOMEM || err==EOVERFLOW || err==EINVAL) { // abort on memory allocation fails in xmalloc mode+ abort();+ }+ #endif+ if (errno==0) {+ errno = (err==EINVAL ? EINVAL : ENOMEM /* compatibility */ );}-#endif}void mi_register_error(mi_error_fun* fun, void* arg) {@@ -621,7 +616,7 @@ void _mi_error_message(int err, const char* fmt, ...) {va_start(args, fmt);mi_show_error_message(fmt, args);va_end(args);- // and call the error handler which may abort (or return normally)+ // and call the error handler which may abort (or return normally, potentially setting errno)if (mi_error_handler != NULL) {mi_error_handler(err, mi_atomic_load_ptr_acquire(void,&mi_error_arg));}@@ -630,14 +625,17 @@ void _mi_error_message(int err, const char* fmt, ...) {}}+mi_decl_noinline mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next) {+ _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next);+ return NULL;+}+// --------------------------------------------------------// Initialize options by checking the environment// --------------------------------------------------------// TODO: implement ourselves to reduce dependencies on the C runtime#include <stdlib.h> // strtol-#include <string.h> // strstr-static void mi_option_init(mi_option_desc_t* desc) {// Read option value from the environment@@ -645,17 +643,17 @@ static void mi_option_init(mi_option_desc_t* desc) {char buf[64+1];_mi_strlcpy(buf, "mimalloc_", sizeof(buf));_mi_strlcat(buf, desc->name, sizeof(buf));- bool found = _mi_getenv(buf, s, sizeof(s));- if (!found && desc->legacy_name != NULL) {+ int err = _mi_getenv(buf, s, sizeof(s));+ if (err==ENOENT && desc->legacy_name != NULL) {_mi_strlcpy(buf, "mimalloc_", sizeof(buf));_mi_strlcat(buf, desc->legacy_name, sizeof(buf));- found = _mi_getenv(buf, s, sizeof(s));- if (found) {+ err = _mi_getenv(buf, s, sizeof(s));+ if (err==0) {_mi_warning_message("environment option \"mimalloc_%s\" is deprecated -- use \"mimalloc_%s\" instead.\n", desc->legacy_name, desc->name);}}- if (found) {+ if (err==0) {size_t len = _mi_strnlen(s, sizeof(buf) - 1);for (size_t i = 0; i < len; i++) {buf[i] = _mi_toupper(s[i]);@@ -665,14 +663,15 @@ static void mi_option_init(mi_option_desc_t* desc) {desc->value = 1;desc->init = MI_OPTION_INITIALIZED;}- else if (_mi_streq(buf,"0") || _mi_streq(buf,"FALSE") || _mi_streq(buf,"NO") || _mi_streq(buf,"OFF")) {+ else if (_mi_streq(buf,"0") || _mi_streq(buf,"FALSE") || _mi_streq(buf,"NO") || _mi_streq(buf,"OFF")) {desc->value = 0;desc->init = MI_OPTION_INITIALIZED;}else {char* end = buf;+ errno = 0;long value = strtol(buf, &end, 10);- if (mi_option_has_size_in_kib(desc->option)) {+ if (errno==0 && mi_option_has_size_in_kib(desc->option)) {// this option is interpreted in KiB to prevent overflow of `long` for large allocations// (long is 32-bit on 64-bit windows, which allows for 4TiB max.)size_t size = (value < 0 ? 0 : (size_t)value);@@ -687,7 +686,7 @@ static void mi_option_init(mi_option_desc_t* desc) {if (overflow || size > (MI_MAX_ALLOC_SIZE / MI_KiB)) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); }value = (size > LONG_MAX ? LONG_MAX : (long)size);}- if (*end == 0) {+ if (errno==0 && *end == 0) {mi_option_set(desc->option, value);}else {@@ -707,7 +706,8 @@ static void mi_option_init(mi_option_desc_t* desc) {}mi_assert_internal(desc->init != MI_OPTION_UNINIT);}- else if (!_mi_preloading()) {+ else if (err==ENOENT) {desc->init = MI_OPTION_DEFAULTED;}+ // and on another error, keep unitialized to try again (can happen during preloading if getenv is not available)}diff --git a/vendored/mimalloc/src/os.c b/vendored/mimalloc/src/os.cindex febecb0c9..15111cf94 100644--- a/vendored/mimalloc/src/os.c+++ b/vendored/mimalloc/src/os.c@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -8,12 +8,13 @@ terms of the MIT license. A copy of the license can be found in the file#include "mimalloc/internal.h"#include "mimalloc/atomic.h"#include "mimalloc/prim.h"+#include "mimalloc/prim-tls.h" // _mi_theap_default for random/* -----------------------------------------------------------Initialization.----------------------------------------------------------- */#ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB-#if MI_INTPTR_SIZE < 8+#if MI_SIZE_SIZE < 8#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 4*MI_MiB // 4 GiB#else#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 32*MI_MiB // 32 GiB@@ -40,6 +41,9 @@ bool _mi_os_has_virtual_reserve(void) {return mi_os_mem_config.has_virtual_reserve;}+bool _mi_os_canuse_thp(void) {+ return mi_os_mem_config.has_transparent_huge_pages;+}// OS (small) page sizesize_t _mi_os_page_size(void) {@@ -54,13 +58,19 @@ size_t _mi_os_large_page_size(void) {// minimal purge size. Can be larger than the page size if transparent huge pages are enabled.size_t _mi_os_minimal_purge_size(void) {size_t minsize = mi_option_get_size(mi_option_minimal_purge_size);- if (minsize != 0) {+ if (minsize != 0) {+ // set by userreturn _mi_align_up(minsize, _mi_os_page_size());}else if (mi_os_mem_config.has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) {+ // don't break up THP pages;+ // we don't do this by default as that can lead to increased memory usage (see issue #1282).+ // on the other hand, breaking up THP pages can lead to performance degradation and perhaps+ // allow_thp should be 2 by default (and the user can set the OS THP setting to [never]).return _mi_os_large_page_size();}else {+ // OS page size by defaultreturn _mi_os_page_size();}}@@ -103,53 +113,57 @@ void _mi_os_init(void) {/* -----------------------------------------------------------Util-------------------------------------------------------------- */-bool _mi_os_decommit(void* addr, size_t size);-bool _mi_os_commit(void* addr, size_t size, bool* is_zero);+bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size);+bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero);// On systems with enough virtual address bits, we can do efficient aligned allocation by using// the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address// space (64TiB) we use this technique. (but see issue #939)-#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID)+#if (MI_SIZE_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID)-// Return a MI_HINT_ALIGN (4MiB) aligned address that is probably available.+// Return a `try_alignment` aligned address that is probably available.// If this returns NULL, the OS will determine the address but on some OS's that may not be// properly aligned which can be more costly as it needs to be adjusted afterwards.-// For a size > 16GiB this always returns NULL in order to guarantee good ASLR randomization;+// In secure mode, for a size > 16GiB this always returns NULL in order to guarantee good ASLR randomization;// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses// in the middle of the 2TiB - 6TiB address range (see issue #372))-#define MI_HINT_ALIGN ((uintptr_t)4 << 20) // 4MiB alignment#define MI_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start#define MI_HINT_AREA ((uintptr_t)4 << 40) // upto (2+4) 6TiB (since before win8 there is "only" 8TiB available to processes)#define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages)-void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)+void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze){static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // = 0// todo: perhaps only do alignment hints if THP is enabled?- if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > MI_HINT_ALIGN) return NULL;+ if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > 16*MI_GiB) return NULL;if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space- size = _mi_align_up(size, MI_HINT_ALIGN);- if (size > 16*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at least 1/(MI_HINT_AREA / 1<<34)- size += MI_HINT_ALIGN; // put in virtual gaps between hinted blocks; this splits VLA's but increases guarded areas.-- uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size);++ size_t req_size = sze + _mi_os_page_size(); // always reserve a bit more to create virtual gaps between hinted blocks.+ req_size += (try_alignment - 1); // ensure we can align in the requested size+ req_size = _mi_align_up(req_size, _mi_os_large_page_size());+ #if (MI_SECURE>=1)+ if (req_size > 32*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<34) = 1/256+ #endif++ uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, req_size);if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initializeuintptr_t init = MI_HINT_BASE;- #if (MI_SECURE>=1 || defined(NDEBUG)) // security: randomize start of aligned allocations unless in debug mode+ #if (MI_SECURE>=1 || !MI_DEBUG) // security: randomize start of aligned allocations unless in debug modemi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this pointconst uintptr_t r = _mi_theap_random_next(theap);- init = init + ((MI_HINT_ALIGN * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB+ init = init + ((MI_MiB * ((r>>17) & 0x3FFFFF)) % MI_HINT_AREA); // (randomly 22 bits)* 1MiB == 0 to 4TiB#endif- uintptr_t expected = hint + size;+ uintptr_t expected = hint + req_size;mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);- hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all+ hint = mi_atomic_add_acq_rel(&aligned_base, req_size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all+ if (hint==0) return NULL;}- mi_assert_internal(hint%MI_HINT_ALIGN == 0);- if (hint%try_alignment != 0) return NULL;- return (void*)hint;+ const uintptr_t hint_align = _mi_align_up(hint,try_alignment);+ mi_assert_internal(hint_align + sze < hint + req_size);+ return (void*)hint_align;}#elsevoid* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) {@@ -173,7 +187,7 @@ size_t _mi_os_secure_guard_page_size(void) {}// In secure mode, try to decommit an area and output a warning if this fails.-bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid) {+bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {if (addr == NULL) return true;#if MI_SECURE > 0bool ok = false;@@ -183,7 +197,7 @@ bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid) {ok = (*(arena->commit_fun))(false /* decommit */, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg);}else {- ok = _mi_os_decommit(addr, _mi_os_secure_guard_page_size());+ ok = _mi_os_decommit(subproc, addr, _mi_os_secure_guard_page_size());}}if (!ok) {@@ -191,18 +205,18 @@ bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid) {}return ok;#else- MI_UNUSED(memid);+ MI_UNUSED(subproc); MI_UNUSED(memid);return true;#endif}// In secure mode, try to decommit an area and output a warning if this fails.-bool _mi_os_secure_guard_page_set_before(void* addr, mi_memid_t memid) {- return _mi_os_secure_guard_page_set_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);+bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {+ return _mi_os_secure_guard_page_set_at(subproc, (uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);}// In secure mode, try to recommit an area-bool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid) {+bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {if (addr == NULL) return true;#if MI_SECURE > 0if (!memid.is_pinned) {@@ -211,18 +225,18 @@ bool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid) {return (*(arena->commit_fun))(true, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg);}else {- return _mi_os_commit(addr, _mi_os_secure_guard_page_size(), NULL);+ return _mi_os_commit(subproc, addr, _mi_os_secure_guard_page_size(), NULL);}}#else- MI_UNUSED(memid);+ MI_UNUSED(subproc); MI_UNUSED(memid);#endifreturn true;}// In secure mode, try to recommit an area-bool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid) {- return _mi_os_secure_guard_page_reset_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);+bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {+ return _mi_os_secure_guard_page_reset_at(subproc, (uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);}@@ -230,23 +244,27 @@ bool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid) {Free memory-------------------------------------------------------------- */-static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc);+static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size);-static void mi_os_prim_free(void* addr, size_t size, size_t commit_size, mi_subproc_t* subproc) {+static void mi_os_prim_free(mi_subproc_t* subproc, void* addr, size_t size, size_t commit_size, bool adjust) {+ mi_assert_internal(subproc!=NULL);mi_assert_internal((size % _mi_os_page_size()) == 0);if (addr == NULL) return; // || _mi_os_is_huge_reserved(addr)int err = _mi_prim_free(addr, size); // allow size==0 (issue #1041)if (err != 0) {_mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);}- if (subproc == NULL) { subproc = _mi_subproc(); } // from `mi_arenas_unsafe_destroy` we pass subproc_main explicitly as we can no longer use the theap pointer- if (commit_size > 0) {- mi_subproc_stat_decrease(subproc, committed, commit_size);+ if (adjust) {+ if (commit_size>0) { mi_subproc_stat_adjust_decrease(subproc, committed, commit_size); }+ mi_subproc_stat_adjust_decrease(subproc, reserved, size);+ }+ else {+ if (commit_size>0) { mi_subproc_stat_decrease(subproc, committed, commit_size); }+ mi_subproc_stat_decrease(subproc, reserved, size);}- mi_subproc_stat_decrease(subproc, reserved, size);}-void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc /* can be NULL */) {+void _mi_os_free_ex(mi_subproc_t* subproc, void* addr, size_t size, bool still_committed, mi_memid_t memid) {if (mi_memkind_is_os(memid.memkind)) {size_t csize = memid.mem.os.size;if (csize==0) { csize = _mi_os_good_alloc_size(size); }@@ -268,10 +286,10 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me// free itif (memid.memkind == MI_MEM_OS_HUGE) {mi_assert(memid.is_pinned);- mi_os_free_huge_os_pages(base, csize, subproc);+ mi_os_free_huge_os_pages(subproc, base, csize);}else {- mi_os_prim_free(base, csize, (still_committed ? commit_size : 0), subproc);+ mi_os_prim_free(subproc, base, csize, (still_committed ? commit_size : 0), false);}}else {@@ -280,8 +298,8 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me}}-void _mi_os_free(void* p, size_t size, mi_memid_t memid) {- _mi_os_free_ex(p, size, true, memid, NULL);+void _mi_os_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) {+ _mi_os_free_ex(subproc, p, size, true, memid);}@@ -291,7 +309,7 @@ void _mi_os_free(void* p, size_t size, mi_memid_t memid) {// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.// Also `hint_addr` is a hint and may be ignored.-static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {+static void* mi_os_prim_alloc_at(mi_subproc_t* subproc, void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);mi_assert_internal(is_zero != NULL);mi_assert_internal(is_large != NULL);@@ -312,11 +330,11 @@ static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignm_mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large);}- mi_os_stat_counter_increase(mmap_calls, 1);+ mi_subproc_stat_counter_increase(subproc, mmap_calls, 1);if (p != NULL) {- mi_os_stat_increase(reserved, size);+ mi_subproc_stat_increase(subproc, reserved, size);if (commit) {- mi_os_stat_increase(committed, size);+ mi_subproc_stat_increase(subproc, committed, size);// seems needed for asan (or `mimalloc-test-api` fails)#ifdef MI_TRACK_ASANif (*is_zero) { mi_track_mem_defined(p,size); }@@ -327,14 +345,14 @@ static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignmreturn p;}-static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {- return mi_os_prim_alloc_at(NULL, size, try_alignment, commit, allow_large, is_large, is_zero);+static void* mi_os_prim_alloc(mi_subproc_t* subproc, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {+ return mi_os_prim_alloc_at(subproc, NULL /* hint addr */, size, try_alignment, commit, allow_large, is_large, is_zero);}// Primitive aligned allocation from the OS.// This function guarantees the allocated memory is aligned.-static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) {+static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) {mi_assert_internal(memid!=NULL);mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);@@ -343,8 +361,12 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commitif (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;size = _mi_align_up(size, _mi_os_page_size());+ #if MI_SIZE_SIZE >= 8+ const bool try_direct_alloc = true;+ #else// try a direct allocation if the alignment is below the default, or less than or equal to 1/4 fraction of the size.const bool try_direct_alloc = (alignment <= mi_os_mem_config.alloc_granularity || alignment <= size/4);+ #endifbool os_is_large = false;bool os_is_zero = false;@@ -352,7 +374,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commitsize_t os_size = size;void* p = NULL;if (try_direct_alloc) {- p = mi_os_prim_alloc(size, alignment, commit, allow_large, &os_is_large, &os_is_zero);+ p = mi_os_prim_alloc(subproc, size, alignment, commit, allow_large, &os_is_large, &os_is_zero);}// aligned already?@@ -366,33 +388,34 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit_mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);}#endif- if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0), NULL); }+ if (p != NULL) { mi_os_prim_free(subproc, p, size, (commit ? size : 0), true /* adjust so we "forget" the previous reservation */); }if (size >= (SIZE_MAX - alignment)) return NULL; // overflowconst size_t over_size = size + alignment;if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block// over-allocate uncommitted (virtual) memory- p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero);+ p = mi_os_prim_alloc(subproc, over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero);if (p == NULL) return NULL;// set p to the aligned part in the full region// note: Windows VirtualFree needs the actual base pointer// this is handled though by having the `base` field in the memidos_base = p; // remember the base- os_size = over_size;+ os_size = over_size; // todo: use size instead as now we over-decrement commit stats on free?p = _mi_align_up_ptr(p, alignment);// explicitly commit only the aligned partif (commit) {- if (!_mi_os_commit(p, size, NULL)) {- mi_os_prim_free(os_base, over_size, 0, NULL);+ if (!_mi_os_commit(subproc, p, size, NULL)) {+ mi_os_prim_free(subproc, os_base, over_size, 0, true);return NULL;}}}else { // mmap can free inside an allocation// overallocate...- p = mi_os_prim_alloc(over_size, 1, commit, false, &os_is_large, &os_is_zero);+ // note: we keep `allow_large` but this only works if we can partially free any large OS pages later on. This seems to be the case though.+ p = mi_os_prim_alloc(subproc, over_size, 1, commit, allow_large /* or false? */, &os_is_large, &os_is_zero);if (p == NULL) return NULL;// and selectively unmap parts around the over-allocated area.@@ -401,8 +424,8 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commitconst size_t mid_size = _mi_align_up(size, _mi_os_page_size());const size_t post_size = over_size - pre_size - mid_size;mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);- if (pre_size > 0) { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0), NULL); }- if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), NULL); }+ if (pre_size > 0) { mi_os_prim_free(subproc, p, pre_size, (commit ? pre_size : 0), true /* adjust */); }+ if (post_size > 0) { mi_os_prim_free(subproc, (uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), true /* adjust */); }// we can return the aligned pointer on `mmap` systemsp = aligned_p;os_base = aligned_p; // since we freed the pre part, `*base == p`.@@ -421,13 +444,13 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commitOS API: alloc and alloc_aligned----------------------------------------------------------- */-void* _mi_os_alloc(size_t size, mi_memid_t* memid) {+void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) {*memid = _mi_memid_none();if (size == 0) return NULL;size = _mi_os_good_alloc_size(size);bool os_is_large = false;bool os_is_zero = false;- void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero);+ void* p = mi_os_prim_alloc(subproc, size, 0, true, false, &os_is_large, &os_is_zero);if (p == NULL) return NULL;*memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large);@@ -436,7 +459,7 @@ void* _mi_os_alloc(size_t size, mi_memid_t* memid) {return p;}-void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid)+void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid){MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings*memid = _mi_memid_none();@@ -444,7 +467,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allosize = _mi_os_good_alloc_size(size);alignment = _mi_align_up(alignment, _mi_os_page_size());- void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, memid );+ void* p = mi_os_prim_alloc_aligned(subproc, size, alignment, commit, allow_large, memid );if (p == NULL) return NULL;mi_assert_internal(memid->mem.os.size >= size);@@ -454,13 +477,13 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo}-mi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_t* memid) {+mi_decl_nodiscard static void* mi_os_ensure_zero(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t* memid) {if (p==NULL || size==0) return p;// ensure committedif (!memid->initially_committed) {bool is_zero = false;- if (!_mi_os_commit(p, size, &is_zero)) {- _mi_os_free(p, size, *memid);+ if (!_mi_os_commit(subproc, p, size, &is_zero)) {+ _mi_os_free(subproc, p, size, *memid);return NULL;}memid->initially_committed = true;@@ -472,9 +495,9 @@ mi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_return p;}-void* _mi_os_zalloc(size_t size, mi_memid_t* memid) {- void* p = _mi_os_alloc(size,memid);- return mi_os_ensure_zero(p, size, memid);+void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) {+ void* p = _mi_os_alloc(subproc, size,memid);+ return mi_os_ensure_zero(subproc, p, size, memid);}/* -----------------------------------------------------------@@ -485,28 +508,29 @@ void* _mi_os_zalloc(size_t size, mi_memid_t* memid) {to use the actual start of the memory region.----------------------------------------------------------- */-void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) {+void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) {mi_assert(offset <= size);mi_assert((alignment % _mi_os_page_size()) == 0);*memid = _mi_memid_none();if (offset > size) return NULL;if (offset == 0) {// regular aligned allocation- return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);+ return _mi_os_alloc_aligned(subproc, size, alignment, commit, allow_large, memid);}else {// overallocate to align at an offsetconst size_t extra = _mi_align_up(offset, alignment) - offset;if (size >= SIZE_MAX - extra) return NULL; // too largeconst size_t oversize = size + extra;- void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid);+ void* const start = _mi_os_alloc_aligned(subproc, oversize, alignment, commit, allow_large, memid);if (start == NULL) return NULL;void* const p = (uint8_t*)start + extra;mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment));// decommit the overallocation at the start+ // note: this double counts the decommit when freeing `memid`. Should we keep commit size in the memid as well?if (commit && extra >= _mi_os_page_size()) {- _mi_os_decommit(start, extra);+ _mi_os_decommit(subproc, start, extra);}return p;}@@ -540,9 +564,10 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t*return mi_os_page_align_areax(true, addr, size, newsize);}-bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) {+bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed) {+ mi_assert_internal(size >= stat_already_committed);if (is_zero != NULL) { *is_zero = false; }- mi_os_stat_counter_increase(commit_calls, 1);+ mi_subproc_stat_counter_increase(subproc, commit_calls, 1);// page align rangesize_t csize;@@ -565,15 +590,15 @@ bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size)if (os_is_zero) { mi_track_mem_defined(start,csize); }else { mi_track_mem_undefined(start,csize); }#endif- mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit+ mi_subproc_stat_increase(subproc, committed, size - stat_already_committed); // for precise commit vs. decommitreturn true;}-bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {- return _mi_os_commit_ex(addr, size, is_zero, size);+bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero) {+ return _mi_os_commit_ex(subproc, addr, size, is_zero, 0);}-static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stat_size) {+static bool mi_os_decommit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* needs_recommit, size_t stat_size) {mi_assert_internal(needs_recommit!=NULL);// page align@@ -588,15 +613,15 @@ static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, siz_mi_warning_message("cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);}else if (*needs_recommit) {- mi_os_stat_decrease(committed, stat_size);+ mi_subproc_stat_decrease(subproc, committed, stat_size);}mi_assert_internal(err == 0);return (err == 0);}-bool _mi_os_decommit(void* addr, size_t size) {+bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size) {bool needs_recommit;- return mi_os_decommit_ex(addr, size, &needs_recommit, size);+ return mi_os_decommit_ex(subproc, addr, size, &needs_recommit, size);}@@ -604,13 +629,13 @@ bool _mi_os_decommit(void* addr, size_t size) {// but may be used later again. This will release physical memory// pages and reduce swapping while keeping the memory committed.// We page align to a conservative area inside the range to reset.-bool _mi_os_reset(void* addr, size_t size) {+bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size) {// page align conservatively within the rangesize_t csize;void* start = mi_os_page_align_area_conservative(addr, size, &csize);if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr)- mi_os_stat_counter_increase(reset, csize);- mi_os_stat_counter_increase(reset_calls, 1);+ mi_subproc_stat_counter_increase(subproc, reset, csize);+ mi_subproc_stat_counter_increase(subproc, reset_calls, 1);#if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSANmemset(start, 0, csize); // pretend it is eagerly reset@@ -624,7 +649,8 @@ bool _mi_os_reset(void* addr, size_t size) {}-void _mi_os_reuse( void* addr, size_t size ) {+void _mi_os_reuse( mi_subproc_t* subproc, void* addr, size_t size ) {+ MI_UNUSED(subproc);// page align conservatively within the rangesize_t csize = 0;void* const start = mi_os_page_align_area_conservative(addr, size, &csize);@@ -637,11 +663,11 @@ void _mi_os_reuse( void* addr, size_t size ) {// either resets or decommits memory, returns true if the memory needs// to be recommitted if it is to be re-used later on.-bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg)+bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg){if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed?- mi_os_stat_counter_increase(purge_calls, 1);- mi_os_stat_counter_increase(purged, size);+ mi_subproc_stat_counter_increase(subproc, purge_calls, 1);+ mi_subproc_stat_counter_increase(subproc, purged, size);if (commit_fun != NULL) {bool decommitted = (*commit_fun)(false, p, size, NULL, commit_fun_arg);@@ -651,12 +677,12 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, m!_mi_preloading()) // don't decommit during preloading (unsafe){bool needs_recommit = true;- mi_os_decommit_ex(p, size, &needs_recommit, stat_size);+ mi_os_decommit_ex(subproc, p, size, &needs_recommit, stat_size);return needs_recommit;}else {if (allow_reset) { // this can sometimes be not allowed if the range is not fully committed (on Windows, we cannot reset uncommitted memory)- _mi_os_reset(p, size);+ _mi_os_reset(subproc, p, size);}return false; // needs no recommit}@@ -664,8 +690,8 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, m// either resets or decommits memory, returns true if the memory needs// to be recommitted if it is to be re-used later on.-bool _mi_os_purge(void* p, size_t size) {- return _mi_os_purge_ex(p, size, true, size, NULL, NULL);+bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size) {+ return _mi_os_purge_ex(subproc, p, size, true, size, NULL, NULL);}@@ -704,14 +730,18 @@ and possibly associated with a specific NUMA node. (use `numa_node>=0`)#define MI_HUGE_OS_PAGE_SIZE (MI_GiB)-#if (MI_INTPTR_SIZE >= 8)+#if (MI_SIZE_SIZE >= 8)// To ensure proper alignment, use our own area for huge OS pagesstatic mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0// Claim an aligned address range for huge pagesstatic uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {if (total_size != NULL) *total_size = 0;- const size_t size = pages * MI_HUGE_OS_PAGE_SIZE;+ size_t size = 0;+ if (mi_mul_overflow(pages,MI_HUGE_OS_PAGE_SIZE,&size)) {+ _mi_warning_message("too many huge pages requested: %zu\n", pages);+ return NULL;+ }uintptr_t start = 0;uintptr_t end = 0;@@ -720,7 +750,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {start = huge_start;if (start == 0) {// Initialize the start address after the 32TiB area- start = ((uintptr_t)8 << 40); // 8TiB virtual start address+ start = ((uintptr_t)32 << 40); // 32TiB virtual start address (after addresses returned by _mi_os_get_aligned_hint)#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug modemi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)if (mi_theap_is_initialized(theap)) { // todo: or no hint at all if we lack randomness?@@ -728,7 +758,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB}else {- _mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, start);+ _mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, (void*)start);}#endif}@@ -747,7 +777,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {#endif// Allocate MI_ARENA_SLICE_ALIGN aligned huge pages-void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) {+void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) {*memid = _mi_memid_none();if (psize != NULL) *psize = 0;if (pages_reserved != NULL) *pages_reserved = 0;@@ -778,21 +808,21 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse// no success, issue a warning and breakif (p != NULL) {_mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr);- mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, NULL);+ mi_os_prim_free(subproc, p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, true /* adjust */);}break;}// success, record itpage++; // increase before timeout check (see issue #711)- mi_os_stat_increase(committed, MI_HUGE_OS_PAGE_SIZE);- mi_os_stat_increase(reserved, MI_HUGE_OS_PAGE_SIZE);+ mi_subproc_stat_increase(subproc, committed, MI_HUGE_OS_PAGE_SIZE);+ mi_subproc_stat_increase(subproc, reserved, MI_HUGE_OS_PAGE_SIZE);// check for timeoutif (max_msecs > 0) {mi_msecs_t elapsed = _mi_clock_end(start_t);if (page >= 1) {- mi_msecs_t estimate = ((elapsed / (page+1)) * pages);+ mi_msecs_t estimate = ((elapsed / (page==0 ? 1 : page)) * pages);if (estimate > 2*max_msecs) { // seems like we are going to timeout, breakelapsed = max_msecs + 1;}@@ -821,11 +851,11 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse// free every huge page in a range individually (as we allocated per page)// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems.-static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc) {+static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size) {if (p==NULL || size==0) return;uint8_t* base = (uint8_t*)p;while (size >= MI_HUGE_OS_PAGE_SIZE) {- mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, subproc);+ mi_os_prim_free(subproc, base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, false /* adjust? */);size -= MI_HUGE_OS_PAGE_SIZE;base += MI_HUGE_OS_PAGE_SIZE;}diff --git a/vendored/mimalloc/src/page-map.c b/vendored/mimalloc/src/page-map.cindex acc0ee9f7..949e37ac3 100644--- a/vendored/mimalloc/src/page-map.c+++ b/vendored/mimalloc/src/page-map.c@@ -1,5 +1,5 @@/*-----------------------------------------------------------------------------Copyright (c) 2023-2025, Microsoft Research, Daan Leijen+Copyright (c) 2023-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -25,8 +25,11 @@ static void mi_page_map_cannot_commit(void) {// A full 256 TiB address space (48 bit) needs a 4 GiB page map.// A full 4 GiB address space (32 bit) needs only a 64 KiB page map.-mi_decl_cache_align uint8_t* _mi_page_map = NULL;-static void* mi_page_map_max_address = NULL;+// Use an initial empty page map so `free(NULL)` works even if mimalloc is not yet initialized (issue #1341)+static uint8_t mi_page_map_empty[1] = { 1 }; // _mi_ptr_page(NULL) == NULL++mi_decl_hidden mi_decl_cache_align _Atomic(uint8_t*) _mi_page_map = mi_page_map_empty;+mi_decl_hidden _Atomic(void*) _mi_page_map_max_address = NULL;static mi_memid_t mi_page_map_memid;#define MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT MI_ARENA_SLICE_SIZE@@ -35,7 +38,7 @@ static mi_bitmap_t* mi_page_map_commit; // one bit per committed 64 KiB entriesmi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count);bool _mi_page_map_init(void) {- size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS);+ size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_MAX_VABITS);if (vbits == 0) {vbits = _mi_os_virtual_address_bits();#if MI_ARCH_X64 // canonical address is limited to the first 128 TiB@@ -45,15 +48,22 @@ bool _mi_page_map_init(void) {if (vbits < MI_ARENA_SLICE_SHIFT) {vbits = MI_ARENA_SLICE_SHIFT;}+ if (vbits < MI_MIN_VABITS) { // cover at least this much for a faster _mi_checked_ptr+ vbits = MI_MIN_VABITS;+ }+ if (vbits > MI_MAX_VABITS) { // limit page map size even if more virtual addresses are available+ vbits = MI_MAX_VABITS;+ }// Allocate the page map and commit bits- mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));+ mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits)));const size_t page_map_size = (MI_ZU(1) << (vbits - MI_ARENA_SLICE_SHIFT));const bool commit = (page_map_size <= 1*MI_MiB || mi_option_is_enabled(mi_option_pagemap_commit)); // _mi_os_has_overcommit(); // commit on-access on Linux systems?const size_t commit_bits = _mi_divide_up(page_map_size, MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);const size_t bitmap_size = (commit ? 0 : mi_bitmap_size(commit_bits, NULL));const size_t reserve_size = bitmap_size + page_map_size;- uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);+ mi_subproc_t* const subproc = _mi_subproc_main();+ uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(subproc, reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);if (base==NULL) {_mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB);return false;@@ -64,13 +74,13 @@ bool _mi_page_map_init(void) {}if (bitmap_size > 0) {mi_page_map_commit = (mi_bitmap_t*)base;- if (!_mi_os_commit(mi_page_map_commit, bitmap_size, NULL)) {+ if (!_mi_os_commit(subproc, mi_page_map_commit, bitmap_size, NULL)) {mi_page_map_cannot_commit();return false;}mi_bitmap_init(mi_page_map_commit, commit_bits, true);}- _mi_page_map = base + bitmap_size;+ mi_atomic_store_ptr_release(uint8_t,&_mi_page_map, base + bitmap_size);// commit the first part so NULL pointers get resolved without an access violationif (!commit) {@@ -79,19 +89,18 @@ bool _mi_page_map_init(void) {return false;}}- _mi_page_map[0] = 1; // so _mi_ptr_page(NULL) == NULL+ mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map)[0] = 1; // so _mi_ptr_page(NULL) == NULLmi_assert_internal(_mi_ptr_page(NULL)==NULL);return true;}-void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) {- mi_assert_internal(subproc != NULL);- mi_assert_internal(_mi_page_map != NULL);- if (_mi_page_map == NULL) return;- _mi_os_free_ex(mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc);- _mi_page_map = NULL;+void _mi_page_map_unsafe_destroy(void) {+ mi_assert_internal(mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map) != NULL);+ if (mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map) == NULL) return;+ _mi_os_free_ex(_mi_subproc_main(), mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, true, mi_page_map_memid);+ mi_atomic_store_ptr_release(uint8_t, &_mi_page_map, NULL);mi_page_map_commit = NULL;- mi_page_map_max_address = NULL;+ mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, NULL);mi_page_map_memid = _mi_memid_none();}@@ -100,6 +109,7 @@ static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) {// is the page map area that contains the page address committed?// we always set the commit bits so we can track what ranges are in-use.// we only actually commit if the map wasn't committed fully already.+ uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map);if (mi_page_map_commit != NULL) {const size_t commit_idx = idx / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;const size_t commit_idx_hi = (idx + slice_count - 1) / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;@@ -107,9 +117,9 @@ static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) {if (mi_bitmap_is_clear(mi_page_map_commit, i)) {// this may race, in which case we do multiple commits (which is ok)bool is_zero;- uint8_t* const start = _mi_page_map + (i * MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);+ uint8_t* const start = page_map + (i * MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);const size_t size = MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;- if (!_mi_os_commit(start, size, &is_zero)) {+ if (!_mi_os_commit(_mi_subproc_main(), start, size, &is_zero)) {mi_page_map_cannot_commit();return false;}@@ -119,8 +129,8 @@ static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) {}}#if MI_DEBUG > 0- _mi_page_map[idx] = 0;- _mi_page_map[idx+slice_count-1] = 0;+ page_map[idx] = 0;+ page_map[idx+slice_count-1] = 0;#endifreturn true;}@@ -137,11 +147,13 @@ static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t*bool _mi_page_map_register(mi_page_t* page) {mi_assert_internal(page != NULL);mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));- mi_assert_internal(_mi_page_map != NULL); // should be initialized before multi-thread access!- if mi_unlikely(_mi_page_map == NULL) {+ mi_assert_internal(mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map) != NULL); // should be initialized before multi-thread access!+ uint8_t* page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map);+ if mi_unlikely(mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map) == NULL) {if (!_mi_page_map_init()) return false;+ page_map = mi_atomic_load_ptr_acquire(uint8_t,&_mi_page_map);}- mi_assert(_mi_page_map!=NULL);+ mi_assert(page_map!=NULL);uint8_t* page_start;size_t slice_count;const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count);@@ -153,37 +165,42 @@ bool _mi_page_map_register(mi_page_t* page) {// set the offsetsfor (size_t i = 0; i < slice_count; i++) {mi_assert_internal(i < 128);- _mi_page_map[idx + i] = (uint8_t)(i+1);+ page_map[idx + i] = (uint8_t)(i+1);}return true;}void _mi_page_map_unregister(mi_page_t* page) {- mi_assert_internal(_mi_page_map != NULL);+ uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map);+ mi_assert_internal(page_map != NULL);+ if (page_map == NULL) return;// get index and countuint8_t* page_start;size_t slice_count;const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count);// unset the offsets- _mi_memzero(_mi_page_map + idx, slice_count);+ _mi_memzero(page_map + idx, slice_count);}void _mi_page_map_unregister_range(void* start, size_t size) {+ uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map);+ mi_assert_internal(page_map!=NULL);+ if (page_map == NULL) return;const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE);const uintptr_t index = _mi_page_map_index(start);// todo: scan the commit bits and clear only those ranges?if (!mi_page_map_ensure_committed(index, slice_count)) { // we commit the range in total;return;}- _mi_memzero(&_mi_page_map[index], slice_count);+ _mi_memzero(&page_map[index], slice_count);}mi_page_t* _mi_safe_ptr_page(const void* p) {- if mi_unlikely(p >= mi_page_map_max_address) return NULL;+ if mi_unlikely(p >= mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL;const uintptr_t idx = _mi_page_map_index(p);if mi_unlikely(mi_page_map_commit != NULL && !mi_bitmap_is_set(mi_page_map_commit, idx/MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT)) return NULL;- const uintptr_t ofs = _mi_page_map[idx];+ const uintptr_t ofs = _mi_page_map_at(idx);if mi_unlikely(ofs == 0) return NULL;return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) - ofs + 1) << MI_ARENA_SLICE_SHIFT);}@@ -196,43 +213,64 @@ mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_att// A 2-level page map#define MI_PAGE_MAP_SUB_SIZE (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*))-#define MI_PAGE_MAP_ENTRIES_PER_CBIT (MI_PAGE_MAP_COUNT < MI_BFIELD_BITS ? 1 : (MI_PAGE_MAP_COUNT / MI_BFIELD_BITS))-mi_decl_cache_align _Atomic(mi_submap_t)* _mi_page_map;-static size_t mi_page_map_count;-static void* mi_page_map_max_address;-static mi_memid_t mi_page_map_memid;-static mi_lock_t mi_page_map_lock;+// Use an initial empty page map so `free(NULL)` works even if mimalloc is not yet initialized (issue #1341)+static mi_page_map_t mi_page_map_empty = {+ MI_ATOMIC_VAR_INIT(1),+ sizeof(mi_page_map_t),+ MI_MEMID_STATIC,+ MI_LOCK_INITIALIZER,+ { MI_ATOMIC_VAR_INIT(NULL) }+};++mi_decl_hidden mi_decl_cache_align _Atomic(mi_page_map_t*) __mi_page_map = MI_ATOMIC_VAR_INIT(&mi_page_map_empty);-// divide the main map in 64 (`MI_BFIELD_BITS`) parts commit those parts on demand-static _Atomic(mi_bfield_t) mi_page_map_commit;+static size_t mi_page_map_count_of_size(size_t size) {+ return (size < sizeof(mi_page_map_t) ? 0 : 1 + (size - sizeof(mi_page_map_t))/sizeof(mi_submap_t));+}++// static void* mi_page_map_addr_of_index(size_t idx) {+// return (void*)((uintptr_t)idx * MI_PAGE_MAP_SUB_COUNT * MI_ARENA_SLICE_SIZE);+// }-mi_decl_nodiscard static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx) {- mi_bfield_t commit = mi_atomic_load_relaxed(&mi_page_map_commit);- const size_t bit_idx = idx/MI_PAGE_MAP_ENTRIES_PER_CBIT;- mi_assert_internal(bit_idx < MI_BFIELD_BITS);- if (pbit_idx != NULL) { *pbit_idx = bit_idx; }- return ((commit & (MI_ZU(1) << bit_idx)) != 0);+mi_decl_nodiscard static mi_decl_noinline bool mi_page_map_commit_entries(mi_page_map_t* pmap, size_t required_idx) {+ const size_t reserved_count = mi_page_map_count_of_size(pmap->reserved_size);+ if mi_unlikely(required_idx >= reserved_count) {+ mi_page_map_cannot_commit();+ return false;+ }+ size_t commit_size = _mi_align_up( sizeof(mi_page_map_t) + (required_idx * sizeof(mi_submap_t)), MI_ARENA_SLICE_SIZE );+ if (pmap->reserved_size < commit_size) { commit_size = pmap->reserved_size; }+ const size_t commit_count = mi_page_map_count_of_size(commit_size);+ mi_assert_internal(commit_count > required_idx);+ mi_assert_internal(commit_count <= reserved_count);+ // note: we rely on uncommitted memory to be zero initialized on the first commit (and further concurrent commits leave the memory as is).+ bool is_zero;+ if mi_unlikely(!_mi_os_commit(_mi_subproc_main(), pmap, commit_size, &is_zero)) {+ mi_page_map_cannot_commit();+ return false;+ }+ mi_assert_internal(is_zero || pmap->memid.initially_zero);+ mi_atomic_store_release(&pmap->committed_count, commit_count);+ // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count));+ return true;}-mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, mi_submap_t* submap) {+mi_decl_nodiscard static bool mi_page_map_ensure_committed(mi_page_map_t* pmap, size_t idx, mi_submap_t* submap) {mi_assert_internal(submap!=NULL && *submap==NULL);- size_t bit_idx;- if mi_unlikely(!mi_page_map_is_committed(idx, &bit_idx)) {- uint8_t* start = (uint8_t*)&_mi_page_map[bit_idx * MI_PAGE_MAP_ENTRIES_PER_CBIT];- if (!_mi_os_commit(start, MI_PAGE_MAP_ENTRIES_PER_CBIT * sizeof(mi_submap_t), NULL)) {- mi_page_map_cannot_commit();- return false;+ if mi_unlikely(idx >= mi_atomic_load_relaxed(&pmap->committed_count)) {+ if (idx >= mi_atomic_load_acquire(&pmap->committed_count)) {+ if (!mi_page_map_commit_entries(pmap,idx)) return false;+ mi_assert_internal(idx < mi_atomic_load_relaxed(&pmap->committed_count));}- mi_atomic_or_acq_rel(&mi_page_map_commit, MI_ZU(1) << bit_idx);}- *submap = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // acquire _mi_page_map_at(idx);+ *submap = mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]);return true;}-// initialize the page map-bool _mi_page_map_init(void) {- size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS);+// initialize the page map+static bool mi_page_map_init_once(void) {+ size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_MAX_VABITS);if (vbits == 0) {vbits = _mi_os_virtual_address_bits();#if MI_ARCH_X64 // canonical address is limited to the first 128 TiB@@ -242,111 +280,145 @@ bool _mi_page_map_init(void) {if (vbits < MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT) {vbits = MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT;}+ if (vbits < MI_MIN_VABITS) { // cover at least this much for a faster _mi_checked_ptr+ vbits = MI_MIN_VABITS;+ }+ if (vbits > MI_MAX_VABITS) { // limit page map size even if more virtual addresses are available+ vbits = MI_MAX_VABITS;+ }// Allocate the page map and commit bitsmi_assert(MI_MAX_VABITS >= vbits);- mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));- mi_page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));- mi_assert(mi_page_map_count <= MI_PAGE_MAP_COUNT);- const size_t os_page_size = _mi_os_page_size();- const size_t page_map_size = _mi_align_up( mi_page_map_count * sizeof(mi_page_t**), os_page_size);- const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;- const size_t reserve_size = page_map_size + submap_size;- #if MI_SECURE- const bool commit = true; // the whole page map is valid and we can reliably check any pointer- #else- const bool commit = page_map_size <= 64*MI_KiB ||+ mi_assert(MI_MIN_VABITS <= vbits);+ const size_t reserve_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));+ const size_t os_page_size = _mi_os_page_size();+ const size_t reserve_size = _mi_align_up( sizeof(mi_page_map_t) + ((reserve_count - 1) * sizeof(mi_submap_t)), os_page_size);+ const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;+ const size_t extra_reserve_size = reserve_size + submap_size;+ const bool commit = (vbits == MI_MIN_VABITS) || (reserve_size <= 64*MI_KiB) || // 42 virtual address bitsmi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit();- #endif- _mi_page_map = (_Atomic(mi_page_t**)*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);- if (_mi_page_map==NULL) {- _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB);+ mi_subproc_t* const subproc = _mi_subproc_main();+ mi_memid_t memid;+ mi_page_map_t* const pmap = (mi_page_map_t*)_mi_os_alloc_aligned(subproc, extra_reserve_size, 1, commit, true /* allow large */, &memid);+ if mi_unlikely(pmap==NULL) {+ _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", extra_reserve_size / MI_KiB);return false;}- if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) {- _mi_warning_message("internal: the page map was committed but not zero initialized!\n");- _mi_memzero_aligned(_mi_page_map, page_map_size);++ // commit+ size_t commit_count;+ if (memid.initially_committed) {+ if (!memid.initially_zero) {+ _mi_warning_message("internal: the page map was committed but not zero initialized!\n");+ _mi_memzero_aligned(pmap, extra_reserve_size);+ memid.initially_zero = true;+ }+ commit_count = mi_page_map_count_of_size(reserve_size);}- mi_atomic_store_release(&mi_page_map_commit, (mi_page_map_memid.initially_committed ? ~MI_ZU(0) : MI_ZU(0)));-+ else {+ // commit first entries up to MI_MIN_VABITS entries+ const size_t min_commit_count = (MI_ZU(1) << (MI_MIN_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));+ const size_t min_commit_size = _mi_align_up( sizeof(mi_page_map_t) + ((min_commit_count-1) * sizeof(mi_submap_t)), os_page_size);+ mi_assert_internal(min_commit_size <= reserve_size);+ bool is_zero;+ if (!_mi_os_commit(subproc,pmap,min_commit_size,&is_zero)) {+ mi_page_map_cannot_commit();+ _mi_os_free(subproc,pmap,extra_reserve_size,memid);+ return false;+ };+ mi_assert_internal(is_zero || memid.initially_zero);+ commit_count = mi_page_map_count_of_size(min_commit_size);+ mi_assert_internal(commit_count >= min_commit_count);+ }+// ensure there is a submap for the NULL address- mi_page_t** const sub0 = (mi_page_t**)((uint8_t*)_mi_page_map + page_map_size); // we reserved a submap part at the end already- if (!mi_page_map_memid.initially_committed) {- if (!_mi_os_commit(sub0, submap_size, NULL)) { // commit full submap (issue #1087)+ mi_page_t** const sub0 = (mi_submap_t)((uint8_t*)pmap + reserve_size); // we reserved a submap part at the end already+ if (!memid.initially_committed) {+ if (!_mi_os_commit(subproc, sub0, submap_size, NULL)) { // commit full submap (issue #1087)mi_page_map_cannot_commit();+ _mi_os_free(subproc,pmap,extra_reserve_size,memid);return false;}}- if (!mi_page_map_memid.initially_zero) { // initialize low addresses with NULL+ if (!memid.initially_zero) { // initialize low addresses with NULL_mi_memzero_aligned(sub0, submap_size);}- mi_submap_t nullsub = NULL;- if (!mi_page_map_ensure_committed(0,&nullsub)) {- mi_page_map_cannot_commit();- return false;- }- mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[0], sub0);- mi_lock_init(&mi_page_map_lock); // initialize late in case the lock init causes allocation-++ // initialize the fields+ pmap->memid = memid;+ pmap->reserved_size = reserve_size;+ mi_lock_init(&pmap->lock);+ mi_atomic_store_release(&pmap->committed_count, commit_count);+ // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count));+ mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[0], sub0);+ mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, pmap);mi_assert_internal(_mi_ptr_page(NULL)==NULL);return true;}+bool _mi_page_map_init(void) {+ bool ok = true;+ mi_atomic_do_once {+ ok = mi_page_map_init_once();+ }+ return ok;+}-void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) {- mi_assert_internal(subproc != NULL);- mi_assert_internal(_mi_page_map != NULL);- if (_mi_page_map == NULL) return;- mi_lock_done(&mi_page_map_lock);- for (size_t idx = 1; idx < mi_page_map_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map)- // free all sub-maps- if (mi_page_map_is_committed(idx, NULL)) {- mi_submap_t sub = _mi_page_map_at(idx);- if (sub != NULL) {- mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false);- _mi_os_free_ex(memid.mem.os.base, memid.mem.os.size, true, memid, subproc);- mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[idx], NULL);- }+void _mi_page_map_unsafe_destroy(void) {+ mi_page_map_t* const pmap = _mi_page_map();+ mi_assert_internal(pmap != NULL);+ if (pmap == NULL || pmap == &mi_page_map_empty) return;+ mi_subproc_t* const subproc = _mi_subproc_main();+ mi_lock_done(&pmap->lock);+ for (size_t idx = 1; idx < pmap->committed_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map)+ // free all sub-maps+ mi_submap_t sub = _mi_page_map_at(pmap,idx);+ if (sub != NULL) {+ mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false);+ _mi_os_free_ex(subproc, memid.mem.os.base, memid.mem.os.size, true, memid);+ mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[idx], NULL);}}- _mi_os_free_ex(_mi_page_map, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc);- _mi_page_map = NULL;- mi_page_map_count = 0;- mi_page_map_memid = _mi_memid_none();- mi_page_map_max_address = NULL;- mi_atomic_store_release(&mi_page_map_commit, (mi_bfield_t)0);+ _mi_os_free_ex(subproc, pmap, pmap->reserved_size, true, pmap->memid);+ mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, &mi_page_map_empty);}--mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(size_t idx, mi_submap_t* submap) {- mi_assert_internal(submap!=NULL && *submap==NULL);+mi_decl_nodiscard static mi_decl_noinline mi_submap_t mi_page_map_alloc_submap_at(mi_page_map_t* pmap, size_t idx) {+ // sub map not yet allocated, alloc nowmi_submap_t sub = NULL;- if (!mi_page_map_ensure_committed(idx, &sub)) {- return false;- }- if mi_unlikely(sub == NULL) {- // sub map not yet allocated, alloc now- mi_lock(&mi_page_map_lock)+ mi_lock(&pmap->lock)+ {+ sub = mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); // reload+ if (sub==NULL) // not yet allocated by another thread?{- sub = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // reload- if (sub==NULL) // not yet allocated by another thread?- {- mi_memid_t memid;- const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;- sub = (mi_submap_t)_mi_os_zalloc(submap_size, &memid);- if (sub==NULL) {- _mi_warning_message("internal error: unable to extend the page map\n");- }- else {- mi_submap_t expect = NULL;- if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &_mi_page_map[idx], &expect, sub)) {- // another thread already allocated it.. free and continue- _mi_os_free(sub, submap_size, memid);- sub = expect;- }+ mi_subproc_t* const subproc = _mi_subproc_main();+ mi_memid_t memid;+ const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;+ sub = (mi_submap_t)_mi_os_zalloc(subproc, submap_size, &memid);+ if (sub==NULL) {+ _mi_warning_message("internal error: unable to extend the page map\n");+ }+ else {+ mi_submap_t expect = NULL;+ if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &pmap->submaps[idx], &expect, sub)) {+ // another thread already allocated it.. free and continue+ _mi_os_free(subproc, sub, submap_size, memid);+ sub = expect;}}}+ }+ return sub;+}++mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(mi_page_map_t* pmap, size_t idx, mi_submap_t* submap) {+ mi_assert_internal(submap!=NULL && *submap==NULL);+ mi_submap_t sub = NULL;+ if (!mi_page_map_ensure_committed(pmap, idx, &sub)) {+ return false;+ }+ if mi_unlikely(sub==NULL) {+ sub = mi_page_map_alloc_submap_at(pmap, idx);if (sub==NULL) return false; // unable to allocate the submap..}mi_assert_internal(sub!=NULL);@@ -354,11 +426,11 @@ mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(size_t idx, mi_submapreturn true;}-static bool mi_page_map_set_range_prim(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {+static bool mi_page_map_set_range_prim(mi_page_map_t* pmap, mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {// is the page map area that contains the page address committed?while (slice_count > 0) {mi_submap_t sub = NULL;- if (!mi_page_map_ensure_submap_at(idx, &sub)) {+ if (!mi_page_map_ensure_submap_at(pmap, idx, &sub)) {return false;};mi_assert_internal(sub!=NULL);@@ -374,11 +446,11 @@ static bool mi_page_map_set_range_prim(mi_page_t* page, size_t idx, size_t sub_ireturn true;}-static bool mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {- if mi_unlikely(!mi_page_map_set_range_prim(page,idx,sub_idx,slice_count)) {+static bool mi_page_map_set_range(mi_page_map_t* pmap, mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {+ if mi_unlikely(!mi_page_map_set_range_prim(pmap, page,idx,sub_idx,slice_count)) {// failed to commit, call again to reset the page pointer if neededif (page!=NULL) {- mi_page_map_set_range_prim(NULL,idx,sub_idx,slice_count);+ mi_page_map_set_range_prim(pmap,NULL,idx,sub_idx,slice_count);}return false;}@@ -396,48 +468,46 @@ static size_t mi_page_map_get_idx(mi_page_t* page, size_t* sub_idx, size_t* slicbool _mi_page_map_register(mi_page_t* page) {mi_assert_internal(page != NULL);mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));- mi_assert_internal(_mi_page_map != NULL); // should be initialized before multi-thread access!- if mi_unlikely(_mi_page_map == NULL) {+ mi_page_map_t* pmap = _mi_page_map();+ mi_assert_internal(pmap != NULL); // should be initialized before multi-thread access!+ if mi_unlikely(pmap == NULL) {if (!_mi_page_map_init()) return false;+ pmap = mi_atomic_load_ptr_acquire(mi_page_map_t,&__mi_page_map);}- mi_assert(_mi_page_map!=NULL);+ mi_assert(pmap!=NULL);size_t slice_count;size_t sub_idx;const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);- return mi_page_map_set_range(page, idx, sub_idx, slice_count);+ return mi_page_map_set_range(pmap, page, idx, sub_idx, slice_count);}void _mi_page_map_unregister(mi_page_t* page) {mi_assert_internal(_mi_page_map != NULL);mi_assert_internal(page != NULL);mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));- if mi_unlikely(_mi_page_map == NULL) return;+ mi_page_map_t* const pmap = _mi_page_map();+ // note: should proceed even if the page was not registered yet (for failure paths in page allocation in `arena.c`)+ if mi_unlikely(pmap == NULL) return;// get index and countsize_t slice_count;size_t sub_idx;const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);// unset the offsets- mi_page_map_set_range(NULL, idx, sub_idx, slice_count);+ mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count);}void _mi_page_map_unregister_range(void* start, size_t size) {- if mi_unlikely(_mi_page_map == NULL) return;+ mi_page_map_t* const pmap = _mi_page_map();+ if mi_unlikely(pmap == NULL) return;const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE);size_t sub_idx;const uintptr_t idx = _mi_page_map_index(start, &sub_idx);- mi_page_map_set_range(NULL, idx, sub_idx, slice_count); // todo: avoid committing if not already committed?+ mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count); // todo: avoid committing if not already committed?}// Return NULL for invalid pointersmi_page_t* _mi_safe_ptr_page(const void* p) {- if (p==NULL) return NULL;- if mi_unlikely(p >= mi_page_map_max_address) return NULL;- size_t sub_idx;- const size_t idx = _mi_page_map_index(p,&sub_idx);- if mi_unlikely(!mi_page_map_is_committed(idx,NULL)) return NULL;- mi_page_t** const sub = _mi_page_map[idx];- if mi_unlikely(sub==NULL) return NULL;- return sub[sub_idx];+ return _mi_checked_ptr_page(p);}mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {diff --git a/vendored/mimalloc/src/page-queue.c b/vendored/mimalloc/src/page-queue.cindex 731672b02..c8e9b5425 100644--- a/vendored/mimalloc/src/page-queue.c+++ b/vendored/mimalloc/src/page-queue.c@@ -21,11 +21,11 @@ terms of the MIT license. A copy of the license can be found in the fileMinimal alignment in machine words (i.e. `sizeof(void*)`)----------------------------------------------------------- */-#if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE)+#if (MI_MAX_ALIGN_SIZE > 4*MI_SIZE_SIZE)#error "define alignment for more than 4x word size for this platform"-#elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE)+#elif (MI_MAX_ALIGN_SIZE > 2*MI_SIZE_SIZE)#define MI_ALIGN4W // 4 machine words minimal alignment-#elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE)+#elif (MI_MAX_ALIGN_SIZE > MI_SIZE_SIZE)#define MI_ALIGN2W // 2 machine words minimal alignment#else// ok, default alignment is 1 word@@ -61,8 +61,8 @@ static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) {// Returns MI_BIN_HUGE if the size is too large.// We use `wsize` for the size in "machine word sizes",// i.e. byte size == `wsize*sizeof(void*)`.-static mi_decl_noinline size_t mi_bin(size_t size) {- size_t wsize = _mi_wsize_from_size(size);+static size_t mi_bin(size_t size) {+ size_t wsize = mi_wsize_from_size(size);#if defined(MI_ALIGN4W)if mi_likely(wsize <= 4) {return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes@@ -112,10 +112,10 @@ size_t _mi_bin_size(size_t bin) {// Good size for allocationmi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept {- if (size <= MI_LARGE_MAX_OBJ_SIZE) {+ if (size <= MI_LARGE_MAX_OBJ_SIZE - MI_PADDING_SIZE) {return _mi_bin_size(mi_bin(size + MI_PADDING_SIZE));}- else if (size <= MI_MAX_ALLOC_SIZE) {+ else if (size <= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) {return _mi_align_up(size + MI_PADDING_SIZE,_mi_os_page_size());}else {@@ -152,10 +152,10 @@ bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq) {for (mi_page_t* page = pq->first; page != NULL; page = page->next) {mi_assert_internal(page->prev == prev);if (mi_page_is_in_full(page)) {- mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2);+ mi_assert_internal(mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2);}else if (mi_page_is_huge(page)) {- mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1);+ mi_assert_internal(mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1);}else {mi_assert_internal(mi_page_block_size(page) == pq->block_size);@@ -212,10 +212,10 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_if (size > MI_SMALL_SIZE_MAX) return;mi_page_t* page = pq->first;- if (pq->first == NULL) page = (mi_page_t*)&_mi_page_empty;+ if (pq->first == NULL) page = _mi_page_empty_get();// find index in the right direct page array- const size_t idx = _mi_wsize_from_size(size);+ const size_t idx = mi_wsize_from_size(size);mi_page_t** const pages_free = theap->pages_free_direct;if (pages_free[idx] == page) return; // already set@@ -232,7 +232,7 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_while( bin == mi_bin(prev->block_size) && prev > &theap->pages[0]) {prev--;}- start = 1 + _mi_wsize_from_size(prev->block_size);+ start = 1 + mi_wsize_from_size(prev->block_size);if (start > idx) start = idx;}@@ -341,6 +341,15 @@ static void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queumi_assert_internal(queue->first == page);}+static void mi_page_queue_move_to_back(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {+ mi_assert_internal(mi_page_theap(page) == theap);+ mi_assert_internal(mi_page_queue_contains(queue, page));+ if (queue->last == page) return;+ mi_page_queue_remove(queue, page);+ mi_page_queue_push_at_end(theap, queue, page);+ mi_assert_internal(queue->last == page);+}+static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) {mi_assert_internal(page != NULL);mi_assert_internal(from->count >= 1);@@ -421,38 +430,3 @@ static void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t// note: we could insert at the front to increase reuse, but it slows down certain benchmarks (like `alloc-test`)mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end of the `to` queue? */, page);}--// Only called from `mi_theap_absorb`.-size_t _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append) {- mi_assert_internal(mi_theap_contains_queue(theap,pq));- mi_assert_internal(pq->block_size == append->block_size);-- if (append->first==NULL) return 0;-- // set append pages to new theap and count- size_t count = 0;- for (mi_page_t* page = append->first; page != NULL; page = page->next) {- mi_page_set_theap(page, theap);- count++;- }- mi_assert_internal(count == append->count);-- if (pq->last==NULL) {- // take over afresh- mi_assert_internal(pq->first==NULL);- pq->first = append->first;- pq->last = append->last;- mi_theap_queue_first_update(theap, pq);- }- else {- // append to end- mi_assert_internal(pq->last!=NULL);- mi_assert_internal(append->first!=NULL);- pq->last->next = append->first;- append->first->prev = pq->last;- pq->last = append->last;- }- pq->count += append->count;-- return count;-}diff --git a/vendored/mimalloc/src/page.c b/vendored/mimalloc/src/page.cindex 97d2fc279..aa669cdd0 100644--- a/vendored/mimalloc/src/page.c+++ b/vendored/mimalloc/src/page.c@@ -15,6 +15,7 @@ terms of the MIT license. A copy of the license can be found in the file#include "mimalloc/internal.h"#include "mimalloc/atomic.h"#include "mimalloc/prim.h"+#include "mimalloc/prim-tls.h"/* -----------------------------------------------------------Definition of page queues for each block size@@ -41,7 +42,7 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page);#if (MI_DEBUG>=3)static size_t mi_page_list_count(mi_page_t* page, mi_block_t* head) {- mi_assert_internal(_mi_ptr_page(page->page_start) == page);+ mi_assert_internal(_mi_ptr_page(mi_page_start(page)) == page);const uint8_t* slice_start = mi_page_slice_start(page);mi_assert_internal(_mi_is_aligned(slice_start,MI_PAGE_ALIGN));size_t count = 0;@@ -82,8 +83,12 @@ static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) {static bool mi_page_is_valid_init(mi_page_t* page) {mi_assert_internal(mi_page_block_size(page) > 0);- mi_assert_internal(page->used <= page->capacity);+ mi_assert_internal(mi_page_used(page) <= page->capacity);mi_assert_internal(page->capacity <= page->reserved);++ mi_assert_internal(page->heap!=NULL);+ mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap);+ mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED);// const size_t bsize = mi_page_block_size(page);// uint8_t* start = mi_page_start(page);@@ -99,7 +104,7 @@ static bool mi_page_is_valid_init(mi_page_t* page) {mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t)));}}- #endif+ #endif#if !MI_TRACK_ENABLED && !MI_TSANmi_block_t* tfree = mi_page_thread_free(page);@@ -109,8 +114,9 @@ static bool mi_page_is_valid_init(mi_page_t* page) {#endifsize_t free_count = mi_page_list_count(page, page->free) + mi_page_list_count(page, page->local_free);- mi_assert_internal(page->used + free_count == page->capacity);+ mi_assert_internal(mi_page_used(page) + free_count == page->capacity);+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));return true;}@@ -123,6 +129,9 @@ bool _mi_page_is_valid(mi_page_t* page) {#endifif (!mi_page_is_abandoned(page)) {//mi_assert_internal(!_mi_process_is_initialized);+ mi_assert_internal(page->heap!=NULL);+ mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap);+ mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED);{mi_page_queue_t* pq = mi_page_queue_of(page);mi_assert_internal(mi_page_queue_contains(pq, page));@@ -130,10 +139,180 @@ bool _mi_page_is_valid(mi_page_t* page) {// mi_assert_internal(mi_theap_contains_queue(mi_page_theap(page),pq));}}+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));+ mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page));return true;}#endif+#if MI_STATS || MI_SAMPLE+// Gets the theap belonging to a page.+static mi_theap_t* mi_theap_of_page(mi_page_t* page) {+ mi_theap_t* theap = page->theap;+ if mi_unlikely(mi_page_thread_id(page) != _mi_prim_thread_id()) {+ theap = _mi_page_associated_theap_peek(page);+ }+ return theap;+}+#endif++#if MI_SAMPLE==1 /* for ==2, the countdown is already done at every `alloc.c:mi_page_alloc_zero` */+static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, size_t alloc_count)+{+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));+ mi_assert_internal(theap!=NULL);+ if (theap->sample_rate==0) return;++ const size_t last_alloc = mi_page_last_alloc(page);+ if (alloc_count <= last_alloc) return;++ // update countdown+ const size_t bsize = mi_page_usable_block_size(page); mi_assert_internal(bsize >= MI_PADDING_SIZE);+ const size_t alloc_diff = alloc_count - last_alloc;+ const uint64_t requested = (uint64_t)alloc_diff * (uint64_t)(bsize - MI_PADDING_SIZE);+ if (requested <= SIZE_MAX && theap->sample_countdown >= (size_t)requested) {+ theap->sample_countdown -= (size_t)requested;+ }+ else {+ theap->sample_requested += (requested - theap->sample_countdown);+ theap->sample_countdown = 0;+ }+}+#endif++#if MI_STATS+// Merge stats from the page into the corresponding theap or heap.+static void mi_theap_page_merge_stats(mi_theap_t* theap, const mi_page_t* page, size_t alloc_count, size_t free_count ) {+ // get heap (as the theap might be NULL)+ mi_heap_t* const heap = mi_page_heap(page);+ mi_assert_internal(theap == NULL || (theap->tld != NULL && _mi_thread_id() == theap->tld->thread_id));+ mi_theapx_stat_counter_increase(heap,theap,pages_stat_updates,1);+ mi_theapx_stat_counter_increase(heap,theap,pages_stat_update_count, alloc_count + free_count);++ // allocation sizes+ const size_t bsize = mi_page_usable_block_size(page);+ const uint64_t allocated = (uint64_t)alloc_count * (uint64_t)bsize;+ const uint64_t freed = (uint64_t)free_count * (uint64_t)bsize;+ #if MI_STATS==1+ const uint64_t requested = allocated - ((uint64_t)alloc_count * MI_PADDING_SIZE);+ #endif+ mi_assert_internal(allocated <= INT64_MAX); // safe to cast to int64_t for stats+ mi_assert_internal(freed <= INT64_MAX);++ // adjust stats+ if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {+ const size_t bin = _mi_bin(bsize);+ // allocations+ if (alloc_count > 0) {+ mi_theapx_stat_counter_increase(heap, theap, malloc_normal_count, alloc_count);+ mi_theapx_stat_increase(heap, theap, malloc_normal, allocated);+ mi_theapx_stat_increase(heap, theap, malloc_bins[bin], alloc_count);+ #if MI_STATS==1+ // use coarse total requested bytes+ mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested);+ #endif+ }+ // frees+ if (free_count > 0) {+ mi_theapx_stat_decrease(heap, theap, malloc_normal, freed);+ mi_theapx_stat_decrease(heap, theap, malloc_bins[bin], free_count);+ }+ }+ else {+ // allocations+ mi_assert_internal(alloc_count<=1);+ mi_assert_internal(free_count<=1);+ if (alloc_count > 0) {+ mi_theapx_stat_counter_increase(heap, theap, malloc_huge_count, alloc_count);+ mi_theapx_stat_increase(heap, theap, malloc_huge, allocated);+ #if MI_STATS==1+ // use coarse total requested bytes+ mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested);+ #endif+ }+ // frees+ if (free_count > 0) {+ mi_theapx_stat_decrease(heap, theap, malloc_huge, freed);+ }+ }+}++// Update stats for a page+static void mi_theap_page_update_stats(mi_theap_t* theap, mi_page_t* page) {+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));+ mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page));++ // get stat counts+ const size_t used = mi_page_used(page);+ const size_t alloc_count = mi_page_alloc_count(page);+ const size_t last_used = mi_page_last_used(page);+ mi_assert_internal(last_used + alloc_count >= used);+ mi_assert_internal(used <= UINT16_MAX);+ const size_t free_count = last_used + alloc_count - used;++ #if MI_SAMPLE==1 // for ==2 it is already counted in every `alloc.c:mi_page_alloc_zero`+ if (theap!=NULL) { mi_theap_adjust_sample_countdown(theap,page,alloc_count); }+ #endif++ // update stats?+ if (alloc_count + free_count == 0) {+ mi_assert_internal(last_used == used);+ return;+ }++ // reset the `alloc_count` (and `last_alloc`), and set `last_used` to `used`+ #if MI_SIZE_SIZE >= 8+ page->xused.used_alloc = (used << 32) | used;+ #else+ page->xused.used_alloc = used;+ page->xlast_used = (uint16_t)used;+ page->xlast_alloc = 0;+ #endif+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));+ mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page));++ mi_theap_page_merge_stats(theap, page, alloc_count, free_count);+}++void _mi_page_update_stats(mi_page_t* page) { // called on abandoned pages etc.+ mi_theap_page_update_stats(mi_theap_of_page(page),page);+}++#else+void _mi_page_update_stats(mi_page_t* page) {+ MI_UNUSED(page);+}+#endif++static void mi_page_update_sample_countdown(mi_page_t* page)+{+ // adjust count down+ const size_t alloc_count = mi_page_alloc_count(page);+ if (alloc_count==0) {+ return;+ }+ else if mi_unlikely(alloc_count>=0x8000) { // if the count could overflow, update stats so the counter is reset+ _mi_page_update_stats(page);+ }+ #if MI_SAMPLE==1 // if ==2 the countdown is already always counted in `alloc.c:mi_page_alloc_zero_ex`+ else {+ mi_theap_t* theap = mi_theap_of_page(page);+ if (theap==NULL) return;+ mi_theap_adjust_sample_countdown(theap,page,alloc_count);+ // update last_alloc to alloc_count+ mi_assert_internal(alloc_count <= UINT16_MAX);+ #if MI_SIZE_SIZE >= 8+ page->xused.used_alloc = (alloc_count << 48) | (page->xused.used_alloc & (~MI_ZU(0) >> 16));+ #else+ page->xlast_alloc = (uint16_t)alloc_count;+ #endif+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));+ mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page));++ }+ #endif+}+/* -----------------------------------------------------------Page collect the `local_free` and `thread_free` lists@@ -155,11 +334,11 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head)// if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free)if mi_unlikely(count > max_count) {- _mi_error_message(EFAULT, "corrupted thread-free list\n");+ _mi_error_message(EFAULT, "corrupted thread-free list (possibly due to a cross-thread double free)\n");return; // the thread-free items cannot be freed}// if `count > page->used` there was another kind memory corruption (either in the page meta-data or in the linked list)- else if mi_unlikely(count > page->used) {+ else if mi_unlikely(count > mi_page_used(page)) {_mi_error_message(EFAULT, "corrupted meta-data in thread-free list\n");return; // the thread-free items cannot be freed}@@ -170,8 +349,8 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head)// update counts nowmi_assert_internal(count <= UINT16_MAX);- mi_assert_internal(page->used >= (uint16_t)count);- page->used = page->used - (uint16_t)count;+ mi_assert_internal(mi_page_used(page) >= count);+ page->xused.used_alloc -= count; // page->used = page->used - (uint16_t)count;}// Collect the local `thread_free` list using an atomic exchange.@@ -192,14 +371,16 @@ static void mi_page_thread_free_collect(mi_page_t* page)mi_page_thread_collect_to_local(page, head);}+// returns `true` if after collection `mi_page_immediate_available` is true.-static bool mi_page_free_quick_collect(mi_page_t* page) {- if (page->free != NULL) return true;+static inline bool mi_page_free_quick_collect(mi_page_t* page) {+ if mi_likely(page->free != NULL) return true;if (page->local_free == NULL) return false;// move local_free to freepage->free = page->local_free;page->local_free = NULL;- page->free_is_zero = false;+ page->free_is_zero = false;+ mi_page_update_sample_countdown(page);return true;}@@ -229,19 +410,20 @@ void _mi_page_free_collect(mi_page_t* page, bool force) {page->local_free = NULL;page->free_is_zero = false;}- }-+ mi_page_update_sample_countdown(page);+ }mi_assert_internal(!force || page->local_free == NULL);}// Collect elements in the thread-free list starting at `head`. This is an optimized// version of `_mi_page_free_collect` to be used from `free.c:_mi_free_collect_mt` that avoids atomic access to `xthread_free`.+// returns a possibly updated expected value for the thread_free pointer.//// `head` must be in the `xthread_free` list. It will not collect `head` itself// so the `used` count is not fully updated in general. However, if the `head` is// the last remaining element, it will be collected and the used count will become `0` (so `mi_page_all_free` becomes true).-void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) {- if (head == NULL) return;+mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) {+ if (head == NULL) return NULL;mi_block_t* next = mi_block_next(page,head); // we cannot collect the head element itself as `page->thread_free` may point to it (and we want to avoid atomic ops)if (next != NULL) {mi_block_set_next(page, head, NULL);@@ -250,13 +432,18 @@ void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) {page->free = page->local_free;page->local_free = NULL;page->free_is_zero = false;- }+ mi_page_update_sample_countdown(page);+ }}- if (page->used == 1) {+ if (mi_page_used(page) == 1) {// all elements are free'd since we skipped the `head` element itselfmi_assert_internal(mi_tf_block(mi_atomic_load_relaxed(&page->xthread_free)) == head);mi_assert_internal(mi_block_next(page,head) == NULL);_mi_page_free_collect(page, false); // collect the final element+ return NULL;+ }+ else {+ return head;}}@@ -265,24 +452,6 @@ void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) {Page fresh and retire----------------------------------------------------------- */-/*-// called from segments when reclaiming abandoned pages-void _mi_page_reclaim(mi_theap_t* theap, mi_page_t* page) {- // mi_page_set_theap(page, theap);- // _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after theap is set)- _mi_page_free_collect(page, false); // ensure used count is up to date-- mi_assert_expensive(mi_page_is_valid_init(page));- // mi_assert_internal(mi_page_theap(page) == theap);- // mi_assert_internal(mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE);-- // TODO: push on full queue immediately if it is full?- mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);- mi_page_queue_push(theap, pq, page);- mi_assert_expensive(_mi_page_is_valid(page));-}-*/-// called from `mi_free` on a reclaim, and fresh_alloc if we get an abandoned pagevoid _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page){@@ -293,6 +462,7 @@ void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page)mi_page_set_theap(page,theap);_mi_page_free_collect(page, false); // ensure used count is up to date+mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);mi_page_queue_push_at_end(theap, pq, page);mi_assert_expensive(_mi_page_is_valid(page));@@ -309,7 +479,7 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {mi_page_set_theap(page, NULL);page->theap = theap; // don't actually set theap to NULL so we can reclaim_on_free within the same theap_mi_arenas_page_abandon(page, theap);- _mi_arenas_collect(false, false, theap->tld); // allow purging+ // _mi_arenas_collect(false, false, theap->tld); // allow purging}}@@ -331,7 +501,9 @@ static mi_page_t* mi_page_fresh_alloc(mi_theap_t* theap, mi_page_queue_t* pq, siif (!mi_page_immediate_available(page)) {if (mi_page_is_expandable(page)) {if (!mi_page_extend_free(theap, page)) {- return NULL; // cannot commit+ // cannot commit+ _mi_page_abandon(page,pq);+ return NULL;};}else {@@ -393,6 +565,7 @@ static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) {// put full pages in a theap local queue (this is for theaps that cannot abandon, for example, if the theap can be destroyed)mi_page_queue_enqueue_from(&mi_page_theap(page)->pages[MI_BIN_FULL], pq, page);_mi_page_free_collect(page, false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set+ _mi_page_update_stats(page); // we must update stats here as mi_theap_collect does not normally visit full pages}}@@ -416,11 +589,11 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) {mi_theap_t* theap = mi_page_theap(page); mi_assert_internal(theap!=NULL);mi_page_set_theap(page,NULL);_mi_arenas_page_free(page, theap);- _mi_arenas_collect(false, false, theap->tld); // allow purging+ // _mi_arenas_collect(false, false, theap->tld); // allow purging}-#define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX // should be less than size for MI_BIN_HUGE-#define MI_RETIRE_CYCLES (16)+#define MI_RETIRE_CYCLES (16) /* keep a retired page around for about 16 "admin cycles" before free'ing it */+#define MI_RETIRE_MAX_PAGES (3) /* keep at most N pages per size bin as retired */// Retire a page with no more used blocks// Important to not retire too quickly though as new@@ -446,12 +619,10 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept {mi_page_queue_t* pq = mi_page_queue_of(page);#if MI_RETIRE_CYCLES > 0const size_t bsize = mi_page_block_size(page);- if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue?- if (pq->last==page && pq->first==page) { // the only page in the queue?+ if mi_likely( pq->count <= MI_RETIRE_MAX_PAGES && !mi_page_queue_is_special(pq)) { // not full or huge queue?+ if (pq->count==1 || bsize < MI_SMALL_SIZE_MAX) {mi_theap_t* theap = mi_page_theap(page);- #if MI_STAT>0mi_theap_stat_counter_increase(theap, pages_retire, 1);- #endifpage->retire_expire = (bsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4);mi_assert_internal(pq >= theap->pages);const size_t index = pq - theap->pages;@@ -460,42 +631,13 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept {if (index > theap->page_retired_max) theap->page_retired_max = index;mi_assert_internal(mi_page_all_free(page));return; // don't free after all- }+ }}#endif_mi_page_free(page, pq);}-// free retired pages: we don't need to look at the entire queues-// since we only retire pages that are at the head position in a queue.-void _mi_theap_collect_retired(mi_theap_t* theap, bool force) {- size_t min = MI_BIN_FULL;- size_t max = 0;- for(size_t bin = theap->page_retired_min; bin <= theap->page_retired_max; bin++) {- mi_page_queue_t* pq = &theap->pages[bin];- mi_page_t* page = pq->first;- if (page != NULL && page->retire_expire != 0) {- if (mi_page_all_free(page)) {- page->retire_expire--;- if (page->retire_expire == 0 || force) {- _mi_page_free(page, pq);- }- else {- // keep retired, update min/max- if (bin < min) min = bin;- if (bin > max) max = bin;- }- }- else {- page->retire_expire = 0;- }- }- }- theap->page_retired_min = min;- theap->page_retired_max = max;-}-/*static void mi_theap_collect_full_pages(mi_theap_t* theap) {// note: normally full pages get immediately abandoned and the full queue is always empty// this path is only used if abandoning is disabled due to a destroy-able theap or options@@ -516,7 +658,47 @@ static void mi_theap_collect_full_pages(mi_theap_t* theap) {page = next;}}-*/++static void mi_page_try_retire(mi_page_queue_t* pq, mi_page_t* page, size_t bin, bool force, size_t* min, size_t* max) {+ mi_assert_internal(page!=NULL && page->retire_expire!=0);+ if (mi_page_all_free(page)) {+ page->retire_expire--;+ if (page->retire_expire == 0 || force) {+ _mi_page_free(page, pq);+ }+ else {+ // keep retired, update min/max+ if (bin < *min) *min = bin;+ if (bin > *max) *max = bin;+ }+ }+ else {+ page->retire_expire = 0;+ }+}++// free retired pages: we don't need to look at the entire queues+// since we only retire pages that are at the head position in a queue.+void _mi_theap_collect_retired(mi_theap_t* theap, bool force) {+ size_t min = MI_BIN_FULL;+ size_t max = 0;+ for(size_t bin = theap->page_retired_min; bin <= theap->page_retired_max; bin++) {+ mi_page_queue_t* pq = &theap->pages[bin];+ mi_page_t* page = pq->first;+ for (int i = 0; i<MI_RETIRE_MAX_PAGES && page!=NULL && page->retire_expire!=0; i++) {+ mi_page_t* next = page->next;+ mi_page_try_retire(pq,page,bin,force,&min,&max);+ page = next;+ }+ }+ theap->page_retired_min = min;+ theap->page_retired_max = max;+ if (!theap->allow_page_abandon) {+ mi_theap_collect_full_pages(theap);+ }+}++/* -----------------------------------------------------------@@ -530,7 +712,7 @@ static void mi_theap_collect_full_pages(mi_theap_t* theap) {#define MI_MIN_SLICES (2)static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* const page, const size_t bsize, const size_t extend) {- #if (MI_SECURE<3)+ #if (MI_SECURE < 2)mi_assert_internal(page->free == NULL);mi_assert_internal(page->local_free == NULL);#endif@@ -557,15 +739,15 @@ static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t*// and initialize the free list by randomly threading through them// set up first element- const uintptr_t r = _mi_theap_random_next(theap);+ const size_t r = _mi_theap_random_next(theap);size_t current = r % slice_count;counts[current]--;mi_block_t* const free_start = blocks[current];// and iterate through the rest; use `random_shuffle` for performance- uintptr_t rnd = _mi_random_shuffle(r|1); // ensure not 0+ size_t rnd = _mi_random_shuffle(r|1); // ensure not 0for (size_t i = 1; i < extend; i++) {- // call random_shuffle only every INTPTR_SIZE rounds- const size_t round = i%MI_INTPTR_SIZE;+ // call random_shuffle only every SIZE_SIZE rounds+ const size_t round = i%MI_SIZE_SIZE;if (round == 0) rnd = _mi_random_shuffle(rnd);// select a random next slice indexsize_t next = ((rnd >> 8*round) & (slice_count-1));@@ -587,7 +769,7 @@ static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t*static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend){- #if (MI_SECURE<3)+ #if (MI_SECURE < 2)mi_assert_internal(page->free == NULL);mi_assert_internal(page->local_free == NULL);#endif@@ -614,7 +796,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, coPage initialize and extend the capacity----------------------------------------------------------- */-#define MI_MAX_EXTEND_SIZE (4*1024) // heuristic, one OS page seems to work well.+#define MI_MAX_EXTEND_SIZE (8*1024) // heuristic, one or two OS pages seems to work well.#if (MI_SECURE>=2)#define MI_MIN_EXTEND (8*MI_SECURE) // extend at least by this many#else@@ -628,7 +810,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co// extra test in malloc? or cache effects?)static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {mi_assert_expensive(mi_page_is_valid_init(page));- #if (MI_SECURE<3)+ #if (MI_SECURE < 2)mi_assert(page->free == NULL);mi_assert(page->local_free == NULL);if (page->free != NULL) return true;@@ -638,10 +820,8 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {size_t page_size;//uint8_t* page_start =mi_page_area(page, &page_size);- #if MI_STAT>0mi_theap_stat_counter_increase(theap, pages_extended, 1);- #endif-+// calculate the extend countconst size_t bsize = mi_page_block_size(page);size_t extend = (size_t)page->reserved - page->capacity;@@ -661,15 +841,30 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {mi_assert_internal(extend < (1UL<<16));// commit on demand?- if (page->slice_committed > 0) {+ const size_t slice_committed = mi_page_slice_committed(page);+ if (slice_committed > 0) {+ // reduce extend if it commits more than an arena slice+ if ((extend * bsize) > MI_ARENA_SLICE_SIZE) {+ extend = _mi_divide_up(MI_ARENA_SLICE_SIZE, bsize);+ }+ // commit required sizeconst size_t needed_size = (page->capacity + extend)*bsize;- const size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), MI_PAGE_MIN_COMMIT_SIZE );- if (needed_commit > page->slice_committed) {- mi_assert_internal(((needed_commit - page->slice_committed) % _mi_os_page_size()) == 0);- if (!_mi_os_commit(mi_page_slice_start(page) + page->slice_committed, needed_commit - page->slice_committed, NULL)) {+ mi_assert_internal(needed_size <= page_size);+ size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), mi_page_min_commit_size());+ #if MI_SECURE>=5+ // the previous alignup could extend the commit into the guard page; re-adjust if needed+ const size_t page_size_commit = _mi_align_up( mi_page_slice_offset_of(page, page_size), _mi_os_page_size() );+ if (needed_commit > page_size_commit) {+ needed_commit = page_size_commit;+ }+ #endif+ if (needed_commit > slice_committed) {+ mi_assert_internal(((needed_commit - slice_committed) % _mi_os_page_size()) == 0);+ if (!_mi_os_commit(_mi_theap_subproc(theap), mi_page_slice_start(page) + slice_committed, needed_commit - slice_committed, NULL)) {return false;}- page->slice_committed = needed_commit;+ mi_assert_internal(needed_commit <= UINT16_MAX * _mi_os_page_size());+ page->slice_pcommitted = (uint16_t)(needed_commit / _mi_os_page_size());}}@@ -682,9 +877,7 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {}// enable the new free listpage->capacity += (uint16_t)extend;- #if MI_STAT>0mi_theap_stat_increase(theap, page_committed, extend * bsize);- #endifmi_assert_expensive(mi_page_is_valid_init(page));return true;}@@ -693,8 +886,8 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) {mi_assert(page != NULL);mi_assert(theap!=NULL);- page->heap = (_mi_is_heap_main(_mi_theap_heap(theap)) ? NULL : _mi_theap_heap(theap)); // faster for `mi_page_associated_theap`- mi_page_set_theap(page, theap);+ // page->heap = (_mi_is_heap_main(_mi_theap_heap(theap)) ? NULL : _mi_theap_heap(theap)); // faster for `mi_page_associated_theap`+ // mi_page_set_theap(page, theap);size_t page_size;uint8_t* page_start = mi_page_area(page, &page_size); MI_UNUSED(page_start);@@ -703,20 +896,25 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) {mi_assert_internal(page->reserved > 0);#if (MI_PADDING || MI_ENCODE_FREELIST)page->keys[0] = _mi_theap_random_next(theap);+ #if MI_PAGE_KEY_COUNT==2page->keys[1] = _mi_theap_random_next(theap);#endif+ #endif#if MI_DEBUG>2if (page->memid.initially_zero) {- mi_track_mem_defined(page->page_start, mi_page_committed(page));+ mi_track_mem_defined(mi_page_start(page), mi_page_committed(page));mi_assert_expensive(mi_mem_is_zero(page_start, mi_page_committed(page)));}#endif+ mi_assert_internal(page->heap != NULL);+ mi_assert_internal(page->heap == _mi_theap_heap(theap));mi_assert_internal(page->theap!=NULL);mi_assert_internal(page->theap == mi_page_theap(page));mi_assert_internal(page->capacity == 0);mi_assert_internal(page->free == NULL);- mi_assert_internal(page->used == 0);+ mi_assert_internal(mi_page_used(page) == 0);+ mi_assert_internal(page->xused.used_alloc == 0);mi_assert_internal(mi_page_is_owned(page));mi_assert_internal(page->xthread_free == 1);mi_assert_internal(page->next == NULL);@@ -725,8 +923,10 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) {mi_assert_internal(!mi_page_has_interior_pointers(page));#if (MI_PADDING || MI_ENCODE_FREELIST)mi_assert_internal(page->keys[0] != 0);+ #if MI_PAGE_KEY_COUNT==2mi_assert_internal(page->keys[1] != 0);#endif+ #endifmi_assert_expensive(mi_page_is_valid_init(page));// initialize an initial free list@@ -749,8 +949,9 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap,long page_full_retain = (pq->block_size > MI_SMALL_MAX_OBJ_SIZE ? 0 : theap->page_full_retain); // only retain small pagesmi_page_t* page_candidate = NULL; // a page with free spacemi_page_t* page = pq->first;+ mi_page_t* const last = pq->last;- while (page != NULL)+ while (page!=NULL){mi_page_t* next = page->next; // remember next (as this page can move to another queue)count++;@@ -774,6 +975,10 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap,mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));mi_page_to_full(page, pq);}+ else if (page!=last && pq->last!=page) {+ // avoid revisiting this page for a while+ mi_page_queue_move_to_back(theap, pq, page);+ }}else {// the page has free space, make it a candidate@@ -787,7 +992,7 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap,page_candidate = page;}// prefer to reuse fuller pages (in the hope the less used page gets freed)- else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) {+ else if (mi_page_used(page) >= mi_page_used(page_candidate) && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) {page_candidate = page;}// if we find a non-expandable candidate, or searched for N pages, return with the best candidate@@ -809,8 +1014,8 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap,mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));mi_page_to_full(page, pq);#endif-- page = next;+ if (page==last) { page=NULL; } // don't revisit earlier pages that moved to the back+ else { page = next; }} // for each pagemi_theap_stat_counter_increase(theap, page_searches, count);@@ -832,7 +1037,7 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap,if (page == NULL) {_mi_theap_collect_retired(theap, false); // perhaps make a page available- page = mi_page_fresh(theap, pq);+ page = mi_page_fresh(theap, pq);mi_assert_internal(page == NULL || mi_page_immediate_available(page));if (page == NULL && first_try) {// out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again@@ -853,63 +1058,48 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap,return page;}---// Find a page with free blocks of `size`.-static mi_page_t* mi_find_free_page(mi_theap_t* theap, mi_page_queue_t* pq) {- // mi_page_queue_t* pq = mi_page_queue(theap, size);+// Look for a page with free blocks of `size` but don't try to search or allocate+static inline mi_page_t* mi_page_queue_lookup_free_first(mi_theap_t* theap, mi_page_queue_t* pq) {mi_assert_internal(!mi_page_queue_is_huge(pq));-- // check the first page: we even do this with candidate search or otherwise we re-search every timemi_page_t* page = pq->first;- if mi_likely(page != NULL && mi_page_free_quick_collect(page)) {+ if mi_likely(page!=NULL && mi_page_free_quick_collect(page)) {+ // fast path#if (MI_SECURE>=2) // in secure mode, we extend half the time to increase randomnessif (page->capacity < page->reserved && ((_mi_theap_random_next(theap) & 1) == 1)) {(void)mi_page_extend_free(theap, page); // ok if this failsmi_assert_internal(mi_page_immediate_available(page));}+ #else+ MI_UNUSED(theap);#endifpage->retire_expire = 0;- return page; // fast path+ mi_assert_internal(mi_page_immediate_available(page));+ mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));+ mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);+ return page;}else {- return mi_page_queue_find_free_ex(theap, pq, true);+ return NULL;}}+// Find a page with free blocks of `size`.+static inline mi_page_t* mi_page_queue_find_free(mi_theap_t* theap, mi_page_queue_t* pq) {+ // mi_page_queue_t* pq = mi_page_queue(theap, size);+ mi_assert_internal(!mi_page_queue_is_huge(pq));-/* ------------------------------------------------------------ Users can register a deferred free function called- when the `free` list is empty. Since the `local_free`- is separate this is deterministically called after- a certain number of allocations.------------------------------------------------------------ */--// The program should only install a single deferred free handler before doing allocation.-static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers)-static _Atomic(void*) deferred_arg;--void _mi_deferred_free(mi_theap_t* theap, bool force) {- theap->heartbeat++;- mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free);- if (fun != NULL && !theap->tld->recurse) {- theap->tld->recurse = true;- void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg);- fun(force, theap->heartbeat, arg);- theap->tld->recurse = false;- }-}--void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept {- mi_atomic_store_ptr_release(void,&deferred_arg, arg);- mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn);+ // check the first page: we even do this with candidate search or otherwise we re-search every time+ mi_page_t* page = mi_page_queue_lookup_free_first(theap,pq);+ if (page==NULL) {+ page = mi_page_queue_find_free_ex(theap, pq, true);+ if (page==NULL) return NULL;+ }+ mi_assert_internal(mi_page_immediate_available(page));+ mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));+ mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);+ return page;}--/* ------------------------------------------------------------ General allocation------------------------------------------------------------ */-// Huge pages contain just one block, and the segment contains just that page.// Huge pages are also use if the requested alignment is very large (> MI_BLOCK_ALIGNMENT_MAX)// so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`.@@ -932,80 +1122,164 @@ static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t pagemi_assert_internal(mi_page_is_abandoned(page));mi_page_set_theap(page, NULL);#endif- mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page));- mi_theap_stat_counter_increase(theap, malloc_huge_count, 1);+ // mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page));+ // mi_theap_stat_counter_increase(theap, malloc_huge_count, 1);}return page;}-// Allocate a page// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_alignment) mi_attr_noexcept {const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`- if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) {+ if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) { // TODO: remove as we check in generic_fallback ?_mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size);return NULL;}mi_page_queue_t* pq = mi_page_queue(theap, (huge_alignment > 0 ? MI_LARGE_MAX_OBJ_SIZE+1 : size));+ mi_page_t* page;// huge allocation?if mi_unlikely(mi_page_queue_is_huge(pq) || req_size > MI_MAX_ALLOC_SIZE) {- return mi_huge_page_alloc(theap,size,huge_alignment,pq);+ page = mi_huge_page_alloc(theap,size,huge_alignment,pq);}else {// otherwise find a page with free blocks in our size segregated queues#if MI_PADDINGmi_assert_internal(size >= MI_PADDING_SIZE);#endif- return mi_find_free_page(theap, pq);+ page = mi_page_queue_find_free(theap,pq);}+ if (page==NULL) return NULL;+ mi_assert_internal(mi_page_block_size(page) >= size);+ mi_assert_internal(mi_page_immediate_available(page));+ mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));+ mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);+ return page;}-// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed.-// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.-// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for-// very large requested alignments in which case we use a huge singleton page.-// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit)-// to use 4 parameters which compiles better on msvc for the malloc fast path.-void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable) mi_attr_noexcept-{- const bool zero = ((zero_huge_alignment & 1) != 0);- const size_t huge_alignment = (zero_huge_alignment & ~1);- #if !MI_THEAP_INITASNULL- mi_assert_internal(theap != NULL);- #endif+/* -----------------------------------------------------------+ Users can register a deferred free function called+ when the `free` list is empty. Since the `local_free`+ is separate this is deterministically called after+ a certain number of allocations.+----------------------------------------------------------- */- // initialize if necessary- if mi_unlikely(!mi_theap_is_initialized(theap)) {- if (theap==&_mi_theap_empty_wrong) {- // we were unable to allocate a theap for a first-class heap- return NULL;- }- // otherwise we initialize the thread and its default theap- mi_thread_init();- theap = _mi_theap_default();- if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; }- mi_assert_internal(_mi_theap_default()==theap);+// The program should only install a single deferred free handler before doing allocation.+static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers)+static _Atomic(void*) deferred_arg;++void _mi_deferred_free(mi_theap_t* theap, bool force) {+ theap->heartbeat++;+ mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free);+ if (fun != NULL && !theap->tld->recurse) {+ theap->tld->recurse = true;+ void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg);+ fun(force, theap->heartbeat, arg);+ theap->tld->recurse = false;+ }+}++void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept {+ mi_atomic_store_ptr_release(void,&deferred_arg, arg);+ mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn);+}+++/* -----------------------------------------------------------+ Admin+----------------------------------------------------------- */+static mi_theap_t* mi_theap_init(mi_theap_t* theap) {+ if mi_likely(mi_theap_is_initialized(theap)) return theap;+ if (theap==&_mi_theap_empty_wrong) {+ // we were unable to allocate a theap for a first-class heap+ return NULL;}+ // otherwise we initialize the thread and its default theap+ theap = _mi_thread_init();+ if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; }+ mi_assert_internal(mi_theap_is_initialized(theap));+ return theap;+}++static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap)+{+ theap = mi_theap_init(theap);+ if (theap==NULL) return NULL;mi_assert_internal(mi_theap_is_initialized(theap));// do administrative tasks every N generic mallocs- if mi_unlikely(++theap->generic_count >= 1000) {+ if mi_unlikely(theap->generic_count >= 1000) {theap->generic_collect_count += theap->generic_count;theap->generic_count = 0;- // call potential deferred free routines- _mi_deferred_free(theap, false);- // free retired pages- _mi_theap_collect_retired(theap, false);- // collect every once in a while (10000 by default)+ // check if the profiler is enabled+ mi_heap_t* const heap = _mi_theap_heap(theap);+ mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler);+ const bool prof_enabled = (prof!=NULL && mi_profiler_is_enabled(prof));+ if (prof_enabled && theap->profile_sample_rate==0) {+ _mi_theap_set_profile_sample_rate(theap,mi_max(1,prof->initial_sample_rate)); // start profiling+ }+ else if (!prof_enabled && theap->profile_sample_rate!=0) {+ _mi_theap_set_profile_sample_rate(theap,0); // stop profiling+ }++ // do a full theap collect every once in a while (10000 by default)const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L);if (theap->generic_collect_count >= generic_collect) {theap->generic_collect_count = 0;mi_theap_collect(theap, false /* force? */);}+ // else if (theap->sample_rate != 0) { // update stats more aggressively if we are sampling+ // mi_theap_collect(theap, false /* force? */); // TODO: make specialized mi_theap_collect_update_stats ?+ // }+ else {+ // otherwise we do a mini-collect+ _mi_deferred_free(theap, false); // call potential deferred free routines+ _mi_theap_collect_retired(theap, false); // free retired pages+ }+ }+ return theap;+}++/* -----------------------------------------------------------+ Generic allocation+----------------------------------------------------------- */++static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage)+{+ // initialize if necessary+ theap = mi_malloc_generic_admin(theap);+ if (theap==NULL) return NULL;++ // check allocation size+ const size_t req_size = size - MI_PADDING_SIZE;+ if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) {+ _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size);+ return NULL;+ }++ // take a sample?+ bool sample_countdown_is_adjusted = false;+ if mi_unlikely(mi_theap_should_sample(theap,req_size)) {+ if (huge_alignment==0 && theap->sample_rate!=0) {+ return _mi_theap_malloc_sampled(theap,req_size,zero,ppage);+ }+ mi_assert_internal(!_mi_is_empty_theap(theap)); // cannot write to the empty theap+ mi_assert_internal(req_size <= MI_SAMPLE_COUNTDOWN_MAX);+ if (theap->sample_rate==0) {+ theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // reset the countdown counter+ }+ #if MI_SAMPLE==2+ else {+ // huge_alignment!=0 so we need to adjust the countdown+ mi_assert_internal(huge_alignment!=0);+ mi_assert_internal(theap->sample_countdown <= SIZE_MAX - req_size);+ sample_countdown_is_adjusted = true;+ theap->sample_countdown += req_size; // adjust such that after `_mi_page_malloc_zero` the countdown is correct again+ }+ #endif}// find (or allocate) a page of the right size@@ -1016,9 +1290,9 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm}if mi_unlikely(page == NULL) { // out of memory- const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`+ if (sample_countdown_is_adjusted) { theap->sample_countdown -= req_size; } // unadjust again if we failed to find a page_mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size);- return NULL;+ return NULL;}mi_assert_internal(mi_page_immediate_available(page));@@ -1026,14 +1300,72 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignmmi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);- // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc)- if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }+ // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc_zero)+ if (ppage!=NULL) { *ppage = page; }void* const p = _mi_page_malloc_zero(theap,page,size,zero);mi_assert_internal(p != NULL);-+ mi_page_update_sample_countdown(page);+// move full pages to the full queue- if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE && mi_page_is_full(page)) {- mi_page_to_full(page, mi_page_queue_of(page));+ // this will also call _mi_page_update_stats for huge pages+ if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE) {+ if (mi_page_is_full(page)) {+ mi_page_to_full(page, mi_page_queue_of(page));+ }+ }+ return p;+}+++// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed.+// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.+// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for+// very large requested alignments in which case we use a huge singleton page.+// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit)+// to use 4 parameters which compiles better on msvc for the malloc fast path.+void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept+{+ #if !MI_THEAP_INITASNULL+ mi_assert_internal(theap != NULL);+ #endif+ const bool zero = ((zero_huge_alignment & 1) != 0);+ const size_t huge_alignment = (zero_huge_alignment & ~1);+ mi_page_t* page = NULL;++ // fast path objects that fit in a small page+ if mi_likely(mi_theap_is_initialized(theap) && ++theap->generic_count < 1000 && huge_alignment==0) {+ const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`+ if (req_size < MI_SMALL_MAX_OBJ_SIZE)+ {+ #if MI_SAMPLE+ if mi_likely(!mi_theap_should_sample(theap,req_size)) // ensure we don't need to take a sample+ #endif+ {+ mi_page_queue_t* pq = mi_page_queue(theap, size);+ mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq));+ page = mi_page_queue_find_free(theap,pq);+ // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE);+ if (page!=NULL) {+ if (ppage!=NULL) { *ppage = page; }+ mi_assert_internal(mi_page_immediate_available(page)); // we should never recurse in _mi_page_malloc_zero+ return _mi_page_malloc_zero(theap,page,size,zero);+ }+ }+ }}+ // otherwise fallback+ return mi_malloc_generic_fallback(theap,size,zero,huge_alignment,ppage);+}++void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept {+ theap = mi_theap_init(theap);+ if (theap==NULL) return NULL;+ const size_t sample_rate = theap->sample_rate;+ const size_t sample_countdown = theap->sample_countdown;+ theap->sample_rate = 0; // prevent a recursive call to mi_theap_malloc_sampled from _mi_malloc_generic+ theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX;+ void* p = _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage);+ theap->sample_rate = sample_rate;+ theap->sample_countdown = sample_countdown;return p;}diff --git a/vendored/mimalloc/src/prim/emscripten/prim.c b/vendored/mimalloc/src/prim/emscripten/prim.cindex 1ba0936e0..992965d56 100644--- a/vendored/mimalloc/src/prim/emscripten/prim.c+++ b/vendored/mimalloc/src/prim/emscripten/prim.c@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen, Alon Zakai+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen, Alon ZakaiThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -61,7 +61,6 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config) {extern void emmalloc_free(void*);int _mi_prim_free(void* addr, size_t size) {- if (size==0) return 0;emmalloc_free(addr);return 0;}@@ -77,7 +76,7 @@ extern void* emmalloc_memalign(size_t alignment, size_t size);int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);*is_large = false;- // TODO: Track the highest address ever seen; first uses of it are zeroes.+ // todo: Track the highest address ever seen; first uses of it are zeroes.// That assumes no one else uses sbrk but us (they could go up,// scribble, and then down), but we could assert on that perhaps.*is_zero = false;@@ -101,7 +100,7 @@ int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commint _mi_prim_commit(void* addr, size_t size, bool* is_zero) {MI_UNUSED(addr); MI_UNUSED(size);- // See TODO above.+ // See todo above.*is_zero = false;return 0;}@@ -155,7 +154,8 @@ size_t _mi_prim_numa_node_count(void) {#include <emscripten/html5.h>mi_msecs_t _mi_prim_clock_now(void) {- return emscripten_date_now();+ // todo: use a monotonic clock instead+ return emscripten_date_now();}@@ -185,12 +185,12 @@ void _mi_prim_out_stderr( const char* msg) {// Environment//-----------------------------------------------------------------bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {+int _mi_prim_getenv(const char* name, char* result, size_t result_size) {// For code size reasons, do not support environ customization for now.MI_UNUSED(name);MI_UNUSED(result);MI_UNUSED(result_size);- return false;+ return 0; // not found}@@ -212,7 +212,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {// use pthread local storage keys to detect thread ending// (and used with MI_TLS_PTHREADS for the default theap)-pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);+pthread_key_t _mi_heap_default_key = MI_PTHREAD_KEY_INVALID;static void mi_pthread_done(void* value) {if (value!=NULL) {@@ -221,18 +221,20 @@ static void mi_pthread_done(void* value) {}void _mi_prim_thread_init_auto_done(void) {- mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));+ mi_assert_internal(_mi_heap_default_key == MI_PTHREAD_KEY_INVALID);pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);}void _mi_prim_thread_done_auto_done(void) {- if (_mi_heap_default_key != (pthread_key_t)(-1)) { // do not leak the key, see issue #809- pthread_key_delete(_mi_heap_default_key);+ pthread_key_t key = _mi_heap_default_key;+ if (key != MI_PTHREAD_KEY_INVALID) { // do not leak the key, see issue #809+ _mi_heap_default_key = MI_PTHREAD_KEY_INVALID;+ pthread_key_delete(key);}}void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {- if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD+ if (_mi_heap_default_key != MI_PTHREAD_KEY_INVALID) { // can happen during recursive invocation on freeBSDpthread_setspecific(_mi_heap_default_key, theap);}}diff --git a/vendored/mimalloc/src/prim/osx/alloc-override-zone.c b/vendored/mimalloc/src/prim/osx/alloc-override-zone.cindex fdbe095c2..e7fc8306b 100644--- a/vendored/mimalloc/src/prim/osx/alloc-override-zone.c+++ b/vendored/mimalloc/src/prim/osx/alloc-override-zone.c@@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file#include "mimalloc.h"#include "mimalloc/internal.h"-+#include "mimalloc/prim-tls.h" // _mi_thread_is_initialized#if defined(MI_MALLOC_OVERRIDE)#if !defined(__APPLE__)@@ -71,11 +71,12 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) {}static void zone_free(malloc_zone_t* zone, void* p) {- if (mi_any_heap_contains(p)) {- mi_free(p); // with the page_map and pagemap_commit=1 we can use the regular free- }- else if (!is_mimalloc_zone(zone)) { // can happen due to interpose- zone->free(zone,p);+ // during C++ thread shutdown `_pthread_tsd_cleanup` may call `zone_free`+ // after mimalloc mi_thread_done, and also on a pointer that was allocated in another subproc.+ if mi_unlikely(!mi_cfree(p)) {+ if (!is_mimalloc_zone(zone)) { // can happen due to interpose+ zone->free(zone,p);+ }}}@@ -134,7 +135,6 @@ static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) {return mi_is_in_heap_region(p);}-/* ------------------------------------------------------Introspection members------------------------------------------------------ */@@ -194,6 +194,14 @@ static boolean_t intro_zone_locked(malloc_zone_t* zone) {return false;}+// Required whenever the zone advertises version >= 9: macOS calls this from the+// atfork_child handler (_malloc_fork_child) without a NULL check. mimalloc keeps+// no zone-level locks that need reinitializing after fork, so a no-op is safe.+// Leaving it NULL makes the forked child jump to address 0 and crash while forking.+static void intro_reinit_lock(malloc_zone_t* zone) {+ MI_UNUSED(zone);+}+/* ------------------------------------------------------At process start, override the default allocator@@ -218,6 +226,9 @@ static malloc_introspection_t mi_introspect = {#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__).statistics = &intro_statistics,.zone_locked = &intro_zone_locked,+#endif+#if defined(MAC_OS_X_VERSION_10_12) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_12) && !defined(__ppc__)+ .reinit_lock = &intro_reinit_lock,#endif};diff --git a/vendored/mimalloc/src/prim/prim-tls.c b/vendored/mimalloc/src/prim/prim-tls.cnew file mode 100644index 000000000..1b79321a0--- /dev/null+++ b/vendored/mimalloc/src/prim/prim-tls.c@@ -0,0 +1,252 @@+/* ----------------------------------------------------------------------------+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen+This is free software; you can redistribute it and/or modify it under the+terms of the MIT license. A copy of the license can be found in the file+"LICENSE" at the root of this distribution.+-----------------------------------------------------------------------------*/++#include "mimalloc.h"+#include "mimalloc/internal.h"+#include "mimalloc/prim.h"+#include "mimalloc/prim-tls.h"++// --------------------------------------------------------------------------+// Implement fast access to the thread local storage for `_mi_theap_default()`+// and `mi_theap_cached()`. See `include/mimalloc/prim-tls.h` for more info+// on the TLS models.+// --------------------------------------------------------------------------++void _mi_tls_slots_init(void);+void _mi_tls_slots_done(void);+void _mi_theap_default_set(mi_theap_t* theap);+void _mi_theap_cached_set(mi_theap_t* theap);+++#if MI_TLS_MODEL_LOCAL+// the thread-local main theap for allocation+mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default = (mi_theap_t*)&_mi_theap_empty;+// the last used non-main theap+mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached = (mi_theap_t*)&_mi_theap_empty;+#endif++mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper = NULL;++mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {+ const mi_threadid_t tid = _mi_prim_thread_id();+ mi_assert_internal( (tid & MI_PAGE_FLAG_MASK) == 0 ); // mimalloc reserves the bottom 2 bits+ return tid;+}++// ----------------------------------------------------------------------------+// Setting the default and cached theap+// ----------------------------------------------------------------------------++#if MI_TLS_MODEL_WIN32++// If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed)+// See <https://en.wikipedia.org/wiki/Win32_Thread_Information_Block> for the offsets.+#if MI_SIZE_SIZE==4+#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_SIZE_SIZE)+#else+#define MI_TLS_DIRECT_FIRST (0x1480 / MI_SIZE_SIZE)+#endif+#define MI_TLS_DIRECT_SLOTS (64)+#define MI_TLS_EXPANSION_SLOTS (1024)++#if !MI_WIN_DIRECT_TLS+// We initially use the last of the expansion slots as the default NULL.+// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc+// before we are initialized :-( (but this seems quite unlikely).+// (todo: another approach could be to use slot 7 (EnvironmentPointer) as the initial slot as that seems to be always NULL)+#define MI_TLS_INITIAL_SLOT MI_TLS_EXPANSION_SLOT+#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1)+#else+// With direct tls we need an initial NULL slot outside the expansion slots+#define MI_TLS_INITIAL_SLOT (5) // Arbitrary user pointer+#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1) // unused+#endif++// in case of errors assign fixed slots (but since we use EFAULT the program should fail anyways)+#define MI_TLS_ERROR_SLOT (5) // arbitrary user pointer+#define MI_TLS_ERROR_EXPANSION_SLOT (7) // environment pointer (only used for OS/2 emulation)+++mi_decl_hidden mi_decl_cache_align _Atomic(size_t) _mi_theap_default_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_SLOT);+mi_decl_hidden _Atomic(size_t) _mi_theap_default_expansion_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_EXPANSION_SLOT);+mi_decl_hidden _Atomic(size_t) _mi_theap_cached_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_SLOT);+mi_decl_hidden _Atomic(size_t) _mi_theap_cached_expansion_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_EXPANSION_SLOT);++static DWORD mi_tls_raw_index_default = TLS_OUT_OF_INDEXES;+static DWORD mi_tls_raw_index_cached = TLS_OUT_OF_INDEXES;++static bool mi_win_tls_slot_alloc(_Atomic(size_t)* slot, _Atomic(size_t)* extended, DWORD* raw_index) {+ // always write slot before extended due to concurrent readers+ const DWORD index = TlsAlloc();+ *raw_index = index;+ if (index==TLS_OUT_OF_INDEXES) {+ mi_atomic_store_release(slot,MI_TLS_ERROR_SLOT);+ mi_atomic_store_release(extended,MI_TLS_ERROR_EXPANSION_SLOT);+ return false;+ }+ else if (index<MI_TLS_DIRECT_SLOTS) {+ mi_atomic_store_release(slot,index + MI_TLS_DIRECT_FIRST);+ mi_atomic_store_release(extended,0);+ return true;+ }+ #if !MI_WIN_DIRECT_TLS+ else if (index < MI_TLS_DIRECT_SLOTS + MI_TLS_EXPANSION_SLOTS - 1) { // check maximum number of expansion slots - 1 (as we use the last one as the default)+ mi_atomic_store_release(slot, MI_TLS_EXPANSION_SLOT);+ mi_atomic_store_release(extended,index - MI_TLS_DIRECT_SLOTS);+ return true;+ }+ #endif+ else {+ // to high an index for us+ _mi_error_message(EFAULT, "returned TLS index was too high (%u)\n", index);+ TlsFree(index);+ *raw_index = TLS_OUT_OF_INDEXES;+ mi_atomic_store_release(slot, MI_TLS_ERROR_SLOT);+ mi_atomic_store_release(extended,MI_TLS_ERROR_EXPANSION_SLOT);+ return false;+ }+}++static void mi_win_tls_slot_free(_Atomic(size_t)*slot, _Atomic(size_t)*extended, DWORD* raw_index) {+ if (*raw_index != TLS_OUT_OF_INDEXES) {+ mi_atomic_store_release(slot, MI_TLS_ERROR_SLOT);+ mi_atomic_store_release(extended, MI_TLS_ERROR_EXPANSION_SLOT);+ TlsFree(*raw_index);+ *raw_index = TLS_OUT_OF_INDEXES;+ }+}++void _mi_tls_slots_init(void) {+ mi_atomic_do_once {+ bool ok = mi_win_tls_slot_alloc(&_mi_theap_default_slot, &_mi_theap_default_expansion_slot, &mi_tls_raw_index_default);+ if (ok) {+ ok = mi_win_tls_slot_alloc(&_mi_theap_cached_slot, &_mi_theap_cached_expansion_slot, &mi_tls_raw_index_cached);+ }+ if (!ok) {+ _mi_error_message(EFAULT, "unable to allocate a fast TLS user slot.\n");+ }+ }+}++void _mi_tls_slots_done(void) {+ mi_win_tls_slot_free(&_mi_theap_default_slot, &_mi_theap_default_expansion_slot, &mi_tls_raw_index_default);+ mi_win_tls_slot_free(&_mi_theap_cached_slot, &_mi_theap_cached_expansion_slot, &mi_tls_raw_index_cached );+}++static void mi_win_tls_slot_set(size_t slot, size_t extended_slot, void* value) {+ mi_assert_internal((slot >= MI_TLS_DIRECT_FIRST && slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) || slot == MI_TLS_EXPANSION_SLOT);+ if (slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) {+ mi_prim_tls_slot_set(slot, value);+ }+ else {+ mi_assert_internal(extended_slot < MI_TLS_EXPANSION_SLOTS);+ TlsSetValue((DWORD)(extended_slot + MI_TLS_DIRECT_SLOTS), value); // use TlsSetValue to initialize the TlsExpansion array if needed+ }+}++#elif MI_TLS_MODEL_PTHREADS++// only for pthreads for now+mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_default_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID);+mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_cached_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID);++static void mi_theap_cached_key_destroy(void* theapv) {+ mi_theap_t* theap = (mi_theap_t*)theapv;+ if (theap!=NULL) {+ _mi_theap_decref(theap);+ }+}++void _mi_tls_slots_init(void) {+ mi_atomic_do_once {+ pthread_key_t key;+ if (_mi_pthread_key_create(&key,NULL,NULL)) { mi_atomic_store_release(&_mi_theap_default_key,key); }+ if (_mi_pthread_key_create(&key,&mi_theap_cached_key_destroy,NULL)) { mi_atomic_store_release(&_mi_theap_cached_key,key); }+ }+}++void _mi_tls_slots_done(void) {+ pthread_key_t key = mi_atomic_exchange_relaxed(&_mi_theap_default_key,MI_PTHREAD_KEY_INVALID);+ if (key!=MI_PTHREAD_KEY_INVALID) { pthread_key_delete(key); }+ key = mi_atomic_exchange_relaxed(&_mi_theap_cached_key,MI_PTHREAD_KEY_INVALID);+ if (key!=MI_PTHREAD_KEY_INVALID) { pthread_key_delete(key); }+}++#elif MI_TLS_MODEL_FIXED++void _mi_tls_slots_init(void) {+ mi_atomic_do_once {+ mi_theap_t* theap = _mi_theap_default();+ if (theap!=NULL) {+ _mi_error_message(EINVAL,"fixed TLS slot is already in use (slot %d = %p)", MI_TLS_MODEL_FIXED_DEFAULT, theap);+ }+ theap = _mi_theap_cached();+ if (theap!=NULL) {+ _mi_error_message(EINVAL,"fixed TLS slot is already in use (slot %d = %p)", MI_TLS_MODEL_FIXED_CACHED, theap);+ }+ }+}++void _mi_tls_slots_done(void) {+ // nothing+}+++#else++void _mi_tls_slots_init(void) {+ // nothing+}++void _mi_tls_slots_done(void) {+ // nothing+}++#endif++void _mi_theap_cached_set(mi_theap_t* theap) {+ mi_theap_t* prev = _mi_theap_cached();+ if (prev==theap) return;+ // set+ _mi_tls_slots_init();+ #if MI_TLS_MODEL_LOCAL+ __mi_theap_cached = theap;+ #elif MI_TLS_MODEL_FIXED+ mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_CACHED, theap);+ #elif MI_TLS_MODEL_WIN32+ mi_win_tls_slot_set(mi_atomic_load_relaxed(&_mi_theap_cached_slot), mi_atomic_load_relaxed(&_mi_theap_cached_expansion_slot), theap);+ #elif MI_TLS_MODEL_PTHREADS+ pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_cached_key);+ if (key!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(key, theap); }+ #endif+ // update refcounts (so cached theap memory keeps available until no longer cached)+ _mi_theap_incref(theap);+ _mi_theap_decref(prev);+}++void _mi_theap_default_set(mi_theap_t* theap) {+ mi_assert_internal(theap != NULL);+ mi_assert_internal(theap->tld != NULL);+ mi_assert_internal(mi_theap_matches_thread(theap));+ _mi_tls_slots_init();+ #if MI_TLS_MODEL_LOCAL+ __mi_theap_default = theap;+ #elif MI_TLS_MODEL_FIXED+ mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_DEFAULT, theap);+ #elif MI_TLS_MODEL_WIN32+ mi_win_tls_slot_set(mi_atomic_load_relaxed(&_mi_theap_default_slot), mi_atomic_load_relaxed(&_mi_theap_default_expansion_slot), theap);+ #elif MI_TLS_MODEL_PTHREADS+ pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_default_key);+ if (key!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(key, theap); }+ #endif++ // set theap main if needed+ if (mi_theap_is_initialized(theap)) {+ // ensure the default theap is passed to `_mi_thread_done` as on some platforms we cannot access TLS at thread termination (as it would allocate again)+ _mi_prim_thread_associate_default_theap(theap);+ }+}diff --git a/vendored/mimalloc/src/prim/prim.c b/vendored/mimalloc/src/prim/prim.cindex 5147bae81..3d3d0c2e8 100644--- a/vendored/mimalloc/src/prim/prim.c+++ b/vendored/mimalloc/src/prim/prim.c@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2018-2023, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -27,11 +27,10 @@ terms of the MIT license. A copy of the license can be found in the file#endif// Generic process initialization-#ifndef MI_PRIM_HAS_PROCESS_ATTACH+#if !defined(MI_PRIM_HAS_PROCESS_ATTACH)#if defined(__GNUC__) || defined(__clang__)- // gcc,clang: use the constructor/destructor attribute- // which for both seem to run before regular constructors/destructors- #if defined(__clang__)+ // gcc,clang: use the constructor/destructor attribute with the highest user priority (<=100 is reserved for the compiler itself)+ #if defined(__clang__) || (__GNUC__ >= 5 && !defined(__APPLE__)) // gcc has support since 4.2 but not on macOS (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=119435)#define mi_attr_constructor __attribute__((constructor(101)))#define mi_attr_destructor __attribute__((destructor(101)))#elsediff --git a/vendored/mimalloc/src/prim/readme.md b/vendored/mimalloc/src/prim/readme.mdindex 380dd3a71..02cb1b90c 100644--- a/vendored/mimalloc/src/prim/readme.md+++ b/vendored/mimalloc/src/prim/readme.md@@ -6,4 +6,7 @@ This is the portability layer where all primitives needed from the OS are define- `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform(and on macOS, `osx/prim.c` defers to `unix/prim.c`).+- `include/mimalloc/prim-tls.h`: primitive TLS model (thread local, pthreads, etc)+- `prim-tls.c`: TLS model implementation+Note: still work in progress, there may still be places in the sources that still depend on OS ifdef's.\ No newline at end of filediff --git a/vendored/mimalloc/src/prim/unix/prim.c b/vendored/mimalloc/src/prim/unix/prim.cindex 93563f587..4e3eafb37 100644--- a/vendored/mimalloc/src/prim/unix/prim.c+++ b/vendored/mimalloc/src/prim/unix/prim.c@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -41,6 +41,13 @@ terms of the MIT license. A copy of the license can be found in the file#else#include <sys/mman.h>#endif+ #if defined(__riscv) || defined(_M_RISCV)+ #if defined(MI_HAS_SYS_HWPROBEH)+ #include <sys/hwprobe.h>+ #elif defined(MI_HAS_ASM_HWPROBEH)+ #include <asm/hwprobe.h>+ #endif+ #endif#elif defined(__APPLE__)#include <AvailabilityMacros.h>#include <TargetConditionals.h>@@ -72,7 +79,7 @@ terms of the MIT license. A copy of the license can be found in the file#define MADV_FREE POSIX_MADV_FREE#endif-#define MI_UNIX_LARGE_PAGE_SIZE (2*MI_MiB) // TODO: can we query the OS for this?+#define MI_UNIX_LARGE_PAGE_SIZE (2*MI_MiB) // todo: can we query the OS for this?//------------------------------------------------------------------------------------// Use syscalls for some primitives to allow for libraries that override open/read/close etc.@@ -150,13 +157,14 @@ static bool unix_detect_thp(void) {#if defined(__linux__)int fd = mi_prim_open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);if (fd >= 0) {- char buf[32];+ char buf[64];ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf));mi_prim_close(fd);// <https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html>// between brackets is the current value, for example: always [madvise] neverif (nread >= 1) {- thp_enabled = (_mi_strnstr(buf,32,"[never]") == NULL);+ if (nread > 64) { nread = 64; }+ thp_enabled = (_mi_strnstr(buf,nread,"[never]") == NULL);}}#endif@@ -205,6 +213,40 @@ static void unix_detect_physical_memory( size_t page_size, size_t* physical_memo#endif}+// Detect the virtual address bits (currently Linux/RISC-V only)+static size_t unix_detect_virtual_address_bits(void) {+ #if defined(__riscv) || defined(_M_RISCV)+ #if defined(RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS)+ struct riscv_hwprobe probe = { .key = RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS, };+ // Prefer the GNU libc interface if available, as it can also use the VDSO+ #if defined(MI_HAS_SYS_HWPROBEH)+ if (__riscv_hwprobe(&probe, 1, 0, NULL, 0) == 0)+ #else+ if (syscall(__NR_riscv_hwprobe, &probe, 1, 0, NULL, 0) == 0)+ #endif+ {+ if (probe.key != -1) { // If a key is unknown to the kernel, its key field will be cleared to -1.+ return (MI_SIZE_BITS - mi_clz((uintptr_t)probe.value));+ }+ }+ #endif+ // Fallback to checking /proc/cpuinfo for older kernels+ const int fd = mi_prim_open("/proc/cpuinfo", O_RDONLY);+ if (fd >= 0) {+ char buf[2048];+ const ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf));+ mi_prim_close(fd);+ if ((nread >= 1) && (nread <= (ssize_t)sizeof(buf))) {+ if (_mi_strnstr(buf, nread, "sv39")) { return 39; }+ else if (_mi_strnstr(buf, nread, "sv48")) { return 48; }+ else if (_mi_strnstr(buf, nread, "sv57")) { return 57; }+ }+ }+ #endif // riscv+ // default+ return MI_MAX_VABITS;+}+void _mi_prim_mem_init( mi_os_mem_config_t* config ){long psize = sysconf(_SC_PAGESIZE);@@ -218,6 +260,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config )config->has_partial_free = true; // mmap can free in partsconfig->has_virtual_reserve = true; // todo: check if this true for NetBSD? (for anonymous mmap with PROT_NONE)config->has_transparent_huge_pages = unix_detect_thp();+ config->virtual_address_bits = unix_detect_virtual_address_bits();// disable transparent huge pages for this process?#if (defined(__linux__) || defined(__ANDROID__)) && defined(PR_GET_THP_DISABLE)@@ -251,7 +294,7 @@ int _mi_prim_free(void* addr, size_t size ) {// return errno on failurestatic int unix_madvise(void* addr, size_t size, int advice) {- #if defined(__sun)+ #if defined(__sun) || defined(_AIX)const int res = madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520)return (res==0 ? 0 : errno);#elif defined(__QNX__)@@ -296,7 +339,7 @@ static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignmen// fall back to regular mmap}#endif- #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)+ #if (MI_SIZE_SIZE >= 8) && !defined(MAP_ALIGNED)// on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocationsif (addr == NULL) {void* hint = _mi_os_get_aligned_hint(try_alignment, size);@@ -333,6 +376,10 @@ static int unix_mmap_fd(void) {#endif}+#if defined(MAP_ALIGNED_SUPER) || defined(MAP_HUGETLB) || defined(MAP_HUGE_1GB) || defined(MAP_HUGE_2MB) || defined(VM_FLAGS_SUPERPAGE_SIZE_2MB)+#define MI_OS_HAS_HUGE_PAGES 1+#endif+static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {#if !defined(MAP_ANONYMOUS)#define MAP_ANONYMOUS MAP_ANON@@ -350,6 +397,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protecprotect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD#endif// huge page allocation+ #if MI_OS_HAS_HUGE_PAGESif (allow_large && (large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))) {static _Atomic(size_t) large_page_try_ok; // = 0;size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);@@ -370,8 +418,8 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int proteclflags |= MAP_HUGETLB;#endif#ifdef MAP_HUGE_1GB- static bool mi_huge_pages_available = true;- if (large_only && (size % MI_GiB) == 0 && mi_huge_pages_available) {+ static _Atomic(size_t) mi_huge_1gib_pages_unavailable;+ if (large_only && (size % MI_GiB) == 0 && (mi_atomic_load_relaxed(&mi_huge_1gib_pages_unavailable)==0)) {lflags |= MAP_HUGE_1GB;}else@@ -390,7 +438,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protecp = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);#ifdef MAP_HUGE_1GBif (p == NULL && (lflags & MAP_HUGE_1GB) == MAP_HUGE_1GB) {- mi_huge_pages_available = false; // don't try huge 1GiB pages again+ mi_atomic_store_relaxed(&mi_huge_1gib_pages_unavailable,1); // don't try huge 1GiB pages againif (large_only) {_mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno);}@@ -404,7 +452,10 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec}}}- }+ } // huge pages+ #else+ MI_UNUSED(large_only);+ #endif// regular allocationif (p == NULL) {*is_large = false;@@ -412,15 +463,17 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec#if !defined(MI_NO_THP)if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) {#if defined(MADV_HUGEPAGE)- // Many Linux systems don't allow MAP_HUGETLB but they support instead- // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE- // though since properly aligned allocations will already use large pages if available- // in that case -- in particular for our large regions (in `memory.c`).- // However, some systems only allow THP if called with explicit `madvise`, so- // when large OS pages are enabled for mimalloc, we call `madvise` anyways.- if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) {- // *is_large = true; // possibly- };+ if (_mi_os_canuse_thp()) {+ // Many Linux systems don't allow MAP_HUGETLB but they support instead+ // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE+ // though since properly aligned allocations will already use large pages if available+ // in that case -- in particular for our large regions (in `memory.c`).+ // However, some systems only allow THP if called with explicit `madvise`, so+ // when large OS pages are enabled for mimalloc, we call `madvise` anyways.+ if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) {+ // *is_large = true; // possibly+ };+ }#elif defined(__sun)struct memcntl_mha cmd = {0};cmd.mha_pagesize = _mi_os_large_page_size();@@ -452,7 +505,7 @@ int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool comm//---------------------------------------------static void unix_mprotect_hint(int err) {- #if defined(__linux__) && (MI_SECURE>=5) // guard page around every mimalloc page+ #if defined(__linux__) && (MI_SECURE>=5 || MI_GUARDED) // guard page around every mimalloc pageif (err == ENOMEM) {_mi_warning_message("The next warning may be caused by a low memory map limit.\n"" On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n"@@ -492,38 +545,39 @@ int _mi_prim_reuse(void* start, size_t size) {int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) {int err = 0;- #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE)- // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097)- err = unix_madvise(start, size, MADV_FREE_REUSABLE);- if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }- #else- // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)- err = unix_madvise(start, size, MADV_DONTNEED);- #endif- // codebase-memory-mcp local change (2026-10-10): macOS always takes the- // recommit path that upstream reserves for MI_DEBUG / MI_SECURE>2 builds.- // In the release path a purged range stays "committed" and is re-accounted as- // a whole with MADV_FREE_REUSE when the arena hands it out again- // (arena.c, _mi_os_reuse), so macOS charges its pages to phys_footprint- // whether or not they are touched; recommitting instead charges only the- // pages that are written. Measured on the indexer (Apple arm64, macOS 26.6),- // peak phys_footprint release -> this change, wall time unchanged and the- // graph byte-identical: TypeScript 1.98 -> 1.09 GB, kubernetes 2.92 -> 1.88 GB,- // dotnet/runtime 12.7 -> 10.6 GB, linux kernel 16.0 -> 13.5 GB. Other- // platforms keep upstream's behaviour (MADV_DONTNEED drops RSS at once).- #if !MI_DEBUG && MI_SECURE<=2 && !defined(__APPLE__)- *needs_recommit = false;+ #if 1+ #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE)+ // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097)+ err = unix_madvise(start, size, MADV_FREE_REUSABLE);+ if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }+ #else+ // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)+ err = unix_madvise(start, size, MADV_DONTNEED);+ #endif+ // codebase-memory-mcp local change (2026-10-10): macOS always takes the+ // recommit path that upstream reserves for MI_DEBUG / MI_SECURE>2 builds.+ // In the release path a purged range stays "committed" and is re-accounted as+ // a whole with MADV_FREE_REUSE when the arena hands it out again+ // (arena.c, _mi_os_reuse), so macOS charges its pages to phys_footprint+ // whether or not they are touched; recommitting instead charges only the+ // pages that are written. Measured on the indexer (Apple arm64, macOS 26.6),+ // peak phys_footprint release -> this change, wall time unchanged and the+ // graph byte-identical: TypeScript 1.98 -> 1.09 GB, kubernetes 2.92 -> 1.88 GB,+ // dotnet/runtime 12.7 -> 10.6 GB, linux kernel 16.0 -> 13.5 GB. Other+ // platforms keep upstream's behaviour (MADV_DONTNEED drops RSS at once).+ #if !MI_DEBUG && MI_SECURE<=2 && !defined(__APPLE__)+ *needs_recommit = false;+ #else+ *needs_recommit = true;+ mprotect(start, size, PROT_NONE);+ #endif#else+ // decommit: use mmap with MAP_FIXED and PROT_NONE to discard the existing memory (and reduce rss)*needs_recommit = true;- mprotect(start, size, PROT_NONE);+ const int fd = unix_mmap_fd();+ void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);+ if (p != start) { err = errno; }#endif- /*- // decommit: use mmap with MAP_FIXED and PROT_NONE to discard the existing memory (and reduce rss)- *needs_recommit = true;- const int fd = unix_mmap_fd();- void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);- if (p != start) { err = errno; }- */return err;}@@ -569,7 +623,7 @@ int _mi_prim_protect(void* start, size_t size, bool protect) {// Huge page allocation//----------------------------------------------#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__)+#if (MI_SIZE_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__)#ifndef MPOL_PREFERRED#define MPOL_PREFERRED 1@@ -590,15 +644,15 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bobool is_large = true;*is_zero = true;*addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large);- if (*addr != NULL && numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes+ if (*addr != NULL && numa_node >= 0 && numa_node < (8*MI_SIZE_SIZE - 1)) { // at most 63 nodesunsigned long numa_mask = (1UL << numa_node);- // TODO: does `mbind` work correctly for huge OS pages? should we+ // todo: does `mbind` work correctly for huge OS pages? should we// use `set_mempolicy` before calling mmap instead?// see: <https://lkml.org/lkml/2017/2/9/875>- long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0);+ long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_SIZE_SIZE, 0);if (err != 0) {err = errno;- _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err);+ _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %ld (0x%lx))\n", numa_node, err, err);}}return (*addr != NULL ? 0 : errno);@@ -623,9 +677,9 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bosize_t _mi_prim_numa_node(void) {#if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu)- unsigned long node = 0;- unsigned long ncpu = 0;- long err = syscall(SYS_getcpu, &ncpu, &node, NULL);+ unsigned int node = 0;+ unsigned int ncpu = 0;+ int err = syscall(SYS_getcpu, &ncpu, &node, NULL);if (err != 0) return 0;return node;#else@@ -635,13 +689,16 @@ size_t _mi_prim_numa_node(void) {size_t _mi_prim_numa_node_count(void) {char buf[128];- unsigned node = 0;- for(node = 0; node < 256; node++) {+ unsigned last_found = 0;+ for(unsigned node = 1; node < 256; node++) {// enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation)- _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1);- if (mi_prim_access(buf,R_OK) != 0) break;+ _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node);+ if (mi_prim_access(buf,R_OK) != 0) {+ if (node - last_found > 4) break; // allow some sparseness of nodes but not more than 4+ }+ else { last_found = node; } // highest found node}- return (node+1);+ return last_found + 1;}#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000@@ -667,7 +724,7 @@ size_t _mi_prim_numa_node_count(void) {#elif defined(__DragonFly__)size_t _mi_prim_numa_node(void) {- // TODO: DragonFly does not seem to provide any userland means to get this information.+ // todo: DragonFly does not seem to provide any userland means to get this information.return 0ul;}@@ -697,35 +754,39 @@ size_t _mi_prim_numa_node_count(void) {#include <time.h>-#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)--mi_msecs_t _mi_prim_clock_now(void) {- struct timespec t;- #ifdef CLOCK_MONOTONIC- clock_gettime(CLOCK_MONOTONIC, &t);+// low resolution timer+static mi_msecs_t mi_prim_clock_now_lowres(void) {+ const int64_t ticks = (int64_t)clock();+ #if !defined(CLOCKS_PER_SEC)+ return ticks;#else- clock_gettime(CLOCK_REALTIME, &t);+ if (CLOCKS_PER_SEC <= 0 || CLOCKS_PER_SEC == 1000) {+ return ticks;+ }+ else if (CLOCKS_PER_SEC > 0 && CLOCKS_PER_SEC < 1000) {+ return ticks * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);+ }+ else {+ return ticks / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);+ }#endif- return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);}-#else--// low resolution timermi_msecs_t _mi_prim_clock_now(void) {- #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)- return (mi_msecs_t)clock();- #elif (CLOCKS_PER_SEC < 1000)- return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);- #else- return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);+ #if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)+ #ifdef CLOCK_MONOTONIC+ const clockid_t clockid = CLOCK_MONOTONIC;+ #else+ const clockid_t clockid = CLOCK_REALTIME;+ #endif+ struct timespec t;+ if (clock_gettime(clockid,&t) == 0) {+ return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000L);+ }#endif+ return mi_prim_clock_now_lowres();}-#endif---//----------------------------------------------------------------// Process info@@ -751,41 +812,46 @@ static mi_msecs_t timeval_secs(const struct timeval* tv) {void _mi_prim_process_info(mi_process_info_t* pinfo){struct rusage rusage;- getrusage(RUSAGE_SELF, &rusage);- pinfo->utime = timeval_secs(&rusage.ru_utime);- pinfo->stime = timeval_secs(&rusage.ru_stime);-#if !defined(__HAIKU__)- pinfo->page_faults = rusage.ru_majflt;-#endif-#if defined(__HAIKU__)- // Haiku does not have (yet?) a way to- // get these stats per process- thread_info tid;- area_info mem;- ssize_t c;- get_thread_info(find_thread(0), &tid);- while (get_next_area_info(tid.team, &c, &mem) == B_OK) {- pinfo->peak_rss += mem.ram_size;- }- pinfo->page_faults = 0;-#elif defined(__APPLE__)- pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes- #ifdef MACH_TASK_BASIC_INFO- struct mach_task_basic_info info;- mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;- if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {- pinfo->current_rss = (size_t)info.resident_size;- }- #else- struct task_basic_info info;- mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT;- if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {- pinfo->current_rss = (size_t)info.resident_size;+ if (getrusage(RUSAGE_SELF, &rusage) == 0) {+ pinfo->utime = timeval_secs(&rusage.ru_utime);+ pinfo->stime = timeval_secs(&rusage.ru_stime);+ #if !defined(__HAIKU__)+ pinfo->page_faults = rusage.ru_majflt;+ #endif+ #if defined(__APPLE__)+ pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes+ #else+ pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB+ #endif}++ #if defined(__HAIKU__)+ // Haiku does not have (yet?) a way to+ // get these stats per process+ thread_info tid;+ if (get_thread_info(find_thread(0), &tid) == B_OK) {+ area_info mem;+ ssize_t c;+ while (get_next_area_info(tid.team, &c, &mem) == B_OK) {+ pinfo->peak_rss += mem.ram_size;+ }+ }+ pinfo->page_faults = 0;+ #elif defined(__APPLE__)+ #ifdef MACH_TASK_BASIC_INFO+ struct mach_task_basic_info info;+ mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;+ if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {+ pinfo->current_rss = (size_t)info.resident_size;+ }+ #else+ struct task_basic_info info;+ mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT;+ if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {+ pinfo->current_rss = (size_t)info.resident_size;+ }+ #endif#endif-#else- pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB-#endif// use defaults for commit}@@ -832,28 +898,28 @@ static char** mi_get_environ(void) {return environ;}#endif-bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {- if (name==NULL) return false;+int _mi_prim_getenv(const char* name, char* result, size_t result_size) {+ if (name==NULL) return -1;const size_t len = _mi_strlen(name);- if (len == 0) return false;+ if (len == 0) return -1;char** env = mi_get_environ();- if (env == NULL) return false;+ if (env == NULL) return -1;// compare up to 10000 entriesfor (int i = 0; i < 10000 && env[i] != NULL; i++) {const char* s = env[i];if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive// found it- _mi_strlcpy(result, s + len + 1, result_size);- return true;+ if (!_mi_strlcpy(result, s + len + 1, result_size)) return -1;+ return 1; // success}}- return false;+ return 0; // not found}#else// fallback: use standard C `getenv` but this cannot be used while initializing the C runtime-bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {+int _mi_prim_getenv(const char* name, char* result, size_t result_size) {// cannot call getenv() when still initializing the C runtime.- if (_mi_preloading()) return false;+ if (_mi_preloading()) return -1; // error, try again laterconst char* s = getenv(name);if (s == NULL) {// we check the upper case name too.@@ -865,9 +931,9 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {buf[len] = 0;s = getenv(buf);}- if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false;- _mi_strlcpy(result, s, result_size);- return true;+ if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return 0; // not found+ if (!_mi_strlcpy(result, s, result_size)) return -1;+ return 1; // success}#endif // !MI_USE_ENVIRON@@ -896,7 +962,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {return true;}-#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) // also for old apple versions < 10.7 (issue #829)+#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) || defined(__CYGWIN__) // also for old apple versions < 10.7 (issue #829)#include <sys/types.h>#include <sys/stat.h>@@ -928,7 +994,10 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {size_t count = 0;while(count < buf_len) {ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count);- if (ret<=0) {+ if (ret==0) {+ break;+ }+ else if (ret<0) {if (errno!=EAGAIN && errno!=EINTR) break;}else {@@ -956,7 +1025,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {// use pthread local storage keys to detect thread ending// (and used with MI_TLS_PTHREADS for the default theap)-pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);+pthread_key_t _mi_heap_default_key = MI_PTHREAD_KEY_INVALID;static void mi_pthread_done(void* value) {if (value!=NULL) {@@ -965,22 +1034,25 @@ static void mi_pthread_done(void* value) {}void _mi_prim_thread_init_auto_done(void) {- mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));+ mi_assert_internal(_mi_heap_default_key == MI_PTHREAD_KEY_INVALID);pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);}void _mi_prim_thread_done_auto_done(void) {- if (_mi_heap_default_key != (pthread_key_t)(-1)) { // do not leak the key, see issue #809- pthread_key_delete(_mi_heap_default_key);+ pthread_key_t key = _mi_heap_default_key;+ if (key != MI_PTHREAD_KEY_INVALID) { // do not leak the key, see issue #809+ _mi_heap_default_key = MI_PTHREAD_KEY_INVALID;+ pthread_key_delete(key);}}void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {- if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD+ if (_mi_heap_default_key != MI_PTHREAD_KEY_INVALID) { // can happen during recursive invocation on freeBSDpthread_setspecific(_mi_heap_default_key, theap);}}+#elsevoid _mi_prim_thread_init_auto_done(void) {diff --git a/vendored/mimalloc/src/prim/wasi/prim.c b/vendored/mimalloc/src/prim/wasi/prim.cindex 790daff02..2d69b7245 100644--- a/vendored/mimalloc/src/prim/wasi/prim.c+++ b/vendored/mimalloc/src/prim/wasi/prim.c@@ -185,33 +185,39 @@ size_t _mi_prim_numa_node_count(void) {#include <time.h>-#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)--mi_msecs_t _mi_prim_clock_now(void) {- struct timespec t;- #ifdef CLOCK_MONOTONIC- clock_gettime(CLOCK_MONOTONIC, &t);+// low resolution timer+static mi_msecs_t mi_prim_clock_now_lowres(void) {+ const int64_t ticks = (int64_t)clock();+ #if !defined(CLOCKS_PER_SEC)+ return ticks;#else- clock_gettime(CLOCK_REALTIME, &t);+ if (CLOCKS_PER_SEC <= 0 || CLOCKS_PER_SEC == 1000) {+ return ticks;+ }+ else if (CLOCKS_PER_SEC > 0 && CLOCKS_PER_SEC < 1000) {+ return ticks * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);+ }+ else {+ return ticks / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);+ }#endif- return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);}-#else--// low resolution timermi_msecs_t _mi_prim_clock_now(void) {- #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)- return (mi_msecs_t)clock();- #elif (CLOCKS_PER_SEC < 1000)- return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);- #else- return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);- #endif+ #if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)+ #ifdef CLOCK_MONOTONIC+ const clockid_t clockid = CLOCK_MONOTONIC;+ #else+ const clockid_t clockid = CLOCK_REALTIME;+ #endif+ struct timespec t;+ if (clock_gettime(clockid,&t) == 0) {+ return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000L);+ }+ #endif+ return mi_prim_clock_now_lowres();}-#endif-//----------------------------------------------------------------// Process info@@ -237,9 +243,9 @@ void _mi_prim_out_stderr( const char* msg ) {// Environment//-----------------------------------------------------------------bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {+int _mi_prim_getenv(const char* name, char* result, size_t result_size) {// cannot call getenv() when still initializing the C runtime.- if (_mi_preloading()) return false;+ if (_mi_preloading()) return -1; // error, try again laterconst char* s = getenv(name);if (s == NULL) {// we check the upper case name too.@@ -251,9 +257,9 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {buf[len] = 0;s = getenv(buf);}- if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false;+ if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return 0; // not found_mi_strlcpy(result, s, result_size);- return true;+ return 1; // found}diff --git a/vendored/mimalloc/src/prim/windows/prim.c b/vendored/mimalloc/src/prim/windows/prim.cindex 51ad3104f..222fa6303 100644--- a/vendored/mimalloc/src/prim/windows/prim.c+++ b/vendored/mimalloc/src/prim/windows/prim.c@@ -13,9 +13,9 @@ terms of the MIT license. A copy of the license can be found in the file#include <stdio.h> // fputs, stderr#include <stdlib.h> // atexit-// xbox has no console IO-#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP | WINAPI_PARTITION_SYSTEM)-#define MI_HAS_CONSOLE_IO+// xbox has no console IO and cannot use LoadLibrary or GetModuleHandle+#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM)+#define MI_WIN_DESKTOP 1#endif//---------------------------------------------@@ -87,6 +87,34 @@ typedef BOOL (__stdcall *PGetPhysicallyInstalledSystemMemory)( PULONGLONG TotalMtypedef BOOL (__stdcall* PGetVersionExW)(LPOSVERSIONINFOW lpVersionInformation);+// Load a library+static HMODULE mi_win_loadlibrary(const TCHAR* library) {+ #if MI_WIN_DESKTOP+ return LoadLibrary(library);+ #else+ return LoadPackagedLibrary(library, 0);+ #endif+}++// Get a library handle (and possibly load it)+static HMODULE mi_win_getlibrary(const TCHAR* library, bool* should_free) {+ #if MI_WIN_DESKTOP+ // avoid calling LoadLibrary for "kernel32", "ntdll", and "kernelbase" (also to avoid hitting the loader lock)+ HMODULE mod = GetModuleHandle(library);+ if (mod!=NULL) {+ *should_free = false;+ return mod;+ }+ #endif+ *should_free = true;+ return mi_win_loadlibrary(library);+}++static void mi_win_freelibrary(HMODULE mod, bool should_free) {+ if (should_free) {+ FreeLibrary(mod);+ }+}//---------------------------------------------// Enable large page support dynamically (if possible)@@ -103,15 +131,17 @@ static bool win_enable_large_os_pages_once(size_t* large_page_size)unsigned long err = 0;HANDLE token = NULL;BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token);+ err = GetLastError();if (ok) {TOKEN_PRIVILEGES tp;ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid);+ err = GetLastError();if (ok) {tp.PrivilegeCount = 1;tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0);+ err = GetLastError();if (ok) {- err = GetLastError();ok = (err == ERROR_SUCCESS);if (ok && large_page_size != NULL && pGetLargePageMinimum != NULL) {*large_page_size = (*pGetLargePageMinimum)();@@ -121,17 +151,19 @@ static bool win_enable_large_os_pages_once(size_t* large_page_size)CloseHandle(token);}if (!ok) {- if (err == 0) err = GetLastError();+ if (err == 0) { err = GetLastError(); }_mi_warning_message("cannot enable large OS page support, error %lu\n", err);}return (ok!=0);}static bool win_enable_large_os_pages(size_t* large_page_size) {+ static size_t win_large_page_size = 0;mi_atomic_do_once {- win_enable_large_os_pages_once(large_page_size);+ win_enable_large_os_pages_once(&win_large_page_size);}- return (_mi_os_large_page_size() > 0);+ if (large_page_size != NULL) { *large_page_size = win_large_page_size; }+ return (win_large_page_size > 0);}@@ -162,21 +194,22 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config )}// get the VirtualAlloc2 function- HINSTANCE hDll = LoadLibrary(TEXT("kernelbase.dll"));+ bool hDllFree;+ HINSTANCE hDll = mi_win_getlibrary(TEXT("kernelbase.dll"), &hDllFree);if (hDll != NULL) {// use VirtualAlloc2FromApp if possible as it is available to Windows store appspVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp");if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2");- FreeLibrary(hDll);+ mi_win_freelibrary(hDll, hDllFree);}// NtAllocateVirtualMemoryEx is used for huge page allocation- hDll = LoadLibrary(TEXT("ntdll.dll"));+ hDll = mi_win_getlibrary(TEXT("ntdll.dll"), &hDllFree);if (hDll != NULL) {pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx");- FreeLibrary(hDll);+ mi_win_freelibrary(hDll, hDllFree);}// Try to use Win7+ numa API- hDll = LoadLibrary(TEXT("kernel32.dll"));+ hDll = mi_win_getlibrary(TEXT("kernel32.dll"), &hDllFree);if (hDll != NULL) {pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, "GetCurrentProcessorNumberEx");pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNodeEx");@@ -205,7 +238,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config )win_minor_version = version.dwMinorVersion;}}- FreeLibrary(hDll);+ mi_win_freelibrary(hDll, hDllFree);}// Enable large/huge OS page support?if (mi_option_is_enabled(mi_option_allow_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {@@ -228,8 +261,9 @@ int _mi_prim_free(void* addr, size_t size ) {// the memory region returned by VirtualAlloc; in that case we need to free using// the start of the region.MEMORY_BASIC_INFORMATION info; _mi_memzero_var(info);- VirtualQuery(addr, &info, sizeof(info));- if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)(4*MI_MiB)) {+ err = (VirtualQuery(addr, &info, sizeof(info)) == 0);+ if (err) { errcode = GetLastError(); }+ if (!err && info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)(4*MI_MiB)) {errcode = 0;err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0);if (err) { errcode = GetLastError(); }@@ -244,7 +278,7 @@ int _mi_prim_free(void* addr, size_t size ) {//---------------------------------------------static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) {- #if (MI_INTPTR_SIZE >= 8)+ #if (MI_SIZE_SIZE >= 8)// on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocationsif (addr == NULL) {void* hint = _mi_os_get_aligned_hint(try_alignment,size);@@ -525,8 +559,9 @@ static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) {static LARGE_INTEGER mfreq; // = 0if (mfreq.QuadPart == 0LL) {LARGE_INTEGER f;- QueryPerformanceFrequency(&f);- mfreq.QuadPart = f.QuadPart/1000LL;+ if (QueryPerformanceFrequency(&f)) {+ mfreq.QuadPart = f.QuadPart/1000LL;+ }if (mfreq.QuadPart == 0) mfreq.QuadPart = 1;}return (mi_msecs_t)(t.QuadPart / mfreq.QuadPart);@@ -534,8 +569,12 @@ static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) {mi_msecs_t _mi_prim_clock_now(void) {LARGE_INTEGER t;- QueryPerformanceCounter(&t);- return mi_to_msecs(t);+ if (QueryPerformanceCounter(&t)) {+ return mi_to_msecs(t);+ }+ else {+ return 0;+ }}@@ -562,29 +601,31 @@ void _mi_prim_process_info(mi_process_info_t* pinfo)FILETIME ut;FILETIME st;FILETIME et;- GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut);- pinfo->utime = filetime_msecs(&ut);- pinfo->stime = filetime_msecs(&st);+ if (GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut)) {+ pinfo->utime = filetime_msecs(&ut);+ pinfo->stime = filetime_msecs(&st);+ }// load psapi on demand- mi_atomic_do_once {- HINSTANCE hDll = LoadLibrary(TEXT("psapi.dll"));+ mi_atomic_do_once{+ HINSTANCE hDll = mi_win_loadlibrary(TEXT("psapi.dll"));if (hDll != NULL) {pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, "GetProcessMemoryInfo");- // FreeLibrary(hDll); // don't free+ // mi_win_freelibrary(hDll, true); // don't free}}// get process info- PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info);if (pGetProcessMemoryInfo != NULL) {- pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info));+ PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info);+ if (pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info))) {+ pinfo->current_rss = (size_t)info.WorkingSetSize;+ pinfo->peak_rss = (size_t)info.PeakWorkingSetSize;+ pinfo->current_commit = (size_t)info.PagefileUsage;+ pinfo->peak_commit = (size_t)info.PeakPagefileUsage;+ pinfo->page_faults = (size_t)info.PageFaultCount;+ }}- pinfo->current_rss = (size_t)info.WorkingSetSize;- pinfo->peak_rss = (size_t)info.PeakWorkingSetSize;- pinfo->current_commit = (size_t)info.PagefileUsage;- pinfo->peak_commit = (size_t)info.PeakPagefileUsage;- pinfo->page_faults = (size_t)info.PageFaultCount;}//----------------------------------------------------------------@@ -596,32 +637,37 @@ void _mi_prim_out_stderr( const char* msg )// on windows with redirection, the C runtime cannot handle locale dependent output// after the main thread closes so we use direct console output.if (!_mi_preloading()) {- // _cputs(msg); // _cputs cannot be used as it aborts when failing to lock the consolestatic HANDLE hcon = INVALID_HANDLE_VALUE;+ #if MI_WIN_DESKTOPstatic bool hconIsConsole = false;+ #endifif (hcon == INVALID_HANDLE_VALUE) {- hcon = GetStdHandle(STD_ERROR_HANDLE);- #ifdef MI_HAS_CONSOLE_IO+ hcon = GetStdHandle(STD_ERROR_HANDLE); // returns NULL if no stderr is available+ #if MI_WIN_DESKTOP+ if (hcon==NULL) {+ AttachConsole(ATTACH_PARENT_PROCESS); // if started from a parent console, try to attach to that+ hcon = GetStdHandle(STD_ERROR_HANDLE);+ }CONSOLE_SCREEN_BUFFER_INFO sbi;- hconIsConsole = ((hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi));+ hconIsConsole = ((hcon != NULL && hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi));#endif}const size_t len = _mi_strlen(msg);if (len > 0 && len < UINT32_MAX) {DWORD written = 0;- if (hconIsConsole) {- #ifdef MI_HAS_CONSOLE_IO- WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL);+ if (hcon != NULL && hcon != INVALID_HANDLE_VALUE) {+ #if MI_WIN_DESKTOP+ if (hconIsConsole) {+ WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL);+ }+ else#endif+ {+ // use direct write in case stderr was redirected+ WriteFile(hcon, msg, (DWORD)len, &written, NULL);+ }}- else if (hcon != INVALID_HANDLE_VALUE) {- // use direct write if stderr was redirected- WriteFile(hcon, msg, (DWORD)len, &written, NULL);- }- else {- // finally fall back to fputs after all- fputs(msg, stderr);- }+ // don't fall back to fputs or _cputs as the crt can have it locked}}}@@ -635,10 +681,10 @@ void _mi_prim_out_stderr( const char* msg )// reliably even when this is invoked before the C runtime is initialized.// i.e. when `_mi_preloading() == true`.// Note: on windows, environment names are not case sensitive.-bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {+int _mi_prim_getenv(const char* name, char* result, size_t result_size) {result[0] = 0;const size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size);- return (len > 0 && len < result_size);+ return (len < result_size ? (len > 0 ? 1 /* success */ : 0 /* not found */) : -1 /* error */);}@@ -673,10 +719,10 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {mi_assert(buf_len <= ULONG_MAX);if (buf_len > ULONG_MAX) return false;mi_atomic_do_once {- HINSTANCE hDll = LoadLibrary(TEXT("bcrypt.dll"));+ HINSTANCE hDll = mi_win_loadlibrary(TEXT("bcrypt.dll"));if (hDll != NULL) {pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, "BCryptGenRandom");- // FreeLibrary(hDll); // don't free+ // mi_win_freelibrary(hDll); // don't free}}if (pBCryptGenRandom == NULL) return false;@@ -737,15 +783,18 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {both static and dynamic linkage (`MI_WIN_INIT_USE_CRT_TLS`).------------------------------------------------------------------------- */#if !defined(MI_WIN_INIT_USE_CRT_TLS) && !defined(MI_WIN_INIT_USE_RAW_DLLMAIN) && !defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && !defined(MI_WIN_INIT_USE_FLS)- #if !defined(__INTEL_LLVM_COMPILER) && !defined(__INTEL_COMPILER)- #define MI_WIN_INIT_USE_CRT_TLS 1+ #if defined(__INTEL_LLVM_COMPILER) || defined(__INTEL_COMPILER)+ #define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* needed for Intel ICX, see issue #1268 */#else- #define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* default for Intel ICX, see issue #1268 */+ #define MI_WIN_INIT_USE_CRT_TLS 1 /* default */#endif#endif#if defined(MI_WIN_INIT_USE_CRT_TLS)- #define MI_PRIM_HAS_PROCESS_ATTACH 1+ #if !defined(__MINGW32__) || defined(MI_MINGW_UCRT64) // on mingw without UCRT use the constructor attribute (in `src/prim/prim.c`)+ #define MI_PRIM_HAS_PROCESS_ATTACH 1+ #endif+// nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.void _mi_prim_thread_init_auto_done(void) {}void _mi_prim_thread_done_auto_done(void) {}@@ -818,10 +867,12 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {#endiftypedef int (mi_cdecl* mi_crt_callback_t)(void);- #if defined(_WIN64)++ #if defined(_WIN64) && defined(_MSC_VER) // 64-bit#pragma comment(linker, "/INCLUDE:_tls_used")#pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")#pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")+ #pragma comment(linker, "/INCLUDE:_mi_crt_callback_init")#pragma const_seg(".CRT$XLB")extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };@@ -834,10 +885,11 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {extern const mi_crt_callback_t _mi_crt_callback_init[];const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };#pragma const_seg()- #else+ #elif defined(_MSC_VER) // 32-bit#pragma comment(linker, "/INCLUDE:__tls_used")#pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")#pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")+ #pragma comment(linker, "/INCLUDE:__mi_crt_callback_init")#pragma data_seg(".CRT$XLB")PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };#pragma data_seg()@@ -847,6 +899,12 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {#pragma data_seg(".CRT$XIB")mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };#pragma data_seg()+ #elif defined(__MINGW32__)+ extern const IMAGE_TLS_DIRECTORY _tls_used;+ __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls+ __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach;+ __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach;+ __attribute__((used, section(".CRT$XIB"))) mi_crt_callback_t _mi_crt_callback_init = &mi_crt_init;#endif#if defined(__cplusplus)@@ -916,7 +974,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {extern "C" {#endif- #if defined(_WIN64)+ #if defined(_WIN64) && defined(_MSC_VER) // 64-bit#pragma comment(linker, "/INCLUDE:_tls_used")#pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")#pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")@@ -928,7 +986,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };#pragma const_seg()- #else+ #elif defined(_MSC_VER) // 32-bit#pragma comment(linker, "/INCLUDE:__tls_used")#pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")#pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")@@ -938,6 +996,11 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {#pragma data_seg(".CRT$XLY")PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };#pragma data_seg()+ #elif defined(__MINGW32__)+ extern const IMAGE_TLS_DIRECTORY _tls_used;+ __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls+ __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach;+ __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach;#endif#if defined(__cplusplus)@@ -985,7 +1048,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {extern "C" {#endif- #if defined(_WIN64)+ #if defined(_WIN64) && defined(_MSC_VER) // 64-bit#pragma comment(linker, "/INCLUDE:_tls_used")#pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")#pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")@@ -997,16 +1060,21 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };#pragma const_seg()- #else+ #elif defined(_MSC_VER) // 32-bit#pragma comment(linker, "/INCLUDE:__tls_used")#pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")#pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")#pragma data_seg(".CRT$XLB")PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };#pragma data_seg()- #pragma data_seg(".CRT$XIY")+ #pragma data_seg(".CRT$XLY")PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };#pragma data_seg()+ #elif defined(__MINGW32__)+ extern const IMAGE_TLS_DIRECTORY _tls_used;+ __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls+ __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach;+ __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach;#endif#if defined(__cplusplus)@@ -1027,21 +1095,31 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {// See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm>#define MI_PRIM_HAS_PROCESS_ATTACH 1- static int mi_process_attach(void) {+ static int mi_cdecl mi_crt_init(void) {mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL);atexit(&_mi_auto_process_done);return 0;}- typedef int(*mi_crt_callback_t)(void);- #if defined(_WIN64)- #pragma comment(linker, "/INCLUDE:_mi_tls_callback")- #pragma section(".CRT$XIU", long, read)- #else- #pragma comment(linker, "/INCLUDE:__mi_tls_callback")++ #if defined(__cplusplus)+ extern "C" {+ #endif+ typedef int (mi_cdecl* mi_crt_callback_t)(void);+ #if defined(_WIN64) // 64-bit+ #pragma comment(linker, "/INCLUDE:_mi_crt_callback_init")+ #pragma const_seg(".CRT$XIU")+ extern const mi_crt_callback_t _mi_crt_callback_init[];+ const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };+ #pragma const_seg()+ #else // 32-bit+ #pragma comment(linker, "/INCLUDE:__mi_crt_callback_init")+ #pragma data_seg(".CRT$XIU")+ mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };+ #pragma data_seg()+ #endif+ #if defined(__cplusplus)+ }#endif- #pragma data_seg(".CRT$XIU")- mi_decl_externc mi_crt_callback_t _mi_tls_callback[] = { &mi_process_attach };- #pragma data_seg()#endif// use the fiber api for calling `_mi_thread_done`.diff --git a/vendored/mimalloc/src/random.c b/vendored/mimalloc/src/random.cindex 75b5cfbb0..3c012adf4 100644--- a/vendored/mimalloc/src/random.c+++ b/vendored/mimalloc/src/random.c@@ -134,18 +134,19 @@ static bool mi_random_is_initialized(mi_random_ctx_t* ctx) {void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) {mi_assert_internal(mi_random_is_initialized(ctx));- mi_assert_internal(ctx != ctx_new);- chacha_split(ctx, (uintptr_t)ctx_new /*nonce*/, ctx_new);+ mi_assert_internal(ctx != ctx_new);+ const uintptr_t nonce_rnd = _mi_random_next(ctx);+ chacha_split(ctx, (uintptr_t)ctx_new ^ nonce_rnd /*nonce*/, ctx_new);}-uintptr_t _mi_random_next(mi_random_ctx_t* ctx) {+size_t _mi_random_next(mi_random_ctx_t* ctx) {mi_assert_internal(mi_random_is_initialized(ctx));- uintptr_t r;+ size_t r;do {- #if MI_INTPTR_SIZE <= 4+ #if MI_SIZE_SIZE <= 4r = chacha_next32(ctx);- #elif MI_INTPTR_SIZE == 8- r = (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx));+ #elif MI_SIZE_SIZE == 8+ r = (((size_t)chacha_next32(ctx) << 32) | chacha_next32(ctx));#else# error "define mi_random_next for this platform"#endif@@ -159,12 +160,12 @@ To initialize a fresh random context.If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR.-----------------------------------------------------------------------------*/-uintptr_t _mi_os_random_weak(uintptr_t extra_seed) {- uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random+size_t _mi_os_random_weak(size_t extra_seed) {+ size_t x = (size_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address randomx ^= _mi_prim_clock_now();// and do a few randomization steps- uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1;- for (uintptr_t i = 0; i < max || x==0; i++, x++) {+ size_t max = ((x ^ (x >> 17)) & 0x0F) + 1;+ for (size_t i = 0; i < max || x==0; i++, x++) {x = _mi_random_shuffle(x);}mi_assert_internal(x != 0);@@ -179,7 +180,7 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) {#if !defined(__wasi__)if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); }#endif- uintptr_t x = _mi_os_random_weak(0);+ size_t x = _mi_os_random_weak(0);for (size_t i = 0; i < 32; i+=4, x++) {x = _mi_random_shuffle(x);key[i] = (uint8_t)(x);@@ -193,6 +194,7 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) {ctx->weak = false;}chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ );+ _mi_memzero(key, sizeof(key));}void _mi_random_init(mi_random_ctx_t* ctx) {diff --git a/vendored/mimalloc/src/sample-guarded.c b/vendored/mimalloc/src/sample-guarded.cnew file mode 100644index 000000000..f7c1fe81e--- /dev/null+++ b/vendored/mimalloc/src/sample-guarded.c@@ -0,0 +1,173 @@+/* ----------------------------------------------------------------------------+Copyright (c) 2019-2026, Microsoft Research, Daan Leijen+This is free software; you can redistribute it and/or modify it under the+terms of the MIT license. A copy of the license can be found in the file+"LICENSE" at the root of this distribution.+-----------------------------------------------------------------------------*/+#include "mimalloc.h"+#include "mimalloc/internal.h"+#include "mimalloc/prim-tls.h"++mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) {+ theap->guarded_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate);+ if (theap->guarded_sample_rate > 1) {+ if (seed == 0) {+ seed = _mi_theap_random_next(theap);+ }+ theap->guarded_sample_countdown = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate`+ }+ else if (theap->guarded_sample_countdown > theap->guarded_sample_rate) {+ theap->guarded_sample_countdown = theap->guarded_sample_rate;+ }+ _mi_theap_update_sample_rate(theap);+}++mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) {+ theap->guarded_size_min = min;+ theap->guarded_size_max = (min > max ? min : max);+}++void _mi_theap_guarded_init(mi_theap_t* theap) {+ #if MI_GUARDED+ mi_theap_guarded_set_sample_rate(theap,+ (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX),+ (size_t)mi_option_get(mi_option_guarded_sample_seed));+ mi_theap_guarded_set_size_bound(theap,+ (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX),+ (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) );+ #else+ mi_theap_guarded_set_sample_rate(theap,0,0);+ #endif+}+++// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true).+// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`)+// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size`+static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t* usable_size) {+ // todo: we can still make padding work by moving it out of the guard page area+ mi_page_t* const page = _mi_ptr_page(block);+ mi_page_set_has_interior_pointers(page, true);+ block->next = MI_BLOCK_TAG_GUARDED;++ // set guard page at the end of the block+ const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local`+ const size_t os_page_size = _mi_os_page_size();+ mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t));+ if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) {+ // should never happen+ mi_free(block);+ return NULL;+ }+ uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size;+ // note: the alignment of the guard page relies on blocks being os_page_size aligned which+ // is ensured in `mi_arena_page_alloc_fresh`.+ mi_assert_internal(_mi_is_aligned(block, os_page_size));+ mi_assert_internal(_mi_is_aligned(guard_page, os_page_size));+ if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) {+ const bool ok = _mi_os_protect(guard_page, os_page_size);+ if mi_unlikely(!ok) {+ _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size);+ }+ }+ else {+ _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size);+ }++ // align pointer just in front of the guard page+ size_t offset = block_size - os_page_size - obj_size;+ mi_assert_internal(offset > sizeof(mi_block_t));+ if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) {+ // give up to place it right in front of the guard page if the offset is too large for unalignment+ offset = MI_PAGE_MAX_OVERALLOC_ALIGN;+ }+ uint8_t* const p = (uint8_t*)block + offset;+ mi_assert_internal(p == guard_page - obj_size || offset >= MI_PAGE_MAX_OVERALLOC_ALIGN);+ if (usable_size != NULL) { *usable_size = (guard_page - p); mi_assert_internal(mi_usable_size(p)==*usable_size); }+ mi_track_align(block, p, offset, obj_size);+ mi_track_mem_defined(block, sizeof(mi_block_t));+ return p;+}++// Allocate a block with a guard page behind it.+mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept+{+ // allocate multiple of page size ending in a guard page+ // ensure minimal alignment requirement?+ if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow+ _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size);+ return NULL;+ }+ const size_t os_page_size = _mi_os_page_size();+ const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE));+ const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE);+ const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size);+ // const size_t threshold = mi_theap_disable_profiler(theap);+ mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap, req_size, false /* don't zero */, ppage);+ // mi_theap_enable_profiler(theap,threshold);+ if (block==NULL) return NULL;+ size_t usable_size = 0;+ void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size);+ if (p == NULL) return NULL;+ if (zero) {+ _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`)+ }++ // stats+ mi_track_malloc(p, usable_size, zero);+ if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); }+ mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1);+ #if MI_STATS+ // adjust request stats to only count the allocated size of the block (and not the guard page)+ mi_theap_stat_counter_decrease(theap, malloc_requested, req_size);+ mi_theap_stat_counter_increase(theap, malloc_requested, size);+ #endif+ #if MI_DEBUG>3+ if (zero) {+ mi_assert_expensive(mi_mem_is_zero(p, size));+ }+ #endif+ #if MI_PAGE_META_SMALL_IS_ALIGNED && MI_DEBUG>=2+ // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail.+ if (size <= MI_SMALL_SIZE_MAX) {+ mi_page_t* const page = _mi_ptr_page(p);+ mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE);+ }+ #endif+ return p;+}+++// Remove guard page when building with MI_GUARDED+#if MI_GUARDED+void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) {+ // if (!mi_block_ptr_is_guarded(block,p)) return;+ mi_assert_internal(mi_block_ptr_is_guarded(block, p));+ mi_assert_internal(mi_page_has_interior_pointers(page));+ mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t));+ mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED);++ const size_t bsize = mi_page_block_size(page);+ const size_t psize = _mi_os_page_size();+ mi_assert_internal(bsize > psize);+ mi_assert_internal(!page->memid.is_pinned);+ void* gpage = (uint8_t*)block + bsize - psize;+ mi_assert_internal(_mi_is_aligned(gpage, psize));+ _mi_os_unprotect(gpage, psize);+}++// unguard a whole page (called from `mi_heap_destroy`)+void _mi_page_unguard_all(mi_page_t* page) {+ if mi_likely(!mi_page_has_interior_pointers(page)) return;+ uint8_t* const start = mi_page_start(page);+ const size_t psize = mi_page_committed(page);+ _mi_os_unprotect(start,psize); // unprotect all at once as we cannot know which blocks are guarded+}+#else+void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) {+ MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p);+}+void _mi_page_unguard_all(mi_page_t* page) {+ MI_UNUSED(page);+}+#endifdiff --git a/vendored/mimalloc/src/sample-profile.c b/vendored/mimalloc/src/sample-profile.cnew file mode 100644index 000000000..f1b5db491--- /dev/null+++ b/vendored/mimalloc/src/sample-profile.c@@ -0,0 +1,264 @@+/* ----------------------------------------------------------------------------+Copyright (c) 2019-2026, Microsoft Research, Daan Leijen+This is free software; you can redistribute it and/or modify it under the+terms of the MIT license. A copy of the license can be found in the file+"LICENSE" at the root of this distribution.+-----------------------------------------------------------------------------*/+#include "mimalloc.h"+#include "mimalloc/internal.h"+#include "mimalloc/prim-tls.h"++//----------------------------------------------------------------------------+// General sampled allocation (called from `page.c:mi_malloc_generic_fallback`)+// This is for both guarded and profiled sampling+//----------------------------------------------------------------------------++size_t _mi_theap_update_sample_rate(mi_theap_t* theap) {+ const size_t old_sample_rate = theap->sample_rate;+ theap->sample_rate = theap->profile_sample_rate;+ if (theap->sample_rate == 0 || (theap->guarded_sample_rate!=0 && theap->sample_rate > theap->guarded_sample_rate)) {+ theap->sample_rate = theap->guarded_sample_rate;+ }+ if (theap->sample_countdown > theap->sample_rate) {+ theap->sample_countdown = theap->sample_rate; // todo: adjust difference?+ }+ return old_sample_rate;+}++// Called when the theap sample countdown <= req_size; should call guarded or profiled allocation.+mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept+{+ // the size has not yet been counted against the countdown (and does not include MI_PADDING_SIZE)+ mi_assert_internal(req_size <= MI_MAX_ALLOC_SIZE);+ mi_assert_internal((theap->sample_countdown==SIZE_MAX && _mi_is_empty_theap(theap)) ||+ (theap->sample_countdown <= MI_SAMPLE_COUNTDOWN_MAX && theap->sample_countdown < req_size));+ const size_t size = req_size + MI_PADDING_SIZE;++ // handle empty theap and disabled sampling+ if (theap->sample_rate==0) {+ if (!_mi_is_empty_theap(theap)) { // avoid writing to the initial empty theap+ theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // avoid the sampling path for a long time+ }+ return _mi_malloc_generic_no_sample(theap,size,zero,ppage);+ }++ // update countdown and total accummulated requested bytes since the last sample+ mi_assert_internal(!_mi_is_empty_theap(theap));+ mi_assert_internal(req_size <= SIZE_MAX/2);+ mi_assert_internal(theap->sample_rate > 0);+ mi_assert_internal(theap->sample_rate <= SIZE_MAX/2);+ mi_assert_internal(theap->sample_rate >= theap->sample_countdown);++ const uint64_t requested = theap->sample_requested = (uint64_t)theap->sample_rate + (uint64_t)(req_size - theap->sample_countdown) + theap->sample_requested;+ mi_assert_internal(requested > 0);+ mi_assert_internal(requested >= req_size);+ theap->sample_countdown = theap->sample_rate; // reset sampling++ // update derived countdowns+ mi_assert_internal(theap->profile_sample_rate!=0 || theap->profile_sample_countdown==0);+ mi_assert_internal(theap->guarded_sample_rate!=0 || theap->guarded_sample_countdown==0);+ bool sample_profile = false;+ bool sample_guarded = false;+ if (theap->profile_sample_rate!=0) {+ if (theap->profile_sample_countdown >= requested) { theap->profile_sample_countdown -= (size_t)requested; } else { sample_profile = true; }+ }+ if (theap->guarded_sample_rate!=0) {+ if (theap->guarded_sample_countdown >= requested) { theap->guarded_sample_countdown -= (size_t)requested; } else { sample_guarded = true; }+ }++ // invoke callback?+ if (sample_profile) {+ theap->profile_sample_countdown = theap->profile_sample_rate; // reset countdown+ theap->sample_requested = 0; // reset requested as we pass it to malloc_profiled+ #if MI_PROFILE+ return _mi_theap_malloc_profiled(theap,size,requested,zero,ppage);+ #endif+ }+ else if (sample_guarded) {+ if (req_size >= theap->guarded_size_min && req_size <= theap->guarded_size_max) {+ // use guarded allocation+ theap->guarded_sample_countdown = theap->guarded_sample_rate; // reset countdown+ #if MI_GUARDED+ return _mi_theap_malloc_guarded(theap,size,zero,ppage);+ #endif+ }+ #if MI_GUARDED+ else {+ // failed size criteria, rewind the sample countdown so we sample asap again+ // todo: can we do better here as this will cause many samples until it fits the size..+ theap->sample_countdown = 0;+ }+ #endif+ }+ // take generic path+ return _mi_malloc_generic_no_sample(theap,size,zero,ppage);+}++++//----------------------------------------------------------------------------+// Util+//----------------------------------------------------------------------------++static mi_profiler_t* mi_heap_profiler(const mi_heap_t* heap) {+ return mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler);+}++static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) {+ mi_heap_t* const heap = _mi_theap_heap(theap);+ mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler);+ if (prof!=NULL && mi_profiler_is_enabled(prof) && prof->on_alloc != NULL) {+ return prof;+ }+ else {+ return NULL;+ }+}++//----------------------------------------------------------------------------+// Profile an allocation+//-----------------------------------------------------------------------------++mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept+{+ mi_assert_internal(theap!=NULL);+ mi_assert_internal(size>=MI_PADDING_SIZE);+ const size_t req_size = size - MI_PADDING_SIZE;+ mi_assert_internal(req_size<=requested_since_last_sample);+ mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap);+ if (prof == NULL) { return _mi_malloc_generic_no_sample(theap,size,zero,ppage); }+ mi_assert_internal(prof!=NULL && prof->on_alloc!=NULL && mi_profiler_is_enabled(prof));++ void* p = NULL;+ size_t new_sample_rate = 0;+ if (prof->on_free==NULL || prof->sample_data_size==0) {+ // just allocate without profiler data+ p = _mi_malloc_generic_no_sample(theap,size,zero,ppage);+ if (p==NULL) { return p; }+ if (prof->on_alloc!=NULL) {+ // we are just allocation profiling (not heap profiling as on_free == NULL)+ new_sample_rate = (*prof->on_alloc)(prof, NULL /* no data */, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) );+ }+ }+ else {+ // overallocate a larger block to store the profiler data+ // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...]+ const size_t sample_data_offset = sizeof(mi_block_t);+ const size_t sample_user_data_size = _mi_align_up(prof->sample_data_size > MI_PROFILE_SAMPLE_DATA_MAX_SIZE ? MI_PROFILE_SAMPLE_DATA_MAX_SIZE : prof->sample_data_size, sizeof(void*));+ const size_t sample_data_size = sizeof(mi_profiler_sample_data_t) + sample_user_data_size; // one void* too many just in case+ const size_t user_offset = _mi_align_up(sample_data_offset + sample_data_size, MI_MAX_ALIGN_SIZE);+ const size_t oversize = user_offset + size;+ mi_page_t* page = NULL;+ mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap,oversize,zero,&page);+ if (block==NULL) return NULL;+ mi_assert_internal(page!=NULL);+ if (ppage!=NULL) { *ppage = page; }+ #if MI_PAGE_META_SMALL_IS_ALIGNED+ // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail.+ // (that is why we need to limit the profiler_data_size as well)+ // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail.+ if (size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); }+ #endif++ // Set up the profiled block as an interior pointer so the interior "slow" path is taken on mi_free (where we catch it to call on_free)+ mi_page_set_has_interior_pointers(page, true);+ block->next = MI_BLOCK_TAG_PROFILED;+ p = (uint8_t*)block + user_offset;+ mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sample_data_offset);+ sample_data->user_data_size = sample_user_data_size;++ // and call the profiler on_alloc+ if (prof->on_alloc!=NULL) {+ new_sample_rate = (*prof->on_alloc)(prof, sample_data, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) );+ }+ }+ if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) {+ _mi_theap_set_profile_sample_rate(theap,new_sample_rate);+ }+ mi_theap_stat_counter_increase(theap,profile_samples,1);+ return p;+}++void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p) {+ mi_assert_internal(mi_block_ptr_is_sampled(block,p));++ // get the heap and profiler+ mi_heap_t* const heap = mi_page_heap(page);+ if (heap==NULL) return;+ mi_profiler_t* prof = mi_heap_profiler(heap);+ if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL) return;++ // call the on_free callback+ mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sizeof(mi_block_t));+ (*prof->on_free)(prof, sample_data, p, heap);+}+++//----------------------------------------------------------------------------+// Profiling API+//-----------------------------------------------------------------------------*++size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate) {+ const size_t old_sample_rate = theap->profile_sample_rate;+ theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate);+ if (theap->profile_sample_countdown > theap->profile_sample_rate) {+ theap->profile_sample_countdown = theap->profile_sample_rate; // todo: adjust difference?+ }+ _mi_theap_update_sample_rate(theap);+ return old_sample_rate;+}++static bool mi_heap_set_profiler(mi_heap_t* heap, mi_profiler_t* profiler) {+ mi_profiler_t* previous = (profiler==NULL ? mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler) : NULL); // don't overwrite unless it is NULL+ return mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t, &heap->profiler, &previous, profiler);+}++mi_decl_export bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler) {+ mi_profiler_stop(profiler);+ return mi_heap_set_profiler(heap,profiler);+}++mi_decl_export void mi_heap_profile_disable(mi_heap_t* heap) {+ static mi_profiler_t mi_nosample_profiler = { NULL, 0, MI_SAMPLE_RATE_MAX, NULL, NULL, NULL };+ mi_heap_set_profiler(heap,NULL); // first clear any previous one+ mi_heap_set_profiler(heap,&mi_nosample_profiler); // and then install the nosample one+}++mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler) {+ mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);+ if (subproc==NULL) return false;+ mi_profiler_stop(profiler);+ mi_profiler_t* previous = (profiler==NULL ? mi_atomic_load_ptr_acquire(mi_profiler_t,&subproc->profiler) : NULL); // don't overwrite unless it is NULL+ if (!mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t,&subproc->profiler, &previous, profiler)) { return false; }+ mi_lock(&subproc->heaps_lock) {+ for (mi_heap_t* heap = subproc->heaps; heap!=NULL; heap = heap->next) {+ mi_heap_set_profiler(heap,profiler);+ }+ }+ return true;+}++mi_decl_export bool mi_profile( mi_profiler_t* profiler) {+ return mi_subproc_profile(mi_subproc_main(),profiler);+}++bool mi_profiler_start(mi_profiler_t* profiler ) {+ if (profiler==NULL) return false;+ const bool was_running = mi_profiler_set_enabled(profiler,true);+ if (was_running) return true;+ // for the main heap, if this is the profiler, start the theap more aggressively+ // otherwise it will be picked up when theaps take the slow generic malloc path.+ mi_heap_t* heap = mi_heap_main();+ if (mi_heap_profiler(heap)==profiler) {+ mi_theap_t* theap = _mi_heap_theap_peek(heap);+ if (theap!=NULL) {+ _mi_theap_set_profile_sample_rate(theap,1);+ }+ }+ return false;+}++bool mi_profiler_stop(mi_profiler_t* profiler) {+ if (profiler==NULL) return true;+ return mi_profiler_set_enabled(profiler,false);+}diff --git a/vendored/mimalloc/src/static.c b/vendored/mimalloc/src/static.cindex 2383f6596..8a4f8c3d5 100644--- a/vendored/mimalloc/src/static.c+++ b/vendored/mimalloc/src/static.c@@ -24,20 +24,23 @@ terms of the MIT license. A copy of the license can be found in the file#include "alloc-aligned.c"#include "alloc-posix.c"#include "arena.c"-#include "arena-meta.c"#include "bitmap.c"#include "heap.c"#include "init.c"#include "libc.c"#include "options.c"#include "os.c"+#include "prim/prim.c" // includes platform specific prim/<platform>/prim.c+#include "prim/prim-tls.c"+#if MI_OSX_ZONE+#include "prim/osx/alloc-override-zone.c"+#endif#include "page.c" // includes page-queue.c#include "page-map.c"#include "random.c"+#include "sample-guarded.c"+#include "sample-profile.c"#include "stats.c"+#include "subproc.c"#include "theap.c"#include "threadlocal.c"-#include "prim/prim.c"-#if MI_OSX_ZONE-#include "prim/osx/alloc-override-zone.c"-#endifdiff --git a/vendored/mimalloc/src/stats.c b/vendored/mimalloc/src/stats.cindex 7ca18e238..c763bfc3c 100644--- a/vendored/mimalloc/src/stats.c+++ b/vendored/mimalloc/src/stats.c@@ -1,5 +1,5 @@/* -----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -8,7 +8,8 @@ terms of the MIT license. A copy of the license can be found in the file#include "mimalloc-stats.h"#include "mimalloc/internal.h"#include "mimalloc/atomic.h"-#include "mimalloc/prim.h"+#include "mimalloc/prim.h" // _mi_prim_clock_now, mi_process_info_t+#include "mimalloc/prim-tls.h"#include <string.h> // memset@@ -38,26 +39,21 @@ static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {if (amount > 0) { stat->total += amount; }}--void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount) {- mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount);-}--void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {- stat->total += amount;-}--void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount) {+void __mi_stat_increase_mt(mi_stat_count_t* stat, uint64_t amount) {+ mi_assert_internal(amount<=INT64_MAX);mi_stat_update_mt(stat, (int64_t)amount);}-void __mi_stat_increase(mi_stat_count_t* stat, size_t amount) {+void __mi_stat_increase(mi_stat_count_t* stat, uint64_t amount) {+ mi_assert_internal(amount<=INT64_MAX);mi_stat_update(stat, (int64_t)amount);}-void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount) {+void __mi_stat_decrease_mt(mi_stat_count_t* stat, uint64_t amount) {+ mi_assert_internal(amount<=INT64_MAX);mi_stat_update_mt(stat, -((int64_t)amount));}-void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) {+void __mi_stat_decrease(mi_stat_count_t* stat, uint64_t amount) {+ mi_assert_internal(amount<=INT64_MAX);mi_stat_update(stat, -((int64_t)amount));}@@ -68,26 +64,33 @@ void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) {static void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount) {if (amount == 0) return;// adjust atomically+ const int64_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&stat->peak);mi_atomic_addi64_relaxed(&stat->current, amount);- mi_atomic_addi64_relaxed(&stat->total, amount);+ const int64_t prev_total = mi_atomic_addi64_relaxed(&stat->total, amount);+ if (prev_total == peak) { mi_atomic_addi64_relaxed(&stat->peak, amount); }}static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) {if (amount == 0) return;stat->current += amount;- stat->total += amount;+ if (stat->total==stat->peak) { stat->peak += amount; }+ stat->total += amount;}-void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount) {+void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, uint64_t amount) {+ mi_assert_internal(amount<=INT64_MAX);mi_stat_adjust_mt(stat, (int64_t)amount);}-void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount) {+void __mi_stat_adjust_increase(mi_stat_count_t* stat, uint64_t amount) {+ mi_assert_internal(amount<=INT64_MAX);mi_stat_adjust(stat, (int64_t)amount);}-void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount) {+void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, uint64_t amount) {+ mi_assert_internal(amount<=INT64_MAX);mi_stat_adjust_mt(stat, -((int64_t)amount));}-void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) {+void __mi_stat_adjust_decrease(mi_stat_count_t* stat, uint64_t amount) {+ mi_assert_internal(amount<=INT64_MAX);mi_stat_adjust(stat, -((int64_t)amount));}@@ -96,15 +99,17 @@ void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) {static void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* src) {if (stat==src) return;mi_atomic_void_addi64_relaxed(&stat->total, &src->total);- const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src->current);+ const int64_t src_peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&src->peak);+ const int64_t src_current = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&src->current);+ const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src_current);// Global current plus thread peak approximates new global peak- // note: peak scores do really not work across threads.+ // note: peak scores do not really work across threads.// we used to just add them together but that often overestimates in practice.// similarly, max does not seem to work well. The current approach// by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112- // for a longer description.- mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src->peak);+ // for a longer description.+ mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src_peak);}static void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counter_t* src) {@@ -122,7 +127,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {// copy all fieldsMI_STAT_FIELDS()- #if MI_STAT>1+ #if MI_STATSfor (size_t i = 0; i <= MI_BIN_HUGE; i++) {mi_stat_count_add_mt(&stats->malloc_bins[i], &src->malloc_bins[i]);}@@ -130,6 +135,9 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {for (size_t i = 0; i <= MI_BIN_HUGE; i++) {mi_stat_count_add_mt(&stats->page_bins[i], &src->page_bins[i]);}+ for (size_t i = 0; i < MI_CBIN_COUNT; i++) {+ mi_stat_count_add_mt(&stats->chunk_bins[i], &src->chunk_bins[i]);+ }}#undef MI_STAT_COUNT@@ -142,7 +150,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {// unit > 0 : size in binary bytes// unit == 0: count as decimal// unit < 0 : count in binary-static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) {+static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, bool limitwidth) {char buf[32]; _mi_memzero_var(buf);int len = 32;const char* suffix = (unit <= 0 ? " " : "B");@@ -167,12 +175,17 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void*_mi_snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix);_mi_snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc);}- _mi_fprintf(out, arg, (fmt==NULL ? "%12s" : fmt), buf);+ if (limitwidth) {+ _mi_fprintf(out, arg, "%12s", buf);+ }+ else {+ _mi_fprintf(out, arg, "%s", buf);+ }}static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {- mi_printf_amount(n,unit,out,arg,NULL);+ mi_printf_amount(n,unit,out,arg,true);}static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {@@ -181,32 +194,31 @@ static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* ar}static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) {- _mi_fprintf(out, arg," %-10s:", msg);+ _mi_fprintf(out, arg," %-12s:", msg);if (unit != 0) {- if (unit > 0) {+ if (unit > 0) { // as KiB (power of two)mi_print_amount(stat->peak, unit, out, arg);mi_print_amount(stat->total, unit, out, arg);- // mi_print_amount(stat->freed, unit, out, arg);mi_print_amount(stat->current, unit, out, arg);mi_print_amount(unit, 1, out, arg);mi_print_count(stat->total, unit, out, arg);}- else {+ else if (unit==-1) { // as K (decimal)mi_print_amount(stat->peak, -1, out, arg);mi_print_amount(stat->total, -1, out, arg);- // mi_print_amount(stat->freed, -1, out, arg);mi_print_amount(stat->current, -1, out, arg);- if (unit == -1) {- _mi_fprintf(out, arg, "%24s", "");- }- else {- mi_print_amount(-unit, 1, out, arg);- mi_print_count((stat->total / -unit), 0, out, arg);- }+ _mi_fprintf(out, arg, "%24s", "");+ }+ else { // as KiB , unit is final count+ mi_print_amount(stat->peak, 1, out, arg);+ mi_print_amount(stat->total, 1, out, arg);+ mi_print_amount(stat->current, -1, out, arg);+ _mi_fprintf(out, arg, "%12s", "");+ mi_print_count(-unit, 0, out, arg);}if (stat->current != 0) {_mi_fprintf(out, arg, " ");- _mi_fprintf(out, arg, (notok == NULL ? "not all freed" : notok));+ _mi_fprintf(out, arg, "%s", (notok == NULL ? "not all freed" : notok));_mi_fprintf(out, arg, "\n");}else {@@ -225,23 +237,23 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_tmi_stat_print_ex(stat, msg, unit, out, arg, NULL);}-#if MI_STAT>1-static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {- _mi_fprintf(out, arg, " %-10s:", msg);+#if MI_STATS+static void mi_stat_total_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) {+ _mi_fprintf(out, arg, " %-12s:", msg);_mi_fprintf(out, arg, "%12s", " "); // no peak- mi_print_amount(stat->total, unit, out, arg);+ mi_print_amount(stat->total, 1, out, arg);_mi_fprintf(out, arg, "\n");}#endifstatic void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {- _mi_fprintf(out, arg, " %-10s:", msg);+ _mi_fprintf(out, arg, " %-12s:", msg);mi_print_amount(stat->total, 0, out, arg);_mi_fprintf(out, arg, "\n");}static void mi_stat_counter_print_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {- _mi_fprintf(out, arg, " %-10s:", msg);+ _mi_fprintf(out, arg, " %-12s:", msg);mi_print_amount(stat->total, 1, out, arg);_mi_fprintf(out, arg, "\n");}@@ -250,16 +262,16 @@ static void mi_stat_average_print(int64_t count, int64_t total, const char* msg,const int64_t avg_tens = (count == 0 ? 0 : (total*10 / count));const int64_t avg_whole = avg_tens/10;const int64_t avg_frac1 = avg_tens%10;- _mi_fprintf(out, arg, " %-10s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1);+ _mi_fprintf(out, arg, " %-12s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1);}static void mi_print_header(const char* name,mi_output_fun* out, void* arg ) {- _mi_fprintf(out, arg, " %-11s %11s %11s %11s %11s %11s\n",+ _mi_fprintf(out, arg, " %-13s %11s %11s %11s %11s %11s\n",name, "peak ", "total ", "current ", "block ", "total# ");}-#if MI_STAT>1+#if MI_STATSstatic bool mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, mi_output_fun* out, void* arg) {bool found = false;char buf[64];@@ -328,13 +340,13 @@ mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_size_t peak_commit;size_t page_faults;mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults);- _mi_fprintf(out, arg, " %-10s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000);- _mi_fprintf(out, arg, " %-10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process",+ _mi_fprintf(out, arg, " %-12s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000);+ _mi_fprintf(out, arg, " %-12s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process",user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults);- mi_printf_amount((int64_t)peak_rss, 1, out, arg, "%s");+ mi_printf_amount((int64_t)peak_rss, 1, out, arg, false);if (peak_commit > 0) {_mi_fprintf(out, arg, ", peak commit: ");- mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s");+ mi_printf_amount((int64_t)peak_commit, 1, out, arg, false);}_mi_fprintf(out, arg, "\n");}@@ -351,19 +363,21 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou_mi_fprintf(out, arg, "%s %zu\n", name, id);if (stats->malloc_normal.total + stats->malloc_huge.total != 0) {- #if MI_STAT>1+ #if MI_STATSmi_print_header("blocks", out, arg);mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, out, arg);#endif- #if MI_STAT- mi_stat_print(&stats->malloc_normal, "binned", (stats->malloc_normal_count.total == 0 ? -1 : 1), out, arg);- mi_stat_print(&stats->malloc_huge, "huge", (stats->malloc_huge_count.total == 0 ? -1 : 1), out, arg);+ #if MI_STATS+ mi_stat_print(&stats->malloc_normal, "binned", -stats->malloc_normal_count.total, out, arg);+ mi_stat_print(&stats->malloc_huge, "huge", -stats->malloc_huge_count.total, out, arg);mi_stat_count_t total = { 0,0,0 };mi_stat_count_add_mt(&total, &stats->malloc_normal);mi_stat_count_add_mt(&total, &stats->malloc_huge);- mi_stat_print_ex(&total, "total", 1, out, arg, "");- #if MI_STAT>1- mi_stat_total_print(&stats->malloc_requested, "malloc req", 1, out, arg);+ mi_stat_print_ex(&total, "total", -(stats->malloc_normal_count.total + stats->malloc_huge_count.total), out, arg, "");+ #if MI_STATS>=2+ mi_stat_total_print(&stats->malloc_requested, "malloc req", out, arg);+ #else+ mi_stat_total_print(&stats->malloc_requested, "malloc req~", out, arg);#endif_mi_fprintf(out, arg, "\n");#endif@@ -377,13 +391,18 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou// mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg);mi_stat_print(&stats->pages, "pages", 0, out, arg);mi_stat_print(&stats->pages_abandoned, "abandoned", 0, out, arg);+ mi_stat_print(&stats->pages_os_abandoned, "os abandoned", 0, out, arg);+ mi_stat_print(&stats->pages_os_allocated, "os allocated", 0, out, arg);mi_stat_counter_print(&stats->pages_reclaim_on_alloc, "reclaima", out, arg);mi_stat_counter_print(&stats->pages_reclaim_on_free, "reclaimf", out, arg);mi_stat_counter_print(&stats->pages_reabandon_full, "reabandon", out, arg);mi_stat_counter_print(&stats->pages_unabandon_busy_wait, "waits", out, arg);mi_stat_counter_print(&stats->pages_extended, "extended", out, arg);mi_stat_counter_print(&stats->pages_retire, "retire", out, arg);+ mi_stat_counter_print(&stats->pages_stat_updates, "stat updates", out, arg);+ mi_stat_average_print(stats->pages_stat_updates.total, stats->pages_stat_update_count.total, "stat avg", out, arg);mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg);+ mi_stat_counter_print(&stats->profile_samples, "prof samples", out, arg);_mi_fprintf(out, arg, "\n");}@@ -405,12 +424,13 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_oumi_stat_print_ex(&stats->heaps, "heaps", 0, out, arg, "");mi_stat_counter_print(&stats->heaps_delete_wait, "heap waits", out, arg);_mi_fprintf(out, arg, "\n");-- mi_print_header("process", out, arg);- mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, "");- _mi_fprintf(out, arg, " %-10s: %5i\n", "numa nodes", _mi_os_numa_node_count());- mi_process_info_print_out(out, arg);}++ mi_print_header("process", out, arg);+ mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, "");+ _mi_fprintf(out, arg, " %-12s: %5i\n", "numa nodes", _mi_os_numa_node_count());+ mi_process_info_print_out(out, arg);+_mi_fprintf(out, arg, "\n");}@@ -432,7 +452,7 @@ void _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from) {mi_assert_internal(to != NULL && from != NULL);if (to == from) return;mi_stats_add(to, from);- _mi_memzero(from, sizeof(mi_stats_t));+ mi_stats_init(from); // zero field and keep the header}static const mi_stats_t* mi_stats_merge_theap_to_heap(mi_theap_t* theap) mi_attr_noexcept {@@ -449,11 +469,16 @@ static const mi_stats_t* mi_heap_get_stats(mi_heap_t* heap) {else return mi_stats_merge_theap_to_heap(theap);}+static const mi_stats_t* mi_theap_get_stats(mi_theap_t* theap) {+ return &theap->stats;+}+// deprecatedvoid mi_stats_reset(void) mi_attr_noexcept {if (!mi_theap_is_initialized(_mi_theap_default())) return;- mi_heap_get_stats(mi_heap_main());- mi_heap_stats_merge_to_subproc(mi_heap_main());+ mi_heap_t* heap_main = mi_heap_main();+ mi_heap_get_stats(heap_main);+ mi_heap_stats_merge_to_subproc(heap_main);}@@ -480,6 +505,10 @@ void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun*if (subproc==NULL) return;mi_heap_print_visit_info_t vinfo = { out, arg };mi_subproc_visit_heaps(subproc_id, &mi_heap_print_visitor, &vinfo);+ if (subproc->theap_meta!=NULL) {+ _mi_stats_print("meta", subproc->subproc_seq, &subproc->theap_meta->stats, out, arg);+ }+ // if (subproc->theap_meta!=NULL) { _mi_stats_merge_into(&subproc->stats, &subproc->theap_meta->stats); }_mi_stats_print("subproc", subproc->subproc_seq, &subproc->stats, out, arg);}@@ -548,8 +577,10 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, spinfo.elapsed = _mi_clock_end(mi_process_start);{ const mi_subproc_t* subproc = _mi_subproc_main();if (subproc!=NULL) {- pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current)));- pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak)));+ const int64_t current = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current));+ const int64_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak));+ pinfo.current_commit = (current < 0 ? 0 : (current < PTRDIFF_MAX ? (size_t)current : PTRDIFF_MAX));+ pinfo.peak_commit = (peak < 0 ? 0 : (peak < PTRDIFF_MAX ? (size_t)peak : PTRDIFF_MAX));}}pinfo.current_rss = pinfo.current_commit;@@ -601,6 +632,14 @@ bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept {return mi_stats_copy(stats, mi_heap_get_stats(heap));}+bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept {+ return mi_stats_copy(stats, mi_theap_get_stats(theap));+}++void mi_theap_stats_merge_to_heap(mi_theap_t* theap) mi_attr_noexcept {+ if (theap == NULL) return;+ _mi_theap_merge_stats(theap);+}static bool mi_cdecl mi_heap_aggregate_visitor(mi_heap_t* heap, void* arg) {mi_stats_t* stats = (mi_stats_t*)arg;@@ -614,6 +653,8 @@ bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attrif (subproc == NULL) return false;if (!mi_stats_copy(stats, &subproc->stats)) return false;mi_subproc_visit_heaps(subproc_id, &mi_heap_aggregate_visitor, stats);+ // hide meta data stats+ // if (subproc->theap_meta!=NULL) { mi_stats_add_into(stats, &subproc->theap_meta->stats); }return true;}@@ -722,9 +763,6 @@ static void mi_json_buf_print_counter_value(mi_json_buf_t* hbuf, const char* nammi_json_buf_print_value(hbuf, name, stat->total);}-#define MI_STAT_COUNT(stat) mi_json_buf_print_count_value(&hbuf, #stat, &stats->stat);-#define MI_STAT_COUNTER(stat) mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat);-static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, char* output_buf) mi_attr_noexcept {if (stats==NULL || stats->size!=sizeof(mi_stats_t) || stats->version!=MI_STAT_VERSION) return NULL;mi_json_buf_t hbuf = { NULL, 0, 0, true };@@ -763,7 +801,13 @@ static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size,mi_json_buf_print(&hbuf, " },\n");// statistics+ #define MI_STAT_COUNT(stat) mi_json_buf_print_count_value(&hbuf, #stat, &stats->stat);+ #define MI_STAT_COUNTER(stat) mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat);+MI_STAT_FIELDS()++ #undef MI_STAT_COUNT+ #undef MI_STAT_COUNTER// size binsmi_json_buf_print(&hbuf, " \"malloc_bins\": [\n");@@ -782,7 +826,7 @@ static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size,}mi_json_buf_print(&hbuf, " ]\n");mi_json_buf_print(&hbuf, "}\n");- if (hbuf.used >= hbuf.size) {+ if (hbuf.used + 1 >= hbuf.size) {// failedif (hbuf.can_realloc) { mi_free(hbuf.buf); }return NULL;diff --git a/vendored/mimalloc/src/subproc.c b/vendored/mimalloc/src/subproc.cnew file mode 100644index 000000000..f42d4a11c--- /dev/null+++ b/vendored/mimalloc/src/subproc.c@@ -0,0 +1,327 @@+/* ----------------------------------------------------------------------------+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen+This is free software; you can redistribute it and/or modify it under the+terms of the MIT license. A copy of the license can be found in the file+"LICENSE" at the root of this distribution.+-----------------------------------------------------------------------------*/++#include "mimalloc.h"+#include "mimalloc/internal.h"+#include "mimalloc/prim-tls.h"++// pre-allocate the main subprocess structure.+static mi_decl_cache_align mi_subproc_t mi_process_subproc_main = mi_init_struct_zero;+static mi_subproc_t* mi_subprocs = NULL;+static mi_lock_t mi_subprocs_lock = MI_LOCK_INITIALIZER;+++/* -----------------------------------------------------------+ Meta-data allocation+ We allocate thread local data and theaps through a dedicated+ theap `subproc.theap_meta` which uses a detached tld with+ a detached thread id. The initial theap_meta is statically+ allocated and can thus be used to allocate on an as yet+ uninitialized thread or process.+ We need to take a lock though to allocate safely on the+ detached `theap_meta`.+----------------------------------------------------------- */++void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid ) {+ mi_assert_internal(subproc->theap_meta != NULL);+ void* p = NULL;+ mi_lock(&subproc->theap_meta_lock) {+ p = mi_theap_zalloc(subproc->theap_meta, size);+ if (memid != NULL) { *memid = (p==NULL ? _mi_memid_none() : _mi_memid_create_malloc(p,size,true) ); }+ }+ return p;+}++void* _mi_meta_zalloc_aligned( mi_subproc_t* subproc, size_t size, size_t aligned, mi_memid_t* memid ) {+ mi_assert_internal(subproc->theap_meta != NULL);+ void* p = NULL;+ mi_lock(&subproc->theap_meta_lock) {+ p = mi_theap_zalloc_aligned(subproc->theap_meta, size, aligned);+ if (memid != NULL) { *memid = (p==NULL ? _mi_memid_none() : _mi_memid_create_malloc(p,size,true) ); }+ }+ return p;+}++void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* oldp, size_t newsize, mi_memid_t* memid ) {+ mi_assert_internal(subproc->theap_meta != NULL);+ // note: since we take a meta lock we cannot use `mi_theap_rezalloc` as that could call `mi_free` which+ // can call `mi_stat_free` which would try to take the meta lock again. See issue #1358.+ void* p = NULL;+ mi_lock(&subproc->theap_meta_lock) {+ p = mi_theap_zalloc(subproc->theap_meta, newsize);+ }+ if (p!=NULL) {+ if (oldp!=NULL) {+ const size_t oldsize = mi_usable_size(oldp);+ const size_t copysize = (newsize < oldsize ? newsize : oldsize);+ _mi_memcpy(p,oldp,copysize);+ if (memid!=NULL) { _mi_meta_free(subproc,oldp,*memid); }+ else { mi_free(oldp); }+ }+ if (memid!=NULL) { *memid = _mi_memid_create_malloc(p,newsize,true); }+ }+ else {+ if (memid!=NULL) { *memid = _mi_memid_none(); }+ }+ return p;+}++void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid) {+ if (p==NULL || mi_memid_needs_no_free(memid)) return;+ if (memid.memkind == MI_MEM_MALLOC) {+ mi_free(p);+ }+ else {+ mi_assert_internal(subproc!=NULL);+ _mi_arenas_free(subproc, p, _mi_memid_size(memid), memid);+ }+}++bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* page) {+ if (page==NULL) return false;+ mi_theap_t* theap = page->theap;+ return (theap != NULL && theap == subproc->theap_meta);+}+++/* -----------------------------------------------------------+ Sub process helpers+----------------------------------------------------------- */++mi_subproc_t* _mi_subproc_main(void) {+ return &mi_process_subproc_main;+}++bool _mi_subproc_is_main(mi_subproc_t* subproc) {+ return (subproc == &mi_process_subproc_main);+}++mi_subproc_t* _mi_subproc(void) {+ mi_theap_t* theap = _mi_theap_default();+ if (theap == NULL || theap->tld == NULL || theap->tld->subproc == NULL) { // see issue #1289 and #1391+ return _mi_subproc_main();+ }+ else {+ return theap->tld->subproc;+ }+}++mi_heap_t* mi_heap_main(void) {+ return _mi_subproc_heap_main(_mi_subproc()); // don't use mi_theap_main_init_get() so this call works during process_init+}+++mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) {+ return (mi_subproc_t*)(subproc_id._mi_subproc_id);+}++mi_subproc_id_t _mi_subproc_to_id(mi_subproc_t* subproc) {+ mi_subproc_id_t id = { subproc };+ return id;+}++mi_subproc_id_t mi_subproc_main(void) {+ return _mi_subproc_to_id(_mi_subproc_main());+}++mi_subproc_id_t mi_subproc_current(void) {+ return _mi_subproc_to_id(_mi_subproc());+}+++/* -----------------------------------------------------------+ Sub process creation+----------------------------------------------------------- */+++static mi_subproc_t* mi_subproc_init(mi_subproc_t* subproc, mi_subproc_t* parent) {+ static _Atomic(size_t) subproc_total_count;+ subproc->parent = parent;+ subproc->subproc_seq = mi_atomic_increment_relaxed(&subproc_total_count);+ mi_stats_header_init(&subproc->stats);+ mi_lock_init(&subproc->arena_reserve_lock);+ mi_lock_init(&subproc->heaps_lock);+ mi_lock_init(&subproc->theap_meta_lock);+ mi_lock(&mi_subprocs_lock) {+ // push on subproc list+ subproc->next = mi_subprocs;+ if (mi_subprocs!=NULL) { mi_subprocs->prev = subproc; }+ mi_subprocs = subproc;+ }+ return subproc;+}++mi_subproc_id_t mi_subproc_new(void) {+ mi_thread_init();+ mi_subproc_t* const parent = _mi_subproc();+ mi_memid_t memid;+ mi_subproc_t* const subproc = (mi_subproc_t*)_mi_meta_zalloc(parent, sizeof(mi_subproc_t), &memid);+ if (subproc == NULL) { return _mi_subproc_to_id(NULL); }+ subproc->memid = memid;++ mi_memid_t theap_memid;+ mi_theap_t* const theap_meta = (mi_theap_t*)_mi_meta_zalloc(parent, sizeof(mi_theap_t), &theap_memid);+ if (theap_meta==NULL) {+ _mi_meta_free(parent, subproc, memid);+ return _mi_subproc_to_id(NULL);+ }+ theap_meta->memid = memid;++ // init subproc+ mi_subproc_init(subproc,parent);++ // init main heap+ mi_heap_t* heap_main = _mi_heap_new_for_subproc(subproc,0,true);+ if (heap_main==NULL) {+ _mi_meta_free(parent, theap_meta, theap_meta->memid);+ mi_subproc_destroy(_mi_subproc_to_id(subproc));+ return _mi_subproc_to_id(NULL);+ }+ mi_assert_internal(subproc->heap_main == heap_main);++ // init meta theap+ mi_assert_internal(parent->theap_meta!=NULL);+ mi_assert_internal(parent->theap_meta->tld!=NULL);+ mi_assert_internal(parent->theap_meta->tld->thread_id == MI_THREADID_DETACHED);+ _mi_theap_init(theap_meta,heap_main,parent->theap_meta->tld /* detached tld */);+ subproc->theap_meta = theap_meta;++ return _mi_subproc_to_id(subproc);+}+++/* -----------------------------------------------------------+ Sub process destruction+----------------------------------------------------------- */++// destroy all subproc resources including arena's, heap's etc.+static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subprocs_lock)+{+ if (subproc==NULL) return;++ // remove from the subproc list+ mi_lock_maybe(&mi_subprocs_lock, acquire_subprocs_lock) {+ if (subproc->next!=NULL) { subproc->next->prev = subproc->prev; }+ if (subproc->prev!=NULL) { subproc->prev->next = subproc->next; }+ else { mi_assert_internal(mi_subprocs==subproc); mi_subprocs = subproc->next; }+ }++ // destroy all subproc heaps+ mi_lock(&subproc->heaps_lock) {+ mi_heap_t* heap = subproc->heaps;+ while (heap != NULL) {+ mi_heap_t* next = heap->next;+ if (heap!=subproc->heap_main) { _mi_heap_force_destroy(heap, false /* don't re-acquire the heaps_lock */); }+ heap = next;+ }+ mi_assert_internal(subproc->heap_main==NULL || subproc->heaps == subproc->heap_main);+ if (subproc->heap_main!=NULL) {+ _mi_thread_locals_thread_done(); // release thread locals that may have been allocated (safe as the main heap uses the fast key)+ if (_mi_subproc_is_main(subproc)) {+ _mi_thread_locals_done();+ }+ _mi_heap_force_destroy(subproc->heap_main, false /* don't re-acquire the heaps_lock */); // no warning if destroying the main heap+ }+ }++ subproc->theap_meta = NULL; // theap meta stats are merged during heap_destroy of the main heap++ if (!_mi_subproc_is_main(subproc)) {+ // merge stats back into the main subproc+ _mi_stats_merge_into(&mi_process_subproc_main.stats, &subproc->stats);+ }++ // remove associated arenas+ _mi_arenas_unsafe_destroy_all(subproc);++ // show stats of the main process (at process end) before releasing the heaps lock+ if (_mi_subproc_is_main(subproc)) {+ if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {+ mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL);+ }+ }++ // todo: should we refcount subprocesses?+ mi_lock_done(&subproc->arena_reserve_lock);+ mi_lock_done(&subproc->heaps_lock);+ mi_lock_done(&subproc->theap_meta_lock);+ _mi_meta_free( subproc->parent, subproc, subproc->memid);+ if (_mi_subproc_is_main(subproc)) {+ // for the main subproc, also release the global page map+ _mi_page_map_unsafe_destroy();+ }+}++void mi_subproc_destroy(mi_subproc_id_t subproc_id) {+ mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);+ if (subproc==NULL || subproc==&mi_process_subproc_main) return;+ mi_subproc_unsafe_destroy(subproc, true /* take lock */);+}++void _mi_subprocs_unsafe_destroy_all(void) {+ mi_lock(&mi_subprocs_lock) {+ mi_subproc_t* subproc = mi_subprocs;+ while (subproc!=NULL) {+ mi_subproc_t* next = subproc->next;+ if (subproc!=&mi_process_subproc_main) {+ mi_subproc_unsafe_destroy(subproc, false /* take mi_subprocs lock */);+ }+ subproc = next;+ }+ }+ mi_subproc_unsafe_destroy(&mi_process_subproc_main, true /* take mi_subprocs lock */);+}+++/* -----------------------------------------------------------+ Sub process various+----------------------------------------------------------- */++void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) {+ mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);+ mi_assert_internal(subproc!=NULL);+ if (subproc==NULL) return;+ mi_assert_internal(subproc->heap_main!=NULL);+ if (subproc->heap_main==NULL) return;+ mi_theap_t* theap = _mi_theap_default();+ if (mi_theap_is_initialized(theap)) {+ if (theap->tld!=NULL && theap->tld->subproc != subproc) {+ _mi_warning_message("unable to add thread to the subprocess as it was already in another subprocess (at %p)\n", theap->tld->subproc);+ }+ return;+ }++ // initialize this thread tld & theap+ _mi_thread_init_with_heap(subproc->heap_main);+}+++bool mi_subproc_visit_heaps(mi_subproc_id_t subproc_id, mi_heap_visit_fun* visitor, void* arg) {+ mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);+ if (subproc==NULL) return false;+ bool ok = true;+ mi_lock(&subproc->heaps_lock) {+ for (mi_heap_t* heap = subproc->heaps; heap!=NULL && ok; heap = heap->next) {+ ok = (*visitor)(heap, arg);+ }+ }+ return ok;+}+++mi_subproc_t* _mi_subproc_main_init(void) {+ mi_lock_init(&mi_subprocs_lock);+ mi_memid_t memid = _mi_memid_create_static(&mi_process_subproc_main,sizeof(mi_subproc_t));+ mi_process_subproc_main.memid = memid;+ mi_subproc_init(&mi_process_subproc_main,NULL);+ return &mi_process_subproc_main;+}++void _mi_subproc_main_done(void) {+ mi_lock_done(&mi_subprocs_lock);+}+diff --git a/vendored/mimalloc/src/theap.c b/vendored/mimalloc/src/theap.cindex ef53c1e07..1731830a7 100644--- a/vendored/mimalloc/src/theap.c+++ b/vendored/mimalloc/src/theap.c@@ -1,5 +1,5 @@/*-----------------------------------------------------------------------------Copyright (c) 2018-2025, Microsoft Research, Daan Leijen+Copyright (c) 2018-2026, Microsoft Research, Daan LeijenThis is free software; you can redistribute it and/or modify it under theterms of the MIT license. A copy of the license can be found in the file"LICENSE" at the root of this distribution.@@ -7,7 +7,8 @@ terms of the MIT license. A copy of the license can be found in the file#include "mimalloc.h"#include "mimalloc/internal.h"-#include "mimalloc/prim.h" // _mi_theap_default+#include "mimalloc/prim.h" // _mi_prim_thread_yield+#include "mimalloc/prim-tls.h" // _mi_theap_default#if defined(_MSC_VER) && (_MSC_VER < 1920)#pragma warning(disable:4204) // non-constant aggregate initializer@@ -23,7 +24,7 @@ typedef bool (theap_page_visitor_fun)(mi_theap_t* theap, mi_page_queue_t* pq, mi// Visit all pages in a theap; returns `false` if break was called.static bool mi_theap_visit_pages(mi_theap_t* theap, theap_page_visitor_fun* fn, bool include_full, void* arg1, void* arg2){- if (theap==NULL || theap->page_count==0) return 0;+ if (theap==NULL || theap->page_count==0) return true;// visit all pages#if MI_DEBUG>1@@ -50,19 +51,24 @@ static bool mi_theap_visit_pages(mi_theap_t* theap, theap_page_visitor_fun* fn,}-#if MI_DEBUG>=2+#if MI_DEBUG>=3static bool mi_theap_page_is_valid(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {MI_UNUSED(arg1);MI_UNUSED(arg2);MI_UNUSED(pq);mi_assert_internal(mi_page_theap(page) == theap);+ mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap);+ mi_assert_internal(page_theap == NULL || theap == page_theap);mi_assert_expensive(_mi_page_is_valid(page));return true;}-#endif-#if MI_DEBUG>=3+static bool mi_theap_is_valid(mi_theap_t* theap) {mi_assert_internal(theap!=NULL);+ mi_heap_t* const heap = _mi_theap_heap_peek(theap);+ mi_assert_internal(heap != NULL);+ mi_theap_t* const heap_theap = _mi_heap_theap_peek(heap); // don't use mi_heap_theap as that may re-initialize the thread+ mi_assert_internal(heap_theap==NULL || heap_theap == theap);mi_theap_visit_pages(theap, &mi_theap_page_is_valid, true, NULL, NULL);for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {mi_assert_internal(_mi_page_queue_is_valid(theap, &theap->pages[bin]));@@ -91,9 +97,10 @@ typedef enum mi_collect_e {static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) {MI_UNUSED(arg2);MI_UNUSED(theap);- mi_assert_internal(mi_theap_page_is_valid(theap, pq, page, NULL, NULL));+ mi_assert_expensive(mi_theap_page_is_valid(theap, pq, page, NULL, NULL));mi_collect_t collect = *((mi_collect_t*)arg_collect);- _mi_page_free_collect(page, collect >= MI_FORCE);+ _mi_page_free_collect(page, collect >= MI_FORCE); // update used count+ _mi_page_update_stats(page);if (mi_page_all_free(page)) {// no more used blocks, possibly free the page.if (collect >= MI_FORCE || page->retire_expire == 0) { // either forced/abandon, or not already retired@@ -108,9 +115,10 @@ static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_pagreturn true; // don't break}-static void mi_theap_merge_stats(mi_theap_t* theap) {- mi_assert_internal(mi_theap_is_initialized(theap));- _mi_stats_merge_into(&_mi_theap_heap(theap)->stats, &theap->stats);+void _mi_theap_merge_stats(mi_theap_t* theap) {+ mi_assert_internal(mi_theap_is_initialized(theap));+ mi_heap_t* const heap = _mi_theap_heap(theap);+ _mi_stats_merge_into(&heap->stats, &theap->stats);}static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect)@@ -124,18 +132,22 @@ static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect)// python/cpython#112532: we may be called from a thread that is not the owner of the theap// const bool is_main_thread = (_mi_is_main_thread() && theap->thread_id == _mi_thread_id());- // collect retired pages- _mi_theap_collect_retired(theap, force);+ // collect retired pages (and full pages if theap->allow_page_abandon is false)+ _mi_theap_collect_retired(theap, force);// collect all pages owned by this threadmi_theap_visit_pages(theap, &mi_theap_page_collect, (collect!=MI_NORMAL), &collect, NULL); // dont normally visit full pages, see issue #1220// collect arenas (this is program wide so don't force purges on abandonment of threads)//mi_atomic_storei64_release(&theap->tld->subproc->purge_expire, 1);- _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld);+ if (collect != MI_ABANDON) {+ _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld);+ }// merge statistics- mi_theap_merge_stats(theap);+ if (mi_option_is_enabled(mi_option_collect_merges_stats)) {+ _mi_theap_merge_stats(theap);+ }}void _mi_theap_collect_abandon(mi_theap_t* theap) {@@ -170,24 +182,41 @@ mi_theap_t* mi_theap_get_default(void) {return theap;}+mi_theap_t* mi_theap_set_default(mi_theap_t* theap) {+ mi_theap_t* const previous = mi_theap_get_default();+ if (mi_theap_is_initialized(theap)) {+ _mi_theap_default_set(theap);+ }+ return previous;+}+++static void mi_theap_options_init(mi_theap_t* theap) {+ theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0);+ theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0);+ theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32);+ theap->is_detached = (theap->tld->thread_id == MI_THREADID_DETACHED);+}+// todo: make order of parameters consistent (but would that break compat with CPython?)void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld){mi_assert_internal(theap!=NULL);mi_assert_internal(heap!=NULL);+ mi_assert_internal(tld!=NULL);mi_memid_t memid = theap->memid;_mi_memcpy_aligned(theap, &_mi_theap_empty, sizeof(mi_theap_t));theap->memid = memid;- theap->refcount = 1;theap->tld = tld; // avoid reading the thread-local tld during initialization- mi_atomic_store_ptr_relaxed(mi_heap_t,&theap->heap,heap);+ mi_atomic_store_release(&theap->refcount,1);+ mi_atomic_store_ptr_release(mi_subproc_t,&theap->subproc,heap->subproc);mi_assert_internal(theap->stats.size == sizeof(mi_stats_t));-- _mi_theap_options_init(theap);+ mi_theap_options_init(theap);+if (theap->tld->is_in_threadpool) {// if we run as part of a thread pool it is better to not arbitrarily reclaim abandoned pages into our theap.// this is checked in `free.c:mi_free_try_collect_mt`- // .. but abandoning is good in this case: halve the full page retain (possibly to 0)+ // .. but abandoning is good in this case: quarter the full page retain (possibly to 0)// (so blocked threads do not hold on to too much memory)if (theap->page_full_retain > 0) {theap->page_full_retain = theap->page_full_retain / 4;@@ -196,29 +225,42 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld)// push on the thread local theaps listmi_theap_t* head = NULL;+ mi_random_ctx_t head_random;mi_lock(&theap->tld->theaps_lock) {head = theap->tld->theaps;theap->tprev = NULL;theap->tnext = head;- if (head!=NULL) { head->tprev = theap; }theap->tld->theaps = theap;+ if (head!=NULL) {+ head->tprev = theap;+ head_random = head->random;+ }}- // initialize random- if (head == NULL) { // first theap in this thread?+ // initialize random if heap==NULL+ if (head==NULL) { // first theap of the first thread?#if defined(_WIN32) && !defined(MI_SHARED_LIB)+ if (tld->thread_seq==0) {_mi_random_init_weak(&theap->random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185)- #else- _mi_random_init(&theap->random);+ }+ else#endif+ {+ _mi_random_init(&theap->random);+ }}else {- _mi_random_split(&head->random, &theap->random);+ _mi_random_split(&head_random, &theap->random); // &theap->random is used as nonce so it is ok if threads capture the same head->random+ }+ // theap->cookie = _mi_theap_random_next(theap) | 1;+ _mi_theap_guarded_init(theap); // needs theap->random+ if (!theap->is_detached) {+ mi_subproc_stat_increase(_mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem}- theap->cookie = _mi_theap_random_next(theap) | 1;- _mi_theap_guarded_init(theap);- mi_subproc_stat_increase(_mi_subproc(),theaps,1);+ // only now set the heap member as it is used to determine if a theap is initialized+ mi_atomic_store_ptr_release(mi_heap_t,&theap->heap,heap);+// push on the heap's theap listmi_lock(&heap->theaps_lock) {head = heap->theaps;@@ -229,33 +271,37 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld)}}-mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) {+mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld) {mi_assert_internal(tld!=NULL);mi_assert_internal(heap!=NULL);+ mi_assert_internal(tld->thread_id == MI_THREADID_DETACHED || _mi_thread_id() == tld->thread_id);+ // mi_assert_internal(_mi_heap_theap_peek(heap)==NULL); // don't access thread locals as this is called on thread init+// allocate and initialize a theapmi_memid_t memid;mi_theap_t* theap;- //if (!_mi_is_heap_main(heap)) {- // theap = (mi_theap_t*)mi_heap_zalloc(mi_heap_main(),sizeof(mi_theap_t));- // memid = _mi_memid_create(MI_MEM_HEAP_MAIN);- // memid.initially_zero = memid.initially_committed = true;- //}- //else+if (heap->exclusive_arena == NULL) {- theap = (mi_theap_t*)_mi_meta_zalloc(sizeof(mi_theap_t), &memid);+ theap = (mi_theap_t*)_mi_meta_zalloc(heap->subproc, sizeof(mi_theap_t), &memid);}else {// theaps associated with a specific arena are allocated in that arena// note: takes up at least one slice which is quite wasteful...const size_t size = _mi_align_up(sizeof(mi_theap_t),MI_ARENA_MIN_OBJ_SIZE);- theap = (mi_theap_t*)_mi_arenas_alloc(heap, size, true, true, heap->exclusive_arena, tld->thread_seq, tld->numa_node, &memid);- mi_assert_internal(memid.mem.os.size >= size);+ theap = (mi_theap_t*)_mi_arenas_alloc(heap, size, true, true, heap->exclusive_arena, tld->thread_seq, tld->numa_node, &memid);}if (theap==NULL) {_mi_error_message(ENOMEM, "unable to allocate theap meta-data\n");return NULL;}+theap->memid = memid;+ return theap;+}++mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) {+ mi_theap_t* theap = _mi_theap_alloc(heap,tld);+ if (theap == NULL) return NULL;_mi_theap_init(theap, heap, tld);return theap;}@@ -266,205 +312,129 @@ uintptr_t _mi_theap_random_next(mi_theap_t* theap) {static void mi_theap_free_mem(mi_theap_t* theap) {if (theap!=NULL) {- mi_subproc_stat_decrease(_mi_subproc(),theaps,1);- // free the used memory- if (theap->memid.memkind == MI_MEM_HEAP_MAIN) { // note: for now unused as it would access theap_default stats in mi_free of the current theap- mi_assert_internal(_mi_is_heap_main(mi_heap_of(theap)));- mi_free(theap);- }- else if (theap->memid.memkind == MI_MEM_META) {- _mi_meta_free(theap, sizeof(*theap), theap->memid);- }- else {- _mi_arenas_free(theap, _mi_align_up(sizeof(*theap),MI_ARENA_MIN_OBJ_SIZE), theap->memid ); // issue #1168, avoid assertion failure+ mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc);+ if (!theap->is_detached) {+ mi_subproc_stat_decrease(subproc,theaps,1);}+ _mi_meta_free(subproc, theap, theap->memid);}}+// we need to reference count theaps due to the _mi_theap_cached thread localsvoid _mi_theap_incref(mi_theap_t* theap) {- if (theap!=NULL && theap->memid.memkind > MI_MEM_STATIC) {+ if (theap!=NULL && !mi_memid_needs_no_free(theap->memid)) {mi_atomic_increment_acq_rel(&theap->refcount);}}void _mi_theap_decref(mi_theap_t* theap) {- if (theap!=NULL && theap->memid.memkind > MI_MEM_STATIC) {+ if (theap!=NULL && !mi_memid_needs_no_free(theap->memid)) {if (mi_atomic_decrement_acq_rel(&theap->refcount) == 1) {mi_theap_free_mem(theap);}}}--// called from `mi_theap_delete` to free the internal theap resources.-bool _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock) {- mi_assert(theap != NULL);- if (theap==NULL) return true;-- mi_heap_t* const heap = mi_atomic_exchange_ptr_acq_rel(mi_heap_t, &theap->heap, NULL);- if (heap==NULL) {- // concurrent interaction, retry in an outer loop (as the other thread may be blocked on our lock)- return false;- }- else {- // merge stats to the owning heap- _mi_stats_merge_into(&heap->stats, &theap->stats);-- // remove ourselves from the heap theaps list- mi_lock_maybe(&heap->theaps_lock, acquire_heap_theaps_lock) {- if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; }- if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; }- else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; }- theap->hnext = theap->hprev = NULL;+// Thread termination and heap delete/destroy might run concurrently+// and we need to ensure we free the memory correctly. A heap or tld+// will first "detach" its theaps so it has a list with theaps that are+// no longer shared, and only then free's the theaps in that list.+// To detach we need to hold both the `heap->theaps_lock` and the `tld->theaps_lock`.+// Due to lock-inversion we need to use `mi_lock_try_acquire` and if that fails+// we back-off, release the outer lock, and try again until we succeed.++// Remove the theaps in this heap from any thread local tld lists.+void _mi_heap_detach_theaps( mi_heap_t* heap ) {+ bool all_detached;+ do {+ all_detached = true;+ mi_lock(&heap->theaps_lock) {+ mi_theap_t* theap = heap->theaps;+ while (theap != NULL) {+ mi_theap_t* next = theap->hnext;+ mi_tld_t* tld = theap->tld;+ if (tld != NULL) {+ if (mi_lock_try_acquire(&tld->theaps_lock)) {+ // remove the theap from the tld theaps list+ if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; }+ if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; }+ else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; }+ theap->tnext = theap->tprev = NULL;+ theap->tld = NULL;+ mi_lock_release(&tld->theaps_lock);+ }+ else {+ all_detached = false;+ }+ }+ theap = next;+ }}-- // remove ourselves from the thread local theaps list- mi_lock_maybe(&theap->tld->theaps_lock, acquire_tld_theaps_lock) {- if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; }- if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; }- else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; }- theap->tnext = theap->tprev = NULL;+ if (!all_detached) {+ mi_subproc_stat_counter_increase(heap->subproc,heaps_delete_wait,1);+ _mi_prim_thread_yield();}- theap->tld = NULL;- _mi_theap_decref(theap);- return true;- }-}---/* ------------------------------------------------------------ Heap destroy------------------------------------------------------------ */-/*--// zero out the page queues-static void mi_theap_reset_pages(mi_theap_t* theap) {- mi_assert_internal(theap != NULL);- mi_assert_internal(mi_theap_is_initialized(theap));- // TODO: copy full empty theap instead?- _mi_memset(&theap->pages_free_direct, 0, sizeof(theap->pages_free_direct));- _mi_memcpy_aligned(&theap->pages, &_mi_theap_empty.pages, sizeof(theap->pages));- // theap->thread_delayed_free = NULL;- theap->page_count = 0;-}--static bool _mi_theap_page_destroy(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {- MI_UNUSED(arg1);- MI_UNUSED(arg2);- MI_UNUSED(pq);-- // ensure no more thread_delayed_free will be added- //_mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false);-- // stats- const size_t bsize = mi_page_block_size(page);- if (bsize > MI_LARGE_MAX_OBJ_SIZE) {- mi_theap_stat_decrease(theap, malloc_huge, bsize);- }- #if (MI_STAT>0)- _mi_page_free_collect(page, false); // update used count- const size_t inuse = page->used;- if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {- mi_theap_stat_decrease(theap, malloc_normal, bsize * inuse);- #if (MI_STAT>1)- mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], inuse);- #endif- }- // mi_theap_stat_decrease(theap, malloc_requested, bsize * inuse); // todo: off for aligned blocks...- #endif-- /// pretend it is all free now- mi_assert_internal(mi_page_thread_free(page) == NULL);- page->used = 0;-- // and free the page- // mi_page_free(page,false);- page->next = NULL;- page->prev = NULL;- mi_page_set_theap(page, NULL);- _mi_arenas_page_free(page, theap);-- return true; // keep going-}--void _mi_theap_destroy_pages(mi_theap_t* theap) {- mi_theap_visit_pages(theap, &_mi_theap_page_destroy, NULL, NULL);- mi_theap_reset_pages(theap);-}--#if MI_TRACK_HEAP_DESTROY-static bool mi_cdecl mi_theap_track_block_free(const mi_theap_t* theap, const mi_theap_area_t* area, void* block, size_t block_size, void* arg) {- MI_UNUSED(theap); MI_UNUSED(area); MI_UNUSED(arg); MI_UNUSED(block_size);- mi_track_free_size(block,mi_usable_size(block));- return true;-}-#endif--void mi_theap_destroy(mi_theap_t* theap) {- mi_assert(theap != NULL);- mi_assert(mi_theap_is_initialized(theap));- mi_assert(!theap->allow_page_reclaim);- mi_assert(!theap->allow_page_abandon);- mi_assert_expensive(mi_theap_is_valid(theap));- if (theap==NULL || !mi_theap_is_initialized(theap)) return;- #if MI_GUARDED- // _mi_warning_message("'mi_theap_destroy' called but MI_GUARDED is enabled -- using `mi_theap_delete` instead (theap at %p)\n", theap);- mi_theap_delete(theap);- return;- #else- if (theap->allow_page_reclaim) {- _mi_warning_message("'mi_theap_destroy' called but ignored as the theap was not created with 'allow_destroy' (theap at %p)\n", theap);- // don't free in case it may contain reclaimed pages,- mi_theap_delete(theap);- }- else {- // track all blocks as freed- #if MI_TRACK_HEAP_DESTROY- mi_theap_visit_blocks(theap, true, mi_theap_track_block_free, NULL);- #endif- // free all pages- _mi_theap_destroy_pages(theap);- mi_theap_free(theap,true);- }- #endif-}--// forcefully destroy all theaps in the current thread-void _mi_theap_unsafe_destroy_all(mi_theap_t* theap) {- mi_assert_internal(theap != NULL);- if (theap == NULL) return;- mi_theap_t* curr = theap->tld->theaps;- while (curr != NULL) {- mi_theap_t* next = curr->next;- if (!curr->allow_page_reclaim) {- mi_theap_destroy(curr);+ } while (!all_detached);+}++// Remove the theaps in this thread from the heaps that own them.+void _mi_tld_detach_theaps( mi_tld_t* tld ) {+ bool all_detached;+ do {+ all_detached = true;+ mi_lock(&tld->theaps_lock) {+ mi_theap_t* theap = tld->theaps;+ while (theap != NULL) {+ mi_theap_t* next = theap->tnext;+ mi_assert_internal(theap->page_count==0);+ mi_heap_t* heap = _mi_theap_heap_peek(theap); // now the heap might be NULL from an earlier iteration+ if (heap != NULL) {+ if (mi_lock_try_acquire(&heap->theaps_lock)) {+ // merge stats into the owning heap stats+ _mi_stats_merge_into(&heap->stats, &theap->stats);+ // remove the theap from the heap list+ if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; }+ if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; }+ else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; }+ theap->hnext = theap->hprev = NULL;+ // and set `heap` to NULL+ mi_atomic_store_ptr_release(mi_heap_t, &theap->heap, NULL);+ mi_lock_release(&heap->theaps_lock);+ }+ else {+ all_detached = false;+ }+ }+ theap = next;+ }}- else {- _mi_theap_destroy_pages(curr);+ if (!all_detached) {+ mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1);+ _mi_prim_thread_yield();}- curr = next;- }+ } while (!all_detached);}-*/++/* ------------------------------------------------------------ Safe Heap delete+ Safe theap delete----------------------------------------------------------- */// Safe delete a theap without freeing any still allocated blocks in that theap.-void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock)-{- mi_assert(theap != NULL);- mi_assert(mi_theap_is_initialized(theap));- mi_assert_expensive(mi_theap_is_valid(theap));- if (theap==NULL || !mi_theap_is_initialized(theap)) return;+// void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock)+// {+// mi_assert(theap != NULL);+// mi_assert(mi_theap_is_initialized(theap));+// mi_assert_expensive(mi_theap_is_valid(theap));+// if (theap==NULL || !mi_theap_is_initialized(theap)) return;- // abandon all pages- _mi_theap_collect_abandon(theap);+// // abandon all pages+// _mi_theap_collect_abandon(theap);- mi_assert_internal(theap->page_count==0);- _mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock);-}+// mi_assert_internal(theap->page_count==0);+// _mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock);+// }@@ -518,7 +488,7 @@ bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena_id) {// reinit direct pages (as we may be in a different process)mi_assert_internal(theap->page_count == 0);for (size_t i = 0; i < MI_PAGES_DIRECT; i++) {- theap->pages_free_direct[i] = (mi_page_t*)&_mi_page_empty;+ theap->pages_free_direct[i] = _mi_page_empty_get();}// push on the thread local theaps list@@ -541,7 +511,7 @@ void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) {area->reserved = page->reserved * bsize;area->committed = page->capacity * bsize;area->blocks = mi_page_start(page);- area->used = page->used; // number of blocks in use (#553)+ area->used = mi_page_used(page); // number of blocks in use (#553)area->block_size = ubsize;area->full_block_size = bsize;area->reserved1 = page;@@ -567,7 +537,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_mi_page_free_collect(page,true); // collect both thread_delayed and local_freemi_assert_internal(page->local_free == NULL);- if (page->used == 0) return true;+ if (mi_page_used(page) == 0) return true;size_t psize;uint8_t* const pstart = mi_page_area(page, &psize);@@ -577,13 +547,13 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi// optimize page with one blockif (page->capacity == 1) {- mi_assert_internal(page->used == 1 && page->free == NULL);+ mi_assert_internal(mi_page_used(page) == 1 && page->free == NULL);return visitor(heap, area, pstart, ubsize, arg);}mi_assert(bsize <= UINT32_MAX);// optimize full pages- if (page->used == page->capacity) {+ if (mi_page_used(page) == page->capacity) {uint8_t* block = pstart;for (size_t i = 0; i < page->capacity; i++) {if (!visitor(heap, area, block, ubsize, arg)) return false;@@ -627,7 +597,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, misize_t bit = blockidx - (bitidx * MI_INTPTR_BITS);free_map[bitidx] |= ((uintptr_t)1 << bit);}- mi_assert_internal(page->capacity == (free_count + page->used));+ mi_assert_internal(page->capacity == (free_count + mi_page_used(page)));// walk through all blocks skipping the free ones#if MI_DEBUG>1@@ -659,10 +629,15 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, miblock += bsize * MI_INTPTR_BITS;}}- mi_assert_internal(page->used == used_count);+ mi_assert_internal(mi_page_used(page) == used_count);return true;}+// bool _mi_page_visit_blocks( mi_page_t* page, mi_block_visit_fun* visitor, void* arg ) {+// mi_heap_area_t area;+// _mi_heap_area_init(&area, page);+// return _mi_theap_area_visit_blocks(&area, page, visitor, arg);+// }// Separate struct to keep `mi_page_t` out of the public interfacediff --git a/vendored/mimalloc/src/threadlocal.c b/vendored/mimalloc/src/threadlocal.cindex eea3329bf..6f280d093 100644--- a/vendored/mimalloc/src/threadlocal.c+++ b/vendored/mimalloc/src/threadlocal.c@@ -6,9 +6,9 @@ terms of the MIT license. A copy of the license can be found in the file-----------------------------------------------------------------------------*//* -----------------------------------------------------------------------------Implement dynamic thread local variables (for heap's).-Unlike most OS native implementations there is no limit on the number-that can be allocated.+ Implement dynamic thread local variables (used by heap's for their theap's).+ Unlike most OS native implementations there is no limit on the number+ that can be allocated.-----------------------------------------------------------------------------*/#include "mimalloc.h"@@ -27,12 +27,41 @@ typedef struct mi_tls_slot_s {typedef struct mi_thread_locals_s {size_t count;+ mi_memid_t memid;mi_tls_slot_t slots[1];} mi_thread_locals_t;-static mi_thread_locals_t mi_thread_locals_empty = { 0, {{0,NULL}} };+static mi_thread_locals_t mi_thread_locals_empty = mi_init_struct_zero;-mi_decl_thread mi_thread_locals_t* mi_thread_locals = &mi_thread_locals_empty; // always point to a valid `mi_thread_locals_t`++/* -----------------------------------------------------------+ We have 2 thread local variable which we implement with either+ a C thread local declaration or using pthread keys.+ - mi_thread_locals: points to an array of thread locals for most keys+ - mi_slot_fast: a single dedicated thread local for slightly faster access. (used for the main heap's theap)+----------------------------------------------------------- */++#if MI_TLS_MODEL_PTHREADS || defined(__APPLE__) // macOS has fast pthreads+// Use pthreads+#define mi_define_thread_local(tp,name,initval) \+ static pthread_key_t __##name##_key = MI_PTHREAD_KEY_INVALID; \+ static inline tp name##_peek(void) { return (tp)mi_pthread_key_get(__##name##_key); } \+ static inline tp name##_get(void) { tp result = name##_peek(); return (result!=NULL ? result : initval); } \+ static inline bool name##_set(tp val) { return mi_pthread_key_set(&__##name##_key,val); } \+ static inline void name##_delete(void) { mi_pthread_key_delete(&__##name##_key); }++#else+// Direct thread locals+#define mi_define_thread_local(tp,name,initval) \+ static mi_decl_thread tp __##name = initval; \+ static inline tp name##_peek(void) { return __##name; } \+ static inline tp name##_get(void) { tp result = __##name; return (result!=NULL ? result : initval); } \+ static inline bool name##_set(tp val) { __##name = val; return true; } \+ static inline void name##_delete(void) { }+#endif++mi_define_thread_local(mi_thread_locals_t*, mi_thread_locals, &mi_thread_locals_empty)+mi_define_thread_local(void*, mi_slot_fast, NULL)/* -----------------------------------------------------------@@ -42,15 +71,18 @@ mi_decl_thread mi_thread_locals_t* mi_thread_locals = &mi_thread_locals_empty;a value, we also set the version of the key.----------------------------------------------------------- */-#if MI_SIZE_BITS < 64-#define MI_TLS_IDX_BITS (MI_SIZE_BITS/2) // half for the index, half for the version+#if MI_SIZE_BITS >= 64+#define MI_TLS_IDX_BITS (MI_SIZE_BITS/4) /* 16 bits for the index, 48 bits for the version */+#elif MI_SIZE_BITS >= 32+#define MI_TLS_IDX_BITS (12) /* 12 bits for index, 20 for the version? */#else-#define MI_TLS_IDX_BITS (MI_SIZE_BITS/4) // 16 bits for the index, 48 bits for the version+#error not enough bits for the version for thread locals#endif#define MI_TLS_IDX_MASK ((MI_ZU(1)<<MI_TLS_IDX_BITS)-1)#define MI_TLS_IDX_MAX MI_TLS_IDX_MASK#define MI_TLS_VERSION_MAX ((MI_ZU(1)<<(MI_SIZE_BITS - MI_TLS_IDX_BITS))-1)+static size_t mi_key_index( size_t key ) {return (key & MI_TLS_IDX_MASK);}@@ -70,36 +102,43 @@ static mi_thread_local_t mi_key_create( size_t index, size_t version ) {// dynamically reallocate the thread local slots when neededstatic mi_thread_locals_t* mi_thread_locals_expand(size_t least_idx) {- mi_thread_locals_t* tls_old = mi_thread_locals;+ mi_thread_locals_t* tls_old = mi_thread_locals_get();const size_t count_old = tls_old->count;size_t count;if (count_old==0) {tls_old = NULL; // so we allocate fresh from mi_thread_locals_emptycount = 16; // start with 16 slots- }+ }else if (count_old >= 1024) {count = count_old + 1024; // at some point increase linearly}else {count = 2*count_old; // and double initially}- if (count <= least_idx) {+ if (count <= least_idx) {count = least_idx + 1;}if (count > MI_TLS_IDX_MAX) { return NULL; } // too large- mi_thread_locals_t* tls = (mi_thread_locals_t*)mi_rezalloc(tls_old, sizeof(mi_thread_locals_t) + count*sizeof(mi_tls_slot_t));+ // allocate as meta (for secure mode)+ // we could also allocate on the main heap; this is recursion safe as that uses the fast local key+ mi_memid_t memid = (tls_old==NULL ? _mi_memid_none() : tls_old->memid);+ mi_thread_locals_t* tls = (mi_thread_locals_t*)_mi_meta_rezalloc(_mi_subproc(), tls_old, sizeof(mi_thread_locals_t) + count*sizeof(mi_tls_slot_t), &memid);if mi_unlikely(tls==NULL) return NULL;+ tls->memid = memid;tls->count = count;- mi_thread_locals = tls;+ mi_thread_locals_set(tls);return tls;}static mi_decl_noinline bool mi_thread_local_set_expand( mi_thread_local_t key, void* val ) {if (val==NULL) return true;- const size_t idx = mi_key_index(key);+ const size_t idx = mi_key_index(key);mi_thread_locals_t* tls = mi_thread_locals_expand(idx);- if (tls==NULL) return false;- mi_assert_internal(tls == mi_thread_locals);+ if (tls==NULL) {+ _mi_error_message(EFAULT,"unable to allocate thread local variables\n");+ return false;+ }+ mi_assert_internal(tls == mi_thread_locals_get());mi_assert_internal(idx < tls->count);tls->slots[idx].value = val;tls->slots[idx].version = mi_key_version(key);@@ -108,8 +147,8 @@ static mi_decl_noinline bool mi_thread_local_set_expand( mi_thread_local_t key,// set a tls slot; returns `true` if successful.// Can return `false` if we could not reallocate the slots array.-bool _mi_thread_local_set( mi_thread_local_t key, void* val ) {- mi_thread_locals_t* tls = mi_thread_locals;+static mi_decl_noinline bool mi_thread_local_set_regular( mi_thread_local_t key, void* val ) {+ mi_thread_locals_t* tls = mi_thread_locals_get();mi_assert_internal(tls!=NULL);mi_assert_internal(key!=0);const size_t idx = mi_key_index(key);@@ -123,25 +162,54 @@ bool _mi_thread_local_set( mi_thread_local_t key, void* val ) {}}-// get a tls slot value-void* _mi_thread_local_get( mi_thread_local_t key ) {- const mi_thread_locals_t* const tls = mi_thread_locals;- mi_assert_internal(tls!=NULL);+bool _mi_thread_local_set( mi_thread_local_t key, void* val ) {mi_assert_internal(key!=0);+ if (key == mi_thread_local_key_fast) {+ return mi_slot_fast_set(val);+ }+ else {+ return mi_thread_local_set_regular(key,val);+ }+}++// get a tls slot value+static mi_decl_noinline void* mi_thread_local_get_regular( mi_thread_local_t key ) {+ const mi_thread_locals_t* const tls = mi_thread_locals_peek();+ if mi_unlikely(tls==NULL) {+ // this can happen if a thread local is accessed after the thread local has been freed+ // from mi_thread_done or mi_process_done.+ // todo: can we remove this check? now we can still call this from process done when stats are printed (which calls mi_heap_theap_peek)+ return NULL;+ }const size_t idx = mi_key_index(key);if mi_likely(idx < tls->count && mi_key_version(key) == tls->slots[idx].version) {return tls->slots[idx].value;}else {- return NULL;+ return NULL;+ }+}++// get a thread local value+void* _mi_thread_local_get( mi_thread_local_t key ) {+ if mi_likely(key == mi_thread_local_key_fast) {+ return mi_slot_fast_get();+ }+ else {+ void* p = mi_thread_local_get_regular(key);+ if (key==0) { mi_assert_internal(p==NULL); }+ return p;}}void _mi_thread_locals_thread_done(void) {- mi_thread_locals_t* const tls = mi_thread_locals;+ mi_thread_locals_t* const tls = mi_thread_locals_peek();if (tls!=NULL && tls->count > 0) {- mi_free(tls);- mi_thread_locals = &mi_thread_locals_empty;+ _mi_meta_free(_mi_subproc(), tls, tls->memid);+ mi_thread_locals_set(NULL);+ }+ if (mi_slot_fast_peek() != NULL) {+ mi_slot_fast_set(NULL);}}@@ -152,6 +220,7 @@ Create and free fresh TLS key'sstatic mi_lock_t mi_thread_locals_lock; // we need a lock in order to re-allocate the slot bitsstatic mi_bitmap_t* mi_thread_locals_free; // reuse an arena bitmap to track which slots were assigned (1=free, 0=in-use)+static mi_memid_t mi_thread_locals_memid; // provenance of mi_thread_locals_freestatic size_t mi_thread_locals_version; // version to be able to reuse slots safelyvoid _mi_thread_locals_init(void) {@@ -161,9 +230,13 @@ void _mi_thread_locals_init(void) {void _mi_thread_locals_done(void) {mi_lock(&mi_thread_locals_lock) {mi_bitmap_t* const slots = mi_thread_locals_free;- mi_free(slots);+ if (slots!=NULL) {+ _mi_meta_free(_mi_subproc_main(), slots, mi_thread_locals_memid);+ }}mi_lock_done(&mi_thread_locals_lock);+ mi_thread_locals_delete();+ mi_slot_fast_delete();}// strange signature but allows us to reuse the arena code for claiming free pages@@ -173,7 +246,7 @@ static bool mi_thread_local_claim_fun(size_t _slice_index, mi_arena_t* _arena, breturn true;}-// When we claim a free slot, we increase the global version counter+// When we claim a free slot, we increase the global version counter// (so if we reuse a slot it will be returning NULL initially when a thread tries to get it)static mi_thread_local_t mi_thread_local_claim(void) {size_t idx = 0;@@ -194,16 +267,21 @@ static bool mi_thread_local_create_expand(void) {const size_t newcount = 1024 + oldcount;if (newcount > MI_TLS_IDX_MAX) { return false; }const size_t newsize = mi_bitmap_size( newcount, NULL );- mi_bitmap_t* newslots = (mi_bitmap_t*)mi_zalloc_aligned(newsize, MI_BCHUNK_SIZE);+ // mi_bitmap_t* newslots = (mi_bitmap_t*)mi_zalloc_aligned(newsize, MI_BCHUNK_SIZE);+ mi_memid_t memid;+ mi_bitmap_t* newslots = (mi_bitmap_t*)_mi_meta_zalloc_aligned(_mi_subproc_main(), newsize, MI_BCHUNK_SIZE, &memid); // always allocate thread locals in the main subprocess+ mi_assert_internal(_mi_is_aligned(newslots,MI_BCHUNK_SIZE));if (newslots==NULL) { return false; }if (slots!=NULL) {// copy over the previous bitmap- _mi_memcpy_aligned(newslots, slots, mi_bitmap_size(oldcount, NULL));- mi_free(slots);+ const size_t oldsize = mi_bitmap_size(oldcount,NULL);+ _mi_memcpy_aligned(newslots, slots, oldsize);+ _mi_meta_free(_mi_subproc_main(), slots, mi_thread_locals_memid);}mi_bitmap_init(newslots, newcount, true /* pretend already zero'd so we do not zero out the copied old entries */);mi_bitmap_unsafe_setN(newslots, oldcount, newcount - oldcount); /* set the new expanded slots as available */mi_thread_locals_free = newslots;+ mi_thread_locals_memid = memid;return true;}@@ -219,6 +297,8 @@ mi_thread_local_t _mi_thread_local_create(void) {}}}+ mi_assert_internal(key!=0);+ mi_assert_internal(key!=mi_thread_local_key_fast);return key;}