Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(mem): vendor mimalloc 3.5.3 so free(NULL) before init is safe

Martin Vogelf9f44b586996
diff --git a/Makefile.cbm b/Makefile.cbm
index 3b93146f1..32124ab12 100644
--- a/Makefile.cbm
+++ b/Makefile.cbm
@@ -45,8 +45,9 @@ endif
# -Wdate-time: with -Werror, any use of __DATE__/__TIME__/__TIMESTAMP__ fails the
# build. Build-time entropy is how one identical source tree produced a new hash
-# on every compile (see the local patch in vendored/mimalloc/src/options.c) —
-# once removed, it must not creep back through a new call site.
+# on every compile (mimalloc's version banner, patched locally until upstream
+# dropped the clock in v3.4.5; see vendored/mimalloc/PATCHES.md) — once
+# removed, it must not creep back through a new call site.
CFLAGS_COMMON = -std=c11 -D_DEFAULT_SOURCE -D_GNU_SOURCE -Wall -Wextra -Werror \
-Wno-unused-parameter -Wno-sign-compare -Wdate-time \
$(GCC_ONLY_FLAGS) \
@@ -182,8 +183,9 @@ endif
ifeq ($(IS_MINGW),yes)
# --no-insert-timestamp: the PE header otherwise carries the link wall clock, so
# two Windows builds of identical source are never byte-identical and every
-# release is a brand-new file to a reputation system. Same motivation as the
-# mimalloc __DATE__ patch; ELF and Mach-O have no equivalent field to clear.
+# release is a brand-new file to a reputation system. Same motivation as
+# dropping mimalloc's __DATE__ banner; ELF and Mach-O have no equivalent field
+# to clear.
WIN32_LIBS := -lws2_32 -lpsapi -lshell32 -ladvapi32 -lbcrypt -Wl,--allow-multiple-definition -Wl,--stack,8388608 -Wl,--no-insert-timestamp -static
endif
@@ -653,10 +655,14 @@ endif
# work inside the allocator (2026-09-17). Committed-bytes accounting, which the
# memory budget reads, is kept in every build (subproc stats, not MI_STAT).
# The TEST build below keeps mimalloc's debug checks.
+# MI_PROFILE=0: mimalloc 3.5.2+ compiles its allocation-sampling profiler in by
+# default, which puts a sample countdown on the allocation path. The product
+# never uses it, so the production allocator leaves it out (the contract in
+# tests/test_prod_allocator_contract.sh checks the banner has no profiling line).
MIMALLOC_CFLAGS = -std=c11 -O2 -w \
-Ivendored/mimalloc/include \
-Ivendored/mimalloc/src \
- -DMI_OVERRIDE=1 -DMI_DEBUG=0 \
+ -DMI_OVERRIDE=1 -DMI_DEBUG=0 -DMI_PROFILE=0 \
$(MIMALLOC_OVERRIDE_DEFINE)
MIMALLOC_CFLAGS_TEST = -std=c11 -g -O1 -w \
-Ivendored/mimalloc/include \
diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md
index 4b76c28e8..17e88a695 100644
--- a/THIRD_PARTY.md
+++ b/THIRD_PARTY.md
@@ -121,10 +121,12 @@ License summary:
| wyhash | `internal/cbm/vendored/wyhash/` | Unlicense (public domain) | [wangyi-fudan/wyhash](https://github.com/wangyi-fudan/wyhash) |
Local modifications to these libraries are documented next to the
-vendored sources (currently only SQLite: `vendored/sqlite3/PATCHES.md`,
-raising the Unix VFS `MAX_PATHNAME` ceiling from 512 to 4096 to match
-CBM's 4 KiB path support). Patches must be reapplied on every upstream
-refresh and are covered by `scripts/vendored-checksums.txt`.
+vendored sources: `vendored/sqlite3/PATCHES.md` (raising the Unix VFS
+`MAX_PATHNAME` ceiling from 512 to 4096 to match CBM's 4 KiB path support)
+and `vendored/mimalloc/PATCHES.md` (the upstream source pin, mimalloc v3.5.3
+at commit `d4881d33`, plus the macOS decommit-recommit change and one comment
+rewording). Patches must be reapplied on every upstream refresh and are
+covered by `scripts/vendored-checksums.txt`.
The graph-UI HTTP server is a first-party implementation
(`src/ui/httpd.c` + `src/ui/http_server.c`) — no third-party HTTP library
diff --git a/scripts/ci/generate-sbom.py b/scripts/ci/generate-sbom.py
index ff2e97efd..1a7d8eacb 100644
--- a/scripts/ci/generate-sbom.py
+++ b/scripts/ci/generate-sbom.py
@@ -40,7 +40,7 @@ sbom = {
"packages": [
{"SPDXID": "SPDXRef-Package-sqlite3", "name": "sqlite3", "versionInfo": "3.51.3", "licenseDeclared": "blessing", "downloadLocation": "https://sqlite.org", "filesAnalyzed": False},
{"SPDXID": "SPDXRef-Package-yyjson", "name": "yyjson", "versionInfo": "0.12.0", "licenseDeclared": "MIT", "downloadLocation": "https://github.com/ibireme/yyjson", "filesAnalyzed": False},
- {"SPDXID": "SPDXRef-Package-mimalloc", "name": "mimalloc", "versionInfo": "3.3.2", "licenseDeclared": "MIT", "downloadLocation": "https://github.com/microsoft/mimalloc", "filesAnalyzed": False},
+ {"SPDXID": "SPDXRef-Package-mimalloc", "name": "mimalloc", "versionInfo": "3.5.3", "licenseDeclared": "MIT", "downloadLocation": "https://github.com/microsoft/mimalloc", "filesAnalyzed": False},
{"SPDXID": "SPDXRef-Package-xxhash", "name": "xxhash", "versionInfo": "0.8.3", "licenseDeclared": "BSD-2-Clause", "downloadLocation": "https://github.com/Cyan4973/xxHash", "filesAnalyzed": False},
{"SPDXID": "SPDXRef-Package-tre", "name": "tre", "versionInfo": "0.8.0", "licenseDeclared": "BSD-2-Clause", "downloadLocation": "https://github.com/laurikari/tre", "filesAnalyzed": False},
{"SPDXID": "SPDXRef-Package-tree-sitter", "name": "tree-sitter", "versionInfo": "0.24.4", "licenseDeclared": "MIT", "downloadLocation": "https://github.com/tree-sitter/tree-sitter", "filesAnalyzed": False},
diff --git a/scripts/vendored-checksums.txt b/scripts/vendored-checksums.txt
index 7762ba82f..845f125a7 100644
--- a/scripts/vendored-checksums.txt
+++ b/scripts/vendored-checksums.txt
@@ -992,50 +992,56 @@ a86b7735823b0408a4f3b922be86d4d25b6f4ab0dab32e03fb77246ca439dcf2 internal/cbm/v
9b4bc8245565c98ccfc61c07749928b57e7c0f6fddb0530c4f6aa1971893d88b internal/cbm/vendored/zstd/zstd.h
66a8c3f71d12ea6e797e4f622f31f3f8f81c41b36f48cad4f5de7d8bfb6aac0a internal/cbm/vendored/zstd/zstd_errors.h
82ade5d7d9b029044b5fbbc326207fc47c2d44ee4dd71e8a559c36d728217de9 vendored/mimalloc/LICENSE
-dd4f25cae53209d45d73f8e6a2b9c219e8fc7434d97f20eba9af1a8b850030fd vendored/mimalloc/include/mimalloc-new-delete.h
+edb95ac4f912bbcde6c1b658e147e757896c881eae249dbb5bbbae66ebdf6deb vendored/mimalloc/PATCHES.md
+0a9c7e6daea2814876eab49697922f98b630c27ab9b089fbe4bd74f6e56edd68 vendored/mimalloc/include/mimalloc-new-delete.h
21fcf61c4443341ac6bf6ea528af31dc7267e8e3456fc64bfd07704503032175 vendored/mimalloc/include/mimalloc-override.h
-5260df227ea1612b8678a06b063cc8f48e739b551554541cdeb4d430a7e257b3 vendored/mimalloc/include/mimalloc-stats.h
-869161433ab2d807a75752d0762accf5b68b6b24ce67e417af46bf181b9b6c72 vendored/mimalloc/include/mimalloc.h
-e9512b521a82cc0cbc262f6060dd70bb4243ac4a7d69d6ab8b1ef8e56f181a21 vendored/mimalloc/include/mimalloc/atomic.h
-57a7297e18ae464a7bfd3fc25a46229008602385697305c69ed289b89ece7631 vendored/mimalloc/include/mimalloc/bits.h
-5aeeff21d667daf4aadcc66f634aa957ecd688d15b277828f367c8c2274251a8 vendored/mimalloc/include/mimalloc/internal.h
-555302d2fcd05e974441d0c9c9208c26e860db4b3f17b9025a08f42e0f4807ba vendored/mimalloc/include/mimalloc/prim.h
-1f378b4b6063c078d2aeb5d916c1d30b5db787a44ab9118e7daf673b086882c6 vendored/mimalloc/include/mimalloc/track.h
-b54e760da2a8d9d8a9cb708e280268c2d194c99bc518b21f72db70100578ad6a vendored/mimalloc/include/mimalloc/types.h
-70420b4430da4bb6b5197fd27c6c0df485249c53a986bc9536c9fb643fc7613b vendored/mimalloc/src/alloc-aligned.c
-6c8e9b0d042481c55791edb7330daec64bedaf26f0e70c9ca40de3cc17a690fe vendored/mimalloc/src/alloc-override.c
-98d0eb877d7b9cdf7e9f857aaf3c739b69f746b91221e0e427e82bdd7cec90b3 vendored/mimalloc/src/alloc-posix.c
-a92d58e47220e5b6d138b83d92498771be4195ee59136a134e4461f6f726b9b8 vendored/mimalloc/src/alloc.c
-86fde644ede2e8d582e53499568a1713516a20326e0350eaa06a28029cece91b vendored/mimalloc/src/arena-meta.c
-a1d6900d2b9a17461aa2a72a7791dfb22f6167b86f7ac6f15f981d9735aa2adb vendored/mimalloc/src/arena.c
-de16033e029488e9ade4152ea1274e1a5ecbdc2c8a8d8ee50b745fd037ee6f6c vendored/mimalloc/src/bitmap.c
-2e7a54ff44592a1fe8fe929e691322622f8aa1e0b1e80954781f332a88dd6cd1 vendored/mimalloc/src/bitmap.h
-c9a845dd7b9020f26d73761d235fb75e7bc4a3682576de750050ac4098211b43 vendored/mimalloc/src/free.c
-ae8e88737c9861e8e071fe5f1e7acc52093bcffd136a359a6b473101e968c14f vendored/mimalloc/src/heap.c
-eca1fe77d72ab2aaccc447c8119056e9c4e8851bda9a95472b573539f76107ad vendored/mimalloc/src/init.c
-ebd95b475f15d5387e6ef3c88a9cb47b88e64d7e55998ae1dc656614676a443b vendored/mimalloc/src/libc.c
-192ce06ac9076445d223205fb340d8a1fc818dbf2b4c4a801dfd2c10bc6c1162 vendored/mimalloc/src/options.c
-011e965f28c0e2c885a6834d51bd756a458dd37ebfed6fd4d9ad6607ce0390ed vendored/mimalloc/src/os.c
-1be3a208c678fc34187c2a8ffb75f3fe1eecc674bbf8b2af4d1f329506b1fad5 vendored/mimalloc/src/page-map.c
-7ec275805c9f00eb66710ffa99817b3ed1dba9826435d08f4866f7935152f6ae vendored/mimalloc/src/page-queue.c
-d3614a59f2407a32e75b051deb1aff31b233f587d325dcc5420ba508551b687e vendored/mimalloc/src/page.c
-7488e9e562ca85d2d757ec48e4d334935263a2333aa05b6061d8520f427c5890 vendored/mimalloc/src/prim/emscripten/prim.c
-ce14eb6dfd55f241095b55f70983931a2d92f7a6a622a9b360cdd70ab3b53208 vendored/mimalloc/src/prim/osx/alloc-override-zone.c
+c0aabbf2fbe54ce55e7dee23cb1891c2d65e0e1f5b18d9fe1a4e733c0c91d996 vendored/mimalloc/include/mimalloc-profile.h
+57aaf0b0db23521a32b0c6d7e0969e37ce69466fcfb0cb9b2d0114c2b56be182 vendored/mimalloc/include/mimalloc-stats.h
+8ec1bff9902bab6682e3ecbf7613ccc915e082ca743fdd0c396d3954323fe8bf vendored/mimalloc/include/mimalloc.h
+7be611200328a99b052eda81abb8ad78b7e03446a0f6d302c1eb31cc9f096398 vendored/mimalloc/include/mimalloc/atomic.h
+1ecfb1bd1a7b6f73c335d21f3411c9ce0de3cbf6710e2d209e6cb6df2baf1ed3 vendored/mimalloc/include/mimalloc/bits.h
+26c10f7b990c51629618c74c64e3263a1fe00da787c2e72c56a79f87393f602c vendored/mimalloc/include/mimalloc/internal.h
+a701ed0c182b4d0039bf683b4ac197011b8b794971a616c76ca64f036bc30d8b vendored/mimalloc/include/mimalloc/prim-tls.h
+1987e8e2eedc07bb181bf2a11a27bec80a5309c32cfa66a56900fb4cbb64b172 vendored/mimalloc/include/mimalloc/prim.h
+31d94e981640bc4c340f21887610a9cd7242af4c5adc578f722ad03cb852db2e vendored/mimalloc/include/mimalloc/track.h
+01abbaf1cc5572675b97f4115f205335e319b93249ee5163fb3bf2ad0fbbceae vendored/mimalloc/include/mimalloc/types.h
+8390ff8d0e6c2954db1d95bfc1a4388b302605a1c5102c0495e5163838186e7f vendored/mimalloc/src/alloc-aligned.c
+6576949f30978f3b7544d9696af12dac9615074a5c2fc05e3681475575bc6ca3 vendored/mimalloc/src/alloc-override.c
+a6dcdef4694964c972e6cf37f197f93c95f321566a3b5ccc1761129257255e93 vendored/mimalloc/src/alloc-posix.c
+e074a8a622f6317d42eb43e48db8cde3345696c96d6749523ef4fd58aeb55bb7 vendored/mimalloc/src/alloc.c
+27fce7003a5108fb9d0c61e92e0cd7870f687f7e3cf12ad19c7fb3954b2a0bc2 vendored/mimalloc/src/arena.c
+b33f7b220358667fb0eddcd6569ec7b79f7ef61daf08448c8448c8d1a8741b09 vendored/mimalloc/src/bitmap.c
+f3a9375ec0b56d515bea4b23732bf9a04964efecc516eb8aaac8b58ebf4512d9 vendored/mimalloc/src/bitmap.h
+56a1c57bd08c49359e19e8cdcd2a11b50c30bb12a53063375374fb3f5c8d6189 vendored/mimalloc/src/free.c
+a33d339568e5a7d16d6bae5650a79b1aef649e9b4ad0261be982580c2971eb64 vendored/mimalloc/src/heap.c
+1452ca684eca6d959eec9eae91750162f7738055f995bd4863f79b87aab88a8c vendored/mimalloc/src/init.c
+b774d5750409006b690ab6262f037d78362ca978729314493636b358249e7fbd vendored/mimalloc/src/libc.c
+5bfe80a0c473371d299840e8ed20831601d4998e4164e2e0bf71303697395824 vendored/mimalloc/src/options.c
+06b2d6d8a4cf3bfcc1822bb09aeaa4e7b741e7a5241f70b4f693b8038f455b00 vendored/mimalloc/src/os.c
+ff3509ae3d4185e9cb2a95e1f35daba3329e998bac917ac7938f1bf06dba0f78 vendored/mimalloc/src/page-map.c
+4128ca8b8d4df1116f73bf9a397903ca9941701c635345b1241295c3a31ff47d vendored/mimalloc/src/page-queue.c
+ad0df1217e96669ef63539c345ea7dc4d637c3269be2d4de36ac3a5a42fb0c12 vendored/mimalloc/src/page.c
+896fc1016a26e10c6cf96259d0524c57e123ac9dcef1ba04f23d0add445286dd vendored/mimalloc/src/prim/emscripten/prim.c
+502efbc8616f18be83713759438a0e5b8017ed215947ddfd48e74b85fd899114 vendored/mimalloc/src/prim/osx/alloc-override-zone.c
247a9952465eb105be03a9962e922b085ce5d52034775551a61cd8594275be73 vendored/mimalloc/src/prim/osx/prim.c
-cc771788e5ba591efb681829a8724f8662a1f50f3db39cf5ebe7738b8ed57e44 vendored/mimalloc/src/prim/prim.c
-792531421c247fca0766ff7630057ec491173578ca4d1919ae34f05eb289ccf4 vendored/mimalloc/src/prim/readme.md
-20594d38ef4fc0c641f20a8664d17ee03ce195ca21b44e036f43cdb7bcd0bd0d vendored/mimalloc/src/prim/unix/prim.c
-bee9c655a17e1e93bf4908e0a902fd3b566628447b1446fcbfa92bca2e76ca15 vendored/mimalloc/src/prim/wasi/prim.c
+884443f66808143d645dc6254c38e27f91b540a667ea23fe0ba1ef1caaf2b114 vendored/mimalloc/src/prim/prim-tls.c
+44419a450e742812d8f3c881d957bdd6cf18f40b90dfe1153084d2b64666f12d vendored/mimalloc/src/prim/prim.c
+28510fce6710d7d08224135e2e986e85995d7907232d3d6e278a82213ca78935 vendored/mimalloc/src/prim/readme.md
+a5ce291e50e965fbeedc632f54bc9f10b78c0548cf7739cd477e21c6e2751b3a vendored/mimalloc/src/prim/unix/prim.c
+2bc862cddb522314d58b3c6ef041e101c25cf04dfdda35cf78f91dbb6140cf46 vendored/mimalloc/src/prim/wasi/prim.c
f70e61a1f2fe63da2f51c581895cf4a0918a9bffb0cf8600171f297141446962 vendored/mimalloc/src/prim/windows/etw-mimalloc.wprp
4f3110ef2054c95cb275be96cb279224d3d46a728c5117bd1435425f382de778 vendored/mimalloc/src/prim/windows/etw.h
445e6a70f9728fcc76a5ba52079336064d5e2257c2969023f1c64bf943776f54 vendored/mimalloc/src/prim/windows/etw.man
-5cde60fcef94cd79ab816030be00f8c1bdf18fab343563508c681ec6e762d1f1 vendored/mimalloc/src/prim/windows/prim.c
+41d96c081cbda6bcad0e8270eab7c3d69f94dfe3d3dcd8a9567569c595984cb0 vendored/mimalloc/src/prim/windows/prim.c
1bcb43eef7fcddf6d8f18c6d5ee9bd350090ed183a1009d4f1c9b4d81a205bb9 vendored/mimalloc/src/prim/windows/readme.md
-a96a1acc75fd6595811b28420cf0495b87fac98c4dbc73aa0d742b6824f607c8 vendored/mimalloc/src/random.c
-4059be2a6676693be3433f5aedc711f56301772e65f3573654c236d6d1cfd074 vendored/mimalloc/src/static.c
-16825bbc252de422982b5728df7eda181d952824f5e3ce958c7fc85feae64b8e vendored/mimalloc/src/stats.c
-7f76bcace4b0578f2063477752138adabd99f7b10a276998e296a312c47a7cfe vendored/mimalloc/src/theap.c
-49be69a4b674e09690a63870830f3ffb05698ed4210949ef5add11957a08325b vendored/mimalloc/src/threadlocal.c
+2bd12844f9de0392fdb00ea70f5ad15649ffd8e49f9a4d2388e15d6469a7b703 vendored/mimalloc/src/random.c
+4edc7700c2dd911cd03919d1098e973d887900ab08a5f41fd9648a79517fca57 vendored/mimalloc/src/sample-guarded.c
+3df74fee931244a302293f7d215dc87e3b5c120ab7df207edd92d0ff19eb3729 vendored/mimalloc/src/sample-profile.c
+fadfb75216098793f9f81e57d6ff3b63914084976526ed7867c9ca212825d784 vendored/mimalloc/src/static.c
+d78e0467008946b40d5356315a9ba76e04354ead65b1aafabbb7bd091c3826a6 vendored/mimalloc/src/stats.c
+41c147e8eb6e960687b79a201dd4498fb80f58923a0a4d73e04e80ef315b8eb6 vendored/mimalloc/src/subproc.c
+91271bc76502c743d109f32201c02271f63a9f6ec51ef207f8b533f6de39df27 vendored/mimalloc/src/theap.c
+01dbaacd5ac6685dc59c40d82a3f8932e5290d4d05f9a441424e4ea53e1ba973 vendored/mimalloc/src/threadlocal.c
cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30 vendored/nomic/LICENSE
893a74d9e3f0a74358344edc999e165739df343b36297967f3fd9cd462022f39 vendored/nomic/NOTICE
dab3009c0d76b0c5e05ad8abc7c9f8f6effca547fea3c0394be96418a85c1081 vendored/nomic/code_tokens.h
diff --git a/vendored/mimalloc/PATCHES.md b/vendored/mimalloc/PATCHES.md
new file mode 100644
index 000000000..fb97f9ace
--- /dev/null
+++ b/vendored/mimalloc/PATCHES.md
@@ -0,0 +1,95 @@
+# Vendored mimalloc: source pin and local patches
+
+## Source pin
+
+- Upstream: <https://github.com/microsoft/mimalloc>
+- Tag: `v3.5.3` (annotated tag object `deac3dcf5f0614884013d411a5c07e16bec96c9f`)
+- Commit: `d4881d338125e1cb7c47ba4cfb398d6f7c0c8d45`
+ (tree `fbc4e8f72287d127c079ada5ede870b7d63e55f5`)
+- Vendored: `LICENSE`, every file under `include/` and every file under `src/`.
+ Nothing else (no build files, tests or docs).
+
+v3.5.3 is upstream's latest published release at the time of the pin
+(GitHub release of 2026-09-17, on the `main3` branch).
+
+v3.5.3 contains upstream commit `b2226c195fe884223f6d6f68ea52df476e57e5c2`
+(mimalloc #1341, first released in v3.4.4): `free(NULL)` must work before the
+allocator is initialized. With 3.3.2's release build, glibc 2.44's
+`newlocale()` freed NULL during libstdc++ start-up, before mimalloc's
+constructor, and every Linux binary died with SIGSEGV (reported with a
+reproduction by Felix Rath in #2562). The preinit fixture in
+`tests/test_prod_allocator_contract.sh` pins it.
+
+## Verifying the vendored bytes
+
+Every vendored file is byte-identical to the tag except the two hunks below.
+Re-check from a fresh bare clone (nothing in it is built or run):
+
+```sh
+git clone --bare https://github.com/microsoft/mimalloc.git /tmp/mimalloc.git
+git --git-dir=/tmp/mimalloc.git rev-parse 'v3.5.3^{commit}' # d4881d33...
+for f in $(git --git-dir=/tmp/mimalloc.git ls-tree -r --name-only v3.5.3 -- LICENSE include src); do
+ [ "$(git --git-dir=/tmp/mimalloc.git rev-parse "v3.5.3:$f")" = \
+ "$(git hash-object "vendored/mimalloc/$f")" ] || echo "differs: $f"
+done
+```
+
+The expected output is exactly the two files named below, and
+`vendored/mimalloc/include` plus `vendored/mimalloc/src` must hold no file the
+tag lacks.
+
+## Local patches (reapply on every refresh)
+
+Each patch is pinned by a check that fails when the patch is dropped.
+
+### 1. macOS decommit always recommits: `src/prim/unix/prim.c`
+
+In `_mi_prim_decommit`, macOS takes the recommit path that upstream reserves for
+`MI_DEBUG` / `MI_SECURE>2` builds, so a purged range is not re-accounted as a
+whole by `MADV_FREE_REUSE` when the arena reuses it. This cut the indexer's peak
+`phys_footprint` substantially (measurements in the inline comment and in commit
+63fb7b3d; re-measured on v3.5.3 when this pin was taken). Linux and Windows keep
+upstream's behaviour.
+
+Pinned by: the decommit fixture in `tests/test_prod_allocator_contract.sh`
+(`needs_recommit` must be true on macOS and upstream's false elsewhere).
+
+### 2. Comment wording: `src/prim/osx/alloc-override-zone.c`
+
+Comment only, no code change: "crash in fork()." reads "crash while forking.".
+`scripts/security-vendored.sh` rejects call spellings such as `fork(` anywhere
+in vendored C, comments included, so the upstream sentence fails that gate.
+(Wording first proposed in #2562.)
+
+Pinned by: `scripts/security-vendored.sh` itself, which reports the pristine
+upstream line as a blocked subprocess call.
+
+## Build configuration
+
+`Makefile.cbm` compiles the production object with `-DMI_PROFILE=0`, which
+leaves out mimalloc's allocation-sampling profiler (on by default since 3.5.2)
+and its sample countdown on the allocation path. Pinned by: the banner fixture
+in `tests/test_prod_allocator_contract.sh` (a profiling build prints a
+`profiling` line). The test build keeps upstream's default.
+
+## Dropped patches
+
+- Reproducible version banner (`src/options.c`, carried from 1f674c80 up to
+ 3.4.4): upstream removed `__DATE__`/`__TIME__` from the banner in v3.4.5
+ (mimalloc #1355). The banner fixture in
+ `tests/test_prod_allocator_contract.sh` still asserts that the first line is
+ exactly `vX.Y.Z` with no build clock, so a future refresh cannot bring it
+ back unnoticed.
+
+## Refresh procedure
+
+1. Replace `LICENSE`, `include/` and `src/` with the new tag's files, deleting
+ files the tag no longer has, and verify the staged tree against the tag
+ before patching.
+2. Reapply the patches above, or drop the ones upstream has absorbed (and move
+ them to "Dropped patches" with the upstream reference).
+3. Update the pin in this file and `versionInfo` in
+ `scripts/ci/generate-sbom.py`, then run `scripts/security-vendored.sh --update`.
+4. After `make -f Makefile.cbm cbm`, run
+ `bash tests/test_prod_allocator_contract.sh build/c` on Linux and macOS
+ (`scripts/test.sh` runs it on every CI leg), and `scripts/security-vendored.sh`.
diff --git a/vendored/mimalloc/include/mimalloc-new-delete.h b/vendored/mimalloc/include/mimalloc-new-delete.h
index c16f4a665..bb1bc5075 100644
--- a/vendored/mimalloc/include/mimalloc-new-delete.h
+++ b/vendored/mimalloc/include/mimalloc-new-delete.h
@@ -37,7 +37,7 @@ terms of the MIT license. A copy of the license can be found in the file
void operator delete (void* p, const std::nothrow_t&) noexcept { mi_free(p); }
void operator delete[](void* p, const std::nothrow_t&) noexcept { mi_free(p); }
- mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); }
+ mi_decl_new(n) void* operator new(std::size_t n) noexcept(false) { return mi_new(n); }
mi_decl_new(n) void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); }
mi_decl_new_nothrow(n) void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); }
@@ -48,7 +48,7 @@ terms of the MIT license. A copy of the license can be found in the file
void operator delete[](void* p, std::size_t n) noexcept { mi_free_size(p,n); };
#endif
- #if (__cplusplus > 201402L || defined(__cpp_aligned_new))
+ #if (__cplusplus > 201402L && defined(__cpp_aligned_new))
void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
diff --git a/vendored/mimalloc/include/mimalloc-profile.h b/vendored/mimalloc/include/mimalloc-profile.h
new file mode 100644
index 000000000..5a1327ff7
--- /dev/null
+++ b/vendored/mimalloc/include/mimalloc-profile.h
@@ -0,0 +1,79 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2024-2026, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#pragma once
+#ifndef MIMALLOC_PROFILE_H
+#define MIMALLOC_PROFILE_H
+
+#include <mimalloc.h>
+#include <stdbool.h> // bool
+#include <stdint.h> // uint64_t
+
+//---------------------------------------------------------------------------
+// Initial support for profiling hooks; still experimental
+//---------------------------------------------------------------------------
+
+#define MI_PROFILE_SAMPLE_DATA_MAX_SIZE (1024)
+
+// User defined profiler; see `test_profile.c` for an example.
+typedef struct mi_profiler_s mi_profiler_t;
+
+// Profiler data is stored together with each sampled allocation (unless the `on_free` field in the profiler is NULL.)
+typedef struct mi_profiler_sample_data_s {
+ size_t user_data_size; // size of the user_data (should be the `mi_profiler_t.sample_data_size`)
+ void* user_data[1]; // default, but can be less or more (up to 1KiB), depending on `sample_data_size`
+} mi_profiler_sample_data_t;
+
+// Profiling callback invoked on each sampled allocation.
+// If `profiler_data!=NULL` (i.e. when `on_free` is not NULL), then `profiler_data->requested_size == requested_size`.
+typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t requested_size, size_t bytes_sample_rate, uint64_t bytes_since_last_sample, const mi_heap_t* heap);
+
+// Profiling callback invoked on each sampled in-place re-allocation.
+typedef size_t (mi_cdecl mi_profiler_on_realloc_inplace_fun)(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t old_size, const mi_heap_t* heap);
+
+// Profiling callback invoked on a previously sampled allocation.
+typedef void (mi_cdecl mi_profiler_on_free_fun )(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, const mi_heap_t* heap);
+
+// A profiler
+// All fields are considered immutable such that they can be copied and accessed concurrently. All fields can be NULL/0.
+struct mi_profiler_s {
+ void* reserved; // opaque; reserved by mimalloc
+ size_t sample_data_size; // size of required profiler data for each sampled allocation (or zero for no data)
+ size_t initial_sample_rate;// initial sample rate in bytes (set to at least 1 or higher) (can be adjusted by `on_alloc`)
+ mi_profiler_on_alloc_fun* on_alloc; // called on a sampled allocation (may be called concurrently)
+ mi_profiler_on_free_fun* on_free; // called on when previous sampled allocation is freed (may be called concurrently)
+ mi_profiler_on_realloc_inplace_fun* on_realloc_inplace; // (as yet unused) called on in-place reallocation of a previous sampled allocation (may be called concurrently)
+ // ... more user fields allowed
+};
+
+// Exported definitions
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// attach a profiler to a particular heap only.
+mi_decl_export bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler);
+
+// disable profiling for a particular heap; useful for a heap that the profiler uses itself for metadata.
+mi_decl_export void mi_heap_profile_disable(mi_heap_t* heap);
+
+// attach a profiler to any (current and future) heaps in a sub-process (unless those heaps disabled profiling)
+mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler);
+
+// attach a profiler to any heaps in the main sub-process
+mi_decl_export bool mi_profile(mi_profiler_t* profiler);
+
+// start sampling
+mi_decl_export bool mi_profiler_start(mi_profiler_t* profiler);
+
+// end sampling
+mi_decl_export bool mi_profiler_stop(mi_profiler_t* profiler);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // MIMALLOC_PROFILE_H
diff --git a/vendored/mimalloc/include/mimalloc-stats.h b/vendored/mimalloc/include/mimalloc-stats.h
index 1c620fd1c..ab45a0715 100644
--- a/vendored/mimalloc/include/mimalloc-stats.h
+++ b/vendored/mimalloc/include/mimalloc-stats.h
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2024-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2024-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -22,7 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file
#elif __cplusplus >= 201103L
#define mi_decl_align(a) alignas(a)
#else
-#define mi_decl_align(a)
+#define mi_decl_align(a) _Alignas(a)
#endif
@@ -49,7 +49,7 @@ typedef struct mi_stat_counter_s {
MI_STAT_COUNT(threads) /* number of threads */ \
MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \
MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \
- MI_STAT_COUNT(malloc_requested) /* malloc requested bytes */ \
+ MI_STAT_COUNTER(malloc_requested) /* malloc requested bytes */ \
\
MI_STAT_COUNTER(mmap_calls) \
MI_STAT_COUNTER(commit_calls) \
@@ -75,11 +75,16 @@ typedef struct mi_stat_counter_s {
/* only on v3 */ \
MI_STAT_COUNT(heaps) \
MI_STAT_COUNT(theaps) \
+ MI_STAT_COUNT(pages_os_abandoned) /* pages in the os abandoned list (bad) */ \
+ MI_STAT_COUNT(pages_os_allocated) /* pages allocated outside arenas (bad) */ \
MI_STAT_COUNTER(pages_reclaim_on_alloc) \
MI_STAT_COUNTER(pages_reclaim_on_free) \
MI_STAT_COUNTER(pages_reabandon_full) \
MI_STAT_COUNTER(pages_unabandon_busy_wait) \
- MI_STAT_COUNTER(heaps_delete_wait)
+ MI_STAT_COUNTER(heaps_delete_wait) \
+ MI_STAT_COUNTER(pages_stat_updates) /* calls to successful page_stat_update */ \
+ MI_STAT_COUNTER(pages_stat_update_count) /* total free/allocs */ \
+ MI_STAT_COUNTER(profile_samples) /* total sampled profiled allocations */ \
// Size bins for chunks
typedef enum mi_chunkbin_e {
@@ -161,6 +166,12 @@ mi_decl_export bool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id
mi_decl_export char* mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL
mi_decl_export size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept;
+// get theap stats (only thread safe on theaps belonging to the calling thread)
+mi_decl_export bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept;
+
+// add the theap stats to the stats of the parent heap and clear the theap stats
+mi_decl_export void mi_theap_stats_merge_to_heap(mi_theap_t* theap) mi_attr_noexcept;
+
#ifdef __cplusplus
}
#endif
diff --git a/vendored/mimalloc/include/mimalloc.h b/vendored/mimalloc/include/mimalloc.h
index f373f5dc8..2985edd94 100644
--- a/vendored/mimalloc/include/mimalloc.h
+++ b/vendored/mimalloc/include/mimalloc.h
@@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
#ifndef MIMALLOC_H
#define MIMALLOC_H
-#define MI_MALLOC_VERSION 30302 // major + 2 digits minor + 2 digits patch
+#define MI_MALLOC_VERSION 30503 // major + 2 digits minor + 2 digits patch
// ------------------------------------------------------
// Compiler specific attributes
@@ -95,8 +95,9 @@ terms of the MIT license. A copy of the license can be found in the file
// Includes
// ------------------------------------------------------
-#include <stddef.h> // size_t
+#include <stddef.h> // size_t, wchar_t
#include <stdbool.h> // bool
+#include <assert.h> // inline assertions
#ifdef __cplusplus
extern "C" {
@@ -119,8 +120,8 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char*
// ------------------------------------------------------
// Extended allocation functions
// ------------------------------------------------------
-#define MI_SMALL_WSIZE_MAX (128)
-#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*))
+#define MI_SMALL_WSIZE_MAX 128
+#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(size_t)) // mimalloc considers `sizeof(size_t)` as the machine word size
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
@@ -133,10 +134,17 @@ mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize)
mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept;
+// `mi_free_size` can be more efficient (as it calls mi_free_small internally).
+mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept;
+
// `mi_free_small` is for special applications like language runtimes.
-// it should only be used to free objects from `mi_(heap_)(m|z)alloc_small` and is potentially a tiny bit faster than `mi_free`
+// it should only be used to free objects from `mi_*alloc_small` and is potentially a tiny bit faster than `mi_free`
mi_decl_export void mi_free_small(void* p) mi_attr_noexcept;
+// As `mi_free_small` but `p` should not be a NULL pointer.
+mi_decl_export void mi_free_small_nonnull(void* p) mi_attr_noexcept;
+
+
// -------------------------------------------------------------------------------------
// Aligned allocation
// Note that `alignment` always follows `size` for consistency with unaligned
@@ -157,12 +165,13 @@ mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t new
// Typed allocation, the type is always the first parameter
// ------------------------------------------------------
-#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp)))
-#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp)))
+#define mi_malloc_tp(tp) ((tp*)mi_malloc_csize(sizeof(tp)))
+#define mi_zalloc_tp(tp) ((tp*)mi_zalloc_csize(sizeof(tp)))
#define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp)))
#define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp)))
#define mi_reallocn_tp(tp,p,n) ((tp*)mi_reallocn(p,n,sizeof(tp)))
#define mi_recalloc_tp(tp,p,n) ((tp*)mi_recalloc(p,n,sizeof(tp)))
+#define mi_free_tp(tp,p) (mi_free_csize(p,sizeof(tp)))
#define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))
#define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
@@ -187,7 +196,7 @@ typedef void (mi_cdecl mi_error_fun)(int err, void* arg);
mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg);
mi_decl_export void mi_collect(bool force) mi_attr_noexcept;
-mi_decl_export int mi_version(void) mi_attr_noexcept;
+mi_decl_export int mi_version(void);
mi_decl_export void mi_options_print(void) mi_attr_noexcept;
mi_decl_export void mi_process_info_print(void) mi_attr_noexcept;
mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
@@ -329,6 +338,7 @@ mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept;
mi_decl_export void mi_arenas_print(void) mi_attr_noexcept;
mi_decl_export size_t mi_arena_min_alignment(void);
mi_decl_export size_t mi_arena_min_size(void);
+mi_decl_export size_t mi_arena_max_object_size(void);
typedef void* mi_arena_id_t;
mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);
@@ -382,8 +392,9 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_calloc(mi_theap
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
mi_decl_nodiscard mi_decl_export void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
-
+mi_decl_nodiscard mi_decl_export void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
// ------------------------------------------------------
// Experimental
@@ -465,25 +476,26 @@ typedef enum mi_option_e {
mi_option_deprecated_purge_extend_delay,
mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's)
mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
- mi_option_visit_abandoned, // allow visiting theap blocks from abandoned threads (=0)
+ mi_option_deprecated_visit_abandoned, // allow visiting theap blocks from abandoned threads (=0)
mi_option_guarded_min, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)
mi_option_guarded_max, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)
mi_option_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)
mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000)
mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)
mi_option_generic_collect, // collect theaps every N (=10000) generic allocation calls
- mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from the current theap, 1 allow always
+ mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallow always, 0 allows if the page originated from the current theap, 1 allow always
mi_option_page_full_retain, // retain N full (small) pages per size class (=2)
mi_option_page_max_candidates, // max candidate pages to consider for allocation (=4)
mi_option_max_vabits, // max user space virtual address bits to consider (=48)
mi_option_pagemap_commit, // commit the full pagemap (to always catch invalid pointer uses) (=0)
mi_option_page_commit_on_demand, // commit page memory on-demand
mi_option_page_max_reclaim, // don't reclaim pages of the same originating theap if we already own N pages (in that size class) (=-1 (unlimited))
- mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=16)
+ mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=32)
mi_option_allow_thp, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.
mi_option_minimal_purge_size, // set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled.
mi_option_arena_max_object_size, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).
- mi_option_arena_is_numa_local, // experimental
+ mi_option_arena_is_numa_local, // experimental: associate local numa node with an initial arena allocation
+ mi_option_collect_merges_stats, // on each theap collection, the theap stats are merged automatically with the parent heap
_mi_option_last,
// legacy option names
mi_option_large_os_pages = mi_option_allow_large_os_pages,
@@ -513,13 +525,13 @@ mi_decl_export void mi_option_set_default(mi_option_t option, long value);
// note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our theap before free-ing.
// -------------------------------------------------------------------------------------------------------
-mi_decl_export void mi_cfree(void* p) mi_attr_noexcept;
+mi_decl_export bool mi_cfree(void* p) mi_attr_noexcept;
mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept;
-mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept;
+mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size); // mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
@@ -530,19 +542,22 @@ mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* ptrp, size_t count, s
mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept;
-mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept;
mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept;
mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept;
mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept;
// wide characters
-#include <wchar.h> // wchar_t
mi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc;
-// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.
-// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).
+// --------------------------------------------------------
+// C++ wrappers
+// The `mi_new` wrappers implement C++ semantics on out-of-memory
+// instead of directly returning `NULL`. (and call `std::get_new_handler`
+// and potentially raise a `std::bad_alloc` exception).
+// --------------------------------------------------------
+
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
@@ -554,6 +569,63 @@ mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount,
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) mi_attr_malloc mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3);
+mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_alloc_new_nothrow(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
+
+
+// --------------------------------------------------------------------------
+// Inlined constant size allocations.
+// These are meant for runtime systems, or overrides where we need the best
+// performance for small, constant-size allocations.
+// These are only better than the regular functions if the size or alignment
+// are indeed constant at the call site.
+// --------------------------------------------------------------------------
+
+// Internal machine word size allocation. `wsize` is the allocation size in machine words (`sizeof(size_t)`)
+mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept mi_attr_malloc;
+mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept mi_attr_malloc;
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept mi_attr_malloc;
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept mi_attr_malloc;
+
+static inline size_t mi_wsize_from_size(size_t size) mi_attr_noexcept {
+ return (size + sizeof(size_t) - 1) / sizeof(size_t);
+}
+
+static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexcept {
+ if (size <= MI_SMALL_SIZE_MAX) { return mi_wmalloc_small(mi_wsize_from_size(size)); } else { return mi_malloc(size); }
+}
+static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept {
+ if (size <= MI_SMALL_SIZE_MAX) { return mi_wzalloc_small(mi_wsize_from_size(size)); } else { return mi_zalloc(size); }
+}
+static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept {
+ assert(theap!=NULL);
+ if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wmalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); }
+}
+static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept {
+ assert(theap!=NULL);
+ if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wzalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); }
+}
+
+static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept {
+ if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); }
+}
+static inline void mi_free_csize_nonnull(void* p, size_t size) mi_attr_noexcept {
+ assert(p!=NULL);
+ if (size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); }
+}
+static inline void mi_free_csize_aligned(void* p, size_t size, size_t aligned) mi_attr_noexcept {
+ if (aligned <= size && size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); }
+}
+static inline void mi_free_csize_aligned_nonnull(void* p, size_t size, size_t aligned) mi_attr_noexcept {
+ assert(p!=NULL);
+ if (aligned <= size && size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); }
+}
+
+// ------------------------------------------------------
+// C++ standard library allocator interface.
+// ------------------------------------------------------
+
#ifdef __cplusplus
}
#endif
diff --git a/vendored/mimalloc/include/mimalloc/atomic.h b/vendored/mimalloc/include/mimalloc/atomic.h
index e4abde6b0..9c2339802 100644
--- a/vendored/mimalloc/include/mimalloc/atomic.h
+++ b/vendored/mimalloc/include/mimalloc/atomic.h
@@ -9,13 +9,14 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_ATOMIC_H
// include windows.h or pthreads.h
-#if defined(_WIN32)
+#if defined(_WIN32) || defined(__CYGWIN__) // we use windows locks on cygwin, but otherwise treat it at unix
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h>
-#elif !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))
-#define MI_USE_PTHREADS
+#endif
+#if MI_TLS_MODEL_PTHREADS || (!defined(_WIN32) && !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)))
+#define MI_USE_PTHREADS 1
#include <pthread.h>
#endif
@@ -215,24 +216,22 @@ static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintpt
(void)(mo);
return (uintptr_t)MI_MSC_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);
}
-static inline void mi_atomic_thread_fence(mi_memory_order mo) {
- (void)(mo);
- _Atomic(uintptr_t) x = 0;
- mi_atomic_exchange_explicit(&x, 1, mo);
-}
static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) {
(void)(mo);
// assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc
#if defined(_M_IX86) || defined(_M_X64)
+ // on x86/x64 we have a strong memory model so any load is acquire
return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
#elif defined(_M_ARM) || defined(_M_ARM64)
if (mo == mi_memory_order_relaxed) {
return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
}
+ #if !defined(__clang__) // work around __ldar missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689
else if (mo <= mi_memory_order_acquire) {
return MI_MSC_XX(__ldar)((volatile const uintptr_t*)p);
}
+ #endif
else {
const uintptr_t u = (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
__dmb(15); // _ARM(64)_BARRIER_SY
@@ -252,9 +251,11 @@ static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, m
if (mo == mi_memory_order_relaxed) {
MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);
}
+ #if !defined(__clang__) // work around __stlr missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689
else if (mo <= mi_memory_order_release) {
MI_MSC_XX(__stlr)((volatile uintptr_t*)p,x);
}
+ #endif
else {
mi_atomic_exchange_explicit(p, x, mo);
}
@@ -272,7 +273,7 @@ static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_o
if (mo == mi_memory_order_relaxed) {
return __iso_volatile_load64((volatile const int64_t*)p);
}
- #if defined(_M_ARM64)
+ #if defined(_M_ARM64) && !defined(__clang__) // work around __ldar64 missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689
else if (mo <= mi_memory_order_acquire) {
return __ldar64((volatile const uintptr_t*)p);
}
@@ -297,7 +298,7 @@ static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi
if (mo == mi_memory_order_relaxed) {
__iso_volatile_store64((volatile int64_t*)p,x);
}
- #if defined(_M_ARM64)
+ #if defined(_M_ARM64) && !defined(__clang__) // work around __stlr64 missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689
else if (mo == mi_memory_order_release) {
__stlr64((volatile uint64_t*)p, (uint64_t)x);
}
@@ -335,18 +336,18 @@ static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const vola
}
}
-static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) {
+static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)* p, int64_t x) {
int64_t current;
do {
current = *p;
} while (current < x && _InterlockedCompareExchange64(p, x, current) != current);
}
-static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) {
+static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t)* p, int64_t i) {
mi_atomic_addi64_relaxed(p, i);
}
-static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) {
+static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t)* p, int64_t* exp, int64_t des) {
const int64_t read = _InterlockedCompareExchange64(p, des, *exp);
if (read == *exp) {
return true;
@@ -358,17 +359,17 @@ static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p,
}
// The pointer macros cast to `uintptr_t`.
-#define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p))
-#define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p))
+#define mi_atomic_load_ptr_acquire(tp,p) ((tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p)))
+#define mi_atomic_load_ptr_relaxed(tp,p) ((tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p)))
#define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))
#define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x))
#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
#define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des)
-#define mi_atomic_exchange_ptr_relaxed(tp,p,x) (tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
-#define mi_atomic_exchange_ptr_release(tp,p,x) (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
-#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x)
+#define mi_atomic_exchange_ptr_relaxed(tp,p,x) ((tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x))
+#define mi_atomic_exchange_ptr_release(tp,p,x) ((tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x))
+#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) ((tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x))
#define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire))
#define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed))
@@ -417,7 +418,7 @@ typedef _Atomic(uintptr_t) mi_atomic_guard_t;
#define mi_lock_maybe(lock,acquire) for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) )
-#if defined(_WIN32)
+#if defined(_WIN32) || defined(__CYGWIN__)
typedef struct mi_lock_s {
SRWLOCK mutex; // slim reader-writer lock
@@ -496,10 +497,10 @@ static inline void mi_lock_release(mi_lock_t* lock) {
lock->mutex.unlock();
}
static inline void mi_lock_init(mi_lock_t* lock) {
- new(&lock->mutex) std::mutex();
+ new(&lock->mutex) std::mutex(); // in-place constructor
}
static inline void mi_lock_done(mi_lock_t* lock) {
- (void)(lock);
+ lock->mutex.~mutex(); // in-place destructor
}
#else
@@ -524,6 +525,7 @@ static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (uintptr_t)1);
}
static inline void mi_lock_acquire(mi_lock_t* lock) {
+ size_t ticks = 0;
for (int i = 0; i < 10000; i++) { // for at most 10000 tries?
if (mi_lock_try_acquire(lock)) return;
_mi_prim_thread_yield();
diff --git a/vendored/mimalloc/include/mimalloc/bits.h b/vendored/mimalloc/include/mimalloc/bits.h
index 617877283..5aa0d8375 100644
--- a/vendored/mimalloc/include/mimalloc/bits.h
+++ b/vendored/mimalloc/include/mimalloc/bits.h
@@ -16,6 +16,15 @@ terms of the MIT license. A copy of the license can be found in the file
#include <stddef.h> // size_t
#include <stdint.h> // int64_t etc
#include <stdbool.h> // bool
+#include <limits.h> // LONG_MAX
+
+#if defined(__cplusplus)
+#define mi_decl_externc extern "C"
+#define mi_init_struct_zero { }
+#else
+#define mi_decl_externc
+#define mi_init_struct_zero { 0 }
+#endif
// ------------------------------------------------------
// Size of a pointer.
@@ -102,13 +111,13 @@ typedef int32_t mi_ssize_t;
#include <intrin.h>
#endif
-#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // msvc
+#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // avx2 implies bmi2
#define __BMI2__ 1
#endif
-#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__)) && !defined(__BMI1__) // msvc
+#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__) || defined(__BMI__)) && !defined(__BMI1__) // bmi2 implies bmi1
#define __BMI1__ 1
#endif
-#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // msvc
+#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // avx2 implies lzcnt
#define __LZCNT__ 1
#endif
@@ -126,9 +135,18 @@ typedef int32_t mi_ssize_t;
#define MI_MAX_VABITS (32)
#endif
+// the MI_MIN_VABITS determine how many bits of the address space are always committed in the page_map
+#if MI_MAX_VABITS <= 32
+#define MI_MIN_VABITS (32)
+#elif MI_MAX_VABITS <= 42
+#define MI_MIN_VABITS MI_MAX_VABITS
+#else
+#define MI_MIN_VABITS (43) /* 8 TiB */
+#endif
+
// use a flat page-map or a 2-level one
#ifndef MI_PAGE_MAP_FLAT
-#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && MI_SECURE==0 && !MI_PAGE_META_IS_SEPARATED
+#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && !MI_SECURE && !MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP
#define MI_PAGE_MAP_FLAT 1
#else
#define MI_PAGE_MAP_FLAT 0
@@ -177,12 +195,16 @@ typedef int32_t mi_ssize_t;
-------------------------------------------------------------------------------- */
size_t _mi_popcount_generic(size_t x);
+extern bool _mi_cpu_has_popcnt;
static inline size_t mi_popcount(size_t x) {
#if mi_has_builtinz(popcount)
return mi_builtinz(popcount)(x);
- #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
- return mi_msc_builtinz(__popcnt)(x);
+ #elif defined(_MSC_VER) && (MI_ARCH_ARM64 || MI_ARCH_ARM32)
+ return mi_msc_builtinz(_CountOneBits)(x);
+ #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86)
+ if (_mi_cpu_has_popcnt) { return mi_msc_builtinz(__popcnt)(x); }
+ else { return _mi_popcount_generic(x); } // see issue #1291
#elif MI_ARCH_X64 && defined(__BMI1__)
return (size_t)_mm_popcnt_u64(x);
#else
diff --git a/vendored/mimalloc/include/mimalloc/internal.h b/vendored/mimalloc/include/mimalloc/internal.h
index febf2559e..b79649c72 100644
--- a/vendored/mimalloc/include/mimalloc/internal.h
+++ b/vendored/mimalloc/include/mimalloc/internal.h
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -40,6 +40,7 @@ terms of the MIT license. A copy of the license can be found in the file
#define mi_decl_weak
#define mi_decl_hidden
#define mi_decl_cold
+#define mi_assume_aligned(p,sz) (p)
#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc
#if !MI_TRACK_ASAN
#define mi_decl_forceinline __attribute__((always_inline)) inline
@@ -50,11 +51,17 @@ terms of the MIT license. A copy of the license can be found in the file
#define mi_decl_thread __thread
#define mi_decl_noreturn __attribute__((noreturn))
#define mi_decl_weak __attribute__((weak))
+#if defined(__MINGW32__) || defined(__CYGWIN__)
+#define mi_decl_hidden
+#else
#define mi_decl_hidden __attribute__((visibility("hidden")))
-#if (__GNUC__ >= 4) || defined(__clang__)
+#endif
+#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
#define mi_decl_cold __attribute__((cold))
+#define mi_assume_aligned(p,sz) __builtin_assume_aligned(p,sz)
#else
#define mi_decl_cold
+#define mi_assume_aligned(p,sz) (p)
#endif
#elif __cplusplus >= 201103L // c++11
#define mi_decl_forceinline inline
@@ -64,6 +71,7 @@ terms of the MIT license. A copy of the license can be found in the file
#define mi_decl_weak
#define mi_decl_hidden
#define mi_decl_cold
+#define mi_assume_aligned(p,sz) (p)
#else
#define mi_decl_forceinline inline
#define mi_decl_noinline
@@ -72,6 +80,7 @@ terms of the MIT license. A copy of the license can be found in the file
#define mi_decl_weak
#define mi_decl_hidden
#define mi_decl_cold
+#define mi_assume_aligned(p,sz) (p)
#endif
#if defined(__GNUC__) || defined(__clang__)
@@ -85,17 +94,7 @@ terms of the MIT license. A copy of the license can be found in the file
#define mi_likely(x) (x)
#endif
-#ifndef __has_builtin
-#define __has_builtin(x) 0
-#endif
-
-#if defined(__cplusplus)
-#define mi_decl_externc extern "C"
-#else
-#define mi_decl_externc
-#endif
-
-#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) // includes clang and icc
+#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__)
#define mi_decl_maybe_unused __attribute__((unused))
#elif __cplusplus >= 201703L // c++17
#define mi_decl_maybe_unused [[maybe_unused]]
@@ -103,13 +102,10 @@ terms of the MIT license. A copy of the license can be found in the file
#define mi_decl_maybe_unused
#endif
-#if defined(__cplusplus)
-#define mi_decl_externc extern "C"
-#else
-#define mi_decl_externc
+#ifndef __has_builtin
+#define __has_builtin(x) 0
#endif
-
#if defined(__EMSCRIPTEN__) && !defined(__wasi__)
#define __wasi__
#endif
@@ -126,19 +122,19 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list
int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...);
char _mi_toupper(char c);
int _mi_strnicmp(const char* s, const char* t, size_t n);
-void _mi_strlcpy(char* dest, const char* src, size_t dest_size);
-void _mi_strlcat(char* dest, const char* src, size_t dest_size);
+bool _mi_strlcpy(char* dest, const char* src, size_t dest_size); // returns true if the entire src was copied
+bool _mi_strlcat(char* dest, const char* src, size_t dest_size); // returns true if the entire src was appended
size_t _mi_strlen(const char* s);
size_t _mi_strnlen(const char* s, size_t max_len);
char* _mi_strnstr(char* s, size_t max_len, const char* pat);
bool _mi_streq(const char* s, const char* t);
-bool _mi_getenv(const char* name, char* result, size_t result_size);
+int _mi_getenv(const char* name, char* result, size_t result_size);
+void _mi_detect_cpu_features(void);
// "options.c"
void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);
void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);
void _mi_raw_message(const char* fmt, ...);
-void _mi_message(const char* fmt, ...);
void _mi_warning_message(const char* fmt, ...);
void _mi_verbose_message(const char* fmt, ...);
void _mi_trace_message(const char* fmt, ...);
@@ -152,79 +148,93 @@ void _mi_random_init(mi_random_ctx_t* ctx);
void _mi_random_init_weak(mi_random_ctx_t* ctx);
void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx);
void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx);
-uintptr_t _mi_random_next(mi_random_ctx_t* ctx);
-uintptr_t _mi_theap_random_next(mi_theap_t* theap);
-uintptr_t _mi_os_random_weak(uintptr_t extra_seed);
-static inline uintptr_t _mi_random_shuffle(uintptr_t x);
+size_t _mi_random_next(mi_random_ctx_t* ctx);
+size_t _mi_theap_random_next(mi_theap_t* theap);
+size_t _mi_os_random_weak(size_t extra_seed);
+static inline size_t _mi_random_shuffle(size_t x);
+
+// prim-tls.c
+void _mi_tls_slots_init(void);
+void _mi_tls_slots_done(void);
+mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;
+void _mi_theap_default_set(mi_theap_t* theap);
+void _mi_theap_cached_set(mi_theap_t* theap);
+
+// subproc.c
+mi_subproc_t* _mi_subproc_main_init(void);
+void _mi_subproc_main_done(void);
+mi_subproc_t* _mi_subproc_main(void);
+bool _mi_subproc_is_main(mi_subproc_t* subproc);
+mi_subproc_t* _mi_subproc(void); // current subproc of this thread
+mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc);
+mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id);
+void _mi_subprocs_unsafe_destroy_all(void);
+
+void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid );
+void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* p, size_t newsize, mi_memid_t* memid );
+void* _mi_meta_zalloc_aligned( mi_subproc_t* subproc, size_t size, size_t alignment, mi_memid_t* memid );
+void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid);
+bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* p);
+
// init.c
-extern mi_decl_hidden mi_decl_cache_align const mi_page_t _mi_page_empty;
+mi_page_t* _mi_page_empty_get(void);
void _mi_auto_process_init(void);
void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept;
bool _mi_is_redirected(void);
bool _mi_allocator_init(const char** message);
void _mi_allocator_done(void);
-bool _mi_is_main_thread(void);
bool _mi_preloading(void); // true while the C runtime is not initialized yet
void _mi_thread_done(mi_theap_t* theap);
+mi_theap_t* _mi_thread_init(void);
+mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap);
+bool _mi_is_empty_theap(const mi_theap_t* theap);
-mi_subproc_t* _mi_subproc(void);
-mi_subproc_t* _mi_subproc_main(void);
-mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc);
-mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id);
-
-mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;
-size_t _mi_thread_seq_id(void) mi_attr_noexcept;
-bool _mi_is_heap_main(const mi_heap_t* heap);
-bool _mi_is_theap_main(const mi_theap_t* theap);
-void _mi_theap_guarded_init(mi_theap_t* theap);
-void _mi_theap_options_init(mi_theap_t* theap);
-mi_theap_t* _mi_theap_default_safe(void); // ensure the returned theap is initialized
-mi_theap_t* _mi_theap_main_safe(void);
-
// os.c
void _mi_os_init(void); // called from process init
-void* _mi_os_alloc(size_t size, mi_memid_t* memid);
-void* _mi_os_zalloc(size_t size, mi_memid_t* memid);
-void _mi_os_free(void* p, size_t size, mi_memid_t memid);
-void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc );
+void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid);
+void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid);
+void _mi_os_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid);
+void _mi_os_free_ex(mi_subproc_t* subproc, void* p, size_t size, bool still_committed, mi_memid_t memid );
size_t _mi_os_page_size(void);
size_t _mi_os_guard_page_size(void);
size_t _mi_os_good_alloc_size(size_t size);
bool _mi_os_has_overcommit(void);
bool _mi_os_has_virtual_reserve(void);
+bool _mi_os_canuse_thp(void);
size_t _mi_os_virtual_address_bits(void);
size_t _mi_os_minimal_purge_size(void);
-bool _mi_os_reset(void* addr, size_t size);
-bool _mi_os_decommit(void* addr, size_t size);
-void _mi_os_reuse(void* p, size_t size);
-mi_decl_nodiscard bool _mi_os_commit(void* p, size_t size, bool* is_zero);
-mi_decl_nodiscard bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size);
+bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size);
+bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size);
+void _mi_os_reuse(mi_subproc_t* subproc, void* p, size_t size);
+mi_decl_nodiscard bool _mi_os_commit(mi_subproc_t* subproc, void* p, size_t size, bool* is_zero);
+mi_decl_nodiscard bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed);
mi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size);
bool _mi_os_unprotect(void* addr, size_t size);
-bool _mi_os_purge(void* p, size_t size);
-bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stats_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg);
+bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size);
+bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stats_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg);
size_t _mi_os_secure_guard_page_size(void);
-bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid);
-bool _mi_os_secure_guard_page_set_before(void* addr, mi_memid_t memid);
-bool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid);
-bool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid);
+bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid);
+bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid);
+bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid);
+bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid);
int _mi_os_numa_node(void);
int _mi_os_numa_node_count(void);
-void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid);
-void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid);
+void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid);
+void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid);
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size);
bool _mi_os_canuse_large_page(size_t size, size_t alignment);
size_t _mi_os_large_page_size(void);
-void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid);
+void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid);
// threadlocal.c
+#define mi_thread_local_key_fast ((mi_thread_local_t)1)
mi_thread_local_t _mi_thread_local_create(void);
void _mi_thread_local_free( mi_thread_local_t key );
@@ -241,8 +251,7 @@ bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_
void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid);
void* _mi_arenas_alloc_aligned(mi_heap_t* heap, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid);
-void _mi_arenas_free(void* p, size_t size, mi_memid_t memid);
-bool _mi_arenas_contain(const void* p);
+void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid);
void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld);
void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc);
@@ -252,59 +261,57 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap
void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */);
bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page);
-// arena-meta.c
-void* _mi_meta_zalloc( size_t size, mi_memid_t* memid );
-void _mi_meta_free(void* p, size_t size, mi_memid_t memid);
-bool _mi_meta_is_meta_page(void* p);
-
// "page-map.c"
bool _mi_page_map_init(void);
mi_decl_nodiscard bool _mi_page_map_register(mi_page_t* page);
void _mi_page_map_unregister(mi_page_t* page);
void _mi_page_map_unregister_range(void* start, size_t size);
mi_page_t* _mi_safe_ptr_page(const void* p);
-void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc);
+void _mi_page_map_unsafe_destroy(void);
// "page.c"
-void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable) mi_attr_noexcept mi_attr_malloc;
+void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc;
+void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc;
void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks
void _mi_page_unfull(mi_page_t* page);
void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq); // free the page
void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread...
-
-size_t _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append);
void _mi_deferred_free(mi_theap_t* theap, bool force);
-
void _mi_page_free_collect(mi_page_t* page, bool force);
-void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head);
+mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head);
mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page);
bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq);
+void _mi_page_update_stats(mi_page_t* page);
size_t _mi_page_stats_bin(const mi_page_t* page); // for stats
size_t _mi_bin_size(size_t bin); // for stats
size_t _mi_bin(size_t size); // for stats
// "theap.c"
+void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld);
+mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld);
mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld);
-void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock);
-void _mi_theap_default_set(mi_theap_t* theap);
-void _mi_theap_cached_set(mi_theap_t* theap);
void _mi_theap_collect_retired(mi_theap_t* theap, bool force);
void _mi_theap_collect_abandon(mi_theap_t* theap);
bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg);
void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page);
-bool _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock);
+
+void _mi_heap_detach_theaps( mi_heap_t* heap );
+void _mi_tld_detach_theaps( mi_tld_t* tld );
void _mi_theap_incref(mi_theap_t* theap);
void _mi_theap_decref(mi_theap_t* theap);
+void _mi_theap_merge_stats(mi_theap_t* theap);
// "heap.c"
+void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id);
void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page);
mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap); // get (and possible create) the theap belonging to a heap
-mi_decl_cold mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap); // get the theap for a heap without initializing (and return NULL in that case)
void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to); // in "arena.c"
void _mi_heap_destroy_pages(mi_heap_t* heap_from); // in "arena.c"
-void _mi_heap_force_destroy(mi_heap_t* heap); // allow destroying the main heap
+void _mi_heap_force_destroy(mi_heap_t* heap, bool acquire_heaps_lock); // allow destroying the main heap
+mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_heap_main);
+bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap);
// "stats.c"
void _mi_stats_init(void);
@@ -316,16 +323,35 @@ mi_msecs_t _mi_clock_start(void);
// "alloc.c"
void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned`
-void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept;
-void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned`
-void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept;
+void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned`
+void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept;
mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p);
void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size);
+
+// "free.c"
+void _mi_free_subproc_safe(void* p) mi_attr_noexcept;
+void _mi_page_unguard_all(mi_page_t* page);
+size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept;
+
#if MI_DEBUG>1
bool _mi_page_is_valid(mi_page_t* page);
#endif
+// "sample-guarded.c"
+mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept;
+void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p);
+void _mi_page_unguard_all(mi_page_t* page);
+void _mi_theap_guarded_init(mi_theap_t* theap);
+
+// "sample-profile.c"
+mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept;
+size_t _mi_theap_update_sample_rate(mi_theap_t* theap);
+
+mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept;
+void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p);
+size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate);
+
// ------------------------------------------------------
// Assertions
@@ -357,20 +383,32 @@ mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const
----------------------------------------------------------- */
// add to stat keeping track of the peak
-void __mi_stat_increase(mi_stat_count_t* stat, size_t amount);
-void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount);
-void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount);
-void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount);
+void __mi_stat_increase(mi_stat_count_t* stat, uint64_t amount);
+void __mi_stat_decrease(mi_stat_count_t* stat, uint64_t amount);
+void __mi_stat_increase_mt(mi_stat_count_t* stat, uint64_t amount);
+void __mi_stat_decrease_mt(mi_stat_count_t* stat, uint64_t amount);
// adjust stat in special cases to compensate for double counting (and does not adjust peak values and can decrease the total)
-void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount);
-void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount);
-void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount);
-void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount);
+void __mi_stat_adjust_increase(mi_stat_count_t* stat, uint64_t amount);
+void __mi_stat_adjust_decrease(mi_stat_count_t* stat, uint64_t amount);
+void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, uint64_t amount);
+void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, uint64_t amount);
// counters can just be increased
-void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
-void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount);
+static inline void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, uint64_t amount) {
+ mi_assert_internal(amount<=INT64_MAX);
+ mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount);
+}
+
+static inline void __mi_stat_counter_increase(mi_stat_counter_t* stat, uint64_t amount) {
+ mi_assert_internal(amount<=INT64_MAX);
+ stat->total += (int64_t)amount;
+}
+
+static inline void __mi_stat_counter_decrease(mi_stat_counter_t* stat, uint64_t amount) {
+ mi_assert_internal(amount<=INT64_MAX);
+ stat->total -= (int64_t)amount;
+}
#define mi_heap_stat_counter_increase(heap,stat,amount) __mi_stat_counter_increase_mt( &(heap)->stats.stat, amount)
#define mi_heap_stat_increase(heap,stat,amount) __mi_stat_increase_mt( &(heap)->stats.stat, amount)
@@ -384,16 +422,60 @@ void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount);
#define mi_subproc_stat_adjust_increase(subproc,stat,amount) __mi_stat_adjust_increase_mt( &(subproc)->stats.stat, amount)
#define mi_subproc_stat_adjust_decrease(subproc,stat,amount) __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amount)
-#define mi_os_stat_counter_increase(stat,amount) mi_subproc_stat_counter_increase(_mi_subproc(),stat,amount)
-#define mi_os_stat_increase(stat,amount) mi_subproc_stat_increase(_mi_subproc(),stat,amount)
-#define mi_os_stat_decrease(stat,amount) mi_subproc_stat_decrease(_mi_subproc(),stat,amount)
-
#define mi_theap_stat_counter_increase(theap,stat,amount) __mi_stat_counter_increase( &(theap)->stats.stat, amount)
+#define mi_theap_stat_counter_decrease(theap,stat,amount) __mi_stat_counter_decrease( &(theap)->stats.stat, amount)
#define mi_theap_stat_increase(theap,stat,amount) __mi_stat_increase( &(theap)->stats.stat, amount)
#define mi_theap_stat_decrease(theap,stat,amount) __mi_stat_decrease( &(theap)->stats.stat, amount)
#define mi_theap_stat_adjust_increase(theap,stat,amnt) __mi_stat_adjust_increase( &(theap)->stats.stat, amnt)
#define mi_theap_stat_adjust_decrease(theap,stat,amnt) __mi_stat_adjust_decrease( &(theap)->stats.stat, amnt)
+#define mi_theapx_stat_counter_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_counter_increase(theap,stat,amount); } else { mi_heap_stat_counter_increase(heap,stat,amount); }
+#define mi_theapx_stat_adjust_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_adjust_decrease(theap,stat,amount); } else { mi_heap_stat_adjust_decrease(heap,stat,amount); }
+#define mi_theapx_stat_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_increase(theap,stat,amount); } else { mi_heap_stat_increase(heap,stat,amount); }
+#define mi_theapx_stat_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_decrease(theap,stat,amount); } else { mi_heap_stat_decrease(heap,stat,amount); }
+
+
+/* -----------------------------------------------------------
+ pthread thread locals
+----------------------------------------------------------- */
+
+#if MI_USE_PTHREADS
+
+#if defined(__APPLE__) && defined(__aarch64__)
+#define MI_PTHREAD_KEY_INVALID ((pthread_key_t)(0)) // nicer codegen
+#else
+#define MI_PTHREAD_KEY_INVALID ((pthread_key_t)(-1))
+#endif
+
+#if defined(__linux__) && defined(__GLIBC__)
+// pthread_getspecific returns NULL for invalid keys. <https://man7.org/linux/man-pages/man3/pthread_getspecific.3p.html>
+// see also: <https://github.com/lattera/glibc/blob/master/nptl/pthread_getspecific.c>
+#define MI_PTHREADS_GET_INVALID_KEY_IS_NULL 1
+#endif
+
+mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruct)(void*), void* init);
+
+static inline void* mi_pthread_key_get(pthread_key_t key) {
+ #if !MI_PTHREADS_GET_INVALID_KEY_IS_NULL
+ if mi_unlikely(key==MI_PTHREAD_KEY_INVALID) return NULL;
+ #endif
+ return pthread_getspecific(key);
+}
+
+static inline bool mi_pthread_key_set(pthread_key_t* pkey, void* val) {
+ if mi_likely(*pkey!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(*pkey,val); return true; }
+ else if (val!=NULL) { return _mi_pthread_key_create(pkey,NULL,val); }
+ else return true;
+}
+
+static inline void mi_pthread_key_delete(pthread_key_t* pkey) {
+ const pthread_key_t key = *pkey;
+ if (key!=MI_PTHREAD_KEY_INVALID) {
+ *pkey = MI_PTHREAD_KEY_INVALID;
+ pthread_key_delete(key);
+ }
+}
+#endif
/* -----------------------------------------------------------
Options (exposed for the debugger)
@@ -440,11 +522,22 @@ typedef struct mi_option_desc_s {
// initialize a local variable to zero; use memset as compilers optimize constant sized memset's
#define _mi_memzero_var(x) memset(&x,0,sizeof(x))
+// minimum
+static inline size_t mi_min(size_t x, size_t y) { return (x <= y ? x : y); }
+
+// maximum
+static inline size_t mi_max(size_t x, size_t y) { return (x >= y ? x : y); }
+
// Is `x` a power of two? (0 is considered a power of two)
static inline bool _mi_is_power_of_two(uintptr_t x) {
return ((x & (x - 1)) == 0);
}
+// valid alignment values are as posix memalign: <https://en.cppreference.com/c/memory/aligned_alloc#Notes>
+static inline bool mi_alignment_is_valid(size_t alignment) {
+ return ((alignment!=0) && _mi_is_power_of_two(alignment));
+}
+
// Is a pointer aligned?
static inline bool _mi_is_aligned(const void* p, size_t alignment) {
return (alignment==0 || ((uintptr_t)p % alignment) == 0);
@@ -453,7 +546,7 @@ static inline bool _mi_is_aligned(const void* p, size_t alignment) {
// Align upwards
static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
mi_assert_internal(alignment != 0);
- uintptr_t mask = alignment - 1;
+ const uintptr_t mask = alignment - 1;
if ((alignment & mask) == 0) { // power of two?
return ((sz + mask) & ~mask);
}
@@ -467,18 +560,19 @@ static inline void* _mi_align_up_ptr(const void* p, size_t alignment) {
return (void*)_mi_align_up((uintptr_t)p, alignment);
}
+// Align down
static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
mi_assert_internal(alignment != 0);
- uintptr_t mask = alignment - 1;
- if ((alignment & mask) == 0) { // power of two?
+ const uintptr_t mask = alignment - 1;
+ if ((alignment & mask) == 0) { // power of two?
return (sz & ~mask);
}
else {
- return ((sz / alignment) * alignment);
+ return ((sz/alignment)*alignment);
}
}
-// align a pointer downwards
+// Align a pointer downwards
static inline void* _mi_align_down_ptr(const void* p, size_t alignment) {
return (void*)_mi_align_down((uintptr_t)p, alignment);
}
@@ -505,13 +599,6 @@ static inline bool mi_mem_is_zero(const void* p, size_t size) {
return true;
}
-// Align a byte size to a size in _machine words_,
-// i.e. byte size == `wsize*sizeof(void*)`.
-static inline size_t _mi_wsize_from_size(size_t size) {
- mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t));
- return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t);
-}
-
// Overflow detecting multiply
#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5))
#include <limits.h> // UINT_MAX, ULONG_MAX
@@ -519,10 +606,10 @@ static inline size_t _mi_wsize_from_size(size_t size) {
#undef _CLOCK_T
#endif
static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
- #if (SIZE_MAX == ULONG_MAX)
- return __builtin_umull_overflow(count, size, (unsigned long *)total);
- #elif (SIZE_MAX == UINT_MAX)
+ #if (SIZE_MAX == UINT_MAX)
return __builtin_umul_overflow(count, size, (unsigned int *)total);
+ #elif (SIZE_MAX == ULONG_MAX)
+ return __builtin_umull_overflow(count, size, (unsigned long *)total);
#else
return __builtin_umulll_overflow(count, size, (unsigned long long *)total);
#endif
@@ -562,32 +649,46 @@ static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* tot
Heap functions
------------------------------------------------------------------------------------------- */
-extern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_THREAD_LOCAL)
-extern mi_decl_hidden const mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated
+extern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_LOCAL)
+extern mi_decl_hidden mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated
+
+static inline mi_heap_t* _mi_theap_heap_peek(const mi_theap_t* theap) {
+ mi_assert_internal(theap!=NULL);
+ return mi_atomic_load_ptr_relaxed(mi_heap_t,&theap->heap);
+}
static inline mi_heap_t* _mi_theap_heap(const mi_theap_t* theap) {
- return mi_atomic_load_ptr_acquire(mi_heap_t,&theap->heap);
+ mi_heap_t* const heap = _mi_theap_heap_peek(theap);
+ mi_assert_internal(heap!=NULL);
+ return heap;
}
static inline bool mi_theap_is_initialized(const mi_theap_t* theap) {
- return (theap != NULL && _mi_theap_heap(theap) != NULL);
+ return (theap != NULL && _mi_theap_heap_peek(theap) != NULL);
+}
+
+static inline mi_subproc_t* _mi_theap_subproc(const mi_theap_t* theap) {
+ mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc);
+ mi_assert_internal(!mi_theap_is_initialized(theap) || _mi_theap_heap(theap)->subproc == subproc);
+ return subproc;
}
-static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t size) {
- mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE));
- const size_t idx = _mi_wsize_from_size(size);
+static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t xsize, bool is_wsize) {
+ mi_assert_internal(is_wsize ? xsize <= (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE) : xsize <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE));
+ const size_t idx = (is_wsize ? xsize : mi_wsize_from_size(xsize));
mi_assert_internal(idx < MI_PAGES_DIRECT);
return theap->pages_free_direct[idx];
}
+static inline bool mi_theap_is_detached(mi_theap_t* theap) {
+ return (theap!=NULL && theap->tld->thread_id == MI_THREADID_DETACHED);
+}
-//static inline uintptr_t _mi_ptr_cookie(const void* p) {
-// extern mi_theap_t _mi_theap_main;
-// mi_assert_internal(_mi_theap_main.cookie != 0);
-// return ((uintptr_t)p ^ _mi_theap_main.cookie);
-//}
-
+static inline bool mi_theap_matches_thread(mi_theap_t* theap) {
+ const mi_threadid_t tid = _mi_thread_id();
+ return (theap==NULL || theap->tld==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap));
+}
/* -----------------------------------------------------------
The page map maps addresses to `mi_page_t` pointers
@@ -598,20 +699,30 @@ static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t
// flat page-map committed on demand, using one byte per slice (64 KiB).
// single indirection and low commit, but large initial virtual reserve (4 GiB with 48 bit virtual addresses)
// used by default on <= 40 bit virtual address spaces.
-extern mi_decl_hidden uint8_t* _mi_page_map;
+extern mi_decl_hidden _Atomic(uint8_t*) _mi_page_map;
+extern mi_decl_hidden _Atomic(void*) _mi_page_map_max_address;
static inline size_t _mi_page_map_index(const void* p) {
return (size_t)((uintptr_t)p >> MI_ARENA_SLICE_SHIFT);
}
+static inline uint8_t _mi_page_map_at(size_t idx) {
+ return mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map)[idx];
+}
+
static inline mi_page_t* _mi_ptr_page_ex(const void* p, bool* valid) {
const size_t idx = _mi_page_map_index(p);
- const size_t ofs = _mi_page_map[idx];
+ const size_t ofs = _mi_page_map_at(idx);
if (valid != NULL) { *valid = (ofs != 0); }
return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) + 1 - ofs) << MI_ARENA_SLICE_SHIFT);
}
static inline mi_page_t* _mi_checked_ptr_page(const void* p) {
+ #if MI_MIN_VABITS < MI_INTPTR_BITS
+ if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) {
+ if (p > mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL;
+ }
+ #endif
bool valid;
mi_page_t* const page = _mi_ptr_page_ex(p, &valid);
return (valid ? page : NULL);
@@ -633,10 +744,17 @@ static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) {
#define MI_PAGE_MAP_SUB_SHIFT (13)
#define MI_PAGE_MAP_SUB_COUNT (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT)
#define MI_PAGE_MAP_SHIFT (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)
-#define MI_PAGE_MAP_COUNT (MI_ZU(1) << MI_PAGE_MAP_SHIFT)
typedef mi_page_t** mi_submap_t;
-extern mi_decl_hidden _Atomic(mi_submap_t)* _mi_page_map;
+typedef struct mi_page_map_s {
+ _Atomic(size_t) committed_count; // currently committed entries
+ size_t reserved_size; // full reserved size (mi_page_map_t + submaps)
+ mi_memid_t memid; // provenance
+ mi_lock_t lock; // used when allocating new submaps
+ _Atomic(mi_submap_t) submaps[1];
+} mi_page_map_t;
+
+extern mi_decl_hidden _Atomic(mi_page_map_t*) __mi_page_map;
static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) {
const size_t u = (size_t)((uintptr_t)p / MI_ARENA_SLICE_SIZE);
@@ -644,33 +762,81 @@ static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) {
return (u / MI_PAGE_MAP_SUB_COUNT);
}
-static inline mi_submap_t _mi_page_map_at(size_t idx) {
- return mi_atomic_load_ptr_relaxed(mi_page_t*, &_mi_page_map[idx]);
+static inline mi_page_map_t* _mi_page_map(void) {
+ return mi_atomic_load_ptr_relaxed(mi_page_map_t,&__mi_page_map);
+}
+
+static inline mi_submap_t _mi_page_map_at(const mi_page_map_t* pmap, size_t idx) {
+ return mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]);
}
static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) {
+ const mi_page_map_t* pmap = _mi_page_map();
size_t sub_idx;
const size_t idx = _mi_page_map_index(p, &sub_idx);
- return (_mi_page_map_at(idx))[sub_idx]; // NULL if p==NULL
+ return _mi_page_map_at(pmap,idx)[sub_idx]; // NULL if p==NULL
}
static inline mi_page_t* _mi_checked_ptr_page(const void* p) {
+ const mi_page_map_t* pmap = _mi_page_map();
size_t sub_idx;
const size_t idx = _mi_page_map_index(p, &sub_idx);
- mi_submap_t const sub = _mi_page_map_at(idx);
+ const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count);
+ if mi_unlikely(idx >= committed_count) return NULL;
+ // #if MI_MIN_VABITS < MI_INTPTR_BITS // is still invalid if free is called before the pagemap is initialized
+ // if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) {
+ // const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count);
+ // if mi_unlikely(idx >= committed_count) return NULL;
+ // }
+ // #endif
+ mi_submap_t const sub = _mi_page_map_at(pmap,idx);
if mi_unlikely(sub == NULL) return NULL;
return sub[sub_idx];
}
#endif
+#if MI_PAGE_META_IS_ALIGNED
+// if the page meta data is aligned in front of pages we can find it efficiently
+// without needing to go through the page map (for valid pointers).
+static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) {
+ mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT);
+ // const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE;
+ const uintptr_t page_idx = ((uintptr_t)p / MI_ARENA_SLICE_SIZE) % (MI_PAGE_META_ALIGNMENT / MI_ARENA_SLICE_SIZE);
+ mi_assert_internal(page_idx <= MI_PAGE_META_ALIGNED_COUNT);
+ #if MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_RISCV // better code on x64/x86/riscv64
+ mi_page_t* const page = (mi_page_t*)((uintptr_t)page_metas | (page_idx * sizeof(mi_page_t)));
+ #else
+ mi_page_t* const page = &page_metas[page_idx];
+ #endif
+ return page;
+
+}
+
+static inline mi_page_t* _mi_aligned_ptr_page(const void* p) {
+ mi_page_t* const page = _mi_aligned_ptr_page0(p);
+ if mi_unlikely(page==NULL) return NULL;
+ #if MI_DEBUG
+ mi_page_t* const cpage = _mi_checked_ptr_page(p);
+ if mi_unlikely(cpage==NULL) {
+ _mi_error_message(EINVAL, "_mi_aligned_ptr_page: invalid pointer: %p\n", p);
+ return NULL;
+ }
+ #endif
+ return mi_atomic_load_ptr_acquire(mi_page_t, &page->self);
+}
+#endif
static inline mi_page_t* _mi_ptr_page(const void* p) {
mi_assert_internal(p==NULL || mi_is_in_heap_region(p));
- #if MI_DEBUG || MI_SECURE || defined(__APPLE__)
- return _mi_checked_ptr_page(p);
- #else
- return _mi_unchecked_ptr_page(p);
+ #if MI_SECURE || MI_FREE_IS_CHECKED
+ return _mi_checked_ptr_page(p);
+ #elif MI_PAGE_META_IS_ALIGNED
+ return _mi_aligned_ptr_page(p);
+ #elif MI_DEBUG
+ return _mi_checked_ptr_page(p);
+ #else
+ return _mi_unchecked_ptr_page(p);
#endif
}
@@ -683,7 +849,8 @@ static inline size_t mi_page_block_size(const mi_page_t* page) {
// Page start
static inline uint8_t* mi_page_start(const mi_page_t* page) {
- return page->page_start;
+ // multiplication must be done in `size_t`
+ return (uint8_t*)page + page->page_offset;
}
static inline size_t mi_page_size(const mi_page_t* page) {
@@ -720,19 +887,27 @@ static inline size_t mi_page_usable_block_size(const mi_page_t* page) {
}
static inline bool mi_page_meta_is_separated(const mi_page_t* page) {
- #if MI_PAGE_META_IS_SEPARATED
- // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_ALIGNED_FREE_SMALL
- return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(page->page_start, MI_ARENA_SLICE_ALIGN));
+ #if MI_PAGE_META_IS_ALIGNED
+ #if MI_PAGE_META_SMALL_IS_ALIGNED
+ return (page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN));
+ #else
+ MI_UNUSED_RELEASE(page);
+ mi_assert_internal(page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN));
+ return true;
+ #endif
+ #elif MI_PAGE_META_IS_SEPARATED
+ // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_SMALL_IS_ALIGNED
+ return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN));
#else
MI_UNUSED(page);
- return false;
+ return false;
#endif
}
static inline uint8_t* mi_page_slice_start(const mi_page_t* page) {
- if (mi_page_meta_is_separated(page)) {
+ if (mi_page_meta_is_separated(page)) {
// page meta info is at a separate location (at `arena->pages`)
- return (uint8_t*)_mi_align_down_ptr(page->page_start, MI_ARENA_SLICE_ALIGN);
+ return (uint8_t*)_mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN);
}
else {
// page meta info is at the start of the page slices
@@ -740,21 +915,57 @@ static inline uint8_t* mi_page_slice_start(const mi_page_t* page) {
}
}
-// This gives the offset relative to the start slice of a page.
+// This gives the offset relative to the start slice of a page.
static inline size_t mi_page_slice_offset_of(const mi_page_t* page, size_t offset_relative_to_page_start) {
- return (page->page_start - mi_page_slice_start(page)) + offset_relative_to_page_start;
+ return (mi_page_start(page) - mi_page_slice_start(page)) + offset_relative_to_page_start;
+}
+
+// How much of the page is committed relative to the slice start? (or 0 if fully committed already)
+static inline size_t mi_page_slice_committed(const mi_page_t* page) {
+ return ((size_t)page->slice_pcommitted * _mi_os_page_size());
}
// Currently committed part of a page
static inline size_t mi_page_committed(const mi_page_t* page) {
- return (page->slice_committed == 0 ? mi_page_size(page) : page->slice_committed - mi_page_slice_offset_of(page,0));
+ const size_t slice_committed = mi_page_slice_committed(page);
+ return (slice_committed == 0 ? mi_page_size(page) : slice_committed - mi_page_slice_offset_of(page,0));
+}
+
+static inline size_t mi_page_used(const mi_page_t* page) {
+ mi_assert_internal(page != NULL);
+ return mi_xused_used_count(page->xused);
+}
+
+static inline void mi_page_used_reset(mi_page_t* page) {
+ page->xused = mi_xused_used_reset(page->xused);
+}
+
+static inline size_t mi_page_alloc_count(const mi_page_t* page) {
+ return mi_xused_alloc_count(page->xused);
+}
+
+static inline size_t mi_page_last_used(const mi_page_t* page) {
+ #if MI_SIZE_SIZE >= 8
+ return (page->xused.used_alloc >> 32) & 0xFFFF;
+ #else
+ return page->xlast_used;
+ #endif
}
+static inline size_t mi_page_last_alloc(const mi_page_t* page) {
+ #if MI_SIZE_SIZE >= 8
+ return (page->xused.used_alloc >> 48) & 0xFFFF;
+ #else
+ return page->xlast_alloc;
+ #endif
+}
+
+
// are all blocks in a page freed?
// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`.
static inline bool mi_page_all_free(const mi_page_t* page) {
mi_assert_internal(page != NULL);
- return (page->used == 0);
+ return (mi_page_used(page)==0);
}
// are there immediately available blocks, i.e. blocks available on the free list.
@@ -773,7 +984,7 @@ static inline bool mi_page_is_expandable(const mi_page_t* page) {
static inline bool mi_page_is_full(const mi_page_t* page) {
- const bool full = (page->reserved == page->used);
+ const bool full = (page->reserved == mi_page_used(page));
mi_assert_internal(!full || page->free == NULL);
return full;
}
@@ -782,14 +993,14 @@ static inline bool mi_page_is_full(const mi_page_t* page) {
static inline bool mi_page_is_mostly_used(const mi_page_t* page) {
if (page==NULL) return true;
uint16_t frac = page->reserved / 8U;
- return (page->reserved - page->used <= frac);
+ return (page->reserved - mi_page_used(page) <= frac);
}
// is more than (n-1)/n'th of a page in use?
static inline bool mi_page_is_used_at_frac(const mi_page_t* page, uint16_t n) {
if (page==NULL) return true;
uint16_t frac = page->reserved / n;
- return (page->reserved - page->used <= frac);
+ return (page->reserved - mi_page_used(page) <= frac);
}
@@ -805,6 +1016,10 @@ static inline mi_page_queue_t* mi_page_queue(const mi_theap_t* theap, size_t siz
return pq;
}
+static inline size_t mi_page_min_commit_size(void) {
+ const size_t psize = _mi_os_page_size();
+ return (MI_PAGE_MIN_COMMIT_SIZE >= psize ? MI_PAGE_MIN_COMMIT_SIZE : psize);
+}
//-----------------------------------------------------------
// Page thread id and flags
@@ -842,7 +1057,8 @@ static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) {
mi_theap_t* const theap = page->theap;
mi_assert_internal(theap!=NULL);
if (theap != NULL) {
- const size_t size = page->capacity * mi_page_block_size(page);
+ mi_assert_internal(page->capacity==page->reserved);
+ const size_t size = page->reserved * mi_page_block_size(page);
if (in_full) { theap->pages_full_size += size; }
else { mi_assert_internal(size <= theap->pages_full_size); theap->pages_full_size -= size; }
}
@@ -893,7 +1109,7 @@ static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) {
static inline mi_theap_t* mi_page_theap(const mi_page_t* page) {
mi_assert_internal(!mi_page_is_abandoned(page));
- mi_assert_internal(page->theap != NULL);
+ mi_assert_internal(page->theap != NULL && page->theap != &_mi_theap_empty);
return page->theap;
}
@@ -906,25 +1122,53 @@ static inline mi_tld_t* mi_page_tld(const mi_page_t* page) {
static inline mi_heap_t* mi_page_heap(const mi_page_t* page) {
mi_heap_t* heap = page->heap;
- // we use NULL for the main heap to make `_mi_page_get_associated_theap` fast in `free.c:mi_abandoned_page_try_reclaim`.
- if mi_likely(heap==NULL) heap = mi_heap_main();
mi_assert_internal(heap != NULL);
return heap;
}
+static inline mi_subproc_t* mi_page_subproc(const mi_page_t* page) {
+ mi_heap_t* const heap = mi_page_heap(page);
+ return heap->subproc;
+}
+
+static inline mi_heap_t* mi_arena_heap_main(const mi_arena_t* arena) {
+ return _mi_subproc_heap_main(arena->subproc);
+}
+
+static inline mi_heap_t* mi_heap_get_heap_main(const mi_heap_t* heap) {
+ return _mi_subproc_heap_main(heap->subproc);
+}
+
+static inline bool _mi_is_heap_main(const mi_heap_t* heap) {
+ mi_assert_internal(heap!=NULL);
+ return (mi_heap_get_heap_main(heap) == heap);
+}
+
+static inline bool _mi_is_process_heap_main(const mi_heap_t* heap) {
+ mi_assert_internal(heap!=NULL);
+ return (_mi_subproc_main()->heap_main == heap);
+}
+
//-----------------------------------------------------------
// Thread free list and ownership
//-----------------------------------------------------------
// Thread free flag helpers
static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) {
- return (mi_block_t*)(tf & ~1);
+ return (mi_block_t*)(tf & ~1);
}
+
static inline bool mi_tf_is_owned(mi_thread_free_t tf) {
return ((tf & 1) == 1);
}
+
static inline mi_thread_free_t mi_tf_create(mi_block_t* block, bool owned) {
- return (mi_thread_free_t)((uintptr_t)block | (owned ? 1 : 0));
+ const uintptr_t base = (uintptr_t)block | (owned ? 1 : 0);
+ return (mi_thread_free_t)base;
+}
+
+static inline mi_thread_free_t mi_tf_set_owned(mi_thread_free_t tf, bool owned) {
+ return mi_tf_create(mi_tf_block(tf), owned);
}
// Thread free access
@@ -935,7 +1179,7 @@ static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) {
// are there any available blocks?
static inline bool mi_page_has_any_available(const mi_page_t* page) {
mi_assert_internal(page != NULL && page->reserved > 0);
- return (page->used < page->reserved || (mi_page_thread_free(page) != NULL));
+ return (mi_page_used(page) < page->reserved || (mi_page_thread_free(page) != NULL));
}
// Owned?
@@ -951,14 +1195,26 @@ static inline bool mi_page_claim_ownership(mi_page_t* page) {
/* -------------------------------------------------------------------
- Guarded objects
+ Guarded and profiled objects
------------------------------------------------------------------- */
-#if MI_GUARDED
+
+#define MI_SAMPLE_RATE_MAX (SIZE_MAX/4)
+#define MI_SAMPLE_COUNTDOWN_MAX (MI_MAX_ALLOC_SIZE)
+
+static inline bool mi_theap_should_sample(mi_theap_t* theap, size_t req_size) {
+ // note: this should return `true` on an empty theap so we initialize it's countdown to `-1`.
+ mi_assert_internal(req_size <= SIZE_MAX/2);
+ // const size_t sample_countdown = theap->sample_countdown - req_size;
+ // return ((mi_ssize_t)sample_countdown < 0);
+ return ((mi_ssize_t)theap->sample_countdown < (mi_ssize_t)req_size);
+}
+
// we always align guarded pointers in a block at an offset
// the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones
#define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0))
+#define MI_BLOCK_TAG_PROFILED ((mi_encoded_t)(1))
#define MI_BLOCK_TAG_GUARDED (~MI_BLOCK_TAG_ALIGNED)
-#endif
+
static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) {
#if MI_GUARDED
@@ -967,33 +1223,28 @@ static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void*
#else
MI_UNUSED(block); MI_UNUSED(p);
return false;
-#endif
+#endif
}
-#if MI_GUARDED
-static inline bool mi_theap_malloc_use_guarded(mi_theap_t* theap, size_t size) {
- // this code is written to result in fast assembly as it is on the hot path for allocation
- const size_t count = theap->guarded_sample_count - 1; // if the rate was 0, this will underflow and count for a long time..
- if mi_likely(count != 0) {
- // no sample
- theap->guarded_sample_count = count;
- return false;
- }
- else if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) {
- // use guarded allocation
- theap->guarded_sample_count = theap->guarded_sample_rate; // reset
- return (theap->guarded_sample_rate != 0);
- }
- else {
- // failed size criteria, rewind count (but don't write to an empty theap)
- if (theap->guarded_sample_rate != 0) { theap->guarded_sample_count = 1; }
- return false;
- }
+static inline bool mi_block_ptr_is_sampled(const mi_block_t* block, const void* p) {
+#if MI_GUARDED || MI_PROFILE
+ const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block;
+ return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next != MI_BLOCK_TAG_ALIGNED);
+#else
+ MI_UNUSED(block); MI_UNUSED(p);
+ return false;
+#endif
}
-mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept;
+static inline bool mi_profiler_is_enabled(const mi_profiler_t* prof) {
+ _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved;
+ return (mi_atomic_load_acquire(penabled) != 0);
+}
-#endif
+static inline bool mi_profiler_set_enabled(mi_profiler_t* prof, bool enable) {
+ _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved;
+ return (mi_atomic_exchange_release(penabled, (enable ? 1 : 0)) != 0);
+}
/* -------------------------------------------------------------------
@@ -1028,25 +1279,46 @@ static inline bool mi_is_in_same_page(const void* p, const void* q) {
}
static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) {
- void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]);
+ const uintptr_t k1 = keys[0];
+ #if MI_PAGE_KEY_COUNT==2
+ const uintptr_t k2 = keys[1];
+ #else
+ const uintptr_t k2 = mi_rotr(k1,13);
+ #endif
+ void* p = (void*)(mi_rotr(x - k1, k1) ^ k2);
return (p==null ? NULL : p);
}
static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) {
- uintptr_t x = (uintptr_t)(p==NULL ? null : p);
- return mi_rotl(x ^ keys[1], keys[0]) + keys[0];
+ const uintptr_t k1 = keys[0];
+ #if MI_PAGE_KEY_COUNT==2
+ const uintptr_t k2 = keys[1];
+ #else
+ const uintptr_t k2 = mi_rotr(k1,13);
+ #endif
+ const uintptr_t x = (uintptr_t)(p==NULL ? null : p);
+ return mi_rotl(x ^ k2, k1) + k1;
}
static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const uintptr_t* keys) {
const uint32_t x = (uint32_t)(mi_ptr_encode(null,p,keys));
// make the lowest byte 0 to prevent spurious read overflows which could be a security issue (issue #951)
+ // also clear bit 9 which we set only when a block is freed.
#if MI_BIG_ENDIAN
- return (x & 0x00FFFFFF);
+ return (x & 0x00FFFEFF);
#else
- return (x & 0xFFFFFF00);
+ return (x & 0xFFFFFE00);
#endif
}
+static inline uint32_t mi_ptr_encode_canary_freed(void) {
+ return (0x00DEAD00); // set bit 9 so it is different from any valid canary
+}
+
+static inline bool mi_ptr_decode_canary_is_freed(uint32_t canary) {
+ return (canary == mi_ptr_encode_canary_freed());
+}
+
static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) {
mi_track_mem_defined(block,sizeof(mi_block_t));
mi_block_t* next;
@@ -1071,14 +1343,15 @@ static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const
mi_track_mem_noaccess(block,sizeof(mi_block_t));
}
+mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next); // in options.c
+
static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) {
#if MI_ENCODE_FREELIST
mi_block_t* next = mi_block_nextx(page,block,page->keys);
// check for free list corruption: is `next` at least in the same page?
- // TODO: check if `next` is `page->block_size` aligned?
- if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) {
- _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next);
- next = NULL;
+ // todo: check if `next` is `page->block_size` aligned?
+ if mi_unlikely(next!=NULL && !mi_page_contains_address(page,next)) {
+ return _mi_block_next_is_corrupted(page,block,next); // returns NULL
}
return next;
#else
@@ -1096,6 +1369,8 @@ static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, c
#endif
}
+
+
/* -----------------------------------------------------------
arena blocks
----------------------------------------------------------- */
@@ -1136,32 +1411,55 @@ static inline mi_memid_t _mi_memid_create_os(void* base, size_t size, bool commi
return memid;
}
-static inline mi_memid_t _mi_memid_create_meta(void* mpage, size_t block_idx, size_t block_count) {
- mi_memid_t memid = _mi_memid_create(MI_MEM_META);
- memid.mem.meta.meta_page = mpage;
- memid.mem.meta.block_index = (uint32_t)block_idx;
- memid.mem.meta.block_count = (uint32_t)block_count;
+static inline mi_memid_t _mi_memid_create_static(void* p, size_t size) {
+ mi_memid_t memid = _mi_memid_create(MI_MEM_STATIC);
+ memid.mem.malloc.base = p;
+ memid.mem.malloc.size = size;
memid.initially_committed = true;
- memid.initially_zero = true;
memid.is_pinned = true;
return memid;
}
+static inline mi_memid_t _mi_memid_create_malloc(void* p, size_t size, bool iszero) {
+ mi_memid_t memid = _mi_memid_create(MI_MEM_MALLOC);
+ memid.mem.malloc.base = p;
+ memid.mem.malloc.size = size;
+ memid.initially_committed = true;
+ memid.initially_zero = iszero;
+ memid.is_pinned = true;
+ return memid;
+}
+
+static inline size_t _mi_memid_size(mi_memid_t memid) {
+ if (mi_memid_is_os(memid)) {
+ return memid.mem.os.size;
+ }
+ else if (memid.memkind == MI_MEM_ARENA) {
+ return mi_size_of_slices(memid.mem.arena.slice_count);
+ }
+ else if (memid.memkind == MI_MEM_MALLOC) {
+ return memid.mem.malloc.size;
+ }
+ else {
+ mi_assert_internal(mi_memid_needs_no_free(memid));
+ return 0;
+ }
+}
// -------------------------------------------------------------------
// Fast "random" shuffle
// -------------------------------------------------------------------
-static inline uintptr_t _mi_random_shuffle(uintptr_t x) {
+static inline size_t _mi_random_shuffle(size_t x) {
if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros
-#if (MI_INTPTR_SIZE>=8)
+#if (MI_SIZE_SIZE>=8)
// by Sebastiano Vigna, see: <http://xoshiro.di.unimi.it/splitmix64.c>
x ^= x >> 30;
x *= 0xbf58476d1ce4e5b9UL;
x ^= x >> 27;
x *= 0x94d049bb133111ebUL;
x ^= x >> 31;
-#elif (MI_INTPTR_SIZE==4)
+#elif (MI_SIZE_SIZE==4)
// by Chris Wellons, see: <https://nullprogram.com/blog/2018/07/31/>
x ^= x >> 16;
x *= 0x7feb352dUL;
@@ -1175,108 +1473,83 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x) {
// ---------------------------------------------------------------------------------
// Provide our own `_mi_memcpy/set` for potential performance optimizations.
-//
-// For now, only on x64/x86 we optimize to `rep movsb/stosb`.
-// Generally, we check for "fast short rep movsb/stosb" (FSRM/FSRS) or "fast enhanced rep movsb" (ERMS) support
-// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253.
-// Todo: we see improvements on win32 but less with glibc; we might want to only enable this on windows.
// ---------------------------------------------------------------------------------
-#if !MI_TRACK_ENABLED && (MI_ARCH_X64 || MI_ARCH_X86) && (defined(_WIN32) || defined(__GNUC__))
-
-extern mi_decl_hidden size_t _mi_cpu_movsb_max; // in init.c
-extern mi_decl_hidden size_t _mi_cpu_stosb_max;
-
-static inline void mi_rep_movsb(void* dst, const void* src, size_t n) {
- #if defined(__GNUC__)
- __asm volatile("rep movsb" : "+D"(dst), "+c"(n), "+S"(src) : : "memory");
- #else
- __movsb((unsigned char*)dst, (const unsigned char*)src, n);
- #endif
-}
-
-static inline void mi_rep_stosb(void* dst, uint8_t val, size_t n) {
- #if defined(__GNUC__)
- __asm volatile("rep stosb" : "+D"(dst), "+c"(n) : "a"(val) : "memory");
- #else
- __stosb((unsigned char*)dst, val, n);
- #endif
-}
-
-static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
- if mi_likely(n <= _mi_cpu_movsb_max) { // has fsrm && n <= 127 (todo: and maybe has erms?)
- mi_rep_movsb(dst, src, n);
- }
- else {
- memcpy(dst, src, n);
- }
+static inline void* _mi_memcpy(void* dst, const void* src, size_t n) {
+ return memcpy(dst, src, n);
}
-static inline void _mi_memset(void* dst, int val, size_t n) {
- if mi_likely(n <= _mi_cpu_stosb_max) { // has fsrs && n <= 127
- mi_rep_stosb(dst, (uint8_t)val, n);
- }
- else {
- memset(dst, val, n);
- }
+static inline void* _mi_memset(void* dst, int val, size_t n) {
+ return memset(dst, val, n);
}
-#else
-
-static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
- memcpy(dst, src, n);
+static inline void* _mi_memset_backward(void* dst, int val, size_t n) {
+ memset((uint8_t*)dst - n, val, n);
+ return dst;
}
-static inline void _mi_memset(void* dst, int val, size_t n) {
- memset(dst, val, n);
+static inline void* _mi_memzero(void* dst, size_t n) {
+ return _mi_memset(dst, 0, n);
}
-#endif
-
-// -------------------------------------------------------------------------------
-// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned
-// This is used for example in `mi_realloc`.
-// -------------------------------------------------------------------------------
-
-#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
-
-// On GCC/CLang we provide a hint that the pointers are word aligned.
-static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
- mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));
- void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);
- const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE);
- _mi_memcpy(adst, asrc, n);
+static inline void* _mi_memzero_backward(void* dst, size_t n) {
+ return _mi_memset_backward(dst, 0, n);
}
-static inline void _mi_memset_aligned(void* dst, int val, size_t n) {
- mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);
- void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);
- _mi_memset(adst, val, n);
+static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
+ // on gcc/clang we can provide a hint that the pointers are word aligned.
+ mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE) && _mi_is_aligned(src,MI_SIZE_SIZE));
+ void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE);
+ const void* asrc = mi_assume_aligned(src, MI_SIZE_SIZE);
+ return _mi_memcpy(adst, asrc, n);
}
-#else
-
-// Default fallback on `_mi_memcpy`
-static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
- mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));
- _mi_memcpy(dst, src, n);
+static inline void* _mi_memset_aligned(void* dst, int val, size_t n) {
+ mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE));
+ void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE);
+ return _mi_memset(adst, val, n);
}
-static inline void _mi_memset_aligned(void* dst, int val, size_t n) {
- mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);
- _mi_memset(dst, val, n);
-}
-
-#endif
-
-static inline void _mi_memzero(void* dst, size_t n) {
- _mi_memset(dst, 0, n);
+static inline void* _mi_memzero_aligned(void* dst, size_t n) {
+ return _mi_memset_aligned(dst, 0, n);
}
-static inline void _mi_memzero_aligned(void* dst, size_t n) {
- _mi_memset_aligned(dst, 0, n);
+// Zero a block: blocks are always aligned with a positive bsize in machine-word bytes.
+static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize) {
+ mi_assert_internal(bsize%MI_SIZE_SIZE == 0);
+ mi_assert_internal(bsize > 0);
+ mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE));
+ mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || _mi_is_aligned(dst,MI_MAX_ALIGN_SIZE));
+
+ // fast memzero for small sizes based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned)
+ // assumes constant memset(p,0,N) gets optimized to fast simd stores by the compiler
+ // note: disabled on riscv for now as a constant memset is not always replaced correctly by current compilers.
+ // (compile with -DMI_USE_MEMZERO16X=0 to disable this)
+ #if (!defined(MI_USE_MEMZERO16X) && !MI_ARCH_RISCV) || (MI_USE_MEMZERO16X != 0) // 16x MI_SIZE_SIZE (128 bytes on 64-bit)
+ if mi_unlikely(bsize < 2*MI_SIZE_SIZE) { // bsize < 16 (8)
+ *((size_t*)dst) = 0;
+ return dst;
+ }
+ mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE));
+ uint8_t* const start = (uint8_t*)mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE);
+ uint8_t* const end = start + bsize; // note: if bsize is always a multiple of 16 then end is always aligned as well (but due to padding this does not hold)
+ if mi_likely(bsize < 8*MI_SIZE_SIZE) { // bsize < 64 (32)
+ const size_t ofs = (bsize>>1)&(2*MI_SIZE_SIZE); mi_assert_internal(bsize < 4*MI_SIZE_SIZE ? ofs==0 : ofs==2*MI_SIZE_SIZE); // ofs == 16 (8)
+ _mi_memzero(start, 2*MI_SIZE_SIZE);
+ _mi_memzero(start+ofs, 2*MI_SIZE_SIZE);
+ _mi_memzero_backward(end-ofs, 2*MI_SIZE_SIZE);
+ _mi_memzero_backward(end, 2*MI_SIZE_SIZE);
+ return dst;
+ }
+ if mi_likely(bsize <= 16*MI_SIZE_SIZE) { // bsize < 128 (64)
+ _mi_memzero(start, 8*MI_SIZE_SIZE);
+ _mi_memzero_backward(end, 8*MI_SIZE_SIZE);
+ return dst;
+ }
+ #endif
+ // fallback to regular memset for larger sizes
+ void* const wdst = mi_assume_aligned(dst,MI_SIZE_SIZE);
+ return _mi_memzero_aligned(wdst, bsize);
}
-
-
#endif // MI_INTERNAL_H
diff --git a/vendored/mimalloc/include/mimalloc/prim-tls.h b/vendored/mimalloc/include/mimalloc/prim-tls.h
new file mode 100644
index 000000000..da78d0193
--- /dev/null
+++ b/vendored/mimalloc/include/mimalloc/prim-tls.h
@@ -0,0 +1,434 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#pragma once
+#ifndef MIMALLOC_PRIM_TLS_H
+#define MIMALLOC_PRIM_TLS_H
+
+#include "types.h"
+#include "internal.h" // mi_decl_hidden
+
+// --------------------------------------------------------------------------
+// We need fast access to both a unique thread id (in `free.c:mi_free`) and
+// to a thread-local theap pointer (in `alloc.c:mi_malloc`).
+//
+// For performance, we tend to use specialized code for various platforms.
+// This leads to quite a few ifdefs but it is just for performance and there
+// is always a portable fallback (based on regular thread local variables).
+//
+// Windows : use NtCurrentTeB and TlsAlloc (MI_TLS_MODEL_WIN32)
+// Linux,FreeBSD : use thread locals with the initial-exec model (MI_TLS_MODEL_LOCAL)
+// macOS : use pthread locals with assembly for the thread-id (MI_TLS_MODEL_PTHREADS)
+// Android,OpenBSD : use pthread locals (MI_TLS_MODEL_PTHREADS). todo: maybe on Android MI_TLS_MODEL_LOCAL is better?
+// --------------------------------------------------------------------------
+
+// static inline void** mi_prim_thread_pointer(void); // get a pointer to the thread local storage
+// static inline void* mi_prim_tls_slot(size_t slot); // directly read an entry from the thread local storage block
+// static inline void mi_prim_tls_slot_set(size_t slot, void* value);
+
+static inline mi_threadid_t _mi_prim_thread_id(void); // get a unique id for a thread
+static inline mi_theap_t* _mi_theap_default(void); // the default thread local theap
+static inline mi_theap_t* _mi_theap_cached(void); // last used thread local theap using the _heap_ api
+static inline bool _mi_thread_is_initialized(void); // a thread is initialized if it has a default theap
+static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap); // get the thread local theap belonging to a heap
+static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap); // get the theap but don't update _mi_theap_cached
+static inline mi_theap_t* _mi_page_associated_theap_peek(const mi_page_t* page); // get the theap associated with a page (used in `mi_free_collect_mt`)
+
+
+// Default TLS model
+#if !defined(MI_TLS_MODEL_LOCAL) && !defined(MI_TLS_MODEL_PTHREADS) && !defined(MI_TLS_MODEL_FIXED) && !defined(MI_TLS_MODEL_WIN32)
+#if defined(_WIN32)
+#define MI_TLS_MODEL_WIN32 1
+#elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__) // and FreeBSD?
+#define MI_TLS_MODEL_PTHREADS 1
+#else
+#define MI_TLS_MODEL_LOCAL 1
+#endif
+#endif
+
+
+//-------------------------------------------------------------------
+// Access to the thread pointer
+//-------------------------------------------------------------------
+
+// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id
+// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)
+// Nevertheless, it seems needed on older graviton platforms (see issue #851).
+// For now, we only enable this for specific platforms.
+#if !defined(MI_USE_BUILTIN_THREAD_POINTER) /* allow user override */
+ #if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
+ && !defined(__CYGWIN__) \
+ && !defined(MI_LIBC_MUSL) \
+ && (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */
+ #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \
+ || (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__riscv)) \
+ || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \
+ || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__) || defined(__riscv)))
+ #define MI_USE_BUILTIN_THREAD_POINTER 1
+ #endif
+ #endif
+#endif
+
+#if defined(MI_PRIM_THREAD_POINTER) // potential user override
+static inline void** mi_prim_thread_pointer(void) {
+ return MI_PRIM_THREAD_POINTER();
+}
+#elif defined(_WIN32)
+static inline void** mi_prim_thread_pointer(void) {
+ return (void**)NtCurrentTeb();
+}
+#elif MI_USE_BUILTIN_THREAD_POINTER
+static inline void** mi_prim_thread_pointer(void) {
+ return (void**)__builtin_thread_pointer();
+}
+#elif defined(__GNUC__) && !defined(__CYGWIN__)
+ #if defined(__aarch64__)
+ static inline void** mi_prim_thread_pointer(void) {
+ void** tcb;
+ #if defined(__APPLE__) // M1, issue rgb(62, 76, 62)
+ __asm__ ("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r" (tcb));
+ #else
+ __asm__ ("mrs %0, tpidr_el0" : "=r" (tcb));
+ #endif
+ return tcb;
+ }
+ #elif defined(__riscv)
+ static inline void** mi_prim_thread_pointer(void) {
+ void** tcb;
+ __asm__ ("mv %0, tp" : "=r" (tcb));
+ return tcb;
+ }
+ #elif defined(__arm__)
+ static inline void** mi_prim_thread_pointer(void) {
+ void** tcb;
+ __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\n\tbic %0, %0, #3" : "=r" (tcb));
+ return tcb;
+ }
+ #elif defined(__i386__)
+ static inline void** mi_prim_thread_pointer(void) {
+ void** tcb;
+ __asm__ ("movl %%gs:0, %0" : "=r" (tcb) : : ); // x86 32-bit always uses GS
+ return tcb;
+ }
+ #elif defined(__x86_64__)
+ static inline void** mi_prim_thread_pointer(void) {
+ void** tcb;
+ #if defined(__APPLE__)
+ __asm__ ("movq %%gs:0, %0" : "=r" (tcb) : : ); // x86_64 macOSX uses GS
+ #elif (MI_SIZE_SIZE==4)
+ __asm__ ("movl %%fs:0, %0" : "=r" (tcb) : : ); // x32 ABI
+ #else
+ __asm__ ("movq %%fs:0, %0" : "=r" (tcb) : : ); // x86_64 Linux, BSD uses FS
+ #endif
+ return tcb;
+ }
+ #else
+ #define MI_NO_THREAD_POINTER (1)
+ #endif
+#elif MI_USE_PTHREADS && defined(__APPLE__)
+static inline void** mi_prim_thread_pointer(void) {
+ return (void**)pthread_self();
+}
+#else
+#define MI_NO_THREAD_POINTER (1)
+#endif
+
+#if !MI_NO_THREAD_POINTER
+#define MI_HAS_TLS_SLOT (1)
+static inline void* mi_prim_tls_slot(size_t slot) {
+ #if defined(_WIN32)
+ #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC)
+ return (void*)__readgsqword((unsigned long)(slot*sizeof(void*))); // direct load at offset from gs
+ #elif _M_IX86 && !defined(_M_ARM64EC)
+ return (void*)__readfsdword((unsigned long)(slot*sizeof(void*))); // direct load at offset from fs
+ #else
+ return mi_prim_thread_pointer()[slot];
+ #endif
+ #else
+ return mi_prim_thread_pointer()[slot];
+ #endif
+}
+
+static inline void mi_prim_tls_slot_set(size_t slot, void* value) {
+ mi_prim_thread_pointer()[slot] = value;
+}
+#endif
+
+/* ----------------------------------------------------------------------------------------
+ Get a unique thread-id for a thread
+---------------------------------------------------------------------------------------- */
+
+// Get a unique id for the current thread.
+#if defined(MI_PRIM_THREAD_ID)
+static inline mi_threadid_t __mi_prim_thread_id(void) {
+ return MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488)
+}
+#elif !MI_NO_THREAD_POINTER
+static inline mi_threadid_t __mi_prim_thread_id(void) {
+ #if defined(__BIONIC__)
+ return (mi_threadid_t)mi_prim_tls_slot(1);
+ #else
+ return (mi_threadid_t)mi_prim_thread_pointer();
+ #endif
+}
+#else
+// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).
+extern mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper;
+static inline mi_threadid_t __mi_prim_thread_id(void) {
+ return (uintptr_t)&__mi_thread_id_helper;
+}
+#endif
+
+static inline mi_threadid_t _mi_prim_thread_id(void) {
+ const mi_threadid_t tid = __mi_prim_thread_id();
+ mi_assert_internal(tid > MI_THREADID_DETACHED);
+ mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear?
+ return tid;
+}
+
+
+/* ----------------------------------------------------------------------------------------
+Get the thread local default theap: `_mi_theap_default()` (and the cached heap `_mi_theap_cached`).
+
+This is inlined here as it is on the fast path for allocation functions.
+We have 4 models:
+
+- MI_TLS_MODEL_LOCAL: use regular thread local (default on Linux, FreeBSD, etc)
+ On most platforms (Linux, FreeBSD, NetBSD, etc), this just returns a
+ thread local variable (`__mi_theap_default`). With the initial-exec TLS model this ensures
+ that the storage will always be available and properly initialized (with an empty theap).
+
+ On some platforms the underlying TLS implementation (or the loader) will call itself `malloc`
+ on a first access to a thread local and recurse in the MI_TLS_MODEL_LOCAL.
+ A way around this is to define MI_TLS_RECURSE_GUARD which adds an extra check if the process
+ is initialized before accessing the thread-local. This is a check in the fast path though
+ so this should be avoided.
+
+- MI_TLS_MODEL_PTHREADS: use `pthread_getspecific`. (default on macOS and OpenBSD, maybe good for Android as well?)
+ Use pthread local storage. Can be as fast as thread locals on many platforms (like recent macOS).
+
+- MI_TLS_MODEL_FIXED: use a fixed slot in the TLS block.
+ This reserves an unused and fixed TLS slot. This is fast and avoids the problem
+ where the underlying TLS implementation (or the loader) will call itself `malloc`
+ on a first access to a thread local (and recurse in the MI_TLS_MODEL_LOCAL).
+ This goes wrong though if the OS or a library uses the same fixed slot, and also
+ prevents multiple instances of mimalloc in the same process.
+
+- MI_TLS_MODEL_WIN32: use a dynamically allocated slot with TlsAlloc. (default on Windows)
+ We use TlsAlloc'd slot. First tries to use one of the "direct" first 64 slots which
+ are the fastest, but falls back to using "expansion" slots when needed (up to 1088 slots).
+ (If the allocated slot happens to always be under 64 for a particular program,
+ one might use cmake with `-DMI_WIN_DIRECT_TLS=ON` to skip the expansion slot test in the fast path.)
+
+Each model should define `MI_THEAP_INITASNULL` to signify that the initial value
+returned from `_mi_theap_default()` can be `NULL` (instead of the address of the empty heap).
+This incurs an extra check in the fast path (but can often be combined in an existing check).
+------------------------------------------------------------------------------------------- */
+
+#if !defined(MI_TLS_RECURSE_GUARD) && MI_TLS_MODEL_LOCAL && defined(__APPLE__)
+#define MI_TLS_RECURSE_GUARD 1 // macOS can allocate on thread-local initialization
+#endif
+
+// Declared this way to optimize register spills and branches
+mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void);
+
+static inline mi_theap_t* __mi_theap_empty(void) {
+ #if __GNUC__
+ __asm(""); // prevent conditional load
+ return (mi_theap_t*)&_mi_theap_empty;
+ #else
+ return _mi_theap_empty_get();
+ #endif
+}
+
+#if MI_TLS_MODEL_LOCAL
+// Thread local with an initial value (default on Linux). Very efficient.
+extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default; // default theap to allocate from
+extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached; // theap from the last used heap
+
+// defined in `init.c`; do not use these directly
+extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called?
+
+static inline mi_theap_t* _mi_theap_default(void) {
+ #if defined(MI_TLS_RECURSE_GUARD)
+ if mi_unlikely(!_mi_process_is_initialized) return _mi_theap_empty_get();
+ #endif
+ return __mi_theap_default;
+}
+
+static inline mi_theap_t* _mi_theap_cached(void) {
+ return __mi_theap_cached;
+}
+
+#elif MI_TLS_MODEL_PTHREADS
+// Dynamic pthread slots. This can be fast depending on the platform (default for macOS and OpenBSD)
+// On some platforms (like macOS), the loader might allocate on thread local declarations which
+// can be avoided with pthreads.
+#define MI_THEAP_INITASNULL 1
+
+extern mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_default_key;
+extern mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_cached_key;
+
+static inline mi_theap_t* _mi_theap_default(void) {
+ pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_default_key);
+ #if defined(__APPLE__) && defined(__aarch64__) && MI_HAS_TLS_SLOT
+ // on apple arm64, the pthread specific slots are direct slots; inline it to avoid a stack frame setup in `mi_malloc`
+ // todo: this is probably also the case on x64 and power pc?
+ if (key == MI_PTHREAD_KEY_INVALID) return NULL;
+ return (mi_theap_t*)mi_prim_tls_slot(key);
+ #else
+ return (mi_theap_t*)mi_pthread_key_get(key);
+ #endif
+}
+
+static inline mi_theap_t* _mi_theap_cached(void) {
+ pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_cached_key);
+ #if defined(__APPLE__) && defined(__aarch64__) && MI_HAS_TLS_SLOT
+ if (key == MI_PTHREAD_KEY_INVALID) return NULL;
+ return (mi_theap_t*)mi_prim_tls_slot(key);
+ #else
+ return (mi_theap_t*)mi_pthread_key_get(key);
+ #endif
+}
+
+#elif MI_TLS_MODEL_WIN32
+// Dynamic TLS slots -- this is the default on Windows.
+#define MI_THEAP_INITASNULL 1
+
+// We try to use direct slots (64 available), but can also use the expansion slots (upto 1024 extra available)
+// See <https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/pebteb/teb/index.htm> for the offsets.
+#if MI_SIZE_SIZE==4
+#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_SIZE_SIZE)
+#else
+#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_SIZE_SIZE)
+#endif
+
+extern mi_decl_hidden _Atomic(size_t) _mi_theap_default_slot;
+extern mi_decl_hidden _Atomic(size_t) _mi_theap_cached_slot;
+extern mi_decl_hidden _Atomic(size_t) _mi_theap_default_expansion_slot;
+extern mi_decl_hidden _Atomic(size_t) _mi_theap_cached_expansion_slot;
+
+static inline mi_theap_t* _mi_theap_default(void) {
+ const size_t slot = mi_atomic_load_relaxed(&_mi_theap_default_slot);
+ mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);
+ #if !MI_WIN_DIRECT_TLS
+ if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?
+ mi_theap_t** const eslots = (mi_theap_t**)theap; // theap is actually the expansion slot entry
+ if mi_likely(eslots!=NULL) { // is it initialized? (on this thread)
+ theap = eslots[mi_atomic_load_relaxed(&_mi_theap_default_expansion_slot)];
+ }
+ }
+ #endif
+ return theap;
+}
+
+static inline mi_theap_t* _mi_theap_cached(void) {
+ const size_t slot = mi_atomic_load_relaxed(&_mi_theap_cached_slot);
+ mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);
+ #if !MI_WIN_DIRECT_TLS
+ if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?
+ mi_theap_t** const eslots = (mi_theap_t**)theap; // theap is the expansion slot entry
+ if mi_likely(eslots!=NULL) { // is it initialized? (on this thread)
+ theap = eslots[mi_atomic_load_relaxed(&_mi_theap_cached_expansion_slot)];
+ }
+ }
+ #endif
+ return theap;
+}
+
+#elif MI_TLS_MODEL_FIXED
+// Fixed TLS slot. Can be the fastest approach, but does not work if there are multiple instances of
+// mimalloc in the same process. Most OS's do not have official user reserved fixed slots so this cannot be
+// guaranteed to work in general.
+#define MI_THEAP_INITASNULL 1
+
+#if !MI_HAS_TLS_SLOT
+#error this platform cannot support MI_TLS_MODEL_FIXED without defining mi_prim_tls_slot
+#endif
+
+#if !defined(MI_TLS_MODEL_FIXED_DEFAULT)
+ #if defined(__APPLE__) && !defined(__POWERPC__) // macOS on arm64 or x64
+ // we use the last two swift framework slots which seem unused.
+ // we may want to use slot 6 and 11 instead which are only used by Windows emulation.
+ // see <https://github.com/apple/darwin-libpthread/blob/main/private/pthread/tsd_private.h#L99> for assigned slots
+ #define MI_TLS_MODEL_FIXED_DEFAULT 108
+ #define MI_TLS_MODEL_FIXED_CACHED 109
+ #elif defined(_WIN32)
+ // we use two seemingly unused fields in the Windows TEB.
+ // see <https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/pebteb/teb/index.htm>
+ #define MI_TLS_MODEL_FIXED_DEFAULT 5 // arbitrary user pointer
+ #define MI_TLS_MODEL_FIXED_CACHED 7 // environment pointer (used by OS2)
+ #else
+ #error define the TLS model fixed slots (or change the TLS model away from MI_TLS_MODEL_FIXED)
+ #endif
+#endif
+
+static inline mi_theap_t* _mi_theap_default(void) {
+ return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_DEFAULT);
+}
+
+static inline mi_theap_t* _mi_theap_cached(void) {
+ return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_CACHED);
+}
+
+#else
+#error "no TLS model is defined for this platform?"
+#endif
+
+
+// Check if a thread is initialized (without using a thread-local if using fixed slots)
+static inline bool _mi_thread_is_initialized(void) {
+ return mi_theap_is_initialized(_mi_theap_default());
+}
+
+// Get (and possible create) the theap belonging to a heap
+// We cache the last accessed theap in `_mi_theap_cached` for better performance.
+static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap) {
+ mi_theap_t* theap = _mi_theap_cached();
+ #if MI_THEAP_INITASNULL
+ if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap;
+ #else
+ if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap;
+ #endif
+ return _mi_heap_theap_get_or_init(heap);
+}
+
+static inline mi_theap_t* _mi_heap_theap_cached(mi_heap_t* heap) {
+ mi_theap_t* theap = _mi_theap_cached();
+ #if MI_THEAP_INITASNULL
+ if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap;
+ #else
+ if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap;
+ #endif
+ return NULL;
+}
+
+// Get the theap belonging to a heap without creating it if it is not yet initialized.
+static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) {
+ mi_theap_t* theap = _mi_theap_cached();
+ #if MI_THEAP_INITASNULL
+ if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap;
+ #else
+ if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap;
+ #endif
+ theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); // don't update the cache on a query
+ mi_assert_internal(theap==NULL || (!_mi_is_empty_theap(theap) && theap->heap==heap));
+ return theap;
+}
+
+// Find the associated theap or NULL if it does not exist (during shutdown)
+// Should be fast as it is called in `free.c:mi_free_try_collect`.
+static inline mi_theap_t* _mi_page_associated_theap_peek(const mi_page_t* page) {
+ mi_heap_t* const heap = mi_page_heap(page);
+ mi_theap_t* const theap = (mi_theap_t*)_mi_thread_local_get(heap->theap);
+ if (theap==NULL || theap->tld==NULL /* heap destroy */) return NULL;
+ if (theap->heap != heap) return NULL; // should never happen, but can happen for a free across subprocesses, which can happen during pthread tls storage deallocation
+ mi_assert_internal(!_mi_is_empty_theap(theap) && mi_theap_matches_thread(theap));
+ // note: for pages allocated by a detached theap, the returned theap may not be detached
+ return theap;
+}
+
+#endif // MI_PRIM_TLS_H
diff --git a/vendored/mimalloc/include/mimalloc/prim.h b/vendored/mimalloc/include/mimalloc/prim.h
index aecfc8074..82b2d9c3d 100644
--- a/vendored/mimalloc/include/mimalloc/prim.h
+++ b/vendored/mimalloc/include/mimalloc/prim.h
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -7,7 +7,8 @@ terms of the MIT license. A copy of the license can be found in the file
#pragma once
#ifndef MIMALLOC_PRIM_H
#define MIMALLOC_PRIM_H
-#include "internal.h" // mi_decl_hidden
+
+#include "types.h"
// --------------------------------------------------------------------------
// This file specifies the primitive portability API.
@@ -23,7 +24,7 @@ terms of the MIT license. A copy of the license can be found in the file
// OS memory configuration
typedef struct mi_os_mem_config_s {
size_t page_size; // default to 4KiB
- size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows)
+ size_t large_page_size; // 0 if not supported, usually 2MiB
size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB)
size_t physical_memory_in_kib; // physical memory size in KiB
size_t virtual_address_bits; // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization)
@@ -107,7 +108,9 @@ void _mi_prim_out_stderr( const char* msg );
// Get an environment variable. (only for options)
// name != NULL, result != NULL, result_size >= 64
-bool _mi_prim_getenv(const char* name, char* result, size_t result_size);
+// Return 1 for success, 0 if not found,
+// and -1 on error (for example, if `getenv` cannot be called yet during preloading).
+int _mi_prim_getenv(const char* name, char* result, size_t result_size);
// Fill a buffer with strong randomness; return `false` on error or if
@@ -126,417 +129,8 @@ void _mi_prim_thread_associate_default_theap(mi_theap_t* theap);
// Is this thread part of a thread pool?
bool _mi_prim_thread_is_in_threadpool(void);
-// Yield to other threads. Should be similar to `sleep(0)`.
+// Yield to other threads. Should be similar to `sleep(0)`.
// Is called only in rare situations and does not have to be lightning fast.
void _mi_prim_thread_yield(void);
-//-------------------------------------------------------------------
-// Access to TLS (thread local storage) slots.
-// We need fast access to both a unique thread id (in `free.c:mi_free`) and
-// to a thread-local theap pointer (in `alloc.c:mi_malloc`).
-// To achieve this we use specialized code for various platforms.
-//-------------------------------------------------------------------
-
-// On some libc + platform combinations we can directly access a thread-local storage (TLS) slot.
-// The TLS layout depends on both the OS and libc implementation so we use specific tests for each main platform.
-// If you test on another platform and it works please send a PR :-)
-// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register.
-//
-// Note: we would like to prefer `__builtin_thread_pointer()` nowadays instead of using assembly,
-// but unfortunately we can not detect support reliably (see issue #883)
-// We also use it on Apple OS as we use a TLS slot for the default theap there.
-#if (defined(_WIN32)) || \
- (defined(__GNUC__) && ( \
- (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \
- || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \
- || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \
- || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
- || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
- ))
-
-static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept {
- void* res;
- const size_t ofs = (slot*sizeof(void*));
- #if defined(_WIN32)
- #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC)
- res = (void*)__readgsqword((unsigned long)ofs); // direct load at offset from gs
- #elif _M_IX86 && !defined(_M_ARM64EC)
- res = (void*)__readfsdword((unsigned long)ofs); // direct load at offset from fs
- #else
- res = ((void**)NtCurrentTeb())[slot]; MI_UNUSED(ofs);
- #endif
- #elif defined(__i386__)
- __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS
- #elif defined(__APPLE__) && defined(__x86_64__)
- __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS
- #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
- __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI
- #elif defined(__x86_64__)
- __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS
- #elif defined(__arm__)
- void** tcb; MI_UNUSED(ofs);
- __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
- res = tcb[slot];
- #elif defined(__aarch64__)
- void** tcb; MI_UNUSED(ofs);
- #if defined(__APPLE__) // M1, issue #343
- __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
- #else
- __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
- #endif
- res = tcb[slot];
- #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781
- MI_UNUSED(ofs);
- res = pthread_getspecific(slot);
- #else
- #define MI_HAS_TLS_SLOT 0
- MI_UNUSED(ofs);
- res = NULL;
- #endif
- return res;
-}
-
-#ifndef MI_HAS_TLS_SLOT
-#define MI_HAS_TLS_SLOT 1
-#endif
-
-// setting a tls slot is only used on macOS for now
-static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
- const size_t ofs = (slot*sizeof(void*));
- #if defined(_WIN32)
- ((void**)NtCurrentTeb())[slot] = value; MI_UNUSED(ofs);
- #elif defined(__i386__)
- __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS
- #elif defined(__APPLE__) && defined(__x86_64__)
- __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS
- #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
- __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI
- #elif defined(__x86_64__)
- __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS
- #elif defined(__arm__)
- void** tcb; MI_UNUSED(ofs);
- __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
- tcb[slot] = value;
- #elif defined(__aarch64__)
- void** tcb; MI_UNUSED(ofs);
- #if defined(__APPLE__) // M1, issue #343
- __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
- #else
- __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
- #endif
- tcb[slot] = value;
- #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781
- MI_UNUSED(ofs);
- pthread_setspecific(slot, value);
- #else
- MI_UNUSED(ofs); MI_UNUSED(value);
- #endif
-}
-
-#endif
-
-
-// defined in `init.c`; do not use these directly
-extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main; // theap belonging to the main heap
-extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called?
-
-
-//-------------------------------------------------------------------
-// Get a fast unique thread id.
-//
-// Getting the thread id should be performant as it is called in the
-// fast path of `_mi_free` and we specialize for various platforms as
-// inlined definitions. Regular code should call `init.c:_mi_thread_id()`.
-// We only require _mi_prim_thread_id() to return a unique id
-// for each thread (unequal to zero).
-//-------------------------------------------------------------------
-
-
-// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id
-// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)
-// Nevertheless, it seems needed on older graviton platforms (see issue #851).
-// For now, we only enable this for specific platforms.
-#if !defined(MI_USE_BUILTIN_THREAD_POINTER) /* allow user override */
- #if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
- && !defined(__CYGWIN__) \
- && !defined(MI_LIBC_MUSL) \
- && (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */
- #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \
- || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \
- || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))
- #define MI_USE_BUILTIN_THREAD_POINTER 1
- #endif
- #endif
-#endif
-
-static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept;
-
-static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
- const mi_threadid_t tid = __mi_prim_thread_id();
- mi_assert_internal(tid > 1);
- mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear?
- return tid;
-}
-
-// Get a unique id for the current thread.
-#if defined(MI_PRIM_THREAD_ID)
-
-static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
- const mi_threadid_t tid = MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488)
- mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits
- return tid;
-}
-
-#elif defined(_WIN32)
-
-static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
- // Windows: works on Intel and ARM in both 32- and 64-bit
- return (uintptr_t)NtCurrentTeb();
-}
-
-#elif MI_USE_BUILTIN_THREAD_POINTER
-
-static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
- // Works on most Unix based platforms with recent compilers
- return (uintptr_t)__builtin_thread_pointer();
-}
-
-#elif MI_HAS_TLS_SLOT
-
-static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
- #if defined(__BIONIC__)
- // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id
- // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86
- return (uintptr_t)mi_prim_tls_slot(1);
- #else
- // in all our other targets, slot 0 is the thread id
- // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h
- // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36
- return (uintptr_t)mi_prim_tls_slot(0);
- #endif
-}
-
-#else
-
-// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).
-static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
- return (uintptr_t)&__mi_theap_main;
-}
-
-#endif
-
-
-
-/* ----------------------------------------------------------------------------------------
-Get the thread local default theap: `_mi_theap_default()` (and the cached heap `_mi_theap_cached`).
-
-This is inlined here as it is on the fast path for allocation functions.
-We have 4 models:
-
-- MI_TLS_MODEL_THREAD_LOCAL: use regular thread local (default on Linux, FreeBSD, etc)
- On most platforms (Linux, FreeBSD, NetBSD, etc), this just returns a
- thread local variable (`__mi_theap_default`). With the initial-exec TLS model this ensures
- that the storage will always be available and properly initialized (with an empty theap).
-
- On some platforms the underlying TLS implementation (or the loader) will call itself `malloc`
- on a first access to a thread local and recurse in the MI_TLS_MODEL_THREAD_LOCAL.
- A way around this is to define MI_TLS_RECURSE_GUARD which adds an extra check if the process
- is initialized before accessing the thread-local. This is a check in the fast path though
- so this should be avoided.
-
-- MI_TLS_MODEL_FIXED_SLOT: use a fixed slot in the TLS block (default on macOS)
- This reserves an unused and fixed TLS slot. This is fast and avoids the problem
- where the underlying TLS implementation (or the loader) will call itself `malloc`
- on a first access to a thread local (and recurse in the MI_TLS_MODEL_THREAD_LOCAL).
- This goes wrong though if the OS or a library uses the same fixed slot.
-
-- MI_TLS_MODEL_DYNAMIC_WIN32: use a dynamically allocated slot with TlsAlloc. (default on Windows)
- Windows has somewhat slow thread locals so by default we use TlsAlloc'd slots which
- can be more efficient. First tries to use one of the "direct" first 64 slots which
- are the fastest, but falls back to using "expansion" slots when needed (up to 1088 slots).
- (If the allocated slot happens to always be under 64 for a particular program,
- one might use cmake with `-DMI_WIN_DIRECT_TLS=ON` to skip the expansion slot test in the fast path.)
-
-- MI_TLS_MODEL_DYNAMIC_PTHREADS: use `pthread_getspecific`. (default on OpenBSD, maybe good for Android as well?)
- Use pthread local storage. Somewhat slow but can work well depending on the platform.
-
-Each model should define `MI_THEAP_INITASNULL` to signify that the initial value
-returned from `_mi_theap_default()` can be `NULL` (instead of the address of the empty heap).
-This incurs an extra check in the fast path (but can often be combined in an existing check).
-------------------------------------------------------------------------------------------- */
-
-static inline mi_theap_t* _mi_theap_default(void);
-static inline mi_theap_t* _mi_theap_cached(void);
-
-#if defined(_WIN32)
- #define MI_TLS_MODEL_DYNAMIC_WIN32 1
-#elif defined(__APPLE__) && MI_HAS_TLS_SLOT && !defined(__POWERPC__) // macOS on arm64 or x64
- // #define MI_TLS_MODEL_DYNAMIC_PTHREADS 1 // also works but a bit slower
- #define MI_TLS_MODEL_FIXED_SLOT 1
- #define MI_TLS_MODEL_FIXED_SLOT_DEFAULT 108 // seems unused. @apple: it would be great to get 2 official slots for custom allocators :-)
- #define MI_TLS_MODEL_FIXED_SLOT_CACHED 109
- // we used before __PTK_FRAMEWORK_OLDGC_KEY9 (89) but that seems used now.
- // see <https://github.com/rweichler/substrate/blob/master/include/pthread_machdep.h>
-#elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__)
- #define MI_TLS_MODEL_DYNAMIC_PTHREADS 1
- // #define MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL 1
-#else
- #define MI_TLS_MODEL_THREAD_LOCAL 1
-#endif
-
-// Declared this way to optimize register spills and branches
-mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void);
-
-static inline mi_theap_t* __mi_theap_empty(void) {
- #if __GNUC__
- __asm(""); // prevent conditional load
- return (mi_theap_t*)&_mi_theap_empty;
- #else
- return _mi_theap_empty_get();
- #endif
-}
-
-#if MI_TLS_MODEL_THREAD_LOCAL
-// Thread local with an initial value (default on Linux). Very efficient.
-
-extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default; // default theap to allocate from
-extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached; // theap from the last used heap
-
-static inline mi_theap_t* _mi_theap_default(void) {
- #if defined(MI_TLS_RECURSE_GUARD)
- if (mi_unlikely(!_mi_process_is_initialized)) return _mi_theap_empty_get();
- #endif
- return __mi_theap_default;
-}
-
-static inline mi_theap_t* _mi_theap_cached(void) {
- return __mi_theap_cached;
-}
-
-#elif MI_TLS_MODEL_FIXED_SLOT
-// Fixed TLS slot (default on macOS).
-#define MI_THEAP_INITASNULL 1
-
-static inline mi_theap_t* _mi_theap_default(void) {
- return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_DEFAULT);
-}
-
-static inline mi_theap_t* _mi_theap_cached(void) {
- return (mi_theap_t*)mi_prim_tls_slot(MI_TLS_MODEL_FIXED_SLOT_CACHED);
-}
-
-#elif MI_TLS_MODEL_DYNAMIC_WIN32
-// Dynamic TLS slot (default on Windows)
-#define MI_THEAP_INITASNULL 1
-
-// We try to use direct slots (64), but can also use the expansion slots (upto 1024 extra available)
-// See <https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/pebteb/teb/index.htm> for the offsets.
-#if MI_SIZE_SIZE==4
-#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_SIZE_SIZE)
-#else
-#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_SIZE_SIZE)
-#endif
-
-extern mi_decl_hidden size_t _mi_theap_default_slot;
-extern mi_decl_hidden size_t _mi_theap_cached_slot;
-extern mi_decl_hidden size_t _mi_theap_default_expansion_slot;
-extern mi_decl_hidden size_t _mi_theap_cached_expansion_slot;
-
-static inline mi_theap_t* _mi_theap_default(void) {
- const size_t slot = _mi_theap_default_slot;
- mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);
- #if !MI_WIN_DIRECT_TLS
- if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?
- if mi_likely(theap!=NULL) { // initialized (on this thread)?
- theap = ((mi_theap_t**)theap)[_mi_theap_default_expansion_slot];
- }
- }
- #endif
- return theap;
-}
-
-static inline mi_theap_t* _mi_theap_cached(void) {
- const size_t slot = _mi_theap_cached_slot;
- mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot);
- #if !MI_WIN_DIRECT_TLS
- if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ?
- if mi_likely(theap!=NULL) { // initialized (on this thread)?
- theap = ((mi_theap_t**)theap)[_mi_theap_cached_expansion_slot];
- }
- }
- #endif
- return theap;
-}
-
-#elif MI_TLS_MODEL_DYNAMIC_PTHREADS
-// Dynamic pthread slot on less common platforms. This is not too bad. (default on OpenBSD)
-#define MI_THEAP_INITASNULL 1
-
-extern mi_decl_hidden pthread_key_t _mi_theap_default_key;
-extern mi_decl_hidden pthread_key_t _mi_theap_cached_key;
-
-static inline mi_theap_t* _mi_theap_default(void) {
- #if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL
- // we can skip this check if using the initial key will return NULL from pthread_getspecific
- if mi_unlikely(_mi_theap_default_key==0) { return NULL; }
- #endif
- return (mi_theap_t*)pthread_getspecific(_mi_theap_default_key);
-}
-
-static inline mi_theap_t* _mi_theap_cached(void) {
- #if !MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL
- // we can skip this check if using the initial key will return NULL from pthread_getspecific
- if mi_unlikely(_mi_theap_cached_key==0) { return NULL; }
- #endif
- return (mi_theap_t*)pthread_getspecific(_mi_theap_cached_key);
-}
-
-#else
-#error "no TLS model is defined for this platform?"
-#endif
-
-
-// Check if a thread is initialized (without using a thread-local if using fixed slots)
-static inline bool _mi_thread_is_initialized(void) {
- return (mi_theap_is_initialized(_mi_theap_default()));
-}
-
-// Get (and possible create) the theap belonging to a heap
-// We cache the last accessed theap in `_mi_theap_cached` for better performance.
-static inline mi_theap_t* _mi_heap_theap(const mi_heap_t* heap) {
- mi_theap_t* theap = _mi_theap_cached();
- #if MI_THEAP_INITASNULL
- if mi_likely(theap!=NULL && _mi_theap_heap(theap)==heap) return theap;
- #else
- if mi_likely(_mi_theap_heap(theap)==heap) return theap;
- #endif
- return _mi_heap_theap_get_or_init(heap);
-}
-
-// Get the theap belonging to a heap without creating in if it is not yet initialized.
-static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) {
- mi_theap_t* theap = _mi_theap_cached();
- #if MI_THEAP_INITASNULL
- if mi_unlikely(theap==NULL || _mi_theap_heap(theap)!=heap)
- #else
- if mi_unlikely(_mi_theap_heap(theap)!=heap)
- #endif
- {
- theap = _mi_heap_theap_get_peek(heap); // don't update the cache on a query (?)
- }
- mi_assert(theap==NULL || _mi_theap_heap(theap)==heap);
- return theap;
-}
-
-// Find the associated theap or NULL if it does not exist (during shutdown)
-// Should be fast as it is called in `free.c:mi_free_try_collect`.
-static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) {
- mi_heap_t* const heap = page->heap;
- mi_theap_t* theap;
- if mi_likely(heap==NULL) { theap = __mi_theap_main; } // note: on macOS accessing the thread_local can cause allocation during thread shutdown (and reinitialize the thread)!
- else { theap = _mi_heap_theap_peek(heap); }
- mi_assert_internal(theap==NULL || _mi_thread_id()==theap->tld->thread_id);
- return theap;
-}
-
#endif // MI_PRIM_H
diff --git a/vendored/mimalloc/include/mimalloc/track.h b/vendored/mimalloc/include/mimalloc/track.h
index 1f4021297..e6a57c8ec 100644
--- a/vendored/mimalloc/include/mimalloc/track.h
+++ b/vendored/mimalloc/include/mimalloc/track.h
@@ -133,18 +133,18 @@ defined, undefined, or not accessible at all:
#endif
-#if MI_PADDING
+#if MI_PADDING_CHECK_BYTES && !MI_GUARDED
#define mi_track_malloc(p,reqsize,zero) \
- if ((p)!=NULL) { \
+ do { if ((p)!=NULL) { \
mi_assert_internal(mi_usable_size(p)==(reqsize)); \
mi_track_malloc_size(p,reqsize,reqsize,zero); \
- }
+ } } while(0)
#else
#define mi_track_malloc(p,reqsize,zero) \
- if ((p)!=NULL) { \
+ do { if ((p)!=NULL) { \
mi_assert_internal(mi_usable_size(p)>=(reqsize)); \
mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \
- }
+ } } while(0)
#endif
#endif // MI_TRACK_H
diff --git a/vendored/mimalloc/include/mimalloc/types.h b/vendored/mimalloc/include/mimalloc/types.h
index 31fdace9c..530c464a3 100644
--- a/vendored/mimalloc/include/mimalloc/types.h
+++ b/vendored/mimalloc/include/mimalloc/types.h
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -24,12 +24,13 @@ terms of the MIT license. A copy of the license can be found in the file
#include <mimalloc-stats.h>
+#include <mimalloc-profile.h>
#include <stddef.h> // ptrdiff_t
#include <stdint.h> // uintptr_t, uint16_t, etc
#include <stdbool.h> // bool
#include <limits.h> // SIZE_MAX etc.
#include <errno.h> // error codes
-#include "bits.h" // size defines (MI_INTPTR_SIZE etc), bit operations
+#include "bits.h" // size defines (MI_SIZE_SIZE etc), bit operations
#include "atomic.h" // _Atomic primitives
// Minimal alignment necessary. On most platforms 16 bytes are needed
@@ -38,6 +39,9 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t)
#endif
+#if MI_MAX_ALIGN_SIZE < MI_INTPTR_SIZE
+#error MI_MAX_ALIGN_SIZE must be at least MI_INTPTR_SIZE
+#endif
// ------------------------------------------------------
// Variants
@@ -51,15 +55,15 @@ terms of the MIT license. A copy of the license can be found in the file
// #define MI_TRACK_ASAN 1
// #define MI_TRACK_ETW 1
-// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).
-// #define MI_STAT 1
+// Define MI_STATS as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).
+// #define MI_STATS 1
// Define MI_SECURE to enable security mitigations
-// #define MI_SECURE 1 // guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption
-// #define MI_SECURE 2 // randomize relative allocation addresses (within mimalloc pages)
-// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)
-// #define MI_SECURE 4 // checks for double free (may be more expensive) (`-DMI_SECURE=ON`)
-// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`)
+// #define MI_SECURE 1 // check invalid pointer free, guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption
+// #define MI_SECURE 2 // randomize relative allocation addresses within mimalloc pages
+// #define MI_SECURE 3 // check buffer overflow, check double free, encode free lists (and detect corrupted free lists)
+// #define MI_SECURE 4 // same as level 3 for now (`-DMI_SECURE=ON`)
+// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`), and byte-precise buffer overflow checks.
#if !defined(MI_SECURE)
#define MI_SECURE 0
@@ -77,29 +81,49 @@ terms of the MIT license. A copy of the license can be found in the file
#endif
#endif
-// Statistics (0=only essential, 1=normal, 2=more fine-grained (expensive) tracking)
-#ifndef MI_STAT
+// Statistics (0=only essential, 1=detailed (fast, can be enabled always), 2=slightly more expensive tracking of precise requested bytes)
+#ifndef MI_STATS
#if (MI_DEBUG>0)
-#define MI_STAT 2
+#define MI_STATS 2
#else
-#define MI_STAT 0
+#define MI_STATS 1
+#endif
#endif
+
+// Enable profiling support (0=off, 1=fast, can be enabled always, 2=allow fine-grained sample rates(<64 KiB), a tad more expensive)
+#ifndef MI_PROFILE
+#define MI_PROFILE 1
#endif
// Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options)
-#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_PAGE_META_ALIGNED_FREE_SMALL
+#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_OPT_FREE_SMALL
#define MI_GUARDED 1
#endif
+// For profiling or guarded pages we need to sample every once in a while. (0=no sampling, 1=fast, 2=allow fine grained sample rates(<64 KiB), a tad more expensive)
+#ifndef MI_SAMPLE
+#if MI_PROFILE>1 || MI_GUARDED>1
+ #define MI_SAMPLE 2
+#elif MI_PROFILE || MI_GUARDED
+ #define MI_SAMPLE 1
+#else
+ #define MI_SAMPLE 0
+#endif
+#endif
+
// Reserve extra padding at the end of each block to be more resilient against theap block overflows.
-// The padding can detect buffer overflow on free.
-#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW))
+// The padding can detect heap-block overflow on free, and provides byte-precise `mi_usable_size`.
+#if !defined(MI_PADDING)
+#if (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW))
#define MI_PADDING 1
+#else
+#define MI_PADDING 0
+#endif
#endif
-// Check padding bytes; allows byte-precise buffer overflow detection
-#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1)
-#define MI_PADDING_CHECK 1
+// Check for byte-precise buffer overflow?
+#if !defined(MI_PADDING_CHECK_BYTES) && MI_PADDING && (MI_SECURE>=5 || MI_DEBUG>=1)
+#define MI_PADDING_CHECK_BYTES 1
#endif
@@ -109,6 +133,17 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_ENCODE_FREELIST 1
#endif
+// Deprecated (double-free is now checked with padding)
+// #if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0))
+// #define MI_CHECK_DOUBLE_FREE 1
+// #endif
+
+#if MI_SECURE>=4 || MI_PADDING || (MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP)
+#define MI_PAGE_KEY_COUNT 2
+#else
+#define MI_PAGE_KEY_COUNT 1
+#endif
+
// Enable large pages for objects between 64KiB and 512KiB.
// This should perhaps be disabled by default as for many workloads the block sizes above 64 KiB
// are quite random which can lead to too many partially used large pages (but see issue #1104).
@@ -116,10 +151,9 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_ENABLE_LARGE_PAGES 1
#endif
-// Place page meta info at the start of the page area or keep it separate?
-// Separate keeps the page info at the arena start (default) which is more secure
-// and reduces wasted space due to alignment and block sizes.
-// (but also reserves more memory up front (about 2MiB per GiB))
+// Place page meta info at the start of the page area or keep it separate?
+// Separate keeps the page info at the arena start (default) which is more secure
+// and reduces wasted space due to alignment and block sizes.
#if !defined(MI_PAGE_META_IS_SEPARATED)
#if MI_PAGE_MAP_FLAT
#define MI_PAGE_META_IS_SEPARATED 0
@@ -128,19 +162,38 @@ terms of the MIT license. A copy of the license can be found in the file
#endif
#endif
+// We can choose to page meta info aligned at the start of every MI_PAGE_META_ALIGNED_CHUNKS
+// This can be used to have a faster `mi_free(_small)` as we can avoid a page_map lookup.
+// This only works if valid pointers are passed to `mi_free` though. However, checked
+// free `mi_cfree` can still use the page map to validate pointers.
+#if !MI_FREE_IS_CHECKED && !MI_FREE_USE_PAGEMAP
+#if MI_PAGE_META_IS_SEPARATED
+#define MI_PAGE_META_IS_ALIGNED 1
+#define MI_PAGE_META_ALIGNED_CHUNKS MI_SIZE_SIZE
+#else
+#warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)"
+#endif
+#endif
+
// We can choose to only put page info of small pages at the start of the page area.
// This can be used to have a slightly faster `mi_free_small` function for specialized
// cases (like language runtime systems).
-#if !defined(MI_PAGE_META_ALIGNED_FREE_SMALL)
-#define MI_PAGE_META_ALIGNED_FREE_SMALL 0
+#if !defined(MI_PAGE_META_SMALL_IS_ALIGNED)
+#if defined(MI_OPT_FREE_SMALL) && MI_OPT_FREE_SMALL==0
+#define MI_PAGE_META_SMALL_IS_ALIGNED 0
+#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE && !MI_GUARDED // guarded can put small allocations into big blocks in a medium page (which would make `mi_free_small` fail)
+#define MI_PAGE_META_SMALL_IS_ALIGNED 1
+#else
+#define MI_PAGE_META_SMALL_IS_ALIGNED 0
+#endif
#endif
// Configuration checks
#if !MI_PAGE_META_IS_SEPARATED && MI_SECURE
#error "secure mode should use separated page infos"
#endif
-#if MI_PAGE_META_ALIGNED_FREE_SMALL && MI_SECURE
-#error "secure mode cannot use MI_PAGE_META_ALIGNED_FREE_SMALL"
+#if MI_PAGE_META_SMALL_IS_ALIGNED && MI_SECURE
+#error "secure mode cannot use MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED)"
#endif
#if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT
#error "cannot have a flat page map with separated page infos"
@@ -149,7 +202,6 @@ terms of the MIT license. A copy of the license can be found in the file
#warning "mimalloc assertions enabled in a release build"
#endif
-
// --------------------------------------------------------------
// Sizes of internal data-structures
// (comments specify sizes on 64-bit, usually 32-bit is halved)
@@ -159,14 +211,14 @@ terms of the MIT license. A copy of the license can be found in the file
#ifndef MI_ARENA_SLICE_SHIFT
#ifdef MI_SMALL_PAGE_SHIFT // backward compatibility
#define MI_ARENA_SLICE_SHIFT MI_SMALL_PAGE_SHIFT
- #elif MI_SECURE>=5 && __APPLE__ && MI_ARCH_ARM64
+ #elif MI_SECURE>=5 && ((__APPLE__ && MI_ARCH_ARM64) || (defined(PAGE_SIZE) && PAGE_SIZE >= 16*MI_KiB))
#define MI_ARENA_SLICE_SHIFT (17) // 128 KiB to not waste too much due to 16 KiB guard pages
#else
#define MI_ARENA_SLICE_SHIFT (13 + MI_SIZE_SHIFT) // 64 KiB (32 KiB on 32-bit)
#endif
#endif
-#if MI_ARENA_SLICE_SHIFT < 12
-#error Arena slices should be at least 4KiB
+#if MI_ARENA_SLICE_SHIFT < 13
+#error Arena slices should be at least 8KiB
#endif
#ifndef MI_BCHUNK_BITS_SHIFT
@@ -182,6 +234,7 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_BCHUNK_BITS (1 << MI_BCHUNK_BITS_SHIFT) // sub-bitmaps in arena's are "bchunks" of 512 bits
#define MI_ARENA_SLICE_SIZE (MI_ZU(1) << MI_ARENA_SLICE_SHIFT) // arena's allocate in slices of 64 KiB
#define MI_ARENA_SLICE_ALIGN (MI_ARENA_SLICE_SIZE)
+#define MI_ARENA_CHUNK_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE)
#define MI_ARENA_MIN_OBJ_SLICES (1)
#define MI_ARENA_MAX_CHUNK_OBJ_SLICES (MI_BCHUNK_BITS) // 32 MiB (or 8 MiB on 32-bit)
@@ -206,11 +259,18 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_BIN_COUNT (MI_BIN_FULL+1)
// We never allocate more than PTRDIFF_MAX (see also <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
-#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX
+#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX
-// Minimal commit for a page on-demand commit (should be >= OS page size)
-#define MI_PAGE_MIN_COMMIT_SIZE MI_ARENA_SLICE_SIZE
+// Minimal commit for a page on-demand commit
+#define MI_PAGE_MIN_COMMIT_SIZE (16*MI_KiB) /* MI_ARENA_SLICE_SIZE */
+#if MI_PAGE_META_IS_ALIGNED
+#define MI_PAGE_META_ALIGNED_COUNT (MI_PAGE_META_ALIGNED_CHUNKS * MI_BCHUNK_BITS)
+#define MI_PAGE_META_ALIGNMENT (MI_PAGE_META_ALIGNED_COUNT * MI_ARENA_SLICE_SIZE) // 256 MiB (32 MiB on 32-bit)
+#define MI_ARENA_ALIGNMENT MI_PAGE_META_ALIGNMENT
+#else
+#define MI_ARENA_ALIGNMENT MI_ARENA_SLICE_ALIGN
+#endif
// ------------------------------------------------------
// Arena's are large reserved areas of memory allocated from
@@ -248,15 +308,14 @@ typedef struct mi_subproc_s mi_subproc_t;
// Memory can reside in arena's, direct OS allocated, meta-data pages, or statically allocated.
// The memid keeps track of this.
typedef enum mi_memkind_e {
- MI_MEM_NONE, // not allocated
+ MI_MEM_NONE, // not allocated (or static)
MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example)
MI_MEM_STATIC, // allocated in a static area and should not be freed (the initial main theap data for example (`init.c`))
- MI_MEM_META, // allocated with the meta data allocator (`arena-meta.c`)
MI_MEM_OS, // allocated from the OS
MI_MEM_OS_HUGE, // allocated as huge OS pages (usually 1GiB, pinned to physical memory)
MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`)
MI_MEM_ARENA, // allocated from an arena (the usual case) (`arena.c`)
- MI_MEM_HEAP_MAIN // allocated in the main heap (for theaps)
+ MI_MEM_MALLOC // allocated with mi_malloc
} mi_memkind_t;
static inline bool mi_memkind_is_os(mi_memkind_t memkind) {
@@ -267,6 +326,7 @@ static inline bool mi_memkind_needs_no_free(mi_memkind_t memkind) {
return (memkind <= MI_MEM_STATIC);
}
+typedef struct mi_meta_page_s mi_meta_page_t;
typedef struct mi_memid_os_info {
void* base; // actual base address of the block (used for offset aligned allocations)
@@ -280,17 +340,16 @@ typedef struct mi_memid_arena_info {
uint32_t slice_count; // allocated slices
} mi_memid_arena_info_t;
-typedef struct mi_memid_meta_info {
- void* meta_page; // meta-page that contains the block
- uint32_t block_index; // block index in the meta-data page
- uint32_t block_count; // allocated blocks
-} mi_memid_meta_info_t;
+typedef struct mi_memid_malloc_info {
+ void* base; // returned pointer
+ size_t size; // allocated size
+} mi_memid_malloc_info_t;
typedef struct mi_memid_s {
union {
- mi_memid_os_info_t os; // only used for MI_MEM_OS
- mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA
- mi_memid_meta_info_t meta; // only used for MI_MEM_META
+ mi_memid_os_info_t os; // only used for MI_MEM_OS(_HUGE/_REMAP)
+ mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA
+ mi_memid_malloc_info_t malloc; // only used for MI_MEM_MALLOC
} mem;
mi_memkind_t memkind;
bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large (2Mib) or huge (1GiB) OS pages)
@@ -298,6 +357,8 @@ typedef struct mi_memid_s {
bool initially_zero; // `true` if the memory was originally zero initialized
} mi_memid_t;
+#define MI_MEMID_INIT(kind) {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ }
+#define MI_MEMID_STATIC MI_MEMID_INIT(MI_MEM_STATIC)
static inline bool mi_memid_is_os(mi_memid_t memid) {
return mi_memkind_is_os(memid.memkind);
@@ -335,6 +396,7 @@ typedef struct mi_block_s {
#define MI_PAGE_IN_FULL_QUEUE MI_ZU(0x01)
#define MI_PAGE_HAS_INTERIOR_POINTERS MI_ZU(0x02)
#define MI_PAGE_FLAG_MASK MI_ZU(0x03)
+#define MI_PAGE_FLAG_BITS (2)
typedef size_t mi_page_flags_t;
// There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue),
@@ -342,7 +404,8 @@ typedef size_t mi_page_flags_t;
// in an arena (in `mi_heap_t.arena_pages.pages_abandoned`) so these can be quickly found for reuse.
// Abandoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked)
#define MI_THREADID_ABANDONED MI_ZU(0)
-#define MI_THREADID_ABANDONED_MAPPED (MI_PAGE_FLAG_MASK + 1)
+#define MI_THREADID_ABANDONED_MAPPED (MI_ZU(1) << MI_PAGE_FLAG_BITS)
+#define MI_THREADID_DETACHED (MI_ZU(2) << MI_PAGE_FLAG_BITS)
// Thread free list.
// Points to a list of blocks that are freed by other threads.
@@ -351,6 +414,29 @@ typedef size_t mi_page_flags_t;
// This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`.
typedef uintptr_t mi_thread_free_t;
+// We store the currently used block count together with the total malloc call count as 16-bit numbers.
+// This is done for better codegen `mi_malloc/mi_free` (where we can increment both at once as `used_alloc += 0x10001` for example).
+// The `last_used` is the `used` count since the statistics are last updated; on 32-bit platforms this
+// is a separate field in `mi_page_t` but on 64-bit we use the upper 32-bits to store it.
+// We need the `alloc_count` and `last_used` to efficiently calculate allocation and free statistics even
+// in a release build; this way we can update the stats in the slow path (`_mi_page_update_stats`).
+typedef union mi_used_s {
+ size_t used_alloc; // used + alloc_count
+ // the following struct is unused but nice for debugging
+ struct {
+ uint16_t used_count;
+ uint16_t alloc_count;
+ #if MI_SIZE_SIZE >= 8
+ uint16_t last_used;
+ uint16_t last_alloc;
+ #endif
+ } debug_le;
+} mi_used_t;
+
+static inline size_t mi_xused_used_count(mi_used_t xused) { return (xused.used_alloc & 0xFFFF); }
+static inline size_t mi_xused_alloc_count(mi_used_t xused) { return ((xused.used_alloc>>16) & 0xFFFF); }
+static inline mi_used_t mi_xused_used_reset(mi_used_t xused) { xused.used_alloc = xused.used_alloc & ~0xFFFF; return xused; }
+
// A page contains blocks of one specific size (`block_size`).
// Each page has three list of free blocks:
// `free` for blocks that can be allocated,
@@ -376,44 +462,48 @@ typedef uintptr_t mi_thread_free_t;
// that case the `xthreadid` is 0 or 4 (4 is for abandoned pages that
// are in the `pages_abandoned` lists of an arena, these are called "mapped" abandoned pages).
// - page flags are in the bottom 3 bits of `xthread_id` for the fast path in `mi_free`.
-// - The layout is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc`
-// - Using `uint16_t` does not seem to slow things down
+// - The layout below is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc`
-typedef struct mi_page_s {
+typedef struct mi_page_s {
+ #if (MI_PAGE_META_IS_ALIGNED)
+ _Atomic(struct mi_page_s*) self; // points to the actual page info (for pages that span multiple slices)
+ #endif
_Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`)
-
mi_block_t* free; // list of available free blocks (`malloc` allocates from this list)
- uint16_t used; // number of blocks in use (including blocks in `thread_free`)
+ mi_used_t xused; // number of blocks in use (including blocks in `thread_free`) (and the allocated count for statistics)
+ #if MI_SIZE_SIZE < 8
+ uint16_t xlast_used; // for statistics; on 64-bit platforms it is in bits 32..47 of xused.
+ uint16_t xlast_alloc; // for sampling; on 64-bit platforms it is in bits 48..63 of xused.
+ #endif
+ mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)
+
+ size_t block_size; // const: size available in each block (always `>0`)
+ size_t page_offset; // const: relative offset from the page to the start of the blocks
uint16_t capacity; // number of blocks committed
uint16_t reserved; // number of blocks reserved in memory
+ uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already)
uint8_t retire_expire; // expiration count for retired blocks
bool free_is_zero; // `true` if the blocks in the free list are zero initialized
-
- mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)
+
+ // next cache line
_Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`)
-
- size_t block_size; // const: size available in each block (always `>0`)
- uint8_t* page_start; // const: start of the blocks
-
- #if (MI_ENCODE_FREELIST || MI_PADDING)
- uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary
- #endif
-
mi_theap_t* theap; // the theap owning this page (may not be valid or NULL for abandoned pages)
mi_heap_t* heap; // const: the heap owning this page
struct mi_page_s* next; // next page owned by the theap with the same `block_size`
struct mi_page_s* prev; // previous page owned by the theap with the same `block_size`
- size_t slice_committed; // committed size relative to the first arena slice of the page data (or 0 if the page is fully committed already)
mi_memid_t memid; // const: provenance of the page memory
+
+ #if (MI_ENCODE_FREELIST || MI_PADDING)
+ uintptr_t keys[MI_PAGE_KEY_COUNT]; // const: one or two random keys to encode the free lists (see `_mi_block_next`) or padding canary
+ #endif
} mi_page_t;
-
// ------------------------------------------------------
// Object sizes
// ------------------------------------------------------
-#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // pages must be aligned on this for the page map.
+#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // page area's must be aligned on this for the page map.
#define MI_PAGE_MIN_START_BLOCK_ALIGN MI_MAX_ALIGN_SIZE // minimal block alignment for the first block in a page (16b)
#define MI_PAGE_MAX_START_BLOCK_ALIGN2 (4*MI_KiB) // maximal block alignment for "power of 2"-sized blocks (such that we guarantee natural alignment)
#define MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4*MI_KiB) // also aligns any multiple of this size to avoid TLB misses.
@@ -423,7 +513,7 @@ typedef struct mi_page_s {
#define MI_SMALL_MAX_OBJ_SIZE ((MI_SMALL_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // = 10 KiB
#if MI_ENABLE_LARGE_PAGES
#define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6) // ~ 84 KiB
-#define MI_LARGE_MAX_OBJ_SIZE (MI_LARGE_PAGE_SIZE/8) // <= 512 KiB // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`
+#define MI_LARGE_MAX_OBJ_SIZE (MI_LARGE_PAGE_SIZE/8) // <= 512 KiB. note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`
#else
#define MI_MEDIUM_MAX_OBJ_SIZE (MI_MEDIUM_PAGE_SIZE/8) // <= 64 KiB
#define MI_LARGE_MAX_OBJ_SIZE MI_MEDIUM_MAX_OBJ_SIZE // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin`
@@ -434,6 +524,23 @@ typedef struct mi_page_s {
#error "mimalloc internal: define more bins"
#endif
+// static invariant: MI_MAX_SINGLETON_BIN >= _mi_bin(MI_LARGE_MAX_OBJ_SIZE) (See init.c for the size bins)
+#if (MI_LARGE_MAX_OBJ_WSIZE <= 8192) // 64 KiB
+#define MI_MAX_SINGLETON_BIN (48)
+#elif (MI_LARGE_MAX_OBJ_WSIZE <= 32768) // 256KiB
+#define MI_MAX_SINGLETON_BIN (56)
+#elif (MI_LARGE_MAX_OBJ_WSIZE <= 65536) // 512KiB
+#define MI_MAX_SINGLETON_BIN (60)
+#else
+#define MI_MAX_SINGLETON_BIN MI_BIN_HUGE
+#endif
+
+// The small object max size must be larger than twice the small size max minus one,
+// such that any aligned allocation with `alignment <= size <= MI_SMALL_SIZE_MAX` will
+// be allocated in a small page (so `mi_free_csize` can delegate correctly to `mi_free_small`.
+#if MI_SMALL_MAX_OBJ_SIZE <= (2*(MI_SMALL_WSIZE_MAX * MI_SIZE_SIZE)-1)
+#error "mimalloc internal: the small object max size is too small"
+#endif
// ------------------------------------------------------
// Page kinds
@@ -488,10 +595,10 @@ typedef struct mi_random_cxt_s {
#if MI_PADDING
typedef struct mi_padding_s {
uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)
- uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes)
+ uint32_t delta; // padding bytes before the block. (mi_full_usable_size(p) - delta == exact allocated bytes)
} mi_padding_t;
#define MI_PADDING_SIZE (sizeof(mi_padding_t))
-#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE)
+#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_SIZE_SIZE - 1) / MI_SIZE_SIZE)
#else
#define MI_PADDING_SIZE 0
#define MI_PADDING_WSIZE 0
@@ -502,11 +609,34 @@ typedef struct mi_padding_s {
// A thread-local heap ("theap") owns a set of thread-local pages.
struct mi_theap_s {
+ // put in front for fast small allocations
+ mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
+
+ // less frequently accessed fields
mi_tld_t* tld; // thread-local data
_Atomic(mi_heap_t*) heap; // the heap this theap belongs to.
- _Atomic(size_t) refcount; // reference count
+ _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction)
+ _Atomic(size_t) refcount; // reference count (needed for safe heap destroy)
+
+ // config
+ long page_full_retain; // how many full pages can be retained per queue (before abandoning them)
+ bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages
+ bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint
+ bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED`
+
+ // sampling
+ size_t sample_countdown; // sample countdown in requested bytes (don't change the field order; see `internal.h:_mi_theap_get_free_small_page`)
+ size_t sample_rate; // current sampling rate in requested bytes (or 0 to disable) (for profiling and guarded mode)
+ uint64_t sample_requested; // total allocated/requested bytes since the last sample
+ size_t profile_sample_rate; // sampling rate in requested bytes for profiling
+ size_t profile_sample_countdown; // countdown in requested bytes for profiling
+ size_t guarded_sample_rate; // sampling rate in requested bytes for guarded objects
+ size_t guarded_sample_countdown; // countdown in requested bytes for guarded objects
+ size_t guarded_size_min; // minimal size for guarded objects
+ size_t guarded_size_max; // maximal size for guarded objects
+
+ // stats
unsigned long long heartbeat; // monotonic heartbeat count
- uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`)
mi_random_ctx_t random; // random number context used for secure allocation
size_t page_count; // total number of pages in the `pages` queues.
size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues)
@@ -515,21 +645,13 @@ struct mi_theap_s {
long generic_count; // how often is `_mi_malloc_generic` called?
long generic_collect_count; // how often is `_mi_malloc_generic` called without collecting?
+ // theaps belong to heaps and threads
mi_theap_t* tnext; // list of theaps in this thread
- mi_theap_t* tprev;
+ mi_theap_t* tprev;
mi_theap_t* hnext; // list of theaps of the owning `heap`
mi_theap_t* hprev;
-
- long page_full_retain; // how many full pages can be retained per queue (before abandoning them)
- bool allow_page_reclaim; // `true` if this theap should not reclaim abandoned pages
- bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint
- #if MI_GUARDED
- size_t guarded_size_min; // minimal size for guarded objects
- size_t guarded_size_max; // maximal size for guarded objects
- size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages)
- size_t guarded_sample_count; // current sample count (counting down to 0)
- #endif
- mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
+
+ // page queues
mi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or "bin")
mi_memid_t memid; // provenance of the theap struct itself (meta or os)
mi_stats_t stats; // thread-local statistics
@@ -566,13 +688,15 @@ typedef struct mi_heap_s {
mi_theap_t* theaps; // list of all thread-local theaps belonging to this heap (using the `hnext`/`hprev` fields)
mi_lock_t theaps_lock; // lock for the theaps list operations
+ _Atomic(mi_profiler_t*) profiler;
+
_Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // total count of abandoned pages in this heap
mi_page_t* os_abandoned_pages; // list of pages that are OS allocated and not in an arena
mi_lock_t os_abandoned_pages_lock; // lock for the os abandoned pages list (this lock protects list operations)
_Atomic(mi_arena_pages_t*) arena_pages[MI_MAX_ARENAS]; // track owned and abandoned pages in the arenas (entries can be NULL)
mi_lock_t arena_pages_lock; // lock to update the arena_pages array
-
+ mi_memid_t memid; // provenance of the heap memory
mi_stats_t stats; // statistics for this heap; periodically updated by merging from each theap
} mi_heap_t;
@@ -590,6 +714,7 @@ struct mi_subproc_s {
size_t subproc_seq; // unique id for sub-processes
mi_subproc_t* next; // list of all sub-processes
mi_subproc_t* prev;
+ _Atomic(mi_meta_page_t*) meta_pages; // meta data pages
_Atomic(size_t) arena_count; // current count of arena's
_Atomic(mi_arena_t*) arenas[MI_MAX_ARENAS]; // arena's of this sub-process
@@ -597,16 +722,22 @@ struct mi_subproc_s {
mi_decl_align(8) // needed on some 32-bit platforms
_Atomic(int64_t) purge_expire; // expiration is set if any arenas can be purged
- _Atomic(mi_heap_t*) heap_main; // main heap for this sub process
+ _Atomic(mi_heap_t*) heap_main; // main heap for this sub process
mi_heap_t* heaps; // heaps belonging to this sub-process
mi_lock_t heaps_lock;
+ mi_theap_t* theap_meta; // detached theap for allocating meta-data
+ mi_lock_t theap_meta_lock; // all allocations in theap_meta need a lock
+
_Atomic(size_t) thread_count; // current threads associated with this sub-process
_Atomic(size_t) thread_total_count; // total created threads associated with this sub-process
_Atomic(size_t) heap_count; // current heaps in this sub-process (== |heaps|)
_Atomic(size_t) heap_total_count; // total created heaps in this sub-process
+ _Atomic(mi_profiler_t*) profiler;
+
mi_memid_t memid; // provenance of this memory block (meta or static)
+ mi_subproc_t* parent; // subproc in which this one was allocated
mi_decl_align(8) // needed on some 32-bit platforms
mi_stats_t stats; // subprocess statistics; updated for arena/OS stats like committed,
// and otherwise merged with heap stats when those are deleted
@@ -646,18 +777,18 @@ struct mi_tld_s {
to reserve large arenas upfront and be able to reuse the memory more effectively.
-----------------------------------------------------------------------------*/
-#define MI_ARENA_BIN_COUNT (MI_BIN_COUNT)
+#define MI_ARENA_BIN_COUNT (MI_MAX_SINGLETON_BIN+1)
#define MI_ARENA_MIN_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) // 32 MiB (or 8 MiB on 32-bit)
-#define MI_ARENA_MAX_SIZE (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE)
+#define MI_ARENA_MAX_SIZE (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE) // 16 GiB
typedef struct mi_bitmap_s mi_bitmap_t; // atomic bitmap (defined in `src/bitmap.h`)
typedef struct mi_bbitmap_s mi_bbitmap_t; // atomic binned bitmap (defined in `src/bitmap.h`)
-typedef struct mi_arena_pages_s {
+struct mi_arena_pages_s {
mi_bitmap_t* pages; // all registered pages (abandoned and owned)
mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT]; // abandoned pages per size bin (a set bit means the start of the page)
// followed by the bitmaps (whose siz`es depend on the arena size)
-} mi_arena_pages_t;
+};
// A memory arena
@@ -665,6 +796,7 @@ typedef struct mi_arena_s {
mi_memid_t memid; // provenance of the memory area
mi_subproc_t* subproc; // subprocess this arena belongs to (`this 'element-of' this->subproc->arenas`)
size_t arena_idx; // index in the arenas array
+ void* start; // actual start of the arena area (the arena_s info may come later due to guard pages etc.)
size_t slice_count; // total size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`)
size_t info_slices; // initial slices reserved for the arena bitmaps
@@ -713,6 +845,10 @@ typedef struct mi_arena_s {
#ifndef EOVERFLOW // count*size overflow
#define EOVERFLOW (75)
#endif
+#ifndef ENOENT // environment variable not found
+#define ENOENT (2)
+#endif
+
/* -----------------------------------------------------------
Debug constants
diff --git a/vendored/mimalloc/src/alloc-aligned.c b/vendored/mimalloc/src/alloc-aligned.c
index 97dc6045b..51f29732c 100644
--- a/vendored/mimalloc/src/alloc-aligned.c
+++ b/vendored/mimalloc/src/alloc-aligned.c
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc.h"
#include "mimalloc/internal.h"
-#include "mimalloc/prim.h" // _mi_theap_default
+#include "mimalloc/prim-tls.h" // _mi_theap_default
#include <string.h> // memset
@@ -18,7 +18,7 @@ terms of the MIT license. A copy of the license can be found in the file
static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) {
// certain blocks are always allocated at a certain natural alignment.
// (see also `arena.c:mi_arenas_page_alloc_fresh`).
- mi_assert_internal(_mi_is_power_of_two(alignment) && (alignment > 0));
+ mi_assert_internal(mi_alignment_is_valid(alignment));
if (alignment > size) return false;
const size_t bsize = mi_good_size(size);
const bool ok = (bsize <= MI_PAGE_MAX_START_BLOCK_ALIGN2 && _mi_is_power_of_two(bsize)) || // power-of-two under N
@@ -27,8 +27,8 @@ static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) {
return ok;
}
-#if MI_GUARDED
-static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero) mi_attr_noexcept {
+#if 0
+static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero, mi_page_t** ppage) mi_attr_noexcept {
// use over allocation for guarded blocksl
#if MI_THEAP_INITASNULL
if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); }
@@ -39,7 +39,7 @@ static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap,
return NULL;
}
const size_t oversize = size + alignment - 1;
- void* const base = _mi_theap_malloc_guarded(theap, oversize, zero);
+ void* const base = _mi_theap_malloc_guarded(theap, oversize, zero, ppage);
if (base==NULL) return NULL;
void* const p = _mi_align_up_ptr(base, alignment);
mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size);
@@ -47,32 +47,18 @@ static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap,
mi_assert_internal(_mi_is_aligned(p, alignment));
return p;
}
-
-static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) {
- #if MI_THEAP_INITASNULL
- if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); }
- #endif
- const size_t rate = theap->guarded_sample_rate;
- // only write if `rate!=0` so we don't write to the constant `_mi_theap_empty`
- if (rate != 0) { theap->guarded_sample_rate = 0; }
- void* p = _mi_theap_malloc_zero(theap, size, zero, usable);
- if (rate != 0) { theap->guarded_sample_rate = rate; }
- return p;
-}
-#else
-static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) {
- return _mi_theap_malloc_zero(theap, size, zero, usable);
-}
#endif
+
// Fallback aligned allocation that over-allocates -- split out for better codegen
-static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept
+static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept
{
mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE));
- mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));
+ mi_assert_internal(mi_alignment_is_valid(alignment));
void* p;
size_t oversize;
+ mi_page_t* page;
if mi_unlikely(alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {
// use OS allocation for large alignments and allocate inside a singleton page (not in an arena)
// This can support alignments >= MI_PAGE_ALIGN by ensuring the object can be aligned
@@ -84,7 +70,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap
}
oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size);
// note: no guarded as alignment > 0
- p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, usable); // the page block size should be large enough to align in the single huge page block
+ p = _mi_theap_malloc_zero(theap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block
if (p == NULL) return NULL;
}
else {
@@ -92,9 +78,11 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap
mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) && alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN);
mi_assert_internal(size < SIZE_MAX - alignment); // `oversize` cannot overflow
oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and alignment 64k, we would allocate a 64k block and pointing just beyond that.
- p = mi_theap_malloc_zero_no_guarded(theap, oversize, zero, usable);
+ p = _mi_theap_malloc_zero(theap, oversize, zero, 0, &page);
if (p == NULL) return NULL;
}
+ mi_assert_internal(page == _mi_ptr_page(p));
+ if (ppage!=NULL) { *ppage = page; }
// .. and align within the allocation
const uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)`
@@ -107,17 +95,20 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap
// and we no longer own it. We should be careful to only read constant fields in the page,
// or use safe atomic access as in `mi_page_set_has_interior_pointers`.
// (we can access the page though since the just allocated pointer keeps it alive)
- mi_page_t* page = _mi_ptr_page(p);
if (aligned_p != p) {
mi_page_set_has_interior_pointers(page, true);
- #if MI_GUARDED
+
+ #if MI_GUARDED || MI_PROFILE
// set tag to aligned so mi_usable_size works with guard pages
+ // note: we could have gotten a profiled or guarded block but that is still ok
+ // and in such case the MI_BLOCK_TAG_ALIGNED is just not at the start of the real heap block
if (adjust >= sizeof(mi_block_t)) {
mi_block_t* const block = (mi_block_t*)p;
block->next = MI_BLOCK_TAG_ALIGNED;
}
#endif
- _mi_padding_shrink(page, (mi_block_t*)p, adjust + size);
+ mi_block_t* const base = _mi_page_ptr_unalign(page, p);
+ _mi_padding_shrink(page, base, adjust + size);
}
// todo: expand padding if overallocated ?
@@ -127,22 +118,13 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap
mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust);
#if MI_DEBUG > 1
mi_page_t* const apage = _mi_ptr_page(aligned_p);
- void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p);
- mi_assert_internal(p == unalign_p);
+ mi_block_t* unalign_p = _mi_page_ptr_unalign(apage, aligned_p);
+ mi_assert_internal(p == (void*)unalign_p || mi_block_ptr_is_sampled(unalign_p, aligned_p));
#endif
- // now zero the block if needed
- //if (alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {
- // // for the tracker, on huge aligned allocations only from the start of the large block is defined
- // mi_track_mem_undefined(aligned_p, size);
- // if (zero) {
- // _mi_memzero_aligned(aligned_p, mi_usable_size(aligned_p));
- // }
- //}
-
if (p != aligned_p) {
mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p));
- #if MI_GUARDED
+ #if MI_GUARDED || MI_PROFILE
mi_track_mem_defined(p, sizeof(mi_block_t));
#endif
}
@@ -150,85 +132,97 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap
}
// Generic primitive aligned allocation -- split out for better codegen
-static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept
+static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept
{
- mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));
+ mi_assert_internal(mi_alignment_is_valid(alignment));
// we don't allocate more than MI_MAX_ALLOC_SIZE (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) {
- _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);
+ _mi_error_message(EINVAL, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);
return NULL;
}
// use regular allocation if it is guaranteed to fit the alignment constraints.
// this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist
// a page with the right block size, and if we always use the over-alloc fallback that would never happen.
- if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) {
- void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, usable);
- mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);
- const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0;
- if mi_likely(is_aligned_or_null) {
- return p;
- }
- else {
- // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct..
- mi_assert(false);
- mi_free(p);
+ #if MI_THEAP_INITASNULL
+ if mi_likely(theap!=NULL)
+ #endif
+ {
+ #if MI_SAMPLE // only try if we would not take a sample
+ if mi_likely(!mi_theap_should_sample(theap,size))
+ #endif
+ {
+ if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment))
+ {
+ mi_page_t* page = NULL;
+ void* p = _mi_theap_malloc_zero(theap, size, zero, 0,&page);
+ if (ppage!=NULL) { *ppage = page; }
+ const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0;
+ if mi_likely(is_aligned_or_null) {
+ return p;
+ }
+ else {
+ // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct..
+ // (but it can happen if this allocation just so happens to be sampled but we checked for that)
+ mi_assert_internal(false); // p==NULL || mi_block_ptr_is_profiled_or_guarded(_mi_page_ptr_unalign(page,p),p));
+ mi_free(p);
+ }
+ }
}
}
// fall back to over-allocation
- return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,usable);
+ return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,ppage);
}
+static mi_decl_cold mi_decl_noinline void* mi_error_bad_alignment(size_t size, size_t alignment, size_t offset) {
+ _mi_error_message(EINVAL, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset);
+ return NULL;
+}
+
// Primitive aligned allocation
-static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept
+static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept
{
// note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size.
- if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)
- #if MI_DEBUG > 0
- _mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\n", size, alignment);
- #endif
- return NULL;
- }
-
- #if MI_GUARDED
- #if MI_THEAP_INITASNULL
- if mi_likely(theap!=NULL)
- #endif
- if (offset==0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN && mi_theap_malloc_use_guarded(theap,size)) {
- return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero);
+ if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two and multiple of void* (see <https://en.cppreference.com/w/c/memory/aligned_alloc#Notes>)
+ return mi_error_bad_alignment(size, alignment, offset);
}
- #endif
// try first if there happens to be a small block available with just the right alignment
// since most small power-of-2 blocks (under MI_PAGE_MAX_BLOCK_START_ALIGN2) are already
// naturally aligned this can be often the case.
- #if MI_THEAP_INITASNULL
- if mi_likely(theap!=NULL)
- #endif
+ if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size)
{
- if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) {
- const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`
- const size_t padsize = size + MI_PADDING_SIZE;
- mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize);
- if mi_likely(page->free != NULL) {
- const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0;
- if mi_likely(is_aligned)
- {
- if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }
- void* p = _mi_page_malloc_zero(theap, page, padsize, zero);
- mi_assert_internal(p != NULL);
- mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);
- mi_track_malloc(p, size, zero);
- return p;
+ #if MI_THEAP_INITASNULL
+ if mi_likely(theap!=NULL)
+ #endif
+ {
+ #if MI_SAMPLE // check if we shouldn't take a sample
+ if mi_likely(!mi_theap_should_sample(theap,size))
+ #endif
+ {
+ const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`
+ const size_t padsize = size + MI_PADDING_SIZE;
+ mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize, false);
+ if mi_likely(page->free != NULL) {
+ const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0;
+ if mi_likely(is_aligned)
+ {
+ if (ppage!=NULL) { *ppage = page; }
+ void* p = _mi_page_malloc_zero(theap, page, padsize, zero); // theap!=NULL
+ mi_assert_internal(p != NULL);
+ mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);
+ mi_track_malloc(p, size, zero);
+ return p;
+ }
}
}
}
}
// fallback to generic aligned allocation
- return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, usable);
+ return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, ppage);
}
@@ -248,7 +242,7 @@ static mi_decl_restrict void* mi_theap_zalloc_aligned_at(mi_theap_t* theap, size
return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, true, NULL);
}
-static mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept {
+mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept {
return mi_theap_zalloc_aligned_at(theap, size, alignment, 0);
}
@@ -275,8 +269,11 @@ mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t a
return mi_theap_malloc_aligned(_mi_theap_default(), size, alignment);
}
-mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {
- return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, block_size);
+mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept {
+ mi_page_t* page;
+ void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, &page);
+ if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); }
+ return p;
}
mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
@@ -287,8 +284,11 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t a
return mi_theap_zalloc_aligned(_mi_theap_default(), size, alignment);
}
-mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept {
- return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, block_size);
+mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept {
+ mi_page_t* page;
+ void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, &page);
+ if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); }
+ return p;
}
mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
@@ -330,24 +330,30 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap,
// ------------------------------------------------------
static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept {
- mi_assert(alignment > 0);
- if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);
+ mi_assert(mi_alignment_is_valid(alignment));
+ if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)
+ return mi_error_bad_alignment(newsize,alignment,offset);
+ }
+ if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero);
if (p == NULL) return mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,zero,NULL);
- size_t size = mi_usable_size(p);
- if (newsize <= size && newsize >= (size - (size / 2))
- && (((uintptr_t)p + offset) % alignment) == 0) {
- return p; // reallocation still fits, is aligned and not more than 25% waste
+ const size_t size = mi_usable_size(p);
+ if (newsize <= size && newsize >= (size - (size / 2)) && (((uintptr_t)p + offset) & (alignment-1)) == 0) {
+ return p; // reallocation still fits, is aligned and not more than 50% waste
}
else {
- // note: we don't zero allocate upfront so we only zero initialize the expanded part
- void* newp = mi_theap_malloc_aligned_at(theap,newsize,alignment,offset);
+ // note: we don't zero allocate upfront so we only zero initialize the expanded part (at the cost of calling mi_usable_size)
+ mi_page_t* newpage;
+ void* const newp = mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,false/*zero?*/,&newpage);
if (newp != NULL) {
- if (zero && newsize > size) {
+ const size_t copy_size = (newsize > size ? size : newsize);
+ const size_t zero_start = (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
+ const size_t usable = _mi_page_usable_size(newpage,newp);
+ mi_assert_internal(usable >= newsize); // use usable for zero'ing, issue #763
+ if (zero && usable > zero_start) {
// also set last word in the previous allocation to zero to ensure any padding is zero-initialized
- size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
- _mi_memzero((uint8_t*)newp + start, newsize - start);
+ _mi_memzero((uint8_t*)newp + zero_start, usable - zero_start);
}
- _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize));
+ _mi_memcpy(newp, p, copy_size); // cannot be aligned due to abitrary offset... (todo: require offset to be a multiple of sizeof(void*)?)
mi_free(p); // only free if successful
}
return newp;
@@ -356,7 +362,7 @@ static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t
static void* mi_theap_realloc_zero_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept {
mi_assert(alignment > 0);
- if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);
+ if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero);
return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,0,zero);
}
diff --git a/vendored/mimalloc/src/alloc-override.c b/vendored/mimalloc/src/alloc-override.c
index 93d066dc1..bcb96b52d 100644
--- a/vendored/mimalloc/src/alloc-override.c
+++ b/vendored/mimalloc/src/alloc-override.c
@@ -127,7 +127,7 @@ typedef void* mi_nothrow_t;
#elif defined(_MSC_VER)
_Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_HYBRIDPATCHABLE
void* __cdecl _expand(_Pre_notnull_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) {
- return mi_expand(_Block, _Size);
+ return mi__expand(_Block, _Size);
}
_Check_return_ _ACRTIMP
size_t __cdecl _msize_base(_Pre_notnull_ void* _Block) _CRT_NOEXCEPT {
@@ -293,7 +293,7 @@ typedef void* mi_nothrow_t;
void _ZdlPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete(void*, std::align_val_t, std::nothrow_t const&)
void _ZdaPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete[](void*, std::align_val_t, std::nothrow_t const&)
- #if (MI_INTPTR_SIZE==8) || (MI_INTPTR_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257
+ #if (MI_SIZE_SIZE==8) || (MI_SIZE_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257
void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit
void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit
void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
@@ -302,7 +302,7 @@ typedef void* mi_nothrow_t;
void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)
void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
- #elif (MI_INTPTR_SIZE==4)
+ #elif (MI_SIZE_SIZE==4)
void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit
void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit
void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); }
@@ -324,14 +324,25 @@ typedef void* mi_nothrow_t;
extern "C" {
#endif
+// defined here instead alloc-posix so we can alias it
+mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept {
+ if (!mi_is_in_heap_region(p)) return 0;
+ return mi_usable_size(p);
+}
+
+mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {
+ if (!mi_is_in_heap_region(p)) return 0;
+ return mi_usable_size(p);
+}
+
#ifndef MI_OSX_IS_INTERPOSED
// Forward Posix/Unix calls as well
void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize)
- size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p)
+ size_t malloc_size(const void* p) MI_FORWARD1(mi_malloc_size,p)
#if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__)
- size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p)
+ size_t malloc_usable_size(void *p) MI_FORWARD1(mi_malloc_usable_size,p)
#else
- size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p)
+ size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_malloc_usable_size,p)
#endif
// No forwarding here due to aliasing/name mangling issues
diff --git a/vendored/mimalloc/src/alloc-posix.c b/vendored/mimalloc/src/alloc-posix.c
index 60639ff49..a22524942 100644
--- a/vendored/mimalloc/src/alloc-posix.c
+++ b/vendored/mimalloc/src/alloc-posix.c
@@ -31,34 +31,16 @@ terms of the MIT license. A copy of the license can be found in the file
#define ENOMEM 12
#endif
-
-mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept {
- // if (!mi_is_in_heap_region(p)) return 0;
- return mi_usable_size(p);
-}
-
-mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {
- // if (!mi_is_in_heap_region(p)) return 0;
- return mi_usable_size(p);
-}
-
mi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept {
return mi_good_size(size);
}
-void mi_cfree(void* p) mi_attr_noexcept {
- if (mi_is_in_heap_region(p)) {
- mi_free(p);
- }
-}
-
-int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept {
+int mi_posix_memalign(void** p, size_t alignment, size_t size) { // mi_attr_noexcept (issue #794)
// Note: The spec dictates we should not modify `*p` on an error. (issue#27)
// <http://man7.org/linux/man-pages/man3/posix_memalign.3.html>
if (p == NULL) return EINVAL;
- if ((alignment % sizeof(void*)) != 0) return EINVAL; // natural alignment
- // it is also required that alignment is a power of 2 and > 0; this is checked in `mi_malloc_aligned`
- if (alignment==0 || !_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2
+ // it is required that alignment is a power of 2 and a multiple of sizeof(void*)
+ if (alignment<sizeof(void*) || !_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2
void* q = mi_malloc_aligned(size, alignment);
if (q==NULL && size != 0) return ENOMEM;
mi_assert_internal(_mi_is_aligned(q,alignment));
@@ -122,7 +104,7 @@ mi_decl_nodiscard int mi_reallocarr( void* ptrp, size_t count, size_t size ) mi_
}
void** op = (void**)ptrp;
if (total == 0) {
- free(*op);
+ mi_free(*op);
*op = NULL;
return 0;
}
diff --git a/vendored/mimalloc/src/alloc.c b/vendored/mimalloc/src/alloc.c
index aa556bd05..a176db77f 100644
--- a/vendored/mimalloc/src/alloc.c
+++ b/vendored/mimalloc/src/alloc.c
@@ -1,6 +1,5 @@
-
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -12,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc.h"
#include "mimalloc/internal.h"
#include "mimalloc/atomic.h"
-#include "mimalloc/prim.h" // _mi_prim_thread_id()
+#include "mimalloc/prim-tls.h" // _mi_prim_thread_id()
#include <string.h> // memset, strlen (for mi_strdup)
#include <stdlib.h> // malloc, abort
@@ -22,35 +21,83 @@ terms of the MIT license. A copy of the license can be found in the file
#include "free.c"
#undef MI_IN_ALLOC_C
+
// ------------------------------------------------------
-// Allocation
+// Core allocation functions.
+// Every allocation goes through `mi_page_malloc_zero`
// ------------------------------------------------------
+#if MI_PADDING
+static mi_decl_noinline void mi_page_block_setup_padding(mi_page_t* page, mi_block_t* block, size_t size) mi_attr_noexcept {
+ const size_t bsize = mi_page_usable_block_size(page);
+ mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize);
+ const ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE));
+ mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta));
+ mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess
+ padding->canary = mi_ptr_encode_canary(page,block,page->keys);
+ padding->delta = (uint32_t)(delta);
+ #if MI_PADDING_CHECK_BYTES
+ if (!mi_page_is_huge(page)) {
+ uint8_t* fill = (uint8_t*)padding - delta;
+ const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes
+ for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; }
+ }
+ #endif
+}
+#endif
+
// Fast allocation in a page: just pop from the free list.
// Fall back to generic allocation only if the list is empty.
-// Note: in release mode the (inlined) routine is about 7 instructions with a single test.
-static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero, size_t* usable) mi_attr_noexcept
+// Note: even though there is a lot of checks etc in the source,
+// in release mode the (inlined) routine is about 7 to 10 instructions with a single test.
+static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, size_t sample_countdown, bool zero, mi_page_t** ppage) mi_attr_noexcept
{
if (page->block_size != 0) { // not the empty theap
mi_assert_internal(mi_page_block_size(page) >= size);
mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));
}
// check the free list
mi_block_t* const block = page->free;
- if mi_unlikely(block == NULL) {
- return _mi_malloc_generic(theap, size, (zero ? 1 : 0), usable);
+ mi_used_t xused = page->xused;
+ xused.used_alloc += 0x10001; // increment both (16-bit) used count and alloc count
+ #if defined(__GNUC__)
+ __asm("" : : : "memory" ); // always load the `used` field before the test
+ #endif
+ if (block == NULL) {
+ return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage);
}
mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);
- if (usable != NULL) { *usable = mi_page_usable_block_size(page); };
+ if (ppage != NULL) { *ppage = page; };
// pop from the free list
- page->free = mi_block_next(page, block);
- page->used++;
+ mi_block_t* next = mi_block_next(page,block);
+ mi_track_mem_undefined(block,sizeof(*block));
+
+ #if MI_SECURE
+ if (!zero) block->next = 0; // don't leak internal data
+ #endif
+
+ page->free = next;
+ page->xused = xused;
mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page);
mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE));
+ #if MI_SAMPLE==2
+ const size_t req_size = size - MI_PADDING_SIZE;
+ mi_assert_internal(theap->sample_countdown == sample_countdown); // we pass it to improve codegen
+ mi_assert_internal(theap->sample_countdown >= req_size);
+ theap->sample_countdown = sample_countdown - req_size;
+ #else
+ MI_UNUSED(sample_countdown);
+ #endif
+
+ #if MI_STATS>=2
+ mi_theap_stat_counter_increase(theap,malloc_requested,size - MI_PADDING_SIZE);
+ #endif
+
#if MI_DEBUG>3
if (page->free_is_zero && size > sizeof(*block)) {
mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block)));
@@ -63,90 +110,68 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_
const size_t bsize = mi_page_usable_block_size(page);
mi_track_mem_undefined(block, bsize);
- #if (MI_STAT>0)
- if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
- mi_theap_stat_increase(theap, malloc_normal, bsize);
- #if (MI_STAT>1)
- mi_theap_stat_counter_increase(theap, malloc_normal_count, 1);
- const size_t bin = _mi_bin(bsize);
- mi_theap_stat_increase(theap, malloc_bins[bin], 1);
- mi_theap_stat_increase(theap, malloc_requested, size - MI_PADDING_SIZE);
- #endif
- }
+ // track per-block statistics
+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));
+
+ // in debug mode initialize with 0xD0
+ #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN
+ if mi_likely(!zero && !mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); }
#endif
// zero the block? note: we need to zero the full block size (issue #63)
- if mi_likely(!zero) {
- // #if MI_SECURE
- block->next = 0; // don't leak internal data
- // #endif
- #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN
- if (!mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); }
- #endif
- }
- else {
+ if mi_unlikely(zero) {
if (!page->free_is_zero) {
- _mi_memzero_aligned(block, bsize);
+ mi_assert_internal(bsize%MI_SIZE_SIZE == 0);
+ #if !MI_PADDING
+ return // use tail-call
+ #endif
+ _mi_memzero_block(block,bsize);
}
else {
- block->next = 0;
- mi_track_mem_defined(block, bsize);
- }
+ block->next = 0;
+ mi_track_mem_defined(block, bsize);
+ }
}
-
+
+ // setup padding (must come after zero'ing)
#if MI_PADDING // && !MI_TRACK_ENABLED
- mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize);
- ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE));
- #if (MI_DEBUG>=2)
- mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta));
- #endif
- mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess
- padding->canary = mi_ptr_encode_canary(page,block,page->keys);
- padding->delta = (uint32_t)(delta);
- #if MI_PADDING_CHECK
- if (!mi_page_is_huge(page)) {
- uint8_t* fill = (uint8_t*)padding - delta;
- const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes
- for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; }
- }
- #endif
+ mi_page_block_setup_padding(page,block,size);
#endif
- return block;
+ return block;
}
// extra entries for improved efficiency in `alloc-aligned.c` (and in `page.c:mi_malloc_generic`.
extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept {
- return mi_page_malloc_zero(theap, page, size, zero, NULL);
+ return mi_page_malloc_zero(theap, page, size, theap->sample_countdown, zero, NULL);
}
-#if MI_GUARDED
-mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept;
-#endif
-
-// main allocation primitives for small and generic allocation
-// internal small size allocation
-static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept
+// Internal small size allocation; assumes the `theap` is non-NULL.
+// This looks up the page directly from a the direct page entries in the theap.
+// The size can be in words (as a wsize); we assume this will be inlined for best code for `mi_wmalloc_small` and similar functions.
+static mi_decl_forceinline mi_decl_restrict void* mi_theap_nonnull_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept
{
+ size_t size = (is_wsize ? xsize * MI_SIZE_SIZE : xsize);
mi_assert(theap != NULL);
mi_assert(size <= MI_SMALL_SIZE_MAX);
#if MI_DEBUG
- const uintptr_t tid = _mi_thread_id();
- mi_assert(theap->tld->thread_id == 0 || theap->tld->thread_id == tid); // theaps are thread local
+ mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local
#endif
- #if (MI_PADDING || MI_GUARDED)
- if mi_unlikely(size == 0) { size = sizeof(void*); }
+ #if (MI_PADDING) // || MI_GUARDED)
+ if mi_unlikely(xsize == 0) { xsize = (is_wsize ? 1 : MI_SIZE_SIZE); size += MI_SIZE_SIZE; }
#endif
- #if MI_GUARDED
- if mi_unlikely(mi_theap_malloc_use_guarded(theap,size)) {
- return _mi_theap_malloc_guarded(theap, size, zero);
- }
+
+ // we only sample if fine-grained sampling is enabled (otherwise we sample in mi_malloc_generic)
+ #if MI_SAMPLE==2
+ if mi_unlikely(mi_theap_should_sample(theap,size)) { return _mi_theap_malloc_sampled(theap,size,zero,ppage); }
#endif
+
+ // get page in constant time
+ mi_page_t* page = _mi_theap_get_free_small_page(theap, xsize + (is_wsize ? MI_PADDING_WSIZE : MI_PADDING_SIZE), is_wsize);
- // get page in constant time, and allocate from it
- mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE);
- void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, usable);
+ // and allocate
+ void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, theap->sample_countdown, zero, ppage);
mi_track_malloc(p,size,zero);
#if MI_DEBUG>3
@@ -157,23 +182,17 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_non
return p;
}
-// internal generic allocation
-static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept
+// Internal generic allocation.
+// Goes to `page.c:_mi_malloc_generic` which handles large and aligned allocations.
+// Every small allocation eventually goes through this generic allocation path if it cannot be satisfied from the small page cache.
+static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept
{
- #if MI_GUARDED
- #if MI_THEAP_INITASNULL
- if (theap!=NULL)
- #endif
- if (huge_alignment==0 && mi_theap_malloc_use_guarded(theap, size)) {
- return _mi_theap_malloc_guarded(theap, size, zero);
- }
- #endif
#if !MI_THEAP_INITASNULL
mi_assert(theap!=NULL);
#endif
- mi_assert(theap==NULL || theap->tld->thread_id == 0 || theap->tld->thread_id == _mi_thread_id()); // theaps are thread local
+ mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local
mi_assert((huge_alignment & 1)==0);
- void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, usable); // note: size can overflow but it is detected in malloc_generic
+ void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, ppage); // note: size can overflow but it is detected in malloc_generic
mi_track_malloc(p, size, zero);
#if MI_DEBUG>3
@@ -184,47 +203,102 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size
return p;
}
-// internal small allocation
-static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept {
- #if MI_THEAP_INITASNULL
+// ------------------------------------------------------------------------
+// Small allocations: mi_malloc_small and variants (zalloc, wmalloc, wzalloc)
+// ------------------------------------------------------------------------
+
+// internal small allocation where the theap can be NULL on some platforms (Windows and macOS)
+static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept {
+ #if !MI_THEAP_INITASNULL
+ return mi_theap_nonnull_xmalloc_small_zero(theap, xsize, is_wsize, zero, ppage);
+ #else
if (theap!=NULL) {
- return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
+ return mi_theap_nonnull_xmalloc_small_zero(theap, xsize, is_wsize, zero, ppage);
}
else {
- return mi_theap_malloc_generic(theap, size, zero, 0, usable); // tailcall
+ return mi_theap_malloc_generic(theap, (is_wsize ? xsize * MI_SIZE_SIZE : xsize), zero, 0, ppage); // tailcall
}
- #else
- return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
#endif
}
-
-// allocate a small block
+// Allocate a small block from a theap; these routines assume a non-NULL theap.
mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {
- return mi_theap_malloc_small_zero(theap, size, false, NULL);
+ mi_assert(theap!=NULL);
+ return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, false, NULL);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept {
+ mi_assert(theap!=NULL);
+ return mi_theap_nonnull_xmalloc_small_zero(theap, wsize, true, false, NULL);
}
+mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {
+ mi_assert(theap!=NULL);
+ return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, true, NULL);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept {
+ mi_assert(theap!=NULL);
+ return mi_theap_nonnull_xmalloc_small_zero(theap, wsize, true, true, NULL);
+}
+
+// Regular small allocation functions that use the default theap.
mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept {
- return mi_theap_malloc_small(_mi_theap_default(), size);
+ return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, NULL);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept {
+ return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, NULL);
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept {
+ return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, false, NULL );
}
+mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept {
+ return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, true, NULL);
+}
+
+
+mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
+ return mi_theap_malloc_small(_mi_heap_theap_get_or_init(heap), size);
+}
mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
- return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, false, NULL);
+ // we could also use: mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(theap), size, false, NULL); }
+ // but the following prevents using a stack frame. We use this to optimize some select functions only.
+ mi_theap_t* const theap = _mi_heap_theap_cached(heap);
+ if mi_likely(theap!=NULL) { return mi_theap_malloc_small(theap, size); }
+ else { return mi_heap_init_malloc_small(heap, size); }
}
-// The main internal allocation functions
-static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept {
+mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
+ return mi_theap_zalloc_small(_mi_heap_theap_get_or_init(heap), size);
+}
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
+ // optimize: return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL);
+ mi_theap_t* const theap = _mi_heap_theap_cached(heap);
+ if mi_likely(theap!=NULL) { return mi_theap_zalloc_small(theap, size); }
+ else { return mi_heap_init_zalloc_small(heap, size); }
+}
+
+
+// ------------------------------------------------------------------------------
+// Main allocation functions: mi_malloc and variants (zalloc, wmalloc, wzalloc)
+// ------------------------------------------------------------------------------
+
+mi_decl_nodiscard static mi_decl_forceinline mi_decl_restrict void* mi_theap_nonnull_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept {
+ mi_assert(theap!=NULL);
// fast path for small objects
if mi_likely(size <= MI_SMALL_SIZE_MAX) {
mi_assert_internal(huge_alignment == 0);
- return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
+ return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, zero, ppage);
}
else {
- return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable);
+ return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage);
}
}
-extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept {
+mi_decl_nodiscard extern mi_decl_forceinline mi_decl_restrict void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept {
// fast path for small objects
#if MI_THEAP_INITASNULL
if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX)
@@ -233,59 +307,67 @@ extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, siz
#endif
{
mi_assert_internal(huge_alignment == 0);
- return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable);
+ return mi_theap_nonnull_xmalloc_small_zero(theap, size, false, zero, ppage);
}
else {
- return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable);
+ return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage);
}
}
-void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept {
- return _mi_theap_malloc_zero_ex(theap, size, zero, 0, usable);
+mi_decl_nodiscard static inline mi_decl_restrict void* mi_theap_nonnull_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {
+ mi_assert(theap!=NULL);
+ return mi_theap_nonnull_malloc_zero(theap, size, false, 0, NULL);
}
+mi_decl_nodiscard static inline mi_decl_restrict void* mi_theap_nonnull_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {
+ mi_assert(theap!=NULL);
+ return mi_theap_nonnull_malloc_zero(theap, size, true, 0, NULL);
+}
// Main allocation functions
+// (note: to reduce variants, we only use nonnull theap variants for small allocations)
mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {
- return _mi_theap_malloc_zero(theap, size, false, NULL);
-}
-
-mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept {
- return mi_theap_malloc(_mi_theap_default(), size);
+ return _mi_theap_malloc_zero(theap, size, false, 0, NULL);
}
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
- return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL);
+mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {
+ return _mi_theap_malloc_zero(theap, size, true, 0, NULL);
}
-
-// zero initialized small block
-mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept {
- return mi_theap_malloc_small_zero(_mi_theap_default(), size, true, NULL);
+mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept {
+ return mi_theap_malloc(_mi_theap_default(), size);
}
-mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept {
- return mi_theap_malloc_small_zero(theap, size, true, NULL);
+mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept {
+ return mi_theap_zalloc(_mi_theap_default(), size);
}
-mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
- return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL);
+mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
+ return mi_theap_nonnull_malloc(_mi_heap_theap_get_or_init(heap), size);
}
+mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
+ // return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL);
+ mi_theap_t* const theap = _mi_heap_theap_cached(heap);
+ if mi_likely(theap!=NULL) { return mi_theap_nonnull_malloc(theap, size); }
+ else { return mi_heap_init_malloc(heap, size); }
-
-mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept {
- return _mi_theap_malloc_zero(theap, size, true, NULL);
}
-mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept {
- return _mi_theap_malloc_zero(_mi_theap_default(), size, true, NULL);
+mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
+ return mi_theap_nonnull_zalloc(_mi_heap_theap_get_or_init(heap), size);
}
-
mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
- return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL);
+ // optimize: return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL);
+ mi_theap_t* const theap = _mi_heap_theap_cached(heap);
+ if mi_likely(theap!=NULL) { return mi_theap_nonnull_zalloc(theap, size); }
+ else { return mi_heap_init_zalloc(heap, size); }
}
+// ----------------------------------------------------------
+// mi_calloc variants
+// ----------------------------------------------------------
+
mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept {
size_t total;
if (mi_count_size_overflow(count,size,&total)) return NULL;
@@ -293,47 +375,64 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_
}
mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept {
- return mi_theap_calloc(_mi_theap_default(),count,size);
+ return mi_theap_calloc(_mi_theap_default(), count, size);
}
mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
- size_t total;
- if (mi_count_size_overflow(count, size, &total)) return NULL;
- return mi_heap_zalloc(heap, total);
+ return mi_theap_calloc(_mi_heap_theap(heap), count, size);
}
+// ----------------------------------------------------------
+// Usable block sizes
+// ----------------------------------------------------------
+
// Return usable size
-mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* usable) mi_attr_noexcept {
- return mi_theap_malloc_small_zero(_mi_theap_default(), size, false, usable);
+static void* mi_ublock_size( void* p, mi_page_t* page, size_t* pblock_size ) {
+ mi_assert_internal(page == _mi_ptr_page(p));
+ if (pblock_size!=NULL) {
+ if (p!=NULL) { *pblock_size = mi_page_block_size(page); }
+ }
+ return p;
+}
+
+mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept {
+ mi_page_t* page;
+ void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, &page);
+ return mi_ublock_size(p,page,pblock_size);
}
-mi_decl_nodiscard mi_decl_restrict void* mi_theap_umalloc(mi_theap_t* theap, size_t size, size_t* usable) mi_attr_noexcept {
- return _mi_theap_malloc_zero_ex(theap, size, false, 0, usable);
+mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept {
+ mi_page_t* page;
+ void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, &page);
+ return mi_ublock_size(p,page,pblock_size);
}
-mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* usable) mi_attr_noexcept {
- return mi_theap_umalloc(_mi_theap_default(), size, usable);
+mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* pblock_size) mi_attr_noexcept {
+ mi_page_t* page;
+ void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, false, 0, &page);
+ return mi_ublock_size(p,page,pblock_size);
}
-mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* usable) mi_attr_noexcept {
- return _mi_theap_malloc_zero_ex(_mi_theap_default(), size, true, 0, usable);
+mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* pblock_size) mi_attr_noexcept {
+ mi_page_t* page;
+ void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, true, 0, &page);
+ return mi_ublock_size(p,page,pblock_size);
}
-mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* usable) mi_attr_noexcept {
+mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* pblock_size) mi_attr_noexcept {
size_t total;
if (mi_count_size_overflow(count,size,&total)) return NULL;
- return mi_uzalloc(total, usable);
+ return mi_uzalloc(total, pblock_size);
}
+// ----------------------------------------------------------
// Uninitialized `calloc`
-static mi_decl_restrict void* mi_theap_mallocn(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept {
- size_t total;
- if (mi_count_size_overflow(count, size, &total)) return NULL;
- return mi_theap_malloc(theap, total);
-}
+// ----------------------------------------------------------
mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept {
- return mi_theap_mallocn(_mi_theap_default(),count,size);
+ size_t total;
+ if (mi_count_size_overflow(count, size, &total)) return NULL;
+ return mi_malloc(total);
}
mi_decl_nodiscard mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
@@ -342,6 +441,9 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t
return mi_heap_malloc(heap, total);
}
+// ----------------------------------------------------------
+// Reallocation
+// ----------------------------------------------------------
// Expand (or shrink) in place (or fail)
void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
@@ -351,14 +453,14 @@ void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
return NULL;
#else
if (p == NULL) return NULL;
- const mi_page_t* const page = mi_validate_ptr_page(p,"mi_expand");
- const size_t size = _mi_usable_size(p,page);
+ const mi_page_t* const page = mi_ptr_page_validate(p,"mi_expand");
+ const size_t size = _mi_page_usable_size(page,p);
if (newsize > size) return NULL;
return p; // it fits
#endif
}
-void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {
+static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept {
// if p == NULL then behave as malloc.
// else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)).
// (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.)
@@ -367,63 +469,80 @@ void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool ze
if (p==NULL) {
page = NULL;
size = 0;
- if (usable_pre!=NULL) { *usable_pre = 0; }
+ if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; }
}
- else {
- page = mi_validate_ptr_page(p,"mi_realloc");
- size = _mi_usable_size(p,page);
- if (usable_pre!=NULL) { *usable_pre = mi_page_usable_block_size(page); }
+ else {
+ page = mi_ptr_page_validate(p,"mi_realloc");
+ if mi_unlikely(page==NULL) { // invalid pointer
+ if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; }
+ if (pblock_size_post!=NULL) { *pblock_size_post = 0; }
+ return NULL;
+ }
+ size = _mi_page_usable_size(page,p);
+ if (pblock_size_pre!=NULL) { *pblock_size_pre = mi_page_block_size(page); }
+ }
+ // check if we can reuse the existing block
+ if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)
+ mi_assert_internal(page!=NULL); // note: page!=NULL (since if p==NULL, we have size=0 and size>=newsize>0
+ #if MI_THEAP_INITASNULL
+ if (theap!=NULL)
+ #endif
+ {
+ if (mi_page_heap(page)==_mi_theap_heap_peek(theap)) { // and within the same heap
+ mi_assert_internal(p!=NULL);
+ // todo: do not track as the usable size is still the same in the free; adjust potential padding?
+ // mi_track_resize(p,size,newsize)
+ // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); }
+ if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(page); }
+ return p; // reallocation still fits and not more than 50% waste
+ }
+ }
}
- if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0 // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)
- && mi_page_heap(page)==_mi_theap_heap(theap)) // and within the same heap
- {
- mi_assert_internal(p!=NULL);
- // todo: do not track as the usable size is still the same in the free; adjust potential padding?
- // mi_track_resize(p,size,newsize)
- // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); }
- if (usable_post!=NULL) { *usable_post = mi_page_usable_block_size(page); }
- return p; // reallocation still fits and not more than 50% waste
- }
- void* newp = mi_theap_umalloc(theap,newsize,usable_post);
+ // note: we don't zero allocate upfront so we only zero initialize the expanded part
+ mi_page_t* newpage; // use block_size for zero-ing, issue #763
+ void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,0,&newpage);
if mi_likely(newp != NULL) {
- if (zero && newsize > size) {
- // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
- const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
- _mi_memzero((uint8_t*)newp + start, newsize - start);
+ if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(newpage); }
+ const size_t copy_size = (newsize > size ? size : newsize);
+ const size_t zero_start = _mi_align_down( (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0), sizeof(intptr_t)); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
+ const size_t usable = _mi_page_usable_size(newpage,newp);
+ mi_assert_internal(usable >= newsize);
+ if (zero && usable > zero_start) {
+ _mi_memzero_aligned((uint8_t*)newp + zero_start, usable - zero_start);
}
else if (newsize == 0) {
((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725)
}
if mi_likely(p != NULL) {
- const size_t copysize = (newsize > size ? size : newsize);
- mi_track_mem_defined(p,copysize); // _mi_useable_size may be too large for byte precise memory tracking..
- _mi_memcpy(newp, p, copysize);
- mi_free(p); // only free the original pointer if successful // todo: optimize since page is known?
+ mi_track_mem_defined(p,copy_size); // _mi_useable_size may be too large for byte precise memory tracking..
+ _mi_memcpy_aligned(newp, p, copy_size);
+ mi_free(p); // only free the original pointer if successful
}
}
return newp;
}
-mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
- return _mi_theap_realloc_zero(theap, p, newsize, false, NULL, NULL);
+void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept {
+ return mi_theap_realloc_zero_ex(theap,p,newsize,zero,NULL,NULL);
}
-static void* mi_theap_reallocn(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept {
- size_t total;
- if (mi_count_size_overflow(count, size, &total)) return NULL;
- return mi_theap_realloc(theap, p, total);
-}
-
-
-// Reallocate but free `p` on errors
-static void* mi_theap_reallocf(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
- void* newp = mi_theap_realloc(theap, p, newsize);
- if (newp==NULL && p!=NULL) mi_free(p);
- return newp;
+// for theap_realloc/rezalloc we allow theap==NULL to reduce variants
+mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
+ if (p==NULL) {
+ return mi_theap_malloc(theap,newsize);
+ }
+ else {
+ return _mi_theap_realloc_zero(theap, p, newsize, false);
+ }
}
-static void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
- return _mi_theap_realloc_zero(theap, p, newsize, true, NULL, NULL);
+mi_decl_nodiscard void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept {
+ if (p==NULL) {
+ return mi_theap_zalloc(theap,newsize);
+ }
+ else {
+ return _mi_theap_realloc_zero(theap, p, newsize, true);
+ }
}
static void* mi_theap_recalloc(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept {
@@ -432,20 +551,27 @@ static void* mi_theap_recalloc(mi_theap_t* theap, void* p, size_t count, size_t
return mi_theap_rezalloc(theap, p, total);
}
-
mi_decl_nodiscard void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept {
return mi_theap_realloc(_mi_theap_default(),p,newsize);
}
mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept {
- return mi_theap_reallocn(_mi_theap_default(),p,count,size);
+ size_t total;
+ if (mi_count_size_overflow(count, size, &total)) return NULL;
+ return mi_theap_realloc(_mi_theap_default(),p,total);
}
-mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept {
- return _mi_theap_realloc_zero(_mi_theap_default(),p,newsize, false, usable_pre, usable_post);
+mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept {
+ return mi_theap_realloc_zero_ex(_mi_theap_default(),p,newsize, false, pblock_size_pre, pblock_size_post);
}
// Reallocate but free `p` on errors
+static void* mi_theap_reallocf(mi_theap_t* xtheap, void* p, size_t newsize) mi_attr_noexcept {
+ void* newp = mi_theap_realloc(xtheap, p, newsize);
+ if (newp==NULL && p!=NULL) mi_free(p);
+ return newp;
+}
+
mi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept {
return mi_theap_reallocf(_mi_theap_default(),p,newsize);
}
@@ -458,13 +584,14 @@ mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_
return mi_theap_recalloc(_mi_theap_default(), p, count, size);
}
-
mi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
return mi_theap_realloc(_mi_heap_theap(heap), p, newsize);
}
mi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {
- return mi_theap_reallocn(_mi_heap_theap(heap), p, count, size);
+ size_t total;
+ if (mi_count_size_overflow(count, size, &total)) return NULL;
+ return mi_theap_realloc(_mi_heap_theap(heap), p, total);
}
// Reallocate but free `p` on errors
@@ -481,17 +608,16 @@ mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count,
}
-
// ------------------------------------------------------
// strdup, strndup, and realpath
// ------------------------------------------------------
// `strdup` using mi_malloc
-mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* theap, const char* s) mi_attr_noexcept {
+mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* xtheap, const char* s) mi_attr_noexcept {
if (s == NULL) return NULL;
size_t len = _mi_strlen(s);
if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1
- char* t = (char*)mi_theap_malloc(theap,len+1);
+ char* t = (char*)mi_theap_malloc(xtheap,len+1);
if (t == NULL) return NULL;
_mi_memcpy(t, s, len);
t[len] = 0;
@@ -507,11 +633,11 @@ mi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const c
}
// `strndup` using mi_malloc
-mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* theap, const char* s, size_t n) mi_attr_noexcept {
+mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* xtheap, const char* s, size_t n) mi_attr_noexcept {
if (s == NULL) return NULL;
const size_t len = _mi_strnlen(s,n); // len <= n
if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1
- char* t = (char*)mi_theap_malloc(theap, len+1);
+ char* t = (char*)mi_theap_malloc(xtheap, len+1);
if (t == NULL) return NULL;
_mi_memcpy(t, s, len);
t[len] = 0;
@@ -533,21 +659,38 @@ mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const
#define PATH_MAX MAX_PATH
#endif
-mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept {
+mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* xtheap, const char* fname, char* resolved_name) mi_attr_noexcept {
// todo: use GetFullPathNameW to allow longer file names
+ if (fname==NULL || *fname==0) {
+ errno = EINVAL;
+ return NULL;
+ }
char buf[PATH_MAX];
DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL);
if (res == 0) {
- errno = GetLastError(); return NULL;
+ DWORD err = GetLastError();
+ switch (err) {
+ case ERROR_LOCK_VIOLATION:
+ case ERROR_SHARING_VIOLATION:
+ case ERROR_INVALID_ACCESS: errno = EACCES; break;
+ case ERROR_INVALID_HANDLE:
+ case ERROR_INVALID_FUNCTION: errno = EINVAL; break;
+ case ERROR_PATH_NOT_FOUND: errno = ENOTDIR; break;
+ case ERROR_FILE_NOT_FOUND: errno = ENOENT; break;
+ case ERROR_NOT_ENOUGH_MEMORY: errno = ENOMEM; break;
+ default: errno = EIO;
+ }
+ return NULL;
}
else if (res > PATH_MAX) {
- errno = EINVAL; return NULL;
+ errno = ENAMETOOLONG;
+ return NULL;
}
else if (resolved_name != NULL) {
return resolved_name;
}
else {
- return mi_theap_strndup(theap, buf, PATH_MAX);
+ return mi_theap_strndup(xtheap, buf, PATH_MAX);
}
}
@@ -573,7 +716,7 @@ static size_t mi_path_max(void) {
return pmax;
}
-char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept {
+char* mi_theap_realpath(mi_theap_t* xtheap, const char* fname, char* resolved_name) mi_attr_noexcept {
if (resolved_name != NULL) {
return realpath(fname,resolved_name);
}
@@ -587,13 +730,13 @@ char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_nam
return result;
*/
const size_t n = mi_path_max();
- char* const buf = (char*)mi_zalloc(n+1);
+ char* const buf = (char*)mi_theap_zalloc(xtheap,n+1);
if (buf == NULL) {
errno = ENOMEM;
return NULL;
}
char* rname = realpath(fname,buf);
- char* result = mi_theap_strndup(theap,rname,n); // ok if `rname==NULL`
+ char* result = mi_theap_strndup(xtheap,rname,n); // ok if `rname==NULL`
mi_free(buf);
return result;
}
@@ -615,8 +758,11 @@ The standard requires calling into `get_new_handler` and
throwing the bad_alloc exception on failure. If we compile
with a C++ compiler we can implement this precisely. If we
use a C compiler we cannot throw a `bad_alloc` exception
-but we call `exit` instead (i.e. not returning).
+but we call `abort` instead (i.e. not returning).
+Also, the standard requires calling the new handler until
+it returns false, but we limit the total calls.
-------------------------------------------------------*/
+#define MI_TRY_NEW_MAX (4)
#ifdef __cplusplus
#include <new>
@@ -638,10 +784,19 @@ static bool mi_try_new_handler(bool nothrow) {
#endif
return false;
}
- else {
+ else if (!nothrow) {
h();
return true;
}
+ else {
+ try {
+ h();
+ }
+ catch(...) { // swallow std::bad_alloc
+ return false; // stop trying
+ }
+ return true;
+ }
}
#else
typedef void (*std_new_handler_t)(void);
@@ -678,22 +833,18 @@ static bool mi_try_new_handler(bool nothrow) {
static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, bool nothrow ) {
void* p = NULL;
- while(p == NULL && mi_try_new_handler(nothrow)) {
+ for(int i = 0; i < MI_TRY_NEW_MAX && p == NULL && mi_try_new_handler(nothrow); i++) {
+ if (size > MI_MAX_ALLOC_SIZE) return NULL; // call try_new_handler at least once
p = mi_theap_malloc(theap,size);
}
return p;
}
-static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) {
- return mi_theap_try_new(_mi_theap_default(), size, nothrow);
-}
-
static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow) {
return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow);
}
-
-mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) {
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) {
void* p = mi_theap_malloc(theap,size);
if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false);
return p;
@@ -709,8 +860,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size
return p;
}
-
-mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) {
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) {
size_t total;
if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc
@@ -730,42 +880,56 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, si
}
+mi_decl_nodiscard mi_decl_restrict void* mi_theap_alloc_new_nothrow(mi_theap_t* theap, size_t size) mi_attr_noexcept {
+ void* p = mi_theap_malloc(theap,size);
+ if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, true);
+ return p;
+}
+
mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept {
- void* p = mi_malloc(size);
- if mi_unlikely(p == NULL) return mi_try_new(size, true);
+ return mi_theap_alloc_new_nothrow(_mi_theap_default(), size);
+}
+
+static mi_decl_noinline void* mi_try_new_aligned(size_t size, size_t alignment, bool nothrow) {
+ void* p = NULL;
+ for(int i = 0; i < MI_TRY_NEW_MAX && p==NULL && mi_try_new_handler(nothrow); i++) {
+ if (!mi_alignment_is_valid(alignment)) return NULL;
+ p = mi_malloc_aligned(size,alignment);
+ }
return p;
}
mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) {
- void* p;
- do {
- p = mi_malloc_aligned(size, alignment);
- }
- while(p == NULL && mi_try_new_handler(false));
+ void* p = mi_malloc_aligned(size, alignment);
+ if mi_unlikely(p==NULL) return mi_try_new_aligned(size,alignment,false);
return p;
}
mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept {
- void* p;
- do {
- p = mi_malloc_aligned(size, alignment);
- }
- while(p == NULL && mi_try_new_handler(true));
+ void* p = mi_malloc_aligned(size, alignment);
+ if mi_unlikely(p==NULL) return mi_try_new_aligned(size,alignment,true);
return p;
}
+static mi_decl_noinline void* mi_try_new_realloc(void* p, size_t newsize) {
+ void* q = NULL;
+ for(int i = 0; i < MI_TRY_NEW_MAX && q==NULL && mi_try_new_handler(false); i++) {
+ if (newsize > MI_MAX_ALLOC_SIZE) return NULL;
+ q = mi_realloc(p,newsize);
+ }
+ return q;
+}
+
mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) {
- void* q;
- do {
- q = mi_realloc(p, newsize);
- } while (q == NULL && mi_try_new_handler(false));
+ void* q = mi_realloc(p, newsize);
+ if (q == NULL) return mi_try_new_realloc(p,newsize);
return q;
}
mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) {
size_t total;
if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) {
- mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc
+ mi_try_new_handler(false);
return NULL;
}
else {
@@ -773,93 +937,6 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) {
}
}
-#if MI_GUARDED
-// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true).
-// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`)
-// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size`
-
-static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) {
- // TODO: we can still make padding work by moving it out of the guard page area
- mi_page_t* const page = _mi_ptr_page(block);
- mi_page_set_has_interior_pointers(page, true);
- block->next = MI_BLOCK_TAG_GUARDED;
-
- // set guard page at the end of the block
- const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local`
- const size_t os_page_size = _mi_os_page_size();
- mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t));
- if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) {
- // should never happen
- mi_free(block);
- return NULL;
- }
- uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size;
- // note: the alignment of the guard page relies on blocks being os_page_size aligned which
- // is ensured in `mi_arena_page_alloc_fresh`.
- mi_assert_internal(_mi_is_aligned(block, os_page_size));
- mi_assert_internal(_mi_is_aligned(guard_page, os_page_size));
- if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) {
- const bool ok = _mi_os_protect(guard_page, os_page_size);
- if mi_unlikely(!ok) {
- _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size);
- }
- }
- else {
- _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size);
- }
-
- // align pointer just in front of the guard page
- size_t offset = block_size - os_page_size - obj_size;
- mi_assert_internal(offset > sizeof(mi_block_t));
- if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) {
- // give up to place it right in front of the guard page if the offset is too large for unalignment
- offset = MI_PAGE_MAX_OVERALLOC_ALIGN;
- }
- uint8_t* const p = (uint8_t*)block + offset;
- mi_assert_internal(p == guard_page - obj_size);
- mi_track_align(block, p, offset, obj_size);
- mi_track_mem_defined(block, sizeof(mi_block_t));
- return p;
-}
-
-mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept
-{
- // allocate multiple of page size ending in a guard page
- // ensure minimal alignment requirement?
- if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow
- _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size);
- return NULL;
- }
- const size_t os_page_size = _mi_os_page_size();
- const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE));
- const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE);
- const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size);
- mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, NULL);
- if (block==NULL) return NULL;
- void* const p = mi_block_ptr_set_guarded(block, obj_size);
- if (p == NULL) return p;
- if (zero) {
- _mi_memzero_aligned(p,obj_size); // we have to zero here as padding might have written here (if the blocksize > reqsize + os_page_size)
- }
-
- // stats
- mi_track_malloc(p, obj_size, zero);
- if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); }
- mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1);
- #if MI_STAT>1
- // adjust stats to only count the allocated size of the block (and not the guard page)
- mi_theap_stat_adjust_decrease(theap, malloc_requested, req_size);
- mi_theap_stat_increase(theap, malloc_requested, size);
- #endif
- #if MI_DEBUG>3
- if (zero) {
- mi_assert_expensive(mi_mem_is_zero(p, size));
- }
- #endif
- return p;
-}
-#endif
-
// ------------------------------------------------------
// ensure explicit external inline definitions are emitted!
// ------------------------------------------------------
@@ -868,10 +945,11 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size,
void* _mi_externs[] = {
(void*)&_mi_page_malloc_zero,
(void*)&_mi_theap_malloc_zero,
- (void*)&_mi_theap_malloc_zero_ex,
(void*)&mi_theap_malloc,
(void*)&mi_theap_zalloc,
(void*)&mi_theap_malloc_small,
+ (void*)&mi_theap_zalloc_small,
+ (void*)&mi_theap_calloc,
(void*)&mi_malloc,
(void*)&mi_malloc_small,
(void*)&mi_zalloc,
diff --git a/vendored/mimalloc/src/arena-meta.c b/vendored/mimalloc/src/arena-meta.c
deleted file mode 100644
index cee38caa1..000000000
--- a/vendored/mimalloc/src/arena-meta.c
+++ /dev/null
@@ -1,179 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2019-2024, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-/* ----------------------------------------------------------------------------
- We have a special "mini" allocator just for allocation of meta-data like
- the theap (`mi_theap_t`) or thread-local data (`mi_tld_t`).
-
- We reuse the bitmap of the arena's for allocation of 64b blocks inside
- an arena slice (64KiB).
- We always ensure that meta data is zero'd (we zero on `free`)
------------------------------------------------------------------------------*/
-
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "bitmap.h"
-
-/* -----------------------------------------------------------
- Meta data allocation
------------------------------------------------------------ */
-
-#define MI_META_PAGE_SIZE MI_ARENA_SLICE_SIZE
-#define MI_META_PAGE_ALIGN MI_ARENA_SLICE_ALIGN
-
-// large enough such that META_MAX_SIZE > 4k (even on 32-bit)
-#define MI_META_BLOCK_SIZE (1 << (16 - MI_BCHUNK_BITS_SHIFT)) // 128 on 64-bit
-#define MI_META_BLOCK_ALIGN MI_META_BLOCK_SIZE
-#define MI_META_BLOCKS_PER_PAGE (MI_META_PAGE_SIZE / MI_META_BLOCK_SIZE) // 512
-#define MI_META_MAX_SIZE (MI_BCHUNK_SIZE * MI_META_BLOCK_SIZE)
-
-#if MI_META_MAX_SIZE <= 4096
-#error "max meta object size should be at least 4KiB"
-#endif
-
-typedef struct mi_meta_page_s {
- _Atomic(struct mi_meta_page_s*) next; // a linked list of meta-data pages (never released)
- mi_memid_t memid; // provenance of the meta-page memory itself
- mi_bbitmap_t blocks_free; // a small bitmap with 1 bit per block.
-} mi_meta_page_t;
-
-static mi_decl_cache_align _Atomic(mi_meta_page_t*) mi_meta_pages = MI_ATOMIC_VAR_INIT(NULL);
-
-
-#if MI_DEBUG > 1
-static mi_meta_page_t* mi_meta_page_of_ptr(void* p, size_t* block_idx) {
- mi_meta_page_t* mpage = (mi_meta_page_t*)((uint8_t*)_mi_align_down_ptr(p,MI_META_PAGE_ALIGN) + _mi_os_secure_guard_page_size());
- if (block_idx != NULL) {
- *block_idx = ((uint8_t*)p - (uint8_t*)mpage) / MI_META_BLOCK_SIZE;
- }
- return mpage;
-}
-#endif
-
-static mi_meta_page_t* mi_meta_page_next( mi_meta_page_t* mpage ) {
- return mi_atomic_load_ptr_acquire(mi_meta_page_t, &mpage->next);
-}
-
-static void* mi_meta_block_start( mi_meta_page_t* mpage, size_t block_idx ) {
- mi_assert_internal(_mi_is_aligned((uint8_t*)mpage - _mi_os_secure_guard_page_size(), MI_META_PAGE_ALIGN));
- mi_assert_internal(block_idx < MI_META_BLOCKS_PER_PAGE);
- void* p = ((uint8_t*)mpage - _mi_os_secure_guard_page_size() + (block_idx * MI_META_BLOCK_SIZE));
- mi_assert_internal(mpage == mi_meta_page_of_ptr(p,NULL));
- return p;
-}
-
-// allocate a fresh meta page and add it to the global list.
-static mi_meta_page_t* mi_meta_page_zalloc(void) {
- // allocate a fresh arena slice
- // note: careful with _mi_subproc as it may recurse into mi_tld and meta_page_zalloc again.. (same with _mi_os_numa_node()...)
- mi_memid_t memid;
- uint8_t* base = (uint8_t*)_mi_arenas_alloc_aligned(mi_heap_main(), MI_META_PAGE_SIZE, MI_META_PAGE_ALIGN, 0,
- true /* commit*/, (MI_SECURE==0) /* allow large? */,
- NULL /* req arena */, 0 /* thread_seq */, -1 /* numa node */, &memid);
- if (base == NULL) return NULL;
- mi_assert_internal(_mi_is_aligned(base,MI_META_PAGE_ALIGN));
- if (!memid.initially_zero) {
- _mi_memzero_aligned(base, MI_ARENA_SLICE_SIZE);
- }
-
- // guard pages
- #if MI_SECURE >= 1
- _mi_os_secure_guard_page_set_at(base, memid);
- _mi_os_secure_guard_page_set_before(base + MI_META_PAGE_SIZE, memid);
- #endif
-
- // initialize the page and free block bitmap
- mi_meta_page_t* mpage = (mi_meta_page_t*)(base + _mi_os_secure_guard_page_size());
- mpage->memid = memid;
- mi_bbitmap_init(&mpage->blocks_free, MI_META_BLOCKS_PER_PAGE, true /* already_zero */);
- const size_t mpage_size = offsetof(mi_meta_page_t,blocks_free) + mi_bbitmap_size(MI_META_BLOCKS_PER_PAGE, NULL);
- const size_t info_blocks = _mi_divide_up(mpage_size,MI_META_BLOCK_SIZE);
- const size_t guard_blocks = _mi_divide_up(_mi_os_secure_guard_page_size(), MI_META_BLOCK_SIZE);
- mi_assert_internal(info_blocks + 2*guard_blocks < MI_META_BLOCKS_PER_PAGE);
- mi_bbitmap_unsafe_setN(&mpage->blocks_free, info_blocks + guard_blocks, MI_META_BLOCKS_PER_PAGE - info_blocks - 2*guard_blocks);
-
- // push atomically in front of the meta page list
- // (note: there is no ABA issue since we never free meta-pages)
- mi_meta_page_t* old = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages);
- do {
- mi_atomic_store_ptr_release(mi_meta_page_t, &mpage->next, old);
- } while(!mi_atomic_cas_ptr_weak_acq_rel(mi_meta_page_t,&mi_meta_pages,&old,mpage));
- return mpage;
-}
-
-
-// allocate meta-data
-mi_decl_noinline void* _mi_meta_zalloc( size_t size, mi_memid_t* pmemid )
-{
- mi_assert_internal(pmemid != NULL);
- size = _mi_align_up(size,MI_META_BLOCK_SIZE);
- if (size == 0 || size > MI_META_MAX_SIZE) return NULL;
- const size_t block_count = _mi_divide_up(size,MI_META_BLOCK_SIZE);
- mi_assert_internal(block_count > 0 && block_count < MI_BCHUNK_BITS);
- mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages);
- mi_meta_page_t* mpage = mpage0;
- while (mpage != NULL) {
- size_t block_idx;
- if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {
- // found and claimed `block_count` blocks
- *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);
- return mi_meta_block_start(mpage,block_idx);
- }
- else {
- mpage = mi_meta_page_next(mpage);
- }
- }
- // failed to find space in existing pages
- if (mi_atomic_load_ptr_acquire(mi_meta_page_t,&mi_meta_pages) != mpage0) {
- // the page list was updated by another thread in the meantime, retry
- return _mi_meta_zalloc(size,pmemid);
- }
- // otherwise, allocate a fresh metapage and try once more
- mpage = mi_meta_page_zalloc();
- if (mpage != NULL) {
- size_t block_idx;
- if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {
- // found and claimed `block_count` blocks
- *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);
- return mi_meta_block_start(mpage,block_idx);
- }
- }
- // if all this failed, allocate from the OS
- return _mi_os_alloc(size, pmemid);
-}
-
-// free meta-data
-mi_decl_noinline void _mi_meta_free(void* p, size_t size, mi_memid_t memid) {
- if (p==NULL) return;
- if (memid.memkind == MI_MEM_META) {
- mi_assert_internal(_mi_divide_up(size, MI_META_BLOCK_SIZE) == memid.mem.meta.block_count);
- const size_t block_count = memid.mem.meta.block_count;
- const size_t block_idx = memid.mem.meta.block_index;
- mi_meta_page_t* mpage = (mi_meta_page_t*)memid.mem.meta.meta_page;
- mi_assert_internal(mi_meta_page_of_ptr(p,NULL) == mpage);
- mi_assert_internal(block_idx + block_count <= MI_META_BLOCKS_PER_PAGE);
- mi_assert_internal(mi_bbitmap_is_clearN(&mpage->blocks_free, block_idx, block_count));
- // we zero on free (and on the initial page allocation) so we don't need a "dirty" map
- _mi_memzero_aligned(mi_meta_block_start(mpage, block_idx), block_count*MI_META_BLOCK_SIZE);
- mi_bbitmap_setN(&mpage->blocks_free, block_idx, block_count);
- }
- else {
- _mi_arenas_free(p,size,memid);
- }
-}
-
-// used for debug output
-bool _mi_meta_is_meta_page(void* p)
-{
- mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t, &mi_meta_pages);
- mi_meta_page_t* mpage = mpage0;
- while (mpage != NULL) {
- if ((void*)mpage == p) return true;
- mpage = mi_meta_page_next(mpage);
- }
- return false;
-}
diff --git a/vendored/mimalloc/src/arena.c b/vendored/mimalloc/src/arena.c
index a5686d554..1b7c8cb33 100644
--- a/vendored/mimalloc/src/arena.c
+++ b/vendored/mimalloc/src/arena.c
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2019-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2019-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -21,9 +21,10 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo
#include "mimalloc.h"
#include "mimalloc/internal.h"
-#include "mimalloc/prim.h"
+#include "mimalloc/prim-tls.h"
#include "bitmap.h"
+
/* -----------------------------------------------------------
Arena id's
----------------------------------------------------------- */
@@ -86,36 +87,56 @@ static bool mi_heap_has_page(mi_heap_t* heap, mi_arena_t* arena, mi_page_t* page
#endif
size_t mi_arena_min_alignment(void) {
- return MI_ARENA_SLICE_ALIGN;
+ return MI_ARENA_ALIGNMENT;
}
size_t mi_arena_min_size(void) {
return MI_ARENA_MIN_SIZE;
}
-static size_t mi_arena_max_object_size(void) {
+// slices reserved for page meta info at the start of aligned chunks
+static size_t mi_arena_page_meta_aligned_slice_count(void) {
+ #if MI_PAGE_META_IS_ALIGNED
+ return _mi_divide_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE);
+ #else
+ return 0;
+ #endif
+}
+
+// fixed limit for the maximum object size in an arena
+static size_t mi_arena_max_fixed_object_size(void) {
+ #if MI_PAGE_META_IS_ALIGNED
+ return (MI_PAGE_META_ALIGNMENT - _mi_align_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE));
+ #else
+ return (MI_ARENA_MAX_SIZE - MI_ARENA_CHUNK_SIZE); // minus an initial chunk to accommodate meta info
+ #endif
+}
+
+// Maximum object size allowed to be allocated in an arena
+size_t mi_arena_max_object_size(void) {
size_t max_size = mi_option_get_size(mi_option_arena_max_object_size);
max_size = _mi_align_up(max_size, MI_ARENA_SLICE_SIZE);
if (max_size <= MI_ARENA_MIN_OBJ_SIZE) {
return MI_ARENA_MIN_OBJ_SIZE;
}
- else if (max_size >= MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE) { // minus a bchunk to accommodate meta info
- return (MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE);
+ else if (max_size >= mi_arena_max_fixed_object_size()) {
+ return mi_arena_max_fixed_object_size();
}
else {
return max_size;
}
}
-mi_decl_nodiscard static bool mi_arena_commit(mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t already_committed) {
+mi_decl_nodiscard static bool mi_arena_commit(mi_subproc_t* subproc, mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t stat_already_committed) {
+ mi_assert_internal(subproc!=NULL);
if (arena != NULL && arena->commit_fun != NULL) {
return (*arena->commit_fun)(true, start, size, is_zero, arena->commit_fun_arg);
}
- else if (already_committed > 0) {
- return _mi_os_commit_ex(start, size, is_zero, already_committed);
+ else if (stat_already_committed > 0) {
+ return _mi_os_commit_ex(subproc, start, size, is_zero, stat_already_committed);
}
else {
- return _mi_os_commit(start, size, is_zero);
+ return _mi_os_commit(subproc, start, size, is_zero);
}
}
@@ -133,7 +154,7 @@ static size_t mi_arena_size(mi_arena_t* arena) {
// Start of the arena memory area
static uint8_t* mi_arena_start(mi_arena_t* arena) {
- return ((uint8_t*)arena);
+ return ((uint8_t*)arena->start);
}
// Start of a slice
@@ -144,17 +165,22 @@ uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) {
mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) {
mi_assert_internal(slice_index < arena->slice_count);
+ #if MI_PAGE_META_IS_ALIGNED
+ mi_page_t* const page = _mi_aligned_ptr_page(mi_arena_slice_start(arena,slice_index)); // todo: optimize?
+ return page;
+ #else
if (arena->pages_meta != NULL) {
mi_page_t* const page = &arena->pages_meta[slice_index];
- #if MI_PAGE_META_ALIGNED_FREE_SMALL
+ #if MI_PAGE_META_SMALL_IS_ALIGNED
// pages with small blocks still have the page at the start of the slice (and set the `block_size` in pages_meta to 0)
if (page->block_size>0) return page;
#else
return page;
#endif
- }
- // fall through (for MI_PAGE_META_ALIGNED_FREE_SMALL)
+ }
+ // fall through (for MI_PAGE_META_SMALL_IS_ALIGNED)
return (mi_page_t*)mi_arena_slice_start(arena,slice_index);
+ #endif
}
// Arena area
@@ -207,7 +233,6 @@ static size_t mi_page_full_size(mi_page_t* page) {
}
}
-
/* -----------------------------------------------------------
Arena Allocation
----------------------------------------------------------- */
@@ -215,6 +240,8 @@ static size_t mi_page_full_size(mi_page_t* page) {
static mi_decl_noinline void* mi_arena_try_alloc_at(
mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid)
{
+ mi_assert_internal(arena!=NULL);
+ mi_assert_internal(slice_count>0);
size_t slice_index;
if (!mi_bbitmap_try_find_and_clearN(arena->slices_free, tseq, slice_count, &slice_index)) return NULL;
@@ -231,6 +258,10 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
mi_assert_internal(already_dirty <= touched_slices);
touched_slices -= already_dirty;
}
+ else {
+ // todo: properly count touched pages with a separate bitmap?
+ touched_slices = 0;
+ }
// set commit state
if (commit) {
@@ -239,7 +270,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
if (already_committed < slice_count) {
// not all committed, try to commit now
bool commit_zero = false;
- if (!mi_arena_commit(arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed))) {
+ if (!mi_arena_commit(arena->subproc, arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(already_committed))) {
// if the commit fails, release ownership, and return NULL;
// note: this does not roll back dirty bits but that is ok.
mi_bbitmap_setN(arena->slices_free, slice_index, slice_count);
@@ -264,7 +295,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
}
else {
// already fully committed.
- _mi_os_reuse(p, mi_size_of_slices(slice_count));
+ _mi_os_reuse(arena->subproc, p, mi_size_of_slices(slice_count));
// if the OS has overcommit, and this is the first time we access these pages, then
// count the commit now (as at arena reserve we didn't count those commits as these are on-demand)
if (_mi_os_has_overcommit() && touched_slices > 0 && !arena->memid.is_pinned /* huge pages, issue #1236 */) {
@@ -424,24 +455,21 @@ static size_t mi_arena_start_idx(mi_heap_t* heap, size_t tseq, size_t arena_cycl
const size_t _arena_count = mi_arenas_get_count(heap->subproc); \
const size_t _arena_cycle = (_arena_count == 0 ? 0 : _arena_count - 1); /* first search the arenas below the last one */ \
/* always start searching in the arena's below the max */ \
- const size_t _start = mi_arena_start_idx(heap,tseq,_arena_cycle); \
+ const size_t _start = (req_arena==NULL ? mi_arena_start_idx(heap,tseq,_arena_cycle) : ((mi_arena_t*)req_arena)->arena_idx); \
+ mi_assert_internal(_start <= _arena_count); \
for (size_t _i = 0; _i < _arena_count; _i++) { \
mi_arena_t* name_arena; \
- if (req_arena != NULL) { \
- name_arena = req_arena; /* if there is a specific req_arena, only search that one */\
- if (_i > 0) break; /* only once */ \
+ size_t _idx; \
+ if (_i < _arena_cycle) { \
+ _idx = _i + _start; \
+ if (_idx >= _arena_cycle) { _idx -= _arena_cycle; } /* adjust so we rotate through the cycle */ \
} \
else { \
- size_t _idx; \
- if (_i < _arena_cycle) { \
- _idx = _i + _start; \
- if (_idx >= _arena_cycle) { _idx -= _arena_cycle; } /* adjust so we rotate through the cycle */ \
- } \
- else { \
- _idx = _i; /* remaining arena's after the cycle */ \
- } \
- name_arena = mi_arena_from_index(heap->subproc,_idx); \
+ _idx = _i; /* remaining arena's after the cycle */ \
} \
+ name_arena = mi_arena_from_index(heap->subproc,_idx); \
+ if (req_arena != NULL && name_arena != req_arena && \
+ (name_arena == NULL || name_arena->parent != req_arena)) continue; /* only the requested arena or its children */ \
if (name_arena != NULL) \
{
@@ -512,7 +540,7 @@ static mi_decl_noinline void* mi_arenas_try_alloc(
if (req_arena != NULL) return NULL;
// don't create arena's while preloading (todo: or should we?)
- if (_mi_preloading()) return NULL;
+ // if (_mi_preloading()) return NULL;
// don't create arena's if OS allocation is disallowed
if (mi_option_is_enabled(mi_option_disallow_os_alloc)) return NULL;
@@ -540,6 +568,7 @@ static mi_decl_noinline void* mi_arenas_try_alloc(
// Allocate from the OS (if allowed)
static void* mi_arena_os_alloc_aligned(
+ mi_subproc_t* subproc,
size_t size, size_t alignment, size_t align_offset,
bool commit, bool allow_large,
mi_arena_id_t req_arena_id, mi_memid_t* memid)
@@ -551,10 +580,10 @@ static void* mi_arena_os_alloc_aligned(
}
if (align_offset > 0) {
- return _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, memid);
+ return _mi_os_alloc_aligned_at_offset(subproc, size, alignment, align_offset, commit, allow_large, memid);
}
else {
- return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);
+ return _mi_os_alloc_aligned(subproc, size, alignment, commit, allow_large, memid);
}
}
@@ -579,7 +608,7 @@ void* _mi_arenas_alloc_aligned( mi_heap_t* heap,
}
// fall back to the OS
- void* p = mi_arena_os_alloc_aligned(size, alignment, align_offset, commit, allow_large, req_arena, memid);
+ void* p = mi_arena_os_alloc_aligned(heap->subproc, size, alignment, align_offset, commit, allow_large, req_arena, memid);
return p;
}
@@ -596,10 +625,10 @@ void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_lar
// release ownership of a page. This may free the page if all blocks were concurrently
// freed in the meantime. Returns true if the page was freed.
-static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theap) {
+static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theapx) {
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_is_abandoned(page));
- mi_assert_internal(_mi_thread_id()==current_theap->tld->thread_id);
+ mi_assert_internal(mi_theap_matches_thread(current_theapx));
mi_thread_free_t tf_new;
mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free);
do {
@@ -607,14 +636,14 @@ static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theap)
while mi_unlikely(mi_tf_block(tf_old) != NULL) {
_mi_page_free_collect(page, false); // update used
if (mi_page_all_free(page)) { // it may become free just before unowning it
- _mi_arenas_page_unabandon(page, current_theap);
- _mi_arenas_page_free(page, current_theap);
+ _mi_arenas_page_unabandon(page, current_theapx);
+ _mi_arenas_page_free(page, current_theapx);
return true;
}
tf_old = mi_atomic_load_relaxed(&page->xthread_free);
}
mi_assert_internal(mi_tf_block(tf_old)==NULL);
- tf_new = mi_tf_create(NULL, false);
+ tf_new = mi_tf_create(NULL, false);
} while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new));
return false;
}
@@ -655,7 +684,8 @@ static mi_arena_t* mi_page_arena_pages(mi_page_t* page, size_t* slice_index, siz
mi_arena_t* const arena = mi_arena_from_memid(page->memid, slice_index, slice_count);
mi_assert_internal(arena != NULL);
if (parena_pages != NULL) {
- mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(mi_page_heap(page), arena);
+ mi_heap_t* heap = mi_page_heap(page);
+ mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena);
mi_assert_internal(arena_pages != NULL);
mi_assert_internal(slice_index==NULL || mi_bitmap_is_set(arena_pages->pages, *slice_index));
*parena_pages = arena_pages;
@@ -697,7 +727,9 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl
MI_UNUSED(slice_count);
const size_t bin = _mi_bin(block_size);
- mi_assert_internal(bin < MI_BIN_COUNT);
+ if (bin >= MI_ARENA_BIN_COUNT) {
+ return NULL; // singleton page size
+ }
// any abandoned in our size class?
mi_assert_internal(heap != NULL);
@@ -729,7 +761,7 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl
_mi_page_free_collect(page, false); // update `used` count
mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
- mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+ mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
@@ -743,8 +775,11 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl
return NULL;
}
-static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid) {
- MI_UNUSED_RELEASE(block_size);
+static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t max_page_meta_count, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid, mi_arena_pages_t** parena_pages ) {
+ MI_UNUSED(max_page_meta_count);
+ mi_assert_internal(parena_pages!=NULL);
+
+ *parena_pages = NULL;
const bool allow_large = (MI_SECURE < 5); // 5 = guard page at end of each arena page
const size_t page_alignment = MI_ARENA_SLICE_ALIGN;
@@ -764,82 +799,173 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_
start = (uint8_t*)mi_arenas_try_alloc(heap, slice_count, page_alignment, commit, allow_large, req_arena, tld->thread_seq, numa_node, memid);
if (start != NULL) {
mi_arena_pages_t* const arena_pages = mi_heap_ensure_arena_pages(heap, memid->mem.arena.arena);
+ *parena_pages = arena_pages;
if (arena_pages==NULL) {
- _mi_arenas_free(start, mi_size_of_slices(slice_count), *memid); // roll back
+ _mi_arenas_free(heap->subproc, start, mi_size_of_slices(slice_count), *memid); // roll back
start = NULL;
}
else {
// note: the following assert should hold if we could check it atomically, but in a concurrent setting we may already allocate in slice_count
// mi_assert_internal(mi_bitmap_is_clearN(arena_pages->pages, memid->mem.arena.slice_index, memid->mem.arena.slice_count));
mi_assert_internal(mi_bitmap_is_clear(arena_pages->pages, memid->mem.arena.slice_index));
- mi_bitmap_set(arena_pages->pages, memid->mem.arena.slice_index);
+ // don't set yet: mi_bitmap_set(arena_pages->pages, memid->mem.arena.slice_index);
}
}
}
// otherwise fall back to the OS
if (start == NULL) {
+ #if MI_PAGE_META_IS_ALIGNED
+ size_t page_offset; // offset in the block for the page area
+ uint8_t* os_start;
+ if (block_alignment < MI_PAGE_META_ALIGNMENT) {
+ page_offset = (block_alignment < MI_PAGE_ALIGN ? MI_PAGE_ALIGN : block_alignment);
+ os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, MI_PAGE_META_ALIGNMENT, 0 /* align offset */, false /* commit */, false /* allow large */, req_arena, memid);
+ }
+ else {
+ // if we allow alignment >= MI_PAGE_META_ALIGNMENT we need to substract 1 from a pointer
+ // in _mi_aligned_ptr_page (and test for (intptr_t)p < 0 instead of NULL). We avoid this by limiting the max alignment.
+ _mi_warning_message("requested alignment is too large (%zu KiB)\n", block_alignment / MI_KiB);
+ errno = EINVAL;
+ return NULL;
+ // page_offset = MI_PAGE_META_ALIGNMENT;
+ // os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, block_alignment, MI_PAGE_META_ALIGNMENT /* align offset */, false /* commit */, false /* allow large */, req_arena, memid);
+ }
+ if (os_start==NULL) return NULL;
+ // commit page info and page
+ const size_t min_page_count = _mi_divide_up(page_offset + MI_ARENA_SLICE_SIZE,MI_ARENA_SLICE_SIZE) + max_page_meta_count;
+ const size_t min_page_meta = min_page_count * sizeof(mi_page_t);
+ mi_assert_internal(min_page_meta < page_offset);
+ bool is_zero;
+ bool ok = mi_arena_commit(heap->subproc,req_arena,os_start,min_page_meta,&is_zero,min_page_meta /* don't count in stats? */);
+ if (ok && commit) {
+ ok = mi_arena_commit(heap->subproc,req_arena,os_start + page_offset,alloc_size,NULL,0);
+ }
+ if (!ok) { _mi_os_free(heap->subproc,os_start,alloc_size+page_offset,*memid); return NULL; }
+ if (!is_zero && !memid->initially_zero) {
+ _mi_memzero_aligned(os_start,min_page_meta);
+ }
+ start = os_start + page_offset;
+ mi_assert_internal(_mi_is_aligned(start,block_alignment));
+ mi_assert_internal(_mi_is_aligned(os_start,MI_PAGE_META_ALIGNMENT));
+ mi_assert_internal(_mi_align_down_ptr(start-1,MI_PAGE_META_ALIGNMENT) == os_start);
+ mi_assert_internal((uint8_t*)_mi_aligned_ptr_page0(start) < os_start + min_page_meta);
+ memid->initially_committed = true; // so we don't commit again
+ #else
if (os_align) {
// note: slice_count already includes the page
- mi_assert_internal(slice_count >= mi_slice_count_of_size(block_size) + mi_slice_count_of_size(page_alignment));
- start = (uint8_t*)mi_arena_os_alloc_aligned(alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid);
+ start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid);
+ mi_assert_internal(_mi_is_aligned(start + page_alignment, block_alignment));
}
else {
- start = (uint8_t*)mi_arena_os_alloc_aligned(alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid);
+ start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid);
}
+ #endif
+ if (start!=NULL) { mi_heap_stat_increase(heap,pages_os_allocated,1); }
}
if (start == NULL) return NULL;
mi_assert_internal(_mi_is_aligned(start, MI_PAGE_ALIGN));
- mi_assert_internal(!os_align || _mi_is_aligned(start + page_alignment, block_alignment));
return start;
}
-static size_t mi_page_block_start(size_t block_size, bool os_align)
+// Only used for non-separate pages
+mi_decl_maybe_unused static size_t mi_page_block_start(size_t block_size, bool os_align)
{
+ size_t offset;
#if MI_GUARDED
// in a guarded build, we align pages with blocks a multiple of an OS page size, to the OS page size
// this ensures that all blocks in such pages are OS page size aligned (which is needed for the guard pages)
const size_t os_page_size = _mi_os_page_size();
mi_assert_internal(MI_PAGE_ALIGN >= os_page_size);
if (!os_align && block_size % os_page_size == 0 && block_size > os_page_size /* at least 2 or more */ ) {
- return _mi_align_up(mi_page_info_size(), os_page_size);
+ offset = _mi_align_up(mi_page_info_size(), os_page_size);
}
else
#endif
if (os_align) {
- return MI_PAGE_ALIGN;
+ offset = MI_PAGE_ALIGN;
}
- else if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) {
+ else if (block_size != 0 && _mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) {
// naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB)
- return _mi_align_up(mi_page_info_size(), block_size);
+ offset = _mi_align_up(mi_page_info_size(), block_size);
+ if (block_size < 64) { offset += 3*block_size; }
+ }
+ else if (block_size != 0 && block_size <= MI_SMALL_SIZE_MAX) {
+ // align small blocks to their size
+ offset = _mi_align_up(mi_page_info_size(), block_size);
}
else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) {
// also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB)
- return _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2);
+ offset = _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2);
}
else {
// otherwise start after the info
- return mi_page_info_size();
+ offset = mi_page_info_size();
+ }
+ return _mi_align_up(offset,MI_MAX_ALIGN_SIZE);
+}
+
+
+// Free a page without modifying page_bin stats
+static void mi_arenas_page_free_prim(mi_page_t* page);
+
+static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_start) {
+ #if MI_PAGE_META_IS_ALIGNED
+ if (memid_slice.memkind == MI_MEM_ARENA || mi_memid_is_os(memid_slice)) {
+ // ensure the meta data is committed
+ if (memid_slice.memkind == MI_MEM_ARENA) {
+ mi_arena_t* const arena = memid_slice.mem.arena.arena;
+ uint8_t* const meta_slices = (uint8_t*)_mi_align_down_ptr(slice_start,MI_PAGE_META_ALIGNMENT);
+ mi_assert_internal(meta_slices >= mi_arena_start(arena));
+ const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE;
+ if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) {
+ // try to commit all page meta slices now
+ const size_t meta_slice_count = mi_arena_page_meta_aligned_slice_count();
+ // the following assertion does not hold in a concurrent setting..
+ // mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, meta_slice_index, meta_slice_count));
+ const size_t commit_size = meta_slice_count * MI_ARENA_SLICE_SIZE;
+ if (!mi_arena_commit(arena->subproc, arena, meta_slices, commit_size, NULL, 0)) {
+ // if the commit fails return NULL
+ return NULL;
+ }
+ // set the commit bits
+ mi_bitmap_setN(arena->slices_committed, meta_slice_index, meta_slice_count, NULL);
+ }
+ }
+ mi_page_t* const page_meta = _mi_aligned_ptr_page0(slice_start);
+ return page_meta;
+ }
+ #else
+ if (memid_slice.memkind == MI_MEM_ARENA) {
+ MI_UNUSED(slice_start);
+ mi_arena_t* const arena = memid_slice.mem.arena.arena;
+ if (arena->pages_meta != NULL) {
+ mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0);
+ return &arena->pages_meta[memid_slice.mem.arena.slice_index];
+ }
}
+ #endif
+ return NULL;
}
// Allocate a fresh page
static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit)
{
- const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);
- const size_t alloc_size = mi_size_of_slices(slice_count);
- mi_memid_t memid = _mi_memid_none();
- uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,block_size,block_alignment,os_align,commit,&memid);
+ const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);
+ const bool singleton = (os_align || block_size > MI_LARGE_MAX_OBJ_SIZE);
+ const size_t max_page_meta_count = (singleton && slice_count > 2 ? 2 : slice_count);
+
+ const size_t alloc_size = mi_size_of_slices(slice_count);
+ mi_memid_t memid = _mi_memid_none();
+ mi_arena_pages_t* arena_pages = NULL;
+ uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,max_page_meta_count,block_alignment,os_align,commit,&memid,&arena_pages);
if (!slice_start) return NULL;
// guard page at the end of mimalloc page?
#if MI_SECURE>=5
mi_assert(alloc_size > _mi_os_secure_guard_page_size());
const size_t page_noguard_size = alloc_size - _mi_os_secure_guard_page_size();
- if (memid.initially_committed) {
- _mi_os_secure_guard_page_set_at(slice_start + page_noguard_size, memid);
- }
#else
const size_t page_noguard_size = alloc_size;
#endif
@@ -850,53 +976,77 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou
size_t block_start = 0;
// allocate page meta info at the arena start?
- if (memid.memkind == MI_MEM_ARENA) {
- mi_arena_t* const arena = memid.mem.arena.arena;
- if (arena->pages_meta != NULL) {
- mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0);
- mi_page_t* const page_meta = &arena->pages_meta[memid.mem.arena.slice_index];
- mi_assert_internal(page_meta->block_size == 0);
- #if MI_PAGE_META_ALIGNED_FREE_SMALL
- // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice,
- // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`)
- if (block_size > MI_SMALL_SIZE_MAX)
+ mi_page_t* const page_meta = mi_arena_page_meta(memid,slice_start);
+ if (page_meta!=NULL) {
+ mi_assert_internal(page_meta->block_size == 0);
+ #if MI_PAGE_META_SMALL_IS_ALIGNED
+ // if `block_size <= MI_SMALL_MAX_OBJ_SIZE` we put the page info in front of the slice,
+ // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`)
+ if (!os_align && block_size <= MI_SMALL_MAX_OBJ_SIZE) {
+ // put page info in front of the slice
+ page = (mi_page_t*)slice_start;
+ block_start = mi_page_block_start(block_size, os_align);
+ }
+ else
+ #endif
+ {
+ page_meta_is_separate = true;
+ page = page_meta;
+ block_start = 0;
+ #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET)
+ #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */
#endif
+ if (block_size >= MI_SIZE_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET &&
+ _mi_is_power_of_two(block_size))
{
- page = page_meta;
- page_meta_is_separate = true;
- block_start = 0;
- #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET)
- #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */
- #endif
- if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) {
- block_start += block_size;
- }
- mi_assert_internal(page->block_size == 0);
- _mi_memzero_aligned(page, sizeof(*page));
+ block_start = _mi_align_up(mi_page_info_size(), block_size); // to maintain natural alignment
+ if (block_size < 64) { block_start += 3*block_size; }
}
+ mi_assert_internal(page->block_size == 0);
+ _mi_memzero_aligned(page, sizeof(*page));
}
}
if (page == NULL) {
+ #if MI_PAGE_META_IS_ALIGNED
+ // can only happen on failing to commit the page meta info
+ _mi_arenas_free(_mi_theap_subproc(theap),slice_start,alloc_size,memid);
+ return NULL;
+ #else
// put page meta info in front of the slice
page = (mi_page_t*)slice_start;
block_start = mi_page_block_start(block_size, os_align);
+ #endif
+ }
+ mi_assert_internal(block_size < MI_MAX_ALIGN_SIZE || block_start % MI_MAX_ALIGN_SIZE == 0);
+ if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) {
+ mi_assert_internal(block_start % block_size == 0); // natural alignment (see also alloc_aligned.c)
}
// commit first block?
size_t commit_size = 0;
if (!memid.initially_committed) {
- commit_size = _mi_align_up(block_start + block_size, MI_PAGE_MIN_COMMIT_SIZE);
+ commit_size = _mi_align_up(block_start + block_size, mi_page_min_commit_size());
if (commit_size > page_noguard_size) { commit_size = page_noguard_size; }
bool is_zero = false;
- if mi_unlikely(!mi_arena_commit( mi_memid_arena(memid), slice_start, commit_size, &is_zero, 0)) {
- _mi_arenas_free(slice_start, alloc_size, memid);
+ if mi_unlikely(!mi_arena_commit( _mi_theap_subproc(theap), mi_memid_arena(memid), slice_start, commit_size, &is_zero, 0)) {
+ _mi_arenas_free(_mi_theap_subproc(theap), slice_start, alloc_size, memid);
return NULL;
}
}
+ // now we can finish initalization and use `mi_arenas_free_page_prim` on error
+
+ // zero initialize the page meta data
if (!memid.initially_zero && !page_meta_is_separate) {
_mi_memzero_aligned(page, sizeof(*page));
}
+ // set the guard page
+ #if MI_SECURE>=5
+ if (memid.initially_committed) {
+ _mi_os_secure_guard_page_set_at(_mi_theap_subproc(theap), slice_start + page_noguard_size, memid);
+ }
+ #endif
+
// claimed free slices: initialize the page partly
if (!memid.initially_zero && memid.initially_committed) {
mi_track_mem_undefined(slice_start, slice_count * MI_ARENA_SLICE_SIZE);
@@ -916,23 +1066,62 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou
const size_t reserved = (os_align ? 1 : (page_noguard_size - block_start) / block_size);
mi_assert_internal(reserved > 0 && reserved <= UINT16_MAX);
- // initialize
- page->reserved = (uint16_t)reserved;
- page->page_start = slice_start + block_start;
+ // initialize the page start
+ uint8_t* const start = slice_start + block_start;
+ mi_assert_internal(start > (uint8_t*)page);
+ page->page_offset = start - (uint8_t*)page;
+
+ // initialize page meta-data
+ page->reserved = (uint16_t)reserved;
page->block_size = block_size;
- page->slice_committed = commit_size;
page->memid = memid;
page->free_is_zero = memid.initially_zero;
+
+ mi_assert_internal((commit && commit_size==0) || (!commit && (commit_size <= UINT16_MAX * _mi_os_page_size())));
+ page->slice_pcommitted = (uint16_t)(commit_size / _mi_os_page_size());
+
+ page->heap = _mi_theap_heap(theap);
+ mi_page_set_theap(page,theap);
+ // mi_assert_internal(mi_page_theap(page) == _mi_heap_theap_peek(page->heap))
+
+ #if MI_PAGE_META_IS_ALIGNED
+ mi_assert_internal(page_meta!=NULL);
+ mi_atomic_store_ptr_release(mi_page_t,&page_meta->self,page);
+ if (slice_count > 1) {
+ // at least two for large singleton blocks as guard pages can have a large offset beyond a single slice
+ for(size_t i = 1; i < max_page_meta_count; i++) {
+ mi_assert_internal(page_meta[i].block_size == 0);
+ mi_atomic_store_ptr_release(mi_page_t,&page_meta[i].self,page);
+ }
+ }
+ #if MI_DEBUG>1
+ mi_page_t* pstart = _mi_aligned_ptr_page0(slice_start);
+ mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pstart->self)==page);
+ if (reserved>1) {
+ mi_page_t* pend = _mi_aligned_ptr_page0(slice_start + (slice_count*MI_ARENA_SLICE_SIZE) - 1);
+ mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pend->self)==page);
+ }
+ #endif
+ #endif
+
mi_assert_internal(page->free==NULL);
mi_assert_internal(page_meta_is_separate == mi_page_meta_is_separated(page));
mi_assert_internal(mi_page_slice_start(page) == slice_start);
+ mi_assert_internal(mi_page_size(page) <= page_noguard_size);
+
+
+ // now register in the arena_pages
+ if (arena_pages!=NULL) {
+ mi_assert_internal(memid.memkind == MI_MEM_ARENA);
+ mi_bitmap_set(arena_pages->pages, memid.mem.arena.slice_index);
+ }
// and own it
mi_page_claim_ownership(page);
// register in the page map
if mi_unlikely(!_mi_page_map_register(page)) {
- _mi_arenas_free( slice_start, alloc_size, memid );
+ mi_arenas_page_free_prim(page);
return NULL;
}
@@ -943,7 +1132,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou
mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page),MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
mi_assert_internal(mi_page_block_size(page) == block_size);
- mi_assert_internal(mi_page_is_abandoned(page));
+ // mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(mi_page_is_owned(page));
return page;
@@ -960,14 +1149,15 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c
// 2. find a free block, potentially allocating a new arena
const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand);
- const bool commit = (slice_count <= mi_slice_count_of_size(MI_PAGE_MIN_COMMIT_SIZE) || // always commit small pages
- (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0));
+ const bool commit = (mi_page_min_commit_size() >= slice_count * MI_ARENA_SLICE_SIZE || // always commit small pages
+ (slice_count >= mi_slice_count_of_size(UINT16_MAX * _mi_os_page_size())) || // always commit pages too large to hold a 32-bit slice_committed
+ (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0));
page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit);
if (page == NULL) return NULL;
mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count);
if (!_mi_page_init(theap, page)) {
- _mi_arenas_free( page, mi_page_full_size(page), page->memid);
+ _mi_arenas_page_free(page,theap);
return NULL;
}
@@ -977,8 +1167,12 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c
// Allocate a page containing one block (very large, or with large alignment)
static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment)
{
+ #if MI_PAGE_META_IS_ALIGNED
+ const size_t info_size = 0;
+ #else
const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);
- const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size());
+ const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size());
+ #endif
#if MI_SECURE < 2
const size_t slice_count = mi_slice_count_of_size(info_size + block_size);
#else
@@ -990,7 +1184,7 @@ static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block
mi_assert(page->reserved == 1);
if (!_mi_page_init(theap, page)) {
- _mi_arenas_free( page, mi_page_full_size(page), page->memid);
+ _mi_arenas_page_free(page,theap);
return NULL;
}
@@ -1000,6 +1194,8 @@ static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block
mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) {
mi_page_t* page;
+ // semi static assert: ensure that all non-singleton block size bins are covered.
+ mi_assert(_mi_bin(MI_LARGE_MAX_OBJ_SIZE) < MI_ARENA_BIN_COUNT);
if mi_unlikely(block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {
mi_assert_internal(_mi_is_power_of_two(block_alignment));
page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment);
@@ -1029,25 +1225,13 @@ mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t bl
return page;
}
-void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {
+static void mi_arenas_page_free_prim(mi_page_t* page) {
mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_all_free(page));
- mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(page->next==NULL && page->prev==NULL);
- mi_assert_internal(current_theapx == NULL || _mi_thread_id()==current_theapx->tld->thread_id);
-
- if (current_theapx != NULL) {
- mi_theap_stat_decrease(current_theapx, page_bins[_mi_page_stats_bin(page)], 1);
- mi_theap_stat_decrease(current_theapx, pages, 1);
- }
- else {
- mi_heap_t* const heap = mi_page_heap(page);
- mi_heap_stat_decrease(heap, page_bins[_mi_page_stats_bin(page)], 1);
- mi_heap_stat_decrease(heap, pages, 1);
- }
-
+
#if MI_DEBUG>1
if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {
size_t bin = _mi_bin(mi_page_block_size(page));
@@ -1056,8 +1240,8 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {
mi_arena_pages_t* arena_pages = NULL;
mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);
mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
- mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
- mi_assert_internal(mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1));
+ mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+ mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1));
mi_assert_internal(mi_bitmap_is_setN(arena_pages->pages, slice_index, 1));
// note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may
// be (temporarily) not true if the free happens while trying to reclaim
@@ -1065,34 +1249,38 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {
}
#endif
+ // unregister page
+ _mi_page_map_unregister(page);
+
// recommit guard page at the end?
// we must do this since we may later allocate large spans over this page and cannot have a guard page in between
#if MI_SECURE >= 5
if (!page->memid.is_pinned) {
- _mi_os_secure_guard_page_reset_before(mi_page_slice_start(page) + mi_page_full_size(page), page->memid);
+ _mi_os_secure_guard_page_reset_before(mi_page_subproc(page), mi_page_slice_start(page) + mi_page_full_size(page), page->memid);
}
#endif
- // unregister page
- _mi_page_map_unregister(page);
+ // and free
if (page->memid.memkind == MI_MEM_ARENA) {
mi_arena_pages_t* arena_pages;
size_t slice_index;
size_t slice_count; MI_UNUSED(slice_count);
mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);
mi_assert_internal(arena_pages!=NULL);
+ mi_assert_internal(arena->subproc == mi_page_subproc(page));
mi_bitmap_clear(arena_pages->pages, slice_index);
- if (page->slice_committed > 0) {
+ const size_t slice_committed = mi_page_slice_committed(page);
+ if (slice_committed > 0) {
// if committed on-demand, set the commit bits to account commit properly
- mi_assert_internal(mi_page_full_size(page) >= page->slice_committed);
- const size_t total_slices = page->slice_committed / MI_ARENA_SLICE_SIZE; // conservative
+ mi_assert_internal(mi_page_full_size(page) >= slice_committed);
+ const size_t total_slices = slice_committed / MI_ARENA_SLICE_SIZE; // conservative
//mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, slice_index, total_slices));
mi_assert_internal(slice_count >= total_slices);
if (total_slices > 0) {
mi_bitmap_setN(arena->slices_committed, slice_index, total_slices, NULL);
}
// any left over?
- const size_t extra = page->slice_committed % MI_ARENA_SLICE_SIZE;
+ const size_t extra = slice_committed % MI_ARENA_SLICE_SIZE;
if (extra > 0) {
// pretend it was decommitted already
mi_subproc_stat_decrease(arena->subproc, committed, extra);
@@ -1102,60 +1290,91 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {
mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
}
}
+ else {
+ mi_heap_stat_decrease(page->heap, pages_os_allocated, 1);
+ }
if (mi_page_meta_is_separated(page)) { page->block_size = 0; } // for assertion checking
- _mi_arenas_free( mi_page_slice_start(page), mi_page_full_size(page), page->memid);
+ _mi_arenas_free( mi_page_subproc(page), mi_page_slice_start(page), mi_page_full_size(page), page->memid);
+}
+
+void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {
+ mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+ mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+ mi_assert_internal(mi_page_is_owned(page));
+ mi_assert_internal(mi_page_all_free(page));
+ mi_assert_internal(mi_page_is_abandoned(page));
+ mi_assert_internal(page->next==NULL && page->prev==NULL);
+ mi_assert_internal(mi_theap_matches_thread(current_theapx));
+
+ mi_heap_t* const heap = mi_page_heap(page);
+ mi_theapx_stat_decrease(heap, current_theapx, page_bins[_mi_page_stats_bin(page)], 1);
+ mi_theapx_stat_decrease(heap, current_theapx, pages, 1);
+ _mi_page_free_collect(page,false); // update used count for cross-thread free's
+ _mi_page_update_stats(page); // and update the stats
+ mi_arenas_page_free_prim(page);
}
/* -----------------------------------------------------------
Arena abandon
----------------------------------------------------------- */
-void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) {
+void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) {
mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(!mi_page_all_free(page));
mi_assert_internal(page->next==NULL && page->prev == NULL);
- mi_assert_internal(_mi_thread_id()==current_theap->tld->thread_id);
+ mi_assert_internal(mi_theap_matches_thread(current_theapx));
// mi_assert_internal(current_theap == _mi_page_associated_theap(page));
- mi_heap_t* heap = mi_page_heap(page); mi_assert_internal(heap==_mi_theap_heap(current_theap));
+ // note: somewhat expensive to update here, but might be good as then we attribute
+ // the current allocations/frees to the current thread/theap. Otherwise it might be
+ // reclaimed later in another thread/theap and those allocations/frees get attributed there...
+ _mi_page_update_stats(page);
+
+ // add to abandoned?
+ mi_heap_t* heap = mi_page_heap(page);
if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {
// make available for allocations
size_t bin = _mi_bin(mi_page_block_size(page));
- size_t slice_index;
- size_t slice_count;
- mi_arena_pages_t* arena_pages = NULL;
- mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena);
-
- mi_assert_internal(!mi_page_is_singleton(page));
- mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
- mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
- mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
-
- mi_page_set_abandoned_mapped(page);
- const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index);
- MI_UNUSED(was_clear); mi_assert_internal(was_clear);
- mi_atomic_increment_relaxed(&heap->abandoned_count[bin]);
- mi_theap_stat_increase(current_theap, pages_abandoned, 1);
+ mi_assert_internal(bin < MI_ARENA_BIN_COUNT);
+ if (bin < MI_ARENA_BIN_COUNT) { // paranoia
+ size_t slice_index;
+ size_t slice_count;
+ mi_arena_pages_t* arena_pages = NULL;
+ mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena);
+
+ mi_assert_internal(!mi_page_is_singleton(page));
+ mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
+ mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+ mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
+
+ mi_page_set_abandoned_mapped(page);
+ const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index);
+ MI_UNUSED(was_clear); mi_assert_internal(was_clear);
+ mi_atomic_increment_relaxed(&heap->abandoned_count[bin]);
+ mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1);
+ mi_abandoned_page_unown(page, current_theapx);
+ return;
+ }
}
- else {
- // page is full (or a singleton), or the page is OS/externally allocated
- // leave as is; it will be reclaimed when an object is free'd in the page
- // but for non-arena pages, add to the subproc list so these can be visited
- if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) {
- mi_lock(&heap->os_abandoned_pages_lock) {
- // push in front
- page->prev = NULL;
- page->next = heap->os_abandoned_pages;
- if (page->next != NULL) { page->next->prev = page; }
- heap->os_abandoned_pages = page;
- }
+ // otherwise,
+ // page is full (or a singleton), or the page is OS/externally allocated
+ // leave as is; it will be reclaimed when an object is free'd in the page
+ // but for non-arena pages, add to the subproc list so these can be visited
+ if (page->memid.memkind != MI_MEM_ARENA) {
+ mi_lock(&heap->os_abandoned_pages_lock) {
+ // push in front
+ page->prev = NULL;
+ page->next = heap->os_abandoned_pages;
+ if (page->next != NULL) { page->next->prev = page; }
+ heap->os_abandoned_pages = page;
}
- mi_theap_stat_increase(current_theap, pages_abandoned, 1);
+ mi_theapx_stat_increase(heap, current_theapx, pages_os_abandoned, 1);
}
- mi_abandoned_page_unown(page, current_theap);
+ mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1);
+ mi_abandoned_page_unown(page, current_theapx);
}
@@ -1174,16 +1393,13 @@ bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) {
}
else {
// do not use _mi_heap_theap as we may call this during shutdown of threads and don't want to reinitialize the theap
- mi_theap_t* const theap = _mi_page_associated_theap_peek(page);
- if (theap == NULL) {
- return false;
- }
- else {
- mi_theap_stat_counter_increase(theap, pages_reabandon_full, 1);
- mi_theap_stat_adjust_decrease(theap, pages_abandoned, 1); // adjust as we are not abandoning fresh
- _mi_arenas_page_abandon(page, theap);
- return true;
- }
+ mi_theap_t* const theapx = _mi_page_associated_theap_peek(page); // can be NULL
+ mi_heap_t* const heap = mi_page_heap(page);
+ // if (theapx==NULL) return false;
+ mi_theapx_stat_counter_increase(heap, theapx, pages_reabandon_full, 1);
+ mi_theapx_stat_adjust_decrease(heap, theapx, pages_abandoned, 1); // adjust as we are not abandoning fresh
+ _mi_arenas_page_abandon(page, theapx);
+ return true;
}
}
@@ -1193,30 +1409,31 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) {
mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_is_abandoned(page));
- mi_assert_internal(current_theapx==NULL || _mi_thread_id()==current_theapx->tld->thread_id);
+ mi_assert_internal(mi_theap_matches_thread(current_theapx));
mi_heap_t* const heap = mi_page_heap(page);
if (mi_page_is_abandoned_mapped(page)) {
mi_assert_internal(page->memid.memkind==MI_MEM_ARENA);
// remove from the abandoned map
- size_t bin = _mi_bin(mi_page_block_size(page));
+ const size_t bin = _mi_bin(mi_page_block_size(page));
+ mi_assert_internal(bin < MI_ARENA_BIN_COUNT);
size_t slice_index;
size_t slice_count;
mi_arena_pages_t* arena_pages;
mi_arena_t* arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena);
mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
- mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+ mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
// this busy waits until a concurrent reader (from alloc_abandoned) is done
- mi_bitmap_clear_once_set(arena_pages->pages_abandoned[bin], slice_index);
+ mi_bitmap_clear_once_set(arena->subproc, arena_pages->pages_abandoned[bin], slice_index);
mi_page_clear_abandoned_mapped(page);
mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]);
}
else {
// page is full (or a singleton), page is OS allocated
// if not an arena page, remove from the subproc os pages list
- if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) {
+ if (page->memid.memkind != MI_MEM_ARENA) {
mi_lock(&heap->os_abandoned_pages_lock) {
if (page->prev != NULL) { page->prev->next = page->next; }
if (page->next != NULL) { page->next->prev = page->prev; }
@@ -1224,14 +1441,10 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) {
page->next = NULL;
page->prev = NULL;
}
+ mi_theapx_stat_decrease(heap, current_theapx, pages_os_abandoned, 1);
}
}
- if (current_theapx!=NULL) {
- mi_theap_stat_decrease(current_theapx, pages_abandoned, 1);
- }
- else {
- mi_heap_stat_decrease(heap, pages_abandoned, 1);
- }
+ mi_theapx_stat_decrease(heap, current_theapx, pages_abandoned, 1);
}
@@ -1241,7 +1454,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) {
static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slices);
static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq);
-void _mi_arenas_free(void* p, size_t size, mi_memid_t memid) {
+void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) {
if (p==NULL) return;
if (size==0) return;
@@ -1250,26 +1463,28 @@ void _mi_arenas_free(void* p, size_t size, mi_memid_t memid) {
if (mi_memkind_is_os(memid.memkind)) {
// was a direct OS allocation, pass through
- _mi_os_free(p, size, memid);
+ _mi_os_free(subproc, p, size, memid);
}
else if (memid.memkind == MI_MEM_ARENA) {
// allocated in an arena
size_t slice_count;
size_t slice_index;
mi_arena_t* arena = mi_arena_from_memid(memid, &slice_index, &slice_count);
+ mi_assert_internal(arena!=NULL);
+ mi_assert_internal(arena->subproc == subproc);
mi_assert_internal((size%MI_ARENA_SLICE_SIZE)==0);
mi_assert_internal((slice_count*MI_ARENA_SLICE_SIZE)==size);
mi_assert_internal(mi_arena_slice_start(arena,slice_index) <= (uint8_t*)p);
mi_assert_internal(mi_arena_slice_start(arena,slice_index) + mi_size_of_slices(slice_count) > (uint8_t*)p);
// checks
if (arena == NULL) {
- _mi_error_message(EINVAL, "trying to free from an invalid arena: %p, size %zu, memid: 0x%zx\n", p, size, memid);
+ _mi_error_message(EINVAL, "trying to free from an invalid arena: %p, size %zu, memkind: 0x%x\n", p, size, memid.memkind);
return;
}
mi_assert_internal(slice_index < arena->slice_count);
mi_assert_internal(slice_index >= mi_arena_info_slices(arena));
if (slice_index < mi_arena_info_slices(arena) || slice_index >= arena->slice_count) {
- _mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memid: 0x%zx\n", p, size, memid);
+ _mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memkind: 0x%x\n", p, size, memid.memkind);
return;
}
@@ -1286,8 +1501,8 @@ void _mi_arenas_free(void* p, size_t size, mi_memid_t memid) {
return;
};
}
- else if (memid.memkind == MI_MEM_META) {
- _mi_meta_free(p, size, memid);
+ else if (memid.memkind == MI_MEM_MALLOC) {
+ _mi_free_subproc_safe(p);
}
else {
// arena was none, external, or static; nothing to do
@@ -1328,10 +1543,10 @@ static bool mi_arenas_contain_ex(const void* p, mi_arena_t* parent) {
return false;
}
-// Is a pointer inside any of our arenas?
-bool _mi_arenas_contain(const void* p) {
- return mi_arenas_contain_ex(p, NULL);
-}
+// // Is a pointer inside any of our arenas?
+// bool _mi_arenas_contain(const void* p) {
+// return mi_arenas_contain_ex(p, NULL);
+// }
// Is a pointer contained in the given arena area?
bool mi_arena_contains(mi_arena_id_t arena_id, const void* p) {
@@ -1357,7 +1572,7 @@ static void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) {
// mi_lock_done(&arena->abandoned_visit_lock);
mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL);
if (mi_memkind_is_os(arena->memid.memkind)) {
- _mi_os_free_ex(mi_arena_start(arena), mi_arena_size(arena), true, arena->memid, subproc); // pass `subproc` to avoid accessing the theap pointer (in `_mi_subproc()`)
+ _mi_os_free_ex(subproc, mi_arena_start(arena), mi_arena_size(arena), true, arena->memid);
}
}
}
@@ -1430,13 +1645,17 @@ static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) {
return size;
}
+static mi_arena_t* mi_arena_info(void* area) {
+ return (mi_arena_t*)((uint8_t*)area + mi_size_of_slices(mi_arena_page_meta_aligned_slice_count()));
+}
+
static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base) {
if (slice_count == 0) slice_count = MI_BCHUNK_BITS;
mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0);
- const size_t base_size = _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE);
+ const size_t base_size = mi_size_of_slices(mi_arena_page_meta_aligned_slice_count()) + _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE);
const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned
const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free
- #if MI_PAGE_META_IS_SEPARATED
+ #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED
const size_t pages_size = slice_count * sizeof(mi_page_t);
#else
const size_t pages_size = 0;
@@ -1457,9 +1676,9 @@ static mi_bitmap_t* mi_arena_bitmap_init(size_t slice_count, uint8_t** base) {
return bitmap;
}
-static mi_bbitmap_t* mi_arena_bbitmap_init(size_t slice_count, uint8_t** base) {
+static mi_bbitmap_t* mi_arena_bbitmap_init(mi_subproc_t* subproc, size_t slice_count, uint8_t** base) {
mi_bbitmap_t* bbitmap = (mi_bbitmap_t*)(*base);
- *base = (*base) + mi_bbitmap_init(bbitmap, slice_count, true /* already zero */);
+ *base = (*base) + mi_bbitmap_init(subproc, bbitmap, slice_count, true /* already zero */);
return bbitmap;
}
@@ -1467,7 +1686,7 @@ static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) {
const size_t slice_count = arena->slice_count;
size_t bitmap_base = 0;
const size_t size = mi_arena_pages_size(slice_count, &bitmap_base);
- mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)mi_heap_zalloc_aligned(mi_heap_main(), size, MI_BCHUNK_SIZE);
+ mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)mi_heap_zalloc_aligned(arena->subproc->heap_main, size, MI_BCHUNK_SIZE);
if (arena_pages==NULL) return NULL;
uint8_t* base = (uint8_t*)arena_pages + bitmap_base;
mi_assert_internal(_mi_is_aligned(base, MI_BCHUNK_SIZE));
@@ -1483,7 +1702,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
int numa_node, bool exclusive,
mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id)
{
- mi_assert_internal(_mi_is_aligned(start,MI_ARENA_SLICE_ALIGN));
+ mi_assert_internal(_mi_is_aligned(start,MI_ARENA_ALIGNMENT));
mi_assert_internal(mi_size_of_slices(slice_count)>=MI_ARENA_MIN_SIZE);
if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now
@@ -1498,13 +1717,11 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
_mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", mi_size_of_slices(slice_count)/MI_KiB, mi_size_of_slices(info_slices+1)/MI_KiB);
return NULL;
}
- else if (info_slices >= MI_ARENA_MAX_CHUNK_OBJ_SLICES) {
- _mi_warning_message("cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)", mi_size_of_slices(slice_count)/MI_MiB, info_slices, MI_ARENA_MAX_CHUNK_OBJ_SLICES);
- return NULL;
- }
-
- mi_arena_t* arena = (mi_arena_t*)start;
-
+ // else if (info_slices >= MI_ARENA_MAX_CHUNK_OBJ_SLICES) {
+ // _mi_warning_message("cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)", mi_size_of_slices(slice_count)/MI_MiB, info_slices, MI_ARENA_MAX_CHUNK_OBJ_SLICES);
+ // return NULL;
+ // }
+
// commit & zero if needed
if (!memid.initially_committed) {
size_t commit_size = mi_size_of_slices(info_slices);
@@ -1512,10 +1729,10 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
if (!memid.is_pinned) { commit_size -= _mi_os_secure_guard_page_size(); }
bool ok = false;
if (commit_fun != NULL) {
- ok = (*commit_fun)(true /* commit */, arena, commit_size, NULL, commit_fun_arg);
+ ok = (*commit_fun)(true /* commit */, start, commit_size, NULL, commit_fun_arg);
}
else {
- ok = _mi_os_commit(arena, commit_size, NULL);
+ ok = _mi_os_commit(subproc, start, commit_size, NULL);
}
if (!ok) {
_mi_warning_message("unable to commit meta-data for OS memory");
@@ -1525,13 +1742,15 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
else if (!memid.is_pinned) {
// if MI_SECURE, set a guard page at the end of the arena info
// todo: this does not respect the commit_fun as the memid is of external memory
- _mi_os_secure_guard_page_set_before((uint8_t*)arena + mi_size_of_slices(info_slices), memid);
+ _mi_os_secure_guard_page_set_before(subproc, (uint8_t*)start + mi_size_of_slices(info_slices), memid);
}
if (!memid.initially_zero) {
- _mi_memzero(arena, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size());
+ _mi_memzero(start, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size());
}
// init
+ mi_arena_t* arena = mi_arena_info(start);
+ arena->start = start;
arena->subproc = subproc;
arena->memid = memid;
arena->is_exclusive = exclusive;
@@ -1551,7 +1770,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
// init bitmaps
uint8_t* base = mi_arena_start(arena) + bitmap_base;
- arena->slices_free = mi_arena_bbitmap_init(slice_count, &base);
+ arena->slices_free = mi_arena_bbitmap_init(subproc, slice_count, &base);
arena->slices_committed = mi_arena_bitmap_init(slice_count, &base);
arena->slices_dirty = mi_arena_bitmap_init(slice_count, &base);
arena->slices_purge = mi_arena_bitmap_init(slice_count, &base);
@@ -1559,7 +1778,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base);
}
- #if MI_PAGE_META_IS_SEPARATED
+ #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED
arena->pages_meta = (mi_page_t*)base;
base += (slice_count * sizeof(mi_page_t));
#else
@@ -1568,7 +1787,23 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
mi_assert_internal(mi_size_of_slices(info_slices) >= (size_t)(base - mi_arena_start(arena)));
// reserve our meta info (and reserve slices outside the memory area)
+ #if MI_PAGE_META_IS_ALIGNED
+ for(size_t i = 0; i < arena->slice_count; i += MI_PAGE_META_ALIGNED_COUNT) {
+ // set all free slices (and skip the slices reserved for the page meta info)
+ const size_t meta_slices = (i==0 ? info_slices : mi_arena_page_meta_aligned_slice_count());
+ const size_t start_idx = (i + meta_slices);
+ size_t count = MI_PAGE_META_ALIGNED_COUNT - meta_slices;
+ if (start_idx < arena->slice_count) {
+ if (count + start_idx > arena->slice_count) {
+ count = arena->slice_count - start_idx;
+ mi_assert_internal(count > 0);
+ }
+ mi_bbitmap_unsafe_setN(arena->slices_free, start_idx, count);
+ }
+ }
+ #else
mi_bbitmap_unsafe_setN(arena->slices_free, info_slices /* start */, arena->slice_count - info_slices);
+ #endif
if (memid.initially_committed) {
mi_bitmap_unsafe_setN(arena->slices_committed, 0, arena->slice_count);
}
@@ -1584,15 +1819,14 @@ static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t s
mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept
{
// checks
- mi_assert(_mi_is_aligned(start, MI_ARENA_SLICE_SIZE));
mi_assert(start!=NULL);
if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); }
if (start==NULL) return false;
- if (!_mi_is_aligned(start, MI_ARENA_SLICE_SIZE)) {
+ if (!_mi_is_aligned(start, MI_ARENA_ALIGNMENT)) {
// we can align the start since the memid tracks the real base of the memory.
- void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_SLICE_SIZE);
+ void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_ALIGNMENT);
const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start;
- if (diff >= size || (size - diff) < MI_ARENA_SLICE_SIZE) {
+ if (diff >= size || (size - diff) < MI_ARENA_ALIGNMENT) {
_mi_warning_message("after alignment, the size of the arena becomes too small (memory at %p with size %zu)\n", start, size);
return false;
}
@@ -1675,12 +1909,18 @@ bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinne
// Reserve a range of regular OS memory
static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) {
if (arena_id != NULL) *arena_id = _mi_arena_id_none();
- size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice
+ if (size <= MI_MAX_ALLOC_SIZE) {
+ size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice
+ }
+ if (size > MI_MAX_ALLOC_SIZE) {
+ _mi_error_message(EOVERFLOW, "memory reservation request is too large (size %zu)\n", size);
+ return ENOMEM;
+ }
mi_memid_t memid;
- void* start = _mi_os_alloc_aligned(size, MI_ARENA_SLICE_ALIGN, commit, allow_large, &memid);
- if (start == NULL) return ENOMEM;
+ void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_ALIGNMENT, commit, allow_large, &memid);
+ if (start == NULL) return ENOMEM;
if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) {
- _mi_os_free_ex(start, size, commit, memid, NULL);
+ _mi_os_free_ex(subproc, start, size, commit, memid);
_mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024));
return ENOMEM;
}
@@ -1756,9 +1996,8 @@ static void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) {
}
static int mi_page_commit_usage(mi_page_t* page) {
- // if (mi_page_size(page) <= MI_PAGE_MIN_COMMIT_SIZE) return 100;
const size_t committed_size = mi_page_committed(page);
- const size_t used_size = page->used * mi_page_block_size(page);
+ const size_t used_size = mi_page_used(page) * mi_page_block_size(page);
return (int)(used_size * 100 / committed_size);
}
@@ -1777,7 +2016,8 @@ static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena,
bit_set_count++;
c = 'p';
color = MI_GRAY;
- if (mi_page_is_singleton(page)) { c = 's'; }
+ if (_mi_meta_is_meta_page(arena->subproc,page)) { c = 'm'; }
+ else if (mi_page_is_singleton(page)) { c = 's'; }
else if (mi_page_is_full(page)) { c = 'f'; }
if (!mi_page_is_abandoned(page)) { c = _mi_toupper(c); }
int commit_usage = mi_page_commit_usage(page);
@@ -1790,8 +2030,13 @@ static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena,
else {
c = '?';
if (bit_of_page > 0) { c = '-'; }
- else if (_mi_meta_is_meta_page(start)) { c = 'm'; color = MI_GRAY; }
+ // else if (_mi_meta_is_meta_page(arena->subproc,start)) { c = 'm'; color = MI_GRAY; }
else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; }
+ #if MI_PAGE_META_IS_ALIGNED
+ else if ((slice_index % MI_PAGE_META_ALIGNED_COUNT) == 0 && (size_t)bit <= mi_arena_page_meta_aligned_slice_count()) {
+ { c = 'i'; color = MI_GRAY; }
+ }
+ #endif
// else if (mi_bitmap_is_setN(arena->pages_purge, slice_index + bit, NULL)) { c = '*'; }
else if (mi_bbitmap_is_setN(arena->slices_free, slice_index+bit,1)) {
if (mi_bitmap_is_set(arena->slices_purge, slice_index + bit)) { c = '~'; color = MI_ORANGE; }
@@ -1898,13 +2143,13 @@ static void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narro
size_t page_total = 0;
for (size_t i = 0; i < max_arenas; i++) {
mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);
- if (arena == NULL) break;
+ if (arena == NULL) continue;
mi_assert(arena->subproc == subproc);
// slice_total += arena->slice_count;
- _mi_raw_message("%sarena %zu at %p: %zu slices (%zu MiB)%s%s, subproc: %p, numa: %i\n",
+ _mi_raw_message("%sarena %zu at %p: %zu slices (%zu MiB)%s%s, subproc: %zu, numa: %i\n",
(arena->parent==NULL ? "" : "(sub)"), i, arena, arena->slice_count, (size_t)(mi_size_of_slices(arena->slice_count)/MI_MiB),
(arena->memid.is_pinned ? ", pinned" : ""), (arena->is_exclusive ? ", exclusive" : ""),
- arena->subproc, arena->numa_node);
+ arena->subproc->subproc_seq, arena->numa_node);
//if (show_inuse) {
// free_total += mi_debug_show_bbitmap("in-use slices", arena->slice_count, arena->slices_free, true, NULL);
//}
@@ -1919,7 +2164,7 @@ static void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narro
// mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena);
// if (arena_pages != NULL)
{
- const char* header1 = "chunks (p:page, f:full, s:singleton, P,F,S:not abandoned, i:arena-info, m:meta-data, ~:free-purgable, _:free-committed, .:free-reserved)";
+ const char* header1 = "chunks (p:page, f:full, s:single, m:meta-data, i:arena-info, P,F,S,M:not abandoned, ~:free-purgable, _:free-committed, .:free-reserved)";
const char* header2 = (narrow ? "\n " : " ");
const char* header3 = "(chunk bin: S:small, M : medium, L : large, X : other)";
page_total += mi_debug_show_chunks(header1, header2, header3, arena->slice_count,
@@ -1951,18 +2196,19 @@ int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_m
if (pages==0) return 0;
if (numa_node < -1) numa_node = -1;
if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count();
+ mi_subproc_t* subproc = _mi_subproc();
size_t hsize = 0;
size_t pages_reserved = 0;
mi_memid_t memid;
- void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid);
+ void* p = _mi_os_alloc_huge_os_pages(subproc, pages, numa_node, timeout_msecs, &pages_reserved, &hsize, &memid);
if (p==NULL || pages_reserved==0) {
_mi_warning_message("failed to reserve %zu GiB huge pages\n", pages);
return ENOMEM;
}
_mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages);
- if (!mi_manage_os_memory_ex2(_mi_subproc(), p, hsize, numa_node, exclusive, memid, NULL, NULL, arena_id)) {
- _mi_os_free(p, hsize, memid);
+ if (!mi_manage_os_memory_ex2(subproc, p, hsize, numa_node, exclusive, memid, NULL, NULL, arena_id)) {
+ _mi_os_free(subproc, p, hsize, memid);
return ENOMEM;
}
return 0;
@@ -2019,7 +2265,14 @@ int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserv
static long mi_arena_purge_delay(void) {
// <0 = no purging allowed, 0=immediate purging, >0=milli-second delay
- return (mi_option_get(mi_option_purge_delay) * mi_option_get(mi_option_arena_purge_mult));
+ const long delay = mi_option_get(mi_option_purge_delay);
+ const long mult = mi_option_get(mi_option_arena_purge_mult);
+ if (delay<0 || mult<0) { return -1; }
+ if (delay==0 || mult==0) { return 0; }
+ size_t total;
+ if (mi_mul_overflow((size_t)delay, (size_t)mult, &total)) { return delay; }
+ if (total > LONG_MAX) { return delay; }
+ return (long)total;
}
// reset or decommit in an arena and update the commit bitmap
@@ -2035,7 +2288,7 @@ static bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_c
size_t already_committed;
mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed); // pretend all committed.. (as we lack a clearN call that counts the already set bits..)
const bool all_committed = (already_committed == slice_count);
- const bool needs_recommit = _mi_os_purge_ex(p, size, all_committed /* allow reset? */, mi_size_of_slices(already_committed), arena->commit_fun, arena->commit_fun_arg);
+ const bool needs_recommit = _mi_os_purge_ex(arena->subproc, p, size, all_committed /* allow reset? */, mi_size_of_slices(already_committed), arena->commit_fun, arena->commit_fun_arg);
if (needs_recommit) {
// no longer committed
@@ -2137,10 +2390,8 @@ static int mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force)
// expired yet?
mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire);
- if (!force) {
- if (expire==0) return -1;
- if (expire > now) return 0;
- }
+ if (expire==0) return -1;
+ if (!force && expire > now) return 0;
// reset expire
mi_atomic_storei64_release(&arena->purge_expire, (mi_msecs_t)0);
@@ -2254,7 +2505,7 @@ bool _mi_heap_visit_blocks(mi_heap_t* heap, bool abandoned_only, bool visit_bloc
mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena);
if (ok && arena_pages != NULL) {
if (abandoned_only) {
- for (size_t bin = 0; ok && bin < MI_BIN_COUNT; bin++) {
+ for (size_t bin = 0; ok && bin < MI_ARENA_BIN_COUNT; bin++) {
// todo: if we had a single abandoned page map as well, this can be faster.
if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) > 0) {
ok = _mi_bitmap_forall_set(arena_pages->pages_abandoned[bin], &mi_heap_visit_page_at, arena, &visit_info);
@@ -2317,25 +2568,35 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are
mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(mi_page_is_owned(page));
- if (page->used==0) {
+ if (mi_page_used(page)==0) {
// free the page
_mi_arenas_page_free(page, theap);
}
else if (heap_target==NULL) {
+ #if MI_GUARDED
+ _mi_page_unguard_all(page); // remove potential interior guard pages
+ #endif
// destroy the page
- page->used=0; // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case
+ mi_page_used_reset(page); // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case
_mi_arenas_page_free(page, theap);
}
else {
// move the page to `heap_target` as an abandoned page
// first remove it from the current heap
const size_t sbin = _mi_page_stats_bin(page);
- size_t slice_index;
- size_t slice_count;
- mi_arena_pages_t* arena_pages = NULL;
- mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);
- mi_assert_internal(mi_bitmap_is_set(arena_pages->pages, slice_index));
- mi_bitmap_clear(arena_pages->pages, slice_index);
+ mi_arena_t* arena = NULL;
+ size_t slice_index = 0;
+ if (page->memid.memkind == MI_MEM_ARENA) {
+ size_t slice_count;
+ mi_arena_pages_t* arena_pages = NULL;
+ arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);
+ mi_assert_internal(mi_bitmap_is_set(arena_pages->pages, slice_index));
+ mi_bitmap_clear(arena_pages->pages, slice_index);
+ }
+ else {
+ // os allocated
+ mi_assert_internal(mi_memid_is_os(page->memid) && page->next == NULL);
+ }
if (theap != NULL) {
mi_theap_stat_decrease(theap, page_bins[sbin], 1);
mi_theap_stat_decrease(theap, pages, 1);
@@ -2347,16 +2608,18 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are
mi_theap_t* theap_target = info->theap_target;
// and then add it to the new target heap
- mi_arena_pages_t* arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);
- if mi_unlikely(arena_pages_target==NULL) {
- // if we cannot allocate this, we move it to the main heap instead (which does not require allocation)
- heap_target = mi_heap_main();
- theap_target = mi_heap_theap(heap_target);
- arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);
- mi_assert_internal(arena_pages_target!=NULL);
- }
- mi_assert_internal(mi_bitmap_is_clear(arena_pages_target->pages, slice_index));
- mi_bitmap_set(arena_pages_target->pages, slice_index);
+ if (arena != NULL) {
+ mi_arena_pages_t* arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);
+ if mi_unlikely(arena_pages_target==NULL) {
+ // if we cannot allocate this, we move it to the main heap instead (which does not require allocation)
+ heap_target = mi_arena_heap_main(arena);
+ theap_target = mi_heap_theap(heap_target); // todo: find through theap_target tld?
+ arena_pages_target = mi_heap_ensure_arena_pages(heap_target, arena);
+ mi_assert_internal(arena_pages_target!=NULL);
+ }
+ mi_assert_internal(mi_bitmap_is_clear(arena_pages_target->pages, slice_index));
+ mi_bitmap_set(arena_pages_target->pages, slice_index);
+ }
page->heap = heap_target;
mi_theap_stat_increase(theap_target, page_bins[sbin], 1);
mi_theap_stat_increase(theap_target, pages, 1);
@@ -2374,7 +2637,7 @@ static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) {
_mi_heap_visit_blocks(heap, false, false, &mi_heap_delete_page, &info);
#if MI_DEBUG>1
// no more arena pages?
- for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
+ for (size_t i = 0; i < MI_MAX_ARENAS; i++) {
mi_arena_pages_t* const arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);
if (arena_pages!=NULL) {
mi_assert_internal(mi_bitmap_is_all_clear(arena_pages->pages));
@@ -2386,7 +2649,7 @@ static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) {
mi_assert_internal(heap->os_abandoned_pages == NULL);
}
// nor arena abandoned pages?
- for (size_t i = 0; i < MI_BIN_COUNT; i++) {
+ for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
mi_assert_internal(mi_atomic_load_relaxed(&heap->abandoned_count[i])==0);
}
#endif
@@ -2394,7 +2657,7 @@ static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) {
void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to) {
if (_mi_is_heap_main(heap_from)) return;
- if (heap_to==NULL) { heap_to = mi_heap_main(); }
+ if (heap_to==NULL) { heap_to = heap_from->subproc->heap_main; }
mi_heap_delete_pages(heap_from, heap_to);
}
@@ -2443,7 +2706,7 @@ mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t*
// adjust abandoned page count
mi_subproc_t* const subproc = arena->subproc;
- for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {
+ for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) {
const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]);
if (count > 0) { mi_atomic_decrement_acq_rel(&subproc->abandoned_count[bin]); }
}
@@ -2503,7 +2766,7 @@ mi_decl_export bool mi_arena_reload(void* start, size_t size, mi_commit_fun_t* c
}
// adjust abandoned page count
- for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {
+ for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) {
const size_t count = mi_bitmap_popcount(arena->pages_abandoned[bin]);
if (count > 0) { mi_atomic_decrement_acq_rel(&arena->subproc->abandoned_count[bin]); }
}
diff --git a/vendored/mimalloc/src/bitmap.c b/vendored/mimalloc/src/bitmap.c
index 67269e8bf..238c4d4c4 100644
--- a/vendored/mimalloc/src/bitmap.c
+++ b/vendored/mimalloc/src/bitmap.c
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2019-2024 Microsoft Research, Daan Leijen
+Copyright (c) 2019-2026 Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -107,9 +107,9 @@ static inline bool mi_bfield_atomic_clear(_Atomic(mi_bfield_t)*b, size_t idx, bo
}
// Clear a bit but only when/once it is set. This is used by concurrent free's while
-// the page is abandoned and mapped. This can incure a busy wait :-( but it should
-// happen almost never (and is accounted for in the stats)
-static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_t idx) {
+// the page is abandoned and mapped. This can incur a busy wait :-( but it should
+// be quite rare (and is accounted for in the stats)
+static inline void mi_bfield_atomic_clear_once_set(mi_subproc_t* subproc, _Atomic(mi_bfield_t)*b, size_t idx) {
mi_assert_internal(idx < MI_BFIELD_BITS);
const mi_bfield_t mask = mi_bfield_mask(1, idx);;
mi_bfield_t old = mi_atomic_load_relaxed(b);
@@ -117,7 +117,7 @@ static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_
if mi_unlikely((old&mask) == 0) {
old = mi_atomic_load_acquire(b);
if ((old&mask)==0) {
- mi_subproc_stat_counter_increase(_mi_subproc(), pages_unabandon_busy_wait, 1);
+ mi_subproc_stat_counter_increase(subproc, pages_unabandon_busy_wait, 1);
}
while ((old&mask)==0) { // busy wait
_mi_prim_thread_yield();
@@ -133,8 +133,7 @@ static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_
// statistics correctly).
static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, size_t* already_set) {
mi_assert_internal(mask != 0);
- mi_bfield_t old = mi_atomic_load_relaxed(b);
- while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)) {}; // try to atomically set the mask bits until success
+ const mi_bfield_t old = mi_atomic_or_acq_rel(b,mask);
if (already_set!=NULL) { *already_set = mi_bfield_popcount(old&mask); }
return ((old&mask) == 0);
}
@@ -143,8 +142,7 @@ static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t
// `all_clear` is set to `true` if the new bfield became zero.
static inline bool mi_bfield_atomic_clear_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, bool* all_clear) {
mi_assert_internal(mask != 0);
- mi_bfield_t old = mi_atomic_load_relaxed(b);
- while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)) {}; // try to atomically clear the mask bits until success
+ const mi_bfield_t old = mi_atomic_and_acq_rel(b,~mask);
if (all_clear != NULL) { *all_clear = ((old&~mask)==0); }
return ((old&mask) == mask);
}
@@ -155,65 +153,79 @@ static inline bool mi_bfield_atomic_setX(_Atomic(mi_bfield_t)*b, size_t* already
return (old==0);
}
-// static inline bool mi_bfield_atomic_clearX(_Atomic(mi_bfield_t)*b, bool* all_clear) {
-// const mi_bfield_t old = mi_atomic_exchange_release(b, mi_bfield_zero());
-// if (all_clear!=NULL) { *all_clear = true; }
-// return (~old==0);
-// }
// ------- mi_bfield_atomic_try_clear ---------------------------------------
-
// Tries to clear a mask atomically, and returns true if the mask bits atomically transitioned from mask to 0
-// and false otherwise (leaving the bit field as is).
+// and false otherwise (leaving the bit field as is). Returns the acquired value back in `expect` (regardless of success).
// `all_clear` is set to `true` if the new bfield became zero.
-static inline bool mi_bfield_atomic_try_clear_mask_of(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, mi_bfield_t expect, bool* all_clear) {
+// `did_temp_clear_bits` is set to `true` some bits were temporarily cleared.
+static inline bool mi_bfield_atomic_try_clear_mask_optimistic(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, mi_bfield_t* previous, bool* did_temp_clear_bits) {
mi_assert_internal(mask != 0);
+ // note: we could also use a strong cas but generally an optimistic atomic and/or is more efficient (at least on arm64)
// try to atomically clear the mask bits
- do {
- if ((expect & mask) != mask) { // are all bits still set?
- if (all_clear != NULL) { *all_clear = (expect == 0); }
- return false;
+ mi_bfield_t old = mi_atomic_and_acq_rel(b,~mask);
+ if (previous!=NULL) { *previous = old; }
+ if mi_likely((old&mask)==mask) {
+ // all bits in the mask transitioned from 1 to 0 atomically
+ return true;
+ }
+ else {
+ // failed to transition all
+ if ((old&mask)!=0) {
+ // restore accidentally cleared ones
+ mi_atomic_or_acq_rel(b, old&mask);
+ if (did_temp_clear_bits!=NULL) { *did_temp_clear_bits = true; }
}
- } while (!mi_atomic_cas_weak_acq_rel(b, &expect, expect & ~mask));
- if (all_clear != NULL) { *all_clear = ((expect & ~mask) == 0); }
- return true;
-}
-
-static inline bool mi_bfield_atomic_try_clear_mask(_Atomic(mi_bfield_t)* b, mi_bfield_t mask, bool* all_clear) {
- mi_assert_internal(mask != 0);
- const mi_bfield_t expect = mi_atomic_load_relaxed(b);
- return mi_bfield_atomic_try_clear_mask_of(b, mask, expect, all_clear);
+ return false;
+ }
}
-// Tries to clear a bit atomically. Returns `true` if the bit transitioned from 1 to 0
-// and `false` otherwise leaving the bfield `b` as-is.
+// Tries to clear a bit atomically. For performance, it assumes there is a good chance of success.
+// Returns `true` if the bit transitioned from 1 to 0
+// and `false` otherwise (leaving the bfield `b` as-is).
// `all_clear` is set to true if the new bfield became zero (and false otherwise)
-mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear(_Atomic(mi_bfield_t)* b, size_t idx, bool* all_clear) {
+mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear_optimistic(_Atomic(mi_bfield_t)* b, size_t idx, mi_bfield_t* previous) {
mi_assert_internal(idx < MI_BFIELD_BITS);
- const mi_bfield_t mask = mi_bfield_one()<<idx;
- return mi_bfield_atomic_try_clear_mask(b, mask, all_clear);
+ const mi_bfield_t mask = ((mi_bfield_t)1<<idx);
+ return mi_bfield_atomic_try_clear_mask_optimistic(b, mask, previous, NULL); // single bit never clears temporarily
}
-// Tries to clear a byte atomically, and returns true if the byte atomically transitioned from 0xFF to 0
+// Tries to clear a byte atomically. For performance, it assumes there is a good chance of success.
+// Returns true if the byte atomically transitioned from 0xFF to 0
// `all_clear` is set to true if the new bfield became zero (and false otherwise)
-mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear8(_Atomic(mi_bfield_t)*b, size_t idx, bool* all_clear) {
+mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear8_optimistic(_Atomic(mi_bfield_t)*b, size_t idx, mi_bfield_t* previous, bool* did_temp_clear_bits) {
mi_assert_internal(idx < MI_BFIELD_BITS);
mi_assert_internal((idx%8)==0);
const mi_bfield_t mask = ((mi_bfield_t)0xFF)<<idx;
- return mi_bfield_atomic_try_clear_mask(b, mask, all_clear);
+ return mi_bfield_atomic_try_clear_mask_optimistic(b, mask, previous, did_temp_clear_bits);
+}
+
+// Try to clear a mask of bits atomically. For performance, it assumes these is a good chance of success.
+// Returns true all bits transitioned from all 1's to 0's.
+// and false otherwise leaving the bit field as-is. `all_clear` is set to true if the new bfield became zero.
+static inline bool mi_bfield_atomic_try_clear_mask(_Atomic(mi_bfield_t)* b, mi_bfield_t mask, bool* all_clear, bool* did_temp_clear_bits) {
+ mi_assert_internal(mask != 0);
+ mi_bfield_t old = mi_atomic_load_relaxed(b); // check first if there is a good chance the mask bits are all set
+ bool cleared = false;
+ if ((old&mask)==mask) {
+ cleared = mi_bfield_atomic_try_clear_mask_optimistic(b, mask, &old, did_temp_clear_bits);
+ }
+ if (all_clear != NULL) { *all_clear = ((cleared ? old&~mask : old) == 0); }
+ return cleared;
}
// Try to clear a full field of bits atomically, and return true all bits transitioned from all 1's to 0's.
// and false otherwise leaving the bit field as-is.
// `all_clear` is set to true if the new bfield became zero (which is always the case if successful).
static inline bool mi_bfield_atomic_try_clearX(_Atomic(mi_bfield_t)*b, bool* all_clear) {
- mi_bfield_t old = mi_bfield_all_set();
- if (mi_atomic_cas_strong_acq_rel(b, &old, mi_bfield_zero())) {
- if (all_clear != NULL) { *all_clear = true; }
- return true;
+ mi_bfield_t old = mi_atomic_load_relaxed(b); // check first if there is a good chance the bits are all set
+ bool cleared = false;
+ if (old==mi_bfield_all_set()) {
+ cleared = mi_atomic_cas_strong_acq_rel(b,&old,mi_bfield_zero());
}
- else return false;
+ if (all_clear != NULL) { *all_clear = (cleared ? true : (old==0)); }
+ return cleared;
}
@@ -331,7 +343,7 @@ mi_decl_noinline static bool mi_bchunk_xsetNC(mi_xset_t set, mi_bchunk_t* chunk,
bool all_clear = false;
const bool transition = (set ? mi_bfield_atomic_set_mask(&chunk->bfields[field], mask, &already_set)
: mi_bfield_atomic_clear_mask(&chunk->bfields[field], mask, &all_clear));
- mi_assert_internal((transition && already_set == 0) || (!transition && already_set > 0));
+ mi_assert_internal(!set || ((transition && already_set == 0) || (!transition && already_set > 0)));
all_transition = all_transition && transition;
total_already_set += already_set;
maybe_all_clear = maybe_all_clear && all_clear;
@@ -369,7 +381,7 @@ static inline bool mi_bchunk_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, b
if (n==1) return mi_bchunk_clear(chunk, cidx, maybe_all_clear);
// if (n==8) return mi_bchunk_clear8(chunk, cidx, maybe_all_clear);
// if (n==MI_BFIELD_BITS) return mi_bchunk_clearX(chunk, cidx, maybe_all_clear);
- // TODO: implement mi_bchunk_xsetNX instead of setNX
+ // todo: implement mi_bchunk_xsetNX instead of setNX
return mi_bchunk_xsetNC(MI_BIT_CLEAR, chunk, cidx, n, NULL, maybe_all_clear);
}
@@ -445,14 +457,14 @@ static inline bool mi_bchunk_is_xsetN(mi_xset_t set, const mi_bchunk_t* chunk, s
// ------- mi_bchunk_try_clear ---------------------------------------
// Clear `0 < n <= MI_BITFIELD_BITS`. Can cross over a bfield boundary.
-static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) {
+static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear, bool* did_temp_clear_bits) {
mi_assert_internal(cidx < MI_BCHUNK_BITS);
mi_assert_internal(n <= MI_BFIELD_BITS);
const size_t i = cidx / MI_BFIELD_BITS;
const size_t idx = cidx % MI_BFIELD_BITS;
if mi_likely(idx + n <= MI_BFIELD_BITS) {
// within one field
- return mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(n, idx), pmaybe_all_clear);
+ return mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(n, idx), pmaybe_all_clear, did_temp_clear_bits);
}
else {
// spanning two fields (todo: use double-word atomic ops?)
@@ -460,14 +472,15 @@ static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t
mi_assert_internal(m < n);
mi_assert_internal(i < MI_BCHUNK_FIELDS - 1);
bool field1_is_clear;
- if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &field1_is_clear)) return false;
+ if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &field1_is_clear, did_temp_clear_bits)) return false;
// try the second field as well
mi_assert_internal(n - m > 0);
mi_assert_internal(n - m < MI_BFIELD_BITS);
bool field2_is_clear;
- if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &field2_is_clear)) {
+ if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &field2_is_clear, did_temp_clear_bits)) {
// we failed to clear the second field, restore the first one
mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), NULL);
+ if (did_temp_clear_bits != NULL) { *did_temp_clear_bits = true; }
return false;
}
if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = field1_is_clear && field2_is_clear; }
@@ -488,7 +501,7 @@ static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t
// and false otherwise leaving all bit fields as is.
// Note: this is the complex one as we need to unwind partial atomic operations if we fail halfway..
// `maybe_all_clear` is set to `true` if all the bfields involved become zero.
-mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) {
+mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear, bool* did_temp_clear_bits) {
mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);
mi_assert_internal(n>0);
if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = true; }
@@ -504,7 +517,7 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t ci
mi_assert_internal(start_field < MI_BCHUNK_FIELDS);
const mi_bfield_t mask_start = mi_bfield_mask(m, start_idx);
bool maybe_all_clear;
- if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_start, &maybe_all_clear)) return false;
+ if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_start, &maybe_all_clear, did_temp_clear_bits)) return false;
// done?
mi_assert_internal(m <= n);
@@ -528,7 +541,7 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t ci
mi_assert_internal(field < MI_BCHUNK_FIELDS);
const mi_bfield_t mask_end = mi_bfield_mask(n, 0);
bool field_is_clear;
- if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_end, &field_is_clear)) goto restore;
+ if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_end, &field_is_clear, did_temp_clear_bits)) goto restore;
maybe_all_clear = maybe_all_clear && field_is_clear;
}
@@ -538,6 +551,7 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t ci
restore:
// `field` is the index of the field that failed to set atomically; we need to restore all previous fields
mi_assert_internal(field > start_field);
+ if (did_temp_clear_bits != NULL) { *did_temp_clear_bits = true; }
while( field > start_field) {
field--;
if (field == start_field) {
@@ -551,11 +565,11 @@ restore:
}
-static inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear) {
+static inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear, bool* did_temp_clear_bits) {
mi_assert_internal(n>0);
// if (n==MI_BFIELD_BITS) return mi_bchunk_try_clearX(chunk, cidx, maybe_all_clear);
- if (n<=MI_BFIELD_BITS) return mi_bchunk_try_clearNX(chunk, cidx, n, maybe_all_clear);
- return mi_bchunk_try_clearNC(chunk, cidx, n, maybe_all_clear);
+ if (n<=MI_BFIELD_BITS) return mi_bchunk_try_clearNX(chunk, cidx, n, maybe_all_clear, did_temp_clear_bits);
+ return mi_bchunk_try_clearNC(chunk, cidx, n, maybe_all_clear, did_temp_clear_bits);
}
@@ -576,19 +590,24 @@ mi_decl_maybe_unused static inline bool mi_mm256_is_zero( __m256i vec) {
}
#endif
+
static inline bool mi_bchunk_try_find_and_clear_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) {
mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS);
- // note: this must be acquire (and not relaxed), or otherwise the AVX code below can loop forever
- // as the compiler won't reload the registers vec1 and vec2 from memory again.
- const mi_bfield_t b = mi_atomic_load_acquire(&chunk->bfields[chunk_idx]);
- size_t idx;
- if (mi_bfield_find_least_bit(b, &idx)) { // find the least bit
- if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], mi_bfield_mask(1,idx), b, NULL)) { // clear it atomically
- *pidx = (chunk_idx*MI_BFIELD_BITS) + idx;
+ _Atomic(mi_bfield_t)* const bfield = &chunk->bfields[chunk_idx];
+ mi_bfield_t b = mi_atomic_load_relaxed(bfield);
+ if (b==0) return false;
+ int tries = 0;
+ do {
+ const mi_bfield_t mask = (b & (~b+1)); // == (b & -b) but avoids a compiler warning -- clear all bits except the least-significant one
+ b = mi_atomic_and_acq_rel(bfield,~mask); // clear the bit and set `b` to the previous value
+ if mi_likely((b&mask)==mask) { // if we transitioned from 1 to 0, we actually cleared it
+ size_t bitidx = 0;
+ mi_bfield_find_least_bit(mask,&bitidx);
+ *pidx = (chunk_idx*MI_BFIELD_BITS) + bitidx;
mi_assert_internal(*pidx < MI_BCHUNK_BITS);
return true;
- }
- }
+ }
+ } while (b!=0 && ++tries <= 4); // limit tries to reduce possible contention
return false;
}
@@ -686,28 +705,36 @@ static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx
return false;
}
-static inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
- mi_assert_internal(n==1); MI_UNUSED(n);
+static inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) {
+ mi_assert_internal(n==1); MI_UNUSED(n); MI_UNUSED(did_temp_clear_bits);
return mi_bchunk_try_find_and_clear(chunk, pidx);
}
-mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) {
- const mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[chunk_idx]);
- // has_set8 has low bit in each byte set if the byte in x == 0xFF
- const mi_bfield_t has_set8 =
- ((~b - MI_BFIELD_LO_BIT8) & // high bit set if byte in x is 0xFF or < 0x7F
- (b & MI_BFIELD_HI_BIT8)) // high bit set if byte in x is >= 0x80
- >> 7; // shift high bit to low bit
- size_t idx;
- if (mi_bfield_find_least_bit(has_set8, &idx)) { // find least 1-bit
- mi_assert_internal(idx <= (MI_BFIELD_BITS - 8));
- mi_assert_internal((idx%8)==0);
- if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], (mi_bfield_t)0xFF << idx, b, NULL)) { // unset the byte atomically
- *pidx = (chunk_idx*MI_BFIELD_BITS) + idx;
- mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS);
- return true;
+mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx, bool* did_temp_clear_bits) {
+ _Atomic(mi_bfield_t)* const bfield = &chunk->bfields[chunk_idx];
+ mi_bfield_t b = mi_atomic_load_relaxed(bfield);
+ if (b==0) return false;
+ int tries = 0;
+ do {
+ // has_set8 has low bit in each byte set if the byte in x == 0xFF
+ const mi_bfield_t has_set8 =
+ ((~b - MI_BFIELD_LO_BIT8) & // high bit set if byte in x is 0xFF or < 0x7F
+ (b & MI_BFIELD_HI_BIT8)) // high bit set if byte in x is >= 0x80
+ >> 7; // shift high bit to low bit
+ size_t bitidx;
+ if (mi_bfield_find_least_bit(has_set8, &bitidx)) { // find least 1-bit
+ mi_assert_internal(bitidx <= (MI_BFIELD_BITS - 8));
+ mi_assert_internal((bitidx%8)==0);
+ if mi_likely(mi_bfield_atomic_try_clear8_optimistic(bfield,bitidx,&b,did_temp_clear_bits)) { // try to clear atomically (and update b)
+ *pidx = (chunk_idx*MI_BFIELD_BITS) + bitidx;
+ mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS);
+ return true;
+ }
}
- }
+ else {
+ return false;
+ }
+ } while(b!=0 && ++tries <= 4); // limit tries to reduce possible contention
return false;
}
@@ -715,9 +742,9 @@ mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchu
// set `*pidx` to its bit index (0 <= *pidx < MI_BCHUNK_BITS) on success.
// Used to find medium size pages in the free blocks.
// todo: try neon version
-static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, size_t* pidx) {
+static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, size_t* pidx, bool* did_temp_clear_bits) {
#if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)
- while (true) {
+ for(int tries=0; tries<4; tries++) { // paranoia: at most 4 tries
// since a cache-line is 64b, load all at once
const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);
const __m256i vec2 = _mm256_load_si256((const __m256i*)chunk->bfields+1);
@@ -733,25 +760,29 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, s
const size_t chunk_idx = bidx / 8;
const size_t idx = (bidx % 8)*8;
mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS);
- if mi_likely(mi_bfield_atomic_try_clear8(&chunk->bfields[chunk_idx], idx, NULL)) { // clear it atomically
+ if mi_likely(mi_bfield_atomic_try_clear8_optimistic(&chunk->bfields[chunk_idx], idx, NULL, did_temp_clear_bits)) { // clear it atomically
*pidx = (chunk_idx*MI_BFIELD_BITS) + idx;
mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS);
return true;
}
// try again
- // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded }
+ // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded
+ // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206)
+ #if defined(__GNUC__)
+ __asm __volatile ("" : : "g"(chunk) : "memory");
+ #endif
}
#else
for (int i = 0; i < MI_BCHUNK_FIELDS; i++) {
- if (mi_bchunk_try_find_and_clear8_at(chunk, i, pidx)) return true;
+ if (mi_bchunk_try_find_and_clear8_at(chunk, i, pidx, did_temp_clear_bits)) return true;
}
- return false;
#endif
+ return false;
}
-static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
- mi_assert_internal(n==8); MI_UNUSED(n);
- return mi_bchunk_try_find_and_clear8(chunk, pidx);
+static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) {
+ mi_assert_internal(n==8); MI_UNUSED(n); MI_UNUSED(did_temp_clear_bits);
+ return mi_bchunk_try_find_and_clear8(chunk, pidx, did_temp_clear_bits);
}
@@ -759,7 +790,7 @@ static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n,
// and try to clear them atomically.
// set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success.
// will cross bfield boundaries.
-mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
+mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) {
if (n == 0 || n > MI_BFIELD_BITS) return false;
const mi_bfield_t mask = mi_bfield_mask(n, 0);
// for all fields in the chunk
@@ -775,7 +806,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk,
const size_t bmask = mask<<idx;
mi_assert_internal(bmask>>idx == mask);
if ((b&bmask) == bmask) { // found a match with all bits set, try clearing atomically
- if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[i], bmask, b0, NULL)) {
+ if mi_likely(mi_bfield_atomic_try_clear_mask_optimistic(&chunk->bfields[i], bmask, &b0, did_temp_clear_bits)) { // updates b0 with the previous value
*pidx = (i*MI_BFIELD_BITS) + idx;
mi_assert_internal(*pidx < MI_BCHUNK_BITS);
mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS);
@@ -783,7 +814,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk,
}
else {
// if we failed to atomically commit, reload b and try again from the start
- b = b0 = mi_atomic_load_acquire(&chunk->bfields[i]);
+ b = b0; // = mi_atomic_load_acquire(&chunk->bfields[i]);
}
}
else {
@@ -803,7 +834,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk,
if (post + pre >= n) {
// it fits -- try to claim it atomically
const size_t cidx = (i*MI_BFIELD_BITS) + (MI_BFIELD_BITS - post);
- if (mi_bchunk_try_clearNX(chunk, cidx, n, NULL)) {
+ if (mi_bchunk_try_clearNX(chunk, cidx, n, NULL, did_temp_clear_bits)) {
// we cleared all atomically
*pidx = cidx;
mi_assert_internal(*pidx < MI_BCHUNK_BITS);
@@ -821,7 +852,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk,
// and try to clear them atomically.
// set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success.
// This can cross bfield boundaries.
-static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
+static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) {
if (n == 0 || n > MI_BCHUNK_BITS) return false; // cannot be more than a chunk
// we first scan ahead to see if there is a range of `n` set bits, and only then try to clear atomically
@@ -870,7 +901,7 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk,
// did we find a range?
if (m==0) {
- if (mi_bchunk_try_clearN(chunk, cidx, n, NULL)) {
+ if (mi_bchunk_try_clearN(chunk, cidx, n, NULL, did_temp_clear_bits)) {
// we cleared all atomically
*pidx = cidx;
mi_assert_internal(*pidx < MI_BCHUNK_BITS);
@@ -888,11 +919,11 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk,
// ------- mi_bchunk_clear_once_set ---------------------------------------
-static inline void mi_bchunk_clear_once_set(mi_bchunk_t* chunk, size_t cidx) {
+static inline void mi_bchunk_clear_once_set(mi_subproc_t* subproc, mi_bchunk_t* chunk, size_t cidx) {
mi_assert_internal(cidx < MI_BCHUNK_BITS);
const size_t i = cidx / MI_BFIELD_BITS;
const size_t idx = cidx % MI_BFIELD_BITS;
- mi_bfield_atomic_clear_once_set(&chunk->bfields[i], idx);
+ mi_bfield_atomic_clear_once_set(subproc, &chunk->bfields[i], idx);
}
@@ -963,7 +994,7 @@ static bool mi_bchunk_bsr(mi_bchunk_t* chunk, size_t* pidx) {
return false;
}
-static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) {
+static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) {
for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) {
i--;
mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);
@@ -1050,7 +1081,7 @@ static void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, si
// start chunk and index
size_t chunk_idx = idx / MI_BCHUNK_BITS;
const size_t cidx = idx % MI_BCHUNK_BITS;
- const size_t ccount = _mi_divide_up(n, MI_BCHUNK_BITS);
+ const size_t ccount = ((cidx + n - 1) / MI_BCHUNK_BITS) + 1;
// first update the chunkmap
mi_bchunk_setN(cmap, chunk_idx, ccount, NULL);
@@ -1323,9 +1354,10 @@ static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk
return true;
}
else {
- // failed to claim it, set abandoned mapping again (unless the page was freed)
+ // failed to claim it, set abandoned mapping again (unless the page was freed and keep_set will be false)
if (keep_set) {
const bool wasclear = mi_bchunk_set(&bitmap->chunks[chunk_idx], cidx, NULL);
+ mi_bitmap_chunkmap_set(bitmap, chunk_idx);
mi_assert_internal(wasclear); MI_UNUSED(wasclear);
}
}
@@ -1355,12 +1387,15 @@ bool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx) {
mi_bfield_t cmap = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]);
size_t cmap_idx;
if (mi_bsr(cmap,&cmap_idx)) {
- // highest chunk
- const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
- size_t cidx;
- if (mi_bchunk_bsr(&bitmap->chunks[chunk_idx], &cidx)) {
- *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;
- return true;
+ // from highest chunk to lowest (scan all in case the cmap entry was stale)
+ for (size_t j = cmap_idx+1; j>0; ) {
+ j--;
+ const size_t chunk_idx = (i*MI_BFIELD_BITS) + j;
+ size_t cidx;
+ if (mi_bchunk_bsr(&bitmap->chunks[chunk_idx], &cidx)) {
+ *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;
+ return true;
+ }
}
}
}
@@ -1388,12 +1423,12 @@ size_t mi_bitmap_popcount(mi_bitmap_t* bitmap) {
// Clear a bit once it is set.
-void mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx) {
+void mi_bitmap_clear_once_set(mi_subproc_t* subproc, mi_bitmap_t* bitmap, size_t idx) {
mi_assert_internal(idx < mi_bitmap_max_bits(bitmap));
const size_t chunk_idx = idx / MI_BCHUNK_BITS;
const size_t cidx = idx % MI_BCHUNK_BITS;
mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap));
- mi_bchunk_clear_once_set(&bitmap->chunks[chunk_idx], cidx);
+ mi_bchunk_clear_once_set(subproc, &bitmap->chunks[chunk_idx], cidx);
}
@@ -1456,6 +1491,8 @@ bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* vis
mi_assert_internal(rng>=1 && rng<=MI_BFIELD_BITS);
mi_assert_internal((idx % MI_BFIELD_BITS) + rng <= MI_BFIELD_BITS);
mi_assert_internal((idx / MI_BCHUNK_BITS) < mi_bitmap_chunk_count(bitmap));
+ // clear rng bits in b
+ b = b & ~mi_bfield_mask(rng, bidx);
if (!visit(idx, rng, arena, arg)) {
// break early: reset the non-visited bits
if (b!=0) {
@@ -1463,8 +1500,6 @@ bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* vis
}
return false;
}
- // clear rng bits in b
- b = b & ~mi_bfield_mask(rng, bidx);
}
mi_assert_internal(rngcount == bpopcount);
}
@@ -1502,14 +1537,15 @@ bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_fo
const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);
mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0); // atomic clear
mi_bfield_t skipped = 0; // but track which bits we skip so we can restore them
- for(size_t shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) { // per `rngslices` to keep alignment
+ size_t shift;
+ for(shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) { // per `rngslices` to keep alignment
const mi_bfield_t rngmask = mi_bfield_mask(rngslices, shift);
if ((b & rngmask) == rngmask) {
const size_t idx = base_idx + shift;
if (!visit(idx, rngslices, arena, arg)) {
// break early: restore non-visited entries
mi_bfield_t notyet_visited = 0;
- if (shift + rngslices < MI_BFIELD_BITS) {
+ if (rngslices + shift < MI_BFIELD_BITS) {
notyet_visited = (b & (~(mi_bfield_t)0 << (shift + rngslices)));
}
mi_assert_internal((notyet_visited & skipped) == 0);
@@ -1523,8 +1559,13 @@ bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_fo
skipped = skipped | (b & rngmask);
}
}
-
+ if (shift < MI_BFIELD_BITS) {
+ // there are some non-visited top bits when `MI_BFIELD_BITS % rngslices != 0`.
+ mi_assert_internal(MI_BFIELD_BITS % rngslices != 0);
+ skipped = skipped | (b & (~(mi_bfield_t)0 << shift));
+ }
if (skipped != 0) {
+ // restore non-visited entries
mi_atomic_or_relaxed(&chunk->bfields[j], skipped);
}
}
@@ -1554,7 +1595,7 @@ size_t mi_bbitmap_size(size_t bit_count, size_t* pchunk_count) {
// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true
// returns the size of the bitmap
-size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero) {
+size_t mi_bbitmap_init(mi_subproc_t* subproc, mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero) {
size_t chunk_count;
const size_t size = mi_bbitmap_size(bit_count, &chunk_count);
if (!already_zero) {
@@ -1562,6 +1603,7 @@ size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zer
}
mi_atomic_store_release(&bbitmap->chunk_count, chunk_count);
mi_assert_internal(mi_atomic_load_relaxed(&bbitmap->chunk_count) <= MI_BITMAP_MAX_CHUNK_COUNT);
+ bbitmap->subproc = subproc;
return size;
}
@@ -1572,27 +1614,16 @@ void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
}
bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx) {
+ // scan for highest zero bit in the bitmap
+ // note: we cannot use the chunkmap since that only conservatively denotes if there might be a set bit in a chuck
+ // todo: bbitmap_init rounds up the bitcount to BCHUNK_BITS and we should skip the top-padding!
const size_t chunk_count = mi_bbitmap_chunk_count(bbitmap);
- const size_t chunkmap_max = _mi_divide_up(chunk_count, MI_BFIELD_BITS);
- size_t skip_at_top = chunk_count % MI_BFIELD_BITS;
- for (size_t i = chunkmap_max; i > 0; ) {
+ for(size_t i = chunk_count; i > 0; ) {
i--;
- mi_bfield_t cmap = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[i]);
- size_t cmap_idx;
- // don't consider top 0 bits; set those to 1 here
- if (skip_at_top > 0) {
- const size_t mask_top = (~mi_bfield_zero()) << (MI_BFIELD_BITS - skip_at_top);
- skip_at_top = 0; // only for the first iteration
- cmap |= mask_top;
- }
- if (mi_bsr(~cmap, &cmap_idx)) {
- // highest chunk
- const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
- size_t cidx;
- if (mi_bchunk_bsr_inv(&bbitmap->chunks[chunk_idx], &cidx)) {
- *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;
- return true;
- }
+ size_t cidx;
+ if (mi_bchunk_bsr_inv(&bbitmap->chunks[i], &cidx)) {
+ *idx = (i * MI_BCHUNK_BITS) + cidx;
+ return true;
}
}
return false;
@@ -1609,11 +1640,11 @@ static void mi_bbitmap_set_chunk_bin(mi_bbitmap_t* bbitmap, size_t chunk_idx, mi
for (mi_chunkbin_t ibin = MI_CBIN_SMALL; ibin < MI_CBIN_NONE; ibin = mi_chunkbin_inc(ibin)) {
if (ibin == bin) {
const bool was_clear = mi_bchunk_set(& bbitmap->chunkmap_bins[ibin], chunk_idx, NULL);
- if (was_clear) { mi_os_stat_increase(chunk_bins[ibin],1); }
+ if (was_clear) { mi_subproc_stat_increase(bbitmap->subproc, chunk_bins[ibin],1); }
}
else {
const bool was_set = mi_bchunk_clear(&bbitmap->chunkmap_bins[ibin], chunk_idx, NULL);
- if (was_set) { mi_os_stat_decrease(chunk_bins[ibin],1); }
+ if (was_set) { mi_subproc_stat_decrease(bbitmap->subproc,chunk_bins[ibin],1); }
}
}
}
@@ -1710,8 +1741,15 @@ bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap));
if (cidx + n > MI_BCHUNK_BITS) return false;
bool maybe_all_clear = false;
- const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear);
- if (cleared && maybe_all_clear) { mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); }
+ bool did_temp_clear_bits = false;
+ const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear, &did_temp_clear_bits);
+ if (cleared && maybe_all_clear) {
+ mi_assert_internal(!did_temp_clear_bits);
+ mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx);
+ } else if (did_temp_clear_bits) {
+ // may have raced with a clearer (in on_find) so set the chunkmap bit conservatively
+ mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false);
+ }
// note: we don't set the size class for an explicit try_clearN (only used by purging)
return cleared;
}
@@ -1753,14 +1791,14 @@ bool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_
(used to find free pages)
-------------------------------------------------------------------------------- */
-typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx);
+typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx, bool* did_temp_clear_bits);
// Go through the bbitmap and for every sequence of `n` set bits, call the visitor function.
// If it returns `true` stop the search.
//
// This is used for finding free blocks and it is important to be efficient (with 2-level bitscan)
// but also reduce fragmentation (through size bins).
-static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find)
+static bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find)
{
// we space out threads to reduce contention
const size_t cmap_max_count = _mi_divide_up(mi_bbitmap_chunk_count(bbitmap),MI_BFIELD_BITS);
@@ -1814,7 +1852,8 @@ static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap,
mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx];
size_t cidx;
- if ((*on_find)(chunk, n, &cidx)) {
+ bool did_temp_clear_bits = false;
+ if ((*on_find)(chunk, n, &cidx, &did_temp_clear_bits)) {
if (cidx==0 && ibin == MI_CBIN_NONE) { // only the first block determines the size bin
// this chunk is now reserved for the `bbin` size class
mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, bbin);
@@ -1824,9 +1863,17 @@ static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap,
return true;
}
else {
+ // note: we may find that all are cleared only on a second iteration (when we fail to clear any bits)
+ // but that is ok as the chunkmap is a conservative approximation.
// todo: should _on_find_ return a boolean if there is a chance all are clear to avoid calling `try_clear?`
- // we may find that all are cleared only on a second iteration but that is ok as the chunkmap is a conservative approximation.
- mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx);
+ // probably not as we already only call `try_clear` once we fail to clear any bits.
+ if (did_temp_clear_bits) {
+ // a concurrent find_and_claim may have cleared the chunkmap bit, restore it now
+ mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false);
+ }
+ else {
+ mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx);
+ }
}
}
mi_bfield_cycle_iterate_end(Y);
@@ -1871,12 +1918,11 @@ bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t
// Try to atomically clear `n` bits starting at `chunk_idx` where `n` can span over multiple chunks
static bool mi_bchunk_try_clearN_(mi_bbitmap_t* bbitmap, size_t chunk_idx, size_t n) {
mi_assert_internal((chunk_idx * MI_BCHUNK_BITS) + n <= mi_bbitmap_max_bits(bbitmap));
-
size_t m = n; // bits to go
size_t count = 0; // chunk count
while (m > 0) {
mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count];
- if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL)) {
+ if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL, NULL)) {
goto rollback;
}
m = (m <= MI_BCHUNK_BITS ? 0 : m - MI_BCHUNK_BITS);
@@ -1890,6 +1936,8 @@ rollback:
count--;
mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count];
mi_bchunk_setN(chunk, 0, MI_BCHUNK_BITS, NULL);
+ // since we may race with clearing, we need to set the chunkmap conservatively
+ mi_bbitmap_chunkmap_set(bbitmap, chunk_idx + count, false);
}
return false;
}
@@ -1926,7 +1974,7 @@ bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t
// did we find a suitable range?
if (count == chunk_req) {
- // now try to claim it!
+ // now try to claim it!
if (mi_bchunk_try_clearN_(bbitmap, chunk_idx, n)) {
*pidx = (chunk_idx * MI_BCHUNK_BITS);
for (size_t i = 0; i < count; i++) {
@@ -1935,6 +1983,11 @@ bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t
mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap));
return true;
}
+ else {
+ // contended: we reset count to retry from the first
+ // (we still skip the first chunk to guarantee progress)
+ count = 0;
+ }
}
// keep searching but skip the scanned range
diff --git a/vendored/mimalloc/src/bitmap.h b/vendored/mimalloc/src/bitmap.h
index ca51a4e04..79eda77fa 100644
--- a/vendored/mimalloc/src/bitmap.h
+++ b/vendored/mimalloc/src/bitmap.h
@@ -71,7 +71,7 @@ typedef size_t mi_bfield_t;
#define MI_BCHUNK_FIELDS (MI_BCHUNK_BITS / MI_BFIELD_BITS) // 8 on both 64- and 32-bit
-// some compiler (msvc in C mode) cannot have expressions in the alignment attribute
+// some compilers (msvc in C mode) cannot have expressions in the alignment attribute
#if MI_BCHUNK_SIZE==64
#define mi_decl_bchunk_align mi_decl_align(64)
#elif MI_BCHUNK_SIZE==32
@@ -95,11 +95,11 @@ typedef mi_bchunk_t mi_bchunkmap_t;
#define MI_BITMAP_MAX_CHUNK_COUNT (MI_BCHUNKMAP_BITS)
#define MI_BITMAP_MIN_CHUNK_COUNT (1)
-#if MI_SIZE_BITS > 32
-#define MI_BITMAP_DEFAULT_CHUNK_COUNT (64) // 2 GiB on 64-bit -- this is for the page map
-#else
+// #if MI_SIZE_BITS > 32
+// #define MI_BITMAP_DEFAULT_CHUNK_COUNT (128) // 4 GiB on 64-bit -- this is for the page map
+// #else
#define MI_BITMAP_DEFAULT_CHUNK_COUNT (1)
-#endif
+// #endif
#define MI_BITMAP_MAX_BIT_COUNT (MI_BITMAP_MAX_CHUNK_COUNT * MI_BCHUNK_BITS) // 16 GiB arena
#define MI_BITMAP_MIN_BIT_COUNT (MI_BITMAP_MIN_CHUNK_COUNT * MI_BCHUNK_BITS) // 32 MiB arena
#define MI_BITMAP_DEFAULT_BIT_COUNT (MI_BITMAP_DEFAULT_CHUNK_COUNT * MI_BCHUNK_BITS) // 2 GiB arena
@@ -199,7 +199,7 @@ mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t
// Atomically clear a bit but only if it is set. Will block otherwise until the bit is set.
// This is used to delay free-ing a page that it at the same time being considered to be
// allocated from `mi_arena_try_abandoned` (and is in the `claim` function of `mi_bitmap_try_find_and_claim`).
-void mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx);
+void mi_bitmap_clear_once_set(mi_subproc_t* subproc, mi_bitmap_t* bitmap, size_t idx);
// If a bit is set in the bitmap, return `true` and set `idx` to the index of the highest bit.
@@ -260,8 +260,9 @@ static inline mi_chunkbin_t mi_chunkbin_of(size_t slice_count) {
typedef mi_decl_bchunk_align struct mi_bbitmap_s {
_Atomic(size_t) chunk_count; // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS)
_Atomic(size_t) chunk_max_accessed; // max chunk index that was once cleared or set
- #if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 2
- size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 2]; // suppress warning on msvc by aligning manually
+ mi_subproc_t* subproc; // constant, for stats
+ #if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 3
+ size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 3]; // suppress warning on msvc by aligning manually
#endif
mi_bchunkmap_t chunkmap;
mi_bchunkmap_t chunkmap_bins[MI_CBIN_COUNT - 1]; // chunkmaps with bit set if the chunk is in that size class (excluding MI_CBIN_NONE)
@@ -288,7 +289,7 @@ bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx);
// Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true
// returns the size of the bitmap.
-size_t mi_bbitmap_init(mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero);
+size_t mi_bbitmap_init(mi_subproc_t* subproc, mi_bbitmap_t* bbitmap, size_t bit_count, bool already_zero);
// Set/clear a sequence of `n` bits in the bitmap (and can cross chunks).
// Not atomic so only use if still local to a thread.
diff --git a/vendored/mimalloc/src/free.c b/vendored/mimalloc/src/free.c
index 6f33340d0..722ee7bf6 100644
--- a/vendored/mimalloc/src/free.c
+++ b/vendored/mimalloc/src/free.c
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -9,15 +9,14 @@ terms of the MIT license. A copy of the license can be found in the file
// add includes help an IDE
#include "mimalloc.h"
#include "mimalloc/internal.h"
-#include "mimalloc/prim.h" // _mi_prim_thread_id()
+#include "mimalloc/prim-tls.h" // _mi_prim_thread_id()
#endif
+
// forward declarations
-static void mi_check_padding(const mi_page_t* page, const mi_block_t* block);
-static bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block);
+mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size);
+mi_decl_nodiscard static bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block);
static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded);
-static void mi_stat_free(const mi_page_t* page, const mi_block_t* block);
-
// ------------------------------------------------------
// Free
@@ -25,22 +24,29 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block);
// regular free of a (thread local) block pointer
// fast path written carefully to prevent spilling on the stack
-static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full)
+static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool check_full)
{
- MI_UNUSED(was_guarded);
// checks
- if mi_unlikely(mi_check_is_double_free(page, block)) return;
- if (!was_guarded) { mi_check_padding(page, block); }
- if (track_stats) { mi_stat_free(page, block); }
+ size_t usable_size;
+ if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return;
+ if mi_unlikely(!mi_check_double_free(page,block)) return; // usually checked with padding
+
+ // mi_stat_free(page, block);
+ mi_track_free_size(block, usable_size);
+
#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN
- memset(block, MI_DEBUG_FREED, mi_page_block_size(page));
+ const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size);
+ _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize);
#endif
- if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); } // faster then mi_usable_size as we already know the page and that p is unaligned
-
+
// actual free: push on the local free list
+ mi_used_t xused = page->xused;
+ xused.used_alloc--; // decrement used count
mi_block_set_next(page, block, page->local_free);
+ page->xused = xused;
page->local_free = block;
- if mi_unlikely(--page->used == 0) {
+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));
+ if mi_unlikely(mi_xused_used_count(xused) == 0) { // is used count zero ?
if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page)
_mi_page_retire(page);
}
@@ -51,75 +57,67 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool
}
// Forward declaration for multi-threaded collect
-static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept;
+static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept;
// Free a block multi-threaded
-static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded) mi_attr_noexcept
+static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_reclaim) mi_attr_noexcept
{
- MI_UNUSED(was_guarded);
- // adjust stats (after padding check and potentially recursive `mi_free` above)
- mi_stat_free(page, block); // stat_free may access the padding
- mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded));
+ size_t usable_size;
+ if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; // checking padding is safe for mt
+
+ // adjust stats (after padding check )
+ // mi_stat_free(page, block); // stat_free may access the padding
+ mi_track_free_size(block, usable_size);
// _mi_padding_shrink(page, block, sizeof(mi_block_t));
-#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading
+ #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading
if (!was_guarded) {
- size_t dbgsize = mi_usable_size(block);
- if (dbgsize > MI_MiB) { dbgsize = MI_MiB; }
+ const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size);
_mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize);
}
-#endif
+ #endif
// push atomically on the page thread free list
+ // mi_theap_t* theap = _mi_page_associated_theap_peek(page);
mi_thread_free_t tf_new;
mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free);
do {
mi_block_set_next(page, block, mi_tf_block(tf_old));
- tf_new = mi_tf_create(block, true /* always use owned: try to claim it if the page is abandoned */);
- } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // todo: release is enough?
+ tf_new = mi_tf_create(block, true);
+ } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new));
// and atomically try to collect the page if it was abandoned
- const bool is_owned_now = !mi_tf_is_owned(tf_old);
- if (is_owned_now) {
+ const bool is_newly_owned = mi_tf_is_owned(tf_new) && !mi_tf_is_owned(tf_old);
+ if (is_newly_owned) {
mi_assert_internal(mi_page_is_abandoned(page));
- mi_free_try_collect_mt(page,block);
+ // mi_assert_internal(!mi_page_is_abandoned_mapped(page));
+ mi_free_try_collect_mt(page, block, allow_reclaim);
}
}
// Adjust a block that was allocated aligned, to the actual start of the block in the page.
// note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the
-// `page_start` and `block_size` fields; however these are constant and the page won't be
+// `page_woffset` and `block_size` fields; however these are constant and the page won't be
// deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently.
-mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {
+static inline mi_block_t* mi_page_ptr_unalign_ex(const mi_page_t* page, const void* p, size_t* poffset) {
mi_assert_internal(page!=NULL && p!=NULL);
const size_t diff = (uint8_t*)p - mi_page_start(page);
const size_t block_size = mi_page_block_size(page);
- const size_t adjust = (_mi_is_power_of_two(block_size) ? diff & (block_size - 1) : diff % block_size);
+ size_t adjust = diff & (block_size - 1);
+ if mi_unlikely(!_mi_is_power_of_two(block_size)) {
+ adjust = diff % block_size;
+ }
+ if (poffset!=NULL) { *poffset = adjust; }
return (mi_block_t*)((uintptr_t)p - adjust);
}
-// forward declaration for a MI_GUARDED build
-#if MI_GUARDED
-static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration
-static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
- if (mi_block_ptr_is_guarded(block, p)) {
- mi_block_unguard(page, block, p);
- return true;
- }
- else {
- return false;
- }
-}
-#else
-static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
- MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p);
- return false;
+mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {
+ return mi_page_ptr_unalign_ex(page,p,NULL);
}
-#endif
-static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, void* p ) {
+static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, const void* p ) {
mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); // should never be an interior pointer
#if MI_SECURE > 0
// in secure mode we always unalign to guard against free-ing interior pointers
@@ -130,63 +128,147 @@ static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, voi
#endif
}
+static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool* was_guarded) mi_attr_noexcept {
+ MI_UNUSED(was_guarded);
+ if mi_likely(!mi_page_has_interior_pointers(page)) {
+ return mi_validate_block_from_ptr(page,p);
+ }
+ else {
+ size_t offset;
+ mi_block_t* const block = mi_page_ptr_unalign_ex(page,p,&offset);
+ #if MI_GUARDED || MI_PROFILE
+ if (offset >= sizeof(mi_block_t)) {
+ #if MI_PROFILE
+ if (block->next == MI_BLOCK_TAG_PROFILED) {
+ _mi_page_profile_on_free(page,block,p);
+ }
+ else
+ #endif
+ #if MI_GUARDED
+ if (block->next == MI_BLOCK_TAG_GUARDED) {
+ _mi_page_block_unguard(page, block, p);
+ *was_guarded = true;
+ }
+ else
+ #endif
+ { /* nothing */ }
+ }
+ #endif
+ return block;
+ }
+}
// free a local pointer (page parameter comes first for better codegen)
static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept {
mi_assert_internal(p!=NULL && page != NULL);
- mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));
- const bool was_guarded = mi_block_check_unguard(page, block, p);
- mi_free_block_local(page, block, was_guarded, true /* track stats */, true /* check for a full page */);
+ bool was_guarded = false;
+ mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded);
+ // mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));
+ // mi_block_check_profiled(page,block,p);
+ // const bool was_guarded = mi_block_check_unguard(page, block, p);
+ mi_free_block_local(page, block, was_guarded, true /* check for a full page */);
}
// free a pointer owned by another thread (page parameter comes first for better codegen)
-static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p) mi_attr_noexcept {
+static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p, bool allow_reclaim) mi_attr_noexcept {
mi_assert_internal(p!=NULL && page != NULL);
- mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));
- const bool was_guarded = mi_block_check_unguard(page, block, p);
- mi_free_block_mt(page, block, was_guarded);
+ bool was_guarded = false;
+ mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded);
+ // mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));
+ // mi_block_check_profiled(page,block,p);
+ // const bool was_guarded = mi_block_check_unguard(page, block, p);
+ mi_free_block_mt(page, block, was_guarded, allow_reclaim);
}
// generic free (for runtime integration)
void mi_decl_noinline _mi_free_generic(mi_page_t* page, bool is_local, void* p) mi_attr_noexcept {
if (is_local) mi_free_generic_local(page,p);
- else mi_free_generic_mt(page,p);
+ else mi_free_generic_mt(page,p,true);
}
// Get the page belonging to a pointer
// Does further checks in debug mode to see if this was a valid pointer.
-static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg)
+static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const char* msg, bool free_small, bool check_p_for_null, mi_page_t** ppage)
{
- MI_UNUSED_RELEASE(msg);
+ MI_UNUSED_RELEASE(msg); MI_UNUSED(free_small);
#if MI_DEBUG
- if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) {
+ if mi_unlikely(!_mi_is_aligned(p,MI_SIZE_SIZE) && !mi_option_is_enabled(mi_option_guarded_precise)) {
_mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p);
- return NULL;
- }
- mi_page_t* page = _mi_safe_ptr_page(p);
- if (p != NULL && page == NULL) {
- _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p);
+ return false;
}
- return page;
- #else
- return _mi_ptr_page(p);
#endif
+
+ mi_page_t* page;
+ #if MI_PAGE_META_SMALL_IS_ALIGNED
+ if (free_small) {
+ const uintptr_t up = _mi_align_down((uintptr_t)p,MI_SMALL_PAGE_SIZE); // like this for codegen on gcc
+ if mi_unlikely(up==0 && check_p_for_null) { return false; }
+ page = (mi_page_t*)up;
+ }
+ else
+ #endif
+ {
+ #if MI_PAGE_META_IS_ALIGNED
+ { page = _mi_aligned_ptr_page0(p); }
+ #else
+ { page = _mi_ptr_page(p); }
+ #endif
+ if mi_unlikely(page==NULL && check_p_for_null) {
+ #if MI_DEBUG
+ if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); }
+ #endif
+ return false;
+ }
+ }
+ #if MI_DEBUG
+ mi_page_t* const cpage = _mi_checked_ptr_page(p);
+ if mi_unlikely(cpage==NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); }
+ #endif
+
+ #if MI_PAGE_META_IS_ALIGNED
+ #if MI_PAGE_META_SMALL_IS_ALIGNED
+ if (free_small) { mi_assert_internal(NULL == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); }
+ else
+ #elif MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE && !MI_GUARDED
+ // for mi_free_small we can avoid a load-acquire (but not when guarded as that may still allocate large blocks)
+ if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); }
+ else
+ #endif
+ { page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); }
+ #endif
+
+ mi_assert_internal(page!=NULL);
+ mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ );
+ #if !MI_GUARDED
+ if (free_small) { mi_assert_internal(page->block_size <= MI_SMALL_MAX_OBJ_SIZE); /* free small should only be called on small pages */ }
+ #endif
+ *ppage = page;
+ return true;
+}
+
+static mi_decl_forceinline bool mi_ptr_page_is_valid(const void* p, const char* msg, mi_page_t** ppage) {
+ return mi_ptr_page_is_valid_ex(p,msg,false /* free_small? */, true /* check_p_for_null */, ppage );
+}
+
+static mi_decl_forceinline mi_page_t* mi_ptr_page_validate(const void* p, const char* msg) {
+ mi_page_t* page;
+ return (mi_ptr_page_is_valid(p,msg,&page) ? page : NULL);
}
// Free a block
// Fast path written carefully to prevent register spilling on the stack
-static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable, mi_page_t* page)
+static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t* pblock_size, bool allow_reclaim)
{
- if mi_unlikely(page==NULL) return; // page will be NULL if p==NULL
mi_assert_internal(p!=NULL && page!=NULL);
- if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }
+ if (pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); }
- const mi_threadid_t xtid = (_mi_prim_thread_id() ^ mi_page_xthread_id(page));
+ const mi_threadid_t ptid = mi_page_xthread_id(page);
+ const mi_threadid_t xtid = (_mi_prim_thread_id() ^ ptid);
if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0`
// thread-local, aligned, and not a full page
mi_block_t* const block = mi_validate_block_from_ptr(page,p);
- mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */);
+ mi_free_block_local(page, block, false /* was guarded */, false /* no need to check if the page is full */);
}
else if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0`
// page is local, but is full or contains (inner) aligned blocks; use generic path
@@ -196,45 +278,125 @@ static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable, mi_page_t* p
else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0`
// blocks are aligned (and not a full page); push on the thread_free list
mi_block_t* const block = mi_validate_block_from_ptr(page,p);
- mi_free_block_mt(page,block,false /* was_guarded */);
+ mi_free_block_mt(page,block,false /* was_guarded */, allow_reclaim);
}
else {
// page is full or contains (inner) aligned blocks; use generic multi-thread path
- mi_free_generic_mt(page, p);
+ mi_free_generic_mt(page, p, allow_reclaim);
}
}
-void mi_free(void* p) mi_attr_noexcept {
- mi_page_t* const page = mi_validate_ptr_page(p,"mi_free");
- mi_free_ex(p, NULL, page);
+void mi_free(void* p) mi_attr_noexcept {
+ mi_page_t* page;
+ if mi_likely(mi_ptr_page_is_valid(p,"mi_free",&page)) {
+ mi_free_nonnull(p, page, NULL, true /* allow reclaim? */);
+ }
}
-void mi_ufree(void* p, size_t* usable) mi_attr_noexcept {
- mi_page_t* const page = mi_validate_ptr_page(p,"mi_ufree");
- mi_free_ex(p, usable, page);
+void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept {
+ mi_page_t* page;
+ if mi_likely(mi_ptr_page_is_valid(p,"mi_ufree",&page)) {
+ mi_free_nonnull(p, page, pblock_size, true /* allow reclaim? */);
+ }
+ else {
+ if (pblock_size!=NULL) { *pblock_size = 0; }
+ }
}
void mi_free_small(void* p) mi_attr_noexcept {
- // We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`.
- // If we keep page info in front of the page area for small objects, we can find the info
- // just by aligning down the pointer instead of looking it up in the page map.
- #if MI_PAGE_META_ALIGNED_FREE_SMALL
- #if MI_GUARDED
- #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as MI_GUARDED is defined"
+ mi_page_t* page;
+ if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small",true /* is_small? */,true /*check p for null*/, &page)) {
+ mi_free_nonnull(p, page, NULL, true /* allow reclaim? */);
+ }
+}
+
+void mi_free_small_nonnull(void* p) mi_attr_noexcept {
+ mi_assert(p!=NULL);
+ mi_page_t* page;
+ if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small_nonnull",true /* is_small? */,false /*check p for null*/, &page)) {
+ mi_free_nonnull(p, page, NULL, true /* allow reclaim? */);
+ }
+}
+
+// Free a pointer that is potentially allocated in a different sub-process
+void _mi_free_subproc_safe(void* p) mi_attr_noexcept {
+ mi_page_t* page;
+ if mi_likely(mi_ptr_page_is_valid(p,"_mi_free_subproc_safe",&page)) {
+ // const bool allow_reclaim = (_mi_subproc() == mi_page_subproc(page));
+ mi_free_nonnull(p, page, NULL, false /* allow reclaim */);
+ }
+}
+
+// ------------------------------------------------------
+// Free variants
+// ------------------------------------------------------
+
+void mi_free_size(void* p, size_t size) mi_attr_noexcept {
+ MI_UNUSED_RELEASE(size);
+ #if MI_DEBUG
+ const mi_page_t* const page = mi_ptr_page_validate(p,"mi_free_size");
+ if (page==NULL) return;
+ mi_assert(p!=NULL);
+ const mi_block_t* block = _mi_page_ptr_unalign(page, p);
+ const size_t usable = _mi_page_usable_size(page,p);
+ if mi_unlikely(size > usable) {
+ const bool is_guarded = mi_block_ptr_is_guarded(block,p);
+ if (!is_guarded) {
+ _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable);
+ mi_free(p);
+ return;
+ }
+ }
+ // const size_t is_aligned = ((void*)block != p);
+ // if mi_unlikely(size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size((is_aligned ? 2 : 1)*MI_SMALL_SIZE_MAX)) { // note: we check *2 in case it was over-aligned
+ // const bool is_guarded = mi_block_ptr_is_guarded(block,p);
+ // if (!is_guarded) {
+ // _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n (maybe a `new[]` was matched with `delete` instead of `delete[]`?)\n", p, size, mi_page_block_size(page));
+ // mi_free(p);
+ // return;
+ // }
+ // }
+ #endif
+ #if MI_PAGE_META_SMALL_IS_ALIGNED || MI_PAGE_META_IS_ALIGNED
+ if mi_likely(size <= MI_SMALL_SIZE_MAX) {
+ mi_free_small(p);
+ }
+ else
+ #endif
+ {
mi_free(p);
- #elif MI_ARENA_SLICE_ALIGN < MI_SMALL_PAGE_SIZE
- #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size"
+ }
+}
+
+void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept {
+ mi_assert(((uintptr_t)p % alignment) == 0);
+ // If the alignment is smaller than the `size`, then for `size <= MI_SMALL_SIZE_MAX`
+ // the block must be allocated within a small page and can thus be handled safely by `mi_free_size`.
+ // (since even with over-allocation the block size will be less than 2*MI_SMALL_SIZE_MAX <= MI_SMALL_OBJ_SIZE_MAX)
+ if mi_likely(alignment <= size) {
+ mi_free_size(p,size);
+ }
+ else {
mi_free(p);
- #else
- mi_page_t* const page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE);
- mi_assert(page == mi_validate_ptr_page(p,"mi_free_small"));
- mi_assert((void*)page == _mi_align_down_ptr(page->page_start,MI_SMALL_PAGE_SIZE));
- mi_assert(page->block_size <= MI_SMALL_SIZE_MAX); // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small`
- mi_free_ex(p, NULL, page);
- #endif
- #else
+ }
+}
+
+void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {
+ MI_UNUSED_RELEASE(alignment);
+ mi_assert(((uintptr_t)p % alignment) == 0);
mi_free(p);
- #endif
+}
+
+// checked free
+bool mi_cfree(void* p) mi_attr_noexcept {
+ mi_page_t* const page = _mi_checked_ptr_page(p);
+ if mi_likely(page!=NULL) {
+ mi_free_nonnull(p, page, NULL, true /* allow reclaim? */);
+ return true;
+ }
+ else {
+ return false;
+ }
}
@@ -270,14 +432,14 @@ static bool mi_abandoned_page_try_reabandon_to_mapped(mi_page_t* page)
// Release ownership of a page. This may free or reabandoned the page if other blocks are concurrently
// freed in the meantime. Returns `true` if the page was freed.
// By passing the captured `expected_thread_free`, we can often avoid calling `mi_page_free_collect`.
-static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expected_thread_free) {
+static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* page_thread_free) {
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(!mi_page_all_free(page));
// try to cas atomically the original free list (`mt_free`) back with the ownership cleared.
- mi_thread_free_t tf_expect = mi_tf_create(expected_thread_free, true);
- mi_thread_free_t tf_new = mi_tf_create(expected_thread_free, false);
- while mi_unlikely(!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_expect, tf_new)) {
+ mi_thread_free_t tf_expect = mi_tf_create(page_thread_free,true);
+ mi_thread_free_t tf_new = mi_tf_set_owned(tf_expect,false);
+ while mi_unlikely(!mi_atomic_cas_strong_acq_rel(&page->xthread_free, &tf_expect, tf_new)) {
mi_assert_internal(mi_tf_is_owned(tf_expect));
// while the xthread_free list is not empty..
while (mi_tf_block(tf_expect) != NULL) {
@@ -286,9 +448,10 @@ static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expec
if (mi_abandoned_page_try_free(page)) return;
if (mi_abandoned_page_try_reabandon_to_mapped(page)) return;
// otherwise continue un-owning
- tf_expect = mi_atomic_load_relaxed(&page->xthread_free);
+ tf_expect = mi_atomic_load_acquire(&page->xthread_free);
}
// and try again to release ownership
+ mi_subproc_stat_counter_increase(mi_page_subproc(page), pages_unabandon_busy_wait, 1);
mi_assert_internal(mi_tf_block(tf_expect)==NULL);
tf_new = mi_tf_create(NULL, false);
}
@@ -315,14 +478,14 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long
mi_assert_internal(!mi_page_all_free(page));
mi_assert_internal(page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE);
mi_assert_internal(reclaim_on_free >= 0);
-
+
// dont reclaim if we just have terminated this thread and we should
// not reinitialize the theap for this thread. (can happen due to thread-local destructors for example -- issue #944)
if (!_mi_thread_is_initialized()) return false;
// get our theap
mi_theap_t* const theap = _mi_page_associated_theap_peek(page);
- if (theap==NULL || !theap->allow_page_reclaim) return false;
+ if (theap==NULL || theap->tld==NULL || !theap->allow_page_reclaim) return false; // see issue #1289
// todo: cache `is_in_threadpool` and `exclusive_arena` directly in the theap for performance?
// set max_reclaim limit
@@ -354,25 +517,27 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long
// We freed a block in an abandoned page (that was not owned). Try to collect
-static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept
+static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept
{
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(mt_free != NULL);
+ // mi_assert_internal(!allow_reclaim || _mi_subproc() == mi_page_subproc(page)); // never reclaim across subprocesses
+
// we own the page now, and it is safe to collect the thread atomic free list
if (page->block_size <= MI_SMALL_SIZE_MAX) {
// use the `_partly` version to avoid atomic operations since we already have the `mt_free` pointing into the thread free list
// (after this the `used` count might be too high (as some blocks may have been concurrently added to the thread free list and are yet uncounted).
// however, if the page became completely free, the used count is guaranteed to be 0.)
mi_assert_internal(page->reserved>=16); // below this even one freed block goes from full to no longer mostly used.
- _mi_page_free_collect_partly(page, mt_free);
+ mt_free = _mi_page_free_collect_partly(page, mt_free);
}
else {
// for larger blocks we use the regular collect
_mi_page_free_collect(page,false /* no force */);
mt_free = NULL; // expected page->xthread_free value after collection
}
- const long reclaim_on_free = _mi_option_get_fast(mi_option_page_reclaim_on_free);
+ const long reclaim_on_free = allow_reclaim && _mi_option_get_fast(mi_option_page_reclaim_on_free);
#if MI_DEBUG > 1
if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); }
if (mi_page_is_full(page)) { mi_assert(mi_page_is_mostly_used(page)); }
@@ -391,112 +556,98 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t*
// ------------------------------------------------------
-// Usable size
+// Usable size
// ------------------------------------------------------
// Bytes available in a block
-static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p) mi_attr_noexcept {
- const mi_block_t* block = _mi_page_ptr_unalign(page, p);
+static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p, const mi_block_t* block) mi_attr_noexcept {
+ // const mi_block_t* block = _mi_page_ptr_unalign(page, p);
const bool is_guarded = mi_block_ptr_is_guarded(block,p);
const size_t size = mi_page_usable_size_of(page, block, is_guarded);
- const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block;
- mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);
- const size_t aligned_size = (size - adjust);
+ mi_assert_internal((void*)p >= (void*)block);
+ const size_t adjust = (uint8_t*)p - (uint8_t*)block;
+ mi_assert_internal(adjust <= size);
+ const size_t aligned_size = (adjust <= size ? size - adjust : 0); // size can be zero if the padding is corrupted
return aligned_size;
}
-static inline size_t _mi_usable_size(const void* p, const mi_page_t* page) mi_attr_noexcept {
+size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept {
if mi_unlikely(page==NULL) return 0;
- if mi_likely(!mi_page_has_interior_pointers(page)) {
- const mi_block_t* block = (const mi_block_t*)p;
+ mi_assert_internal(mi_ptr_page_validate(p,"_mi_page_usable_size") == page);
+ const mi_block_t* block = _mi_page_ptr_unalign(page, p); // for safety, always unalign
+ if mi_likely((const void*)block==page) {
return mi_page_usable_size_of(page, block, false /* is guarded */);
}
else {
// split out to separate routine for improved code generation
- return mi_page_usable_aligned_size_of(page, p);
+ return mi_page_usable_aligned_size_of(page, p, block);
}
}
mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept {
- const mi_page_t* const page = mi_validate_ptr_page(p,"mi_usable_size");
- return _mi_usable_size(p,page);
-}
-
-
-// ------------------------------------------------------
-// Free variants
-// ------------------------------------------------------
-
-void mi_free_size(void* p, size_t size) mi_attr_noexcept {
- MI_UNUSED_RELEASE(size);
- #if MI_DEBUG
- const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size");
- const size_t available = _mi_usable_size(p,page);
- mi_assert(p == NULL || size <= available || available == 0 /* invalid pointer */ );
- #endif
- mi_free(p);
-}
-
-void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept {
- MI_UNUSED_RELEASE(alignment);
- mi_assert(((uintptr_t)p % alignment) == 0);
- mi_free_size(p,size);
-}
-
-void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {
- MI_UNUSED_RELEASE(alignment);
- mi_assert(((uintptr_t)p % alignment) == 0);
- mi_free(p);
+ const mi_page_t* const page = mi_ptr_page_validate(p,"mi_usable_size");
+ return _mi_page_usable_size(page,p);
}
// ------------------------------------------------------
+// Deprecated: double free is usually checked with padding now
+// as that is faster and works better for cross-thread free'ing.
// Check for double free in secure and debug mode
// This is somewhat expensive so only enabled for secure mode 4
// ------------------------------------------------------
-#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0))
+#if MI_SECURE>=3 && !MI_PADDING
// linear check if the free list contains a specific element
-static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) {
- while (list != NULL) {
+static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem, const char* list_kind) {
+ const size_t max_count = page->capacity; // can never hold more blocks than the capacity
+ size_t count = 0;
+ while (list != NULL && count <= max_count) { // double-free can create cycles so we limit the number of iterations
if (elem==list) return true;
list = mi_block_next(page, list);
+ count++;
+ }
+ if mi_unlikely(count > max_count) {
+ _mi_error_message(EFAULT, "corrupted %s list (possibly due to a double free)\n", list_kind);
}
return false;
}
-static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) {
+static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const mi_block_t* block) {
// The decoded value is in the same page (or NULL).
// Walk the free lists to verify positively if it is already freed
- if (mi_list_contains(page, page->free, block) ||
- mi_list_contains(page, page->local_free, block) ||
- mi_list_contains(page, mi_page_thread_free(page), block))
+ if (mi_list_contains(page, page->free, block, "free") ||
+ mi_list_contains(page, page->local_free, block, "local free") ||
+ mi_list_contains(page, mi_page_thread_free(page), block, "thread free"))
{
_mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page));
- return true;
+ return false;
}
- return false;
+ return true;
}
-#define mi_track_page(page,access) { size_t psize; void* pstart = _mi_page_start(_mi_page_segment(page),page,&psize); mi_track_mem_##access( pstart, psize); }
+// Used for double free checking to avoid checking free lists too frequently
+static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi_block_t* block) {
+ mi_block_t* n = mi_block_nextx(page,block,page->keys);
+ return (_mi_is_aligned(block,MI_SIZE_SIZE) && // quick check: aligned pointer?
+ (n==NULL || mi_page_contains_address(page,n))); // quick check: in the same page or NULL?
+}
-static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {
- bool is_double_free = false;
- mi_block_t* n = mi_block_nextx(page, block, page->keys); // pretend it is freed, and get the decoded first field
- if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer?
- (n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL?
+// check if `block` was free'd before
+static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) {
+ if mi_unlikely(mi_block_could_be_double_free(page,block)) // quick check: next field is aligned in the same page or NULL?
{
// Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free?
// (continue in separate function to improve code generation)
- is_double_free = mi_check_is_double_freex(page, block);
+ return mi_check_double_freex(page, block);
}
- return is_double_free;
+ else return true;
}
#else
-static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) {
+static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) {
MI_UNUSED(page);
MI_UNUSED(block);
- return false;
+ return true;
}
#endif
@@ -506,30 +657,31 @@ static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block
// ---------------------------------------------------------------------------
#if MI_PADDING // && !MI_TRACK_ENABLED
-static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) {
+static inline bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize, bool* double_free) {
*bsize = mi_page_usable_block_size(page);
- const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize);
+ mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize);
mi_track_mem_defined(padding,sizeof(mi_padding_t));
*delta = padding->delta;
- uint32_t canary = padding->canary;
- uintptr_t keys[2];
- keys[0] = page->keys[0];
- keys[1] = page->keys[1];
- bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize);
+ const uint32_t canary = padding->canary;
+ const bool ok = (mi_ptr_encode_canary(page,block,page->keys) == canary && *delta <= *bsize);
+ if (double_free!=NULL) {
+ if mi_unlikely(!ok) { *double_free = mi_ptr_decode_canary_is_freed(canary); } // double free?
+ else { padding->canary = mi_ptr_encode_canary_freed(); } // mark as freed
+ }
mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
return ok;
}
// Return the exact usable size of a block.
static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {
- if (is_guarded) {
+ if mi_unlikely(is_guarded) {
const size_t bsize = mi_page_block_size(page);
return (bsize - _mi_os_page_size());
}
else {
size_t bsize;
size_t delta;
- bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
+ bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL);
mi_assert_internal(ok); mi_assert_internal(delta <= bsize);
return (ok ? bsize - delta : 0);
}
@@ -540,9 +692,10 @@ static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* bl
// contain the pointer for the delayed list, then shrink the padding (by decreasing delta)
// so it will later not trigger an overflow error in `mi_free_block`.
void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {
+ if (mi_block_ptr_is_guarded(block,block + 1)) return;
size_t bsize;
size_t delta;
- bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
+ bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL);
mi_assert_internal(ok);
if (!ok || (bsize - delta) >= min_size) return; // usually already enough space
mi_assert_internal(bsize >= min_size);
@@ -555,9 +708,15 @@ void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const si
mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
}
#else
-static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {
- MI_UNUSED(is_guarded); MI_UNUSED(block);
- return mi_page_usable_block_size(page);
+static inline size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {
+ MI_UNUSED(block);
+ if mi_unlikely(is_guarded) {
+ const size_t bsize = mi_page_block_size(page);
+ return (bsize - _mi_os_page_size());
+ }
+ else {
+ return mi_page_usable_block_size(page);
+ }
}
void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {
@@ -565,16 +724,17 @@ void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const si
}
#endif
-#if MI_PADDING && MI_PADDING_CHECK
+#if MI_PADDING
-static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) {
+static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong, bool* is_double_free) {
size_t bsize;
size_t delta;
- bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
+ bool ok = mi_page_decode_padding(page, block, &delta, &bsize, is_double_free );
*size = *wrong = bsize;
if (!ok) return false;
mi_assert_internal(bsize >= delta);
*size = bsize - delta;
+ #if MI_PADDING_CHECK_BYTES
if (!mi_page_is_huge(page)) {
uint8_t* fill = (uint8_t*)block + bsize - delta;
const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes
@@ -588,41 +748,65 @@ static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, si
}
mi_track_mem_noaccess(fill, maxpad);
}
+ #endif
return ok;
}
-static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {
- size_t size;
- size_t wrong;
- if (!mi_verify_padding(page,block,&size,&wrong)) {
- _mi_error_message(EFAULT, "buffer overflow in theap block %p of size %zu: write after %zu bytes\n", block, size, wrong );
+mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) {
+ if mi_unlikely(is_guarded) {
+ const size_t bsize = mi_page_block_size(page);
+ *usable_size = (bsize - _mi_os_page_size());
+ return true;
+ }
+ else {
+ size_t wrong;
+ bool is_double_free;
+ if mi_unlikely(!mi_verify_padding(page,block,usable_size,&wrong,&is_double_free)) {
+ if (is_double_free) {
+ _mi_error_message(EAGAIN, "double free detected of heap block %p with size %zu\n", block, *usable_size);
+ }
+ else {
+ _mi_error_message(EFAULT, "buffer overflow in heap block %p of size %zu: write after %zu bytes\n", block, *usable_size, wrong );
+ }
+ return false;
+ }
+ return true;
}
}
#else
-static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {
- MI_UNUSED(page);
- MI_UNUSED(block);
+mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) {
+ *usable_size = mi_page_usable_size_of(page,block,is_guarded);
+ return true;
}
#endif
// only maintain stats for smaller objects if requested
-#if (MI_STAT>0)
+#if 0 && (MI_STATS>0)
static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
- MI_UNUSED(block);
- mi_theap_t* const theap = _mi_theap_default();
- if (!mi_theap_is_initialized(theap)) return; // (for now) skip statistics if free'd after thread_done was called (usually a thread cleanup call by the OS)
+ MI_UNUSED(block);
+ mi_theap_t* theap = _mi_theap_default();
+ mi_lock_t* lock = NULL;
+ mi_subproc_t* const subproc = mi_page_subproc(page);
+ mi_theap_t* const theap_meta = subproc->theap_meta;
+ if mi_unlikely(!mi_theap_is_initialized(theap) || // can happen if free'd after thread_done was called (usually a thread cleanup call by the OS)
+ // page->theap == subproc->theap_meta .. but we cannot read `theap` if we don't own the page
+ (theap_meta != NULL && mi_page_thread_id(page) == theap_meta->tld->thread_id)) {
+ theap = theap_meta;
+ lock = &subproc->theap_meta_lock;
+ mi_lock_acquire(lock);
+ }
const size_t bsize = mi_page_usable_block_size(page);
- // #if (MI_STAT>1)
+ // #if (MI_STATS>=2)
// const size_t usize = mi_page_usable_size_of(page, block);
// mi_theap_stat_decrease(theap, malloc_requested, usize);
// #endif
if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
mi_theap_stat_decrease(theap, malloc_normal, bsize);
- #if (MI_STAT > 1)
+ #if (MI_STATS > 1)
mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], 1);
#endif
}
@@ -630,29 +814,13 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
const size_t bpsize = mi_page_block_size(page); // match stat in page.c:mi_huge_page_alloc
mi_theap_stat_decrease(theap, malloc_huge, bpsize);
}
+
+ if mi_unlikely(lock!=NULL) {
+ mi_lock_release(lock);
+ }
}
#else
void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
MI_UNUSED(page); MI_UNUSED(block);
}
#endif
-
-
-// Remove guard page when building with MI_GUARDED
-#if MI_GUARDED
-static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) {
- MI_UNUSED(p);
- mi_assert_internal(mi_block_ptr_is_guarded(block, p));
- mi_assert_internal(mi_page_has_interior_pointers(page));
- mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t));
- mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED);
-
- const size_t bsize = mi_page_block_size(page);
- const size_t psize = _mi_os_page_size();
- mi_assert_internal(bsize > psize);
- mi_assert_internal(!page->memid.is_pinned);
- void* gpage = (uint8_t*)block + bsize - psize;
- mi_assert_internal(_mi_is_aligned(gpage, psize));
- _mi_os_unprotect(gpage, psize);
-}
-#endif
diff --git a/vendored/mimalloc/src/heap.c b/vendored/mimalloc/src/heap.c
index f0a016527..d7f1db132 100644
--- a/vendored/mimalloc/src/heap.c
+++ b/vendored/mimalloc/src/heap.c
@@ -1,5 +1,5 @@
/*----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -7,7 +7,8 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc.h"
#include "mimalloc/internal.h"
-#include "mimalloc/prim.h" // _mi_theap_default
+#include "mimalloc/prim.h" // _mi_prim_thread_yield
+#include "mimalloc/prim-tls.h" // _mi_heap_theap
/* -----------------------------------------------------------
@@ -15,7 +16,7 @@ terms of the MIT license. A copy of the license can be found in the file
----------------------------------------------------------- */
mi_theap_t* mi_heap_theap(mi_heap_t* heap) {
- return _mi_heap_theap(heap);
+ return _mi_heap_theap(heap); // in prim.h
}
void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node) {
@@ -30,64 +31,66 @@ void mi_heap_stats_merge_to_subproc(mi_heap_t* heap) {
void mi_heap_stats_merge_to_main(mi_heap_t* heap) {
if (heap==NULL) return;
- _mi_stats_merge_into(&mi_heap_main()->stats, &heap->stats);
+ _mi_stats_merge_into(&mi_heap_get_heap_main(heap)->stats, &heap->stats);
}
+bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap) {
+ mi_assert_internal((uintptr_t)theap == 1 || _mi_theap_heap(theap)==heap);
+ mi_assert_internal(!_mi_is_empty_theap(theap));
+ mi_assert_internal(heap->theap != 0);
+ return _mi_thread_local_set(heap->theap,theap);
+}
+
+// mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap) {
+// mi_theap_t* theap;
+// mi_assert_internal(heap->theap != 0);
+// if mi_likely(heap->theap!=0) { // paranoia
+// theap = (mi_theap_t*)_mi_thread_local_get(heap->theap);
+// }
+// else {
+// _mi_error_message(EFAULT, "no thread-local reserved for heap (%p)\n", heap);
+// return NULL;
+// }
+// mi_assert_internal(!_mi_is_empty_theap(theap));
+// mi_assert_internal(theap->heap == heap); // this goes wrong if using main heaps across subprocesses (as all share the same key)
+// return theap;
+// }
+
+
static mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_heap)
{
mi_heap_t* heap = (mi_heap_t*)const_heap;
mi_assert_internal(heap!=NULL);
- if (_mi_is_heap_main(heap)) {
- // this can be called if the (main) thread is not yet initialized (as no allocation happened)
- // but `theap_main_init_get()` will call `mi_thread_init()`
- mi_theap_t* const theap = _mi_theap_main_safe();
- mi_assert_internal(theap!=NULL && _mi_is_heap_main(_mi_theap_heap(theap)));
- return theap;
+ // initialize thread first in case this is the main heap
+ // (which may allocate the default theap already for the main heap)
+ if (!_mi_thread_is_initialized()) {
+ mi_thread_init();
}
- // otherwise initialize the theap for this heap
- // get the thread local
- mi_assert_internal(heap->theap != 0);
- if (heap->theap==0) { // paranoia
- _mi_error_message(EFAULT, "no thread-local reserved for heap (%p)\n", heap);
- return NULL;
- }
+ // get the thread local theap
mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap);
// create a fresh theap?
if (theap==NULL) {
- // set first an invalid value to ensure the thread local storage is allocated
- if (!_mi_thread_local_set(heap->theap, (mi_theap_t*)1)) {
- _mi_error_message(EFAULT, "unable to allocate memory for thread local storage\n");
- return NULL;
- }
- // then allocate the theap
- theap = _mi_theap_create(heap, _mi_theap_default_safe()->tld);
- _mi_thread_local_set(heap->theap, theap); // Cannot fail now as it was set before. Always set so the local is valid or NULL (and not 1)
+ // allocate a fresh theap
+ theap = _mi_theap_create(heap, mi_theap_get_default()->tld); // sets the theap thread local
if (theap==NULL) {
_mi_error_message(EFAULT, "unable to allocate memory for a thread local heap\n");
return NULL;
- }
+ }
+ _mi_heap_theap_set(heap, theap);
+ mi_assert_internal(theap == (mi_theap_t*)_mi_thread_local_get(heap->theap));
}
return theap;
}
-// get the theap for a heap without initializing (and return NULL in that case)
-mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap) {
- if (heap==NULL || _mi_is_heap_main(heap)) {
- return _mi_theap_main_safe();
- }
- else {
- return (mi_theap_t*)_mi_thread_local_get(heap->theap);
- }
-}
-
// get (and possibly create) the theap belonging to a heap
-mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap)
+mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap)
{
- mi_theap_t* theap = _mi_heap_theap_peek(heap);
+ mi_assert_internal(heap->theap != 0);
+ mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap);
if mi_unlikely(theap==NULL) {
theap = mi_heap_init_theap(heap);
if (theap==NULL) { return (mi_theap_t*)&_mi_theap_empty_wrong; } // this will return NULL from page.c:_mi_malloc_generic
@@ -96,28 +99,15 @@ mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap)
return theap;
}
-
-mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) {
- // always allocate heap data in the (subprocess) main heap
- mi_heap_t* const heap_main = mi_heap_main();
- // todo: allocate heap data in the exclusive arena ?
- mi_heap_t* const heap = (mi_heap_t*)mi_heap_zalloc( heap_main, sizeof(mi_heap_t) );
- if (heap==NULL) return NULL;
-
- // reserve a thread local slot for this heap (see also issue #1230)
- const mi_thread_local_t theap_slot = _mi_thread_local_create();
- if (theap_slot == 0) {
- _mi_error_message(EFAULT, "unable to dynamically create a thread local for a heap\n");
- mi_free(heap);
- return NULL;
- }
-
+void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap_slot, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id)
+{
// init fields
heap->theap = theap_slot;
- heap->subproc = heap_main->subproc;
- heap->heap_seq = mi_atomic_increment_relaxed(&heap_main->subproc->heap_total_count);
+ heap->subproc = subproc;
+ heap->heap_seq = mi_atomic_increment_relaxed(&subproc->heap_total_count);
heap->exclusive_arena = _mi_arena_from_id(exclusive_arena_id);
heap->numa_node = -1; // no initial affinity
+ heap->profiler = mi_atomic_load_ptr_acquire(mi_profiler_t,&subproc->profiler);
mi_stats_header_init(&heap->stats);
mi_lock_init(&heap->theaps_lock);
mi_lock_init(&heap->os_abandoned_pages_lock);
@@ -131,94 +121,141 @@ mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) {
if (head!=NULL) { head->prev = heap; }
heap->subproc->heaps = heap;
}
- mi_atomic_increment_relaxed(&heap_main->subproc->heap_count);
- mi_subproc_stat_increase(heap_main->subproc, heaps, 1);
+ mi_atomic_increment_relaxed(&subproc->heap_count);
+ mi_subproc_stat_increase(subproc, heaps, 1);
+ mi_assert_internal(_mi_is_heap_main(heap) ? heap->theap == mi_thread_local_key_fast : heap->theap != 0);
+}
+
+mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_main_heap) {
+ mi_assert_internal(is_main_heap ? (subproc->heap_main == NULL && subproc->parent != NULL) : subproc->heap_main != NULL);
+ // heap data is allocated in the current subproc
+ mi_heap_t* const heap_main = (is_main_heap ? subproc->parent->heap_main : subproc->heap_main);
+ // todo: allocate heap data in the exclusive arena ?
+ mi_heap_t* const heap = (mi_heap_t*)mi_heap_zalloc( heap_main, sizeof(mi_heap_t) );
+ if (heap==NULL) return NULL;
+ // reserve a thread local slot for this heap (see also issue #1230)
+ mi_thread_local_t theap_slot = (is_main_heap ? mi_thread_local_key_fast : _mi_thread_local_create());
+ if (theap_slot == 0) {
+ _mi_error_message(EFAULT, "unable to dynamically create a thread local for a heap\n");
+ mi_free(heap);
+ return NULL;
+ }
+ if (is_main_heap) {
+ mi_assert_internal(subproc->heap_main == NULL);
+ subproc->heap_main = heap;
+ }
+ _mi_heap_init(heap, theap_slot, subproc, exclusive_arena_id);
return heap;
}
+mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) {
+ // `mi_heap_new` may be the very first mimalloc call in a process, in which case the
+ // main heap does not exist yet and `_mi_heap_new_for_subproc` would allocate from a NULL `subproc->heap_main`.
+ mi_thread_init();
+ return _mi_heap_new_for_subproc(_mi_subproc(), exclusive_arena_id, false);
+}
+
mi_heap_t* mi_heap_new(void) {
return mi_heap_new_in_arena(0);
}
// free all theaps belonging to this heap (without deleting their pages as we do this arena wise for efficiency)
static void mi_heap_free_theaps(mi_heap_t* heap) {
- // This can run concurrently with a thread that terminates (see `init.c:mi_thread_theaps_done`),
+ // This can run concurrently with a thread that terminates (see `init.c:mi_thread_theaps_done`),
// and we need to ensure we free theaps atomically.
- // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock
- // potential concurrent thread termination which tries to remove the theap from our theaps list.
- bool all_freed;
- do {
- all_freed = true;
- mi_theap_t* theap = NULL;
- mi_lock(&heap->theaps_lock) {
- theap = heap->theaps;
- while(theap != NULL) {
- mi_theap_t* next = theap->hnext;
- if (!_mi_theap_free(theap, false /* dont re-acquire the heap->theaps_lock */, true /* acquire the tld->theaps_lock though */ )) {
- all_freed = false;
- }
- theap = next;
- }
- }
- if (!all_freed) {
- mi_heap_stat_counter_increase(heap,heaps_delete_wait,1);
- _mi_prim_thread_yield();
+
+ // We first detach our theaps list from any thread local lists
+ _mi_heap_detach_theaps(heap);
+
+ // Now we can safely free the theaps
+ mi_lock(&heap->theaps_lock) { // paranoia
+ mi_theap_t* theap = heap->theaps;
+ heap->theaps = NULL;
+ while(theap != NULL) {
+ mi_theap_t* next = theap->hnext;
+ theap->hnext = NULL;
+ theap->hprev = NULL;
+ mi_assert_internal(theap->tld==NULL);
+ // merge stats into the owning heap stats
+ _mi_stats_merge_into(&heap->stats, &theap->stats);
+ // and free
+ _mi_theap_decref(theap); // a cached entry can still point to the theap
+ theap = next;
}
- else {
- mi_assert_internal(heap->theaps==NULL);
- }
+ }
+
+ // set the theap thread local to NULL (so _mi_page_associated_theap does not read from a freed theap (through delete pages -> page_update_stats))
+ if (!_mi_is_process_heap_main(heap)) {
+ _mi_thread_local_free(heap->theap);
+ heap->theap = 0;
}
- while(!all_freed);
}
// free the heap resources (assuming the pages are already moved/destroyed, and all theaps have been freed)
-static void mi_heap_free(mi_heap_t* heap) {
- mi_assert_internal(heap!=NULL && !_mi_is_heap_main(heap));
+static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) {
+ mi_assert_internal(heap!=NULL); // && !_mi_is_process_heap_main(heap));
// free all arena pages infos
- mi_lock(&heap->arena_pages_lock) {
- for (size_t i = 0; i < MI_MAX_ARENAS; i++) {
- mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);
- if (arena_pages!=NULL) {
- mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL);
- mi_free(arena_pages);
+ const bool is_main = _mi_is_heap_main(heap);
+ if (!is_main) { // pages for the main heap are pre-allocated in the arenas
+ mi_lock(&heap->arena_pages_lock) {
+ for (size_t i = 0; i < MI_MAX_ARENAS; i++) {
+ mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);
+ if (arena_pages!=NULL) {
+ mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL);
+ _mi_free_subproc_safe(arena_pages);
+ }
}
}
}
// remove the heap from the subproc
- mi_heap_stats_merge_to_main(heap);
+ if (!is_main) {
+ mi_heap_stats_merge_to_main(heap);
+ }
+ else {
+ _mi_stats_merge_into(&heap->subproc->stats,&heap->stats);
+ }
mi_atomic_decrement_relaxed(&heap->subproc->heap_count);
mi_subproc_stat_decrease(heap->subproc, heaps, 1);
- mi_lock(&heap->subproc->heaps_lock) {
+ mi_lock_maybe(&heap->subproc->heaps_lock, acquire_heaps_lock) {
if (heap->next!=NULL) { heap->next->prev = heap->prev; }
if (heap->prev!=NULL) { heap->prev->next = heap->next; }
else { heap->subproc->heaps = heap->next; }
}
- _mi_thread_local_free(heap->theap);
mi_lock_done(&heap->theaps_lock);
mi_lock_done(&heap->os_abandoned_pages_lock);
mi_lock_done(&heap->arena_pages_lock);
- mi_free(heap);
+ if (!_mi_is_process_heap_main(heap)) {
+ // _mi_thread_local_free(heap->theap);
+ _mi_free_subproc_safe(heap);
+ }
}
void mi_heap_delete(mi_heap_t* heap) {
if (heap==NULL) return;
- if (_mi_is_heap_main(heap)) {
+ mi_heap_t* heap_main = mi_heap_get_heap_main(heap);
+ if (heap == heap_main) {
_mi_warning_message("cannot delete the main heap\n");
return;
}
mi_heap_free_theaps(heap);
- _mi_heap_move_pages(heap, mi_heap_main());
- mi_heap_free(heap);
+ _mi_heap_move_pages(heap, heap_main);
+ mi_heap_free(heap,true /* acquire subproc->heaps_lock */);
}
-void _mi_heap_force_destroy(mi_heap_t* heap) {
+void _mi_heap_force_destroy(mi_heap_t* heap, bool acquire_heaps_lock) {
if (heap==NULL) return;
mi_heap_free_theaps(heap);
_mi_heap_destroy_pages(heap);
- if (!_mi_is_heap_main(heap)) { mi_heap_free(heap); } // todo: release locks of the main heap?
+ // if (_mi_subproc_main()->heap_main == heap) {
+ // _mi_stats_merge_into(&heap->subproc->stats,&heap->stats);
+ // }
+ // else
+ {
+ mi_heap_free(heap, acquire_heaps_lock); // todo: release locks of the main heap?
+ }
}
void mi_heap_destroy(mi_heap_t* heap) {
@@ -227,17 +264,18 @@ void mi_heap_destroy(mi_heap_t* heap) {
_mi_warning_message("cannot destroy the main heap\n");
return;
}
- _mi_heap_force_destroy(heap);
+ _mi_heap_force_destroy(heap,true /* acquire subproc->heaps_lock */);
}
mi_heap_t* mi_heap_of(const void* p) {
- mi_page_t* page = _mi_safe_ptr_page(p);
+ mi_page_t* const page = _mi_safe_ptr_page(p);
if (page==NULL) return NULL;
return mi_page_heap(page);
}
bool mi_any_heap_contains(const void* p) {
- return (mi_heap_of(p)!=NULL);
+ mi_page_t* const page = _mi_safe_ptr_page(p);
+ return (page!=NULL);
}
bool mi_heap_contains(const mi_heap_t* heap, const void* p) {
@@ -256,8 +294,8 @@ bool mi_check_owned(const void* p) {
bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept {
if (p==NULL) return false;
const mi_page_t* const page = _mi_safe_ptr_page(p); // Get the page containing this pointer
- if (page==NULL || page->used==page->capacity || page->capacity < page->reserved) return false;
- // If the page is the head of the queue, it is currently being used for
+ if (page==NULL || mi_page_used(page)==page->capacity || page->capacity < page->reserved) return false;
+ // If the page is the head of the queue, it is currently being used for
// allocations; we skip it to avoid immediate thrashing.
if (page->prev == NULL) return false;
@@ -265,9 +303,9 @@ bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc
const mi_heap_t* const page_heap = mi_page_heap(page);
if (page_heap==NULL) return false;
if (heap!=NULL && page_heap!=heap) return false;
-
+
// check utilization
if (page->capacity==0) return false;
if (perc_threshold>=100) return true;
- return (perc_threshold >= ((100UL*page->used) / page->capacity));
+ return (perc_threshold >= ((100UL*mi_page_used(page)) / page->capacity));
}
diff --git a/vendored/mimalloc/src/init.c b/vendored/mimalloc/src/init.c
index 1fabd8918..9b0ded5b5 100644
--- a/vendored/mimalloc/src/init.c
+++ b/vendored/mimalloc/src/init.c
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -7,44 +7,61 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc.h"
#include "mimalloc/internal.h"
#include "mimalloc/prim.h"
+#include "mimalloc/prim-tls.h"
#include <string.h> // memcpy, memset
#include <stdlib.h> // atexit
-#define MI_MEMID_INIT(kind) {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ }
-#define MI_MEMID_STATIC MI_MEMID_INIT(MI_MEM_STATIC)
-
// Empty page used to initialize the small free pages array
-const mi_page_t _mi_page_empty = {
+static const mi_page_t mi_page_empty = {
+ #if MI_PAGE_META_IS_ALIGNED
+ MI_ATOMIC_VAR_INIT(NULL), // self
+ #endif
MI_ATOMIC_VAR_INIT(0), // xthread_id
NULL, // free
- 0, // used
+ {0}, // xused
+ #if MI_SIZE_SIZE < 8
+ 0, // xlast_used
+ 0, // xlast_alloc
+ #endif
+ NULL, // local_free
+ 0, // block_size
+ 0, // page_offset
0, // capacity
0, // reserved capacity
+ 0, // slice_pcommitted
0, // retire_expire
false, // is_zero
- NULL, // local_free
MI_ATOMIC_VAR_INIT(0), // xthread_free
- 0, // block_size
- NULL, // page_start
- #if (MI_PADDING || MI_ENCODE_FREELIST)
- { 0, 0 }, // keys
- #endif
NULL, // theap
NULL, // heap
NULL, NULL, // next, prev
- MI_ARENA_SLICE_SIZE, // page_committed
- MI_MEMID_STATIC // memid
+ MI_MEMID_STATIC, // memid
+ #if (MI_PADDING || MI_ENCODE_FREELIST)
+ #if MI_PAGE_KEY_COUNT==2
+ { 0, 0 }, // keys
+ #else
+ { 0 }, // key
+ #endif
+ #endif
};
-#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty)
+#define MI_PAGE_EMPTY() ((mi_page_t*)&mi_page_empty)
+
+#if MI_SMALL_WSIZE_MAX == 128
+#define MI_INIT_PAGES_DIRECT(p) MI_INIT128(p)
+#elif MI_SMALL_WSIZE_MAX == 256
+#define MI_INIT_PAGES_DIRECT(p) MI_INIT128(p), MI_INIT128(p)
+#else
+#error define initializer for direct pages
+#endif
-#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8)
-#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
+#if (MI_PADDING>0) && (MI_SIZE_SIZE >= 8)
+#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
#elif (MI_PADDING>0)
-#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
+#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
#else
-#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() }
+#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY() }
#endif
@@ -70,15 +87,16 @@ const mi_page_t _mi_page_empty = {
#define MI_STAT_COUNT(stat) {0,0,0},
#define MI_STAT_COUNTER(stat) {0},
-#define MI_STATS_NULL \
- MI_STAT_FIELDS() \
- \
- { MI_INIT4(MI_STAT_COUNT_NULL) }, \
- { { 0 }, { 0 }, { 0 }, { 0 } }, \
- \
- { MI_INIT74(MI_STAT_COUNT_NULL) }, \
- { MI_INIT74(MI_STAT_COUNT_NULL) }, \
- { MI_INIT5(MI_STAT_COUNT_NULL) }
+#define MI_STATS_FIELDS_NULL \
+ MI_STAT_FIELDS() /* regular stat fields */ \
+ { MI_INIT4(MI_STAT_COUNT_NULL) }, /* stat reserved */ \
+ { { 0 }, { 0 }, { 0 }, { 0 } }, /* stat counter reserved */ \
+ { MI_INIT74(MI_STAT_COUNT_NULL) }, /* malloc_bins */ \
+ { MI_INIT74(MI_STAT_COUNT_NULL) }, /* page bins */ \
+ { MI_INIT5(MI_STAT_COUNT_NULL) } /* chunk bins */
+
+#define MI_STATS_NULL \
+ { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_FIELDS_NULL }
// --------------------------------------------------------
// Statically allocate an empty theap as the initial
@@ -89,21 +107,11 @@ const mi_page_t _mi_page_empty = {
// may lead to allocation itself on some platforms)
// --------------------------------------------------------
-static mi_decl_cache_align mi_subproc_t subproc_main
-#if __cplusplus
- = { }; // empty initializer to prevent running the constructor (with msvc)
-#else
- = { 0 }; // C zero initialize
-#endif
-
-static mi_subproc_t* subprocs = &subproc_main;
-static mi_lock_t subprocs_lock;
-
-static mi_decl_cache_align mi_tld_t tld_empty = {
- 0, // thread_id
+static mi_decl_cache_align mi_tld_t mi_tld_detached = {
+ MI_THREADID_DETACHED, // thread_id
0, // thread_seq
0, // default numa node
- &subproc_main, // subproc
+ NULL, // subproc
NULL, // theaps list
MI_LOCK_INITIALIZER, // theaps lock
false, // recurse
@@ -111,37 +119,21 @@ static mi_decl_cache_align mi_tld_t tld_empty = {
MI_MEMID_STATIC // memid
};
-mi_decl_cache_align const mi_theap_t _mi_theap_empty = {
- &tld_empty, // tld
+mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = {
+ MI_SMALL_PAGES_EMPTY, // direct small pages
+ &mi_tld_detached, // tld
MI_ATOMIC_VAR_INIT(NULL), // heap
+ MI_ATOMIC_VAR_INIT(NULL), // subproc
MI_ATOMIC_VAR_INIT(1), // refcount
- 0, // heartbeat
- 0, // cookie
- { {0}, {0}, 0, true }, // random
- 0, // page count
- MI_BIN_FULL, 0, // page retired min/max
- 0, // pages_full_size
- 0, 0, // generic count
- NULL, NULL, // tnext, tprev
- NULL, NULL, // hnext, hprev
0, // full page retain
false, // allow reclaim
true, // allow abandon
- #if MI_GUARDED
- 0, 0, 0, 1, // sample count is 1 so we never write to it (see `internal.h:mi_theap_malloc_use_guarded`)
- #endif
- MI_SMALL_PAGES_EMPTY,
- MI_PAGE_QUEUES_EMPTY,
- MI_MEMID_STATIC,
- { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats
-};
-
-mi_decl_cache_align const mi_theap_t _mi_theap_empty_wrong = {
- &tld_empty, // tld
- MI_ATOMIC_VAR_INIT(NULL), // heap
- MI_ATOMIC_VAR_INIT(1), // refcount
+ true, // is_detached
+ ~MI_ZU(0), // sample countdown: "-1" (with a sample rate of 0, so we won't write to the empty theap with MI_SAMPLE==2)
+ 0, 0, // sample rate, requested
+ 0, 0, // profile rate, countdown
+ 0, 0, 0, 0, // guarded rate, countdown, min, max
0, // heartbeat
- 0, // cookie
{ {0}, {0}, 0, true }, // random
0, // page count
MI_BIN_FULL, 0, // page retired min/max
@@ -149,128 +141,38 @@ mi_decl_cache_align const mi_theap_t _mi_theap_empty_wrong = {
0, 0, // generic count
NULL, NULL, // tnext, tprev
NULL, NULL, // hnext, hprev
- 0, // full page retain
- false, // allow reclaim
- true, // allow abandon
- #if MI_GUARDED
- 0, 0, 0, 1, // sample count is 1 so we never write to it (see `internal.h:mi_theap_malloc_use_guarded`)
- #endif
- MI_SMALL_PAGES_EMPTY,
MI_PAGE_QUEUES_EMPTY,
MI_MEMID_STATIC,
- { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats
+ MI_STATS_NULL, // stats
};
-// Heap for the main thread
-
-extern mi_decl_hidden mi_decl_cache_align mi_theap_t theap_main;
-extern mi_decl_hidden mi_decl_cache_align mi_heap_t heap_main;
-
-static mi_decl_cache_align mi_tld_t tld_main = {
- 0, // thread_id
- 0, // thread_seq
- 0, // numa node
- &subproc_main, // subproc
- &theap_main, // theaps list
- MI_LOCK_INITIALIZER, // theaps lock
- false, // recurse
- false, // is_in_threadpool
- MI_MEMID_STATIC // memid
-};
-
-mi_decl_cache_align mi_theap_t theap_main = {
- &tld_main, // thread local data
- MI_ATOMIC_VAR_INIT(&heap_main), // main heap
- MI_ATOMIC_VAR_INIT(1), // refcount
- 0, // heartbeat
- 0, // initial cookie
- { {0x846ca68b}, {0}, 0, true }, // random
- 0, // page count
- MI_BIN_FULL, 0, // page retired min/max
- 0, // pages_full_size
- 0, 0, // generic count
- NULL, NULL, // tnext, tprev
- NULL, NULL, // hnext, hprev
- 2, // full page retain
- true, // allow page reclaim
- true, // allow page abandon
- #if MI_GUARDED
- 0, 0, 0, 0,
- #endif
- MI_SMALL_PAGES_EMPTY,
- MI_PAGE_QUEUES_EMPTY,
- MI_MEMID_STATIC,
- { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats
-};
-
-mi_decl_cache_align mi_heap_t heap_main
-#if __cplusplus
- = { }; // empty initializer to prevent running the constructor (with msvc)
-#else
- = { 0 }; // C zero initialize
-#endif
-
-// the theap belonging to the main heap
-mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_main = NULL;
+#undef MI_STAT_COUNT
+#undef MI_STAT_COUNTER
-mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
- mi_threadid_t tid = _mi_prim_thread_id();
- mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits
- return tid;
-}
-#if MI_TLS_MODEL_THREAD_LOCAL
-// the thread-local main theap for allocation
-mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default = (mi_theap_t*)&_mi_theap_empty;
-// the last used non-main theap
-mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached = (mi_theap_t*)&_mi_theap_empty;
-#endif
+// pre-allocate the process heap, and meta-data theap
+static mi_decl_cache_align mi_heap_t mi_process_heap_main = mi_init_struct_zero;
+static mi_decl_cache_align mi_theap_t mi_process_theap_meta = mi_init_struct_zero;
-bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`.
+// pre-allocate the initial tld and theap for the main thread (this is not strictly needed but nice for stats)
+static mi_decl_cache_align mi_tld_t mi_process_tld_main = mi_init_struct_zero;
+static mi_decl_cache_align mi_theap_t mi_process_theap_main = mi_init_struct_zero;
-mi_stats_t _mi_stats_main = { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL };
+mi_decl_hidden mi_decl_cache_align mi_theap_t _mi_theap_empty_wrong = mi_init_struct_zero; // used for error paths
+mi_decl_hidden bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`.
-#undef MI_STAT_COUNT
-#undef MI_STAT_COUNTER
-
-
-#if MI_GUARDED
-mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) {
- theap->guarded_sample_rate = sample_rate;
- theap->guarded_sample_count = sample_rate; // count down samples
- if (theap->guarded_sample_rate > 1) {
- if (seed == 0) {
- seed = _mi_theap_random_next(theap);
- }
- theap->guarded_sample_count = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate`
- }
-}
-mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) {
- theap->guarded_size_min = min;
- theap->guarded_size_max = (min > max ? min : max);
+mi_page_t* _mi_page_empty_get(void) {
+ return (mi_page_t*)&mi_page_empty;
}
-void _mi_theap_guarded_init(mi_theap_t* theap) {
- mi_theap_guarded_set_sample_rate(theap,
- (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX),
- (size_t)mi_option_get(mi_option_guarded_sample_seed));
- mi_theap_guarded_set_size_bound(theap,
- (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX),
- (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) );
-}
-#else
-mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) {
- MI_UNUSED(theap); MI_UNUSED(sample_rate); MI_UNUSED(seed);
+mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void) {
+ return (mi_theap_t*)&_mi_theap_empty;
}
-mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) {
- MI_UNUSED(theap); MI_UNUSED(min); MI_UNUSED(max);
-}
-void _mi_theap_guarded_init(mi_theap_t* theap) {
- MI_UNUSED(theap);
+bool _mi_is_empty_theap(const mi_theap_t* theap) {
+ return (theap == &_mi_theap_empty);
}
-#endif
/* -----------------------------------------------------------
Initialization
@@ -278,365 +180,212 @@ void _mi_theap_guarded_init(mi_theap_t* theap) {
can cause allocation and induce recursion during initialization.
----------------------------------------------------------- */
+static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc);
-// Initialize main subproc
-static void mi_subproc_main_init(void) {
- if (subproc_main.memid.memkind != MI_MEM_STATIC) {
- subproc_main.memid = _mi_memid_create(MI_MEM_STATIC);
- subproc_main.heaps = &heap_main;
- subproc_main.heap_total_count = 1;
- subproc_main.heap_count = 1;
- mi_atomic_store_ptr_release(mi_heap_t, &subproc_main.heap_main, &heap_main);
- __mi_stat_increase_mt(&subproc_main.stats.heaps, 1);
- mi_stats_header_init(&subproc_main.stats);
- mi_lock_init(&subproc_main.arena_reserve_lock);
- mi_lock_init(&subproc_main.heaps_lock);
- mi_lock_init(&subprocs_lock);
- mi_lock_init(&tld_empty.theaps_lock);
- }
-}
+// Initialize main heap
+static void mi_heap_main_init_once(void) {
+ mi_memid_t memid_static = _mi_memid_create(MI_MEM_STATIC);
+ _mi_memcpy(&_mi_theap_empty_wrong,&_mi_theap_empty,sizeof(_mi_theap_empty_wrong));
-// Initialize main tld
-static void mi_tld_main_init(void) {
- if (tld_main.thread_id == 0) {
- tld_main.thread_id = _mi_prim_thread_id();
- mi_lock_init(&tld_main.theaps_lock);
- }
-}
+ // initialize the main subprocess
+ mi_subproc_t* subproc_main = _mi_subproc_main_init();
-void _mi_theap_options_init(mi_theap_t* theap) {
- theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0);
- theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0);
- theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32);
-}
+ // detached tld for mi_theap_empty (and theap_meta)
+ mi_tld_detached.memid = memid_static;
+ mi_tld_init(&mi_tld_detached, 0, subproc_main);
-// Initialization of the (statically allocated) main theap, and the main tld and subproc.
-static void mi_theap_main_init(void) {
- if mi_unlikely(theap_main.memid.memkind != MI_MEM_STATIC) {
- // theap
- theap_main.memid = _mi_memid_create(MI_MEM_STATIC);
- #if defined(__APPLE__) || defined(_WIN32) && !defined(MI_SHARED_LIB)
- _mi_random_init_weak(&theap_main.random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185)
- #else
- _mi_random_init(&theap_main.random);
- #endif
- theap_main.cookie = _mi_theap_random_next(&theap_main);
- _mi_theap_options_init(&theap_main);
- _mi_theap_guarded_init(&theap_main);
- }
-}
+ // main process heap
+ mi_process_heap_main.memid = memid_static;
+ mi_atomic_store_ptr_release(mi_heap_t,&subproc_main->heap_main,&mi_process_heap_main);
+ _mi_heap_init(&mi_process_heap_main,mi_thread_local_key_fast,subproc_main,0);
-// Initialize main heap
-static void mi_heap_main_init(void) {
- if mi_unlikely(heap_main.subproc == NULL) {
- heap_main.subproc = &subproc_main;
- heap_main.theaps = &theap_main;
+ // detached theap for allocating meta-data (we can allocate on this without having an initialized thread)
+ mi_process_theap_meta.memid = memid_static;
+ _mi_theap_init(&mi_process_theap_meta,&mi_process_heap_main,&mi_tld_detached);
+ mi_process_theap_meta.allow_page_abandon = false; // for security, don't share with other threads
+ mi_process_theap_meta.page_full_retain = 2;
+ mi_process_theap_meta.sample_rate = 0; // no sampling for meta data
+ mi_process_theap_meta.sample_countdown = 0;
+ subproc_main->theap_meta = &mi_process_theap_meta;
- mi_theap_main_init();
- mi_subproc_main_init();
- mi_tld_main_init();
+ // mi_heap_theap_set(&mi_process_heap_main,&mi_process_theap_main); // set in `mi_thread_init(_theap_default)`
+}
- mi_lock_init(&heap_main.theaps_lock);
- mi_lock_init(&heap_main.os_abandoned_pages_lock);
- mi_lock_init(&heap_main.arena_pages_lock);
+static void mi_heap_main_init(void) {
+ mi_atomic_do_once {
+ mi_heap_main_init_once();
}
}
+mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) {
+ mi_heap_t* heap = mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main);
+ if mi_likely(heap!=NULL) {
+ return heap;
+ }
+ else if (_mi_subproc_is_main(subproc)) {
+ mi_heap_main_init();
+ mi_assert_internal(mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main) != NULL);
+ return mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main);
+ }
+ else {
+ mi_assert_internal(false);
+ return &mi_process_heap_main;
+ }
+}
/* -----------------------------------------------------------
Thread local data
----------------------------------------------------------- */
-// Allocate fresh tld
-static mi_tld_t* mi_tld_alloc(void) {
- if (_mi_is_main_thread()) {
- mi_atomic_increment_relaxed(&tld_main.subproc->thread_count);
- return &tld_main;
+static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc) {
+ tld->subproc = subproc;
+ tld->theaps = NULL;
+ mi_lock_init(&tld->theaps_lock);
+ if (tld->thread_id == MI_THREADID_DETACHED) {
+ tld->numa_node = -1;
}
else {
- // allocate tld meta-data
- // note: we need to be careful to not access the tld from `_mi_meta_zalloc`
- // (and in turn from `_mi_arena_alloc_aligned` and `_mi_os_alloc_aligned`).
- mi_memid_t memid;
- mi_tld_t* tld = (mi_tld_t*)_mi_meta_zalloc(sizeof(mi_tld_t), &memid);
- if (tld==NULL) {
- _mi_error_message(ENOMEM, "unable to allocate memory for thread local data\n");
- return NULL;
- }
- tld->memid = memid;
- tld->theaps = NULL;
- mi_lock_init(&tld->theaps_lock);
- tld->subproc = &subproc_main;
tld->numa_node = _mi_os_numa_node();
tld->thread_id = _mi_prim_thread_id();
- tld->thread_seq = mi_atomic_increment_relaxed(&tld->subproc->thread_total_count);
tld->is_in_threadpool = _mi_prim_thread_is_in_threadpool();
+ tld->thread_seq = tseq;
mi_atomic_increment_relaxed(&tld->subproc->thread_count);
- return tld;
}
+ return tld;
}
-#define MI_TLD_INVALID ((mi_tld_t*)1)
-
-mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) {
- mi_lock_done(&tld->theaps_lock);
- if (tld != NULL && tld != MI_TLD_INVALID) {
- mi_atomic_decrement_relaxed(&tld->subproc->thread_count);
- _mi_meta_free(tld, sizeof(mi_tld_t), tld->memid);
- }
- #if 0
- // do not read/write to `thread_tld` on older macOS <= 14 as that will re-initialize the thread local storage
- // (since we are calling this during pthread shutdown)
- // (and this could happen on other systems as well, so let's never do it)
- thread_tld = MI_TLD_INVALID;
- #endif
-}
-
-// return the thread local heap ensuring it is initialized (and not `NULL` or `&_mi_theap_empty`);
-mi_theap_t* _mi_theap_default_safe(void) {
- mi_theap_t* theap = _mi_theap_default();
- if mi_likely(mi_theap_is_initialized(theap)) return theap;
- mi_thread_init();
- mi_assert_internal(mi_theap_is_initialized(_mi_theap_default()));
- return _mi_theap_default();
-}
-
-// return the main theap ensuring it is initialized.
-mi_theap_t* _mi_theap_main_safe(void) {
- mi_theap_t* theap = __mi_theap_main;
- if mi_unlikely(theap==NULL) { // if thread_init or default_set was never called
- mi_thread_init(); // sets the default slot to the main theap
- theap = _mi_theap_default();
- mi_assert_internal(theap!=NULL);
- mi_assert_internal(_mi_is_theap_main(theap));
- if (_mi_is_theap_main(theap)) {
- __mi_theap_main = theap;
- }
- }
- mi_assert_internal(theap!=NULL && _mi_is_theap_main(theap));
- return theap;
-}
-
-
-mi_subproc_t* _mi_subproc_main(void) {
- return &subproc_main;
-}
+// Allocate fresh tld
+static mi_tld_t* mi_tld_create(mi_subproc_t* subproc) {
+ mi_assert_internal(subproc->theap_meta != NULL); // should be initialized on the main thread before other threads allocate
+ const size_t tseq = mi_atomic_increment_relaxed(&subproc->thread_total_count);
-mi_subproc_t* _mi_subproc(void) {
- // should work without doing initialization (as it may be called from `_mi_tld -> mi_tld_alloc ... -> os_alloc -> _mi_subproc()`
- // todo: this will still fail on OS systems where the first access to a thread-local causes allocation.
- // on such systems we can check for this with the _mi_prim_get_default_theap as those are protected (by being
- // stored in a TLS slot for example)
- mi_theap_t* theap = _mi_theap_default();
- if (theap == NULL) {
- return _mi_subproc_main();
+ mi_memid_t memid;
+ mi_tld_t* tld;
+ if (_mi_subproc_is_main(subproc) && tseq==0 /* first tld */) {
+ tld = &mi_process_tld_main;
+ memid = _mi_memid_create_static(tld,sizeof(*tld));
}
else {
- return theap->tld->subproc; // avoid using thread local storage (`thread_tld`)
- }
-}
-
-mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) {
- mi_heap_t* heap = mi_atomic_load_ptr_relaxed(mi_heap_t,&subproc->heap_main);
- if mi_likely(heap!=NULL) {
- return heap;
+ tld = (mi_tld_t*)_mi_meta_zalloc(subproc, sizeof(mi_tld_t), &memid);
}
- else {
- mi_heap_main_init();
- mi_assert_internal(mi_atomic_load_relaxed(&subproc->heap_main) != NULL);
- return mi_atomic_load_ptr_relaxed(mi_heap_t,&subproc->heap_main);
+ if (tld==NULL) {
+ _mi_error_message(ENOMEM, "unable to allocate memory for thread local data\n");
+ return NULL;
}
+ tld->memid = memid;
+ return mi_tld_init(tld,tseq,subproc);
}
-mi_heap_t* mi_heap_main(void) {
- return _mi_subproc_heap_main(_mi_subproc()); // don't use mi_theap_main_init_get() so this call works during process_init
-}
-
-bool _mi_is_heap_main(const mi_heap_t* heap) {
- mi_assert_internal(heap!=NULL);
- return (_mi_subproc_heap_main(heap->subproc) == heap);
+mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) {
+ if (tld==NULL) return;
+ mi_atomic_decrement_relaxed(&tld->subproc->thread_count);
+ tld->thread_id = (mi_threadid_t)(~0); // it is best to set an invalid tid for tld_main as sometimes the same thread-id
+ // is reused by the OS after a thread has terminated. (see issue #1287)
+ mi_lock_done(&tld->theaps_lock);
+ _mi_meta_free(tld->subproc, tld, tld->memid); // note: safe for static tld
}
-bool _mi_is_theap_main(const mi_theap_t* theap) {
- return (mi_theap_is_initialized(theap) && _mi_is_heap_main(_mi_theap_heap(theap)));
-}
/* -----------------------------------------------------------
- Sub process
+ Thread Init
----------------------------------------------------------- */
-
-mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) {
- return (mi_subproc_t*)(subproc_id._mi_subproc_id);
-}
-
-mi_subproc_id_t _mi_subproc_to_id(mi_subproc_t* subproc) {
- mi_subproc_id_t id = { subproc };
- return id;
-}
-
-mi_subproc_id_t mi_subproc_main(void) {
- return _mi_subproc_to_id(_mi_subproc_main());
-}
-
-mi_subproc_id_t mi_subproc_current(void) {
- return _mi_subproc_to_id(_mi_subproc());
-}
-
-mi_subproc_id_t mi_subproc_new(void) {
- static _Atomic(size_t) subproc_total_count;
- mi_memid_t memid;
- mi_subproc_t* subproc = (mi_subproc_t*)_mi_meta_zalloc(sizeof(mi_subproc_t),&memid);
- if (subproc == NULL) return _mi_subproc_to_id(NULL);
- subproc->memid = memid;
- subproc->subproc_seq = mi_atomic_increment_relaxed(&subproc_total_count) + 1;
- mi_stats_header_init(&subproc->stats);
- mi_lock_init(&subproc->arena_reserve_lock);
- mi_lock_init(&subproc->heaps_lock);
- mi_lock(&subprocs_lock) {
- // push on subproc list
- subproc->next = subprocs;
- if (subprocs!=NULL) { subprocs->prev = subproc; }
- subprocs = subproc;
+#if MI_DEBUG || defined(MI_TLS_RECURSE_GUARD)
+static mi_theap_t* mi_heap_check_for_existing_theap(mi_heap_t* heap) {
+ const mi_threadid_t tid = _mi_thread_id();
+ mi_theap_t* thread_theap = NULL;
+ mi_lock(&heap->theaps_lock) {
+ for(mi_theap_t* theap = heap->theaps; theap != NULL; theap = theap->hnext ) {
+ if (theap->tld->thread_id == tid) {
+ thread_theap = theap;
+ break;
+ }
+ }
}
- return _mi_subproc_to_id(subproc);
+ return thread_theap;
}
+#endif
-// destroy all subproc resources including arena's, heap's etc.
-static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subprocs_lock)
+// Initialize thread
+mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap_main)
{
- if (subproc==NULL) return;
+ // ensure our process has started already
+ mi_process_init();
- // remove from the subproc list
- mi_lock_maybe(&subprocs_lock, acquire_subprocs_lock) {
- if (subproc->next!=NULL) { subproc->next->prev = subproc->prev; }
- if (subproc->prev!=NULL) { subproc->prev->next = subproc->next; }
- else { mi_assert_internal(subprocs==subproc); subprocs = subproc->next; }
- }
+ // if the theap_default is already set we have already initialized
+ mi_theap_t* theap = _mi_theap_default();
+ if (mi_theap_is_initialized(theap)) return theap;
+
+ // initialize the default theap
+ // note: we cannot access thread-locals yet as that can cause (recursive) allocation
+ // (on macOS <= 14 for example where the loader allocates thread-local data on demand).
+ if (heap_main==NULL) {
+ heap_main = mi_heap_main();
+ mi_assert_internal(heap_main == &mi_process_heap_main);
+ }
+ mi_assert_internal(heap_main!=NULL);
+
+ #if MI_DEBUG || defined(MI_TLS_RECURSE_GUARD)
+ theap = mi_heap_check_for_existing_theap(heap_main); // recursion check
+ #if !defined(MI_TLS_RECURSE_GUARD)
+ mi_assert_internal(theap==NULL);
+ #endif
+ #else
+ theap = NULL;
+ #endif
- // destroy all subproc heaps
- mi_lock(&subproc->heaps_lock) {
- mi_heap_t* heap = subproc->heaps;
- while (heap != NULL) {
- mi_heap_t* next = heap->next;
- if (heap!=subproc->heap_main) { mi_heap_destroy(heap); }
- heap = next;
+ if (theap==NULL) {
+ // allocated the tld
+ mi_tld_t* tld = mi_tld_create(heap_main->subproc);
+ if (tld==NULL) return NULL; // out-of-memory on tld allocation
+ // allocate and initialize the theap for the main heap
+ if (tld==&mi_process_tld_main) {
+ theap = &mi_process_theap_main; // initial theap is pre-allocated
+ theap->memid = _mi_memid_create_static(theap,sizeof(*theap));
+ }
+ else {
+ theap = _mi_theap_alloc(heap_main,tld); // otherwise meta allocate
+ if (theap==NULL) { mi_tld_free(tld); return NULL; } // out-of-memory on theap allocation
}
- mi_assert_internal(subproc->heaps == subproc->heap_main);
- _mi_heap_force_destroy(subproc->heap_main); // no warning if destroying the main heap
+ _mi_theap_init(theap,heap_main,tld);
}
- // remove associated arenas
- _mi_arenas_unsafe_destroy_all(subproc);
+ // now initialize the thread
+ _mi_theap_default_set(theap);
+ // and only then set the heap_theap field as that accesses thread locals
+ _mi_heap_theap_set(heap_main, theap); // todo: can fail!
- // merge stats back into the main subproc?
- if (subproc!=&subproc_main) {
- _mi_stats_merge_into(&subproc_main.stats, &subproc->stats);
- }
+ mi_assert_internal(mi_theap_is_initialized(theap));
+ mi_theap_t* const heap_theap = (heap_main==NULL ? NULL : (mi_theap_t*)_mi_thread_local_get(heap_main->theap));
+ mi_assert_internal(heap_main==NULL || heap_theap == theap); MI_UNUSED_RELEASE(heap_theap);
- // safe to release
- // todo: should we refcount subprocesses?
- mi_lock_done(&subproc->arena_reserve_lock);
- mi_lock_done(&subproc->heaps_lock);
- if (subproc!=&subproc_main) {
- _mi_meta_free(subproc, sizeof(mi_subproc_t), subproc->memid);
- }
- else {
- // for the main subproc, also release the global page map
- _mi_page_map_unsafe_destroy(&subproc_main);
- }
-}
-
-void mi_subproc_destroy(mi_subproc_id_t subproc_id) {
- mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
- if (subproc==NULL || subproc==&subproc_main) return;
- mi_subproc_unsafe_destroy(subproc, true /* take lock */);
+ mi_subproc_stat_increase(_mi_theap_subproc(theap), threads, 1); // or theap stats and wait for merge?
+ // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());
+ return theap;
}
-static void mi_subprocs_unsafe_destroy_all(void) {
- mi_lock(&subprocs_lock) {
- mi_subproc_t* subproc = subprocs;
- while (subproc!=NULL) {
- mi_subproc_t* next = subproc->next;
- if (subproc!=&subproc_main) {
- mi_subproc_unsafe_destroy(subproc, false /* take subprocs lock */);
- }
- subproc = next;
- }
- }
- mi_subproc_unsafe_destroy(&subproc_main, true /* take subprocs lock */);
+mi_theap_t* _mi_thread_init(void) {
+ return _mi_thread_init_with_heap(NULL);
}
-
-void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) {
- mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
- mi_tld_t* const tld = _mi_theap_default_safe()->tld;
- mi_assert(tld->subproc== &subproc_main);
- if (tld->subproc != &subproc_main) {
- _mi_warning_message("unable to add thread to the subprocess as it was already in another subprocess (id: %p)\n", subproc);
- return;
- }
- tld->subproc = subproc;
- tld->thread_seq = mi_atomic_increment_relaxed(&subproc->thread_total_count);
- mi_atomic_decrement_relaxed(&subproc_main.thread_count);
- mi_atomic_increment_relaxed(&subproc->thread_count);
+void mi_decl_noinline mi_thread_init(void) mi_attr_noexcept {
+ _mi_thread_init();
}
-bool mi_subproc_visit_heaps(mi_subproc_id_t subproc_id, mi_heap_visit_fun* visitor, void* arg) {
- mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
- if (subproc==NULL) return false;
- bool ok = true;
- mi_lock(&subproc->heaps_lock) {
- for (mi_heap_t* heap = subproc->heaps; heap!=NULL && ok; heap = heap->next) {
- ok = (*visitor)(heap, arg);
- }
- }
- return ok;
-}
-
/* -----------------------------------------------------------
- Allocate theap data
+ Theaps done
----------------------------------------------------------- */
-// Initialize the thread local default theap, called from `mi_thread_init`
-static mi_theap_t* _mi_thread_init_theap_default(void) {
- mi_theap_t* theap = _mi_theap_default();
- if (mi_theap_is_initialized(theap)) return theap;
- if (_mi_is_main_thread()) {
- mi_heap_main_init();
- theap = &theap_main;
- }
- else {
- // allocates tld data
- // note: we cannot access thread-locals yet as that can cause (recursive) allocation
- // (on macOS <= 14 for example where the loader allocates thread-local data on demand).
- mi_tld_t* tld = mi_tld_alloc();
- if (tld==NULL) return NULL; // things are very wrong if this fails (out of memory)
- // allocate and initialize the theap for the main heap
- theap = _mi_theap_create(mi_heap_main(), tld);
- }
- // associate the theap with this thread
- // (this is safe, on macOS for example, the theap is set in a dedicated TLS slot and thus does not cause recursive allocation)
- _mi_theap_default_set(theap);
- return theap;
-}
-
-
// Free the thread local theaps
static void mi_thread_theaps_done(mi_tld_t* tld)
{
- // reset the thread local theaps
- _mi_theap_default_set((mi_theap_t*)&_mi_theap_empty);
- _mi_theap_cached_set((mi_theap_t*)&_mi_theap_empty);
- __mi_theap_main = NULL;
-
// abandon the pages of all theaps in this thread
mi_lock(&tld->theaps_lock) {
mi_theap_t* theap = tld->theaps;
while (theap != NULL) {
- mi_theap_t* next = theap->tnext;
+ mi_theap_t* next = theap->tnext;
// never destroy theaps; if a dll is linked statically with mimalloc,
// there may still be delete/free calls after the mi_fls_done is called. Issue #207
_mi_theap_collect_abandon(theap);
@@ -645,39 +394,38 @@ static void mi_thread_theaps_done(mi_tld_t* tld)
}
}
- // free the theaps of this thread.
- // This can run concurrently with a `mi_heap_free_theaps` and we need to ensure we free theaps atomically.
- // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock
- // potential concurrent `mi_heap_free_theaps` which tries to remove the theap from our theaps list.
- bool all_freed;
- do {
- all_freed = true;
- mi_lock(&tld->theaps_lock) {
- mi_theap_t* theap = tld->theaps;
- while (theap != NULL) {
- mi_theap_t* next = theap->tnext;
- mi_assert_internal(theap->page_count==0);
- if (!_mi_theap_free(theap, true /* acquire heap->theaps_lock */, false /* dont re-acquire the tld->theaps_lock*/ )) {
- all_freed = false;
- }
- theap = next;
- }
- }
- if (!all_freed) {
- mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1);
- _mi_prim_thread_yield();
- }
- else {
- mi_assert_internal(tld->theaps==NULL);
+ // reset the thread local theaps
+ // note: do this after abandon as page->heap may be NULL and mi_heap_main should return the heap
+ // belonging to the right subprocess
+ _mi_theap_default_set((mi_theap_t*)&_mi_theap_empty);
+ _mi_theap_cached_set((mi_theap_t*)&_mi_theap_empty);
+
+ // We might run concurrently with a `mi_heap_free_theaps` and we need to ensure we free theaps atomically.
+ // we first detach our theaps list from any heaps
+ _mi_tld_detach_theaps(tld);
+
+ // no heaps point to our theaps anymore, free them
+ mi_lock(&tld->theaps_lock) { // paranoia
+ mi_theap_t* theap = tld->theaps;
+ tld->theaps = NULL;
+ while (theap != NULL) {
+ mi_theap_t* next = theap->tnext;
+ mi_assert_internal(theap->page_count==0);
+ mi_assert_internal(_mi_theap_heap_peek(theap)==NULL);
+ theap->tld = NULL;
+ theap->tnext = NULL;
+ theap->tprev = NULL;
+ mi_assert_internal(mi_atomic_load_relaxed(&theap->refcount) == 1); // as the cached entry is set to empty
+ _mi_theap_decref(theap);
+ theap = next;
}
- } while (!all_freed);
+ }
mi_assert(_mi_theap_default()==(mi_theap_t*)&_mi_theap_empty); // careful to not re-initialize the default theap during theap_delete
mi_assert(!mi_theap_is_initialized(_mi_theap_default()));
}
-
// --------------------------------------------------------
// Try to run `mi_thread_done()` automatically so any memory
// owned by the thread but not yet released can be abandoned
@@ -694,48 +442,27 @@ static void mi_thread_theaps_done(mi_tld_t* tld)
// to set up the thread local keys.
// --------------------------------------------------------
-// Set up handlers so `mi_thread_done` is called automatically
+// Set up hooks so `mi_thread_done` is called automatically
static void mi_process_setup_auto_thread_done(void) {
mi_atomic_do_once {
_mi_prim_thread_init_auto_done();
- _mi_theap_default_set(&theap_main);
+ mi_theap_t* theap = _mi_theap_default();
+ mi_assert_internal(mi_theap_is_initialized(theap));
+ if (mi_theap_is_initialized(theap)) {
+ _mi_theap_default_set(theap);
+ }
}
}
-
-bool _mi_is_main_thread(void) {
- return (tld_main.thread_id==0 || tld_main.thread_id == _mi_thread_id());
-}
-
-
-// Initialize thread
-void mi_thread_init(void) mi_attr_noexcept
-{
- // ensure our process has started already
- mi_process_init();
- // if the theap_default is already set we have already initialized
- if (_mi_thread_is_initialized()) return;
-
- // initialize the default theap
- if (_mi_thread_init_theap_default() == NULL) return; // out-of-memory on tld/theap allocation
-
- mi_heap_stat_increase(mi_heap_main(), threads, 1);
- // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());
-}
-
void mi_thread_done(void) mi_attr_noexcept {
_mi_thread_done(NULL);
}
void _mi_thread_done(mi_theap_t* _theap_main)
-{
+{
// NULL can be passed on some platforms
if (_theap_main==NULL) {
- _theap_main = __mi_theap_main; // don't call `mi_theap_main_safe` as that re-initializes the thread
- if (_theap_main==NULL) { // can happen if `mi_theap_main_safe` is never called; but then the default is main
- _theap_main = _mi_theap_default();
- mi_assert_internal(_theap_main==NULL || _mi_is_theap_main(_theap_main));
- }
+ _theap_main = _mi_theap_default();
}
// prevent re-entrancy through theap_done/theap_set_default_direct (issue #699)
@@ -743,14 +470,14 @@ void _mi_thread_done(mi_theap_t* _theap_main)
return;
}
+ // get the current tld
+ mi_tld_t* const tld = _theap_main->tld;
+
// release dynamic thread_local's
_mi_thread_locals_thread_done();
- // note: we store the tld as we should avoid reading `thread_tld` at this point (to avoid reinitializing the thread local storage)
- mi_tld_t* const tld = _theap_main->tld;
-
// adjust stats
- mi_heap_stat_decrease(_mi_subproc_heap_main(tld->subproc), threads, 1); // todo: or `_theap_main->heap`?
+ mi_subproc_stat_decrease(tld->subproc, threads, 1); // todo: or `_theap_main->heap`?
// check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local theaps...
if (tld->thread_id != _mi_prim_thread_id()) return;
@@ -762,225 +489,16 @@ void _mi_thread_done(mi_theap_t* _theap_main)
mi_tld_free(tld);
}
-
-mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void) {
- return (mi_theap_t*)&_mi_theap_empty;
-}
-
-#if MI_TLS_MODEL_DYNAMIC_WIN32
-
-// If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed)
-// See <https://en.wikipedia.org/wiki/Win32_Thread_Information_Block> for the offsets.
-#if MI_SIZE_SIZE==4
-#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_SIZE_SIZE)
-#else
-#define MI_TLS_DIRECT_FIRST (0x1480 / MI_SIZE_SIZE)
-#endif
-#define MI_TLS_DIRECT_SLOTS (64)
-#define MI_TLS_EXPANSION_SLOTS (1024)
-
-#if !MI_WIN_DIRECT_TLS
-#define MI_TLS_INITIAL_SLOT MI_TLS_EXPANSION_SLOT
-#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1)
-#else
-// with only direct entries, use the "arbitrary user data" field
-// and assume it is NULL (see also <http://www.nynaeve.net/?p=98>)
-#define MI_TLS_INITIAL_SLOT (5)
-#define MI_TLS_INITIAL_EXPANSION_SLOT (0)
-#endif
-
-// we initially use the last of the expansion slots as the default NULL.
-// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc (which is quite unlikely)
-mi_decl_hidden mi_decl_cache_align size_t _mi_theap_default_slot = MI_TLS_INITIAL_SLOT;
-mi_decl_hidden size_t _mi_theap_default_expansion_slot = MI_TLS_INITIAL_EXPANSION_SLOT;
-mi_decl_hidden size_t _mi_theap_cached_slot = MI_TLS_INITIAL_SLOT;
-mi_decl_hidden size_t _mi_theap_cached_expansion_slot = MI_TLS_INITIAL_EXPANSION_SLOT;
-
-static DWORD mi_tls_raw_index_default = TLS_OUT_OF_INDEXES;
-static DWORD mi_tls_raw_index_cached = TLS_OUT_OF_INDEXES;
-
-static bool mi_win_tls_slot_alloc(size_t* slot, size_t* extended, DWORD* raw_index) {
- const DWORD index = TlsAlloc();
- *raw_index = index;
- if (index==TLS_OUT_OF_INDEXES) {
- *extended = 0;
- *slot = 0;
- return false;
- }
- else if (index<MI_TLS_DIRECT_SLOTS) {
- *extended = 0;
- *slot = index + MI_TLS_DIRECT_FIRST;
- return true;
- }
- #if !MI_WIN_DIRECT_TLS
- else if (index < MI_TLS_DIRECT_SLOTS + MI_TLS_EXPANSION_SLOTS - 1) { // check maximum number of expansion slots - 1 (as we use the last one as the default)
- *extended = index - MI_TLS_DIRECT_SLOTS;
- *slot = MI_TLS_EXPANSION_SLOT;
- return true;
- }
- #endif
- else {
- // to high an index for us
- _mi_error_message(EFAULT, "returned tls index was too high (%u)\n", index);
- TlsFree(index);
- *raw_index = TLS_OUT_OF_INDEXES;
- *extended = 0;
- *slot = 0;
- return false;
- }
-}
-
-static void mi_win_tls_slot_free(DWORD* raw_index) {
- if (*raw_index != TLS_OUT_OF_INDEXES) {
- TlsFree(*raw_index);
- *raw_index = TLS_OUT_OF_INDEXES;
- }
-}
-
-static void mi_tls_slots_init(void) {
- mi_atomic_do_once {
- bool ok = mi_win_tls_slot_alloc(&_mi_theap_default_slot, &_mi_theap_default_expansion_slot, &mi_tls_raw_index_default);
- if (ok) {
- ok = mi_win_tls_slot_alloc(&_mi_theap_cached_slot, &_mi_theap_cached_expansion_slot, &mi_tls_raw_index_cached);
- }
- if (!ok) {
- _mi_error_message(EFAULT, "unable to allocate fast TLS user slot (0x%zx)\n", _mi_theap_cached_slot);
- }
- }
-}
-
-static void mi_tls_slots_done(void) {
- mi_win_tls_slot_free(&mi_tls_raw_index_default);
- mi_win_tls_slot_free(&mi_tls_raw_index_cached );
-}
-
-static void mi_win_tls_slot_set(size_t slot, size_t extended_slot, void* value) {
- mi_assert_internal((slot >= MI_TLS_DIRECT_FIRST && slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) || slot == MI_TLS_EXPANSION_SLOT);
- if (slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) {
- mi_prim_tls_slot_set(slot, value);
- }
- else {
- mi_assert_internal(extended_slot < MI_TLS_EXPANSION_SLOTS);
- TlsSetValue((DWORD)(extended_slot + MI_TLS_DIRECT_SLOTS), value); // use TlsSetValue to initialize the TlsExpansion array if needed
- }
-}
-
-#elif MI_TLS_MODEL_DYNAMIC_PTHREADS
-
-// only for pthreads for now
-mi_decl_hidden pthread_key_t _mi_theap_default_key = 0;
-mi_decl_hidden pthread_key_t _mi_theap_cached_key = 0;
-
-// create a non-zero pthread key
-static int mi_pthread_key_create( pthread_key_t* pkey ) {
- pthread_key_t key;
- int err = pthread_key_create(&key, NULL);
- if (err!=0) return err;
- if (key==0) {
- // if we get a zero key, create another one as we use 0 for an invalid key
- pthread_key_t key2;
- err = pthread_key_create(&key2, NULL);
- pthread_key_delete(key); // delete the old key
- if (err!=0) return err;
- key = key2;
- }
- mi_assert_internal(key!=0);
- *pkey = key;
- return 0;
-}
-
-static void mi_tls_slots_init(void) {
- mi_atomic_do_once {
- int err = mi_pthread_key_create(&_mi_theap_default_key);
- if (err==0) {
- err = mi_pthread_key_create(&_mi_theap_cached_key);
- }
- if (err!=0) {
- _mi_error_message(EFAULT, "unable to allocate pthread keys (error %d)\n", err);
- }
- }
-}
-
-static void mi_tls_slots_done(void) {
- if (_mi_theap_default_key != 0) {
- pthread_key_delete(_mi_theap_default_key);
- _mi_theap_default_key = 0;
- }
- if (_mi_theap_cached_key != 0) {
- pthread_key_delete(_mi_theap_cached_key);
- _mi_theap_cached_key = 0;
- }
-}
-
-#else
-
-static void mi_tls_slots_init(void) {
- // nothing
-}
-
-static void mi_tls_slots_done(void) {
- // nothing
-}
-
-#endif
-
-void _mi_theap_cached_set(mi_theap_t* theap) {
- mi_theap_t* prev = _mi_theap_cached();
- if (prev==theap) return;
- // set
- mi_tls_slots_init();
- #if MI_TLS_MODEL_THREAD_LOCAL
- __mi_theap_cached = theap;
- #elif MI_TLS_MODEL_FIXED_SLOT
- mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_SLOT_CACHED, theap);
- #elif MI_TLS_MODEL_DYNAMIC_WIN32
- mi_win_tls_slot_set(_mi_theap_cached_slot, _mi_theap_cached_expansion_slot, theap);
- #elif MI_TLS_MODEL_DYNAMIC_PTHREADS
- if (_mi_theap_cached_key!=0) pthread_setspecific(_mi_theap_cached_key, theap);
- #endif
- // update refcounts (so cached theap memory keeps available until no longer cached)
- _mi_theap_incref(theap);
- _mi_theap_decref(prev);
-}
-
-void _mi_theap_default_set(mi_theap_t* theap) {
- mi_theap_t* const theap_old = _mi_theap_default();
- mi_assert_internal(theap != NULL);
- mi_assert_internal(theap->tld->thread_id==0 || theap->tld->thread_id==_mi_thread_id());
- mi_tls_slots_init();
- #if MI_TLS_MODEL_THREAD_LOCAL
- __mi_theap_default = theap;
- #elif MI_TLS_MODEL_FIXED_SLOT
- mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_SLOT_DEFAULT, theap);
- #elif MI_TLS_MODEL_DYNAMIC_WIN32
- mi_win_tls_slot_set(_mi_theap_default_slot, _mi_theap_default_expansion_slot, theap);
- #elif MI_TLS_MODEL_DYNAMIC_PTHREADS
- if (_mi_theap_default_key!=0) pthread_setspecific(_mi_theap_default_key, theap);
- #endif
-
- // set theap main if needed
- if (mi_theap_is_initialized(theap)) {
- // ensure the default theap is passed to `_mi_thread_done` as on some platforms we cannot access TLS at thread termination (as it would allocate again)
- _mi_prim_thread_associate_default_theap(theap);
- if (_mi_is_heap_main(_mi_theap_heap(theap))) {
- __mi_theap_main = theap;
- }
- }
-
- // ensure either the default slot contains the main theap, or __mi_theap_main is initialized
- if (mi_theap_is_initialized(theap_old) && _mi_is_heap_main(_mi_theap_heap(theap_old))) {
- __mi_theap_main = theap_old;
- }
-}
-
void mi_thread_set_in_threadpool(void) mi_attr_noexcept {
- mi_theap_t* theap = _mi_theap_default_safe();
+ mi_theap_t* theap = mi_theap_get_default();
theap->tld->is_in_threadpool = true;
}
+
// --------------------------------------------------------
-// Run functions on process init/done, and thread init/done
+// Process init and done
// --------------------------------------------------------
+
static bool os_preloading = true; // true until this module is initialized
// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false.
@@ -993,20 +511,12 @@ mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept {
return _mi_is_redirected();
}
-// Called once by the process loader from `src/prim/prim.c`
+// Called once by the process loader from `src/prim/prim.c` before `main` is called.
void _mi_auto_process_init(void) {
- // mi_heap_main_init();
- // #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD)
- // volatile mi_theap_t* dummy = __mi_theap_default; // access TLS to allocate it before setting tls_initialized to true;
- // if (dummy == NULL) return; // use dummy or otherwise the access may get optimized away (issue #697)
- // #endif
-
os_preloading = false;
- mi_assert_internal(_mi_is_main_thread());
mi_process_init();
- mi_process_setup_auto_thread_done();
- _mi_thread_locals_init();
+
_mi_options_post_init(); // now we can print to stderr
if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n");
@@ -1018,98 +528,42 @@ void _mi_auto_process_init(void) {
}
// reseed random
- _mi_random_reinit_if_weak(&theap_main.random);
-}
-
-// CPU features
-mi_decl_cache_align size_t _mi_cpu_movsb_max = 0; // for size <= max, rep movsb is fast
-mi_decl_cache_align size_t _mi_cpu_stosb_max = 0; // for size <= max, rep stosb is fast
-mi_decl_cache_align bool _mi_cpu_has_popcnt = false;
-
-#if (MI_ARCH_X64 || MI_ARCH_X86)
-#if defined(__GNUC__)
-// #include <cpuid.h>
-static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {
- // note: use explicit assembly instead of __get_cpuid as we need the sublevel (in ecx)
- // (on Ubuntu 22 with WSL the __get_cpuid does not clear ecx for level 7 which is incorrect).
- uint32_t eax, ebx, ecx, edx;
- __asm __volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "a"(level), "c"(sublevel) : );
- regs4[0] = eax;
- regs4[1] = ebx;
- regs4[2] = ecx;
- regs4[3] = edx;
- return true;
-}
-
-#elif defined(_MSC_VER)
-static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {
- __cpuidex((int32_t*)regs4, (int32_t)level, (int32_t)sublevel);
- return true;
-}
-#else
-static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {
- MI_UNUSED(regs4); MI_UNUSED(level); MI_UNUSED(sublevel);
- return false;
-}
-#endif
-
-static void mi_detect_cpu_features(void) {
- // FSRM for fast short rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017))
- // EMRS for fast enhanced rep movsb/stosb support (not used at the moment, memcpy always seems faster?)
- // FSRS for fast short rep stosb
- bool amd = false;
- bool fsrm = false;
- // bool erms = false;
- bool fsrs = false;
- uint32_t cpu_info[4];
- if (mi_cpuid(cpu_info, 0, 0)) {
- amd = (cpu_info[2]==0x444d4163); // (Auth enti cAMD)
- }
- if (mi_cpuid(cpu_info, 7, 0)) {
- fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>
- // erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>
- }
- if (mi_cpuid(cpu_info, 7, 1)) {
- fsrs = ((cpu_info[1] & (1 << 11)) != 0); // bit 11 of EBX: see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=1:_Extended_Features>
- }
- if (mi_cpuid(cpu_info, 1, 0)) {
- _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see <https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits>
- }
-
- if (fsrm) {
- _mi_cpu_movsb_max = 127;
- }
- if (fsrs || (amd && fsrm)) { // fsrm on amd implies fsrs, see: https://marc.info/?l=git-commits-head&m=168186277717803
- _mi_cpu_stosb_max = 127;
+ mi_theap_t* theap = _mi_theap_default();
+ if (theap != NULL) {
+ _mi_random_reinit_if_weak(&theap->random);
+ mi_subproc_t* subproc = _mi_theap_subproc(theap);
+ if (subproc->theap_meta != NULL) {
+ mi_lock(&subproc->theap_meta_lock) {
+ _mi_random_reinit_if_weak(&subproc->theap_meta->random);
+ }
+ }
}
}
-#else
-static void mi_detect_cpu_features(void) {
- #if MI_ARCH_ARM64
- _mi_cpu_has_popcnt = true;
+
+// Initialize the process; called by thread_init, the process loader, or an initial allocation (perhaps by the loader or a system library)
+static void mi_process_init_once(void) {
+ #if defined(__CYGWIN__) // we need to kickstart the cygwin runtime
+ __mi_thread_id_helper = NULL;
#endif
-}
-#endif
+ _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id());
+ _mi_detect_cpu_features();
+ _mi_options_init(); // read environment (if possible)
+ _mi_stats_init(); // start timer
+ _mi_os_init(); // primitive dependent
-// Initialize the process; called by thread_init or the process loader
-static void mi_process_init_once(void) mi_attr_noexcept {
- _mi_process_is_initialized = true;
- _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id());
+ mi_heap_main_init(); // before page_map_init so stats are working
+ _mi_page_map_init(); // todo: this could fail.. should we abort in that case?
+ mi_thread_init();
- mi_detect_cpu_features();
- _mi_options_init();
- _mi_stats_init();
- _mi_os_init();
// the following can potentially allocate (on freeBSD for pthread keys)
- // todo: do 2-phase so we can use stats at first, then later init the keys?
- mi_heap_main_init(); // before page_map_init so stats are working
- _mi_page_map_init(); // todo: this could fail.. should we abort in that case?
- mi_thread_init();
- _mi_process_is_initialized = true;
+ _mi_tls_slots_init(); // pthread key create
+ _mi_thread_locals_init(); // pthread key create
+ _mi_process_is_initialized = true; // before `mi_process_setup_auto_thread_done` so `_mi_theap_default` returns the current theap with `MI_TLS_RECURSE_GUARD`
+ mi_process_setup_auto_thread_done(); // after the above mi_thread_init so it can add the current theap
- #if defined(_WIN32) && defined(MI_WIN_USE_FLS)
+ #if defined(_WIN32) && defined(MI_WIN_INIT_USE_FLS)
// On windows, when building as a static lib the FLS cleanup happens to early for the main thread.
// To avoid this, set the FLS value for the main thread to NULL so the fls cleanup
// will not call _mi_thread_done on the (still executing) main thread. See issue #508.
@@ -1137,16 +591,14 @@ static void mi_process_init_once(void) mi_attr_noexcept {
// Initialize the process; called by thread_init or the process loader
void mi_process_init(void) mi_attr_noexcept {
- // #if _MSC_VER < 1920
- // mi_heap_main_init(); // vs2017 can dynamically re-initialize _mi_heap_main
- // #endif
mi_atomic_do_once {
mi_process_init_once();
}
}
-// Called when the process is done (cdecl as it is used with `at_exit` on some platforms)
-void mi_cdecl mi_process_done(void) mi_attr_noexcept {
+
+// Called when the process is done
+static void mi_process_done_once(void) {
// only shutdown if we were initialized
if (!_mi_process_is_initialized) return;
// ensure we are called once
@@ -1154,8 +606,8 @@ void mi_cdecl mi_process_done(void) mi_attr_noexcept {
if (process_done) return;
process_done = true;
- // free dynamic thread locals (if used at all)
- _mi_thread_locals_done();
+ // decref any cached theap
+ _mi_theap_cached_set(_mi_theap_empty_get());
// release any thread specific resources and ensure _mi_thread_done is called on all but the main thread
_mi_prim_thread_done_auto_done();
@@ -1170,30 +622,47 @@ void mi_cdecl mi_process_done(void) mi_attr_noexcept {
#endif
// done with tracking tools
- mi_track_done()
+ mi_track_done();
// Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free
// since after process_done there might still be other code running that calls `free` (like at_exit routines,
// or C-runtime termination code.
+ mi_subproc_t* subproc_main = _mi_subproc_main();
if (mi_option_is_enabled(mi_option_destroy_on_exit)) {
- mi_subprocs_unsafe_destroy_all(); // destroys all subprocs, arenas, and the page_map!
+ _mi_subprocs_unsafe_destroy_all(); // destroys all mi_subprocs, arenas, thread locals, and the page_map!
}
else {
- mi_heap_stats_merge_to_subproc(mi_heap_main());
- }
-
- // careful now to no longer access any allocator functionality
- if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {
- mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL);
+ // free dynamic thread locals (if used at all)
+ _mi_thread_locals_thread_done();
+ _mi_thread_locals_done();
+ if (subproc_main->heap_main != NULL) {
+ if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {
+ mi_theap_collect(subproc_main->theap_meta, false /* force */); // update stats of all pages in theap_meta
+ _mi_theap_merge_stats(subproc_main->theap_meta);
+ _mi_theap_merge_stats(_mi_theap_default()); // _mi_thread_locals_done can free
+ mi_heap_stats_merge_to_subproc(subproc_main->heap_main);
+ mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); // note: can try to access (the now freed) thread_locals in mi_heap_theap_peek
+ }
+ }
}
- mi_lock_done(&subprocs_lock);
- mi_tls_slots_done();
+
+ _mi_tls_slots_done();
+ _mi_subproc_main_done();
_mi_allocator_done();
- _mi_verbose_message("process done: 0x%zx\n", tld_main.thread_id);
+ _mi_verbose_message("process done %zu\n", sizeof(mi_page_t)); // : 0x%zx\n", mi_process_tld_main.thread_id);
os_preloading = true; // don't call the C runtime anymore
}
+
+// Call when the process is done (cdecl as it is used with `at_exit` on some platforms)
+void mi_cdecl mi_process_done(void) mi_attr_noexcept {
+ mi_atomic_do_once {
+ mi_process_done_once();
+ }
+}
+
+// Called automatically when the process is done (cdecl as it is used with `at_exit` on some platforms)
void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept {
- if (_mi_option_get_fast(mi_option_destroy_on_exit)>1) return;
+ if (_mi_option_get_fast(mi_option_destroy_on_exit)>=2) return; // allow disabling auto process done
mi_process_done();
}
diff --git a/vendored/mimalloc/src/libc.c b/vendored/mimalloc/src/libc.c
index c74003a14..8f211230d 100644
--- a/vendored/mimalloc/src/libc.c
+++ b/vendored/mimalloc/src/libc.c
@@ -1,12 +1,12 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2024, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
-----------------------------------------------------------------------------*/
// --------------------------------------------------------
-// This module defines various std libc functions to reduce
+// This module defines various standard libc functions to reduce
// the dependency on libc, and also prevent errors caused
// by some libc implementations when called before `main`
// executes (due to malloc redirection)
@@ -39,8 +39,8 @@ bool _mi_streq(const char* s, const char* t) {
return (*s == *t);
}
-void _mi_strlcpy(char* dest, const char* src, size_t dest_size) {
- if (dest==NULL || src==NULL || dest_size == 0) return;
+bool _mi_strlcpy(char* dest, const char* src, size_t dest_size) {
+ if (dest==NULL || src==NULL || dest_size == 0) return (src==NULL || *src==0);
// copy until end of src, or when dest is (almost) full
while (*src != 0 && dest_size > 1) {
*dest++ = *src++;
@@ -48,23 +48,24 @@ void _mi_strlcpy(char* dest, const char* src, size_t dest_size) {
}
// always zero terminate
*dest = 0;
+ return (*src == 0);
}
-void _mi_strlcat(char* dest, const char* src, size_t dest_size) {
- if (dest==NULL || src==NULL || dest_size == 0) return;
+bool _mi_strlcat(char* dest, const char* src, size_t dest_size) {
+ if (dest==NULL || src==NULL || dest_size == 0) return (src==NULL || *src==0);
// find end of string in the dest buffer
while (*dest != 0 && dest_size > 1) {
dest++;
dest_size--;
}
// and catenate
- _mi_strlcpy(dest, src, dest_size);
+ return _mi_strlcpy(dest, src, dest_size);
}
size_t _mi_strnlen(const char* s, size_t max_len) {
if (s==NULL) return 0;
size_t len = 0;
- while(s[len] != 0 && len < max_len) { len++; }
+ while(len < max_len && s[len] != 0) { len++; }
return len;
}
@@ -76,7 +77,7 @@ char* _mi_strnstr(char* s, size_t max_len, const char* pat) {
if (s==NULL) return NULL;
if (pat==NULL) return s;
const size_t m = _mi_strnlen(s, max_len);
- const size_t n = _mi_strlen(pat);
+ const size_t n = _mi_strlen(pat);
for (size_t start = 0; start + n <= m; start++) {
size_t i = 0;
while (i<n && pat[i]==s[start+i]) {
@@ -88,16 +89,18 @@ char* _mi_strnstr(char* s, size_t max_len, const char* pat) {
}
#ifdef MI_NO_GETENV
-bool _mi_getenv(const char* name, char* result, size_t result_size) {
+int _mi_getenv(const char* name, char* result, size_t result_size) {
MI_UNUSED(name);
MI_UNUSED(result);
MI_UNUSED(result_size);
- return false;
+ return ENOENT;
}
#else
-bool _mi_getenv(const char* name, char* result, size_t result_size) {
- if (name==NULL || result == NULL || result_size < 64) return false;
- return _mi_prim_getenv(name,result,result_size);
+int _mi_getenv(const char* name, char* result, size_t result_size) {
+ if (name==NULL || result == NULL || result_size < 64) return ENOENT;
+ // change the result of _mi_prim_getenv to an errno result
+ const int res = _mi_prim_getenv(name,result,result_size);
+ return (res > 0 ? 0 : (res == 0 ? ENOENT : EAGAIN));
}
#endif
@@ -116,14 +119,14 @@ bool _mi_atomic_once_enter(mi_atomic_once_t* once) {
}
const mi_threadid_t current_tid = _mi_thread_id();
if (once_tid == current_tid) {
- return false; // recursive invocation; we need this for process_init for example
- }
+ return false; // recursive invocation; don't block on ourselves
+ }
mi_lock_acquire(&once->lock);
uintptr_t expected = 0;
if (mi_atomic_cas_strong_acq_rel(&once->tid, &expected, current_tid)) { // could use atomic_load/store as well
return true; // should execute and release
- }
+ }
else {
mi_lock_release(&once->lock);
return false; // already another thread entered and released
@@ -134,8 +137,72 @@ void _mi_atomic_once_release(mi_atomic_once_t* once) {
if (mi_atomic_load_acquire(&once->tid)>1) { // paranoia
mi_atomic_store_release(&once->tid,1); // done executing
mi_lock_release(&once->lock);
- }
+ }
+}
+
+#if MI_USE_PTHREADS
+mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruct)(void*), void* init) {
+ int err = pthread_key_create(pkey,destruct);
+ if mi_unlikely(err!=0) {
+ *pkey = MI_PTHREAD_KEY_INVALID;
+ _mi_error_message(ENOMEM,"unable to allocate a thread local variable (error %d)\n", err);
+ return false;
+ }
+ mi_assert_internal(*pkey != MI_PTHREAD_KEY_INVALID);
+ if (init!=NULL) {
+ pthread_setspecific(*pkey,init);
+ };
+ mi_assert_internal(pthread_getspecific(*pkey)==init);
+ return true;
}
+#endif
+
+// --------------------------------------------------------
+// Detect CPU features
+// --------------------------------------------------------
+mi_decl_cache_align bool _mi_cpu_has_popcnt = false;
+
+#if (MI_ARCH_X64 || MI_ARCH_X86)
+#if defined(__GNUC__)
+// #include <cpuid.h>
+static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {
+ // note: use explicit assembly instead of __get_cpuid as we need the sublevel (in ecx)
+ // (on Ubuntu 22 with WSL the __get_cpuid does not clear ecx for level 7 which is incorrect).
+ uint32_t eax, ebx, ecx, edx;
+ __asm __volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "a"(level), "c"(sublevel) : );
+ regs4[0] = eax;
+ regs4[1] = ebx;
+ regs4[2] = ecx;
+ regs4[3] = edx;
+ return true;
+}
+
+#elif defined(_MSC_VER)
+static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {
+ __cpuidex((int32_t*)regs4, (int32_t)level, (int32_t)sublevel);
+ return true;
+}
+#else
+static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) {
+ MI_UNUSED(regs4); MI_UNUSED(level); MI_UNUSED(sublevel);
+ return false;
+}
+#endif
+
+void _mi_detect_cpu_features(void) {
+ uint32_t cpu_info[4];
+ if (mi_cpuid(cpu_info, 1, 0)) {
+ _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see <https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits>
+ }
+}
+
+#else
+void _mi_detect_cpu_features(void) {
+ #if MI_ARCH_ARM64 || defined(__riscv_zbb) || defined(__riscv_b)
+ _mi_cpu_has_popcnt = true;
+ #endif
+}
+#endif
// --------------------------------------------------------
@@ -196,22 +263,20 @@ static void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char*
mi_outc('0',out,end);
}
else {
- // output digits in reverse
- char* start = *out;
- while (x > 0) {
+ #define MI_MAX_OUT_DIGITS (160) /* a 512 bit number has 155 digits */
+ char num[MI_MAX_OUT_DIGITS];
+ int dcount = 0;
+ while(x>0 && dcount < MI_MAX_OUT_DIGITS) {
char digit = (char)(x % base);
- mi_outc((digit <= 9 ? '0' + digit : 'A' + digit - 10),out,end);
+ num[dcount++] = (digit <= 9 ? '0' + digit : 'A' + digit - 10);
x = x / base;
}
+ if (dcount>=MI_MAX_OUT_DIGITS) return; // don't output anything?
if (prefix != 0) {
mi_outc(prefix, out, end);
}
- size_t len = *out - start;
- // and reverse in-place
- for (size_t i = 0; i < (len / 2); i++) {
- char c = start[len - i - 1];
- start[len - i - 1] = start[i];
- start[i] = c;
+ while(dcount-- > 0) {
+ mi_outc(num[dcount], out, end);
}
}
}
@@ -258,7 +323,10 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {
if (c >= '1' && c <= '9') {
width = (c - '0'); MI_NEXTC();
while (c >= '0' && c <= '9') {
- width = (10 * width) + (c - '0'); MI_NEXTC();
+ if (width < SIZE_MAX/1024) { // no overflow
+ width = (10 * width) + (c - '0');
+ }
+ MI_NEXTC();
}
if (c == 0) break; // extra check due to while
}
@@ -297,7 +365,7 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {
if (width == 0 && (c == 'x' || c == 'p')) {
if (c == 'p') { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(void*))); }
if (width == 0) { width = 2; }
- fill = '0';
+ if (alignright) { fill = '0'; }
}
mi_out_num(x, (c == 'x' || c == 'p' ? 16 : 10), numplus, &out, end);
}
@@ -350,6 +418,7 @@ int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...) {
return written;
}
+#undef MI_NEXTC
// --------------------------------------------------------
@@ -365,7 +434,7 @@ static size_t mi_ctz_generic32(uint32_t x) {
31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9
};
if (x==0) return 32;
- return debruijn[(uint32_t)((x & -(int32_t)x) * (uint32_t)(0x077CB531U)) >> 27];
+ return debruijn[(uint32_t)((x & (~x + 1U)) * (uint32_t)(0x077CB531U)) >> 27];
}
static size_t mi_clz_generic32(uint32_t x) {
diff --git a/vendored/mimalloc/src/options.c b/vendored/mimalloc/src/options.c
index c2d8fa914..eaa2119de 100644
--- a/vendored/mimalloc/src/options.c
+++ b/vendored/mimalloc/src/options.c
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -17,7 +17,7 @@ static long mi_max_warning_count = 16; // stop outputting warnings after this (u
static void mi_add_stderr_output(void);
-int mi_version(void) mi_attr_noexcept {
+int mi_version(void) {
return MI_MALLOC_VERSION;
}
@@ -49,7 +49,7 @@ int mi_version(void) mi_attr_noexcept {
// in KiB
#ifndef MI_DEFAULT_ARENA_RESERVE
- #if (MI_INTPTR_SIZE>4)
+ #if (MI_SIZE_SIZE>4)
#define MI_DEFAULT_ARENA_RESERVE 1024L*1024L
#else
#define MI_DEFAULT_ARENA_RESERVE 128L*1024L
@@ -77,15 +77,15 @@ int mi_version(void) mi_attr_noexcept {
#endif
#ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE
-#if MI_GUARDED && !MI_DEBUG
-#define MI_DEFAULT_GUARDED_SAMPLE_RATE 4000
+#if MI_GUARDED && MI_DEBUG
+#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 /* MI_MiB */
#else
-#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0
+#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0
#endif
#endif
#ifndef MI_DEFAULT_PAGEMAP_COMMIT
-#if defined(__APPLE__) // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access
+#if defined(__APPLE__) && MI_PAGE_MAP_FLAT // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access
#define MI_DEFAULT_PAGEMAP_COMMIT 1
#else
#define MI_DEFAULT_PAGEMAP_COMMIT 0
@@ -104,10 +104,15 @@ int mi_version(void) mi_attr_noexcept {
#if defined(__ANDROID__)
#define MI_DEFAULT_ALLOW_THP 0
#else
-#define MI_DEFAULT_ALLOW_THP 1
+// #define MI_DEFAULT_ALLOW_THP 1 // allow THP but purging may split up THP pages
+#define MI_DEFAULT_ALLOW_THP 2 // allow THP and set the minimal purge size to 2MiB to avoid breaking them up
#endif
#endif
+#ifndef MI_DEFAULT_COLLECT_MERGES_STATS
+#define MI_DEFAULT_COLLECT_MERGES_STATS 1
+#endif
+
// Static options
static mi_option_desc_t mi_options[_mi_option_last] =
{
@@ -121,7 +126,7 @@ static mi_option_desc_t mi_options[_mi_option_last] =
{ MI_DEFAULT_VERBOSE, MI_OPTION_UNINIT, MI_OPTION(verbose) },
// some of the following options are experimental and not all combinations are allowed.
- { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit) },
+ { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit) },
{ MI_DEFAULT_ARENA_EAGER_COMMIT,
MI_OPTION_UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux)
{ 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit)
@@ -134,9 +139,9 @@ static mi_option_desc_t mi_options[_mi_option_last] =
MI_OPTION_UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`)
{ 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread
{ 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free
- { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_abandoned_page_purge) },
+ { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_abandoned_page_purge) },
{ 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_reset) }, // reset segment memory on free (needs eager commit)
- { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit_delay) },
+ { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit_delay) },
{ 1000,MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) }, // purge delay in milli-seconds
{ 0, MI_OPTION_UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes.
{ 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas)
@@ -146,15 +151,11 @@ static mi_option_desc_t mi_options[_mi_option_last] =
{ 10, MI_OPTION_UNINIT, MI_OPTION(deprecated_max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try.
{ 0, MI_OPTION_UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees!
{ MI_DEFAULT_ARENA_RESERVE, MI_OPTION_UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`)
- { 1, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's
+ { 4, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's
{ 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(deprecated_purge_extend_delay, decommit_extend_delay) },
{ MI_DEFAULT_DISALLOW_ARENA_ALLOC, MI_OPTION_UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's)
{ 400, MI_OPTION_UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries.
-#if defined(MI_VISIT_ABANDONED)
- { 1, MI_OPTION_INITIALIZED, MI_OPTION(visit_abandoned) }, // allow visiting theap blocks in abandoned segments; requires taking locks during reclaim.
-#else
- { 0, MI_OPTION_UNINIT, MI_OPTION(visit_abandoned) },
-#endif
+ { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_visit_abandoned) },
{ 0, MI_OPTION_UNINIT, MI_OPTION(guarded_min) }, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects
{ MI_GiB, MI_OPTION_UNINIT, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects
{ 0, MI_OPTION_UNINIT, MI_OPTION(guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)
@@ -175,16 +176,18 @@ static mi_option_desc_t mi_options[_mi_option_last] =
MI_OPTION_UNINIT, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class
{ MI_DEFAULT_ALLOW_THP,
MI_OPTION_UNINIT, MI_OPTION(allow_thp) }, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process.
- { 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if `mi_option_allow_thp==2`).
- { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE,
- MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).
+ { 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if THP is enabled).
+ { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE,
+ MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).
{ 0, MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) }, // associate local numa node with an initial arena allocation
+ { MI_DEFAULT_COLLECT_MERGES_STATS,
+ MI_OPTION_UNINIT, MI_OPTION(collect_merges_stats) }, // on each theap collect, stats are merged with the parent heap
};
static void mi_option_init(mi_option_desc_t* desc);
static bool mi_option_has_size_in_kib(mi_option_t option) {
- return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve ||
+ return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve ||
option == mi_option_minimal_purge_size || option == mi_option_arena_max_object_size);
}
@@ -203,7 +206,7 @@ void _mi_options_init(void) {
_mi_warning_message("option 'allow_large_os_pages' is disabled to allow for guarded objects\n");
}
}
- #endif
+ #endif
}
// called at actual process load, it should be safe to print now
@@ -221,15 +224,6 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_
const int vermajor = MI_MALLOC_VERSION/10000;
const int verminor = (MI_MALLOC_VERSION%10000)/100;
const int verpatch = (MI_MALLOC_VERSION%100);
- /* CBM LOCAL PATCH (keep across mimalloc refreshes): upstream appends
- * "(built on __DATE__, __TIME__)" here. mimalloc is linked into every cbm
- * binary, so those two macros made every build byte-unique — two compilations
- * of identical source seconds apart could never produce the same hash. That
- * defeats reproducible builds, and it also means a released artifact can never
- * inherit an antivirus false-positive determination made about its
- * predecessor, because every rebuild is a brand-new file to a reputation
- * system. The banner keeps the version; the wall clock is not diagnostics.
- * -Wdate-time in CFLAGS_COMMON makes any new use a build failure. */
_mi_fprintf(out, arg, "v%i.%i.%i%s%s\n", vermajor, verminor, verpatch,
#if defined(MI_CMAKE_BUILD_TYPE)
", " mi_stringify(MI_CMAKE_BUILD_TYPE)
@@ -252,16 +246,34 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_
_mi_fprintf(out, arg, "option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : ""));
}
- // show build configuration
- _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG );
- _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE );
- _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL);
+ // show build configuration
+ #if MI_PAGE_META_IS_ALIGNED && MI_PAGE_META_SMALL_IS_ALIGNED
+ _mi_fprintf(out, arg, "free mode : (small) aligned, page size: %zu\n", sizeof(mi_page_t));
+ #elif MI_PAGE_META_IS_ALIGNED
+ _mi_fprintf(out, arg, "free mode : aligned, page size: %zu\n", sizeof(mi_page_t));
+ #elif MI_PAGE_META_SMALL_IS_ALIGNED
+ _mi_fprintf(out, arg, "free mode : small aligned + pagemap, page size: %zu\n", sizeof(mi_page_t));
+ #elif MI_FREE_IS_CHECKED
+ _mi_fprintf(out, arg, "free mode : checked, page size: %zu\n", sizeof(mi_page_t));
+ #else
+ _mi_fprintf(out, arg, "free mode : pagemap, page size: %zu\n", sizeof(mi_page_t));
+ #endif
+ #if MI_ENCODE_FREELIST
+ _mi_fprintf(out, arg, "free lists : encoded with %d key(s)\n", MI_PAGE_KEY_COUNT);
+ #endif
#if MI_GUARDED
- _mi_fprintf(out, arg, "guarded build: %s\n", mi_option_get(mi_option_guarded_sample_rate) != 0 ? "enabled" : "disabled");
+ _mi_fprintf(out, arg, "guarded mode: %s, rate=%ld\n", MI_SAMPLE==2 ? "fine-grained" : "enabled", mi_option_get(mi_option_guarded_sample_rate));
+ #endif
+ #if MI_PROFILE
+ _mi_fprintf(out, arg, "profiling : %s\n", MI_SAMPLE==2 ? "fine-grained" : "enabled");
#endif
#if MI_TSAN
_mi_fprintf(out, arg, "thread santizer enabled\n");
#endif
+ _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL);
+ _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG );
+ _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE );
+ _mi_fprintf(out, arg, "padding : %s\n", (MI_PADDING==0 ? "none" : (MI_PADDING==1 ? "enabled" : "byte precise")));
}
mi_decl_export void mi_options_print(void) mi_attr_noexcept {
@@ -446,25 +458,12 @@ static void mi_add_stderr_output(void) {
static _Atomic(size_t) error_count; // = 0; // when >= max_error_count stop emitting errors
static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings
-// When overriding malloc, we may recurse into mi_vfprintf if an allocation
-// inside the C runtime causes another message.
-// In some cases (like on macOS) the loader already allocates which
-// calls into mimalloc; if we then access thread locals (like `recurse`)
-// this may crash as the access may call _tlv_bootstrap that tries to
-// (recursively) invoke malloc again to allocate space for the thread local
-// variables on demand. This is why we use a _mi_preloading test on such
-// platforms. However, C code generator may move the initial thread local address
-// load before the `if` and we therefore split it out in a separate function.
-static mi_decl_thread bool recurse = false;
-
static mi_decl_noinline bool mi_recurse_enter_prim(void) {
- if (recurse) return false;
- recurse = true;
return true;
}
static mi_decl_noinline void mi_recurse_exit_prim(void) {
- recurse = false;
+ /* nothing */
}
static bool mi_recurse_enter(void) {
@@ -482,7 +481,7 @@ static void mi_recurse_exit(void) {
}
void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) {
- if (out==NULL || (void*)out==(void*)stdout || (void*)out==(void*)stderr) { // TODO: use mi_out_stderr for stderr?
+ if (out==NULL || (void*)out==(void*)stdout || (void*)out==(void*)stderr) { // todo: use mi_out_stderr for stderr?
if (!mi_recurse_enter()) return;
out = mi_out_get_default(&arg);
if (prefix != NULL) out(prefix, arg);
@@ -514,7 +513,7 @@ void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) {
}
static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) {
- if (prefix != NULL && _mi_strnlen(prefix,33) <= 32 && !_mi_is_main_thread()) {
+ if (prefix != NULL && _mi_strnlen(prefix,33) <= 32) { // && !_mi_is_main_thread()) {
char tprefix[64];
_mi_snprintf(tprefix, sizeof(tprefix), "%sthread 0x%tx: ", prefix, (uintptr_t)_mi_thread_id());
mi_vfprintf(out, arg, tprefix, fmt, args);
@@ -531,13 +530,6 @@ void _mi_raw_message(const char* fmt, ...) {
va_end(args);
}
-void _mi_message(const char* fmt, ...) {
- va_list args;
- va_start(args, fmt);
- mi_vfprintf_thread(NULL, NULL, "mimalloc: ", fmt, args);
- va_end(args);
-}
-
void _mi_trace_message(const char* fmt, ...) {
if (mi_option_get(mi_option_verbose) <= 1) return; // only with verbose level 2 or higher
va_list args;
@@ -590,24 +582,27 @@ static _Atomic(void*) mi_error_arg; // = NULL
static void mi_error_default(int err) {
MI_UNUSED(err);
-#if (MI_DEBUG>0)
- if (err==EFAULT) {
- #ifdef _MSC_VER
- __debugbreak();
- #endif
- abort();
- }
-#endif
-#if (MI_SECURE>0)
- if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data)
- abort();
- }
-#endif
-#if defined(MI_XMALLOC)
- if (err==ENOMEM || err==EOVERFLOW) { // abort on memory allocation fails in xmalloc mode
- abort();
+ #if (MI_DEBUG>0)
+ if (err==EFAULT) {
+ #ifdef _MSC_VER
+ __debugbreak();
+ #endif
+ abort();
+ }
+ #endif
+ #if (MI_SECURE>0)
+ if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data)
+ abort();
+ }
+ #endif
+ #if defined(MI_XMALLOC)
+ if (err==ENOMEM || err==EOVERFLOW || err==EINVAL) { // abort on memory allocation fails in xmalloc mode
+ abort();
+ }
+ #endif
+ if (errno==0) {
+ errno = (err==EINVAL ? EINVAL : ENOMEM /* compatibility */ );
}
-#endif
}
void mi_register_error(mi_error_fun* fun, void* arg) {
@@ -621,7 +616,7 @@ void _mi_error_message(int err, const char* fmt, ...) {
va_start(args, fmt);
mi_show_error_message(fmt, args);
va_end(args);
- // and call the error handler which may abort (or return normally)
+ // and call the error handler which may abort (or return normally, potentially setting errno)
if (mi_error_handler != NULL) {
mi_error_handler(err, mi_atomic_load_ptr_acquire(void,&mi_error_arg));
}
@@ -630,14 +625,17 @@ void _mi_error_message(int err, const char* fmt, ...) {
}
}
+mi_decl_noinline mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next) {
+ _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next);
+ return NULL;
+}
+
// --------------------------------------------------------
// Initialize options by checking the environment
// --------------------------------------------------------
// TODO: implement ourselves to reduce dependencies on the C runtime
#include <stdlib.h> // strtol
-#include <string.h> // strstr
-
static void mi_option_init(mi_option_desc_t* desc) {
// Read option value from the environment
@@ -645,17 +643,17 @@ static void mi_option_init(mi_option_desc_t* desc) {
char buf[64+1];
_mi_strlcpy(buf, "mimalloc_", sizeof(buf));
_mi_strlcat(buf, desc->name, sizeof(buf));
- bool found = _mi_getenv(buf, s, sizeof(s));
- if (!found && desc->legacy_name != NULL) {
+ int err = _mi_getenv(buf, s, sizeof(s));
+ if (err==ENOENT && desc->legacy_name != NULL) {
_mi_strlcpy(buf, "mimalloc_", sizeof(buf));
_mi_strlcat(buf, desc->legacy_name, sizeof(buf));
- found = _mi_getenv(buf, s, sizeof(s));
- if (found) {
+ err = _mi_getenv(buf, s, sizeof(s));
+ if (err==0) {
_mi_warning_message("environment option \"mimalloc_%s\" is deprecated -- use \"mimalloc_%s\" instead.\n", desc->legacy_name, desc->name);
}
}
- if (found) {
+ if (err==0) {
size_t len = _mi_strnlen(s, sizeof(buf) - 1);
for (size_t i = 0; i < len; i++) {
buf[i] = _mi_toupper(s[i]);
@@ -665,14 +663,15 @@ static void mi_option_init(mi_option_desc_t* desc) {
desc->value = 1;
desc->init = MI_OPTION_INITIALIZED;
}
- else if (_mi_streq(buf,"0") || _mi_streq(buf,"FALSE") || _mi_streq(buf,"NO") || _mi_streq(buf,"OFF")) {
+ else if (_mi_streq(buf,"0") || _mi_streq(buf,"FALSE") || _mi_streq(buf,"NO") || _mi_streq(buf,"OFF")) {
desc->value = 0;
desc->init = MI_OPTION_INITIALIZED;
}
else {
char* end = buf;
+ errno = 0;
long value = strtol(buf, &end, 10);
- if (mi_option_has_size_in_kib(desc->option)) {
+ if (errno==0 && mi_option_has_size_in_kib(desc->option)) {
// this option is interpreted in KiB to prevent overflow of `long` for large allocations
// (long is 32-bit on 64-bit windows, which allows for 4TiB max.)
size_t size = (value < 0 ? 0 : (size_t)value);
@@ -687,7 +686,7 @@ static void mi_option_init(mi_option_desc_t* desc) {
if (overflow || size > (MI_MAX_ALLOC_SIZE / MI_KiB)) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); }
value = (size > LONG_MAX ? LONG_MAX : (long)size);
}
- if (*end == 0) {
+ if (errno==0 && *end == 0) {
mi_option_set(desc->option, value);
}
else {
@@ -707,7 +706,8 @@ static void mi_option_init(mi_option_desc_t* desc) {
}
mi_assert_internal(desc->init != MI_OPTION_UNINIT);
}
- else if (!_mi_preloading()) {
+ else if (err==ENOENT) {
desc->init = MI_OPTION_DEFAULTED;
}
+ // and on another error, keep unitialized to try again (can happen during preloading if getenv is not available)
}
diff --git a/vendored/mimalloc/src/os.c b/vendored/mimalloc/src/os.c
index febecb0c9..15111cf94 100644
--- a/vendored/mimalloc/src/os.c
+++ b/vendored/mimalloc/src/os.c
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -8,12 +8,13 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc/internal.h"
#include "mimalloc/atomic.h"
#include "mimalloc/prim.h"
+#include "mimalloc/prim-tls.h" // _mi_theap_default for random
/* -----------------------------------------------------------
Initialization.
----------------------------------------------------------- */
#ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB
-#if MI_INTPTR_SIZE < 8
+#if MI_SIZE_SIZE < 8
#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 4*MI_MiB // 4 GiB
#else
#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 32*MI_MiB // 32 GiB
@@ -40,6 +41,9 @@ bool _mi_os_has_virtual_reserve(void) {
return mi_os_mem_config.has_virtual_reserve;
}
+bool _mi_os_canuse_thp(void) {
+ return mi_os_mem_config.has_transparent_huge_pages;
+}
// OS (small) page size
size_t _mi_os_page_size(void) {
@@ -54,13 +58,19 @@ size_t _mi_os_large_page_size(void) {
// minimal purge size. Can be larger than the page size if transparent huge pages are enabled.
size_t _mi_os_minimal_purge_size(void) {
size_t minsize = mi_option_get_size(mi_option_minimal_purge_size);
- if (minsize != 0) {
+ if (minsize != 0) {
+ // set by user
return _mi_align_up(minsize, _mi_os_page_size());
}
else if (mi_os_mem_config.has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) {
+ // don't break up THP pages;
+ // we don't do this by default as that can lead to increased memory usage (see issue #1282).
+ // on the other hand, breaking up THP pages can lead to performance degradation and perhaps
+ // allow_thp should be 2 by default (and the user can set the OS THP setting to [never]).
return _mi_os_large_page_size();
}
else {
+ // OS page size by default
return _mi_os_page_size();
}
}
@@ -103,53 +113,57 @@ void _mi_os_init(void) {
/* -----------------------------------------------------------
Util
-------------------------------------------------------------- */
-bool _mi_os_decommit(void* addr, size_t size);
-bool _mi_os_commit(void* addr, size_t size, bool* is_zero);
+bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size);
+bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero);
// On systems with enough virtual address bits, we can do efficient aligned allocation by using
// the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address
// space (64TiB) we use this technique. (but see issue #939)
-#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID)
+#if (MI_SIZE_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID)
-// Return a MI_HINT_ALIGN (4MiB) aligned address that is probably available.
+// Return a `try_alignment` aligned address that is probably available.
// If this returns NULL, the OS will determine the address but on some OS's that may not be
// properly aligned which can be more costly as it needs to be adjusted afterwards.
-// For a size > 16GiB this always returns NULL in order to guarantee good ASLR randomization;
+// In secure mode, for a size > 16GiB this always returns NULL in order to guarantee good ASLR randomization;
// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses
// in the middle of the 2TiB - 6TiB address range (see issue #372))
-#define MI_HINT_ALIGN ((uintptr_t)4 << 20) // 4MiB alignment
#define MI_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start
#define MI_HINT_AREA ((uintptr_t)4 << 40) // upto (2+4) 6TiB (since before win8 there is "only" 8TiB available to processes)
#define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages)
-void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)
+void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze)
{
static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // = 0
// todo: perhaps only do alignment hints if THP is enabled?
- if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > MI_HINT_ALIGN) return NULL;
+ if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > 16*MI_GiB) return NULL;
if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space
- size = _mi_align_up(size, MI_HINT_ALIGN);
- if (size > 16*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at least 1/(MI_HINT_AREA / 1<<34)
- size += MI_HINT_ALIGN; // put in virtual gaps between hinted blocks; this splits VLA's but increases guarded areas.
-
- uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size);
+
+ size_t req_size = sze + _mi_os_page_size(); // always reserve a bit more to create virtual gaps between hinted blocks.
+ req_size += (try_alignment - 1); // ensure we can align in the requested size
+ req_size = _mi_align_up(req_size, _mi_os_large_page_size());
+ #if (MI_SECURE>=1)
+ if (req_size > 32*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<34) = 1/256
+ #endif
+
+ uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, req_size);
if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize
uintptr_t init = MI_HINT_BASE;
- #if (MI_SECURE>=1 || defined(NDEBUG)) // security: randomize start of aligned allocations unless in debug mode
+ #if (MI_SECURE>=1 || !MI_DEBUG) // security: randomize start of aligned allocations unless in debug mode
mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)
if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point
const uintptr_t r = _mi_theap_random_next(theap);
- init = init + ((MI_HINT_ALIGN * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB
+ init = init + ((MI_MiB * ((r>>17) & 0x3FFFFF)) % MI_HINT_AREA); // (randomly 22 bits)* 1MiB == 0 to 4TiB
#endif
- uintptr_t expected = hint + size;
+ uintptr_t expected = hint + req_size;
mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);
- hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all
+ hint = mi_atomic_add_acq_rel(&aligned_base, req_size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all
+ if (hint==0) return NULL;
}
- mi_assert_internal(hint%MI_HINT_ALIGN == 0);
- if (hint%try_alignment != 0) return NULL;
- return (void*)hint;
+ const uintptr_t hint_align = _mi_align_up(hint,try_alignment);
+ mi_assert_internal(hint_align + sze < hint + req_size);
+ return (void*)hint_align;
}
#else
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
@@ -173,7 +187,7 @@ size_t _mi_os_secure_guard_page_size(void) {
}
// In secure mode, try to decommit an area and output a warning if this fails.
-bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid) {
+bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {
if (addr == NULL) return true;
#if MI_SECURE > 0
bool ok = false;
@@ -183,7 +197,7 @@ bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid) {
ok = (*(arena->commit_fun))(false /* decommit */, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg);
}
else {
- ok = _mi_os_decommit(addr, _mi_os_secure_guard_page_size());
+ ok = _mi_os_decommit(subproc, addr, _mi_os_secure_guard_page_size());
}
}
if (!ok) {
@@ -191,18 +205,18 @@ bool _mi_os_secure_guard_page_set_at(void* addr, mi_memid_t memid) {
}
return ok;
#else
- MI_UNUSED(memid);
+ MI_UNUSED(subproc); MI_UNUSED(memid);
return true;
#endif
}
// In secure mode, try to decommit an area and output a warning if this fails.
-bool _mi_os_secure_guard_page_set_before(void* addr, mi_memid_t memid) {
- return _mi_os_secure_guard_page_set_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);
+bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {
+ return _mi_os_secure_guard_page_set_at(subproc, (uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);
}
// In secure mode, try to recommit an area
-bool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid) {
+bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {
if (addr == NULL) return true;
#if MI_SECURE > 0
if (!memid.is_pinned) {
@@ -211,18 +225,18 @@ bool _mi_os_secure_guard_page_reset_at(void* addr, mi_memid_t memid) {
return (*(arena->commit_fun))(true, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg);
}
else {
- return _mi_os_commit(addr, _mi_os_secure_guard_page_size(), NULL);
+ return _mi_os_commit(subproc, addr, _mi_os_secure_guard_page_size(), NULL);
}
}
#else
- MI_UNUSED(memid);
+ MI_UNUSED(subproc); MI_UNUSED(memid);
#endif
return true;
}
// In secure mode, try to recommit an area
-bool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid) {
- return _mi_os_secure_guard_page_reset_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);
+bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {
+ return _mi_os_secure_guard_page_reset_at(subproc, (uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);
}
@@ -230,23 +244,27 @@ bool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid) {
Free memory
-------------------------------------------------------------- */
-static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc);
+static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size);
-static void mi_os_prim_free(void* addr, size_t size, size_t commit_size, mi_subproc_t* subproc) {
+static void mi_os_prim_free(mi_subproc_t* subproc, void* addr, size_t size, size_t commit_size, bool adjust) {
+ mi_assert_internal(subproc!=NULL);
mi_assert_internal((size % _mi_os_page_size()) == 0);
if (addr == NULL) return; // || _mi_os_is_huge_reserved(addr)
int err = _mi_prim_free(addr, size); // allow size==0 (issue #1041)
if (err != 0) {
_mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
}
- if (subproc == NULL) { subproc = _mi_subproc(); } // from `mi_arenas_unsafe_destroy` we pass subproc_main explicitly as we can no longer use the theap pointer
- if (commit_size > 0) {
- mi_subproc_stat_decrease(subproc, committed, commit_size);
+ if (adjust) {
+ if (commit_size>0) { mi_subproc_stat_adjust_decrease(subproc, committed, commit_size); }
+ mi_subproc_stat_adjust_decrease(subproc, reserved, size);
+ }
+ else {
+ if (commit_size>0) { mi_subproc_stat_decrease(subproc, committed, commit_size); }
+ mi_subproc_stat_decrease(subproc, reserved, size);
}
- mi_subproc_stat_decrease(subproc, reserved, size);
}
-void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc /* can be NULL */) {
+void _mi_os_free_ex(mi_subproc_t* subproc, void* addr, size_t size, bool still_committed, mi_memid_t memid) {
if (mi_memkind_is_os(memid.memkind)) {
size_t csize = memid.mem.os.size;
if (csize==0) { csize = _mi_os_good_alloc_size(size); }
@@ -268,10 +286,10 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me
// free it
if (memid.memkind == MI_MEM_OS_HUGE) {
mi_assert(memid.is_pinned);
- mi_os_free_huge_os_pages(base, csize, subproc);
+ mi_os_free_huge_os_pages(subproc, base, csize);
}
else {
- mi_os_prim_free(base, csize, (still_committed ? commit_size : 0), subproc);
+ mi_os_prim_free(subproc, base, csize, (still_committed ? commit_size : 0), false);
}
}
else {
@@ -280,8 +298,8 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me
}
}
-void _mi_os_free(void* p, size_t size, mi_memid_t memid) {
- _mi_os_free_ex(p, size, true, memid, NULL);
+void _mi_os_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) {
+ _mi_os_free_ex(subproc, p, size, true, memid);
}
@@ -291,7 +309,7 @@ void _mi_os_free(void* p, size_t size, mi_memid_t memid) {
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
// Also `hint_addr` is a hint and may be ignored.
-static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
+static void* mi_os_prim_alloc_at(mi_subproc_t* subproc, void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
mi_assert_internal(is_zero != NULL);
mi_assert_internal(is_large != NULL);
@@ -312,11 +330,11 @@ static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignm
_mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large);
}
- mi_os_stat_counter_increase(mmap_calls, 1);
+ mi_subproc_stat_counter_increase(subproc, mmap_calls, 1);
if (p != NULL) {
- mi_os_stat_increase(reserved, size);
+ mi_subproc_stat_increase(subproc, reserved, size);
if (commit) {
- mi_os_stat_increase(committed, size);
+ mi_subproc_stat_increase(subproc, committed, size);
// seems needed for asan (or `mimalloc-test-api` fails)
#ifdef MI_TRACK_ASAN
if (*is_zero) { mi_track_mem_defined(p,size); }
@@ -327,14 +345,14 @@ static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignm
return p;
}
-static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
- return mi_os_prim_alloc_at(NULL, size, try_alignment, commit, allow_large, is_large, is_zero);
+static void* mi_os_prim_alloc(mi_subproc_t* subproc, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
+ return mi_os_prim_alloc_at(subproc, NULL /* hint addr */, size, try_alignment, commit, allow_large, is_large, is_zero);
}
// Primitive aligned allocation from the OS.
// This function guarantees the allocated memory is aligned.
-static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) {
+static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) {
mi_assert_internal(memid!=NULL);
mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
@@ -343,8 +361,12 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
size = _mi_align_up(size, _mi_os_page_size());
+ #if MI_SIZE_SIZE >= 8
+ const bool try_direct_alloc = true;
+ #else
// try a direct allocation if the alignment is below the default, or less than or equal to 1/4 fraction of the size.
const bool try_direct_alloc = (alignment <= mi_os_mem_config.alloc_granularity || alignment <= size/4);
+ #endif
bool os_is_large = false;
bool os_is_zero = false;
@@ -352,7 +374,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
size_t os_size = size;
void* p = NULL;
if (try_direct_alloc) {
- p = mi_os_prim_alloc(size, alignment, commit, allow_large, &os_is_large, &os_is_zero);
+ p = mi_os_prim_alloc(subproc, size, alignment, commit, allow_large, &os_is_large, &os_is_zero);
}
// aligned already?
@@ -366,33 +388,34 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
_mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);
}
#endif
- if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0), NULL); }
+ if (p != NULL) { mi_os_prim_free(subproc, p, size, (commit ? size : 0), true /* adjust so we "forget" the previous reservation */); }
if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
const size_t over_size = size + alignment;
if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block
// over-allocate uncommitted (virtual) memory
- p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero);
+ p = mi_os_prim_alloc(subproc, over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero);
if (p == NULL) return NULL;
// set p to the aligned part in the full region
// note: Windows VirtualFree needs the actual base pointer
// this is handled though by having the `base` field in the memid
os_base = p; // remember the base
- os_size = over_size;
+ os_size = over_size; // todo: use size instead as now we over-decrement commit stats on free?
p = _mi_align_up_ptr(p, alignment);
// explicitly commit only the aligned part
if (commit) {
- if (!_mi_os_commit(p, size, NULL)) {
- mi_os_prim_free(os_base, over_size, 0, NULL);
+ if (!_mi_os_commit(subproc, p, size, NULL)) {
+ mi_os_prim_free(subproc, os_base, over_size, 0, true);
return NULL;
}
}
}
else { // mmap can free inside an allocation
// overallocate...
- p = mi_os_prim_alloc(over_size, 1, commit, false, &os_is_large, &os_is_zero);
+ // note: we keep `allow_large` but this only works if we can partially free any large OS pages later on. This seems to be the case though.
+ p = mi_os_prim_alloc(subproc, over_size, 1, commit, allow_large /* or false? */, &os_is_large, &os_is_zero);
if (p == NULL) return NULL;
// and selectively unmap parts around the over-allocated area.
@@ -401,8 +424,8 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
const size_t mid_size = _mi_align_up(size, _mi_os_page_size());
const size_t post_size = over_size - pre_size - mid_size;
mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
- if (pre_size > 0) { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0), NULL); }
- if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), NULL); }
+ if (pre_size > 0) { mi_os_prim_free(subproc, p, pre_size, (commit ? pre_size : 0), true /* adjust */); }
+ if (post_size > 0) { mi_os_prim_free(subproc, (uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), true /* adjust */); }
// we can return the aligned pointer on `mmap` systems
p = aligned_p;
os_base = aligned_p; // since we freed the pre part, `*base == p`.
@@ -421,13 +444,13 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
OS API: alloc and alloc_aligned
----------------------------------------------------------- */
-void* _mi_os_alloc(size_t size, mi_memid_t* memid) {
+void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) {
*memid = _mi_memid_none();
if (size == 0) return NULL;
size = _mi_os_good_alloc_size(size);
bool os_is_large = false;
bool os_is_zero = false;
- void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero);
+ void* p = mi_os_prim_alloc(subproc, size, 0, true, false, &os_is_large, &os_is_zero);
if (p == NULL) return NULL;
*memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large);
@@ -436,7 +459,7 @@ void* _mi_os_alloc(size_t size, mi_memid_t* memid) {
return p;
}
-void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid)
+void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid)
{
MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings
*memid = _mi_memid_none();
@@ -444,7 +467,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo
size = _mi_os_good_alloc_size(size);
alignment = _mi_align_up(alignment, _mi_os_page_size());
- void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, memid );
+ void* p = mi_os_prim_alloc_aligned(subproc, size, alignment, commit, allow_large, memid );
if (p == NULL) return NULL;
mi_assert_internal(memid->mem.os.size >= size);
@@ -454,13 +477,13 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo
}
-mi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_t* memid) {
+mi_decl_nodiscard static void* mi_os_ensure_zero(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t* memid) {
if (p==NULL || size==0) return p;
// ensure committed
if (!memid->initially_committed) {
bool is_zero = false;
- if (!_mi_os_commit(p, size, &is_zero)) {
- _mi_os_free(p, size, *memid);
+ if (!_mi_os_commit(subproc, p, size, &is_zero)) {
+ _mi_os_free(subproc, p, size, *memid);
return NULL;
}
memid->initially_committed = true;
@@ -472,9 +495,9 @@ mi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_
return p;
}
-void* _mi_os_zalloc(size_t size, mi_memid_t* memid) {
- void* p = _mi_os_alloc(size,memid);
- return mi_os_ensure_zero(p, size, memid);
+void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) {
+ void* p = _mi_os_alloc(subproc, size,memid);
+ return mi_os_ensure_zero(subproc, p, size, memid);
}
/* -----------------------------------------------------------
@@ -485,28 +508,29 @@ void* _mi_os_zalloc(size_t size, mi_memid_t* memid) {
to use the actual start of the memory region.
----------------------------------------------------------- */
-void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) {
+void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) {
mi_assert(offset <= size);
mi_assert((alignment % _mi_os_page_size()) == 0);
*memid = _mi_memid_none();
if (offset > size) return NULL;
if (offset == 0) {
// regular aligned allocation
- return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);
+ return _mi_os_alloc_aligned(subproc, size, alignment, commit, allow_large, memid);
}
else {
// overallocate to align at an offset
const size_t extra = _mi_align_up(offset, alignment) - offset;
if (size >= SIZE_MAX - extra) return NULL; // too large
const size_t oversize = size + extra;
- void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid);
+ void* const start = _mi_os_alloc_aligned(subproc, oversize, alignment, commit, allow_large, memid);
if (start == NULL) return NULL;
void* const p = (uint8_t*)start + extra;
mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment));
// decommit the overallocation at the start
+ // note: this double counts the decommit when freeing `memid`. Should we keep commit size in the memid as well?
if (commit && extra >= _mi_os_page_size()) {
- _mi_os_decommit(start, extra);
+ _mi_os_decommit(subproc, start, extra);
}
return p;
}
@@ -540,9 +564,10 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t*
return mi_os_page_align_areax(true, addr, size, newsize);
}
-bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) {
+bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed) {
+ mi_assert_internal(size >= stat_already_committed);
if (is_zero != NULL) { *is_zero = false; }
- mi_os_stat_counter_increase(commit_calls, 1);
+ mi_subproc_stat_counter_increase(subproc, commit_calls, 1);
// page align range
size_t csize;
@@ -565,15 +590,15 @@ bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size)
if (os_is_zero) { mi_track_mem_defined(start,csize); }
else { mi_track_mem_undefined(start,csize); }
#endif
- mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit
+ mi_subproc_stat_increase(subproc, committed, size - stat_already_committed); // for precise commit vs. decommit
return true;
}
-bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
- return _mi_os_commit_ex(addr, size, is_zero, size);
+bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero) {
+ return _mi_os_commit_ex(subproc, addr, size, is_zero, 0);
}
-static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stat_size) {
+static bool mi_os_decommit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* needs_recommit, size_t stat_size) {
mi_assert_internal(needs_recommit!=NULL);
// page align
@@ -588,15 +613,15 @@ static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, siz
_mi_warning_message("cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
}
else if (*needs_recommit) {
- mi_os_stat_decrease(committed, stat_size);
+ mi_subproc_stat_decrease(subproc, committed, stat_size);
}
mi_assert_internal(err == 0);
return (err == 0);
}
-bool _mi_os_decommit(void* addr, size_t size) {
+bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size) {
bool needs_recommit;
- return mi_os_decommit_ex(addr, size, &needs_recommit, size);
+ return mi_os_decommit_ex(subproc, addr, size, &needs_recommit, size);
}
@@ -604,13 +629,13 @@ bool _mi_os_decommit(void* addr, size_t size) {
// but may be used later again. This will release physical memory
// pages and reduce swapping while keeping the memory committed.
// We page align to a conservative area inside the range to reset.
-bool _mi_os_reset(void* addr, size_t size) {
+bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size) {
// page align conservatively within the range
size_t csize;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr)
- mi_os_stat_counter_increase(reset, csize);
- mi_os_stat_counter_increase(reset_calls, 1);
+ mi_subproc_stat_counter_increase(subproc, reset, csize);
+ mi_subproc_stat_counter_increase(subproc, reset_calls, 1);
#if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN
memset(start, 0, csize); // pretend it is eagerly reset
@@ -624,7 +649,8 @@ bool _mi_os_reset(void* addr, size_t size) {
}
-void _mi_os_reuse( void* addr, size_t size ) {
+void _mi_os_reuse( mi_subproc_t* subproc, void* addr, size_t size ) {
+ MI_UNUSED(subproc);
// page align conservatively within the range
size_t csize = 0;
void* const start = mi_os_page_align_area_conservative(addr, size, &csize);
@@ -637,11 +663,11 @@ void _mi_os_reuse( void* addr, size_t size ) {
// either resets or decommits memory, returns true if the memory needs
// to be recommitted if it is to be re-used later on.
-bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg)
+bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg)
{
if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed?
- mi_os_stat_counter_increase(purge_calls, 1);
- mi_os_stat_counter_increase(purged, size);
+ mi_subproc_stat_counter_increase(subproc, purge_calls, 1);
+ mi_subproc_stat_counter_increase(subproc, purged, size);
if (commit_fun != NULL) {
bool decommitted = (*commit_fun)(false, p, size, NULL, commit_fun_arg);
@@ -651,12 +677,12 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, m
!_mi_preloading()) // don't decommit during preloading (unsafe)
{
bool needs_recommit = true;
- mi_os_decommit_ex(p, size, &needs_recommit, stat_size);
+ mi_os_decommit_ex(subproc, p, size, &needs_recommit, stat_size);
return needs_recommit;
}
else {
if (allow_reset) { // this can sometimes be not allowed if the range is not fully committed (on Windows, we cannot reset uncommitted memory)
- _mi_os_reset(p, size);
+ _mi_os_reset(subproc, p, size);
}
return false; // needs no recommit
}
@@ -664,8 +690,8 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, m
// either resets or decommits memory, returns true if the memory needs
// to be recommitted if it is to be re-used later on.
-bool _mi_os_purge(void* p, size_t size) {
- return _mi_os_purge_ex(p, size, true, size, NULL, NULL);
+bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size) {
+ return _mi_os_purge_ex(subproc, p, size, true, size, NULL, NULL);
}
@@ -704,14 +730,18 @@ and possibly associated with a specific NUMA node. (use `numa_node>=0`)
#define MI_HUGE_OS_PAGE_SIZE (MI_GiB)
-#if (MI_INTPTR_SIZE >= 8)
+#if (MI_SIZE_SIZE >= 8)
// To ensure proper alignment, use our own area for huge OS pages
static mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0
// Claim an aligned address range for huge pages
static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
if (total_size != NULL) *total_size = 0;
- const size_t size = pages * MI_HUGE_OS_PAGE_SIZE;
+ size_t size = 0;
+ if (mi_mul_overflow(pages,MI_HUGE_OS_PAGE_SIZE,&size)) {
+ _mi_warning_message("too many huge pages requested: %zu\n", pages);
+ return NULL;
+ }
uintptr_t start = 0;
uintptr_t end = 0;
@@ -720,7 +750,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
start = huge_start;
if (start == 0) {
// Initialize the start address after the 32TiB area
- start = ((uintptr_t)8 << 40); // 8TiB virtual start address
+ start = ((uintptr_t)32 << 40); // 32TiB virtual start address (after addresses returned by _mi_os_get_aligned_hint)
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode
mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)
if (mi_theap_is_initialized(theap)) { // todo: or no hint at all if we lack randomness?
@@ -728,7 +758,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB
}
else {
- _mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, start);
+ _mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, (void*)start);
}
#endif
}
@@ -747,7 +777,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
#endif
// Allocate MI_ARENA_SLICE_ALIGN aligned huge pages
-void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) {
+void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) {
*memid = _mi_memid_none();
if (psize != NULL) *psize = 0;
if (pages_reserved != NULL) *pages_reserved = 0;
@@ -778,21 +808,21 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
// no success, issue a warning and break
if (p != NULL) {
_mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr);
- mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, NULL);
+ mi_os_prim_free(subproc, p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, true /* adjust */);
}
break;
}
// success, record it
page++; // increase before timeout check (see issue #711)
- mi_os_stat_increase(committed, MI_HUGE_OS_PAGE_SIZE);
- mi_os_stat_increase(reserved, MI_HUGE_OS_PAGE_SIZE);
+ mi_subproc_stat_increase(subproc, committed, MI_HUGE_OS_PAGE_SIZE);
+ mi_subproc_stat_increase(subproc, reserved, MI_HUGE_OS_PAGE_SIZE);
// check for timeout
if (max_msecs > 0) {
mi_msecs_t elapsed = _mi_clock_end(start_t);
if (page >= 1) {
- mi_msecs_t estimate = ((elapsed / (page+1)) * pages);
+ mi_msecs_t estimate = ((elapsed / (page==0 ? 1 : page)) * pages);
if (estimate > 2*max_msecs) { // seems like we are going to timeout, break
elapsed = max_msecs + 1;
}
@@ -821,11 +851,11 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
// free every huge page in a range individually (as we allocated per page)
// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems.
-static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc) {
+static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size) {
if (p==NULL || size==0) return;
uint8_t* base = (uint8_t*)p;
while (size >= MI_HUGE_OS_PAGE_SIZE) {
- mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, subproc);
+ mi_os_prim_free(subproc, base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, false /* adjust? */);
size -= MI_HUGE_OS_PAGE_SIZE;
base += MI_HUGE_OS_PAGE_SIZE;
}
diff --git a/vendored/mimalloc/src/page-map.c b/vendored/mimalloc/src/page-map.c
index acc0ee9f7..949e37ac3 100644
--- a/vendored/mimalloc/src/page-map.c
+++ b/vendored/mimalloc/src/page-map.c
@@ -1,5 +1,5 @@
/*----------------------------------------------------------------------------
-Copyright (c) 2023-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2023-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -25,8 +25,11 @@ static void mi_page_map_cannot_commit(void) {
// A full 256 TiB address space (48 bit) needs a 4 GiB page map.
// A full 4 GiB address space (32 bit) needs only a 64 KiB page map.
-mi_decl_cache_align uint8_t* _mi_page_map = NULL;
-static void* mi_page_map_max_address = NULL;
+// Use an initial empty page map so `free(NULL)` works even if mimalloc is not yet initialized (issue #1341)
+static uint8_t mi_page_map_empty[1] = { 1 }; // _mi_ptr_page(NULL) == NULL
+
+mi_decl_hidden mi_decl_cache_align _Atomic(uint8_t*) _mi_page_map = mi_page_map_empty;
+mi_decl_hidden _Atomic(void*) _mi_page_map_max_address = NULL;
static mi_memid_t mi_page_map_memid;
#define MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT MI_ARENA_SLICE_SIZE
@@ -35,7 +38,7 @@ static mi_bitmap_t* mi_page_map_commit; // one bit per committed 64 KiB entries
mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count);
bool _mi_page_map_init(void) {
- size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS);
+ size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_MAX_VABITS);
if (vbits == 0) {
vbits = _mi_os_virtual_address_bits();
#if MI_ARCH_X64 // canonical address is limited to the first 128 TiB
@@ -45,15 +48,22 @@ bool _mi_page_map_init(void) {
if (vbits < MI_ARENA_SLICE_SHIFT) {
vbits = MI_ARENA_SLICE_SHIFT;
}
+ if (vbits < MI_MIN_VABITS) { // cover at least this much for a faster _mi_checked_ptr
+ vbits = MI_MIN_VABITS;
+ }
+ if (vbits > MI_MAX_VABITS) { // limit page map size even if more virtual addresses are available
+ vbits = MI_MAX_VABITS;
+ }
// Allocate the page map and commit bits
- mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));
+ mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits)));
const size_t page_map_size = (MI_ZU(1) << (vbits - MI_ARENA_SLICE_SHIFT));
const bool commit = (page_map_size <= 1*MI_MiB || mi_option_is_enabled(mi_option_pagemap_commit)); // _mi_os_has_overcommit(); // commit on-access on Linux systems?
const size_t commit_bits = _mi_divide_up(page_map_size, MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);
const size_t bitmap_size = (commit ? 0 : mi_bitmap_size(commit_bits, NULL));
const size_t reserve_size = bitmap_size + page_map_size;
- uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);
+ mi_subproc_t* const subproc = _mi_subproc_main();
+ uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(subproc, reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);
if (base==NULL) {
_mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB);
return false;
@@ -64,13 +74,13 @@ bool _mi_page_map_init(void) {
}
if (bitmap_size > 0) {
mi_page_map_commit = (mi_bitmap_t*)base;
- if (!_mi_os_commit(mi_page_map_commit, bitmap_size, NULL)) {
+ if (!_mi_os_commit(subproc, mi_page_map_commit, bitmap_size, NULL)) {
mi_page_map_cannot_commit();
return false;
}
mi_bitmap_init(mi_page_map_commit, commit_bits, true);
}
- _mi_page_map = base + bitmap_size;
+ mi_atomic_store_ptr_release(uint8_t,&_mi_page_map, base + bitmap_size);
// commit the first part so NULL pointers get resolved without an access violation
if (!commit) {
@@ -79,19 +89,18 @@ bool _mi_page_map_init(void) {
return false;
}
}
- _mi_page_map[0] = 1; // so _mi_ptr_page(NULL) == NULL
+ mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map)[0] = 1; // so _mi_ptr_page(NULL) == NULL
mi_assert_internal(_mi_ptr_page(NULL)==NULL);
return true;
}
-void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) {
- mi_assert_internal(subproc != NULL);
- mi_assert_internal(_mi_page_map != NULL);
- if (_mi_page_map == NULL) return;
- _mi_os_free_ex(mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc);
- _mi_page_map = NULL;
+void _mi_page_map_unsafe_destroy(void) {
+ mi_assert_internal(mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map) != NULL);
+ if (mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map) == NULL) return;
+ _mi_os_free_ex(_mi_subproc_main(), mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, true, mi_page_map_memid);
+ mi_atomic_store_ptr_release(uint8_t, &_mi_page_map, NULL);
mi_page_map_commit = NULL;
- mi_page_map_max_address = NULL;
+ mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, NULL);
mi_page_map_memid = _mi_memid_none();
}
@@ -100,6 +109,7 @@ static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) {
// is the page map area that contains the page address committed?
// we always set the commit bits so we can track what ranges are in-use.
// we only actually commit if the map wasn't committed fully already.
+ uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map);
if (mi_page_map_commit != NULL) {
const size_t commit_idx = idx / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;
const size_t commit_idx_hi = (idx + slice_count - 1) / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;
@@ -107,9 +117,9 @@ static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) {
if (mi_bitmap_is_clear(mi_page_map_commit, i)) {
// this may race, in which case we do multiple commits (which is ok)
bool is_zero;
- uint8_t* const start = _mi_page_map + (i * MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);
+ uint8_t* const start = page_map + (i * MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT);
const size_t size = MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT;
- if (!_mi_os_commit(start, size, &is_zero)) {
+ if (!_mi_os_commit(_mi_subproc_main(), start, size, &is_zero)) {
mi_page_map_cannot_commit();
return false;
}
@@ -119,8 +129,8 @@ static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) {
}
}
#if MI_DEBUG > 0
- _mi_page_map[idx] = 0;
- _mi_page_map[idx+slice_count-1] = 0;
+ page_map[idx] = 0;
+ page_map[idx+slice_count-1] = 0;
#endif
return true;
}
@@ -137,11 +147,13 @@ static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t*
bool _mi_page_map_register(mi_page_t* page) {
mi_assert_internal(page != NULL);
mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
- mi_assert_internal(_mi_page_map != NULL); // should be initialized before multi-thread access!
- if mi_unlikely(_mi_page_map == NULL) {
+ mi_assert_internal(mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map) != NULL); // should be initialized before multi-thread access!
+ uint8_t* page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map);
+ if mi_unlikely(mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map) == NULL) {
if (!_mi_page_map_init()) return false;
+ page_map = mi_atomic_load_ptr_acquire(uint8_t,&_mi_page_map);
}
- mi_assert(_mi_page_map!=NULL);
+ mi_assert(page_map!=NULL);
uint8_t* page_start;
size_t slice_count;
const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count);
@@ -153,37 +165,42 @@ bool _mi_page_map_register(mi_page_t* page) {
// set the offsets
for (size_t i = 0; i < slice_count; i++) {
mi_assert_internal(i < 128);
- _mi_page_map[idx + i] = (uint8_t)(i+1);
+ page_map[idx + i] = (uint8_t)(i+1);
}
return true;
}
void _mi_page_map_unregister(mi_page_t* page) {
- mi_assert_internal(_mi_page_map != NULL);
+ uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map);
+ mi_assert_internal(page_map != NULL);
+ if (page_map == NULL) return;
// get index and count
uint8_t* page_start;
size_t slice_count;
const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count);
// unset the offsets
- _mi_memzero(_mi_page_map + idx, slice_count);
+ _mi_memzero(page_map + idx, slice_count);
}
void _mi_page_map_unregister_range(void* start, size_t size) {
+ uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map);
+ mi_assert_internal(page_map!=NULL);
+ if (page_map == NULL) return;
const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE);
const uintptr_t index = _mi_page_map_index(start);
// todo: scan the commit bits and clear only those ranges?
if (!mi_page_map_ensure_committed(index, slice_count)) { // we commit the range in total;
return;
}
- _mi_memzero(&_mi_page_map[index], slice_count);
+ _mi_memzero(&page_map[index], slice_count);
}
mi_page_t* _mi_safe_ptr_page(const void* p) {
- if mi_unlikely(p >= mi_page_map_max_address) return NULL;
+ if mi_unlikely(p >= mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL;
const uintptr_t idx = _mi_page_map_index(p);
if mi_unlikely(mi_page_map_commit != NULL && !mi_bitmap_is_set(mi_page_map_commit, idx/MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT)) return NULL;
- const uintptr_t ofs = _mi_page_map[idx];
+ const uintptr_t ofs = _mi_page_map_at(idx);
if mi_unlikely(ofs == 0) return NULL;
return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) - ofs + 1) << MI_ARENA_SLICE_SHIFT);
}
@@ -196,43 +213,64 @@ mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_att
// A 2-level page map
#define MI_PAGE_MAP_SUB_SIZE (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*))
-#define MI_PAGE_MAP_ENTRIES_PER_CBIT (MI_PAGE_MAP_COUNT < MI_BFIELD_BITS ? 1 : (MI_PAGE_MAP_COUNT / MI_BFIELD_BITS))
-mi_decl_cache_align _Atomic(mi_submap_t)* _mi_page_map;
-static size_t mi_page_map_count;
-static void* mi_page_map_max_address;
-static mi_memid_t mi_page_map_memid;
-static mi_lock_t mi_page_map_lock;
+// Use an initial empty page map so `free(NULL)` works even if mimalloc is not yet initialized (issue #1341)
+static mi_page_map_t mi_page_map_empty = {
+ MI_ATOMIC_VAR_INIT(1),
+ sizeof(mi_page_map_t),
+ MI_MEMID_STATIC,
+ MI_LOCK_INITIALIZER,
+ { MI_ATOMIC_VAR_INIT(NULL) }
+};
+
+mi_decl_hidden mi_decl_cache_align _Atomic(mi_page_map_t*) __mi_page_map = MI_ATOMIC_VAR_INIT(&mi_page_map_empty);
-// divide the main map in 64 (`MI_BFIELD_BITS`) parts commit those parts on demand
-static _Atomic(mi_bfield_t) mi_page_map_commit;
+static size_t mi_page_map_count_of_size(size_t size) {
+ return (size < sizeof(mi_page_map_t) ? 0 : 1 + (size - sizeof(mi_page_map_t))/sizeof(mi_submap_t));
+}
+
+// static void* mi_page_map_addr_of_index(size_t idx) {
+// return (void*)((uintptr_t)idx * MI_PAGE_MAP_SUB_COUNT * MI_ARENA_SLICE_SIZE);
+// }
-mi_decl_nodiscard static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx) {
- mi_bfield_t commit = mi_atomic_load_relaxed(&mi_page_map_commit);
- const size_t bit_idx = idx/MI_PAGE_MAP_ENTRIES_PER_CBIT;
- mi_assert_internal(bit_idx < MI_BFIELD_BITS);
- if (pbit_idx != NULL) { *pbit_idx = bit_idx; }
- return ((commit & (MI_ZU(1) << bit_idx)) != 0);
+mi_decl_nodiscard static mi_decl_noinline bool mi_page_map_commit_entries(mi_page_map_t* pmap, size_t required_idx) {
+ const size_t reserved_count = mi_page_map_count_of_size(pmap->reserved_size);
+ if mi_unlikely(required_idx >= reserved_count) {
+ mi_page_map_cannot_commit();
+ return false;
+ }
+ size_t commit_size = _mi_align_up( sizeof(mi_page_map_t) + (required_idx * sizeof(mi_submap_t)), MI_ARENA_SLICE_SIZE );
+ if (pmap->reserved_size < commit_size) { commit_size = pmap->reserved_size; }
+ const size_t commit_count = mi_page_map_count_of_size(commit_size);
+ mi_assert_internal(commit_count > required_idx);
+ mi_assert_internal(commit_count <= reserved_count);
+ // note: we rely on uncommitted memory to be zero initialized on the first commit (and further concurrent commits leave the memory as is).
+ bool is_zero;
+ if mi_unlikely(!_mi_os_commit(_mi_subproc_main(), pmap, commit_size, &is_zero)) {
+ mi_page_map_cannot_commit();
+ return false;
+ }
+ mi_assert_internal(is_zero || pmap->memid.initially_zero);
+ mi_atomic_store_release(&pmap->committed_count, commit_count);
+ // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count));
+ return true;
}
-mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, mi_submap_t* submap) {
+mi_decl_nodiscard static bool mi_page_map_ensure_committed(mi_page_map_t* pmap, size_t idx, mi_submap_t* submap) {
mi_assert_internal(submap!=NULL && *submap==NULL);
- size_t bit_idx;
- if mi_unlikely(!mi_page_map_is_committed(idx, &bit_idx)) {
- uint8_t* start = (uint8_t*)&_mi_page_map[bit_idx * MI_PAGE_MAP_ENTRIES_PER_CBIT];
- if (!_mi_os_commit(start, MI_PAGE_MAP_ENTRIES_PER_CBIT * sizeof(mi_submap_t), NULL)) {
- mi_page_map_cannot_commit();
- return false;
+ if mi_unlikely(idx >= mi_atomic_load_relaxed(&pmap->committed_count)) {
+ if (idx >= mi_atomic_load_acquire(&pmap->committed_count)) {
+ if (!mi_page_map_commit_entries(pmap,idx)) return false;
+ mi_assert_internal(idx < mi_atomic_load_relaxed(&pmap->committed_count));
}
- mi_atomic_or_acq_rel(&mi_page_map_commit, MI_ZU(1) << bit_idx);
}
- *submap = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // acquire _mi_page_map_at(idx);
+ *submap = mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]);
return true;
}
-// initialize the page map
-bool _mi_page_map_init(void) {
- size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS);
+// initialize the page map
+static bool mi_page_map_init_once(void) {
+ size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_MAX_VABITS);
if (vbits == 0) {
vbits = _mi_os_virtual_address_bits();
#if MI_ARCH_X64 // canonical address is limited to the first 128 TiB
@@ -242,111 +280,145 @@ bool _mi_page_map_init(void) {
if (vbits < MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT) {
vbits = MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT;
}
+ if (vbits < MI_MIN_VABITS) { // cover at least this much for a faster _mi_checked_ptr
+ vbits = MI_MIN_VABITS;
+ }
+ if (vbits > MI_MAX_VABITS) { // limit page map size even if more virtual addresses are available
+ vbits = MI_MAX_VABITS;
+ }
// Allocate the page map and commit bits
mi_assert(MI_MAX_VABITS >= vbits);
- mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));
- mi_page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));
- mi_assert(mi_page_map_count <= MI_PAGE_MAP_COUNT);
- const size_t os_page_size = _mi_os_page_size();
- const size_t page_map_size = _mi_align_up( mi_page_map_count * sizeof(mi_page_t**), os_page_size);
- const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;
- const size_t reserve_size = page_map_size + submap_size;
- #if MI_SECURE
- const bool commit = true; // the whole page map is valid and we can reliably check any pointer
- #else
- const bool commit = page_map_size <= 64*MI_KiB ||
+ mi_assert(MI_MIN_VABITS <= vbits);
+ const size_t reserve_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));
+ const size_t os_page_size = _mi_os_page_size();
+ const size_t reserve_size = _mi_align_up( sizeof(mi_page_map_t) + ((reserve_count - 1) * sizeof(mi_submap_t)), os_page_size);
+ const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;
+ const size_t extra_reserve_size = reserve_size + submap_size;
+ const bool commit = (vbits == MI_MIN_VABITS) || (reserve_size <= 64*MI_KiB) || // 42 virtual address bits
mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit();
- #endif
- _mi_page_map = (_Atomic(mi_page_t**)*)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid);
- if (_mi_page_map==NULL) {
- _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB);
+ mi_subproc_t* const subproc = _mi_subproc_main();
+ mi_memid_t memid;
+ mi_page_map_t* const pmap = (mi_page_map_t*)_mi_os_alloc_aligned(subproc, extra_reserve_size, 1, commit, true /* allow large */, &memid);
+ if mi_unlikely(pmap==NULL) {
+ _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", extra_reserve_size / MI_KiB);
return false;
}
- if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) {
- _mi_warning_message("internal: the page map was committed but not zero initialized!\n");
- _mi_memzero_aligned(_mi_page_map, page_map_size);
+
+ // commit
+ size_t commit_count;
+ if (memid.initially_committed) {
+ if (!memid.initially_zero) {
+ _mi_warning_message("internal: the page map was committed but not zero initialized!\n");
+ _mi_memzero_aligned(pmap, extra_reserve_size);
+ memid.initially_zero = true;
+ }
+ commit_count = mi_page_map_count_of_size(reserve_size);
}
- mi_atomic_store_release(&mi_page_map_commit, (mi_page_map_memid.initially_committed ? ~MI_ZU(0) : MI_ZU(0)));
-
+ else {
+ // commit first entries up to MI_MIN_VABITS entries
+ const size_t min_commit_count = (MI_ZU(1) << (MI_MIN_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));
+ const size_t min_commit_size = _mi_align_up( sizeof(mi_page_map_t) + ((min_commit_count-1) * sizeof(mi_submap_t)), os_page_size);
+ mi_assert_internal(min_commit_size <= reserve_size);
+ bool is_zero;
+ if (!_mi_os_commit(subproc,pmap,min_commit_size,&is_zero)) {
+ mi_page_map_cannot_commit();
+ _mi_os_free(subproc,pmap,extra_reserve_size,memid);
+ return false;
+ };
+ mi_assert_internal(is_zero || memid.initially_zero);
+ commit_count = mi_page_map_count_of_size(min_commit_size);
+ mi_assert_internal(commit_count >= min_commit_count);
+ }
+
// ensure there is a submap for the NULL address
- mi_page_t** const sub0 = (mi_page_t**)((uint8_t*)_mi_page_map + page_map_size); // we reserved a submap part at the end already
- if (!mi_page_map_memid.initially_committed) {
- if (!_mi_os_commit(sub0, submap_size, NULL)) { // commit full submap (issue #1087)
+ mi_page_t** const sub0 = (mi_submap_t)((uint8_t*)pmap + reserve_size); // we reserved a submap part at the end already
+ if (!memid.initially_committed) {
+ if (!_mi_os_commit(subproc, sub0, submap_size, NULL)) { // commit full submap (issue #1087)
mi_page_map_cannot_commit();
+ _mi_os_free(subproc,pmap,extra_reserve_size,memid);
return false;
}
}
- if (!mi_page_map_memid.initially_zero) { // initialize low addresses with NULL
+ if (!memid.initially_zero) { // initialize low addresses with NULL
_mi_memzero_aligned(sub0, submap_size);
}
- mi_submap_t nullsub = NULL;
- if (!mi_page_map_ensure_committed(0,&nullsub)) {
- mi_page_map_cannot_commit();
- return false;
- }
- mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[0], sub0);
- mi_lock_init(&mi_page_map_lock); // initialize late in case the lock init causes allocation
-
+
+ // initialize the fields
+ pmap->memid = memid;
+ pmap->reserved_size = reserve_size;
+ mi_lock_init(&pmap->lock);
+ mi_atomic_store_release(&pmap->committed_count, commit_count);
+ // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count));
+ mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[0], sub0);
+ mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, pmap);
mi_assert_internal(_mi_ptr_page(NULL)==NULL);
return true;
}
+bool _mi_page_map_init(void) {
+ bool ok = true;
+ mi_atomic_do_once {
+ ok = mi_page_map_init_once();
+ }
+ return ok;
+}
-void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) {
- mi_assert_internal(subproc != NULL);
- mi_assert_internal(_mi_page_map != NULL);
- if (_mi_page_map == NULL) return;
- mi_lock_done(&mi_page_map_lock);
- for (size_t idx = 1; idx < mi_page_map_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map)
- // free all sub-maps
- if (mi_page_map_is_committed(idx, NULL)) {
- mi_submap_t sub = _mi_page_map_at(idx);
- if (sub != NULL) {
- mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false);
- _mi_os_free_ex(memid.mem.os.base, memid.mem.os.size, true, memid, subproc);
- mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[idx], NULL);
- }
+void _mi_page_map_unsafe_destroy(void) {
+ mi_page_map_t* const pmap = _mi_page_map();
+ mi_assert_internal(pmap != NULL);
+ if (pmap == NULL || pmap == &mi_page_map_empty) return;
+ mi_subproc_t* const subproc = _mi_subproc_main();
+ mi_lock_done(&pmap->lock);
+ for (size_t idx = 1; idx < pmap->committed_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map)
+ // free all sub-maps
+ mi_submap_t sub = _mi_page_map_at(pmap,idx);
+ if (sub != NULL) {
+ mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false);
+ _mi_os_free_ex(subproc, memid.mem.os.base, memid.mem.os.size, true, memid);
+ mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[idx], NULL);
}
}
- _mi_os_free_ex(_mi_page_map, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc);
- _mi_page_map = NULL;
- mi_page_map_count = 0;
- mi_page_map_memid = _mi_memid_none();
- mi_page_map_max_address = NULL;
- mi_atomic_store_release(&mi_page_map_commit, (mi_bfield_t)0);
+ _mi_os_free_ex(subproc, pmap, pmap->reserved_size, true, pmap->memid);
+ mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, &mi_page_map_empty);
}
-
-mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(size_t idx, mi_submap_t* submap) {
- mi_assert_internal(submap!=NULL && *submap==NULL);
+mi_decl_nodiscard static mi_decl_noinline mi_submap_t mi_page_map_alloc_submap_at(mi_page_map_t* pmap, size_t idx) {
+ // sub map not yet allocated, alloc now
mi_submap_t sub = NULL;
- if (!mi_page_map_ensure_committed(idx, &sub)) {
- return false;
- }
- if mi_unlikely(sub == NULL) {
- // sub map not yet allocated, alloc now
- mi_lock(&mi_page_map_lock)
+ mi_lock(&pmap->lock)
+ {
+ sub = mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); // reload
+ if (sub==NULL) // not yet allocated by another thread?
{
- sub = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // reload
- if (sub==NULL) // not yet allocated by another thread?
- {
- mi_memid_t memid;
- const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;
- sub = (mi_submap_t)_mi_os_zalloc(submap_size, &memid);
- if (sub==NULL) {
- _mi_warning_message("internal error: unable to extend the page map\n");
- }
- else {
- mi_submap_t expect = NULL;
- if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &_mi_page_map[idx], &expect, sub)) {
- // another thread already allocated it.. free and continue
- _mi_os_free(sub, submap_size, memid);
- sub = expect;
- }
+ mi_subproc_t* const subproc = _mi_subproc_main();
+ mi_memid_t memid;
+ const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;
+ sub = (mi_submap_t)_mi_os_zalloc(subproc, submap_size, &memid);
+ if (sub==NULL) {
+ _mi_warning_message("internal error: unable to extend the page map\n");
+ }
+ else {
+ mi_submap_t expect = NULL;
+ if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &pmap->submaps[idx], &expect, sub)) {
+ // another thread already allocated it.. free and continue
+ _mi_os_free(subproc, sub, submap_size, memid);
+ sub = expect;
}
}
}
+ }
+ return sub;
+}
+
+mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(mi_page_map_t* pmap, size_t idx, mi_submap_t* submap) {
+ mi_assert_internal(submap!=NULL && *submap==NULL);
+ mi_submap_t sub = NULL;
+ if (!mi_page_map_ensure_committed(pmap, idx, &sub)) {
+ return false;
+ }
+ if mi_unlikely(sub==NULL) {
+ sub = mi_page_map_alloc_submap_at(pmap, idx);
if (sub==NULL) return false; // unable to allocate the submap..
}
mi_assert_internal(sub!=NULL);
@@ -354,11 +426,11 @@ mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(size_t idx, mi_submap
return true;
}
-static bool mi_page_map_set_range_prim(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {
+static bool mi_page_map_set_range_prim(mi_page_map_t* pmap, mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {
// is the page map area that contains the page address committed?
while (slice_count > 0) {
mi_submap_t sub = NULL;
- if (!mi_page_map_ensure_submap_at(idx, &sub)) {
+ if (!mi_page_map_ensure_submap_at(pmap, idx, &sub)) {
return false;
};
mi_assert_internal(sub!=NULL);
@@ -374,11 +446,11 @@ static bool mi_page_map_set_range_prim(mi_page_t* page, size_t idx, size_t sub_i
return true;
}
-static bool mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {
- if mi_unlikely(!mi_page_map_set_range_prim(page,idx,sub_idx,slice_count)) {
+static bool mi_page_map_set_range(mi_page_map_t* pmap, mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {
+ if mi_unlikely(!mi_page_map_set_range_prim(pmap, page,idx,sub_idx,slice_count)) {
// failed to commit, call again to reset the page pointer if needed
if (page!=NULL) {
- mi_page_map_set_range_prim(NULL,idx,sub_idx,slice_count);
+ mi_page_map_set_range_prim(pmap,NULL,idx,sub_idx,slice_count);
}
return false;
}
@@ -396,48 +468,46 @@ static size_t mi_page_map_get_idx(mi_page_t* page, size_t* sub_idx, size_t* slic
bool _mi_page_map_register(mi_page_t* page) {
mi_assert_internal(page != NULL);
mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
- mi_assert_internal(_mi_page_map != NULL); // should be initialized before multi-thread access!
- if mi_unlikely(_mi_page_map == NULL) {
+ mi_page_map_t* pmap = _mi_page_map();
+ mi_assert_internal(pmap != NULL); // should be initialized before multi-thread access!
+ if mi_unlikely(pmap == NULL) {
if (!_mi_page_map_init()) return false;
+ pmap = mi_atomic_load_ptr_acquire(mi_page_map_t,&__mi_page_map);
}
- mi_assert(_mi_page_map!=NULL);
+ mi_assert(pmap!=NULL);
size_t slice_count;
size_t sub_idx;
const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);
- return mi_page_map_set_range(page, idx, sub_idx, slice_count);
+ return mi_page_map_set_range(pmap, page, idx, sub_idx, slice_count);
}
void _mi_page_map_unregister(mi_page_t* page) {
mi_assert_internal(_mi_page_map != NULL);
mi_assert_internal(page != NULL);
mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
- if mi_unlikely(_mi_page_map == NULL) return;
+ mi_page_map_t* const pmap = _mi_page_map();
+ // note: should proceed even if the page was not registered yet (for failure paths in page allocation in `arena.c`)
+ if mi_unlikely(pmap == NULL) return;
// get index and count
size_t slice_count;
size_t sub_idx;
const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);
// unset the offsets
- mi_page_map_set_range(NULL, idx, sub_idx, slice_count);
+ mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count);
}
void _mi_page_map_unregister_range(void* start, size_t size) {
- if mi_unlikely(_mi_page_map == NULL) return;
+ mi_page_map_t* const pmap = _mi_page_map();
+ if mi_unlikely(pmap == NULL) return;
const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE);
size_t sub_idx;
const uintptr_t idx = _mi_page_map_index(start, &sub_idx);
- mi_page_map_set_range(NULL, idx, sub_idx, slice_count); // todo: avoid committing if not already committed?
+ mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count); // todo: avoid committing if not already committed?
}
// Return NULL for invalid pointers
mi_page_t* _mi_safe_ptr_page(const void* p) {
- if (p==NULL) return NULL;
- if mi_unlikely(p >= mi_page_map_max_address) return NULL;
- size_t sub_idx;
- const size_t idx = _mi_page_map_index(p,&sub_idx);
- if mi_unlikely(!mi_page_map_is_committed(idx,NULL)) return NULL;
- mi_page_t** const sub = _mi_page_map[idx];
- if mi_unlikely(sub==NULL) return NULL;
- return sub[sub_idx];
+ return _mi_checked_ptr_page(p);
}
mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
diff --git a/vendored/mimalloc/src/page-queue.c b/vendored/mimalloc/src/page-queue.c
index 731672b02..c8e9b5425 100644
--- a/vendored/mimalloc/src/page-queue.c
+++ b/vendored/mimalloc/src/page-queue.c
@@ -21,11 +21,11 @@ terms of the MIT license. A copy of the license can be found in the file
Minimal alignment in machine words (i.e. `sizeof(void*)`)
----------------------------------------------------------- */
-#if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE)
+#if (MI_MAX_ALIGN_SIZE > 4*MI_SIZE_SIZE)
#error "define alignment for more than 4x word size for this platform"
-#elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE)
+#elif (MI_MAX_ALIGN_SIZE > 2*MI_SIZE_SIZE)
#define MI_ALIGN4W // 4 machine words minimal alignment
-#elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE)
+#elif (MI_MAX_ALIGN_SIZE > MI_SIZE_SIZE)
#define MI_ALIGN2W // 2 machine words minimal alignment
#else
// ok, default alignment is 1 word
@@ -61,8 +61,8 @@ static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) {
// Returns MI_BIN_HUGE if the size is too large.
// We use `wsize` for the size in "machine word sizes",
// i.e. byte size == `wsize*sizeof(void*)`.
-static mi_decl_noinline size_t mi_bin(size_t size) {
- size_t wsize = _mi_wsize_from_size(size);
+static size_t mi_bin(size_t size) {
+ size_t wsize = mi_wsize_from_size(size);
#if defined(MI_ALIGN4W)
if mi_likely(wsize <= 4) {
return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
@@ -112,10 +112,10 @@ size_t _mi_bin_size(size_t bin) {
// Good size for allocation
mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept {
- if (size <= MI_LARGE_MAX_OBJ_SIZE) {
+ if (size <= MI_LARGE_MAX_OBJ_SIZE - MI_PADDING_SIZE) {
return _mi_bin_size(mi_bin(size + MI_PADDING_SIZE));
}
- else if (size <= MI_MAX_ALLOC_SIZE) {
+ else if (size <= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) {
return _mi_align_up(size + MI_PADDING_SIZE,_mi_os_page_size());
}
else {
@@ -152,10 +152,10 @@ bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq) {
for (mi_page_t* page = pq->first; page != NULL; page = page->next) {
mi_assert_internal(page->prev == prev);
if (mi_page_is_in_full(page)) {
- mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2);
+ mi_assert_internal(mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2);
}
else if (mi_page_is_huge(page)) {
- mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1);
+ mi_assert_internal(mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1);
}
else {
mi_assert_internal(mi_page_block_size(page) == pq->block_size);
@@ -212,10 +212,10 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_
if (size > MI_SMALL_SIZE_MAX) return;
mi_page_t* page = pq->first;
- if (pq->first == NULL) page = (mi_page_t*)&_mi_page_empty;
+ if (pq->first == NULL) page = _mi_page_empty_get();
// find index in the right direct page array
- const size_t idx = _mi_wsize_from_size(size);
+ const size_t idx = mi_wsize_from_size(size);
mi_page_t** const pages_free = theap->pages_free_direct;
if (pages_free[idx] == page) return; // already set
@@ -232,7 +232,7 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_
while( bin == mi_bin(prev->block_size) && prev > &theap->pages[0]) {
prev--;
}
- start = 1 + _mi_wsize_from_size(prev->block_size);
+ start = 1 + mi_wsize_from_size(prev->block_size);
if (start > idx) start = idx;
}
@@ -341,6 +341,15 @@ static void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queu
mi_assert_internal(queue->first == page);
}
+static void mi_page_queue_move_to_back(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) {
+ mi_assert_internal(mi_page_theap(page) == theap);
+ mi_assert_internal(mi_page_queue_contains(queue, page));
+ if (queue->last == page) return;
+ mi_page_queue_remove(queue, page);
+ mi_page_queue_push_at_end(theap, queue, page);
+ mi_assert_internal(queue->last == page);
+}
+
static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) {
mi_assert_internal(page != NULL);
mi_assert_internal(from->count >= 1);
@@ -421,38 +430,3 @@ static void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t
// note: we could insert at the front to increase reuse, but it slows down certain benchmarks (like `alloc-test`)
mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end of the `to` queue? */, page);
}
-
-// Only called from `mi_theap_absorb`.
-size_t _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append) {
- mi_assert_internal(mi_theap_contains_queue(theap,pq));
- mi_assert_internal(pq->block_size == append->block_size);
-
- if (append->first==NULL) return 0;
-
- // set append pages to new theap and count
- size_t count = 0;
- for (mi_page_t* page = append->first; page != NULL; page = page->next) {
- mi_page_set_theap(page, theap);
- count++;
- }
- mi_assert_internal(count == append->count);
-
- if (pq->last==NULL) {
- // take over afresh
- mi_assert_internal(pq->first==NULL);
- pq->first = append->first;
- pq->last = append->last;
- mi_theap_queue_first_update(theap, pq);
- }
- else {
- // append to end
- mi_assert_internal(pq->last!=NULL);
- mi_assert_internal(append->first!=NULL);
- pq->last->next = append->first;
- append->first->prev = pq->last;
- pq->last = append->last;
- }
- pq->count += append->count;
-
- return count;
-}
diff --git a/vendored/mimalloc/src/page.c b/vendored/mimalloc/src/page.c
index 97d2fc279..aa669cdd0 100644
--- a/vendored/mimalloc/src/page.c
+++ b/vendored/mimalloc/src/page.c
@@ -15,6 +15,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc/internal.h"
#include "mimalloc/atomic.h"
#include "mimalloc/prim.h"
+#include "mimalloc/prim-tls.h"
/* -----------------------------------------------------------
Definition of page queues for each block size
@@ -41,7 +42,7 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page);
#if (MI_DEBUG>=3)
static size_t mi_page_list_count(mi_page_t* page, mi_block_t* head) {
- mi_assert_internal(_mi_ptr_page(page->page_start) == page);
+ mi_assert_internal(_mi_ptr_page(mi_page_start(page)) == page);
const uint8_t* slice_start = mi_page_slice_start(page);
mi_assert_internal(_mi_is_aligned(slice_start,MI_PAGE_ALIGN));
size_t count = 0;
@@ -82,8 +83,12 @@ static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) {
static bool mi_page_is_valid_init(mi_page_t* page) {
mi_assert_internal(mi_page_block_size(page) > 0);
- mi_assert_internal(page->used <= page->capacity);
+ mi_assert_internal(mi_page_used(page) <= page->capacity);
mi_assert_internal(page->capacity <= page->reserved);
+
+ mi_assert_internal(page->heap!=NULL);
+ mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap);
+ mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED);
// const size_t bsize = mi_page_block_size(page);
// uint8_t* start = mi_page_start(page);
@@ -99,7 +104,7 @@ static bool mi_page_is_valid_init(mi_page_t* page) {
mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t)));
}
}
- #endif
+ #endif
#if !MI_TRACK_ENABLED && !MI_TSAN
mi_block_t* tfree = mi_page_thread_free(page);
@@ -109,8 +114,9 @@ static bool mi_page_is_valid_init(mi_page_t* page) {
#endif
size_t free_count = mi_page_list_count(page, page->free) + mi_page_list_count(page, page->local_free);
- mi_assert_internal(page->used + free_count == page->capacity);
+ mi_assert_internal(mi_page_used(page) + free_count == page->capacity);
+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));
return true;
}
@@ -123,6 +129,9 @@ bool _mi_page_is_valid(mi_page_t* page) {
#endif
if (!mi_page_is_abandoned(page)) {
//mi_assert_internal(!_mi_process_is_initialized);
+ mi_assert_internal(page->heap!=NULL);
+ mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap);
+ mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED);
{
mi_page_queue_t* pq = mi_page_queue_of(page);
mi_assert_internal(mi_page_queue_contains(pq, page));
@@ -130,10 +139,180 @@ bool _mi_page_is_valid(mi_page_t* page) {
// mi_assert_internal(mi_theap_contains_queue(mi_page_theap(page),pq));
}
}
+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));
+ mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page));
return true;
}
#endif
+#if MI_STATS || MI_SAMPLE
+// Gets the theap belonging to a page.
+static mi_theap_t* mi_theap_of_page(mi_page_t* page) {
+ mi_theap_t* theap = page->theap;
+ if mi_unlikely(mi_page_thread_id(page) != _mi_prim_thread_id()) {
+ theap = _mi_page_associated_theap_peek(page);
+ }
+ return theap;
+}
+#endif
+
+#if MI_SAMPLE==1 /* for ==2, the countdown is already done at every `alloc.c:mi_page_alloc_zero` */
+static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, size_t alloc_count)
+{
+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));
+ mi_assert_internal(theap!=NULL);
+ if (theap->sample_rate==0) return;
+
+ const size_t last_alloc = mi_page_last_alloc(page);
+ if (alloc_count <= last_alloc) return;
+
+ // update countdown
+ const size_t bsize = mi_page_usable_block_size(page); mi_assert_internal(bsize >= MI_PADDING_SIZE);
+ const size_t alloc_diff = alloc_count - last_alloc;
+ const uint64_t requested = (uint64_t)alloc_diff * (uint64_t)(bsize - MI_PADDING_SIZE);
+ if (requested <= SIZE_MAX && theap->sample_countdown >= (size_t)requested) {
+ theap->sample_countdown -= (size_t)requested;
+ }
+ else {
+ theap->sample_requested += (requested - theap->sample_countdown);
+ theap->sample_countdown = 0;
+ }
+}
+#endif
+
+#if MI_STATS
+// Merge stats from the page into the corresponding theap or heap.
+static void mi_theap_page_merge_stats(mi_theap_t* theap, const mi_page_t* page, size_t alloc_count, size_t free_count ) {
+ // get heap (as the theap might be NULL)
+ mi_heap_t* const heap = mi_page_heap(page);
+ mi_assert_internal(theap == NULL || (theap->tld != NULL && _mi_thread_id() == theap->tld->thread_id));
+ mi_theapx_stat_counter_increase(heap,theap,pages_stat_updates,1);
+ mi_theapx_stat_counter_increase(heap,theap,pages_stat_update_count, alloc_count + free_count);
+
+ // allocation sizes
+ const size_t bsize = mi_page_usable_block_size(page);
+ const uint64_t allocated = (uint64_t)alloc_count * (uint64_t)bsize;
+ const uint64_t freed = (uint64_t)free_count * (uint64_t)bsize;
+ #if MI_STATS==1
+ const uint64_t requested = allocated - ((uint64_t)alloc_count * MI_PADDING_SIZE);
+ #endif
+ mi_assert_internal(allocated <= INT64_MAX); // safe to cast to int64_t for stats
+ mi_assert_internal(freed <= INT64_MAX);
+
+ // adjust stats
+ if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
+ const size_t bin = _mi_bin(bsize);
+ // allocations
+ if (alloc_count > 0) {
+ mi_theapx_stat_counter_increase(heap, theap, malloc_normal_count, alloc_count);
+ mi_theapx_stat_increase(heap, theap, malloc_normal, allocated);
+ mi_theapx_stat_increase(heap, theap, malloc_bins[bin], alloc_count);
+ #if MI_STATS==1
+ // use coarse total requested bytes
+ mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested);
+ #endif
+ }
+ // frees
+ if (free_count > 0) {
+ mi_theapx_stat_decrease(heap, theap, malloc_normal, freed);
+ mi_theapx_stat_decrease(heap, theap, malloc_bins[bin], free_count);
+ }
+ }
+ else {
+ // allocations
+ mi_assert_internal(alloc_count<=1);
+ mi_assert_internal(free_count<=1);
+ if (alloc_count > 0) {
+ mi_theapx_stat_counter_increase(heap, theap, malloc_huge_count, alloc_count);
+ mi_theapx_stat_increase(heap, theap, malloc_huge, allocated);
+ #if MI_STATS==1
+ // use coarse total requested bytes
+ mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested);
+ #endif
+ }
+ // frees
+ if (free_count > 0) {
+ mi_theapx_stat_decrease(heap, theap, malloc_huge, freed);
+ }
+ }
+}
+
+// Update stats for a page
+static void mi_theap_page_update_stats(mi_theap_t* theap, mi_page_t* page) {
+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));
+ mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page));
+
+ // get stat counts
+ const size_t used = mi_page_used(page);
+ const size_t alloc_count = mi_page_alloc_count(page);
+ const size_t last_used = mi_page_last_used(page);
+ mi_assert_internal(last_used + alloc_count >= used);
+ mi_assert_internal(used <= UINT16_MAX);
+ const size_t free_count = last_used + alloc_count - used;
+
+ #if MI_SAMPLE==1 // for ==2 it is already counted in every `alloc.c:mi_page_alloc_zero`
+ if (theap!=NULL) { mi_theap_adjust_sample_countdown(theap,page,alloc_count); }
+ #endif
+
+ // update stats?
+ if (alloc_count + free_count == 0) {
+ mi_assert_internal(last_used == used);
+ return;
+ }
+
+ // reset the `alloc_count` (and `last_alloc`), and set `last_used` to `used`
+ #if MI_SIZE_SIZE >= 8
+ page->xused.used_alloc = (used << 32) | used;
+ #else
+ page->xused.used_alloc = used;
+ page->xlast_used = (uint16_t)used;
+ page->xlast_alloc = 0;
+ #endif
+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));
+ mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page));
+
+ mi_theap_page_merge_stats(theap, page, alloc_count, free_count);
+}
+
+void _mi_page_update_stats(mi_page_t* page) { // called on abandoned pages etc.
+ mi_theap_page_update_stats(mi_theap_of_page(page),page);
+}
+
+#else
+void _mi_page_update_stats(mi_page_t* page) {
+ MI_UNUSED(page);
+}
+#endif
+
+static void mi_page_update_sample_countdown(mi_page_t* page)
+{
+ // adjust count down
+ const size_t alloc_count = mi_page_alloc_count(page);
+ if (alloc_count==0) {
+ return;
+ }
+ else if mi_unlikely(alloc_count>=0x8000) { // if the count could overflow, update stats so the counter is reset
+ _mi_page_update_stats(page);
+ }
+ #if MI_SAMPLE==1 // if ==2 the countdown is already always counted in `alloc.c:mi_page_alloc_zero_ex`
+ else {
+ mi_theap_t* theap = mi_theap_of_page(page);
+ if (theap==NULL) return;
+ mi_theap_adjust_sample_countdown(theap,page,alloc_count);
+ // update last_alloc to alloc_count
+ mi_assert_internal(alloc_count <= UINT16_MAX);
+ #if MI_SIZE_SIZE >= 8
+ page->xused.used_alloc = (alloc_count << 48) | (page->xused.used_alloc & (~MI_ZU(0) >> 16));
+ #else
+ page->xlast_alloc = (uint16_t)alloc_count;
+ #endif
+ mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));
+ mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page));
+
+ }
+ #endif
+}
+
/* -----------------------------------------------------------
Page collect the `local_free` and `thread_free` lists
@@ -155,11 +334,11 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head)
// if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free)
if mi_unlikely(count > max_count) {
- _mi_error_message(EFAULT, "corrupted thread-free list\n");
+ _mi_error_message(EFAULT, "corrupted thread-free list (possibly due to a cross-thread double free)\n");
return; // the thread-free items cannot be freed
}
// if `count > page->used` there was another kind memory corruption (either in the page meta-data or in the linked list)
- else if mi_unlikely(count > page->used) {
+ else if mi_unlikely(count > mi_page_used(page)) {
_mi_error_message(EFAULT, "corrupted meta-data in thread-free list\n");
return; // the thread-free items cannot be freed
}
@@ -170,8 +349,8 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head)
// update counts now
mi_assert_internal(count <= UINT16_MAX);
- mi_assert_internal(page->used >= (uint16_t)count);
- page->used = page->used - (uint16_t)count;
+ mi_assert_internal(mi_page_used(page) >= count);
+ page->xused.used_alloc -= count; // page->used = page->used - (uint16_t)count;
}
// Collect the local `thread_free` list using an atomic exchange.
@@ -192,14 +371,16 @@ static void mi_page_thread_free_collect(mi_page_t* page)
mi_page_thread_collect_to_local(page, head);
}
+
// returns `true` if after collection `mi_page_immediate_available` is true.
-static bool mi_page_free_quick_collect(mi_page_t* page) {
- if (page->free != NULL) return true;
+static inline bool mi_page_free_quick_collect(mi_page_t* page) {
+ if mi_likely(page->free != NULL) return true;
if (page->local_free == NULL) return false;
// move local_free to free
page->free = page->local_free;
page->local_free = NULL;
- page->free_is_zero = false;
+ page->free_is_zero = false;
+ mi_page_update_sample_countdown(page);
return true;
}
@@ -229,19 +410,20 @@ void _mi_page_free_collect(mi_page_t* page, bool force) {
page->local_free = NULL;
page->free_is_zero = false;
}
- }
-
+ mi_page_update_sample_countdown(page);
+ }
mi_assert_internal(!force || page->local_free == NULL);
}
// Collect elements in the thread-free list starting at `head`. This is an optimized
// version of `_mi_page_free_collect` to be used from `free.c:_mi_free_collect_mt` that avoids atomic access to `xthread_free`.
+// returns a possibly updated expected value for the thread_free pointer.
//
// `head` must be in the `xthread_free` list. It will not collect `head` itself
// so the `used` count is not fully updated in general. However, if the `head` is
// the last remaining element, it will be collected and the used count will become `0` (so `mi_page_all_free` becomes true).
-void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) {
- if (head == NULL) return;
+mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) {
+ if (head == NULL) return NULL;
mi_block_t* next = mi_block_next(page,head); // we cannot collect the head element itself as `page->thread_free` may point to it (and we want to avoid atomic ops)
if (next != NULL) {
mi_block_set_next(page, head, NULL);
@@ -250,13 +432,18 @@ void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) {
page->free = page->local_free;
page->local_free = NULL;
page->free_is_zero = false;
- }
+ mi_page_update_sample_countdown(page);
+ }
}
- if (page->used == 1) {
+ if (mi_page_used(page) == 1) {
// all elements are free'd since we skipped the `head` element itself
mi_assert_internal(mi_tf_block(mi_atomic_load_relaxed(&page->xthread_free)) == head);
mi_assert_internal(mi_block_next(page,head) == NULL);
_mi_page_free_collect(page, false); // collect the final element
+ return NULL;
+ }
+ else {
+ return head;
}
}
@@ -265,24 +452,6 @@ void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) {
Page fresh and retire
----------------------------------------------------------- */
-/*
-// called from segments when reclaiming abandoned pages
-void _mi_page_reclaim(mi_theap_t* theap, mi_page_t* page) {
- // mi_page_set_theap(page, theap);
- // _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after theap is set)
- _mi_page_free_collect(page, false); // ensure used count is up to date
-
- mi_assert_expensive(mi_page_is_valid_init(page));
- // mi_assert_internal(mi_page_theap(page) == theap);
- // mi_assert_internal(mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE);
-
- // TODO: push on full queue immediately if it is full?
- mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);
- mi_page_queue_push(theap, pq, page);
- mi_assert_expensive(_mi_page_is_valid(page));
-}
-*/
-
// called from `mi_free` on a reclaim, and fresh_alloc if we get an abandoned page
void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page)
{
@@ -293,6 +462,7 @@ void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page)
mi_page_set_theap(page,theap);
_mi_page_free_collect(page, false); // ensure used count is up to date
+
mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);
mi_page_queue_push_at_end(theap, pq, page);
mi_assert_expensive(_mi_page_is_valid(page));
@@ -309,7 +479,7 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
mi_page_set_theap(page, NULL);
page->theap = theap; // don't actually set theap to NULL so we can reclaim_on_free within the same theap
_mi_arenas_page_abandon(page, theap);
- _mi_arenas_collect(false, false, theap->tld); // allow purging
+ // _mi_arenas_collect(false, false, theap->tld); // allow purging
}
}
@@ -331,7 +501,9 @@ static mi_page_t* mi_page_fresh_alloc(mi_theap_t* theap, mi_page_queue_t* pq, si
if (!mi_page_immediate_available(page)) {
if (mi_page_is_expandable(page)) {
if (!mi_page_extend_free(theap, page)) {
- return NULL; // cannot commit
+ // cannot commit
+ _mi_page_abandon(page,pq);
+ return NULL;
};
}
else {
@@ -393,6 +565,7 @@ static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) {
// put full pages in a theap local queue (this is for theaps that cannot abandon, for example, if the theap can be destroyed)
mi_page_queue_enqueue_from(&mi_page_theap(page)->pages[MI_BIN_FULL], pq, page);
_mi_page_free_collect(page, false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set
+ _mi_page_update_stats(page); // we must update stats here as mi_theap_collect does not normally visit full pages
}
}
@@ -416,11 +589,11 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) {
mi_theap_t* theap = mi_page_theap(page); mi_assert_internal(theap!=NULL);
mi_page_set_theap(page,NULL);
_mi_arenas_page_free(page, theap);
- _mi_arenas_collect(false, false, theap->tld); // allow purging
+ // _mi_arenas_collect(false, false, theap->tld); // allow purging
}
-#define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX // should be less than size for MI_BIN_HUGE
-#define MI_RETIRE_CYCLES (16)
+#define MI_RETIRE_CYCLES (16) /* keep a retired page around for about 16 "admin cycles" before free'ing it */
+#define MI_RETIRE_MAX_PAGES (3) /* keep at most N pages per size bin as retired */
// Retire a page with no more used blocks
// Important to not retire too quickly though as new
@@ -446,12 +619,10 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept {
mi_page_queue_t* pq = mi_page_queue_of(page);
#if MI_RETIRE_CYCLES > 0
const size_t bsize = mi_page_block_size(page);
- if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue?
- if (pq->last==page && pq->first==page) { // the only page in the queue?
+ if mi_likely( pq->count <= MI_RETIRE_MAX_PAGES && !mi_page_queue_is_special(pq)) { // not full or huge queue?
+ if (pq->count==1 || bsize < MI_SMALL_SIZE_MAX) {
mi_theap_t* theap = mi_page_theap(page);
- #if MI_STAT>0
mi_theap_stat_counter_increase(theap, pages_retire, 1);
- #endif
page->retire_expire = (bsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4);
mi_assert_internal(pq >= theap->pages);
const size_t index = pq - theap->pages;
@@ -460,42 +631,13 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept {
if (index > theap->page_retired_max) theap->page_retired_max = index;
mi_assert_internal(mi_page_all_free(page));
return; // don't free after all
- }
+ }
}
#endif
_mi_page_free(page, pq);
}
-// free retired pages: we don't need to look at the entire queues
-// since we only retire pages that are at the head position in a queue.
-void _mi_theap_collect_retired(mi_theap_t* theap, bool force) {
- size_t min = MI_BIN_FULL;
- size_t max = 0;
- for(size_t bin = theap->page_retired_min; bin <= theap->page_retired_max; bin++) {
- mi_page_queue_t* pq = &theap->pages[bin];
- mi_page_t* page = pq->first;
- if (page != NULL && page->retire_expire != 0) {
- if (mi_page_all_free(page)) {
- page->retire_expire--;
- if (page->retire_expire == 0 || force) {
- _mi_page_free(page, pq);
- }
- else {
- // keep retired, update min/max
- if (bin < min) min = bin;
- if (bin > max) max = bin;
- }
- }
- else {
- page->retire_expire = 0;
- }
- }
- }
- theap->page_retired_min = min;
- theap->page_retired_max = max;
-}
-/*
static void mi_theap_collect_full_pages(mi_theap_t* theap) {
// note: normally full pages get immediately abandoned and the full queue is always empty
// this path is only used if abandoning is disabled due to a destroy-able theap or options
@@ -516,7 +658,47 @@ static void mi_theap_collect_full_pages(mi_theap_t* theap) {
page = next;
}
}
-*/
+
+static void mi_page_try_retire(mi_page_queue_t* pq, mi_page_t* page, size_t bin, bool force, size_t* min, size_t* max) {
+ mi_assert_internal(page!=NULL && page->retire_expire!=0);
+ if (mi_page_all_free(page)) {
+ page->retire_expire--;
+ if (page->retire_expire == 0 || force) {
+ _mi_page_free(page, pq);
+ }
+ else {
+ // keep retired, update min/max
+ if (bin < *min) *min = bin;
+ if (bin > *max) *max = bin;
+ }
+ }
+ else {
+ page->retire_expire = 0;
+ }
+}
+
+// free retired pages: we don't need to look at the entire queues
+// since we only retire pages that are at the head position in a queue.
+void _mi_theap_collect_retired(mi_theap_t* theap, bool force) {
+ size_t min = MI_BIN_FULL;
+ size_t max = 0;
+ for(size_t bin = theap->page_retired_min; bin <= theap->page_retired_max; bin++) {
+ mi_page_queue_t* pq = &theap->pages[bin];
+ mi_page_t* page = pq->first;
+ for (int i = 0; i<MI_RETIRE_MAX_PAGES && page!=NULL && page->retire_expire!=0; i++) {
+ mi_page_t* next = page->next;
+ mi_page_try_retire(pq,page,bin,force,&min,&max);
+ page = next;
+ }
+ }
+ theap->page_retired_min = min;
+ theap->page_retired_max = max;
+ if (!theap->allow_page_abandon) {
+ mi_theap_collect_full_pages(theap);
+ }
+}
+
+
/* -----------------------------------------------------------
@@ -530,7 +712,7 @@ static void mi_theap_collect_full_pages(mi_theap_t* theap) {
#define MI_MIN_SLICES (2)
static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* const page, const size_t bsize, const size_t extend) {
- #if (MI_SECURE<3)
+ #if (MI_SECURE < 2)
mi_assert_internal(page->free == NULL);
mi_assert_internal(page->local_free == NULL);
#endif
@@ -557,15 +739,15 @@ static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t*
// and initialize the free list by randomly threading through them
// set up first element
- const uintptr_t r = _mi_theap_random_next(theap);
+ const size_t r = _mi_theap_random_next(theap);
size_t current = r % slice_count;
counts[current]--;
mi_block_t* const free_start = blocks[current];
// and iterate through the rest; use `random_shuffle` for performance
- uintptr_t rnd = _mi_random_shuffle(r|1); // ensure not 0
+ size_t rnd = _mi_random_shuffle(r|1); // ensure not 0
for (size_t i = 1; i < extend; i++) {
- // call random_shuffle only every INTPTR_SIZE rounds
- const size_t round = i%MI_INTPTR_SIZE;
+ // call random_shuffle only every SIZE_SIZE rounds
+ const size_t round = i%MI_SIZE_SIZE;
if (round == 0) rnd = _mi_random_shuffle(rnd);
// select a random next slice index
size_t next = ((rnd >> 8*round) & (slice_count-1));
@@ -587,7 +769,7 @@ static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t*
static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend)
{
- #if (MI_SECURE<3)
+ #if (MI_SECURE < 2)
mi_assert_internal(page->free == NULL);
mi_assert_internal(page->local_free == NULL);
#endif
@@ -614,7 +796,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co
Page initialize and extend the capacity
----------------------------------------------------------- */
-#define MI_MAX_EXTEND_SIZE (4*1024) // heuristic, one OS page seems to work well.
+#define MI_MAX_EXTEND_SIZE (8*1024) // heuristic, one or two OS pages seems to work well.
#if (MI_SECURE>=2)
#define MI_MIN_EXTEND (8*MI_SECURE) // extend at least by this many
#else
@@ -628,7 +810,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co
// extra test in malloc? or cache effects?)
static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {
mi_assert_expensive(mi_page_is_valid_init(page));
- #if (MI_SECURE<3)
+ #if (MI_SECURE < 2)
mi_assert(page->free == NULL);
mi_assert(page->local_free == NULL);
if (page->free != NULL) return true;
@@ -638,10 +820,8 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {
size_t page_size;
//uint8_t* page_start =
mi_page_area(page, &page_size);
- #if MI_STAT>0
mi_theap_stat_counter_increase(theap, pages_extended, 1);
- #endif
-
+
// calculate the extend count
const size_t bsize = mi_page_block_size(page);
size_t extend = (size_t)page->reserved - page->capacity;
@@ -661,15 +841,30 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {
mi_assert_internal(extend < (1UL<<16));
// commit on demand?
- if (page->slice_committed > 0) {
+ const size_t slice_committed = mi_page_slice_committed(page);
+ if (slice_committed > 0) {
+ // reduce extend if it commits more than an arena slice
+ if ((extend * bsize) > MI_ARENA_SLICE_SIZE) {
+ extend = _mi_divide_up(MI_ARENA_SLICE_SIZE, bsize);
+ }
+ // commit required size
const size_t needed_size = (page->capacity + extend)*bsize;
- const size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), MI_PAGE_MIN_COMMIT_SIZE );
- if (needed_commit > page->slice_committed) {
- mi_assert_internal(((needed_commit - page->slice_committed) % _mi_os_page_size()) == 0);
- if (!_mi_os_commit(mi_page_slice_start(page) + page->slice_committed, needed_commit - page->slice_committed, NULL)) {
+ mi_assert_internal(needed_size <= page_size);
+ size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), mi_page_min_commit_size());
+ #if MI_SECURE>=5
+ // the previous alignup could extend the commit into the guard page; re-adjust if needed
+ const size_t page_size_commit = _mi_align_up( mi_page_slice_offset_of(page, page_size), _mi_os_page_size() );
+ if (needed_commit > page_size_commit) {
+ needed_commit = page_size_commit;
+ }
+ #endif
+ if (needed_commit > slice_committed) {
+ mi_assert_internal(((needed_commit - slice_committed) % _mi_os_page_size()) == 0);
+ if (!_mi_os_commit(_mi_theap_subproc(theap), mi_page_slice_start(page) + slice_committed, needed_commit - slice_committed, NULL)) {
return false;
}
- page->slice_committed = needed_commit;
+ mi_assert_internal(needed_commit <= UINT16_MAX * _mi_os_page_size());
+ page->slice_pcommitted = (uint16_t)(needed_commit / _mi_os_page_size());
}
}
@@ -682,9 +877,7 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {
}
// enable the new free list
page->capacity += (uint16_t)extend;
- #if MI_STAT>0
mi_theap_stat_increase(theap, page_committed, extend * bsize);
- #endif
mi_assert_expensive(mi_page_is_valid_init(page));
return true;
}
@@ -693,8 +886,8 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {
mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) {
mi_assert(page != NULL);
mi_assert(theap!=NULL);
- page->heap = (_mi_is_heap_main(_mi_theap_heap(theap)) ? NULL : _mi_theap_heap(theap)); // faster for `mi_page_associated_theap`
- mi_page_set_theap(page, theap);
+ // page->heap = (_mi_is_heap_main(_mi_theap_heap(theap)) ? NULL : _mi_theap_heap(theap)); // faster for `mi_page_associated_theap`
+ // mi_page_set_theap(page, theap);
size_t page_size;
uint8_t* page_start = mi_page_area(page, &page_size); MI_UNUSED(page_start);
@@ -703,20 +896,25 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) {
mi_assert_internal(page->reserved > 0);
#if (MI_PADDING || MI_ENCODE_FREELIST)
page->keys[0] = _mi_theap_random_next(theap);
+ #if MI_PAGE_KEY_COUNT==2
page->keys[1] = _mi_theap_random_next(theap);
#endif
+ #endif
#if MI_DEBUG>2
if (page->memid.initially_zero) {
- mi_track_mem_defined(page->page_start, mi_page_committed(page));
+ mi_track_mem_defined(mi_page_start(page), mi_page_committed(page));
mi_assert_expensive(mi_mem_is_zero(page_start, mi_page_committed(page)));
}
#endif
+ mi_assert_internal(page->heap != NULL);
+ mi_assert_internal(page->heap == _mi_theap_heap(theap));
mi_assert_internal(page->theap!=NULL);
mi_assert_internal(page->theap == mi_page_theap(page));
mi_assert_internal(page->capacity == 0);
mi_assert_internal(page->free == NULL);
- mi_assert_internal(page->used == 0);
+ mi_assert_internal(mi_page_used(page) == 0);
+ mi_assert_internal(page->xused.used_alloc == 0);
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(page->xthread_free == 1);
mi_assert_internal(page->next == NULL);
@@ -725,8 +923,10 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) {
mi_assert_internal(!mi_page_has_interior_pointers(page));
#if (MI_PADDING || MI_ENCODE_FREELIST)
mi_assert_internal(page->keys[0] != 0);
+ #if MI_PAGE_KEY_COUNT==2
mi_assert_internal(page->keys[1] != 0);
#endif
+ #endif
mi_assert_expensive(mi_page_is_valid_init(page));
// initialize an initial free list
@@ -749,8 +949,9 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap,
long page_full_retain = (pq->block_size > MI_SMALL_MAX_OBJ_SIZE ? 0 : theap->page_full_retain); // only retain small pages
mi_page_t* page_candidate = NULL; // a page with free space
mi_page_t* page = pq->first;
+ mi_page_t* const last = pq->last;
- while (page != NULL)
+ while (page!=NULL)
{
mi_page_t* next = page->next; // remember next (as this page can move to another queue)
count++;
@@ -774,6 +975,10 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap,
mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));
mi_page_to_full(page, pq);
}
+ else if (page!=last && pq->last!=page) {
+ // avoid revisiting this page for a while
+ mi_page_queue_move_to_back(theap, pq, page);
+ }
}
else {
// the page has free space, make it a candidate
@@ -787,7 +992,7 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap,
page_candidate = page;
}
// prefer to reuse fuller pages (in the hope the less used page gets freed)
- else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) {
+ else if (mi_page_used(page) >= mi_page_used(page_candidate) && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) {
page_candidate = page;
}
// if we find a non-expandable candidate, or searched for N pages, return with the best candidate
@@ -809,8 +1014,8 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap,
mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));
mi_page_to_full(page, pq);
#endif
-
- page = next;
+ if (page==last) { page=NULL; } // don't revisit earlier pages that moved to the back
+ else { page = next; }
} // for each page
mi_theap_stat_counter_increase(theap, page_searches, count);
@@ -832,7 +1037,7 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap,
if (page == NULL) {
_mi_theap_collect_retired(theap, false); // perhaps make a page available
- page = mi_page_fresh(theap, pq);
+ page = mi_page_fresh(theap, pq);
mi_assert_internal(page == NULL || mi_page_immediate_available(page));
if (page == NULL && first_try) {
// out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again
@@ -853,63 +1058,48 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap,
return page;
}
-
-
-// Find a page with free blocks of `size`.
-static mi_page_t* mi_find_free_page(mi_theap_t* theap, mi_page_queue_t* pq) {
- // mi_page_queue_t* pq = mi_page_queue(theap, size);
+// Look for a page with free blocks of `size` but don't try to search or allocate
+static inline mi_page_t* mi_page_queue_lookup_free_first(mi_theap_t* theap, mi_page_queue_t* pq) {
mi_assert_internal(!mi_page_queue_is_huge(pq));
-
- // check the first page: we even do this with candidate search or otherwise we re-search every time
mi_page_t* page = pq->first;
- if mi_likely(page != NULL && mi_page_free_quick_collect(page)) {
+ if mi_likely(page!=NULL && mi_page_free_quick_collect(page)) {
+ // fast path
#if (MI_SECURE>=2) // in secure mode, we extend half the time to increase randomness
if (page->capacity < page->reserved && ((_mi_theap_random_next(theap) & 1) == 1)) {
(void)mi_page_extend_free(theap, page); // ok if this fails
mi_assert_internal(mi_page_immediate_available(page));
}
+ #else
+ MI_UNUSED(theap);
#endif
page->retire_expire = 0;
- return page; // fast path
+ mi_assert_internal(mi_page_immediate_available(page));
+ mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+ mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+ return page;
}
else {
- return mi_page_queue_find_free_ex(theap, pq, true);
+ return NULL;
}
}
+// Find a page with free blocks of `size`.
+static inline mi_page_t* mi_page_queue_find_free(mi_theap_t* theap, mi_page_queue_t* pq) {
+ // mi_page_queue_t* pq = mi_page_queue(theap, size);
+ mi_assert_internal(!mi_page_queue_is_huge(pq));
-/* -----------------------------------------------------------
- Users can register a deferred free function called
- when the `free` list is empty. Since the `local_free`
- is separate this is deterministically called after
- a certain number of allocations.
------------------------------------------------------------ */
-
-// The program should only install a single deferred free handler before doing allocation.
-static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers)
-static _Atomic(void*) deferred_arg;
-
-void _mi_deferred_free(mi_theap_t* theap, bool force) {
- theap->heartbeat++;
- mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free);
- if (fun != NULL && !theap->tld->recurse) {
- theap->tld->recurse = true;
- void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg);
- fun(force, theap->heartbeat, arg);
- theap->tld->recurse = false;
- }
-}
-
-void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept {
- mi_atomic_store_ptr_release(void,&deferred_arg, arg);
- mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn);
+ // check the first page: we even do this with candidate search or otherwise we re-search every time
+ mi_page_t* page = mi_page_queue_lookup_free_first(theap,pq);
+ if (page==NULL) {
+ page = mi_page_queue_find_free_ex(theap, pq, true);
+ if (page==NULL) return NULL;
+ }
+ mi_assert_internal(mi_page_immediate_available(page));
+ mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+ mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+ return page;
}
-
-/* -----------------------------------------------------------
- General allocation
------------------------------------------------------------ */
-
// Huge pages contain just one block, and the segment contains just that page.
// Huge pages are also use if the requested alignment is very large (> MI_BLOCK_ALIGNMENT_MAX)
// so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`.
@@ -932,80 +1122,164 @@ static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page
mi_assert_internal(mi_page_is_abandoned(page));
mi_page_set_theap(page, NULL);
#endif
- mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page));
- mi_theap_stat_counter_increase(theap, malloc_huge_count, 1);
+ // mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page));
+ // mi_theap_stat_counter_increase(theap, malloc_huge_count, 1);
}
return page;
}
-
// Allocate a page
// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.
static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_alignment) mi_attr_noexcept {
const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`
- if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) {
+ if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) { // TODO: remove as we check in generic_fallback ?
_mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size);
return NULL;
}
mi_page_queue_t* pq = mi_page_queue(theap, (huge_alignment > 0 ? MI_LARGE_MAX_OBJ_SIZE+1 : size));
+ mi_page_t* page;
// huge allocation?
if mi_unlikely(mi_page_queue_is_huge(pq) || req_size > MI_MAX_ALLOC_SIZE) {
- return mi_huge_page_alloc(theap,size,huge_alignment,pq);
+ page = mi_huge_page_alloc(theap,size,huge_alignment,pq);
}
else {
// otherwise find a page with free blocks in our size segregated queues
#if MI_PADDING
mi_assert_internal(size >= MI_PADDING_SIZE);
#endif
- return mi_find_free_page(theap, pq);
+ page = mi_page_queue_find_free(theap,pq);
}
+ if (page==NULL) return NULL;
+ mi_assert_internal(mi_page_block_size(page) >= size);
+ mi_assert_internal(mi_page_immediate_available(page));
+ mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
+ mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
+ return page;
}
-// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed.
-// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.
-// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for
-// very large requested alignments in which case we use a huge singleton page.
-// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit)
-// to use 4 parameters which compiles better on msvc for the malloc fast path.
-void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable) mi_attr_noexcept
-{
- const bool zero = ((zero_huge_alignment & 1) != 0);
- const size_t huge_alignment = (zero_huge_alignment & ~1);
- #if !MI_THEAP_INITASNULL
- mi_assert_internal(theap != NULL);
- #endif
+/* -----------------------------------------------------------
+ Users can register a deferred free function called
+ when the `free` list is empty. Since the `local_free`
+ is separate this is deterministically called after
+ a certain number of allocations.
+----------------------------------------------------------- */
- // initialize if necessary
- if mi_unlikely(!mi_theap_is_initialized(theap)) {
- if (theap==&_mi_theap_empty_wrong) {
- // we were unable to allocate a theap for a first-class heap
- return NULL;
- }
- // otherwise we initialize the thread and its default theap
- mi_thread_init();
- theap = _mi_theap_default();
- if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; }
- mi_assert_internal(_mi_theap_default()==theap);
+// The program should only install a single deferred free handler before doing allocation.
+static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers)
+static _Atomic(void*) deferred_arg;
+
+void _mi_deferred_free(mi_theap_t* theap, bool force) {
+ theap->heartbeat++;
+ mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free);
+ if (fun != NULL && !theap->tld->recurse) {
+ theap->tld->recurse = true;
+ void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg);
+ fun(force, theap->heartbeat, arg);
+ theap->tld->recurse = false;
+ }
+}
+
+void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept {
+ mi_atomic_store_ptr_release(void,&deferred_arg, arg);
+ mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn);
+}
+
+
+/* -----------------------------------------------------------
+ Admin
+----------------------------------------------------------- */
+static mi_theap_t* mi_theap_init(mi_theap_t* theap) {
+ if mi_likely(mi_theap_is_initialized(theap)) return theap;
+ if (theap==&_mi_theap_empty_wrong) {
+ // we were unable to allocate a theap for a first-class heap
+ return NULL;
}
+ // otherwise we initialize the thread and its default theap
+ theap = _mi_thread_init();
+ if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; }
+ mi_assert_internal(mi_theap_is_initialized(theap));
+ return theap;
+}
+
+static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap)
+{
+ theap = mi_theap_init(theap);
+ if (theap==NULL) return NULL;
mi_assert_internal(mi_theap_is_initialized(theap));
// do administrative tasks every N generic mallocs
- if mi_unlikely(++theap->generic_count >= 1000) {
+ if mi_unlikely(theap->generic_count >= 1000) {
theap->generic_collect_count += theap->generic_count;
theap->generic_count = 0;
- // call potential deferred free routines
- _mi_deferred_free(theap, false);
- // free retired pages
- _mi_theap_collect_retired(theap, false);
- // collect every once in a while (10000 by default)
+ // check if the profiler is enabled
+ mi_heap_t* const heap = _mi_theap_heap(theap);
+ mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler);
+ const bool prof_enabled = (prof!=NULL && mi_profiler_is_enabled(prof));
+ if (prof_enabled && theap->profile_sample_rate==0) {
+ _mi_theap_set_profile_sample_rate(theap,mi_max(1,prof->initial_sample_rate)); // start profiling
+ }
+ else if (!prof_enabled && theap->profile_sample_rate!=0) {
+ _mi_theap_set_profile_sample_rate(theap,0); // stop profiling
+ }
+
+ // do a full theap collect every once in a while (10000 by default)
const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L);
if (theap->generic_collect_count >= generic_collect) {
theap->generic_collect_count = 0;
mi_theap_collect(theap, false /* force? */);
}
+ // else if (theap->sample_rate != 0) { // update stats more aggressively if we are sampling
+ // mi_theap_collect(theap, false /* force? */); // TODO: make specialized mi_theap_collect_update_stats ?
+ // }
+ else {
+ // otherwise we do a mini-collect
+ _mi_deferred_free(theap, false); // call potential deferred free routines
+ _mi_theap_collect_retired(theap, false); // free retired pages
+ }
+ }
+ return theap;
+}
+
+/* -----------------------------------------------------------
+ Generic allocation
+----------------------------------------------------------- */
+
+static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage)
+{
+ // initialize if necessary
+ theap = mi_malloc_generic_admin(theap);
+ if (theap==NULL) return NULL;
+
+ // check allocation size
+ const size_t req_size = size - MI_PADDING_SIZE;
+ if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) {
+ _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size);
+ return NULL;
+ }
+
+ // take a sample?
+ bool sample_countdown_is_adjusted = false;
+ if mi_unlikely(mi_theap_should_sample(theap,req_size)) {
+ if (huge_alignment==0 && theap->sample_rate!=0) {
+ return _mi_theap_malloc_sampled(theap,req_size,zero,ppage);
+ }
+ mi_assert_internal(!_mi_is_empty_theap(theap)); // cannot write to the empty theap
+ mi_assert_internal(req_size <= MI_SAMPLE_COUNTDOWN_MAX);
+ if (theap->sample_rate==0) {
+ theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // reset the countdown counter
+ }
+ #if MI_SAMPLE==2
+ else {
+ // huge_alignment!=0 so we need to adjust the countdown
+ mi_assert_internal(huge_alignment!=0);
+ mi_assert_internal(theap->sample_countdown <= SIZE_MAX - req_size);
+ sample_countdown_is_adjusted = true;
+ theap->sample_countdown += req_size; // adjust such that after `_mi_page_malloc_zero` the countdown is correct again
+ }
+ #endif
}
// find (or allocate) a page of the right size
@@ -1016,9 +1290,9 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm
}
if mi_unlikely(page == NULL) { // out of memory
- const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`
+ if (sample_countdown_is_adjusted) { theap->sample_countdown -= req_size; } // unadjust again if we failed to find a page
_mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size);
- return NULL;
+ return NULL;
}
mi_assert_internal(mi_page_immediate_available(page));
@@ -1026,14 +1300,72 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm
mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
- // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc)
- if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }
+ // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc_zero)
+ if (ppage!=NULL) { *ppage = page; }
void* const p = _mi_page_malloc_zero(theap,page,size,zero);
mi_assert_internal(p != NULL);
-
+ mi_page_update_sample_countdown(page);
+
// move full pages to the full queue
- if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE && mi_page_is_full(page)) {
- mi_page_to_full(page, mi_page_queue_of(page));
+ // this will also call _mi_page_update_stats for huge pages
+ if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE) {
+ if (mi_page_is_full(page)) {
+ mi_page_to_full(page, mi_page_queue_of(page));
+ }
+ }
+ return p;
+}
+
+
+// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed.
+// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.
+// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for
+// very large requested alignments in which case we use a huge singleton page.
+// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit)
+// to use 4 parameters which compiles better on msvc for the malloc fast path.
+void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept
+{
+ #if !MI_THEAP_INITASNULL
+ mi_assert_internal(theap != NULL);
+ #endif
+ const bool zero = ((zero_huge_alignment & 1) != 0);
+ const size_t huge_alignment = (zero_huge_alignment & ~1);
+ mi_page_t* page = NULL;
+
+ // fast path objects that fit in a small page
+ if mi_likely(mi_theap_is_initialized(theap) && ++theap->generic_count < 1000 && huge_alignment==0) {
+ const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`
+ if (req_size < MI_SMALL_MAX_OBJ_SIZE)
+ {
+ #if MI_SAMPLE
+ if mi_likely(!mi_theap_should_sample(theap,req_size)) // ensure we don't need to take a sample
+ #endif
+ {
+ mi_page_queue_t* pq = mi_page_queue(theap, size);
+ mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq));
+ page = mi_page_queue_find_free(theap,pq);
+ // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE);
+ if (page!=NULL) {
+ if (ppage!=NULL) { *ppage = page; }
+ mi_assert_internal(mi_page_immediate_available(page)); // we should never recurse in _mi_page_malloc_zero
+ return _mi_page_malloc_zero(theap,page,size,zero);
+ }
+ }
+ }
}
+ // otherwise fallback
+ return mi_malloc_generic_fallback(theap,size,zero,huge_alignment,ppage);
+}
+
+void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept {
+ theap = mi_theap_init(theap);
+ if (theap==NULL) return NULL;
+ const size_t sample_rate = theap->sample_rate;
+ const size_t sample_countdown = theap->sample_countdown;
+ theap->sample_rate = 0; // prevent a recursive call to mi_theap_malloc_sampled from _mi_malloc_generic
+ theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX;
+ void* p = _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage);
+ theap->sample_rate = sample_rate;
+ theap->sample_countdown = sample_countdown;
return p;
}
diff --git a/vendored/mimalloc/src/prim/emscripten/prim.c b/vendored/mimalloc/src/prim/emscripten/prim.c
index 1ba0936e0..992965d56 100644
--- a/vendored/mimalloc/src/prim/emscripten/prim.c
+++ b/vendored/mimalloc/src/prim/emscripten/prim.c
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen, Alon Zakai
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen, Alon Zakai
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -61,7 +61,6 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config) {
extern void emmalloc_free(void*);
int _mi_prim_free(void* addr, size_t size) {
- if (size==0) return 0;
emmalloc_free(addr);
return 0;
}
@@ -77,7 +76,7 @@ extern void* emmalloc_memalign(size_t alignment, size_t size);
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);
*is_large = false;
- // TODO: Track the highest address ever seen; first uses of it are zeroes.
+ // todo: Track the highest address ever seen; first uses of it are zeroes.
// That assumes no one else uses sbrk but us (they could go up,
// scribble, and then down), but we could assert on that perhaps.
*is_zero = false;
@@ -101,7 +100,7 @@ int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool comm
int _mi_prim_commit(void* addr, size_t size, bool* is_zero) {
MI_UNUSED(addr); MI_UNUSED(size);
- // See TODO above.
+ // See todo above.
*is_zero = false;
return 0;
}
@@ -155,7 +154,8 @@ size_t _mi_prim_numa_node_count(void) {
#include <emscripten/html5.h>
mi_msecs_t _mi_prim_clock_now(void) {
- return emscripten_date_now();
+ // todo: use a monotonic clock instead
+ return emscripten_date_now();
}
@@ -185,12 +185,12 @@ void _mi_prim_out_stderr( const char* msg) {
// Environment
//----------------------------------------------------------------
-bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
+int _mi_prim_getenv(const char* name, char* result, size_t result_size) {
// For code size reasons, do not support environ customization for now.
MI_UNUSED(name);
MI_UNUSED(result);
MI_UNUSED(result_size);
- return false;
+ return 0; // not found
}
@@ -212,7 +212,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {
// use pthread local storage keys to detect thread ending
// (and used with MI_TLS_PTHREADS for the default theap)
-pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);
+pthread_key_t _mi_heap_default_key = MI_PTHREAD_KEY_INVALID;
static void mi_pthread_done(void* value) {
if (value!=NULL) {
@@ -221,18 +221,20 @@ static void mi_pthread_done(void* value) {
}
void _mi_prim_thread_init_auto_done(void) {
- mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));
+ mi_assert_internal(_mi_heap_default_key == MI_PTHREAD_KEY_INVALID);
pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);
}
void _mi_prim_thread_done_auto_done(void) {
- if (_mi_heap_default_key != (pthread_key_t)(-1)) { // do not leak the key, see issue #809
- pthread_key_delete(_mi_heap_default_key);
+ pthread_key_t key = _mi_heap_default_key;
+ if (key != MI_PTHREAD_KEY_INVALID) { // do not leak the key, see issue #809
+ _mi_heap_default_key = MI_PTHREAD_KEY_INVALID;
+ pthread_key_delete(key);
}
}
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
- if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD
+ if (_mi_heap_default_key != MI_PTHREAD_KEY_INVALID) { // can happen during recursive invocation on freeBSD
pthread_setspecific(_mi_heap_default_key, theap);
}
}
diff --git a/vendored/mimalloc/src/prim/osx/alloc-override-zone.c b/vendored/mimalloc/src/prim/osx/alloc-override-zone.c
index fdbe095c2..e7fc8306b 100644
--- a/vendored/mimalloc/src/prim/osx/alloc-override-zone.c
+++ b/vendored/mimalloc/src/prim/osx/alloc-override-zone.c
@@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc.h"
#include "mimalloc/internal.h"
-
+#include "mimalloc/prim-tls.h" // _mi_thread_is_initialized
#if defined(MI_MALLOC_OVERRIDE)
#if !defined(__APPLE__)
@@ -71,11 +71,12 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) {
}
static void zone_free(malloc_zone_t* zone, void* p) {
- if (mi_any_heap_contains(p)) {
- mi_free(p); // with the page_map and pagemap_commit=1 we can use the regular free
- }
- else if (!is_mimalloc_zone(zone)) { // can happen due to interpose
- zone->free(zone,p);
+ // during C++ thread shutdown `_pthread_tsd_cleanup` may call `zone_free`
+ // after mimalloc mi_thread_done, and also on a pointer that was allocated in another subproc.
+ if mi_unlikely(!mi_cfree(p)) {
+ if (!is_mimalloc_zone(zone)) { // can happen due to interpose
+ zone->free(zone,p);
+ }
}
}
@@ -134,7 +135,6 @@ static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) {
return mi_is_in_heap_region(p);
}
-
/* ------------------------------------------------------
Introspection members
------------------------------------------------------ */
@@ -194,6 +194,14 @@ static boolean_t intro_zone_locked(malloc_zone_t* zone) {
return false;
}
+// Required whenever the zone advertises version >= 9: macOS calls this from the
+// atfork_child handler (_malloc_fork_child) without a NULL check. mimalloc keeps
+// no zone-level locks that need reinitializing after fork, so a no-op is safe.
+// Leaving it NULL makes the forked child jump to address 0 and crash while forking.
+static void intro_reinit_lock(malloc_zone_t* zone) {
+ MI_UNUSED(zone);
+}
+
/* ------------------------------------------------------
At process start, override the default allocator
@@ -218,6 +226,9 @@ static malloc_introspection_t mi_introspect = {
#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) && !defined(__ppc__)
.statistics = &intro_statistics,
.zone_locked = &intro_zone_locked,
+#endif
+#if defined(MAC_OS_X_VERSION_10_12) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_12) && !defined(__ppc__)
+ .reinit_lock = &intro_reinit_lock,
#endif
};
diff --git a/vendored/mimalloc/src/prim/prim-tls.c b/vendored/mimalloc/src/prim/prim-tls.c
new file mode 100644
index 000000000..1b79321a0
--- /dev/null
+++ b/vendored/mimalloc/src/prim/prim-tls.c
@@ -0,0 +1,252 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"
+#include "mimalloc/prim-tls.h"
+
+// --------------------------------------------------------------------------
+// Implement fast access to the thread local storage for `_mi_theap_default()`
+// and `mi_theap_cached()`. See `include/mimalloc/prim-tls.h` for more info
+// on the TLS models.
+// --------------------------------------------------------------------------
+
+void _mi_tls_slots_init(void);
+void _mi_tls_slots_done(void);
+void _mi_theap_default_set(mi_theap_t* theap);
+void _mi_theap_cached_set(mi_theap_t* theap);
+
+
+#if MI_TLS_MODEL_LOCAL
+// the thread-local main theap for allocation
+mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default = (mi_theap_t*)&_mi_theap_empty;
+// the last used non-main theap
+mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached = (mi_theap_t*)&_mi_theap_empty;
+#endif
+
+mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper = NULL;
+
+mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
+ const mi_threadid_t tid = _mi_prim_thread_id();
+ mi_assert_internal( (tid & MI_PAGE_FLAG_MASK) == 0 ); // mimalloc reserves the bottom 2 bits
+ return tid;
+}
+
+// ----------------------------------------------------------------------------
+// Setting the default and cached theap
+// ----------------------------------------------------------------------------
+
+#if MI_TLS_MODEL_WIN32
+
+// If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed)
+// See <https://en.wikipedia.org/wiki/Win32_Thread_Information_Block> for the offsets.
+#if MI_SIZE_SIZE==4
+#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_SIZE_SIZE)
+#else
+#define MI_TLS_DIRECT_FIRST (0x1480 / MI_SIZE_SIZE)
+#endif
+#define MI_TLS_DIRECT_SLOTS (64)
+#define MI_TLS_EXPANSION_SLOTS (1024)
+
+#if !MI_WIN_DIRECT_TLS
+// We initially use the last of the expansion slots as the default NULL.
+// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc
+// before we are initialized :-( (but this seems quite unlikely).
+// (todo: another approach could be to use slot 7 (EnvironmentPointer) as the initial slot as that seems to be always NULL)
+#define MI_TLS_INITIAL_SLOT MI_TLS_EXPANSION_SLOT
+#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1)
+#else
+// With direct tls we need an initial NULL slot outside the expansion slots
+#define MI_TLS_INITIAL_SLOT (5) // Arbitrary user pointer
+#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1) // unused
+#endif
+
+// in case of errors assign fixed slots (but since we use EFAULT the program should fail anyways)
+#define MI_TLS_ERROR_SLOT (5) // arbitrary user pointer
+#define MI_TLS_ERROR_EXPANSION_SLOT (7) // environment pointer (only used for OS/2 emulation)
+
+
+mi_decl_hidden mi_decl_cache_align _Atomic(size_t) _mi_theap_default_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_SLOT);
+mi_decl_hidden _Atomic(size_t) _mi_theap_default_expansion_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_EXPANSION_SLOT);
+mi_decl_hidden _Atomic(size_t) _mi_theap_cached_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_SLOT);
+mi_decl_hidden _Atomic(size_t) _mi_theap_cached_expansion_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_EXPANSION_SLOT);
+
+static DWORD mi_tls_raw_index_default = TLS_OUT_OF_INDEXES;
+static DWORD mi_tls_raw_index_cached = TLS_OUT_OF_INDEXES;
+
+static bool mi_win_tls_slot_alloc(_Atomic(size_t)* slot, _Atomic(size_t)* extended, DWORD* raw_index) {
+ // always write slot before extended due to concurrent readers
+ const DWORD index = TlsAlloc();
+ *raw_index = index;
+ if (index==TLS_OUT_OF_INDEXES) {
+ mi_atomic_store_release(slot,MI_TLS_ERROR_SLOT);
+ mi_atomic_store_release(extended,MI_TLS_ERROR_EXPANSION_SLOT);
+ return false;
+ }
+ else if (index<MI_TLS_DIRECT_SLOTS) {
+ mi_atomic_store_release(slot,index + MI_TLS_DIRECT_FIRST);
+ mi_atomic_store_release(extended,0);
+ return true;
+ }
+ #if !MI_WIN_DIRECT_TLS
+ else if (index < MI_TLS_DIRECT_SLOTS + MI_TLS_EXPANSION_SLOTS - 1) { // check maximum number of expansion slots - 1 (as we use the last one as the default)
+ mi_atomic_store_release(slot, MI_TLS_EXPANSION_SLOT);
+ mi_atomic_store_release(extended,index - MI_TLS_DIRECT_SLOTS);
+ return true;
+ }
+ #endif
+ else {
+ // to high an index for us
+ _mi_error_message(EFAULT, "returned TLS index was too high (%u)\n", index);
+ TlsFree(index);
+ *raw_index = TLS_OUT_OF_INDEXES;
+ mi_atomic_store_release(slot, MI_TLS_ERROR_SLOT);
+ mi_atomic_store_release(extended,MI_TLS_ERROR_EXPANSION_SLOT);
+ return false;
+ }
+}
+
+static void mi_win_tls_slot_free(_Atomic(size_t)*slot, _Atomic(size_t)*extended, DWORD* raw_index) {
+ if (*raw_index != TLS_OUT_OF_INDEXES) {
+ mi_atomic_store_release(slot, MI_TLS_ERROR_SLOT);
+ mi_atomic_store_release(extended, MI_TLS_ERROR_EXPANSION_SLOT);
+ TlsFree(*raw_index);
+ *raw_index = TLS_OUT_OF_INDEXES;
+ }
+}
+
+void _mi_tls_slots_init(void) {
+ mi_atomic_do_once {
+ bool ok = mi_win_tls_slot_alloc(&_mi_theap_default_slot, &_mi_theap_default_expansion_slot, &mi_tls_raw_index_default);
+ if (ok) {
+ ok = mi_win_tls_slot_alloc(&_mi_theap_cached_slot, &_mi_theap_cached_expansion_slot, &mi_tls_raw_index_cached);
+ }
+ if (!ok) {
+ _mi_error_message(EFAULT, "unable to allocate a fast TLS user slot.\n");
+ }
+ }
+}
+
+void _mi_tls_slots_done(void) {
+ mi_win_tls_slot_free(&_mi_theap_default_slot, &_mi_theap_default_expansion_slot, &mi_tls_raw_index_default);
+ mi_win_tls_slot_free(&_mi_theap_cached_slot, &_mi_theap_cached_expansion_slot, &mi_tls_raw_index_cached );
+}
+
+static void mi_win_tls_slot_set(size_t slot, size_t extended_slot, void* value) {
+ mi_assert_internal((slot >= MI_TLS_DIRECT_FIRST && slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) || slot == MI_TLS_EXPANSION_SLOT);
+ if (slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) {
+ mi_prim_tls_slot_set(slot, value);
+ }
+ else {
+ mi_assert_internal(extended_slot < MI_TLS_EXPANSION_SLOTS);
+ TlsSetValue((DWORD)(extended_slot + MI_TLS_DIRECT_SLOTS), value); // use TlsSetValue to initialize the TlsExpansion array if needed
+ }
+}
+
+#elif MI_TLS_MODEL_PTHREADS
+
+// only for pthreads for now
+mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_default_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID);
+mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_cached_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID);
+
+static void mi_theap_cached_key_destroy(void* theapv) {
+ mi_theap_t* theap = (mi_theap_t*)theapv;
+ if (theap!=NULL) {
+ _mi_theap_decref(theap);
+ }
+}
+
+void _mi_tls_slots_init(void) {
+ mi_atomic_do_once {
+ pthread_key_t key;
+ if (_mi_pthread_key_create(&key,NULL,NULL)) { mi_atomic_store_release(&_mi_theap_default_key,key); }
+ if (_mi_pthread_key_create(&key,&mi_theap_cached_key_destroy,NULL)) { mi_atomic_store_release(&_mi_theap_cached_key,key); }
+ }
+}
+
+void _mi_tls_slots_done(void) {
+ pthread_key_t key = mi_atomic_exchange_relaxed(&_mi_theap_default_key,MI_PTHREAD_KEY_INVALID);
+ if (key!=MI_PTHREAD_KEY_INVALID) { pthread_key_delete(key); }
+ key = mi_atomic_exchange_relaxed(&_mi_theap_cached_key,MI_PTHREAD_KEY_INVALID);
+ if (key!=MI_PTHREAD_KEY_INVALID) { pthread_key_delete(key); }
+}
+
+#elif MI_TLS_MODEL_FIXED
+
+void _mi_tls_slots_init(void) {
+ mi_atomic_do_once {
+ mi_theap_t* theap = _mi_theap_default();
+ if (theap!=NULL) {
+ _mi_error_message(EINVAL,"fixed TLS slot is already in use (slot %d = %p)", MI_TLS_MODEL_FIXED_DEFAULT, theap);
+ }
+ theap = _mi_theap_cached();
+ if (theap!=NULL) {
+ _mi_error_message(EINVAL,"fixed TLS slot is already in use (slot %d = %p)", MI_TLS_MODEL_FIXED_CACHED, theap);
+ }
+ }
+}
+
+void _mi_tls_slots_done(void) {
+ // nothing
+}
+
+
+#else
+
+void _mi_tls_slots_init(void) {
+ // nothing
+}
+
+void _mi_tls_slots_done(void) {
+ // nothing
+}
+
+#endif
+
+void _mi_theap_cached_set(mi_theap_t* theap) {
+ mi_theap_t* prev = _mi_theap_cached();
+ if (prev==theap) return;
+ // set
+ _mi_tls_slots_init();
+ #if MI_TLS_MODEL_LOCAL
+ __mi_theap_cached = theap;
+ #elif MI_TLS_MODEL_FIXED
+ mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_CACHED, theap);
+ #elif MI_TLS_MODEL_WIN32
+ mi_win_tls_slot_set(mi_atomic_load_relaxed(&_mi_theap_cached_slot), mi_atomic_load_relaxed(&_mi_theap_cached_expansion_slot), theap);
+ #elif MI_TLS_MODEL_PTHREADS
+ pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_cached_key);
+ if (key!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(key, theap); }
+ #endif
+ // update refcounts (so cached theap memory keeps available until no longer cached)
+ _mi_theap_incref(theap);
+ _mi_theap_decref(prev);
+}
+
+void _mi_theap_default_set(mi_theap_t* theap) {
+ mi_assert_internal(theap != NULL);
+ mi_assert_internal(theap->tld != NULL);
+ mi_assert_internal(mi_theap_matches_thread(theap));
+ _mi_tls_slots_init();
+ #if MI_TLS_MODEL_LOCAL
+ __mi_theap_default = theap;
+ #elif MI_TLS_MODEL_FIXED
+ mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_DEFAULT, theap);
+ #elif MI_TLS_MODEL_WIN32
+ mi_win_tls_slot_set(mi_atomic_load_relaxed(&_mi_theap_default_slot), mi_atomic_load_relaxed(&_mi_theap_default_expansion_slot), theap);
+ #elif MI_TLS_MODEL_PTHREADS
+ pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_default_key);
+ if (key!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(key, theap); }
+ #endif
+
+ // set theap main if needed
+ if (mi_theap_is_initialized(theap)) {
+ // ensure the default theap is passed to `_mi_thread_done` as on some platforms we cannot access TLS at thread termination (as it would allocate again)
+ _mi_prim_thread_associate_default_theap(theap);
+ }
+}
diff --git a/vendored/mimalloc/src/prim/prim.c b/vendored/mimalloc/src/prim/prim.c
index 5147bae81..3d3d0c2e8 100644
--- a/vendored/mimalloc/src/prim/prim.c
+++ b/vendored/mimalloc/src/prim/prim.c
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -27,11 +27,10 @@ terms of the MIT license. A copy of the license can be found in the file
#endif
// Generic process initialization
-#ifndef MI_PRIM_HAS_PROCESS_ATTACH
+#if !defined(MI_PRIM_HAS_PROCESS_ATTACH)
#if defined(__GNUC__) || defined(__clang__)
- // gcc,clang: use the constructor/destructor attribute
- // which for both seem to run before regular constructors/destructors
- #if defined(__clang__)
+ // gcc,clang: use the constructor/destructor attribute with the highest user priority (<=100 is reserved for the compiler itself)
+ #if defined(__clang__) || (__GNUC__ >= 5 && !defined(__APPLE__)) // gcc has support since 4.2 but not on macOS (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=119435)
#define mi_attr_constructor __attribute__((constructor(101)))
#define mi_attr_destructor __attribute__((destructor(101)))
#else
diff --git a/vendored/mimalloc/src/prim/readme.md b/vendored/mimalloc/src/prim/readme.md
index 380dd3a71..02cb1b90c 100644
--- a/vendored/mimalloc/src/prim/readme.md
+++ b/vendored/mimalloc/src/prim/readme.md
@@ -6,4 +6,7 @@ This is the portability layer where all primitives needed from the OS are define
- `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform
(and on macOS, `osx/prim.c` defers to `unix/prim.c`).
+- `include/mimalloc/prim-tls.h`: primitive TLS model (thread local, pthreads, etc)
+- `prim-tls.c`: TLS model implementation
+
Note: still work in progress, there may still be places in the sources that still depend on OS ifdef's.
\ No newline at end of file
diff --git a/vendored/mimalloc/src/prim/unix/prim.c b/vendored/mimalloc/src/prim/unix/prim.c
index 93563f587..4e3eafb37 100644
--- a/vendored/mimalloc/src/prim/unix/prim.c
+++ b/vendored/mimalloc/src/prim/unix/prim.c
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -41,6 +41,13 @@ terms of the MIT license. A copy of the license can be found in the file
#else
#include <sys/mman.h>
#endif
+ #if defined(__riscv) || defined(_M_RISCV)
+ #if defined(MI_HAS_SYS_HWPROBEH)
+ #include <sys/hwprobe.h>
+ #elif defined(MI_HAS_ASM_HWPROBEH)
+ #include <asm/hwprobe.h>
+ #endif
+ #endif
#elif defined(__APPLE__)
#include <AvailabilityMacros.h>
#include <TargetConditionals.h>
@@ -72,7 +79,7 @@ terms of the MIT license. A copy of the license can be found in the file
#define MADV_FREE POSIX_MADV_FREE
#endif
-#define MI_UNIX_LARGE_PAGE_SIZE (2*MI_MiB) // TODO: can we query the OS for this?
+#define MI_UNIX_LARGE_PAGE_SIZE (2*MI_MiB) // todo: can we query the OS for this?
//------------------------------------------------------------------------------------
// Use syscalls for some primitives to allow for libraries that override open/read/close etc.
@@ -150,13 +157,14 @@ static bool unix_detect_thp(void) {
#if defined(__linux__)
int fd = mi_prim_open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
if (fd >= 0) {
- char buf[32];
+ char buf[64];
ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf));
mi_prim_close(fd);
// <https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html>
// between brackets is the current value, for example: always [madvise] never
if (nread >= 1) {
- thp_enabled = (_mi_strnstr(buf,32,"[never]") == NULL);
+ if (nread > 64) { nread = 64; }
+ thp_enabled = (_mi_strnstr(buf,nread,"[never]") == NULL);
}
}
#endif
@@ -205,6 +213,40 @@ static void unix_detect_physical_memory( size_t page_size, size_t* physical_memo
#endif
}
+// Detect the virtual address bits (currently Linux/RISC-V only)
+static size_t unix_detect_virtual_address_bits(void) {
+ #if defined(__riscv) || defined(_M_RISCV)
+ #if defined(RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS)
+ struct riscv_hwprobe probe = { .key = RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS, };
+ // Prefer the GNU libc interface if available, as it can also use the VDSO
+ #if defined(MI_HAS_SYS_HWPROBEH)
+ if (__riscv_hwprobe(&probe, 1, 0, NULL, 0) == 0)
+ #else
+ if (syscall(__NR_riscv_hwprobe, &probe, 1, 0, NULL, 0) == 0)
+ #endif
+ {
+ if (probe.key != -1) { // If a key is unknown to the kernel, its key field will be cleared to -1.
+ return (MI_SIZE_BITS - mi_clz((uintptr_t)probe.value));
+ }
+ }
+ #endif
+ // Fallback to checking /proc/cpuinfo for older kernels
+ const int fd = mi_prim_open("/proc/cpuinfo", O_RDONLY);
+ if (fd >= 0) {
+ char buf[2048];
+ const ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf));
+ mi_prim_close(fd);
+ if ((nread >= 1) && (nread <= (ssize_t)sizeof(buf))) {
+ if (_mi_strnstr(buf, nread, "sv39")) { return 39; }
+ else if (_mi_strnstr(buf, nread, "sv48")) { return 48; }
+ else if (_mi_strnstr(buf, nread, "sv57")) { return 57; }
+ }
+ }
+ #endif // riscv
+ // default
+ return MI_MAX_VABITS;
+}
+
void _mi_prim_mem_init( mi_os_mem_config_t* config )
{
long psize = sysconf(_SC_PAGESIZE);
@@ -218,6 +260,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config )
config->has_partial_free = true; // mmap can free in parts
config->has_virtual_reserve = true; // todo: check if this true for NetBSD? (for anonymous mmap with PROT_NONE)
config->has_transparent_huge_pages = unix_detect_thp();
+ config->virtual_address_bits = unix_detect_virtual_address_bits();
// disable transparent huge pages for this process?
#if (defined(__linux__) || defined(__ANDROID__)) && defined(PR_GET_THP_DISABLE)
@@ -251,7 +294,7 @@ int _mi_prim_free(void* addr, size_t size ) {
// return errno on failure
static int unix_madvise(void* addr, size_t size, int advice) {
- #if defined(__sun)
+ #if defined(__sun) || defined(_AIX)
const int res = madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520)
return (res==0 ? 0 : errno);
#elif defined(__QNX__)
@@ -296,7 +339,7 @@ static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignmen
// fall back to regular mmap
}
#endif
- #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)
+ #if (MI_SIZE_SIZE >= 8) && !defined(MAP_ALIGNED)
// on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations
if (addr == NULL) {
void* hint = _mi_os_get_aligned_hint(try_alignment, size);
@@ -333,6 +376,10 @@ static int unix_mmap_fd(void) {
#endif
}
+#if defined(MAP_ALIGNED_SUPER) || defined(MAP_HUGETLB) || defined(MAP_HUGE_1GB) || defined(MAP_HUGE_2MB) || defined(VM_FLAGS_SUPERPAGE_SIZE_2MB)
+#define MI_OS_HAS_HUGE_PAGES 1
+#endif
+
static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {
#if !defined(MAP_ANONYMOUS)
#define MAP_ANONYMOUS MAP_ANON
@@ -350,6 +397,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec
protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
#endif
// huge page allocation
+ #if MI_OS_HAS_HUGE_PAGES
if (allow_large && (large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))) {
static _Atomic(size_t) large_page_try_ok; // = 0;
size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);
@@ -370,8 +418,8 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec
lflags |= MAP_HUGETLB;
#endif
#ifdef MAP_HUGE_1GB
- static bool mi_huge_pages_available = true;
- if (large_only && (size % MI_GiB) == 0 && mi_huge_pages_available) {
+ static _Atomic(size_t) mi_huge_1gib_pages_unavailable;
+ if (large_only && (size % MI_GiB) == 0 && (mi_atomic_load_relaxed(&mi_huge_1gib_pages_unavailable)==0)) {
lflags |= MAP_HUGE_1GB;
}
else
@@ -390,7 +438,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec
p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);
#ifdef MAP_HUGE_1GB
if (p == NULL && (lflags & MAP_HUGE_1GB) == MAP_HUGE_1GB) {
- mi_huge_pages_available = false; // don't try huge 1GiB pages again
+ mi_atomic_store_relaxed(&mi_huge_1gib_pages_unavailable,1); // don't try huge 1GiB pages again
if (large_only) {
_mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno);
}
@@ -404,7 +452,10 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec
}
}
}
- }
+ } // huge pages
+ #else
+ MI_UNUSED(large_only);
+ #endif
// regular allocation
if (p == NULL) {
*is_large = false;
@@ -412,15 +463,17 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec
#if !defined(MI_NO_THP)
if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) {
#if defined(MADV_HUGEPAGE)
- // Many Linux systems don't allow MAP_HUGETLB but they support instead
- // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE
- // though since properly aligned allocations will already use large pages if available
- // in that case -- in particular for our large regions (in `memory.c`).
- // However, some systems only allow THP if called with explicit `madvise`, so
- // when large OS pages are enabled for mimalloc, we call `madvise` anyways.
- if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) {
- // *is_large = true; // possibly
- };
+ if (_mi_os_canuse_thp()) {
+ // Many Linux systems don't allow MAP_HUGETLB but they support instead
+ // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE
+ // though since properly aligned allocations will already use large pages if available
+ // in that case -- in particular for our large regions (in `memory.c`).
+ // However, some systems only allow THP if called with explicit `madvise`, so
+ // when large OS pages are enabled for mimalloc, we call `madvise` anyways.
+ if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) {
+ // *is_large = true; // possibly
+ };
+ }
#elif defined(__sun)
struct memcntl_mha cmd = {0};
cmd.mha_pagesize = _mi_os_large_page_size();
@@ -452,7 +505,7 @@ int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool comm
//---------------------------------------------
static void unix_mprotect_hint(int err) {
- #if defined(__linux__) && (MI_SECURE>=5) // guard page around every mimalloc page
+ #if defined(__linux__) && (MI_SECURE>=5 || MI_GUARDED) // guard page around every mimalloc page
if (err == ENOMEM) {
_mi_warning_message("The next warning may be caused by a low memory map limit.\n"
" On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n"
@@ -492,38 +545,39 @@ int _mi_prim_reuse(void* start, size_t size) {
int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) {
int err = 0;
- #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE)
- // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097)
- err = unix_madvise(start, size, MADV_FREE_REUSABLE);
- if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }
- #else
- // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)
- err = unix_madvise(start, size, MADV_DONTNEED);
- #endif
- // codebase-memory-mcp local change (2026-10-10): macOS always takes the
- // recommit path that upstream reserves for MI_DEBUG / MI_SECURE>2 builds.
- // In the release path a purged range stays "committed" and is re-accounted as
- // a whole with MADV_FREE_REUSE when the arena hands it out again
- // (arena.c, _mi_os_reuse), so macOS charges its pages to phys_footprint
- // whether or not they are touched; recommitting instead charges only the
- // pages that are written. Measured on the indexer (Apple arm64, macOS 26.6),
- // peak phys_footprint release -> this change, wall time unchanged and the
- // graph byte-identical: TypeScript 1.98 -> 1.09 GB, kubernetes 2.92 -> 1.88 GB,
- // dotnet/runtime 12.7 -> 10.6 GB, linux kernel 16.0 -> 13.5 GB. Other
- // platforms keep upstream's behaviour (MADV_DONTNEED drops RSS at once).
- #if !MI_DEBUG && MI_SECURE<=2 && !defined(__APPLE__)
- *needs_recommit = false;
+ #if 1
+ #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE)
+ // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097)
+ err = unix_madvise(start, size, MADV_FREE_REUSABLE);
+ if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }
+ #else
+ // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)
+ err = unix_madvise(start, size, MADV_DONTNEED);
+ #endif
+ // codebase-memory-mcp local change (2026-10-10): macOS always takes the
+ // recommit path that upstream reserves for MI_DEBUG / MI_SECURE>2 builds.
+ // In the release path a purged range stays "committed" and is re-accounted as
+ // a whole with MADV_FREE_REUSE when the arena hands it out again
+ // (arena.c, _mi_os_reuse), so macOS charges its pages to phys_footprint
+ // whether or not they are touched; recommitting instead charges only the
+ // pages that are written. Measured on the indexer (Apple arm64, macOS 26.6),
+ // peak phys_footprint release -> this change, wall time unchanged and the
+ // graph byte-identical: TypeScript 1.98 -> 1.09 GB, kubernetes 2.92 -> 1.88 GB,
+ // dotnet/runtime 12.7 -> 10.6 GB, linux kernel 16.0 -> 13.5 GB. Other
+ // platforms keep upstream's behaviour (MADV_DONTNEED drops RSS at once).
+ #if !MI_DEBUG && MI_SECURE<=2 && !defined(__APPLE__)
+ *needs_recommit = false;
+ #else
+ *needs_recommit = true;
+ mprotect(start, size, PROT_NONE);
+ #endif
#else
+ // decommit: use mmap with MAP_FIXED and PROT_NONE to discard the existing memory (and reduce rss)
*needs_recommit = true;
- mprotect(start, size, PROT_NONE);
+ const int fd = unix_mmap_fd();
+ void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);
+ if (p != start) { err = errno; }
#endif
- /*
- // decommit: use mmap with MAP_FIXED and PROT_NONE to discard the existing memory (and reduce rss)
- *needs_recommit = true;
- const int fd = unix_mmap_fd();
- void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);
- if (p != start) { err = errno; }
- */
return err;
}
@@ -569,7 +623,7 @@ int _mi_prim_protect(void* start, size_t size, bool protect) {
// Huge page allocation
//---------------------------------------------
-#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__)
+#if (MI_SIZE_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__)
#ifndef MPOL_PREFERRED
#define MPOL_PREFERRED 1
@@ -590,15 +644,15 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo
bool is_large = true;
*is_zero = true;
*addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large);
- if (*addr != NULL && numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes
+ if (*addr != NULL && numa_node >= 0 && numa_node < (8*MI_SIZE_SIZE - 1)) { // at most 63 nodes
unsigned long numa_mask = (1UL << numa_node);
- // TODO: does `mbind` work correctly for huge OS pages? should we
+ // todo: does `mbind` work correctly for huge OS pages? should we
// use `set_mempolicy` before calling mmap instead?
// see: <https://lkml.org/lkml/2017/2/9/875>
- long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0);
+ long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_SIZE_SIZE, 0);
if (err != 0) {
err = errno;
- _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err);
+ _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %ld (0x%lx))\n", numa_node, err, err);
}
}
return (*addr != NULL ? 0 : errno);
@@ -623,9 +677,9 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo
size_t _mi_prim_numa_node(void) {
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu)
- unsigned long node = 0;
- unsigned long ncpu = 0;
- long err = syscall(SYS_getcpu, &ncpu, &node, NULL);
+ unsigned int node = 0;
+ unsigned int ncpu = 0;
+ int err = syscall(SYS_getcpu, &ncpu, &node, NULL);
if (err != 0) return 0;
return node;
#else
@@ -635,13 +689,16 @@ size_t _mi_prim_numa_node(void) {
size_t _mi_prim_numa_node_count(void) {
char buf[128];
- unsigned node = 0;
- for(node = 0; node < 256; node++) {
+ unsigned last_found = 0;
+ for(unsigned node = 1; node < 256; node++) {
// enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation)
- _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1);
- if (mi_prim_access(buf,R_OK) != 0) break;
+ _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node);
+ if (mi_prim_access(buf,R_OK) != 0) {
+ if (node - last_found > 4) break; // allow some sparseness of nodes but not more than 4
+ }
+ else { last_found = node; } // highest found node
}
- return (node+1);
+ return last_found + 1;
}
#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000
@@ -667,7 +724,7 @@ size_t _mi_prim_numa_node_count(void) {
#elif defined(__DragonFly__)
size_t _mi_prim_numa_node(void) {
- // TODO: DragonFly does not seem to provide any userland means to get this information.
+ // todo: DragonFly does not seem to provide any userland means to get this information.
return 0ul;
}
@@ -697,35 +754,39 @@ size_t _mi_prim_numa_node_count(void) {
#include <time.h>
-#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)
-
-mi_msecs_t _mi_prim_clock_now(void) {
- struct timespec t;
- #ifdef CLOCK_MONOTONIC
- clock_gettime(CLOCK_MONOTONIC, &t);
+// low resolution timer
+static mi_msecs_t mi_prim_clock_now_lowres(void) {
+ const int64_t ticks = (int64_t)clock();
+ #if !defined(CLOCKS_PER_SEC)
+ return ticks;
#else
- clock_gettime(CLOCK_REALTIME, &t);
+ if (CLOCKS_PER_SEC <= 0 || CLOCKS_PER_SEC == 1000) {
+ return ticks;
+ }
+ else if (CLOCKS_PER_SEC > 0 && CLOCKS_PER_SEC < 1000) {
+ return ticks * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);
+ }
+ else {
+ return ticks / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);
+ }
#endif
- return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);
}
-#else
-
-// low resolution timer
mi_msecs_t _mi_prim_clock_now(void) {
- #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)
- return (mi_msecs_t)clock();
- #elif (CLOCKS_PER_SEC < 1000)
- return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);
- #else
- return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);
+ #if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)
+ #ifdef CLOCK_MONOTONIC
+ const clockid_t clockid = CLOCK_MONOTONIC;
+ #else
+ const clockid_t clockid = CLOCK_REALTIME;
+ #endif
+ struct timespec t;
+ if (clock_gettime(clockid,&t) == 0) {
+ return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000L);
+ }
#endif
+ return mi_prim_clock_now_lowres();
}
-#endif
-
-
-
//----------------------------------------------------------------
// Process info
@@ -751,41 +812,46 @@ static mi_msecs_t timeval_secs(const struct timeval* tv) {
void _mi_prim_process_info(mi_process_info_t* pinfo)
{
struct rusage rusage;
- getrusage(RUSAGE_SELF, &rusage);
- pinfo->utime = timeval_secs(&rusage.ru_utime);
- pinfo->stime = timeval_secs(&rusage.ru_stime);
-#if !defined(__HAIKU__)
- pinfo->page_faults = rusage.ru_majflt;
-#endif
-#if defined(__HAIKU__)
- // Haiku does not have (yet?) a way to
- // get these stats per process
- thread_info tid;
- area_info mem;
- ssize_t c;
- get_thread_info(find_thread(0), &tid);
- while (get_next_area_info(tid.team, &c, &mem) == B_OK) {
- pinfo->peak_rss += mem.ram_size;
- }
- pinfo->page_faults = 0;
-#elif defined(__APPLE__)
- pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes
- #ifdef MACH_TASK_BASIC_INFO
- struct mach_task_basic_info info;
- mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
- if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
- pinfo->current_rss = (size_t)info.resident_size;
- }
- #else
- struct task_basic_info info;
- mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT;
- if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
- pinfo->current_rss = (size_t)info.resident_size;
+ if (getrusage(RUSAGE_SELF, &rusage) == 0) {
+ pinfo->utime = timeval_secs(&rusage.ru_utime);
+ pinfo->stime = timeval_secs(&rusage.ru_stime);
+ #if !defined(__HAIKU__)
+ pinfo->page_faults = rusage.ru_majflt;
+ #endif
+ #if defined(__APPLE__)
+ pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes
+ #else
+ pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB
+ #endif
}
+
+ #if defined(__HAIKU__)
+ // Haiku does not have (yet?) a way to
+ // get these stats per process
+ thread_info tid;
+ if (get_thread_info(find_thread(0), &tid) == B_OK) {
+ area_info mem;
+ ssize_t c;
+ while (get_next_area_info(tid.team, &c, &mem) == B_OK) {
+ pinfo->peak_rss += mem.ram_size;
+ }
+ }
+ pinfo->page_faults = 0;
+ #elif defined(__APPLE__)
+ #ifdef MACH_TASK_BASIC_INFO
+ struct mach_task_basic_info info;
+ mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
+ if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
+ pinfo->current_rss = (size_t)info.resident_size;
+ }
+ #else
+ struct task_basic_info info;
+ mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT;
+ if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
+ pinfo->current_rss = (size_t)info.resident_size;
+ }
+ #endif
#endif
-#else
- pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB
-#endif
// use defaults for commit
}
@@ -832,28 +898,28 @@ static char** mi_get_environ(void) {
return environ;
}
#endif
-bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
- if (name==NULL) return false;
+int _mi_prim_getenv(const char* name, char* result, size_t result_size) {
+ if (name==NULL) return -1;
const size_t len = _mi_strlen(name);
- if (len == 0) return false;
+ if (len == 0) return -1;
char** env = mi_get_environ();
- if (env == NULL) return false;
+ if (env == NULL) return -1;
// compare up to 10000 entries
for (int i = 0; i < 10000 && env[i] != NULL; i++) {
const char* s = env[i];
if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive
// found it
- _mi_strlcpy(result, s + len + 1, result_size);
- return true;
+ if (!_mi_strlcpy(result, s + len + 1, result_size)) return -1;
+ return 1; // success
}
}
- return false;
+ return 0; // not found
}
#else
// fallback: use standard C `getenv` but this cannot be used while initializing the C runtime
-bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
+int _mi_prim_getenv(const char* name, char* result, size_t result_size) {
// cannot call getenv() when still initializing the C runtime.
- if (_mi_preloading()) return false;
+ if (_mi_preloading()) return -1; // error, try again later
const char* s = getenv(name);
if (s == NULL) {
// we check the upper case name too.
@@ -865,9 +931,9 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
buf[len] = 0;
s = getenv(buf);
}
- if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false;
- _mi_strlcpy(result, s, result_size);
- return true;
+ if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return 0; // not found
+ if (!_mi_strlcpy(result, s, result_size)) return -1;
+ return 1; // success
}
#endif // !MI_USE_ENVIRON
@@ -896,7 +962,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {
return true;
}
-#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) // also for old apple versions < 10.7 (issue #829)
+#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) || defined(__CYGWIN__) // also for old apple versions < 10.7 (issue #829)
#include <sys/types.h>
#include <sys/stat.h>
@@ -928,7 +994,10 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {
size_t count = 0;
while(count < buf_len) {
ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count);
- if (ret<=0) {
+ if (ret==0) {
+ break;
+ }
+ else if (ret<0) {
if (errno!=EAGAIN && errno!=EINTR) break;
}
else {
@@ -956,7 +1025,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {
// use pthread local storage keys to detect thread ending
// (and used with MI_TLS_PTHREADS for the default theap)
-pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);
+pthread_key_t _mi_heap_default_key = MI_PTHREAD_KEY_INVALID;
static void mi_pthread_done(void* value) {
if (value!=NULL) {
@@ -965,22 +1034,25 @@ static void mi_pthread_done(void* value) {
}
void _mi_prim_thread_init_auto_done(void) {
- mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));
+ mi_assert_internal(_mi_heap_default_key == MI_PTHREAD_KEY_INVALID);
pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);
}
void _mi_prim_thread_done_auto_done(void) {
- if (_mi_heap_default_key != (pthread_key_t)(-1)) { // do not leak the key, see issue #809
- pthread_key_delete(_mi_heap_default_key);
+ pthread_key_t key = _mi_heap_default_key;
+ if (key != MI_PTHREAD_KEY_INVALID) { // do not leak the key, see issue #809
+ _mi_heap_default_key = MI_PTHREAD_KEY_INVALID;
+ pthread_key_delete(key);
}
}
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
- if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD
+ if (_mi_heap_default_key != MI_PTHREAD_KEY_INVALID) { // can happen during recursive invocation on freeBSD
pthread_setspecific(_mi_heap_default_key, theap);
}
}
+
#else
void _mi_prim_thread_init_auto_done(void) {
diff --git a/vendored/mimalloc/src/prim/wasi/prim.c b/vendored/mimalloc/src/prim/wasi/prim.c
index 790daff02..2d69b7245 100644
--- a/vendored/mimalloc/src/prim/wasi/prim.c
+++ b/vendored/mimalloc/src/prim/wasi/prim.c
@@ -185,33 +185,39 @@ size_t _mi_prim_numa_node_count(void) {
#include <time.h>
-#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)
-
-mi_msecs_t _mi_prim_clock_now(void) {
- struct timespec t;
- #ifdef CLOCK_MONOTONIC
- clock_gettime(CLOCK_MONOTONIC, &t);
+// low resolution timer
+static mi_msecs_t mi_prim_clock_now_lowres(void) {
+ const int64_t ticks = (int64_t)clock();
+ #if !defined(CLOCKS_PER_SEC)
+ return ticks;
#else
- clock_gettime(CLOCK_REALTIME, &t);
+ if (CLOCKS_PER_SEC <= 0 || CLOCKS_PER_SEC == 1000) {
+ return ticks;
+ }
+ else if (CLOCKS_PER_SEC > 0 && CLOCKS_PER_SEC < 1000) {
+ return ticks * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);
+ }
+ else {
+ return ticks / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);
+ }
#endif
- return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000);
}
-#else
-
-// low resolution timer
mi_msecs_t _mi_prim_clock_now(void) {
- #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)
- return (mi_msecs_t)clock();
- #elif (CLOCKS_PER_SEC < 1000)
- return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);
- #else
- return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);
- #endif
+ #if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)
+ #ifdef CLOCK_MONOTONIC
+ const clockid_t clockid = CLOCK_MONOTONIC;
+ #else
+ const clockid_t clockid = CLOCK_REALTIME;
+ #endif
+ struct timespec t;
+ if (clock_gettime(clockid,&t) == 0) {
+ return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000L);
+ }
+ #endif
+ return mi_prim_clock_now_lowres();
}
-#endif
-
//----------------------------------------------------------------
// Process info
@@ -237,9 +243,9 @@ void _mi_prim_out_stderr( const char* msg ) {
// Environment
//----------------------------------------------------------------
-bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
+int _mi_prim_getenv(const char* name, char* result, size_t result_size) {
// cannot call getenv() when still initializing the C runtime.
- if (_mi_preloading()) return false;
+ if (_mi_preloading()) return -1; // error, try again later
const char* s = getenv(name);
if (s == NULL) {
// we check the upper case name too.
@@ -251,9 +257,9 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
buf[len] = 0;
s = getenv(buf);
}
- if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return false;
+ if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return 0; // not found
_mi_strlcpy(result, s, result_size);
- return true;
+ return 1; // found
}
diff --git a/vendored/mimalloc/src/prim/windows/prim.c b/vendored/mimalloc/src/prim/windows/prim.c
index 51ad3104f..222fa6303 100644
--- a/vendored/mimalloc/src/prim/windows/prim.c
+++ b/vendored/mimalloc/src/prim/windows/prim.c
@@ -13,9 +13,9 @@ terms of the MIT license. A copy of the license can be found in the file
#include <stdio.h> // fputs, stderr
#include <stdlib.h> // atexit
-// xbox has no console IO
-#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP | WINAPI_PARTITION_SYSTEM)
-#define MI_HAS_CONSOLE_IO
+// xbox has no console IO and cannot use LoadLibrary or GetModuleHandle
+#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM)
+#define MI_WIN_DESKTOP 1
#endif
//---------------------------------------------
@@ -87,6 +87,34 @@ typedef BOOL (__stdcall *PGetPhysicallyInstalledSystemMemory)( PULONGLONG TotalM
typedef BOOL (__stdcall* PGetVersionExW)(LPOSVERSIONINFOW lpVersionInformation);
+// Load a library
+static HMODULE mi_win_loadlibrary(const TCHAR* library) {
+ #if MI_WIN_DESKTOP
+ return LoadLibrary(library);
+ #else
+ return LoadPackagedLibrary(library, 0);
+ #endif
+}
+
+// Get a library handle (and possibly load it)
+static HMODULE mi_win_getlibrary(const TCHAR* library, bool* should_free) {
+ #if MI_WIN_DESKTOP
+ // avoid calling LoadLibrary for "kernel32", "ntdll", and "kernelbase" (also to avoid hitting the loader lock)
+ HMODULE mod = GetModuleHandle(library);
+ if (mod!=NULL) {
+ *should_free = false;
+ return mod;
+ }
+ #endif
+ *should_free = true;
+ return mi_win_loadlibrary(library);
+}
+
+static void mi_win_freelibrary(HMODULE mod, bool should_free) {
+ if (should_free) {
+ FreeLibrary(mod);
+ }
+}
//---------------------------------------------
// Enable large page support dynamically (if possible)
@@ -103,15 +131,17 @@ static bool win_enable_large_os_pages_once(size_t* large_page_size)
unsigned long err = 0;
HANDLE token = NULL;
BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token);
+ err = GetLastError();
if (ok) {
TOKEN_PRIVILEGES tp;
ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid);
+ err = GetLastError();
if (ok) {
tp.PrivilegeCount = 1;
tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0);
+ err = GetLastError();
if (ok) {
- err = GetLastError();
ok = (err == ERROR_SUCCESS);
if (ok && large_page_size != NULL && pGetLargePageMinimum != NULL) {
*large_page_size = (*pGetLargePageMinimum)();
@@ -121,17 +151,19 @@ static bool win_enable_large_os_pages_once(size_t* large_page_size)
CloseHandle(token);
}
if (!ok) {
- if (err == 0) err = GetLastError();
+ if (err == 0) { err = GetLastError(); }
_mi_warning_message("cannot enable large OS page support, error %lu\n", err);
}
return (ok!=0);
}
static bool win_enable_large_os_pages(size_t* large_page_size) {
+ static size_t win_large_page_size = 0;
mi_atomic_do_once {
- win_enable_large_os_pages_once(large_page_size);
+ win_enable_large_os_pages_once(&win_large_page_size);
}
- return (_mi_os_large_page_size() > 0);
+ if (large_page_size != NULL) { *large_page_size = win_large_page_size; }
+ return (win_large_page_size > 0);
}
@@ -162,21 +194,22 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config )
}
// get the VirtualAlloc2 function
- HINSTANCE hDll = LoadLibrary(TEXT("kernelbase.dll"));
+ bool hDllFree;
+ HINSTANCE hDll = mi_win_getlibrary(TEXT("kernelbase.dll"), &hDllFree);
if (hDll != NULL) {
// use VirtualAlloc2FromApp if possible as it is available to Windows store apps
pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp");
if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2");
- FreeLibrary(hDll);
+ mi_win_freelibrary(hDll, hDllFree);
}
// NtAllocateVirtualMemoryEx is used for huge page allocation
- hDll = LoadLibrary(TEXT("ntdll.dll"));
+ hDll = mi_win_getlibrary(TEXT("ntdll.dll"), &hDllFree);
if (hDll != NULL) {
pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx");
- FreeLibrary(hDll);
+ mi_win_freelibrary(hDll, hDllFree);
}
// Try to use Win7+ numa API
- hDll = LoadLibrary(TEXT("kernel32.dll"));
+ hDll = mi_win_getlibrary(TEXT("kernel32.dll"), &hDllFree);
if (hDll != NULL) {
pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, "GetCurrentProcessorNumberEx");
pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNodeEx");
@@ -205,7 +238,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config )
win_minor_version = version.dwMinorVersion;
}
}
- FreeLibrary(hDll);
+ mi_win_freelibrary(hDll, hDllFree);
}
// Enable large/huge OS page support?
if (mi_option_is_enabled(mi_option_allow_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
@@ -228,8 +261,9 @@ int _mi_prim_free(void* addr, size_t size ) {
// the memory region returned by VirtualAlloc; in that case we need to free using
// the start of the region.
MEMORY_BASIC_INFORMATION info; _mi_memzero_var(info);
- VirtualQuery(addr, &info, sizeof(info));
- if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)(4*MI_MiB)) {
+ err = (VirtualQuery(addr, &info, sizeof(info)) == 0);
+ if (err) { errcode = GetLastError(); }
+ if (!err && info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)(4*MI_MiB)) {
errcode = 0;
err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0);
if (err) { errcode = GetLastError(); }
@@ -244,7 +278,7 @@ int _mi_prim_free(void* addr, size_t size ) {
//---------------------------------------------
static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) {
- #if (MI_INTPTR_SIZE >= 8)
+ #if (MI_SIZE_SIZE >= 8)
// on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations
if (addr == NULL) {
void* hint = _mi_os_get_aligned_hint(try_alignment,size);
@@ -525,8 +559,9 @@ static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) {
static LARGE_INTEGER mfreq; // = 0
if (mfreq.QuadPart == 0LL) {
LARGE_INTEGER f;
- QueryPerformanceFrequency(&f);
- mfreq.QuadPart = f.QuadPart/1000LL;
+ if (QueryPerformanceFrequency(&f)) {
+ mfreq.QuadPart = f.QuadPart/1000LL;
+ }
if (mfreq.QuadPart == 0) mfreq.QuadPart = 1;
}
return (mi_msecs_t)(t.QuadPart / mfreq.QuadPart);
@@ -534,8 +569,12 @@ static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) {
mi_msecs_t _mi_prim_clock_now(void) {
LARGE_INTEGER t;
- QueryPerformanceCounter(&t);
- return mi_to_msecs(t);
+ if (QueryPerformanceCounter(&t)) {
+ return mi_to_msecs(t);
+ }
+ else {
+ return 0;
+ }
}
@@ -562,29 +601,31 @@ void _mi_prim_process_info(mi_process_info_t* pinfo)
FILETIME ut;
FILETIME st;
FILETIME et;
- GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut);
- pinfo->utime = filetime_msecs(&ut);
- pinfo->stime = filetime_msecs(&st);
+ if (GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut)) {
+ pinfo->utime = filetime_msecs(&ut);
+ pinfo->stime = filetime_msecs(&st);
+ }
// load psapi on demand
- mi_atomic_do_once {
- HINSTANCE hDll = LoadLibrary(TEXT("psapi.dll"));
+ mi_atomic_do_once{
+ HINSTANCE hDll = mi_win_loadlibrary(TEXT("psapi.dll"));
if (hDll != NULL) {
pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, "GetProcessMemoryInfo");
- // FreeLibrary(hDll); // don't free
+ // mi_win_freelibrary(hDll, true); // don't free
}
}
// get process info
- PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info);
if (pGetProcessMemoryInfo != NULL) {
- pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info));
+ PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info);
+ if (pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info))) {
+ pinfo->current_rss = (size_t)info.WorkingSetSize;
+ pinfo->peak_rss = (size_t)info.PeakWorkingSetSize;
+ pinfo->current_commit = (size_t)info.PagefileUsage;
+ pinfo->peak_commit = (size_t)info.PeakPagefileUsage;
+ pinfo->page_faults = (size_t)info.PageFaultCount;
+ }
}
- pinfo->current_rss = (size_t)info.WorkingSetSize;
- pinfo->peak_rss = (size_t)info.PeakWorkingSetSize;
- pinfo->current_commit = (size_t)info.PagefileUsage;
- pinfo->peak_commit = (size_t)info.PeakPagefileUsage;
- pinfo->page_faults = (size_t)info.PageFaultCount;
}
//----------------------------------------------------------------
@@ -596,32 +637,37 @@ void _mi_prim_out_stderr( const char* msg )
// on windows with redirection, the C runtime cannot handle locale dependent output
// after the main thread closes so we use direct console output.
if (!_mi_preloading()) {
- // _cputs(msg); // _cputs cannot be used as it aborts when failing to lock the console
static HANDLE hcon = INVALID_HANDLE_VALUE;
+ #if MI_WIN_DESKTOP
static bool hconIsConsole = false;
+ #endif
if (hcon == INVALID_HANDLE_VALUE) {
- hcon = GetStdHandle(STD_ERROR_HANDLE);
- #ifdef MI_HAS_CONSOLE_IO
+ hcon = GetStdHandle(STD_ERROR_HANDLE); // returns NULL if no stderr is available
+ #if MI_WIN_DESKTOP
+ if (hcon==NULL) {
+ AttachConsole(ATTACH_PARENT_PROCESS); // if started from a parent console, try to attach to that
+ hcon = GetStdHandle(STD_ERROR_HANDLE);
+ }
CONSOLE_SCREEN_BUFFER_INFO sbi;
- hconIsConsole = ((hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi));
+ hconIsConsole = ((hcon != NULL && hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi));
#endif
}
const size_t len = _mi_strlen(msg);
if (len > 0 && len < UINT32_MAX) {
DWORD written = 0;
- if (hconIsConsole) {
- #ifdef MI_HAS_CONSOLE_IO
- WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL);
+ if (hcon != NULL && hcon != INVALID_HANDLE_VALUE) {
+ #if MI_WIN_DESKTOP
+ if (hconIsConsole) {
+ WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL);
+ }
+ else
#endif
+ {
+ // use direct write in case stderr was redirected
+ WriteFile(hcon, msg, (DWORD)len, &written, NULL);
+ }
}
- else if (hcon != INVALID_HANDLE_VALUE) {
- // use direct write if stderr was redirected
- WriteFile(hcon, msg, (DWORD)len, &written, NULL);
- }
- else {
- // finally fall back to fputs after all
- fputs(msg, stderr);
- }
+ // don't fall back to fputs or _cputs as the crt can have it locked
}
}
}
@@ -635,10 +681,10 @@ void _mi_prim_out_stderr( const char* msg )
// reliably even when this is invoked before the C runtime is initialized.
// i.e. when `_mi_preloading() == true`.
// Note: on windows, environment names are not case sensitive.
-bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
+int _mi_prim_getenv(const char* name, char* result, size_t result_size) {
result[0] = 0;
const size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size);
- return (len > 0 && len < result_size);
+ return (len < result_size ? (len > 0 ? 1 /* success */ : 0 /* not found */) : -1 /* error */);
}
@@ -673,10 +719,10 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {
mi_assert(buf_len <= ULONG_MAX);
if (buf_len > ULONG_MAX) return false;
mi_atomic_do_once {
- HINSTANCE hDll = LoadLibrary(TEXT("bcrypt.dll"));
+ HINSTANCE hDll = mi_win_loadlibrary(TEXT("bcrypt.dll"));
if (hDll != NULL) {
pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, "BCryptGenRandom");
- // FreeLibrary(hDll); // don't free
+ // mi_win_freelibrary(hDll); // don't free
}
}
if (pBCryptGenRandom == NULL) return false;
@@ -737,15 +783,18 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
both static and dynamic linkage (`MI_WIN_INIT_USE_CRT_TLS`).
------------------------------------------------------------------------- */
#if !defined(MI_WIN_INIT_USE_CRT_TLS) && !defined(MI_WIN_INIT_USE_RAW_DLLMAIN) && !defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && !defined(MI_WIN_INIT_USE_FLS)
- #if !defined(__INTEL_LLVM_COMPILER) && !defined(__INTEL_COMPILER)
- #define MI_WIN_INIT_USE_CRT_TLS 1
+ #if defined(__INTEL_LLVM_COMPILER) || defined(__INTEL_COMPILER)
+ #define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* needed for Intel ICX, see issue #1268 */
#else
- #define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* default for Intel ICX, see issue #1268 */
+ #define MI_WIN_INIT_USE_CRT_TLS 1 /* default */
#endif
#endif
#if defined(MI_WIN_INIT_USE_CRT_TLS)
- #define MI_PRIM_HAS_PROCESS_ATTACH 1
+ #if !defined(__MINGW32__) || defined(MI_MINGW_UCRT64) // on mingw without UCRT use the constructor attribute (in `src/prim/prim.c`)
+ #define MI_PRIM_HAS_PROCESS_ATTACH 1
+ #endif
+
// nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.
void _mi_prim_thread_init_auto_done(void) {}
void _mi_prim_thread_done_auto_done(void) {}
@@ -818,10 +867,12 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
#endif
typedef int (mi_cdecl* mi_crt_callback_t)(void);
- #if defined(_WIN64)
+
+ #if defined(_WIN64) && defined(_MSC_VER) // 64-bit
#pragma comment(linker, "/INCLUDE:_tls_used")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
+ #pragma comment(linker, "/INCLUDE:_mi_crt_callback_init")
#pragma const_seg(".CRT$XLB")
extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];
const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
@@ -834,10 +885,11 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
extern const mi_crt_callback_t _mi_crt_callback_init[];
const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
#pragma const_seg()
- #else
+ #elif defined(_MSC_VER) // 32-bit
#pragma comment(linker, "/INCLUDE:__tls_used")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
+ #pragma comment(linker, "/INCLUDE:__mi_crt_callback_init")
#pragma data_seg(".CRT$XLB")
PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
#pragma data_seg()
@@ -847,6 +899,12 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
#pragma data_seg(".CRT$XIB")
mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
#pragma data_seg()
+ #elif defined(__MINGW32__)
+ extern const IMAGE_TLS_DIRECTORY _tls_used;
+ __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls
+ __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach;
+ __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach;
+ __attribute__((used, section(".CRT$XIB"))) mi_crt_callback_t _mi_crt_callback_init = &mi_crt_init;
#endif
#if defined(__cplusplus)
@@ -916,7 +974,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
extern "C" {
#endif
- #if defined(_WIN64)
+ #if defined(_WIN64) && defined(_MSC_VER) // 64-bit
#pragma comment(linker, "/INCLUDE:_tls_used")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
@@ -928,7 +986,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];
const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
#pragma const_seg()
- #else
+ #elif defined(_MSC_VER) // 32-bit
#pragma comment(linker, "/INCLUDE:__tls_used")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
@@ -938,6 +996,11 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
#pragma data_seg(".CRT$XLY")
PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
#pragma data_seg()
+ #elif defined(__MINGW32__)
+ extern const IMAGE_TLS_DIRECTORY _tls_used;
+ __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls
+ __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach;
+ __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach;
#endif
#if defined(__cplusplus)
@@ -985,7 +1048,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
extern "C" {
#endif
- #if defined(_WIN64)
+ #if defined(_WIN64) && defined(_MSC_VER) // 64-bit
#pragma comment(linker, "/INCLUDE:_tls_used")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
@@ -997,16 +1060,21 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];
const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };
#pragma const_seg()
- #else
+ #elif defined(_MSC_VER) // 32-bit
#pragma comment(linker, "/INCLUDE:__tls_used")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
#pragma data_seg(".CRT$XLB")
PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };
#pragma data_seg()
- #pragma data_seg(".CRT$XIY")
+ #pragma data_seg(".CRT$XLY")
PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };
#pragma data_seg()
+ #elif defined(__MINGW32__)
+ extern const IMAGE_TLS_DIRECTORY _tls_used;
+ __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls
+ __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach;
+ __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach;
#endif
#if defined(__cplusplus)
@@ -1027,21 +1095,31 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
// See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm>
#define MI_PRIM_HAS_PROCESS_ATTACH 1
- static int mi_process_attach(void) {
+ static int mi_cdecl mi_crt_init(void) {
mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL);
atexit(&_mi_auto_process_done);
return 0;
}
- typedef int(*mi_crt_callback_t)(void);
- #if defined(_WIN64)
- #pragma comment(linker, "/INCLUDE:_mi_tls_callback")
- #pragma section(".CRT$XIU", long, read)
- #else
- #pragma comment(linker, "/INCLUDE:__mi_tls_callback")
+
+ #if defined(__cplusplus)
+ extern "C" {
+ #endif
+ typedef int (mi_cdecl* mi_crt_callback_t)(void);
+ #if defined(_WIN64) // 64-bit
+ #pragma comment(linker, "/INCLUDE:_mi_crt_callback_init")
+ #pragma const_seg(".CRT$XIU")
+ extern const mi_crt_callback_t _mi_crt_callback_init[];
+ const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
+ #pragma const_seg()
+ #else // 32-bit
+ #pragma comment(linker, "/INCLUDE:__mi_crt_callback_init")
+ #pragma data_seg(".CRT$XIU")
+ mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
+ #pragma data_seg()
+ #endif
+ #if defined(__cplusplus)
+ }
#endif
- #pragma data_seg(".CRT$XIU")
- mi_decl_externc mi_crt_callback_t _mi_tls_callback[] = { &mi_process_attach };
- #pragma data_seg()
#endif
// use the fiber api for calling `_mi_thread_done`.
diff --git a/vendored/mimalloc/src/random.c b/vendored/mimalloc/src/random.c
index 75b5cfbb0..3c012adf4 100644
--- a/vendored/mimalloc/src/random.c
+++ b/vendored/mimalloc/src/random.c
@@ -134,18 +134,19 @@ static bool mi_random_is_initialized(mi_random_ctx_t* ctx) {
void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) {
mi_assert_internal(mi_random_is_initialized(ctx));
- mi_assert_internal(ctx != ctx_new);
- chacha_split(ctx, (uintptr_t)ctx_new /*nonce*/, ctx_new);
+ mi_assert_internal(ctx != ctx_new);
+ const uintptr_t nonce_rnd = _mi_random_next(ctx);
+ chacha_split(ctx, (uintptr_t)ctx_new ^ nonce_rnd /*nonce*/, ctx_new);
}
-uintptr_t _mi_random_next(mi_random_ctx_t* ctx) {
+size_t _mi_random_next(mi_random_ctx_t* ctx) {
mi_assert_internal(mi_random_is_initialized(ctx));
- uintptr_t r;
+ size_t r;
do {
- #if MI_INTPTR_SIZE <= 4
+ #if MI_SIZE_SIZE <= 4
r = chacha_next32(ctx);
- #elif MI_INTPTR_SIZE == 8
- r = (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx));
+ #elif MI_SIZE_SIZE == 8
+ r = (((size_t)chacha_next32(ctx) << 32) | chacha_next32(ctx));
#else
# error "define mi_random_next for this platform"
#endif
@@ -159,12 +160,12 @@ To initialize a fresh random context.
If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR.
-----------------------------------------------------------------------------*/
-uintptr_t _mi_os_random_weak(uintptr_t extra_seed) {
- uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random
+size_t _mi_os_random_weak(size_t extra_seed) {
+ size_t x = (size_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random
x ^= _mi_prim_clock_now();
// and do a few randomization steps
- uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1;
- for (uintptr_t i = 0; i < max || x==0; i++, x++) {
+ size_t max = ((x ^ (x >> 17)) & 0x0F) + 1;
+ for (size_t i = 0; i < max || x==0; i++, x++) {
x = _mi_random_shuffle(x);
}
mi_assert_internal(x != 0);
@@ -179,7 +180,7 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) {
#if !defined(__wasi__)
if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); }
#endif
- uintptr_t x = _mi_os_random_weak(0);
+ size_t x = _mi_os_random_weak(0);
for (size_t i = 0; i < 32; i+=4, x++) {
x = _mi_random_shuffle(x);
key[i] = (uint8_t)(x);
@@ -193,6 +194,7 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) {
ctx->weak = false;
}
chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ );
+ _mi_memzero(key, sizeof(key));
}
void _mi_random_init(mi_random_ctx_t* ctx) {
diff --git a/vendored/mimalloc/src/sample-guarded.c b/vendored/mimalloc/src/sample-guarded.c
new file mode 100644
index 000000000..f7c1fe81e
--- /dev/null
+++ b/vendored/mimalloc/src/sample-guarded.c
@@ -0,0 +1,173 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2019-2026, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim-tls.h"
+
+mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) {
+ theap->guarded_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate);
+ if (theap->guarded_sample_rate > 1) {
+ if (seed == 0) {
+ seed = _mi_theap_random_next(theap);
+ }
+ theap->guarded_sample_countdown = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate`
+ }
+ else if (theap->guarded_sample_countdown > theap->guarded_sample_rate) {
+ theap->guarded_sample_countdown = theap->guarded_sample_rate;
+ }
+ _mi_theap_update_sample_rate(theap);
+}
+
+mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) {
+ theap->guarded_size_min = min;
+ theap->guarded_size_max = (min > max ? min : max);
+}
+
+void _mi_theap_guarded_init(mi_theap_t* theap) {
+ #if MI_GUARDED
+ mi_theap_guarded_set_sample_rate(theap,
+ (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX),
+ (size_t)mi_option_get(mi_option_guarded_sample_seed));
+ mi_theap_guarded_set_size_bound(theap,
+ (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX),
+ (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) );
+ #else
+ mi_theap_guarded_set_sample_rate(theap,0,0);
+ #endif
+}
+
+
+// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true).
+// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`)
+// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size`
+static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t* usable_size) {
+ // todo: we can still make padding work by moving it out of the guard page area
+ mi_page_t* const page = _mi_ptr_page(block);
+ mi_page_set_has_interior_pointers(page, true);
+ block->next = MI_BLOCK_TAG_GUARDED;
+
+ // set guard page at the end of the block
+ const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local`
+ const size_t os_page_size = _mi_os_page_size();
+ mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t));
+ if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) {
+ // should never happen
+ mi_free(block);
+ return NULL;
+ }
+ uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size;
+ // note: the alignment of the guard page relies on blocks being os_page_size aligned which
+ // is ensured in `mi_arena_page_alloc_fresh`.
+ mi_assert_internal(_mi_is_aligned(block, os_page_size));
+ mi_assert_internal(_mi_is_aligned(guard_page, os_page_size));
+ if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) {
+ const bool ok = _mi_os_protect(guard_page, os_page_size);
+ if mi_unlikely(!ok) {
+ _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size);
+ }
+ }
+ else {
+ _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size);
+ }
+
+ // align pointer just in front of the guard page
+ size_t offset = block_size - os_page_size - obj_size;
+ mi_assert_internal(offset > sizeof(mi_block_t));
+ if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) {
+ // give up to place it right in front of the guard page if the offset is too large for unalignment
+ offset = MI_PAGE_MAX_OVERALLOC_ALIGN;
+ }
+ uint8_t* const p = (uint8_t*)block + offset;
+ mi_assert_internal(p == guard_page - obj_size || offset >= MI_PAGE_MAX_OVERALLOC_ALIGN);
+ if (usable_size != NULL) { *usable_size = (guard_page - p); mi_assert_internal(mi_usable_size(p)==*usable_size); }
+ mi_track_align(block, p, offset, obj_size);
+ mi_track_mem_defined(block, sizeof(mi_block_t));
+ return p;
+}
+
+// Allocate a block with a guard page behind it.
+mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept
+{
+ // allocate multiple of page size ending in a guard page
+ // ensure minimal alignment requirement?
+ if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow
+ _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size);
+ return NULL;
+ }
+ const size_t os_page_size = _mi_os_page_size();
+ const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE));
+ const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE);
+ const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size);
+ // const size_t threshold = mi_theap_disable_profiler(theap);
+ mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap, req_size, false /* don't zero */, ppage);
+ // mi_theap_enable_profiler(theap,threshold);
+ if (block==NULL) return NULL;
+ size_t usable_size = 0;
+ void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size);
+ if (p == NULL) return NULL;
+ if (zero) {
+ _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`)
+ }
+
+ // stats
+ mi_track_malloc(p, usable_size, zero);
+ if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); }
+ mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1);
+ #if MI_STATS
+ // adjust request stats to only count the allocated size of the block (and not the guard page)
+ mi_theap_stat_counter_decrease(theap, malloc_requested, req_size);
+ mi_theap_stat_counter_increase(theap, malloc_requested, size);
+ #endif
+ #if MI_DEBUG>3
+ if (zero) {
+ mi_assert_expensive(mi_mem_is_zero(p, size));
+ }
+ #endif
+ #if MI_PAGE_META_SMALL_IS_ALIGNED && MI_DEBUG>=2
+ // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail.
+ if (size <= MI_SMALL_SIZE_MAX) {
+ mi_page_t* const page = _mi_ptr_page(p);
+ mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE);
+ }
+ #endif
+ return p;
+}
+
+
+// Remove guard page when building with MI_GUARDED
+#if MI_GUARDED
+void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) {
+ // if (!mi_block_ptr_is_guarded(block,p)) return;
+ mi_assert_internal(mi_block_ptr_is_guarded(block, p));
+ mi_assert_internal(mi_page_has_interior_pointers(page));
+ mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t));
+ mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED);
+
+ const size_t bsize = mi_page_block_size(page);
+ const size_t psize = _mi_os_page_size();
+ mi_assert_internal(bsize > psize);
+ mi_assert_internal(!page->memid.is_pinned);
+ void* gpage = (uint8_t*)block + bsize - psize;
+ mi_assert_internal(_mi_is_aligned(gpage, psize));
+ _mi_os_unprotect(gpage, psize);
+}
+
+// unguard a whole page (called from `mi_heap_destroy`)
+void _mi_page_unguard_all(mi_page_t* page) {
+ if mi_likely(!mi_page_has_interior_pointers(page)) return;
+ uint8_t* const start = mi_page_start(page);
+ const size_t psize = mi_page_committed(page);
+ _mi_os_unprotect(start,psize); // unprotect all at once as we cannot know which blocks are guarded
+}
+#else
+void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) {
+ MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p);
+}
+void _mi_page_unguard_all(mi_page_t* page) {
+ MI_UNUSED(page);
+}
+#endif
diff --git a/vendored/mimalloc/src/sample-profile.c b/vendored/mimalloc/src/sample-profile.c
new file mode 100644
index 000000000..f1b5db491
--- /dev/null
+++ b/vendored/mimalloc/src/sample-profile.c
@@ -0,0 +1,264 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2019-2026, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim-tls.h"
+
+//----------------------------------------------------------------------------
+// General sampled allocation (called from `page.c:mi_malloc_generic_fallback`)
+// This is for both guarded and profiled sampling
+//----------------------------------------------------------------------------
+
+size_t _mi_theap_update_sample_rate(mi_theap_t* theap) {
+ const size_t old_sample_rate = theap->sample_rate;
+ theap->sample_rate = theap->profile_sample_rate;
+ if (theap->sample_rate == 0 || (theap->guarded_sample_rate!=0 && theap->sample_rate > theap->guarded_sample_rate)) {
+ theap->sample_rate = theap->guarded_sample_rate;
+ }
+ if (theap->sample_countdown > theap->sample_rate) {
+ theap->sample_countdown = theap->sample_rate; // todo: adjust difference?
+ }
+ return old_sample_rate;
+}
+
+// Called when the theap sample countdown <= req_size; should call guarded or profiled allocation.
+mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept
+{
+ // the size has not yet been counted against the countdown (and does not include MI_PADDING_SIZE)
+ mi_assert_internal(req_size <= MI_MAX_ALLOC_SIZE);
+ mi_assert_internal((theap->sample_countdown==SIZE_MAX && _mi_is_empty_theap(theap)) ||
+ (theap->sample_countdown <= MI_SAMPLE_COUNTDOWN_MAX && theap->sample_countdown < req_size));
+ const size_t size = req_size + MI_PADDING_SIZE;
+
+ // handle empty theap and disabled sampling
+ if (theap->sample_rate==0) {
+ if (!_mi_is_empty_theap(theap)) { // avoid writing to the initial empty theap
+ theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // avoid the sampling path for a long time
+ }
+ return _mi_malloc_generic_no_sample(theap,size,zero,ppage);
+ }
+
+ // update countdown and total accummulated requested bytes since the last sample
+ mi_assert_internal(!_mi_is_empty_theap(theap));
+ mi_assert_internal(req_size <= SIZE_MAX/2);
+ mi_assert_internal(theap->sample_rate > 0);
+ mi_assert_internal(theap->sample_rate <= SIZE_MAX/2);
+ mi_assert_internal(theap->sample_rate >= theap->sample_countdown);
+
+ const uint64_t requested = theap->sample_requested = (uint64_t)theap->sample_rate + (uint64_t)(req_size - theap->sample_countdown) + theap->sample_requested;
+ mi_assert_internal(requested > 0);
+ mi_assert_internal(requested >= req_size);
+ theap->sample_countdown = theap->sample_rate; // reset sampling
+
+ // update derived countdowns
+ mi_assert_internal(theap->profile_sample_rate!=0 || theap->profile_sample_countdown==0);
+ mi_assert_internal(theap->guarded_sample_rate!=0 || theap->guarded_sample_countdown==0);
+ bool sample_profile = false;
+ bool sample_guarded = false;
+ if (theap->profile_sample_rate!=0) {
+ if (theap->profile_sample_countdown >= requested) { theap->profile_sample_countdown -= (size_t)requested; } else { sample_profile = true; }
+ }
+ if (theap->guarded_sample_rate!=0) {
+ if (theap->guarded_sample_countdown >= requested) { theap->guarded_sample_countdown -= (size_t)requested; } else { sample_guarded = true; }
+ }
+
+ // invoke callback?
+ if (sample_profile) {
+ theap->profile_sample_countdown = theap->profile_sample_rate; // reset countdown
+ theap->sample_requested = 0; // reset requested as we pass it to malloc_profiled
+ #if MI_PROFILE
+ return _mi_theap_malloc_profiled(theap,size,requested,zero,ppage);
+ #endif
+ }
+ else if (sample_guarded) {
+ if (req_size >= theap->guarded_size_min && req_size <= theap->guarded_size_max) {
+ // use guarded allocation
+ theap->guarded_sample_countdown = theap->guarded_sample_rate; // reset countdown
+ #if MI_GUARDED
+ return _mi_theap_malloc_guarded(theap,size,zero,ppage);
+ #endif
+ }
+ #if MI_GUARDED
+ else {
+ // failed size criteria, rewind the sample countdown so we sample asap again
+ // todo: can we do better here as this will cause many samples until it fits the size..
+ theap->sample_countdown = 0;
+ }
+ #endif
+ }
+ // take generic path
+ return _mi_malloc_generic_no_sample(theap,size,zero,ppage);
+}
+
+
+
+//----------------------------------------------------------------------------
+// Util
+//----------------------------------------------------------------------------
+
+static mi_profiler_t* mi_heap_profiler(const mi_heap_t* heap) {
+ return mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler);
+}
+
+static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) {
+ mi_heap_t* const heap = _mi_theap_heap(theap);
+ mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler);
+ if (prof!=NULL && mi_profiler_is_enabled(prof) && prof->on_alloc != NULL) {
+ return prof;
+ }
+ else {
+ return NULL;
+ }
+}
+
+//----------------------------------------------------------------------------
+// Profile an allocation
+//-----------------------------------------------------------------------------
+
+mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept
+{
+ mi_assert_internal(theap!=NULL);
+ mi_assert_internal(size>=MI_PADDING_SIZE);
+ const size_t req_size = size - MI_PADDING_SIZE;
+ mi_assert_internal(req_size<=requested_since_last_sample);
+ mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap);
+ if (prof == NULL) { return _mi_malloc_generic_no_sample(theap,size,zero,ppage); }
+ mi_assert_internal(prof!=NULL && prof->on_alloc!=NULL && mi_profiler_is_enabled(prof));
+
+ void* p = NULL;
+ size_t new_sample_rate = 0;
+ if (prof->on_free==NULL || prof->sample_data_size==0) {
+ // just allocate without profiler data
+ p = _mi_malloc_generic_no_sample(theap,size,zero,ppage);
+ if (p==NULL) { return p; }
+ if (prof->on_alloc!=NULL) {
+ // we are just allocation profiling (not heap profiling as on_free == NULL)
+ new_sample_rate = (*prof->on_alloc)(prof, NULL /* no data */, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) );
+ }
+ }
+ else {
+ // overallocate a larger block to store the profiler data
+ // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...]
+ const size_t sample_data_offset = sizeof(mi_block_t);
+ const size_t sample_user_data_size = _mi_align_up(prof->sample_data_size > MI_PROFILE_SAMPLE_DATA_MAX_SIZE ? MI_PROFILE_SAMPLE_DATA_MAX_SIZE : prof->sample_data_size, sizeof(void*));
+ const size_t sample_data_size = sizeof(mi_profiler_sample_data_t) + sample_user_data_size; // one void* too many just in case
+ const size_t user_offset = _mi_align_up(sample_data_offset + sample_data_size, MI_MAX_ALIGN_SIZE);
+ const size_t oversize = user_offset + size;
+ mi_page_t* page = NULL;
+ mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap,oversize,zero,&page);
+ if (block==NULL) return NULL;
+ mi_assert_internal(page!=NULL);
+ if (ppage!=NULL) { *ppage = page; }
+ #if MI_PAGE_META_SMALL_IS_ALIGNED
+ // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail.
+ // (that is why we need to limit the profiler_data_size as well)
+ // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail.
+ if (size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); }
+ #endif
+
+ // Set up the profiled block as an interior pointer so the interior "slow" path is taken on mi_free (where we catch it to call on_free)
+ mi_page_set_has_interior_pointers(page, true);
+ block->next = MI_BLOCK_TAG_PROFILED;
+ p = (uint8_t*)block + user_offset;
+ mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sample_data_offset);
+ sample_data->user_data_size = sample_user_data_size;
+
+ // and call the profiler on_alloc
+ if (prof->on_alloc!=NULL) {
+ new_sample_rate = (*prof->on_alloc)(prof, sample_data, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) );
+ }
+ }
+ if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) {
+ _mi_theap_set_profile_sample_rate(theap,new_sample_rate);
+ }
+ mi_theap_stat_counter_increase(theap,profile_samples,1);
+ return p;
+}
+
+void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p) {
+ mi_assert_internal(mi_block_ptr_is_sampled(block,p));
+
+ // get the heap and profiler
+ mi_heap_t* const heap = mi_page_heap(page);
+ if (heap==NULL) return;
+ mi_profiler_t* prof = mi_heap_profiler(heap);
+ if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL) return;
+
+ // call the on_free callback
+ mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sizeof(mi_block_t));
+ (*prof->on_free)(prof, sample_data, p, heap);
+}
+
+
+//----------------------------------------------------------------------------
+// Profiling API
+//-----------------------------------------------------------------------------*
+
+size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate) {
+ const size_t old_sample_rate = theap->profile_sample_rate;
+ theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate);
+ if (theap->profile_sample_countdown > theap->profile_sample_rate) {
+ theap->profile_sample_countdown = theap->profile_sample_rate; // todo: adjust difference?
+ }
+ _mi_theap_update_sample_rate(theap);
+ return old_sample_rate;
+}
+
+static bool mi_heap_set_profiler(mi_heap_t* heap, mi_profiler_t* profiler) {
+ mi_profiler_t* previous = (profiler==NULL ? mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler) : NULL); // don't overwrite unless it is NULL
+ return mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t, &heap->profiler, &previous, profiler);
+}
+
+mi_decl_export bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler) {
+ mi_profiler_stop(profiler);
+ return mi_heap_set_profiler(heap,profiler);
+}
+
+mi_decl_export void mi_heap_profile_disable(mi_heap_t* heap) {
+ static mi_profiler_t mi_nosample_profiler = { NULL, 0, MI_SAMPLE_RATE_MAX, NULL, NULL, NULL };
+ mi_heap_set_profiler(heap,NULL); // first clear any previous one
+ mi_heap_set_profiler(heap,&mi_nosample_profiler); // and then install the nosample one
+}
+
+mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler) {
+ mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
+ if (subproc==NULL) return false;
+ mi_profiler_stop(profiler);
+ mi_profiler_t* previous = (profiler==NULL ? mi_atomic_load_ptr_acquire(mi_profiler_t,&subproc->profiler) : NULL); // don't overwrite unless it is NULL
+ if (!mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t,&subproc->profiler, &previous, profiler)) { return false; }
+ mi_lock(&subproc->heaps_lock) {
+ for (mi_heap_t* heap = subproc->heaps; heap!=NULL; heap = heap->next) {
+ mi_heap_set_profiler(heap,profiler);
+ }
+ }
+ return true;
+}
+
+mi_decl_export bool mi_profile( mi_profiler_t* profiler) {
+ return mi_subproc_profile(mi_subproc_main(),profiler);
+}
+
+bool mi_profiler_start(mi_profiler_t* profiler ) {
+ if (profiler==NULL) return false;
+ const bool was_running = mi_profiler_set_enabled(profiler,true);
+ if (was_running) return true;
+ // for the main heap, if this is the profiler, start the theap more aggressively
+ // otherwise it will be picked up when theaps take the slow generic malloc path.
+ mi_heap_t* heap = mi_heap_main();
+ if (mi_heap_profiler(heap)==profiler) {
+ mi_theap_t* theap = _mi_heap_theap_peek(heap);
+ if (theap!=NULL) {
+ _mi_theap_set_profile_sample_rate(theap,1);
+ }
+ }
+ return false;
+}
+
+bool mi_profiler_stop(mi_profiler_t* profiler) {
+ if (profiler==NULL) return true;
+ return mi_profiler_set_enabled(profiler,false);
+}
diff --git a/vendored/mimalloc/src/static.c b/vendored/mimalloc/src/static.c
index 2383f6596..8a4f8c3d5 100644
--- a/vendored/mimalloc/src/static.c
+++ b/vendored/mimalloc/src/static.c
@@ -24,20 +24,23 @@ terms of the MIT license. A copy of the license can be found in the file
#include "alloc-aligned.c"
#include "alloc-posix.c"
#include "arena.c"
-#include "arena-meta.c"
#include "bitmap.c"
#include "heap.c"
#include "init.c"
#include "libc.c"
#include "options.c"
#include "os.c"
+#include "prim/prim.c" // includes platform specific prim/<platform>/prim.c
+#include "prim/prim-tls.c"
+#if MI_OSX_ZONE
+#include "prim/osx/alloc-override-zone.c"
+#endif
#include "page.c" // includes page-queue.c
#include "page-map.c"
#include "random.c"
+#include "sample-guarded.c"
+#include "sample-profile.c"
#include "stats.c"
+#include "subproc.c"
#include "theap.c"
#include "threadlocal.c"
-#include "prim/prim.c"
-#if MI_OSX_ZONE
-#include "prim/osx/alloc-override-zone.c"
-#endif
diff --git a/vendored/mimalloc/src/stats.c b/vendored/mimalloc/src/stats.c
index 7ca18e238..c763bfc3c 100644
--- a/vendored/mimalloc/src/stats.c
+++ b/vendored/mimalloc/src/stats.c
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -8,7 +8,8 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc-stats.h"
#include "mimalloc/internal.h"
#include "mimalloc/atomic.h"
-#include "mimalloc/prim.h"
+#include "mimalloc/prim.h" // _mi_prim_clock_now, mi_process_info_t
+#include "mimalloc/prim-tls.h"
#include <string.h> // memset
@@ -38,26 +39,21 @@ static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
if (amount > 0) { stat->total += amount; }
}
-
-void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount) {
- mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount);
-}
-
-void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {
- stat->total += amount;
-}
-
-void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount) {
+void __mi_stat_increase_mt(mi_stat_count_t* stat, uint64_t amount) {
+ mi_assert_internal(amount<=INT64_MAX);
mi_stat_update_mt(stat, (int64_t)amount);
}
-void __mi_stat_increase(mi_stat_count_t* stat, size_t amount) {
+void __mi_stat_increase(mi_stat_count_t* stat, uint64_t amount) {
+ mi_assert_internal(amount<=INT64_MAX);
mi_stat_update(stat, (int64_t)amount);
}
-void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount) {
+void __mi_stat_decrease_mt(mi_stat_count_t* stat, uint64_t amount) {
+ mi_assert_internal(amount<=INT64_MAX);
mi_stat_update_mt(stat, -((int64_t)amount));
}
-void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) {
+void __mi_stat_decrease(mi_stat_count_t* stat, uint64_t amount) {
+ mi_assert_internal(amount<=INT64_MAX);
mi_stat_update(stat, -((int64_t)amount));
}
@@ -68,26 +64,33 @@ void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) {
static void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount) {
if (amount == 0) return;
// adjust atomically
+ const int64_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&stat->peak);
mi_atomic_addi64_relaxed(&stat->current, amount);
- mi_atomic_addi64_relaxed(&stat->total, amount);
+ const int64_t prev_total = mi_atomic_addi64_relaxed(&stat->total, amount);
+ if (prev_total == peak) { mi_atomic_addi64_relaxed(&stat->peak, amount); }
}
static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) {
if (amount == 0) return;
stat->current += amount;
- stat->total += amount;
+ if (stat->total==stat->peak) { stat->peak += amount; }
+ stat->total += amount;
}
-void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount) {
+void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, uint64_t amount) {
+ mi_assert_internal(amount<=INT64_MAX);
mi_stat_adjust_mt(stat, (int64_t)amount);
}
-void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount) {
+void __mi_stat_adjust_increase(mi_stat_count_t* stat, uint64_t amount) {
+ mi_assert_internal(amount<=INT64_MAX);
mi_stat_adjust(stat, (int64_t)amount);
}
-void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount) {
+void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, uint64_t amount) {
+ mi_assert_internal(amount<=INT64_MAX);
mi_stat_adjust_mt(stat, -((int64_t)amount));
}
-void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) {
+void __mi_stat_adjust_decrease(mi_stat_count_t* stat, uint64_t amount) {
+ mi_assert_internal(amount<=INT64_MAX);
mi_stat_adjust(stat, -((int64_t)amount));
}
@@ -96,15 +99,17 @@ void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) {
static void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* src) {
if (stat==src) return;
mi_atomic_void_addi64_relaxed(&stat->total, &src->total);
- const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src->current);
+ const int64_t src_peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&src->peak);
+ const int64_t src_current = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&src->current);
+ const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src_current);
// Global current plus thread peak approximates new global peak
- // note: peak scores do really not work across threads.
+ // note: peak scores do not really work across threads.
// we used to just add them together but that often overestimates in practice.
// similarly, max does not seem to work well. The current approach
// by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112
- // for a longer description.
- mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src->peak);
+ // for a longer description.
+ mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src_peak);
}
static void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counter_t* src) {
@@ -122,7 +127,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
// copy all fields
MI_STAT_FIELDS()
- #if MI_STAT>1
+ #if MI_STATS
for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
mi_stat_count_add_mt(&stats->malloc_bins[i], &src->malloc_bins[i]);
}
@@ -130,6 +135,9 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
mi_stat_count_add_mt(&stats->page_bins[i], &src->page_bins[i]);
}
+ for (size_t i = 0; i < MI_CBIN_COUNT; i++) {
+ mi_stat_count_add_mt(&stats->chunk_bins[i], &src->chunk_bins[i]);
+ }
}
#undef MI_STAT_COUNT
@@ -142,7 +150,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
// unit > 0 : size in binary bytes
// unit == 0: count as decimal
// unit < 0 : count in binary
-static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) {
+static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, bool limitwidth) {
char buf[32]; _mi_memzero_var(buf);
int len = 32;
const char* suffix = (unit <= 0 ? " " : "B");
@@ -167,12 +175,17 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void*
_mi_snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix);
_mi_snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc);
}
- _mi_fprintf(out, arg, (fmt==NULL ? "%12s" : fmt), buf);
+ if (limitwidth) {
+ _mi_fprintf(out, arg, "%12s", buf);
+ }
+ else {
+ _mi_fprintf(out, arg, "%s", buf);
+ }
}
static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {
- mi_printf_amount(n,unit,out,arg,NULL);
+ mi_printf_amount(n,unit,out,arg,true);
}
static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) {
@@ -181,32 +194,31 @@ static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* ar
}
static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) {
- _mi_fprintf(out, arg," %-10s:", msg);
+ _mi_fprintf(out, arg," %-12s:", msg);
if (unit != 0) {
- if (unit > 0) {
+ if (unit > 0) { // as KiB (power of two)
mi_print_amount(stat->peak, unit, out, arg);
mi_print_amount(stat->total, unit, out, arg);
- // mi_print_amount(stat->freed, unit, out, arg);
mi_print_amount(stat->current, unit, out, arg);
mi_print_amount(unit, 1, out, arg);
mi_print_count(stat->total, unit, out, arg);
}
- else {
+ else if (unit==-1) { // as K (decimal)
mi_print_amount(stat->peak, -1, out, arg);
mi_print_amount(stat->total, -1, out, arg);
- // mi_print_amount(stat->freed, -1, out, arg);
mi_print_amount(stat->current, -1, out, arg);
- if (unit == -1) {
- _mi_fprintf(out, arg, "%24s", "");
- }
- else {
- mi_print_amount(-unit, 1, out, arg);
- mi_print_count((stat->total / -unit), 0, out, arg);
- }
+ _mi_fprintf(out, arg, "%24s", "");
+ }
+ else { // as KiB , unit is final count
+ mi_print_amount(stat->peak, 1, out, arg);
+ mi_print_amount(stat->total, 1, out, arg);
+ mi_print_amount(stat->current, -1, out, arg);
+ _mi_fprintf(out, arg, "%12s", "");
+ mi_print_count(-unit, 0, out, arg);
}
if (stat->current != 0) {
_mi_fprintf(out, arg, " ");
- _mi_fprintf(out, arg, (notok == NULL ? "not all freed" : notok));
+ _mi_fprintf(out, arg, "%s", (notok == NULL ? "not all freed" : notok));
_mi_fprintf(out, arg, "\n");
}
else {
@@ -225,23 +237,23 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t
mi_stat_print_ex(stat, msg, unit, out, arg, NULL);
}
-#if MI_STAT>1
-static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {
- _mi_fprintf(out, arg, " %-10s:", msg);
+#if MI_STATS
+static void mi_stat_total_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) {
+ _mi_fprintf(out, arg, " %-12s:", msg);
_mi_fprintf(out, arg, "%12s", " "); // no peak
- mi_print_amount(stat->total, unit, out, arg);
+ mi_print_amount(stat->total, 1, out, arg);
_mi_fprintf(out, arg, "\n");
}
#endif
static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {
- _mi_fprintf(out, arg, " %-10s:", msg);
+ _mi_fprintf(out, arg, " %-12s:", msg);
mi_print_amount(stat->total, 0, out, arg);
_mi_fprintf(out, arg, "\n");
}
static void mi_stat_counter_print_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {
- _mi_fprintf(out, arg, " %-10s:", msg);
+ _mi_fprintf(out, arg, " %-12s:", msg);
mi_print_amount(stat->total, 1, out, arg);
_mi_fprintf(out, arg, "\n");
}
@@ -250,16 +262,16 @@ static void mi_stat_average_print(int64_t count, int64_t total, const char* msg,
const int64_t avg_tens = (count == 0 ? 0 : (total*10 / count));
const int64_t avg_whole = avg_tens/10;
const int64_t avg_frac1 = avg_tens%10;
- _mi_fprintf(out, arg, " %-10s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1);
+ _mi_fprintf(out, arg, " %-12s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1);
}
static void mi_print_header(const char* name,mi_output_fun* out, void* arg ) {
- _mi_fprintf(out, arg, " %-11s %11s %11s %11s %11s %11s\n",
+ _mi_fprintf(out, arg, " %-13s %11s %11s %11s %11s %11s\n",
name, "peak ", "total ", "current ", "block ", "total# ");
}
-#if MI_STAT>1
+#if MI_STATS
static bool mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, mi_output_fun* out, void* arg) {
bool found = false;
char buf[64];
@@ -328,13 +340,13 @@ mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_
size_t peak_commit;
size_t page_faults;
mi_process_info(&elapsed, &user_time, &sys_time, &current_rss, &peak_rss, &current_commit, &peak_commit, &page_faults);
- _mi_fprintf(out, arg, " %-10s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000);
- _mi_fprintf(out, arg, " %-10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process",
+ _mi_fprintf(out, arg, " %-12s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000);
+ _mi_fprintf(out, arg, " %-12s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process",
user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults);
- mi_printf_amount((int64_t)peak_rss, 1, out, arg, "%s");
+ mi_printf_amount((int64_t)peak_rss, 1, out, arg, false);
if (peak_commit > 0) {
_mi_fprintf(out, arg, ", peak commit: ");
- mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s");
+ mi_printf_amount((int64_t)peak_commit, 1, out, arg, false);
}
_mi_fprintf(out, arg, "\n");
}
@@ -351,19 +363,21 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou
_mi_fprintf(out, arg, "%s %zu\n", name, id);
if (stats->malloc_normal.total + stats->malloc_huge.total != 0) {
- #if MI_STAT>1
+ #if MI_STATS
mi_print_header("blocks", out, arg);
mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, out, arg);
#endif
- #if MI_STAT
- mi_stat_print(&stats->malloc_normal, "binned", (stats->malloc_normal_count.total == 0 ? -1 : 1), out, arg);
- mi_stat_print(&stats->malloc_huge, "huge", (stats->malloc_huge_count.total == 0 ? -1 : 1), out, arg);
+ #if MI_STATS
+ mi_stat_print(&stats->malloc_normal, "binned", -stats->malloc_normal_count.total, out, arg);
+ mi_stat_print(&stats->malloc_huge, "huge", -stats->malloc_huge_count.total, out, arg);
mi_stat_count_t total = { 0,0,0 };
mi_stat_count_add_mt(&total, &stats->malloc_normal);
mi_stat_count_add_mt(&total, &stats->malloc_huge);
- mi_stat_print_ex(&total, "total", 1, out, arg, "");
- #if MI_STAT>1
- mi_stat_total_print(&stats->malloc_requested, "malloc req", 1, out, arg);
+ mi_stat_print_ex(&total, "total", -(stats->malloc_normal_count.total + stats->malloc_huge_count.total), out, arg, "");
+ #if MI_STATS>=2
+ mi_stat_total_print(&stats->malloc_requested, "malloc req", out, arg);
+ #else
+ mi_stat_total_print(&stats->malloc_requested, "malloc req~", out, arg);
#endif
_mi_fprintf(out, arg, "\n");
#endif
@@ -377,13 +391,18 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou
// mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg);
mi_stat_print(&stats->pages, "pages", 0, out, arg);
mi_stat_print(&stats->pages_abandoned, "abandoned", 0, out, arg);
+ mi_stat_print(&stats->pages_os_abandoned, "os abandoned", 0, out, arg);
+ mi_stat_print(&stats->pages_os_allocated, "os allocated", 0, out, arg);
mi_stat_counter_print(&stats->pages_reclaim_on_alloc, "reclaima", out, arg);
mi_stat_counter_print(&stats->pages_reclaim_on_free, "reclaimf", out, arg);
mi_stat_counter_print(&stats->pages_reabandon_full, "reabandon", out, arg);
mi_stat_counter_print(&stats->pages_unabandon_busy_wait, "waits", out, arg);
mi_stat_counter_print(&stats->pages_extended, "extended", out, arg);
mi_stat_counter_print(&stats->pages_retire, "retire", out, arg);
+ mi_stat_counter_print(&stats->pages_stat_updates, "stat updates", out, arg);
+ mi_stat_average_print(stats->pages_stat_updates.total, stats->pages_stat_update_count.total, "stat avg", out, arg);
mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg);
+ mi_stat_counter_print(&stats->profile_samples, "prof samples", out, arg);
_mi_fprintf(out, arg, "\n");
}
@@ -405,12 +424,13 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou
mi_stat_print_ex(&stats->heaps, "heaps", 0, out, arg, "");
mi_stat_counter_print(&stats->heaps_delete_wait, "heap waits", out, arg);
_mi_fprintf(out, arg, "\n");
-
- mi_print_header("process", out, arg);
- mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, "");
- _mi_fprintf(out, arg, " %-10s: %5i\n", "numa nodes", _mi_os_numa_node_count());
- mi_process_info_print_out(out, arg);
}
+
+ mi_print_header("process", out, arg);
+ mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, "");
+ _mi_fprintf(out, arg, " %-12s: %5i\n", "numa nodes", _mi_os_numa_node_count());
+ mi_process_info_print_out(out, arg);
+
_mi_fprintf(out, arg, "\n");
}
@@ -432,7 +452,7 @@ void _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from) {
mi_assert_internal(to != NULL && from != NULL);
if (to == from) return;
mi_stats_add(to, from);
- _mi_memzero(from, sizeof(mi_stats_t));
+ mi_stats_init(from); // zero field and keep the header
}
static const mi_stats_t* mi_stats_merge_theap_to_heap(mi_theap_t* theap) mi_attr_noexcept {
@@ -449,11 +469,16 @@ static const mi_stats_t* mi_heap_get_stats(mi_heap_t* heap) {
else return mi_stats_merge_theap_to_heap(theap);
}
+static const mi_stats_t* mi_theap_get_stats(mi_theap_t* theap) {
+ return &theap->stats;
+}
+
// deprecated
void mi_stats_reset(void) mi_attr_noexcept {
if (!mi_theap_is_initialized(_mi_theap_default())) return;
- mi_heap_get_stats(mi_heap_main());
- mi_heap_stats_merge_to_subproc(mi_heap_main());
+ mi_heap_t* heap_main = mi_heap_main();
+ mi_heap_get_stats(heap_main);
+ mi_heap_stats_merge_to_subproc(heap_main);
}
@@ -480,6 +505,10 @@ void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun*
if (subproc==NULL) return;
mi_heap_print_visit_info_t vinfo = { out, arg };
mi_subproc_visit_heaps(subproc_id, &mi_heap_print_visitor, &vinfo);
+ if (subproc->theap_meta!=NULL) {
+ _mi_stats_print("meta", subproc->subproc_seq, &subproc->theap_meta->stats, out, arg);
+ }
+ // if (subproc->theap_meta!=NULL) { _mi_stats_merge_into(&subproc->stats, &subproc->theap_meta->stats); }
_mi_stats_print("subproc", subproc->subproc_seq, &subproc->stats, out, arg);
}
@@ -548,8 +577,10 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, s
pinfo.elapsed = _mi_clock_end(mi_process_start);
{ const mi_subproc_t* subproc = _mi_subproc_main();
if (subproc!=NULL) {
- pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current)));
- pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak)));
+ const int64_t current = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current));
+ const int64_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak));
+ pinfo.current_commit = (current < 0 ? 0 : (current < PTRDIFF_MAX ? (size_t)current : PTRDIFF_MAX));
+ pinfo.peak_commit = (peak < 0 ? 0 : (peak < PTRDIFF_MAX ? (size_t)peak : PTRDIFF_MAX));
}
}
pinfo.current_rss = pinfo.current_commit;
@@ -601,6 +632,14 @@ bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept {
return mi_stats_copy(stats, mi_heap_get_stats(heap));
}
+bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept {
+ return mi_stats_copy(stats, mi_theap_get_stats(theap));
+}
+
+void mi_theap_stats_merge_to_heap(mi_theap_t* theap) mi_attr_noexcept {
+ if (theap == NULL) return;
+ _mi_theap_merge_stats(theap);
+}
static bool mi_cdecl mi_heap_aggregate_visitor(mi_heap_t* heap, void* arg) {
mi_stats_t* stats = (mi_stats_t*)arg;
@@ -614,6 +653,8 @@ bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr
if (subproc == NULL) return false;
if (!mi_stats_copy(stats, &subproc->stats)) return false;
mi_subproc_visit_heaps(subproc_id, &mi_heap_aggregate_visitor, stats);
+ // hide meta data stats
+ // if (subproc->theap_meta!=NULL) { mi_stats_add_into(stats, &subproc->theap_meta->stats); }
return true;
}
@@ -722,9 +763,6 @@ static void mi_json_buf_print_counter_value(mi_json_buf_t* hbuf, const char* nam
mi_json_buf_print_value(hbuf, name, stat->total);
}
-#define MI_STAT_COUNT(stat) mi_json_buf_print_count_value(&hbuf, #stat, &stats->stat);
-#define MI_STAT_COUNTER(stat) mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat);
-
static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, char* output_buf) mi_attr_noexcept {
if (stats==NULL || stats->size!=sizeof(mi_stats_t) || stats->version!=MI_STAT_VERSION) return NULL;
mi_json_buf_t hbuf = { NULL, 0, 0, true };
@@ -763,7 +801,13 @@ static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size,
mi_json_buf_print(&hbuf, " },\n");
// statistics
+ #define MI_STAT_COUNT(stat) mi_json_buf_print_count_value(&hbuf, #stat, &stats->stat);
+ #define MI_STAT_COUNTER(stat) mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat);
+
MI_STAT_FIELDS()
+
+ #undef MI_STAT_COUNT
+ #undef MI_STAT_COUNTER
// size bins
mi_json_buf_print(&hbuf, " \"malloc_bins\": [\n");
@@ -782,7 +826,7 @@ static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size,
}
mi_json_buf_print(&hbuf, " ]\n");
mi_json_buf_print(&hbuf, "}\n");
- if (hbuf.used >= hbuf.size) {
+ if (hbuf.used + 1 >= hbuf.size) {
// failed
if (hbuf.can_realloc) { mi_free(hbuf.buf); }
return NULL;
diff --git a/vendored/mimalloc/src/subproc.c b/vendored/mimalloc/src/subproc.c
new file mode 100644
index 000000000..f42d4a11c
--- /dev/null
+++ b/vendored/mimalloc/src/subproc.c
@@ -0,0 +1,327 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim-tls.h"
+
+// pre-allocate the main subprocess structure.
+static mi_decl_cache_align mi_subproc_t mi_process_subproc_main = mi_init_struct_zero;
+static mi_subproc_t* mi_subprocs = NULL;
+static mi_lock_t mi_subprocs_lock = MI_LOCK_INITIALIZER;
+
+
+/* -----------------------------------------------------------
+ Meta-data allocation
+ We allocate thread local data and theaps through a dedicated
+ theap `subproc.theap_meta` which uses a detached tld with
+ a detached thread id. The initial theap_meta is statically
+ allocated and can thus be used to allocate on an as yet
+ uninitialized thread or process.
+ We need to take a lock though to allocate safely on the
+ detached `theap_meta`.
+----------------------------------------------------------- */
+
+void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid ) {
+ mi_assert_internal(subproc->theap_meta != NULL);
+ void* p = NULL;
+ mi_lock(&subproc->theap_meta_lock) {
+ p = mi_theap_zalloc(subproc->theap_meta, size);
+ if (memid != NULL) { *memid = (p==NULL ? _mi_memid_none() : _mi_memid_create_malloc(p,size,true) ); }
+ }
+ return p;
+}
+
+void* _mi_meta_zalloc_aligned( mi_subproc_t* subproc, size_t size, size_t aligned, mi_memid_t* memid ) {
+ mi_assert_internal(subproc->theap_meta != NULL);
+ void* p = NULL;
+ mi_lock(&subproc->theap_meta_lock) {
+ p = mi_theap_zalloc_aligned(subproc->theap_meta, size, aligned);
+ if (memid != NULL) { *memid = (p==NULL ? _mi_memid_none() : _mi_memid_create_malloc(p,size,true) ); }
+ }
+ return p;
+}
+
+void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* oldp, size_t newsize, mi_memid_t* memid ) {
+ mi_assert_internal(subproc->theap_meta != NULL);
+ // note: since we take a meta lock we cannot use `mi_theap_rezalloc` as that could call `mi_free` which
+ // can call `mi_stat_free` which would try to take the meta lock again. See issue #1358.
+ void* p = NULL;
+ mi_lock(&subproc->theap_meta_lock) {
+ p = mi_theap_zalloc(subproc->theap_meta, newsize);
+ }
+ if (p!=NULL) {
+ if (oldp!=NULL) {
+ const size_t oldsize = mi_usable_size(oldp);
+ const size_t copysize = (newsize < oldsize ? newsize : oldsize);
+ _mi_memcpy(p,oldp,copysize);
+ if (memid!=NULL) { _mi_meta_free(subproc,oldp,*memid); }
+ else { mi_free(oldp); }
+ }
+ if (memid!=NULL) { *memid = _mi_memid_create_malloc(p,newsize,true); }
+ }
+ else {
+ if (memid!=NULL) { *memid = _mi_memid_none(); }
+ }
+ return p;
+}
+
+void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid) {
+ if (p==NULL || mi_memid_needs_no_free(memid)) return;
+ if (memid.memkind == MI_MEM_MALLOC) {
+ mi_free(p);
+ }
+ else {
+ mi_assert_internal(subproc!=NULL);
+ _mi_arenas_free(subproc, p, _mi_memid_size(memid), memid);
+ }
+}
+
+bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* page) {
+ if (page==NULL) return false;
+ mi_theap_t* theap = page->theap;
+ return (theap != NULL && theap == subproc->theap_meta);
+}
+
+
+/* -----------------------------------------------------------
+ Sub process helpers
+----------------------------------------------------------- */
+
+mi_subproc_t* _mi_subproc_main(void) {
+ return &mi_process_subproc_main;
+}
+
+bool _mi_subproc_is_main(mi_subproc_t* subproc) {
+ return (subproc == &mi_process_subproc_main);
+}
+
+mi_subproc_t* _mi_subproc(void) {
+ mi_theap_t* theap = _mi_theap_default();
+ if (theap == NULL || theap->tld == NULL || theap->tld->subproc == NULL) { // see issue #1289 and #1391
+ return _mi_subproc_main();
+ }
+ else {
+ return theap->tld->subproc;
+ }
+}
+
+mi_heap_t* mi_heap_main(void) {
+ return _mi_subproc_heap_main(_mi_subproc()); // don't use mi_theap_main_init_get() so this call works during process_init
+}
+
+
+mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) {
+ return (mi_subproc_t*)(subproc_id._mi_subproc_id);
+}
+
+mi_subproc_id_t _mi_subproc_to_id(mi_subproc_t* subproc) {
+ mi_subproc_id_t id = { subproc };
+ return id;
+}
+
+mi_subproc_id_t mi_subproc_main(void) {
+ return _mi_subproc_to_id(_mi_subproc_main());
+}
+
+mi_subproc_id_t mi_subproc_current(void) {
+ return _mi_subproc_to_id(_mi_subproc());
+}
+
+
+/* -----------------------------------------------------------
+ Sub process creation
+----------------------------------------------------------- */
+
+
+static mi_subproc_t* mi_subproc_init(mi_subproc_t* subproc, mi_subproc_t* parent) {
+ static _Atomic(size_t) subproc_total_count;
+ subproc->parent = parent;
+ subproc->subproc_seq = mi_atomic_increment_relaxed(&subproc_total_count);
+ mi_stats_header_init(&subproc->stats);
+ mi_lock_init(&subproc->arena_reserve_lock);
+ mi_lock_init(&subproc->heaps_lock);
+ mi_lock_init(&subproc->theap_meta_lock);
+ mi_lock(&mi_subprocs_lock) {
+ // push on subproc list
+ subproc->next = mi_subprocs;
+ if (mi_subprocs!=NULL) { mi_subprocs->prev = subproc; }
+ mi_subprocs = subproc;
+ }
+ return subproc;
+}
+
+mi_subproc_id_t mi_subproc_new(void) {
+ mi_thread_init();
+ mi_subproc_t* const parent = _mi_subproc();
+ mi_memid_t memid;
+ mi_subproc_t* const subproc = (mi_subproc_t*)_mi_meta_zalloc(parent, sizeof(mi_subproc_t), &memid);
+ if (subproc == NULL) { return _mi_subproc_to_id(NULL); }
+ subproc->memid = memid;
+
+ mi_memid_t theap_memid;
+ mi_theap_t* const theap_meta = (mi_theap_t*)_mi_meta_zalloc(parent, sizeof(mi_theap_t), &theap_memid);
+ if (theap_meta==NULL) {
+ _mi_meta_free(parent, subproc, memid);
+ return _mi_subproc_to_id(NULL);
+ }
+ theap_meta->memid = memid;
+
+ // init subproc
+ mi_subproc_init(subproc,parent);
+
+ // init main heap
+ mi_heap_t* heap_main = _mi_heap_new_for_subproc(subproc,0,true);
+ if (heap_main==NULL) {
+ _mi_meta_free(parent, theap_meta, theap_meta->memid);
+ mi_subproc_destroy(_mi_subproc_to_id(subproc));
+ return _mi_subproc_to_id(NULL);
+ }
+ mi_assert_internal(subproc->heap_main == heap_main);
+
+ // init meta theap
+ mi_assert_internal(parent->theap_meta!=NULL);
+ mi_assert_internal(parent->theap_meta->tld!=NULL);
+ mi_assert_internal(parent->theap_meta->tld->thread_id == MI_THREADID_DETACHED);
+ _mi_theap_init(theap_meta,heap_main,parent->theap_meta->tld /* detached tld */);
+ subproc->theap_meta = theap_meta;
+
+ return _mi_subproc_to_id(subproc);
+}
+
+
+/* -----------------------------------------------------------
+ Sub process destruction
+----------------------------------------------------------- */
+
+// destroy all subproc resources including arena's, heap's etc.
+static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subprocs_lock)
+{
+ if (subproc==NULL) return;
+
+ // remove from the subproc list
+ mi_lock_maybe(&mi_subprocs_lock, acquire_subprocs_lock) {
+ if (subproc->next!=NULL) { subproc->next->prev = subproc->prev; }
+ if (subproc->prev!=NULL) { subproc->prev->next = subproc->next; }
+ else { mi_assert_internal(mi_subprocs==subproc); mi_subprocs = subproc->next; }
+ }
+
+ // destroy all subproc heaps
+ mi_lock(&subproc->heaps_lock) {
+ mi_heap_t* heap = subproc->heaps;
+ while (heap != NULL) {
+ mi_heap_t* next = heap->next;
+ if (heap!=subproc->heap_main) { _mi_heap_force_destroy(heap, false /* don't re-acquire the heaps_lock */); }
+ heap = next;
+ }
+ mi_assert_internal(subproc->heap_main==NULL || subproc->heaps == subproc->heap_main);
+ if (subproc->heap_main!=NULL) {
+ _mi_thread_locals_thread_done(); // release thread locals that may have been allocated (safe as the main heap uses the fast key)
+ if (_mi_subproc_is_main(subproc)) {
+ _mi_thread_locals_done();
+ }
+ _mi_heap_force_destroy(subproc->heap_main, false /* don't re-acquire the heaps_lock */); // no warning if destroying the main heap
+ }
+ }
+
+ subproc->theap_meta = NULL; // theap meta stats are merged during heap_destroy of the main heap
+
+ if (!_mi_subproc_is_main(subproc)) {
+ // merge stats back into the main subproc
+ _mi_stats_merge_into(&mi_process_subproc_main.stats, &subproc->stats);
+ }
+
+ // remove associated arenas
+ _mi_arenas_unsafe_destroy_all(subproc);
+
+ // show stats of the main process (at process end) before releasing the heaps lock
+ if (_mi_subproc_is_main(subproc)) {
+ if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {
+ mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL);
+ }
+ }
+
+ // todo: should we refcount subprocesses?
+ mi_lock_done(&subproc->arena_reserve_lock);
+ mi_lock_done(&subproc->heaps_lock);
+ mi_lock_done(&subproc->theap_meta_lock);
+ _mi_meta_free( subproc->parent, subproc, subproc->memid);
+ if (_mi_subproc_is_main(subproc)) {
+ // for the main subproc, also release the global page map
+ _mi_page_map_unsafe_destroy();
+ }
+}
+
+void mi_subproc_destroy(mi_subproc_id_t subproc_id) {
+ mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
+ if (subproc==NULL || subproc==&mi_process_subproc_main) return;
+ mi_subproc_unsafe_destroy(subproc, true /* take lock */);
+}
+
+void _mi_subprocs_unsafe_destroy_all(void) {
+ mi_lock(&mi_subprocs_lock) {
+ mi_subproc_t* subproc = mi_subprocs;
+ while (subproc!=NULL) {
+ mi_subproc_t* next = subproc->next;
+ if (subproc!=&mi_process_subproc_main) {
+ mi_subproc_unsafe_destroy(subproc, false /* take mi_subprocs lock */);
+ }
+ subproc = next;
+ }
+ }
+ mi_subproc_unsafe_destroy(&mi_process_subproc_main, true /* take mi_subprocs lock */);
+}
+
+
+/* -----------------------------------------------------------
+ Sub process various
+----------------------------------------------------------- */
+
+void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) {
+ mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
+ mi_assert_internal(subproc!=NULL);
+ if (subproc==NULL) return;
+ mi_assert_internal(subproc->heap_main!=NULL);
+ if (subproc->heap_main==NULL) return;
+ mi_theap_t* theap = _mi_theap_default();
+ if (mi_theap_is_initialized(theap)) {
+ if (theap->tld!=NULL && theap->tld->subproc != subproc) {
+ _mi_warning_message("unable to add thread to the subprocess as it was already in another subprocess (at %p)\n", theap->tld->subproc);
+ }
+ return;
+ }
+
+ // initialize this thread tld & theap
+ _mi_thread_init_with_heap(subproc->heap_main);
+}
+
+
+bool mi_subproc_visit_heaps(mi_subproc_id_t subproc_id, mi_heap_visit_fun* visitor, void* arg) {
+ mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
+ if (subproc==NULL) return false;
+ bool ok = true;
+ mi_lock(&subproc->heaps_lock) {
+ for (mi_heap_t* heap = subproc->heaps; heap!=NULL && ok; heap = heap->next) {
+ ok = (*visitor)(heap, arg);
+ }
+ }
+ return ok;
+}
+
+
+mi_subproc_t* _mi_subproc_main_init(void) {
+ mi_lock_init(&mi_subprocs_lock);
+ mi_memid_t memid = _mi_memid_create_static(&mi_process_subproc_main,sizeof(mi_subproc_t));
+ mi_process_subproc_main.memid = memid;
+ mi_subproc_init(&mi_process_subproc_main,NULL);
+ return &mi_process_subproc_main;
+}
+
+void _mi_subproc_main_done(void) {
+ mi_lock_done(&mi_subprocs_lock);
+}
+
diff --git a/vendored/mimalloc/src/theap.c b/vendored/mimalloc/src/theap.c
index ef53c1e07..1731830a7 100644
--- a/vendored/mimalloc/src/theap.c
+++ b/vendored/mimalloc/src/theap.c
@@ -1,5 +1,5 @@
/*----------------------------------------------------------------------------
-Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -7,7 +7,8 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc.h"
#include "mimalloc/internal.h"
-#include "mimalloc/prim.h" // _mi_theap_default
+#include "mimalloc/prim.h" // _mi_prim_thread_yield
+#include "mimalloc/prim-tls.h" // _mi_theap_default
#if defined(_MSC_VER) && (_MSC_VER < 1920)
#pragma warning(disable:4204) // non-constant aggregate initializer
@@ -23,7 +24,7 @@ typedef bool (theap_page_visitor_fun)(mi_theap_t* theap, mi_page_queue_t* pq, mi
// Visit all pages in a theap; returns `false` if break was called.
static bool mi_theap_visit_pages(mi_theap_t* theap, theap_page_visitor_fun* fn, bool include_full, void* arg1, void* arg2)
{
- if (theap==NULL || theap->page_count==0) return 0;
+ if (theap==NULL || theap->page_count==0) return true;
// visit all pages
#if MI_DEBUG>1
@@ -50,19 +51,24 @@ static bool mi_theap_visit_pages(mi_theap_t* theap, theap_page_visitor_fun* fn,
}
-#if MI_DEBUG>=2
+#if MI_DEBUG>=3
static bool mi_theap_page_is_valid(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
MI_UNUSED(arg1);
MI_UNUSED(arg2);
MI_UNUSED(pq);
mi_assert_internal(mi_page_theap(page) == theap);
+ mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap);
+ mi_assert_internal(page_theap == NULL || theap == page_theap);
mi_assert_expensive(_mi_page_is_valid(page));
return true;
}
-#endif
-#if MI_DEBUG>=3
+
static bool mi_theap_is_valid(mi_theap_t* theap) {
mi_assert_internal(theap!=NULL);
+ mi_heap_t* const heap = _mi_theap_heap_peek(theap);
+ mi_assert_internal(heap != NULL);
+ mi_theap_t* const heap_theap = _mi_heap_theap_peek(heap); // don't use mi_heap_theap as that may re-initialize the thread
+ mi_assert_internal(heap_theap==NULL || heap_theap == theap);
mi_theap_visit_pages(theap, &mi_theap_page_is_valid, true, NULL, NULL);
for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {
mi_assert_internal(_mi_page_queue_is_valid(theap, &theap->pages[bin]));
@@ -91,9 +97,10 @@ typedef enum mi_collect_e {
static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) {
MI_UNUSED(arg2);
MI_UNUSED(theap);
- mi_assert_internal(mi_theap_page_is_valid(theap, pq, page, NULL, NULL));
+ mi_assert_expensive(mi_theap_page_is_valid(theap, pq, page, NULL, NULL));
mi_collect_t collect = *((mi_collect_t*)arg_collect);
- _mi_page_free_collect(page, collect >= MI_FORCE);
+ _mi_page_free_collect(page, collect >= MI_FORCE); // update used count
+ _mi_page_update_stats(page);
if (mi_page_all_free(page)) {
// no more used blocks, possibly free the page.
if (collect >= MI_FORCE || page->retire_expire == 0) { // either forced/abandon, or not already retired
@@ -108,9 +115,10 @@ static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_pag
return true; // don't break
}
-static void mi_theap_merge_stats(mi_theap_t* theap) {
- mi_assert_internal(mi_theap_is_initialized(theap));
- _mi_stats_merge_into(&_mi_theap_heap(theap)->stats, &theap->stats);
+void _mi_theap_merge_stats(mi_theap_t* theap) {
+ mi_assert_internal(mi_theap_is_initialized(theap));
+ mi_heap_t* const heap = _mi_theap_heap(theap);
+ _mi_stats_merge_into(&heap->stats, &theap->stats);
}
static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect)
@@ -124,18 +132,22 @@ static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect)
// python/cpython#112532: we may be called from a thread that is not the owner of the theap
// const bool is_main_thread = (_mi_is_main_thread() && theap->thread_id == _mi_thread_id());
- // collect retired pages
- _mi_theap_collect_retired(theap, force);
+ // collect retired pages (and full pages if theap->allow_page_abandon is false)
+ _mi_theap_collect_retired(theap, force);
// collect all pages owned by this thread
mi_theap_visit_pages(theap, &mi_theap_page_collect, (collect!=MI_NORMAL), &collect, NULL); // dont normally visit full pages, see issue #1220
// collect arenas (this is program wide so don't force purges on abandonment of threads)
//mi_atomic_storei64_release(&theap->tld->subproc->purge_expire, 1);
- _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld);
+ if (collect != MI_ABANDON) {
+ _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld);
+ }
// merge statistics
- mi_theap_merge_stats(theap);
+ if (mi_option_is_enabled(mi_option_collect_merges_stats)) {
+ _mi_theap_merge_stats(theap);
+ }
}
void _mi_theap_collect_abandon(mi_theap_t* theap) {
@@ -170,24 +182,41 @@ mi_theap_t* mi_theap_get_default(void) {
return theap;
}
+mi_theap_t* mi_theap_set_default(mi_theap_t* theap) {
+ mi_theap_t* const previous = mi_theap_get_default();
+ if (mi_theap_is_initialized(theap)) {
+ _mi_theap_default_set(theap);
+ }
+ return previous;
+}
+
+
+static void mi_theap_options_init(mi_theap_t* theap) {
+ theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0);
+ theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0);
+ theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32);
+ theap->is_detached = (theap->tld->thread_id == MI_THREADID_DETACHED);
+}
+
// todo: make order of parameters consistent (but would that break compat with CPython?)
void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld)
{
mi_assert_internal(theap!=NULL);
mi_assert_internal(heap!=NULL);
+ mi_assert_internal(tld!=NULL);
mi_memid_t memid = theap->memid;
_mi_memcpy_aligned(theap, &_mi_theap_empty, sizeof(mi_theap_t));
theap->memid = memid;
- theap->refcount = 1;
theap->tld = tld; // avoid reading the thread-local tld during initialization
- mi_atomic_store_ptr_relaxed(mi_heap_t,&theap->heap,heap);
+ mi_atomic_store_release(&theap->refcount,1);
+ mi_atomic_store_ptr_release(mi_subproc_t,&theap->subproc,heap->subproc);
mi_assert_internal(theap->stats.size == sizeof(mi_stats_t));
-
- _mi_theap_options_init(theap);
+ mi_theap_options_init(theap);
+
if (theap->tld->is_in_threadpool) {
// if we run as part of a thread pool it is better to not arbitrarily reclaim abandoned pages into our theap.
// this is checked in `free.c:mi_free_try_collect_mt`
- // .. but abandoning is good in this case: halve the full page retain (possibly to 0)
+ // .. but abandoning is good in this case: quarter the full page retain (possibly to 0)
// (so blocked threads do not hold on to too much memory)
if (theap->page_full_retain > 0) {
theap->page_full_retain = theap->page_full_retain / 4;
@@ -196,29 +225,42 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld)
// push on the thread local theaps list
mi_theap_t* head = NULL;
+ mi_random_ctx_t head_random;
mi_lock(&theap->tld->theaps_lock) {
head = theap->tld->theaps;
theap->tprev = NULL;
theap->tnext = head;
- if (head!=NULL) { head->tprev = theap; }
theap->tld->theaps = theap;
+ if (head!=NULL) {
+ head->tprev = theap;
+ head_random = head->random;
+ }
}
- // initialize random
- if (head == NULL) { // first theap in this thread?
+ // initialize random if heap==NULL
+ if (head==NULL) { // first theap of the first thread?
#if defined(_WIN32) && !defined(MI_SHARED_LIB)
+ if (tld->thread_seq==0) {
_mi_random_init_weak(&theap->random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185)
- #else
- _mi_random_init(&theap->random);
+ }
+ else
#endif
+ {
+ _mi_random_init(&theap->random);
+ }
}
else {
- _mi_random_split(&head->random, &theap->random);
+ _mi_random_split(&head_random, &theap->random); // &theap->random is used as nonce so it is ok if threads capture the same head->random
+ }
+ // theap->cookie = _mi_theap_random_next(theap) | 1;
+ _mi_theap_guarded_init(theap); // needs theap->random
+ if (!theap->is_detached) {
+ mi_subproc_stat_increase(_mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem
}
- theap->cookie = _mi_theap_random_next(theap) | 1;
- _mi_theap_guarded_init(theap);
- mi_subproc_stat_increase(_mi_subproc(),theaps,1);
+ // only now set the heap member as it is used to determine if a theap is initialized
+ mi_atomic_store_ptr_release(mi_heap_t,&theap->heap,heap);
+
// push on the heap's theap list
mi_lock(&heap->theaps_lock) {
head = heap->theaps;
@@ -229,33 +271,37 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld)
}
}
-mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) {
+mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld) {
mi_assert_internal(tld!=NULL);
mi_assert_internal(heap!=NULL);
+ mi_assert_internal(tld->thread_id == MI_THREADID_DETACHED || _mi_thread_id() == tld->thread_id);
+ // mi_assert_internal(_mi_heap_theap_peek(heap)==NULL); // don't access thread locals as this is called on thread init
+
// allocate and initialize a theap
mi_memid_t memid;
mi_theap_t* theap;
- //if (!_mi_is_heap_main(heap)) {
- // theap = (mi_theap_t*)mi_heap_zalloc(mi_heap_main(),sizeof(mi_theap_t));
- // memid = _mi_memid_create(MI_MEM_HEAP_MAIN);
- // memid.initially_zero = memid.initially_committed = true;
- //}
- //else
+
if (heap->exclusive_arena == NULL) {
- theap = (mi_theap_t*)_mi_meta_zalloc(sizeof(mi_theap_t), &memid);
+ theap = (mi_theap_t*)_mi_meta_zalloc(heap->subproc, sizeof(mi_theap_t), &memid);
}
else {
// theaps associated with a specific arena are allocated in that arena
// note: takes up at least one slice which is quite wasteful...
const size_t size = _mi_align_up(sizeof(mi_theap_t),MI_ARENA_MIN_OBJ_SIZE);
- theap = (mi_theap_t*)_mi_arenas_alloc(heap, size, true, true, heap->exclusive_arena, tld->thread_seq, tld->numa_node, &memid);
- mi_assert_internal(memid.mem.os.size >= size);
+ theap = (mi_theap_t*)_mi_arenas_alloc(heap, size, true, true, heap->exclusive_arena, tld->thread_seq, tld->numa_node, &memid);
}
if (theap==NULL) {
_mi_error_message(ENOMEM, "unable to allocate theap meta-data\n");
return NULL;
}
+
theap->memid = memid;
+ return theap;
+}
+
+mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) {
+ mi_theap_t* theap = _mi_theap_alloc(heap,tld);
+ if (theap == NULL) return NULL;
_mi_theap_init(theap, heap, tld);
return theap;
}
@@ -266,205 +312,129 @@ uintptr_t _mi_theap_random_next(mi_theap_t* theap) {
static void mi_theap_free_mem(mi_theap_t* theap) {
if (theap!=NULL) {
- mi_subproc_stat_decrease(_mi_subproc(),theaps,1);
- // free the used memory
- if (theap->memid.memkind == MI_MEM_HEAP_MAIN) { // note: for now unused as it would access theap_default stats in mi_free of the current theap
- mi_assert_internal(_mi_is_heap_main(mi_heap_of(theap)));
- mi_free(theap);
- }
- else if (theap->memid.memkind == MI_MEM_META) {
- _mi_meta_free(theap, sizeof(*theap), theap->memid);
- }
- else {
- _mi_arenas_free(theap, _mi_align_up(sizeof(*theap),MI_ARENA_MIN_OBJ_SIZE), theap->memid ); // issue #1168, avoid assertion failure
+ mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc);
+ if (!theap->is_detached) {
+ mi_subproc_stat_decrease(subproc,theaps,1);
}
+ _mi_meta_free(subproc, theap, theap->memid);
}
}
+// we need to reference count theaps due to the _mi_theap_cached thread locals
void _mi_theap_incref(mi_theap_t* theap) {
- if (theap!=NULL && theap->memid.memkind > MI_MEM_STATIC) {
+ if (theap!=NULL && !mi_memid_needs_no_free(theap->memid)) {
mi_atomic_increment_acq_rel(&theap->refcount);
}
}
void _mi_theap_decref(mi_theap_t* theap) {
- if (theap!=NULL && theap->memid.memkind > MI_MEM_STATIC) {
+ if (theap!=NULL && !mi_memid_needs_no_free(theap->memid)) {
if (mi_atomic_decrement_acq_rel(&theap->refcount) == 1) {
mi_theap_free_mem(theap);
}
}
}
-
-// called from `mi_theap_delete` to free the internal theap resources.
-bool _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock) {
- mi_assert(theap != NULL);
- if (theap==NULL) return true;
-
- mi_heap_t* const heap = mi_atomic_exchange_ptr_acq_rel(mi_heap_t, &theap->heap, NULL);
- if (heap==NULL) {
- // concurrent interaction, retry in an outer loop (as the other thread may be blocked on our lock)
- return false;
- }
- else {
- // merge stats to the owning heap
- _mi_stats_merge_into(&heap->stats, &theap->stats);
-
- // remove ourselves from the heap theaps list
- mi_lock_maybe(&heap->theaps_lock, acquire_heap_theaps_lock) {
- if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; }
- if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; }
- else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; }
- theap->hnext = theap->hprev = NULL;
+// Thread termination and heap delete/destroy might run concurrently
+// and we need to ensure we free the memory correctly. A heap or tld
+// will first "detach" its theaps so it has a list with theaps that are
+// no longer shared, and only then free's the theaps in that list.
+// To detach we need to hold both the `heap->theaps_lock` and the `tld->theaps_lock`.
+// Due to lock-inversion we need to use `mi_lock_try_acquire` and if that fails
+// we back-off, release the outer lock, and try again until we succeed.
+
+// Remove the theaps in this heap from any thread local tld lists.
+void _mi_heap_detach_theaps( mi_heap_t* heap ) {
+ bool all_detached;
+ do {
+ all_detached = true;
+ mi_lock(&heap->theaps_lock) {
+ mi_theap_t* theap = heap->theaps;
+ while (theap != NULL) {
+ mi_theap_t* next = theap->hnext;
+ mi_tld_t* tld = theap->tld;
+ if (tld != NULL) {
+ if (mi_lock_try_acquire(&tld->theaps_lock)) {
+ // remove the theap from the tld theaps list
+ if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; }
+ if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; }
+ else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; }
+ theap->tnext = theap->tprev = NULL;
+ theap->tld = NULL;
+ mi_lock_release(&tld->theaps_lock);
+ }
+ else {
+ all_detached = false;
+ }
+ }
+ theap = next;
+ }
}
-
- // remove ourselves from the thread local theaps list
- mi_lock_maybe(&theap->tld->theaps_lock, acquire_tld_theaps_lock) {
- if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; }
- if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; }
- else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; }
- theap->tnext = theap->tprev = NULL;
+ if (!all_detached) {
+ mi_subproc_stat_counter_increase(heap->subproc,heaps_delete_wait,1);
+ _mi_prim_thread_yield();
}
- theap->tld = NULL;
- _mi_theap_decref(theap);
- return true;
- }
-}
-
-
-/* -----------------------------------------------------------
- Heap destroy
------------------------------------------------------------ */
-/*
-
-// zero out the page queues
-static void mi_theap_reset_pages(mi_theap_t* theap) {
- mi_assert_internal(theap != NULL);
- mi_assert_internal(mi_theap_is_initialized(theap));
- // TODO: copy full empty theap instead?
- _mi_memset(&theap->pages_free_direct, 0, sizeof(theap->pages_free_direct));
- _mi_memcpy_aligned(&theap->pages, &_mi_theap_empty.pages, sizeof(theap->pages));
- // theap->thread_delayed_free = NULL;
- theap->page_count = 0;
-}
-
-static bool _mi_theap_page_destroy(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
- MI_UNUSED(arg1);
- MI_UNUSED(arg2);
- MI_UNUSED(pq);
-
- // ensure no more thread_delayed_free will be added
- //_mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false);
-
- // stats
- const size_t bsize = mi_page_block_size(page);
- if (bsize > MI_LARGE_MAX_OBJ_SIZE) {
- mi_theap_stat_decrease(theap, malloc_huge, bsize);
- }
- #if (MI_STAT>0)
- _mi_page_free_collect(page, false); // update used count
- const size_t inuse = page->used;
- if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
- mi_theap_stat_decrease(theap, malloc_normal, bsize * inuse);
- #if (MI_STAT>1)
- mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], inuse);
- #endif
- }
- // mi_theap_stat_decrease(theap, malloc_requested, bsize * inuse); // todo: off for aligned blocks...
- #endif
-
- /// pretend it is all free now
- mi_assert_internal(mi_page_thread_free(page) == NULL);
- page->used = 0;
-
- // and free the page
- // mi_page_free(page,false);
- page->next = NULL;
- page->prev = NULL;
- mi_page_set_theap(page, NULL);
- _mi_arenas_page_free(page, theap);
-
- return true; // keep going
-}
-
-void _mi_theap_destroy_pages(mi_theap_t* theap) {
- mi_theap_visit_pages(theap, &_mi_theap_page_destroy, NULL, NULL);
- mi_theap_reset_pages(theap);
-}
-
-#if MI_TRACK_HEAP_DESTROY
-static bool mi_cdecl mi_theap_track_block_free(const mi_theap_t* theap, const mi_theap_area_t* area, void* block, size_t block_size, void* arg) {
- MI_UNUSED(theap); MI_UNUSED(area); MI_UNUSED(arg); MI_UNUSED(block_size);
- mi_track_free_size(block,mi_usable_size(block));
- return true;
-}
-#endif
-
-void mi_theap_destroy(mi_theap_t* theap) {
- mi_assert(theap != NULL);
- mi_assert(mi_theap_is_initialized(theap));
- mi_assert(!theap->allow_page_reclaim);
- mi_assert(!theap->allow_page_abandon);
- mi_assert_expensive(mi_theap_is_valid(theap));
- if (theap==NULL || !mi_theap_is_initialized(theap)) return;
- #if MI_GUARDED
- // _mi_warning_message("'mi_theap_destroy' called but MI_GUARDED is enabled -- using `mi_theap_delete` instead (theap at %p)\n", theap);
- mi_theap_delete(theap);
- return;
- #else
- if (theap->allow_page_reclaim) {
- _mi_warning_message("'mi_theap_destroy' called but ignored as the theap was not created with 'allow_destroy' (theap at %p)\n", theap);
- // don't free in case it may contain reclaimed pages,
- mi_theap_delete(theap);
- }
- else {
- // track all blocks as freed
- #if MI_TRACK_HEAP_DESTROY
- mi_theap_visit_blocks(theap, true, mi_theap_track_block_free, NULL);
- #endif
- // free all pages
- _mi_theap_destroy_pages(theap);
- mi_theap_free(theap,true);
- }
- #endif
-}
-
-// forcefully destroy all theaps in the current thread
-void _mi_theap_unsafe_destroy_all(mi_theap_t* theap) {
- mi_assert_internal(theap != NULL);
- if (theap == NULL) return;
- mi_theap_t* curr = theap->tld->theaps;
- while (curr != NULL) {
- mi_theap_t* next = curr->next;
- if (!curr->allow_page_reclaim) {
- mi_theap_destroy(curr);
+ } while (!all_detached);
+}
+
+// Remove the theaps in this thread from the heaps that own them.
+void _mi_tld_detach_theaps( mi_tld_t* tld ) {
+ bool all_detached;
+ do {
+ all_detached = true;
+ mi_lock(&tld->theaps_lock) {
+ mi_theap_t* theap = tld->theaps;
+ while (theap != NULL) {
+ mi_theap_t* next = theap->tnext;
+ mi_assert_internal(theap->page_count==0);
+ mi_heap_t* heap = _mi_theap_heap_peek(theap); // now the heap might be NULL from an earlier iteration
+ if (heap != NULL) {
+ if (mi_lock_try_acquire(&heap->theaps_lock)) {
+ // merge stats into the owning heap stats
+ _mi_stats_merge_into(&heap->stats, &theap->stats);
+ // remove the theap from the heap list
+ if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; }
+ if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; }
+ else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; }
+ theap->hnext = theap->hprev = NULL;
+ // and set `heap` to NULL
+ mi_atomic_store_ptr_release(mi_heap_t, &theap->heap, NULL);
+ mi_lock_release(&heap->theaps_lock);
+ }
+ else {
+ all_detached = false;
+ }
+ }
+ theap = next;
+ }
}
- else {
- _mi_theap_destroy_pages(curr);
+ if (!all_detached) {
+ mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1);
+ _mi_prim_thread_yield();
}
- curr = next;
- }
+ } while (!all_detached);
}
-*/
+
+
/* -----------------------------------------------------------
- Safe Heap delete
+ Safe theap delete
----------------------------------------------------------- */
// Safe delete a theap without freeing any still allocated blocks in that theap.
-void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock)
-{
- mi_assert(theap != NULL);
- mi_assert(mi_theap_is_initialized(theap));
- mi_assert_expensive(mi_theap_is_valid(theap));
- if (theap==NULL || !mi_theap_is_initialized(theap)) return;
+// void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock)
+// {
+// mi_assert(theap != NULL);
+// mi_assert(mi_theap_is_initialized(theap));
+// mi_assert_expensive(mi_theap_is_valid(theap));
+// if (theap==NULL || !mi_theap_is_initialized(theap)) return;
- // abandon all pages
- _mi_theap_collect_abandon(theap);
+// // abandon all pages
+// _mi_theap_collect_abandon(theap);
- mi_assert_internal(theap->page_count==0);
- _mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock);
-}
+// mi_assert_internal(theap->page_count==0);
+// _mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock);
+// }
@@ -518,7 +488,7 @@ bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena_id) {
// reinit direct pages (as we may be in a different process)
mi_assert_internal(theap->page_count == 0);
for (size_t i = 0; i < MI_PAGES_DIRECT; i++) {
- theap->pages_free_direct[i] = (mi_page_t*)&_mi_page_empty;
+ theap->pages_free_direct[i] = _mi_page_empty_get();
}
// push on the thread local theaps list
@@ -541,7 +511,7 @@ void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) {
area->reserved = page->reserved * bsize;
area->committed = page->capacity * bsize;
area->blocks = mi_page_start(page);
- area->used = page->used; // number of blocks in use (#553)
+ area->used = mi_page_used(page); // number of blocks in use (#553)
area->block_size = ubsize;
area->full_block_size = bsize;
area->reserved1 = page;
@@ -567,7 +537,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi
_mi_page_free_collect(page,true); // collect both thread_delayed and local_free
mi_assert_internal(page->local_free == NULL);
- if (page->used == 0) return true;
+ if (mi_page_used(page) == 0) return true;
size_t psize;
uint8_t* const pstart = mi_page_area(page, &psize);
@@ -577,13 +547,13 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi
// optimize page with one block
if (page->capacity == 1) {
- mi_assert_internal(page->used == 1 && page->free == NULL);
+ mi_assert_internal(mi_page_used(page) == 1 && page->free == NULL);
return visitor(heap, area, pstart, ubsize, arg);
}
mi_assert(bsize <= UINT32_MAX);
// optimize full pages
- if (page->used == page->capacity) {
+ if (mi_page_used(page) == page->capacity) {
uint8_t* block = pstart;
for (size_t i = 0; i < page->capacity; i++) {
if (!visitor(heap, area, block, ubsize, arg)) return false;
@@ -627,7 +597,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi
size_t bit = blockidx - (bitidx * MI_INTPTR_BITS);
free_map[bitidx] |= ((uintptr_t)1 << bit);
}
- mi_assert_internal(page->capacity == (free_count + page->used));
+ mi_assert_internal(page->capacity == (free_count + mi_page_used(page)));
// walk through all blocks skipping the free ones
#if MI_DEBUG>1
@@ -659,10 +629,15 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi
block += bsize * MI_INTPTR_BITS;
}
}
- mi_assert_internal(page->used == used_count);
+ mi_assert_internal(mi_page_used(page) == used_count);
return true;
}
+// bool _mi_page_visit_blocks( mi_page_t* page, mi_block_visit_fun* visitor, void* arg ) {
+// mi_heap_area_t area;
+// _mi_heap_area_init(&area, page);
+// return _mi_theap_area_visit_blocks(&area, page, visitor, arg);
+// }
// Separate struct to keep `mi_page_t` out of the public interface
diff --git a/vendored/mimalloc/src/threadlocal.c b/vendored/mimalloc/src/threadlocal.c
index eea3329bf..6f280d093 100644
--- a/vendored/mimalloc/src/threadlocal.c
+++ b/vendored/mimalloc/src/threadlocal.c
@@ -6,9 +6,9 @@ terms of the MIT license. A copy of the license can be found in the file
-----------------------------------------------------------------------------*/
/* ----------------------------------------------------------------------------
-Implement dynamic thread local variables (for heap's).
-Unlike most OS native implementations there is no limit on the number
-that can be allocated.
+ Implement dynamic thread local variables (used by heap's for their theap's).
+ Unlike most OS native implementations there is no limit on the number
+ that can be allocated.
-----------------------------------------------------------------------------*/
#include "mimalloc.h"
@@ -27,12 +27,41 @@ typedef struct mi_tls_slot_s {
typedef struct mi_thread_locals_s {
size_t count;
+ mi_memid_t memid;
mi_tls_slot_t slots[1];
} mi_thread_locals_t;
-static mi_thread_locals_t mi_thread_locals_empty = { 0, {{0,NULL}} };
+static mi_thread_locals_t mi_thread_locals_empty = mi_init_struct_zero;
-mi_decl_thread mi_thread_locals_t* mi_thread_locals = &mi_thread_locals_empty; // always point to a valid `mi_thread_locals_t`
+
+/* -----------------------------------------------------------
+ We have 2 thread local variable which we implement with either
+ a C thread local declaration or using pthread keys.
+ - mi_thread_locals: points to an array of thread locals for most keys
+ - mi_slot_fast: a single dedicated thread local for slightly faster access. (used for the main heap's theap)
+----------------------------------------------------------- */
+
+#if MI_TLS_MODEL_PTHREADS || defined(__APPLE__) // macOS has fast pthreads
+// Use pthreads
+#define mi_define_thread_local(tp,name,initval) \
+ static pthread_key_t __##name##_key = MI_PTHREAD_KEY_INVALID; \
+ static inline tp name##_peek(void) { return (tp)mi_pthread_key_get(__##name##_key); } \
+ static inline tp name##_get(void) { tp result = name##_peek(); return (result!=NULL ? result : initval); } \
+ static inline bool name##_set(tp val) { return mi_pthread_key_set(&__##name##_key,val); } \
+ static inline void name##_delete(void) { mi_pthread_key_delete(&__##name##_key); }
+
+#else
+// Direct thread locals
+#define mi_define_thread_local(tp,name,initval) \
+ static mi_decl_thread tp __##name = initval; \
+ static inline tp name##_peek(void) { return __##name; } \
+ static inline tp name##_get(void) { tp result = __##name; return (result!=NULL ? result : initval); } \
+ static inline bool name##_set(tp val) { __##name = val; return true; } \
+ static inline void name##_delete(void) { }
+#endif
+
+mi_define_thread_local(mi_thread_locals_t*, mi_thread_locals, &mi_thread_locals_empty)
+mi_define_thread_local(void*, mi_slot_fast, NULL)
/* -----------------------------------------------------------
@@ -42,15 +71,18 @@ mi_decl_thread mi_thread_locals_t* mi_thread_locals = &mi_thread_locals_empty;
a value, we also set the version of the key.
----------------------------------------------------------- */
-#if MI_SIZE_BITS < 64
-#define MI_TLS_IDX_BITS (MI_SIZE_BITS/2) // half for the index, half for the version
+#if MI_SIZE_BITS >= 64
+#define MI_TLS_IDX_BITS (MI_SIZE_BITS/4) /* 16 bits for the index, 48 bits for the version */
+#elif MI_SIZE_BITS >= 32
+#define MI_TLS_IDX_BITS (12) /* 12 bits for index, 20 for the version? */
#else
-#define MI_TLS_IDX_BITS (MI_SIZE_BITS/4) // 16 bits for the index, 48 bits for the version
+#error not enough bits for the version for thread locals
#endif
#define MI_TLS_IDX_MASK ((MI_ZU(1)<<MI_TLS_IDX_BITS)-1)
#define MI_TLS_IDX_MAX MI_TLS_IDX_MASK
#define MI_TLS_VERSION_MAX ((MI_ZU(1)<<(MI_SIZE_BITS - MI_TLS_IDX_BITS))-1)
+
static size_t mi_key_index( size_t key ) {
return (key & MI_TLS_IDX_MASK);
}
@@ -70,36 +102,43 @@ static mi_thread_local_t mi_key_create( size_t index, size_t version ) {
// dynamically reallocate the thread local slots when needed
static mi_thread_locals_t* mi_thread_locals_expand(size_t least_idx) {
- mi_thread_locals_t* tls_old = mi_thread_locals;
+ mi_thread_locals_t* tls_old = mi_thread_locals_get();
const size_t count_old = tls_old->count;
size_t count;
if (count_old==0) {
tls_old = NULL; // so we allocate fresh from mi_thread_locals_empty
count = 16; // start with 16 slots
- }
+ }
else if (count_old >= 1024) {
count = count_old + 1024; // at some point increase linearly
}
else {
count = 2*count_old; // and double initially
}
- if (count <= least_idx) {
+ if (count <= least_idx) {
count = least_idx + 1;
}
if (count > MI_TLS_IDX_MAX) { return NULL; } // too large
- mi_thread_locals_t* tls = (mi_thread_locals_t*)mi_rezalloc(tls_old, sizeof(mi_thread_locals_t) + count*sizeof(mi_tls_slot_t));
+ // allocate as meta (for secure mode)
+ // we could also allocate on the main heap; this is recursion safe as that uses the fast local key
+ mi_memid_t memid = (tls_old==NULL ? _mi_memid_none() : tls_old->memid);
+ mi_thread_locals_t* tls = (mi_thread_locals_t*)_mi_meta_rezalloc(_mi_subproc(), tls_old, sizeof(mi_thread_locals_t) + count*sizeof(mi_tls_slot_t), &memid);
if mi_unlikely(tls==NULL) return NULL;
+ tls->memid = memid;
tls->count = count;
- mi_thread_locals = tls;
+ mi_thread_locals_set(tls);
return tls;
}
static mi_decl_noinline bool mi_thread_local_set_expand( mi_thread_local_t key, void* val ) {
if (val==NULL) return true;
- const size_t idx = mi_key_index(key);
+ const size_t idx = mi_key_index(key);
mi_thread_locals_t* tls = mi_thread_locals_expand(idx);
- if (tls==NULL) return false;
- mi_assert_internal(tls == mi_thread_locals);
+ if (tls==NULL) {
+ _mi_error_message(EFAULT,"unable to allocate thread local variables\n");
+ return false;
+ }
+ mi_assert_internal(tls == mi_thread_locals_get());
mi_assert_internal(idx < tls->count);
tls->slots[idx].value = val;
tls->slots[idx].version = mi_key_version(key);
@@ -108,8 +147,8 @@ static mi_decl_noinline bool mi_thread_local_set_expand( mi_thread_local_t key,
// set a tls slot; returns `true` if successful.
// Can return `false` if we could not reallocate the slots array.
-bool _mi_thread_local_set( mi_thread_local_t key, void* val ) {
- mi_thread_locals_t* tls = mi_thread_locals;
+static mi_decl_noinline bool mi_thread_local_set_regular( mi_thread_local_t key, void* val ) {
+ mi_thread_locals_t* tls = mi_thread_locals_get();
mi_assert_internal(tls!=NULL);
mi_assert_internal(key!=0);
const size_t idx = mi_key_index(key);
@@ -123,25 +162,54 @@ bool _mi_thread_local_set( mi_thread_local_t key, void* val ) {
}
}
-// get a tls slot value
-void* _mi_thread_local_get( mi_thread_local_t key ) {
- const mi_thread_locals_t* const tls = mi_thread_locals;
- mi_assert_internal(tls!=NULL);
+bool _mi_thread_local_set( mi_thread_local_t key, void* val ) {
mi_assert_internal(key!=0);
+ if (key == mi_thread_local_key_fast) {
+ return mi_slot_fast_set(val);
+ }
+ else {
+ return mi_thread_local_set_regular(key,val);
+ }
+}
+
+// get a tls slot value
+static mi_decl_noinline void* mi_thread_local_get_regular( mi_thread_local_t key ) {
+ const mi_thread_locals_t* const tls = mi_thread_locals_peek();
+ if mi_unlikely(tls==NULL) {
+ // this can happen if a thread local is accessed after the thread local has been freed
+ // from mi_thread_done or mi_process_done.
+ // todo: can we remove this check? now we can still call this from process done when stats are printed (which calls mi_heap_theap_peek)
+ return NULL;
+ }
const size_t idx = mi_key_index(key);
if mi_likely(idx < tls->count && mi_key_version(key) == tls->slots[idx].version) {
return tls->slots[idx].value;
}
else {
- return NULL;
+ return NULL;
+ }
+}
+
+// get a thread local value
+void* _mi_thread_local_get( mi_thread_local_t key ) {
+ if mi_likely(key == mi_thread_local_key_fast) {
+ return mi_slot_fast_get();
+ }
+ else {
+ void* p = mi_thread_local_get_regular(key);
+ if (key==0) { mi_assert_internal(p==NULL); }
+ return p;
}
}
void _mi_thread_locals_thread_done(void) {
- mi_thread_locals_t* const tls = mi_thread_locals;
+ mi_thread_locals_t* const tls = mi_thread_locals_peek();
if (tls!=NULL && tls->count > 0) {
- mi_free(tls);
- mi_thread_locals = &mi_thread_locals_empty;
+ _mi_meta_free(_mi_subproc(), tls, tls->memid);
+ mi_thread_locals_set(NULL);
+ }
+ if (mi_slot_fast_peek() != NULL) {
+ mi_slot_fast_set(NULL);
}
}
@@ -152,6 +220,7 @@ Create and free fresh TLS key's
static mi_lock_t mi_thread_locals_lock; // we need a lock in order to re-allocate the slot bits
static mi_bitmap_t* mi_thread_locals_free; // reuse an arena bitmap to track which slots were assigned (1=free, 0=in-use)
+static mi_memid_t mi_thread_locals_memid; // provenance of mi_thread_locals_free
static size_t mi_thread_locals_version; // version to be able to reuse slots safely
void _mi_thread_locals_init(void) {
@@ -161,9 +230,13 @@ void _mi_thread_locals_init(void) {
void _mi_thread_locals_done(void) {
mi_lock(&mi_thread_locals_lock) {
mi_bitmap_t* const slots = mi_thread_locals_free;
- mi_free(slots);
+ if (slots!=NULL) {
+ _mi_meta_free(_mi_subproc_main(), slots, mi_thread_locals_memid);
+ }
}
mi_lock_done(&mi_thread_locals_lock);
+ mi_thread_locals_delete();
+ mi_slot_fast_delete();
}
// strange signature but allows us to reuse the arena code for claiming free pages
@@ -173,7 +246,7 @@ static bool mi_thread_local_claim_fun(size_t _slice_index, mi_arena_t* _arena, b
return true;
}
-// When we claim a free slot, we increase the global version counter
+// When we claim a free slot, we increase the global version counter
// (so if we reuse a slot it will be returning NULL initially when a thread tries to get it)
static mi_thread_local_t mi_thread_local_claim(void) {
size_t idx = 0;
@@ -194,16 +267,21 @@ static bool mi_thread_local_create_expand(void) {
const size_t newcount = 1024 + oldcount;
if (newcount > MI_TLS_IDX_MAX) { return false; }
const size_t newsize = mi_bitmap_size( newcount, NULL );
- mi_bitmap_t* newslots = (mi_bitmap_t*)mi_zalloc_aligned(newsize, MI_BCHUNK_SIZE);
+ // mi_bitmap_t* newslots = (mi_bitmap_t*)mi_zalloc_aligned(newsize, MI_BCHUNK_SIZE);
+ mi_memid_t memid;
+ mi_bitmap_t* newslots = (mi_bitmap_t*)_mi_meta_zalloc_aligned(_mi_subproc_main(), newsize, MI_BCHUNK_SIZE, &memid); // always allocate thread locals in the main subprocess
+ mi_assert_internal(_mi_is_aligned(newslots,MI_BCHUNK_SIZE));
if (newslots==NULL) { return false; }
if (slots!=NULL) {
// copy over the previous bitmap
- _mi_memcpy_aligned(newslots, slots, mi_bitmap_size(oldcount, NULL));
- mi_free(slots);
+ const size_t oldsize = mi_bitmap_size(oldcount,NULL);
+ _mi_memcpy_aligned(newslots, slots, oldsize);
+ _mi_meta_free(_mi_subproc_main(), slots, mi_thread_locals_memid);
}
mi_bitmap_init(newslots, newcount, true /* pretend already zero'd so we do not zero out the copied old entries */);
mi_bitmap_unsafe_setN(newslots, oldcount, newcount - oldcount); /* set the new expanded slots as available */
mi_thread_locals_free = newslots;
+ mi_thread_locals_memid = memid;
return true;
}
@@ -219,6 +297,8 @@ mi_thread_local_t _mi_thread_local_create(void) {
}
}
}
+ mi_assert_internal(key!=0);
+ mi_assert_internal(key!=mi_thread_local_key_fast);
return key;
}