Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: no expert-cache loan for the prompt path (Prefill::relayout and the refill into given-back allocations), auto expert cache capped at 1500 slots; CI: Metal build job on a macOS runner
Niko1221e6dbbc312a5e
diff --git a/.github/workflows/metal-build.yml b/.github/workflows/metal-build.ymlnew file mode 100644index 000000000..7a69696fc--- /dev/null+++ b/.github/workflows/metal-build.yml@@ -0,0 +1,51 @@+name: Metal build++# Compile and link the Metal engine on a macOS runner (no GPU needed: the kernels are compiled at run time). The shared host files+# (src/core/verify.cpp, src/prefill, src/program/generate.cpp) call kernel entry points that exist only for CUDA / HIP until Metal gets a+# stub or an implementation; a missing one is a link error here, not on a user's Mac (owner round 9, G6).++on:+ pull_request:+ paths:+ - 'src/**'+ - 'include/**'+ - 'metal/**'+ - 'third_party/**'+ - 'CMakeLists.txt'+ - '.github/workflows/metal-build.yml'+ workflow_dispatch:++permissions:+ contents: read++jobs:+ metal-link:+ runs-on: macos-14+ timeout-minutes: 45+ env:+ # setup.py's LLAMA_CPP_COMMIT and metal/setup_mac.py's metal-cpp pin+ LLAMA_CPP_COMMIT: 3cf03257f219afbe7334045ff7c6a06ac68c627d+ METAL_CPP_URL: https://developer.apple.com/metal/cpp/files/metal-cpp_26.zip+ METAL_CPP_SHA256: 4df3c078b9aadcb516212e9cb03004cbc5ce9a3e9c068fa3144d021db585a3a4+ steps:+ - uses: actions/checkout@v4+ - name: Tools+ run: brew install ninja+ - name: metal-cpp+ run: |+ curl -fsSL "$METAL_CPP_URL" -o metal-cpp.zip+ echo "$METAL_CPP_SHA256 metal-cpp.zip" | shasum -a 256 -c -+ unzip -q metal-cpp.zip -d _mcpp+ - name: llama.cpp (ggml) at the pinned commit+ run: |+ mkdir llama.cpp && cd llama.cpp+ git init -q+ git remote add origin https://github.com/ggml-org/llama.cpp.git+ git fetch -q --depth 1 origin "$LLAMA_CPP_COMMIT"+ git checkout -q FETCH_HEAD+ - name: Configure+ run: |+ cmake -S metal -B build-metal -G Ninja -DCMAKE_BUILD_TYPE=Release \+ -DMETAL_CPP_DIR="$PWD/_mcpp/metal-cpp" -DSTRATA_GGML_DIR="$PWD/llama.cpp"+ - name: Build and link the engine+ run: cmake --build build-metal --target stratadiff --git a/src/program/generate.cpp b/src/program/generate.cppindex edf3f52c8..fd6187ffd 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -6024,6 +6024,19 @@ int main(int argc, char** argv) {const int64_t blob = (int64_t) strata::kernels::cpu::expert_layout().max_blob;int64_t slots = ((int64_t) free_b - reserve) / blob;if (!profile.empty()) slots = std::min<int64_t>(slots, (int64_t) profile.size());+#if defined(STRATA_GPU_METAL)+ // Metal: the decode window reads its experts from its own arena (metal/src/engine/window.cpp), and the device copies the shared host+ // code makes - this cache included - are given back before the arena is sized. An auto cache of everything that is "free" (14.56 GiB on+ // a 32 GB Mac) only held the memory the arena and the prompt buffers need, left 85 MiB free at the first request, and was touched+ // again after it was given back. 1500 slots is the cache the identity gates run with; --expert-cache N sets another.+ constexpr int64_t kMetalAutoCacheSlots = 1500;+ if (slots > kMetalAutoCacheSlots) {+ std::fprintf(stderr, "strata generate: expert cache auto: %d slots, not %lld: on a Mac the engine reads its experts from its own arena and "+ "this cache would only hold memory the arena and the prompt buffers need (--expert-cache N sets it)\n",+ (int) kMetalAutoCacheSlots, (long long) slots);+ slots = kMetalAutoCacheSlots;+ }+#endifo.expert_cache = (int) std::max<int64_t>(slots, 0);std::fprintf(stderr, "strata generate: expert cache auto: %.2f GiB free, %d MiB reserved (+%lld MiB for the ""draft head) -> %d slots\n",@@ -8626,6 +8639,15 @@ int main(int argc, char** argv) {"%lld MiB free\n",(long long) o.prefill_chunk, (long long) (borrow_bytes >> 20), (long long) (free_b >> 20));}+#if defined(STRATA_GPU_METAL)+ // Metal: the prompt path's buffers are the Metal window's own (a region of its expert arena, metal/src/engine/window.cpp) - there is+ // no loan of expert-cache slots. Taking one anyway made short prompts end in `Prefill::relayout` ("a chunk of at most 0": the Metal+ // Prefill has no borrowed layout) and long ones refill "lent" slots through a copy into device allocations the Metal window had given back.+ for (PfPart& p : pf_parts) { p.first = -1; p.first_now = -1; }+ lend_first = -1;+ borrow = nullptr;+ borrow_bytes = 0;+#endifbool any_loan = borrow != nullptr;for (size_t i = 1; i < pf_parts.size(); ++i) any_loan = any_loan || pf_parts[i].first >= 0;if (any_loan) {