Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 cpu lane: exact AVX2 rows for 7..64+ tokens an expert (the prompt path): decode once, u8 activations, no per-token sign; IQ1_M too

Niko12217e9b0dcc2584
diff --git a/src/dsv41/ds4_gg_cpu.cpp b/src/dsv41/ds4_gg_cpu.cpp
index 3244c13bf..0215296fa 100644
--- a/src/dsv41/ds4_gg_cpu.cpp
+++ b/src/dsv41/ds4_gg_cpu.cpp
@@ -112,7 +112,7 @@ static bool own_ok(int type) {
std::memcpy(&w[o], &d, sizeof d);
}
}
- for (int nt : {1, 2, 3, 4, 5, 6, 8}) {
+ for (int nt : {1, 2, 3, 4, 5, 6, 8, 16, 33}) {
std::vector<std::vector<uint8_t>> a((size_t) nt, std::vector<uint8_t>(ab));
std::vector<const void*> ap((size_t) nt);
std::vector<std::vector<float>> o0((size_t) nt, std::vector<float>(rows)), o1 = o0;
@@ -164,8 +164,14 @@ static void dots(int type, const uint8_t* w, size_t rb, int n, const void* const
void gg_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff, int r0,
int r1, float limit) {
- static thread_local std::vector<float> tmp[8];
- float* uo[8];
+ constexpr int kMaxTok = 64; // (the prompt path: a chunk's tokens an expert)
+ static thread_local std::vector<float> tmp[kMaxTok];
+ float* uo[kMaxTok];
+ if (nt > kMaxTok) { // (more than a pass takes: in pieces)
+ for (int t0 = 0; t0 < nt; t0 += kMaxTok)
+ gg_cpu_gate_up(f, gate, up, act + t0, nt - t0 < kMaxTok ? nt - t0 : kMaxTok, ff + t0, r0, r1, limit);
+ return;
+ }
for (int t = 0; t < nt; ++t) {
tmp[t].resize((size_t) r1);
uo[t] = tmp[t].data();
diff --git a/src/kernels/cpu/iq_avx2_rows.inl b/src/kernels/cpu/iq_avx2_rows.inl
index c30735448..6c6eb02c1 100644
--- a/src/kernels/cpu/iq_avx2_rows.inl
+++ b/src/kernels/cpu/iq_avx2_rows.inl
@@ -350,6 +350,188 @@ void dot_rows(const uint8_t* w, size_t row_bytes, int n, const void* const* act,
}
}
+// ---- many tokens an expert (the prompt: a chunk's tokens routed to one expert), EX rows only.
+// The weights of a 256-value block are decoded once into signed int8 (the grid magnitude times the sign) and every token of the
+// call applies them. The activations come as u8 (q8 + 128, made once per call for all the rows): maddubs(act_u8, w_s8) needs no
+// per-token sign, and sum(w * q) = sum(w * (q + 128)) - 128 * sum(w) with the second sum made once per weight block (corr), so the
+// integer sums are exactly ggml's, lane for lane (no int16 overflow: |w| <= 62 for these grids, 255 * 62 * 2 < 32767).
+// A token costs, per 32 values, one maddubs, one madd and one add (the single-token form has a sign on top).
+constexpr int kWideTokens = 64;
+
+STRATA_ROWS_FN inline void wide_prep(const block_q8_K* y, int nb, uint8_t* u) {
+ const __m256i flip = _mm256_set1_epi8((char) 0x80);
+ for (int i = 0; i < nb; ++i)
+ for (int c = 0; c < 8; ++c)
+ _mm256_storeu_si256((__m256i*) (u + (size_t) i * 256 + 32 * c),
+ _mm256_xor_si256(_mm256_loadu_si256((const __m256i*) (y[i].qs + 32 * c)), flip));
+}
+
+template <int TY> STRATA_ROWS_FN
+void dot_rows_wide(const uint8_t* w, size_t row_bytes, int n, const void* const* act, int T, float* const* out, int r0, int r1) {
+ // tokens in tiles of kTile and rows in groups of kRows: a tile's activation slice of one block (kTile * 256 B) and the group's
+ // accumulators (kRows * kTile * 32 B) stay in L1 while the group's rows go past, the tile's whole activation in L2
+#ifndef STRATA_WIDE_TILE
+#define STRATA_WIDE_TILE 16
+#endif
+#ifndef STRATA_WIDE_ROWS
+#define STRATA_WIDE_ROWS 8
+#endif
+ constexpr int kTile = STRATA_WIDE_TILE, kRows = STRATA_WIDE_ROWS;
+ const int nb = n / QK_K;
+ static thread_local std::vector<uint8_t> ubuf;
+ static thread_local std::vector<float> accb;
+ ubuf.resize((size_t) (T < kTile ? T : kTile) * n);
+ accb.resize((size_t) kRows * kTile * 8);
+ const __m256i one8 = _mm256_set1_epi8(1);
+ const int pf = prefetch_distance();
+ for (int t0 = 0; t0 < T; t0 += kTile) {
+ const int tt = T - t0 < kTile ? T - t0 : kTile;
+ const block_q8_K* y[kTile];
+ for (int t = 0; t < tt; ++t) {
+ y[t] = (const block_q8_K*) act[t0 + t];
+ wide_prep(y[t], nb, ubuf.data() + (size_t) t * n);
+ }
+ for (int rg = r0; rg < r1; rg += kRows) {
+ const int rr = r1 - rg < kRows ? r1 - rg : kRows;
+ for (int k = 0; k < rr * tt; ++k) _mm256_storeu_ps(&accb[(size_t) k * 8], _mm256_setzero_ps());
+ for (int i = 0; i < nb; ++i) {
+ for (int q = 0; q < rr; ++q) {
+ const uint8_t* blk = w + (size_t) (rg + q) * row_bytes + (size_t) i * Fmt32<TY>::bytes;
+ rows_ahead(blk, pf);
+ __m256i w8[8], sc[8];
+ __m256i corr = _mm256_setzero_si256();
+ for (int j = 0; j < 4; ++j)
+ for (int half = 0; half < 2; ++half) {
+ __m256i g, sgn, s;
+ Fmt32<TY>::decode(blk, j, half, g, sgn, s);
+ const int k = 2 * j + half;
+ w8[k] = _mm256_sign_epi8(g, sgn);
+ sc[k] = s;
+ corr = madd_add(corr, _mm256_maddubs_epi16(one8, w8[k]), s);
+ }
+ const __m256i corr128 = _mm256_slli_epi32(corr, 7);
+ const float dx = h2f(u16(blk));
+ float* accq = &accb[(size_t) q * tt * 8];
+ for (int t = 0; t < tt; ++t) {
+ const uint8_t* u = ubuf.data() + (size_t) t * n + (size_t) i * 256;
+ __m256i a0 = _mm256_setzero_si256(), a1 = _mm256_setzero_si256();
+ for (int k = 0; k < 8; k += 2) {
+ a0 = madd_add(a0, _mm256_maddubs_epi16(_mm256_loadu_si256((const __m256i*) (u + 32 * k)), w8[k]), sc[k]);
+ a1 = madd_add(a1, _mm256_maddubs_epi16(_mm256_loadu_si256((const __m256i*) (u + 32 * k + 32)), w8[k + 1]), sc[k + 1]);
+ }
+ const __m256i s = _mm256_sub_epi32(_mm256_add_epi32(a0, a1), corr128);
+ float* ap = accq + (size_t) t * 8;
+ _mm256_storeu_ps(ap, _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(s), _mm256_loadu_ps(ap)));
+ }
+ }
+ }
+ for (int q = 0; q < rr; ++q)
+ for (int t = 0; t < tt; ++t)
+ out[t0 + t][rg + q] = Fmt32<TY>::K * ggml_hsum8(_mm256_loadu_ps(&accb[((size_t) q * tt + t) * 8]));
+ }
+ }
+}
+
+// IQ1_M the same way: its grid bytes are already signed (-1, 0, 1), and the delta dot keeps the pair sums of the activation
+// (maddubs(1, q8), made once per tile). Two float accumulators a (row, token), ggml's sum d*sumi1 and sum d*sumi2.
+STRATA_ROWS_FN
+void dot_rows_wide_iq1m(const uint8_t* w, size_t row_bytes, int n, const void* const* act, int T, float* const* out, int r0, int r1) {
+ constexpr int kTile = STRATA_WIDE_TILE, kRows = STRATA_WIDE_ROWS;
+ const int nb = n / QK_K;
+ static thread_local std::vector<uint8_t> ubuf;
+ static thread_local std::vector<int16_t> pbuf;
+ static thread_local std::vector<float> accb;
+ ubuf.resize((size_t) (T < kTile ? T : kTile) * n);
+ pbuf.resize((size_t) (T < kTile ? T : kTile) * n);
+ accb.resize((size_t) kRows * kTile * 16);
+ const __m256i one8 = _mm256_set1_epi8(1);
+ const __m256i mask = _mm256_set1_epi16(0x7);
+ const __m256i mone = _mm256_set1_epi16(1);
+ const __m256i mtwo8 = _mm256_set1_epi8(2);
+ const __m256i scales_shift = _mm256_set_epi64x(9, 3, 6, 0);
+ const __m256i nib_shift = _mm256_setr_epi32(0, 4, 8, 12, 16, 20, 24, 28);
+ const __m256i m7 = _mm256_set1_epi32(7), m8 = _mm256_set1_epi32(8);
+ const int pf = prefetch_distance();
+ for (int t0 = 0; t0 < T; t0 += kTile) {
+ const int tt = T - t0 < kTile ? T - t0 : kTile;
+ const block_q8_K* y[kTile];
+ for (int t = 0; t < tt; ++t) {
+ y[t] = (const block_q8_K*) act[t0 + t];
+ wide_prep(y[t], nb, ubuf.data() + (size_t) t * n);
+ iq1m_pre_build(y[t], nb, pbuf.data() + (size_t) t * n);
+ }
+ for (int rg = r0; rg < r1; rg += kRows) {
+ const int rr = r1 - rg < kRows ? r1 - rg : kRows;
+ for (int k = 0; k < rr * tt * 2; ++k) _mm256_storeu_ps(&accb[(size_t) k * 8], _mm256_setzero_ps());
+ for (int i = 0; i < nb; ++i) {
+ for (int q = 0; q < rr; ++q) {
+ const uint8_t* blk = w + (size_t) (rg + q) * row_bytes + (size_t) i * 56;
+ rows_ahead(blk, pf);
+ const uint8_t* qs = blk;
+ const uint8_t* qh = blk + 32;
+ const uint8_t* scb = blk + 48;
+ const uint16_t sc0 = u16(scb), sc1 = u16(scb + 2), sc2 = u16(scb + 4), sc3 = u16(scb + 6);
+ const uint16_t d16 = (uint16_t) ((sc0 >> 12) | ((sc1 >> 8) & 0x00f0) | ((sc2 >> 4) & 0x0f00) | (sc3 & 0xf000));
+ __m256i scales = _mm256_set1_epi64x((long long) u64(scb));
+ scales = _mm256_srlv_epi64(scales, scales_shift);
+ scales = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scales, mask), 1), mone);
+ __m256i sidx1 = _mm256_set1_epi16(0x0100);
+ __m256i sidx2 = _mm256_add_epi8(sidx1, _mm256_set1_epi8(8));
+ __m256i g1[4], g2[4], sa[4], sb[4], ssa[4], ssb[4];
+ __m256i corr = _mm256_setzero_si256();
+ for (int it = 0; it < 4; ++it) {
+ const __m256i v8 = _mm256_srlv_epi32(_mm256_set1_epi32((int) u32(qh + 4 * it)), nib_shift);
+ const __m256i idx = _mm256_or_si256(_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) (qs + 8 * it))),
+ _mm256_slli_epi32(_mm256_and_si256(v8, m7), 8));
+ alignas(32) int32_t ix[8];
+ _mm256_store_si256((__m256i*) ix, idx);
+ g1[it] = _mm256_set_epi64x((long long) iq1s_grid[ix[3]], (long long) iq1s_grid[ix[2]],
+ (long long) iq1s_grid[ix[1]], (long long) iq1s_grid[ix[0]]);
+ g2[it] = _mm256_set_epi64x((long long) iq1s_grid[ix[7]], (long long) iq1s_grid[ix[6]],
+ (long long) iq1s_grid[ix[5]], (long long) iq1s_grid[ix[4]]);
+ const __m256i neg = _mm256_cmpeq_epi32(_mm256_and_si256(v8, m8), m8);
+ const __m256i n1 = _mm256_cvtepi32_epi64(_mm256_castsi256_si128(neg));
+ const __m256i n2 = _mm256_cvtepi32_epi64(_mm256_extracti128_si256(neg, 1));
+ sa[it] = _mm256_shuffle_epi8(scales, sidx1);
+ sb[it] = _mm256_shuffle_epi8(scales, sidx2);
+ ssa[it] = _mm256_sign_epi16(sa[it], _mm256_or_si256(n1, mone));
+ ssb[it] = _mm256_sign_epi16(sb[it], _mm256_or_si256(n2, mone));
+ sidx1 = _mm256_add_epi8(sidx1, mtwo8);
+ sidx2 = _mm256_add_epi8(sidx2, mtwo8);
+ corr = madd_add(madd_add(corr, _mm256_maddubs_epi16(one8, g1[it]), sa[it]), _mm256_maddubs_epi16(one8, g2[it]), sb[it]);
+ }
+ const __m256i corr128 = _mm256_slli_epi32(corr, 7);
+ const float dx = h2f(d16);
+ float* accq = &accb[(size_t) q * tt * 16];
+ for (int t = 0; t < tt; ++t) {
+ const uint8_t* u = ubuf.data() + (size_t) t * n + (size_t) i * 256;
+ const int16_t* pp = pbuf.data() + (size_t) t * n + (size_t) i * 128;
+ __m256i a0 = _mm256_setzero_si256(), a1 = _mm256_setzero_si256();
+ __m256i b0 = _mm256_setzero_si256(), b1 = _mm256_setzero_si256();
+ for (int it = 0; it < 4; ++it) {
+ a0 = madd_add(a0, _mm256_maddubs_epi16(_mm256_loadu_si256((const __m256i*) (u + 64 * it)), g1[it]), sa[it]);
+ a1 = madd_add(a1, _mm256_maddubs_epi16(_mm256_loadu_si256((const __m256i*) (u + 64 * it + 32)), g2[it]), sb[it]);
+ b0 = madd_add(b0, _mm256_loadu_si256((const __m256i*) (pp + 32 * it)), ssa[it]);
+ b1 = madd_add(b1, _mm256_loadu_si256((const __m256i*) (pp + 32 * it + 16)), ssb[it]);
+ }
+ const __m256i s1 = _mm256_sub_epi32(_mm256_add_epi32(a0, a1), corr128);
+ const __m256i s2 = _mm256_add_epi32(b0, b1);
+ const __m256 d = _mm256_set1_ps(y[t][i].d * dx);
+ float* ap = accq + (size_t) t * 16;
+ _mm256_storeu_ps(ap, _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(s1), _mm256_loadu_ps(ap)));
+ _mm256_storeu_ps(ap + 8, _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(s2), _mm256_loadu_ps(ap + 8)));
+ }
+ }
+ }
+ for (int q = 0; q < rr; ++q)
+ for (int t = 0; t < tt; ++t) {
+ const float* ap = &accb[((size_t) q * tt + t) * 16];
+ out[t0 + t][rg + q] = ggml_hsum8(_mm256_loadu_ps(ap)) + 0.125f * ggml_hsum8(_mm256_loadu_ps(ap + 8));
+ }
+ }
+ }
+}
+
template <int TY, bool EX> STRATA_ROWS_FN
void gu_rows_nt(int nt, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act,
float* const* ff, int r0, int r1) {
@@ -368,6 +550,13 @@ void gu_rows_nt(int nt, const uint8_t* blob, size_t gu_row, size_t up_off, int n
template <int TY, bool EX> STRATA_ROWS_FN
void dot_rows_nt(int nt, const uint8_t* w, size_t row_bytes, int n, const void* const* act, float* const* out, int r0,
int r1) {
+ if constexpr (EX) {
+ if (nt >= 7) { // (a prompt chunk's tokens: one decode of a block for all of them)
+ if constexpr (TY == 29) dot_rows_wide_iq1m(w, row_bytes, n, act, nt, out, r0, r1);
+ else dot_rows_wide<TY>(w, row_bytes, n, act, nt, out, r0, r1);
+ return;
+ }
+ }
switch (nt) {
case 1: dot_rows<TY, 1, EX>(w, row_bytes, n, act, out, r0, r1); break;
case 2: dot_rows<TY, 2, EX>(w, row_bytes, n, act, out, r0, r1); break;
diff --git a/tests/dsv41/gg_dot_bench.cpp b/tests/dsv41/gg_dot_bench.cpp
index 03b002e10..27604acbd 100644
--- a/tests/dsv41/gg_dot_bench.cpp
+++ b/tests/dsv41/gg_dot_bench.cpp
@@ -14,6 +14,7 @@
#include <atomic>
#include <chrono>
#include <cmath>
+#include <functional>
#include <cstdio>
#include <cstdlib>
#include <cstring>
@@ -26,10 +27,103 @@ using clk = std::chrono::steady_clock;
static double secs(clk::time_point a) { return std::chrono::duration<double>(clk::now() - a).count(); }
+// ---- --expert: the engine's expert pipeline on random experts (gate+up+SwiGLU, quantize h, down) in jobs of 64 rows over N threads, for
+// ggml-cpu's dots (mode 0), the default rule (2) and the AVX2 rows always (1); GB/s of all three matrices' bytes, outputs compared bit for bit.
+static void fill_blocks(uint8_t* w, size_t bytes, int type, uint64_t seed) {
+ uint64_t st = 0x9E3779B97F4A7C15ull ^ (seed * 0xD1B54A32D192ED03ull);
+ auto next = [&st] { st ^= st << 13; st ^= st >> 7; st ^= st << 17; return st; };
+ for (size_t o = 0; o + 8 <= bytes; o += 8) { const uint64_t v = next(); std::memcpy(w + o, &v, 8); }
+ const size_t bb = ggml_type_size((ggml_type) type);
+ for (size_t o = 0; o + bb <= bytes; o += bb) {
+ const ggml_fp16_t d = ggml_fp32_to_fp16(0.002f + 0.001f * (float) (next() % 16));
+ if (type == 29) {
+ uint16_t sc[4];
+ std::memcpy(sc, w + o + 48, 8);
+ for (int k = 0; k < 4; ++k) sc[k] = (uint16_t) ((sc[k] & 0x0fff) | ((d >> (4 * k)) & 0xf) << 12);
+ std::memcpy(w + o + 48, sc, 8);
+ } else {
+ std::memcpy(w + o, &d, sizeof d);
+ }
+ }
+}
+
+static int run_expert(int tg, int td, int threads, long mb, int nt, int reps) {
+ ggml_cpu_init();
+ const int E = 5120, FF = 2304;
+ IqCpu f;
+ std::string err;
+ if (!gg_cpu_fmt(tg, td, E, FF, f, err)) { std::fprintf(stderr, "gg_cpu_fmt: %s\n", err.c_str()); return 2; }
+ const size_t gbytes = f.gu_row * FF, dbytes = f.d_row * E, ebytes = 2 * gbytes + dbytes;
+ const int ne = (int) std::max(2L, (long) ((size_t) mb * 1000000 / ebytes));
+ std::vector<uint8_t> gw((size_t) ne * gbytes), uw((size_t) ne * gbytes), dw((size_t) ne * dbytes);
+ for (int e = 0; e < ne; ++e) {
+ fill_blocks(&gw[(size_t) e * gbytes], gbytes, tg, 3 * (uint64_t) e + 1);
+ fill_blocks(&uw[(size_t) e * gbytes], gbytes, tg, 3 * (uint64_t) e + 2);
+ fill_blocks(&dw[(size_t) e * dbytes], dbytes, td, 3 * (uint64_t) e + 3);
+ }
+ uint64_t st = 12345;
+ auto next = [&st] { st ^= st << 13; st ^= st >> 7; st ^= st << 17; return st; };
+ std::vector<std::vector<uint8_t>> act((size_t) nt, std::vector<uint8_t>(f.act_bytes));
+ for (int t = 0; t < nt; ++t) {
+ std::vector<float> x((size_t) E);
+ for (auto& v : x) v = ((float) (next() % 2001) - 1000.0f) * 1e-3f;
+ gg_cpu_quant(f.gu_act, x.data(), E, act[(size_t) t].data());
+ }
+ std::vector<float> h((size_t) ne * nt * FF), y((size_t) ne * nt * E);
+ std::vector<uint8_t> hq((size_t) ne * nt * f.h_bytes);
+ const int RB = 64;
+ auto run_jobs = [&](long njobs, const std::function<void(long)>& fn) {
+ std::atomic<long> cur{0};
+ std::vector<std::thread> th;
+ for (int k = 0; k < threads; ++k)
+ th.emplace_back([&] { for (;;) { const long j = cur.fetch_add(1); if (j >= njobs) break; fn(j); } });
+ for (auto& t : th) t.join();
+ };
+ auto pass = [&](int mode) {
+ gg_cpu_set_iq256(mode);
+ double best = 1e30;
+ for (int r = 0; r < reps + 1; ++r) {
+ const auto t0 = clk::now();
+ const long gb = (FF + RB - 1) / RB, db = (E + RB - 1) / RB;
+ run_jobs((long) ne * gb, [&](long j) {
+ const int e = (int) (j / gb), r0 = (int) (j % gb) * RB, r1 = std::min(FF, r0 + RB);
+ const void* ap[64];
+ float* hp[64];
+ for (int t = 0; t < nt; ++t) { ap[t] = act[(size_t) t].data(); hp[t] = &h[((size_t) e * nt + t) * FF]; }
+ gg_cpu_gate_up(f, &gw[(size_t) e * gbytes], &uw[(size_t) e * gbytes], ap, nt, hp, r0, r1, 10.0f);
+ });
+ run_jobs((long) ne * nt, [&](long j) { gg_cpu_quant(f.d_act, &h[(size_t) j * FF], FF, &hq[(size_t) j * f.h_bytes]); });
+ run_jobs((long) ne * db, [&](long j) {
+ const int e = (int) (j / db), r0 = (int) (j % db) * RB, r1 = std::min(E, r0 + RB);
+ const void* hp[64];
+ float* yp[64];
+ for (int t = 0; t < nt; ++t) { hp[t] = &hq[((size_t) e * nt + t) * f.h_bytes]; yp[t] = &y[((size_t) e * nt + t) * E]; }
+ gg_cpu_down(f, &dw[(size_t) e * dbytes], hp, nt, yp, r0, r1);
+ });
+ if (r > 0) best = std::min(best, secs(t0));
+ }
+ return best;
+ };
+ const double gb = (double) ne * ebytes / 1e9;
+ std::vector<float> y0;
+ double t0s = 0;
+ int rc = 0;
+ for (int mode : {0, 2, 1}) {
+ const double t = pass(mode);
+ if (mode == 0) { y0 = y; t0s = t; }
+ const bool same = std::memcmp(y0.data(), y.data(), y.size() * 4) == 0;
+ std::printf("expert gate/up %d down %d nt %d threads %d: %d experts, %.2f GB | mode %s %.3f s = %.1f GB/s (x%.2f) %s\n", tg, td, nt,
+ threads, ne, gb, mode == 0 ? "ggml " : mode == 2 ? "auto " : "own ", t, gb / t, t0s / t, same ? "bit-identical" : "DIFFERENT");
+ if (!same) rc = 1;
+ }
+ return rc;
+}
+
int main(int argc, char** argv) {
int type = 18, threads = 32, nt = 1, reps = 3;
long mb = 3072;
- bool exact = false;
+ bool exact = false, expert = false;
+ int dtype = -1;
for (int i = 1; i < argc; ++i) {
std::string a = argv[i];
auto val = [&] { return i + 1 < argc ? std::atol(argv[++i]) : 0L; };
@@ -39,7 +133,10 @@ int main(int argc, char** argv) {
else if (a == "--nt") nt = (int) val();
else if (a == "--reps") reps = (int) val();
else if (a == "--exact") exact = true;
+ else if (a == "--expert") expert = true;
+ else if (a == "--dtype") dtype = (int) val();
}
+ if (expert) return run_expert(type, dtype < 0 ? type : dtype, threads, mb, nt, reps);
ggml_cpu_init();
const int n = 5120; // a gate / up row
IqCpu f;