Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 win: --dspark auto (not loaded when the experts do not fit the GPU), --ids @file, h100sim
Niko12210b48e23fa567
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex 608923b04..1276dc636 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -22,6 +22,8 @@#include <chrono>#include <condition_variable>#include <cstdio>+#include <fstream>+#include <sstream>#include <cstdlib>#include <cstring>#include <deque>@@ -677,6 +679,7 @@ int main(int argc, char** argv) {const std::string v = val("--dspark");spec = v != "off";o.dspark = v != "none"; // none: not even loaded+ o.dspark_auto = v == "auto";} else if (a == "--dspark-conf") o.dspark_conf = (float) std::atof(val("--dspark-conf").c_str());else if (a == "--tp") o.tp = std::max(1, std::atoi(val("--tp").c_str()));else if (a == "--prefill-chunk") o.prefill_chunk = std::max(0, std::atoi(val("--prefill-chunk").c_str()));@@ -764,6 +767,12 @@ int main(int argc, char** argv) {}return 0;}+ if (!ids_s.empty() && ids_s[0] == '@') { // --ids @file: the ids in a file (a command line is 32 KB on Windows)+ std::ifstream idf(ids_s.substr(1));+ std::stringstream ss;+ ss << idf.rdbuf();+ ids_s = ss.str();+ }const std::vector<int> ids = parse_ids(ids_s);if (!images.empty()) { // --images: a GENI file's pictures, for the prompt's spansstd::vector<Pic> pics;diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex bd23b83c0..c0fcf21f6 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -1899,6 +1899,17 @@ struct Model::Impl {if (o.vision_only) return true;// layers: roles and the sources they read+ if (o.dspark_auto && o.dspark && g.n_mtp > 0) { // (see Options::dspark_auto)+ double exp_b = 0, room_b = 0;+ for (const auto& kt : pk.t)+ if (kt.first.rfind("layers.", 0) == 0 && kt.first.find(".ffn.experts.") != std::string::npos) exp_b += (double) kt.second.bytes;+ for (const Dev& d : devs) room_b += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20));+ if (exp_b > 1.5 * room_b) {+ o.dspark = false;+ std::fprintf(stderr, "ds4: DSpark not loaded: the routed experts (%.0f GB) are far more than the GPU holds (%.0f GB), where its verify "+ "rounds read 3.3x a token's bytes and gain nothing (--dspark on loads it anyway)\n", exp_b / 1e9, room_b / 1e9);+ }+ }if (!o.dspark) g.n_mtp = 0;L.resize((size_t) (g.nl + g.n_mtp));int last_src = -1;diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hppindex c291d13ac..dc5925cf4 100644--- a/src/dsv41/ds4_model.hpp+++ b/src/dsv41/ds4_model.hpp@@ -37,6 +37,10 @@ struct Options {// most (never past the RAM the process can still get less 8 GB); 0: off, the OS page cache as beforedouble ssd_cache_gb = -1.0; // < 0: auto (the RAM the process can get less a floor, at most 48 GB; off below 4 GB), 0: offint ssd_policy = 0;+ // dspark_auto: the drafter is loaded only where the routed experts fit the GPUs (more than 1.5x what they have, the verify rounds+ // read 3.3x the bytes of a token and measured no faster than plain decoding, and the drafter's 7 GB of experts take the VRAM the+ // trunk's hottest would have had); --dspark on / off / none says it outright+ bool dspark_auto = true;bool dspark = true; // load the DSpark drafter (the checkpoint's mtp.* blocks) for speculative decodingfloat dspark_conf = 0.5f; // a draft is verified while its confidence (sigmoid) is at least this// tensor parallelism: runs of tp GPUs of one kind (the same memory) take the same layers, each a share of thediff --git a/tools/dsv41/win/h100sim.py b/tools/dsv41/win/h100sim.pynew file mode 100644index 000000000..e362375d9--- /dev/null+++ b/tools/dsv41/win/h100sim.py@@ -0,0 +1,132 @@+"""h100sim.py - replay the H100 routing corpus (1.78M tokens, code/agent/chat/multi/math/long, DS4_TRACE format, see+phaseA-tests/h100/trace_tools.py) through an LRU expert cache at this PC's sizes, with and without a profile warm start.++ h100sim.py <trace.bin[.zst]> <profile.bin> [--tokens N] [--skip S] [--caps 30,40,44] [--sizes sizes.json]++Prints the demand hit rate by bytes after a warm-up of `--warm` tokens, for: empty start LRU; LRU started with the profile's+hottest experts in (the engine's preload) and the same cold-start window's hit rate (first 2000 tokens), and a static top-N+(profile only, no adaptation)."""+import argparse+import collections+import io+import json+import struct+import sys++import numpy as np+++def decode(path, max_tokens, skip):+ if path.endswith('.zst'):+ import zstandard+ raw = zstandard.ZstdDecompressor().stream_reader(open(path, 'rb')).read()+ else:+ raw = open(path, 'rb').read()+ buf = np.frombuffer(raw, dtype=np.uint8)+ pos, n = 0, len(buf)+ per = {l: [] for l in range(40)}+ got = [0] * 40+ while pos < n:+ b0 = int(buf[pos])+ if b0 & 0x80:+ l = b0 & 0x7F+ T = int(buf[pos + 1]) | (int(buf[pos + 2]) << 8)+ pos += 3+ else:+ l = b0+ T = int(buf[pos + 1]) & 0x7F+ pos += 2+ sz = T * 12+ if pos + sz > n:+ break+ per[l].append(buf[pos:pos + sz].view(np.int16).reshape(T, 6))+ pos += sz+ if min(got) >= skip + max_tokens:+ break+ got[l] += T+ arr = []+ for l in range(40):+ a = np.concatenate(per[l])[skip:skip + max_tokens]+ arr.append(a)+ m = min(len(a) for a in arr)+ return np.stack([a[:m] for a in arr], axis=1) # [tokens][layer][6]+++def lru(acc_iter, size, cap, init=None, warm=0):+ od = collections.OrderedDict()+ used = 0+ if init:+ for k in init:+ od[k] = size[k[0]]+ used += size[k[0]]+ hb = tb = 0.0+ first_hb = first_tb = 0.0+ for i, (t, k) in enumerate(acc_iter):+ sz = size[k[0]]+ cnt = t >= warm+ early = t < 2000+ if k in od:+ od.move_to_end(k)+ if cnt:+ hb += sz+ if early:+ first_hb += sz+ else:+ od[k] = sz+ used += sz+ while used > cap:+ _, s0 = od.popitem(last=False)+ used -= s0+ if cnt:+ tb += sz+ if early:+ first_tb += sz+ return 100 * hb / max(tb, 1), 100 * first_hb / max(first_tb, 1)+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument('trace')+ ap.add_argument('profile')+ ap.add_argument('--tokens', type=int, default=40000)+ ap.add_argument('--skip', type=int, default=0)+ ap.add_argument('--warm', type=int, default=4000)+ ap.add_argument('--caps', default='30,40')+ ap.add_argument('--sizes', default=r'C:\ds-ista-work\sizes.json')+ a = ap.parse_args()+ d = json.load(open(a.sizes))+ size = {int(k): float(v) for k, v in d['exp_layer'].items()}+ tr = decode(a.trace, a.tokens, a.skip)+ prof = np.fromfile(a.profile, dtype=np.uint32).reshape(-1, 384)[:40]+ print('trace', tr.shape, 'tokens', len(tr))++ def acc():+ for t in range(len(tr)):+ for l in range(40):+ for e in tr[t, l]:+ yield t, (l, int(e))+ dens = sorted(((prof[l, e] / size[l], l, e) for l in range(40) for e in range(384)), reverse=True)+ for cg in [float(x) for x in a.caps.split(',')]:+ cap = cg * 1e9+ r0, f0 = lru(acc(), size, cap, None, a.warm)+ order, used = [], 0.0+ for _, l, e in dens:+ if used + size[l] > cap:+ break+ order.append((l, e))+ used += size[l]+ r1, f1 = lru(acc(), size, cap, list(reversed(order)), a.warm) # hottest last = most recent+ # static: top-N of the profile, no adaptation+ sel = set(order)+ hb = tb = 0.0+ for t, k in acc():+ if t >= a.warm:+ tb += size[k[0]]+ if k in sel:+ hb += size[k[0]]+ print('cap %.0f GB: LRU from empty: steady hit %.1f%% (first 2000 tokens %.1f%%); LRU from the profile\'s hottest: steady %.1f%% (first 2000 tokens %.1f%%); static top-N alone %.1f%%' % (+ cg, r0, f0, r1, f1, 100 * hb / tb))+++if __name__ == '__main__':+ main()