Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 win: --dspark auto (not loaded when the experts do not fit the GPU), --ids @file, h100sim

Niko12210b48e23fa567
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
index 608923b04..1276dc636 100644
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@ -22,6 +22,8 @@
#include <chrono>
#include <condition_variable>
#include <cstdio>
+#include <fstream>
+#include <sstream>
#include <cstdlib>
#include <cstring>
#include <deque>
@@ -677,6 +679,7 @@ int main(int argc, char** argv) {
const std::string v = val("--dspark");
spec = v != "off";
o.dspark = v != "none"; // none: not even loaded
+ o.dspark_auto = v == "auto";
} else if (a == "--dspark-conf") o.dspark_conf = (float) std::atof(val("--dspark-conf").c_str());
else if (a == "--tp") o.tp = std::max(1, std::atoi(val("--tp").c_str()));
else if (a == "--prefill-chunk") o.prefill_chunk = std::max(0, std::atoi(val("--prefill-chunk").c_str()));
@@ -764,6 +767,12 @@ int main(int argc, char** argv) {
}
return 0;
}
+ if (!ids_s.empty() && ids_s[0] == '@') { // --ids @file: the ids in a file (a command line is 32 KB on Windows)
+ std::ifstream idf(ids_s.substr(1));
+ std::stringstream ss;
+ ss << idf.rdbuf();
+ ids_s = ss.str();
+ }
const std::vector<int> ids = parse_ids(ids_s);
if (!images.empty()) { // --images: a GENI file's pictures, for the prompt's spans
std::vector<Pic> pics;
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index bd23b83c0..c0fcf21f6 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -1899,6 +1899,17 @@ struct Model::Impl {
if (o.vision_only) return true;
// layers: roles and the sources they read
+ if (o.dspark_auto && o.dspark && g.n_mtp > 0) { // (see Options::dspark_auto)
+ double exp_b = 0, room_b = 0;
+ for (const auto& kt : pk.t)
+ if (kt.first.rfind("layers.", 0) == 0 && kt.first.find(".ffn.experts.") != std::string::npos) exp_b += (double) kt.second.bytes;
+ for (const Dev& d : devs) room_b += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20));
+ if (exp_b > 1.5 * room_b) {
+ o.dspark = false;
+ std::fprintf(stderr, "ds4: DSpark not loaded: the routed experts (%.0f GB) are far more than the GPU holds (%.0f GB), where its verify "
+ "rounds read 3.3x a token's bytes and gain nothing (--dspark on loads it anyway)\n", exp_b / 1e9, room_b / 1e9);
+ }
+ }
if (!o.dspark) g.n_mtp = 0;
L.resize((size_t) (g.nl + g.n_mtp));
int last_src = -1;
diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hpp
index c291d13ac..dc5925cf4 100644
--- a/src/dsv41/ds4_model.hpp
+++ b/src/dsv41/ds4_model.hpp
@@ -37,6 +37,10 @@ struct Options {
// most (never past the RAM the process can still get less 8 GB); 0: off, the OS page cache as before
double ssd_cache_gb = -1.0; // < 0: auto (the RAM the process can get less a floor, at most 48 GB; off below 4 GB), 0: off
int ssd_policy = 0;
+ // dspark_auto: the drafter is loaded only where the routed experts fit the GPUs (more than 1.5x what they have, the verify rounds
+ // read 3.3x the bytes of a token and measured no faster than plain decoding, and the drafter's 7 GB of experts take the VRAM the
+ // trunk's hottest would have had); --dspark on / off / none says it outright
+ bool dspark_auto = true;
bool dspark = true; // load the DSpark drafter (the checkpoint's mtp.* blocks) for speculative decoding
float dspark_conf = 0.5f; // a draft is verified while its confidence (sigmoid) is at least this
// tensor parallelism: runs of tp GPUs of one kind (the same memory) take the same layers, each a share of the
diff --git a/tools/dsv41/win/h100sim.py b/tools/dsv41/win/h100sim.py
new file mode 100644
index 000000000..e362375d9
--- /dev/null
+++ b/tools/dsv41/win/h100sim.py
@@ -0,0 +1,132 @@
+"""h100sim.py - replay the H100 routing corpus (1.78M tokens, code/agent/chat/multi/math/long, DS4_TRACE format, see
+phaseA-tests/h100/trace_tools.py) through an LRU expert cache at this PC's sizes, with and without a profile warm start.
+
+ h100sim.py <trace.bin[.zst]> <profile.bin> [--tokens N] [--skip S] [--caps 30,40,44] [--sizes sizes.json]
+
+Prints the demand hit rate by bytes after a warm-up of `--warm` tokens, for: empty start LRU; LRU started with the profile's
+hottest experts in (the engine's preload) and the same cold-start window's hit rate (first 2000 tokens), and a static top-N
+(profile only, no adaptation)."""
+import argparse
+import collections
+import io
+import json
+import struct
+import sys
+
+import numpy as np
+
+
+def decode(path, max_tokens, skip):
+ if path.endswith('.zst'):
+ import zstandard
+ raw = zstandard.ZstdDecompressor().stream_reader(open(path, 'rb')).read()
+ else:
+ raw = open(path, 'rb').read()
+ buf = np.frombuffer(raw, dtype=np.uint8)
+ pos, n = 0, len(buf)
+ per = {l: [] for l in range(40)}
+ got = [0] * 40
+ while pos < n:
+ b0 = int(buf[pos])
+ if b0 & 0x80:
+ l = b0 & 0x7F
+ T = int(buf[pos + 1]) | (int(buf[pos + 2]) << 8)
+ pos += 3
+ else:
+ l = b0
+ T = int(buf[pos + 1]) & 0x7F
+ pos += 2
+ sz = T * 12
+ if pos + sz > n:
+ break
+ per[l].append(buf[pos:pos + sz].view(np.int16).reshape(T, 6))
+ pos += sz
+ if min(got) >= skip + max_tokens:
+ break
+ got[l] += T
+ arr = []
+ for l in range(40):
+ a = np.concatenate(per[l])[skip:skip + max_tokens]
+ arr.append(a)
+ m = min(len(a) for a in arr)
+ return np.stack([a[:m] for a in arr], axis=1) # [tokens][layer][6]
+
+
+def lru(acc_iter, size, cap, init=None, warm=0):
+ od = collections.OrderedDict()
+ used = 0
+ if init:
+ for k in init:
+ od[k] = size[k[0]]
+ used += size[k[0]]
+ hb = tb = 0.0
+ first_hb = first_tb = 0.0
+ for i, (t, k) in enumerate(acc_iter):
+ sz = size[k[0]]
+ cnt = t >= warm
+ early = t < 2000
+ if k in od:
+ od.move_to_end(k)
+ if cnt:
+ hb += sz
+ if early:
+ first_hb += sz
+ else:
+ od[k] = sz
+ used += sz
+ while used > cap:
+ _, s0 = od.popitem(last=False)
+ used -= s0
+ if cnt:
+ tb += sz
+ if early:
+ first_tb += sz
+ return 100 * hb / max(tb, 1), 100 * first_hb / max(first_tb, 1)
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument('trace')
+ ap.add_argument('profile')
+ ap.add_argument('--tokens', type=int, default=40000)
+ ap.add_argument('--skip', type=int, default=0)
+ ap.add_argument('--warm', type=int, default=4000)
+ ap.add_argument('--caps', default='30,40')
+ ap.add_argument('--sizes', default=r'C:\ds-ista-work\sizes.json')
+ a = ap.parse_args()
+ d = json.load(open(a.sizes))
+ size = {int(k): float(v) for k, v in d['exp_layer'].items()}
+ tr = decode(a.trace, a.tokens, a.skip)
+ prof = np.fromfile(a.profile, dtype=np.uint32).reshape(-1, 384)[:40]
+ print('trace', tr.shape, 'tokens', len(tr))
+
+ def acc():
+ for t in range(len(tr)):
+ for l in range(40):
+ for e in tr[t, l]:
+ yield t, (l, int(e))
+ dens = sorted(((prof[l, e] / size[l], l, e) for l in range(40) for e in range(384)), reverse=True)
+ for cg in [float(x) for x in a.caps.split(',')]:
+ cap = cg * 1e9
+ r0, f0 = lru(acc(), size, cap, None, a.warm)
+ order, used = [], 0.0
+ for _, l, e in dens:
+ if used + size[l] > cap:
+ break
+ order.append((l, e))
+ used += size[l]
+ r1, f1 = lru(acc(), size, cap, list(reversed(order)), a.warm) # hottest last = most recent
+ # static: top-N of the profile, no adaptation
+ sel = set(order)
+ hb = tb = 0.0
+ for t, k in acc():
+ if t >= a.warm:
+ tb += size[k[0]]
+ if k in sel:
+ hb += size[k[0]]
+ print('cap %.0f GB: LRU from empty: steady hit %.1f%% (first 2000 tokens %.1f%%); LRU from the profile\'s hottest: steady %.1f%% (first 2000 tokens %.1f%%); static top-N alone %.1f%%' % (
+ cg, r0, f0, r1, f1, 100 * hb / tb))
+
+
+if __name__ == '__main__':
+ main()