Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 win: pack_gguf reads the IST-DAS Engram parameters from tensors, IQ1_M type, packinfo 64-bit sizes, DS4_TRACE_AHEAD, cachesim
Niko1221eb6c0a23f0e2
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 0562e5b94..774cfce73 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -5259,13 +5259,31 @@ struct Model::Impl {int nslots = 0;std::vector<std::pair<int, int>> picks; // (expert, t * kPicks + i), CPU-lane onlyif (trace_f != nullptr && !l.mtp) { // DS4_TRACE=file: one record a route request: u8 layer, u8 T, T x k int16 expert ids- uint8_t hd[2] = {(uint8_t) li, (uint8_t) T};+ // DS4_TRACE_AHEAD=1: T has bit 7 set and the guesses follow: u8 na1, u8 na2, T x (na1 + na2) int16 (the+ // next layer's likely picks by rank, then the one after's), as the lookahead made them for THIS layer's+ // predecessors (the request carries the guesses for the layers after it)+ static const bool with_ahead = std::getenv("DS4_TRACE_AHEAD") != nullptr;+ uint8_t hd[2] = {(uint8_t) li, (uint8_t) (T | (with_ahead ? 0x80 : 0))};std::fwrite(hd, 1, 2, trace_f);for (int t = 0; t < T; ++t)for (int i = 0; i < l.k_exp; ++i) {const int16_t v = (int16_t) r.ids[t][i];std::fwrite(&v, 2, 1, trace_f);}+ if (with_ahead) {+ const uint8_t na[2] = {(uint8_t) std::max(0, std::min(r.na[0], kAhead1)), (uint8_t) std::max(0, std::min(r.na[1], kPicks))};+ std::fwrite(na, 1, 2, trace_f);+ for (int t = 0; t < T; ++t) {+ for (int i = 0; i < na[0]; ++i) {+ const int16_t v = (int16_t) r.ahead[t][i];+ std::fwrite(&v, 2, 1, trace_f);+ }+ for (int i = 0; i < na[1]; ++i) {+ const int16_t v = (int16_t) r.ahead[t][kAhead1 + i];+ std::fwrite(&v, 2, 1, trace_f);+ }+ }+ }}const bool dsk = disk_skip_w > 0.0f && !l.mtp && li >= disk_skip_from && li < g.nl;for (int t = 0; t < T; ++t) {diff --git a/src/dsv41/ds4_packinfo.cpp b/src/dsv41/ds4_packinfo.cppindex c71a264eb..bced9da70 100644--- a/src/dsv41/ds4_packinfo.cpp+++ b/src/dsv41/ds4_packinfo.cpp@@ -25,7 +25,7 @@ namespace {// and not in the pack's tensor list is fine; a tensor whose type is not here is the CPU lane's (ggml-cpu's row dots).bool device_type(const std::string& t) {static const std::set<std::string> ok = {"MXFP4", "Q2_K", "Q3_K", "Q4_K", "Q5_K", "Q6_K", "IQ2_XXS", "IQ2_XS",- "IQ2_S", "IQ3_XXS", "IQ3_S", "IQ4_XS", "TQK6", "TQK7", "TQK5X", "TQK4X",+ "IQ2_S", "IQ3_XXS", "IQ3_S", "IQ4_XS", "IQ1_M", "TQK6", "TQK7", "TQK5X", "TQK4X","TQK5T", "TQK4T"};return ok.count(t) != 0;}@@ -127,8 +127,13 @@ int pack_info(const std::string& dir, std::FILE* out) {++missing;continue;}+#if defined(_WIN32)+ _fseeki64(f, 0, SEEK_END); // (long is 32 bits there: the weights are far past 2 GB)+ const long long sz = _ftelli64(f);+#elsestd::fseek(f, 0, SEEK_END);const long long sz = std::ftell(f);+#endifstd::fclose(f);if ((double) sz + 0.5 < n) {std::fprintf(out, " SHORT %s: %.2f GB there, %.2f GB wanted\n", path.c_str(), (double) sz / 1e9, n / 1e9);diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.pyindex 4fe27ec94..3d39d4346 100644--- a/tools/dsv41/pack_gguf.py+++ b/tools/dsv41/pack_gguf.py@@ -40,7 +40,7 @@ TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 30: ("BF16", 1# the low-bit expert types (the Maya quants' experts; the engine's ds4_iq kernels)10: ("Q2_K", 256, 84), 11: ("Q3_K", 256, 110), 12: ("Q4_K", 256, 144), 13: ("Q5_K", 256, 176),14: ("Q6_K", 256, 210), 16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98),- 20: ("IQ4_NL", 32, 18), 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136),+ 19: ("IQ1_S", 256, 50), 29: ("IQ1_M", 256, 56), 20: ("IQ4_NL", 32, 18), 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136),# the trellis types (tools/trellis/tqenc.hpp; 145 / 146 are the agentionai fork's ggml ids, 224-227 Strata's own)145: ("TQK6", 128, 26), 146: ("TQK7", 128, 30), 224: ("TQK5X", 128, 22), 225: ("TQK4X", 128, 18),226: ("TQK5T", 128, 22), 227: ("TQK4T", 128, 18)}@@ -268,8 +268,12 @@ def int_tensors_to_engram(gs, kv: dict):with g.path.open("rb") as f:f.seek(g.data_start + off)kv["%s.engram.%s" % (a, key)] = list(struct.unpack("<%d%s" % (n, fmt), f.read(n * sz)))- if "%s.engram.pad_token_id" % a not in kv and "%s.engram_pad_token_id" % a in kv:- kv["%s.engram.pad_token_id" % a] = kv["%s.engram_pad_token_id" % a]+ # the IST-DAS spelling of the Engram's scalar keys -> the dotted ones this pack reads+ for dotted, theirs in (("engram.layers", "engram.layer_ids"), ("engram.max_ngram_size", "engram_max_ngram_size"),+ ("engram.head_count", "engram_n_heads"), ("engram.head_dim", "engram_head_dim"),+ ("engram.pad_token_id", "engram_pad_token_id")):+ if "%s.%s" % (a, dotted) not in kv and "%s.%s" % (a, theirs) in kv:+ kv["%s.%s" % (a, dotted)] = kv["%s.%s" % (a, theirs)]def split_ple(g: Gguf, kv: dict):@@ -444,8 +448,8 @@ def main() -> int:# Looked for only when the quant's shards leave some out.have = {t[0] for g in gs for t in g.tensors}arch = kv["general.architecture"]- layers = kv.get("%s.engram.layers" % arch, [])int_tensors_to_engram(gs, kv)+ layers = kv.get("%s.engram.layers" % arch, [])def has_part(il, part, names):if "blk.%d.%s.weight" % (il, part) in names:diff --git a/tools/dsv41/win/cachesim.py b/tools/dsv41/win/cachesim.pynew file mode 100644index 000000000..e89027823--- /dev/null+++ b/tools/dsv41/win/cachesim.py@@ -0,0 +1,342 @@+"""cachesim.py - offline expert-cache policies on routing traces (DS4_TRACE files of strata-dsv41).++A trace is a run of records: u8 layer, u8 T (bit 7: guesses follow), T x k int16 expert ids [, u8 na1, u8 na2, T x (na1 + na2)+int16 guesses for the next layer and the one after]. One decode step (or verify window) is the records of layers 0..nl-1 in order.++Sizes are in bytes (an expert's real size in its layer: sizes.json), capacities in GB. Policies:+ lru one global LRU over (layer, expert)+ lfu frequency with exponential decay (LRFU / CRF), evicts the lowest score+ static the N hottest by training-trace frequency (a shipped per-set profile), no adaptation+ static+lru half the capacity pinned from the profile, half LRU+ belady the offline optimum (farthest next use): the ceiling for any predictor of reuse+ lru+ahead LRU plus a prefetch of the guesses for the next layer (top m): counts the wasted bytes+Reports: hit rate (by accesses and by bytes), SSD bytes per token (misses + wasted prefetch), the share of misses a guess covered.+"""+import argparse+import collections+import heapq+import json+import math+import os+import struct+import sys+++def read_trace(path, k=6):+ """-> list of steps; a step = list of (layer, [rows of expert ids], guesses or None)."""+ data = open(path, 'rb').read()+ pos, n = 0, len(data)+ steps, cur, last_layer = [], [], -1+ while pos + 2 <= n:+ layer, t = data[pos], data[pos + 1]+ pos += 2+ has_ahead = bool(t & 0x80)+ t &= 0x7f+ rows = []+ for _ in range(t):+ rows.append(list(struct.unpack_from('<%dh' % k, data, pos)))+ pos += 2 * k+ guesses = None+ if has_ahead:+ na1, na2 = data[pos], data[pos + 1]+ pos += 2+ guesses = []+ for _ in range(t):+ g1 = list(struct.unpack_from('<%dh' % na1, data, pos)) if na1 else []+ pos += 2 * na1+ g2 = list(struct.unpack_from('<%dh' % na2, data, pos)) if na2 else []+ pos += 2 * na2+ guesses.append((g1, g2))+ if layer <= last_layer and cur:+ steps.append(cur)+ cur = []+ last_layer = layer+ cur.append((layer, rows, guesses))+ if cur:+ steps.append(cur)+ return steps+++def accesses(steps):+ """the flat access stream: (step, layer, expert) with the union over a window's rows, in layer order"""+ out = []+ for si, st in enumerate(steps):+ for layer, rows, _ in st:+ seen = []+ for r in rows:+ for e in r:+ if e not in seen:+ seen.append(e)+ for e in seen:+ out.append((si, layer, e))+ return out+++class Sizes:+ def __init__(self, path):+ d = json.load(open(path))+ el = d['exp_layer']+ self.s = {int(k): float(v) for k, v in el.items()}++ def __call__(self, layer):+ return self.s.get(layer, 8.6e6)+++def sim_lru(acc, size, cap):+ od = collections.OrderedDict()+ used = hits = hitb = tot = totb = 0+ for _, l, e in acc:+ k = (l, e)+ sz = size(l)+ tot += 1+ totb += sz+ if k in od:+ od.move_to_end(k)+ hits += 1+ hitb += sz+ continue+ od[k] = sz+ used += sz+ while used > cap:+ _, s0 = od.popitem(last=False)+ used -= s0+ return hits, hitb, tot, totb+++def sim_lfu(acc, size, cap, half_life_acc=20000):+ lam = math.log(2) / half_life_acc+ score, last, res = {}, {}, {}+ heap = []+ used = hits = hitb = tot = totb = 0+ for t, (_, l, e) in enumerate(acc):+ k = (l, e)+ sz = size(l)+ tot += 1+ totb += sz+ # CRF: decayed count, kept in log space: v = log(count) + lam * t+ if k in res:+ hits += 1+ hitb += sz+ old = score.get(k)+ v = lam * t+ score[k] = v if old is None else v + math.log1p(math.exp(old - v)) if old - v < 50 else old+ if k not in res:+ res[k] = sz+ used += sz+ heapq.heappush(heap, (score[k], k))+ while used > cap and heap:+ sc, kk = heapq.heappop(heap)+ if kk not in res or score[kk] != sc:+ continue+ if kk == k and len(res) == 1:+ break+ used -= res.pop(kk)+ return hits, hitb, tot, totb+++def freq_table(acc):+ c = collections.Counter((l, e) for _, l, e in acc)+ return c+++def pick_static(freq, size, cap):+ keys = sorted(freq, key=lambda k: -freq[k] / size(k[0]))+ chosen, used = set(), 0+ for k in keys:+ s = size(k[0])+ if used + s > cap:+ continue+ chosen.add(k)+ used += s+ return chosen+++def sim_static(acc, size, chosen):+ hits = hitb = tot = totb = 0+ for _, l, e in acc:+ sz = size(l)+ tot += 1+ totb += sz+ if (l, e) in chosen:+ hits += 1+ hitb += sz+ return hits, hitb, tot, totb+++def sim_static_lru(acc, size, cap, chosen, pinned_bytes):+ od = collections.OrderedDict()+ used = hits = hitb = tot = totb = 0+ room = cap - pinned_bytes+ for _, l, e in acc:+ k = (l, e)+ sz = size(l)+ tot += 1+ totb += sz+ if k in chosen:+ hits += 1+ hitb += sz+ continue+ if k in od:+ od.move_to_end(k)+ hits += 1+ hitb += sz+ continue+ od[k] = sz+ used += sz+ while used > room:+ _, s0 = od.popitem(last=False)+ used -= s0+ return hits, hitb, tot, totb+++def sim_belady(acc, size, cap):+ n = len(acc)+ nxt = [0] * n+ last = {}+ for i in range(n - 1, -1, -1):+ k = (acc[i][1], acc[i][2])+ nxt[i] = last.get(k, n + 1)+ last[k] = i+ res = {} # k -> next use+ heap = [] # (-next use, k)+ used = hits = hitb = tot = totb = 0+ for i, (_, l, e) in enumerate(acc):+ k = (l, e)+ sz = size(l)+ tot += 1+ totb += sz+ if k in res:+ hits += 1+ hitb += sz+ res[k] = nxt[i]+ heapq.heappush(heap, (-nxt[i], k))+ continue+ if nxt[i] > n: # never used again: do not keep+ continue+ res[k] = nxt[i]+ used += sz+ heapq.heappush(heap, (-nxt[i], k))+ while used > cap and heap:+ nu, kk = heapq.heappop(heap)+ if kk not in res or res[kk] != -nu:+ continue+ used -= size(kk[0])+ del res[kk]+ return hits, hitb, tot, totb+++def sim_lru_ahead(steps, size, cap, m=6, depth=1):+ """LRU + prefetch of the top-m guesses (from the record of layer L-1 for layer L): a miss that a guess covered is a 'hidden'+ miss (its read started a layer early), a guess nobody used is wasted bytes."""+ od = collections.OrderedDict()+ used = 0+ hits = tot = miss = hidden = 0+ ssd = 0.0+ wasted = 0.0+ pend = {} # (layer, expert) -> bytes prefetched, not yet used this step+ nsteps = len(steps)++ def put(k, sz):+ nonlocal used+ if k in od:+ od.move_to_end(k)+ return+ od[k] = sz+ used += sz+ while used > cap:+ _, s0 = od.popitem(last=False)+ used -= s0++ for st in steps:+ pend.clear()+ for layer, rows, guesses in st:+ sz = size(layer)+ seen = []+ for r in rows:+ for e in r:+ if e not in seen:+ seen.append(e)+ for e in seen:+ k = (layer, e)+ tot += 1+ if k in pend:+ hidden += 1+ del pend[k]+ miss += 1+ put(k, sz)+ elif k in od:+ hits += 1+ od.move_to_end(k)+ else:+ miss += 1+ ssd += sz+ put(k, sz)+ for k, b in list(pend.items()):+ if k[0] <= layer:+ wasted += b+ del pend[k]+ if guesses:+ nl = layer + 1+ gs = []+ for g1, g2 in guesses:+ for e in g1[:m]:+ if e not in gs:+ gs.append(e)+ for e in gs:+ k = (nl, e)+ if k in od or k in pend:+ continue+ pend[k] = size(nl)+ ssd += size(nl)+ for b in pend.values():+ wasted += b+ return dict(hit=hits / max(1, tot), hidden=hidden / max(1, tot), ssd_per_step_MB=ssd / max(1, nsteps) / 1e6,+ wasted_per_step_MB=wasted / max(1, nsteps) / 1e6)+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument('traces', nargs='+', help='trace files: all are test sets; --train-on names the profile source')+ ap.add_argument('--sizes', default='C:/ds-ista-work/sizes.json')+ ap.add_argument('--caps', default='20,30,40,44', help='cache sizes in GB (VRAM hot set + pinned RAM)')+ ap.add_argument('--train', nargs='*', default=[], help='traces the static profile is built from (others than the test)')+ ap.add_argument('--skip', default='', help='policies to skip (comma): lru,lfu,static,static+lru,belady,ahead')+ a = ap.parse_args()+ size = Sizes(a.sizes)+ skip = set(a.skip.split(','))+ train_acc = []+ for t in a.train:+ train_acc += accesses(read_trace(t))+ for path in a.traces:+ steps = read_trace(path)+ acc = accesses(steps)+ nsteps = len(steps)+ print('== %s: %d steps, %d expert accesses (%.1f a step)' % (os.path.basename(path), nsteps, len(acc), len(acc) / max(1, nsteps)))+ tb = sum(size(l) for _, l, _ in acc) / nsteps+ print(' bytes a step (union per layer): %.2f GB' % (tb / 1e9))+ print(' %-14s %6s %9s %9s %12s' % ('policy', 'capGB', 'hit(acc)', 'hit(bytes)', 'SSD MB/step'))+ for cg in [float(x) for x in a.caps.split(',')]:+ cap = cg * 1e9+ rows = []+ if 'lru' not in skip:+ rows.append(('lru', sim_lru(acc, size, cap)))+ if 'lfu' not in skip:+ rows.append(('lfu(decay)', sim_lfu(acc, size, cap)))+ if train_acc and 'static' not in skip:+ ch = pick_static(freq_table(train_acc), size, cap)+ rows.append(('static', sim_static(acc, size, ch)))+ ch2 = pick_static(freq_table(train_acc), size, cap / 2)+ rows.append(('static+lru', sim_static_lru(acc, size, cap, ch2, cap / 2)))+ if 'belady' not in skip:+ rows.append(('belady', sim_belady(acc, size, cap)))+ for name, (h, hb, t, tb_) in rows:+ print(' %-14s %6.0f %8.1f%% %8.1f%% %12.1f' % (name, cg, 100 * h / t, 100 * hb / tb_, (tb_ - hb) / nsteps / 1e6))+ if 'ahead' not in skip and any(g for st in steps for _, _, g in st):+ for m in (6, 8, 12):+ r = sim_lru_ahead(steps, size, cap, m=m)+ print(' lru+ahead(top%-2d) %4.0f hit %.1f%% (+%.1f%% hidden) SSD %.0f MB/step (wasted %.0f)' % (+ m, cg, 100 * r['hit'], 100 * r['hidden'], r['ssd_per_step_MB'], r['wasted_per_step_MB']))+++if __name__ == '__main__':+ main()diff --git a/tools/dsv41/win/hwbench.cu b/tools/dsv41/win/hwbench.cuindex 6903ffc0d..0d03d3bf3 100644--- a/tools/dsv41/win/hwbench.cu+++ b/tools/dsv41/win/hwbench.cu@@ -61,7 +61,7 @@ static void bench_pcie() {}static void bench_nvme(const char* path, size_t block, int qd, double secs, bool seq) {- HANDLE f = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, nullptr);+ HANDLE f = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, nullptr);if (f == INVALID_HANDLE_VALUE) { printf("open failed %lu\n", GetLastError()); return; }LARGE_INTEGER sz; GetFileSizeEx(f, &sz);HANDLE port = CreateIoCompletionPort(f, nullptr, 0, 1);