Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 win: pack_gguf reads the IST-DAS Engram parameters from tensors, IQ1_M type, packinfo 64-bit sizes, DS4_TRACE_AHEAD, cachesim

Niko1221eb6c0a23f0e2
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 0562e5b94..774cfce73 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -5259,13 +5259,31 @@ struct Model::Impl {
int nslots = 0;
std::vector<std::pair<int, int>> picks; // (expert, t * kPicks + i), CPU-lane only
if (trace_f != nullptr && !l.mtp) { // DS4_TRACE=file: one record a route request: u8 layer, u8 T, T x k int16 expert ids
- uint8_t hd[2] = {(uint8_t) li, (uint8_t) T};
+ // DS4_TRACE_AHEAD=1: T has bit 7 set and the guesses follow: u8 na1, u8 na2, T x (na1 + na2) int16 (the
+ // next layer's likely picks by rank, then the one after's), as the lookahead made them for THIS layer's
+ // predecessors (the request carries the guesses for the layers after it)
+ static const bool with_ahead = std::getenv("DS4_TRACE_AHEAD") != nullptr;
+ uint8_t hd[2] = {(uint8_t) li, (uint8_t) (T | (with_ahead ? 0x80 : 0))};
std::fwrite(hd, 1, 2, trace_f);
for (int t = 0; t < T; ++t)
for (int i = 0; i < l.k_exp; ++i) {
const int16_t v = (int16_t) r.ids[t][i];
std::fwrite(&v, 2, 1, trace_f);
}
+ if (with_ahead) {
+ const uint8_t na[2] = {(uint8_t) std::max(0, std::min(r.na[0], kAhead1)), (uint8_t) std::max(0, std::min(r.na[1], kPicks))};
+ std::fwrite(na, 1, 2, trace_f);
+ for (int t = 0; t < T; ++t) {
+ for (int i = 0; i < na[0]; ++i) {
+ const int16_t v = (int16_t) r.ahead[t][i];
+ std::fwrite(&v, 2, 1, trace_f);
+ }
+ for (int i = 0; i < na[1]; ++i) {
+ const int16_t v = (int16_t) r.ahead[t][kAhead1 + i];
+ std::fwrite(&v, 2, 1, trace_f);
+ }
+ }
+ }
}
const bool dsk = disk_skip_w > 0.0f && !l.mtp && li >= disk_skip_from && li < g.nl;
for (int t = 0; t < T; ++t) {
diff --git a/src/dsv41/ds4_packinfo.cpp b/src/dsv41/ds4_packinfo.cpp
index c71a264eb..bced9da70 100644
--- a/src/dsv41/ds4_packinfo.cpp
+++ b/src/dsv41/ds4_packinfo.cpp
@@ -25,7 +25,7 @@ namespace {
// and not in the pack's tensor list is fine; a tensor whose type is not here is the CPU lane's (ggml-cpu's row dots).
bool device_type(const std::string& t) {
static const std::set<std::string> ok = {"MXFP4", "Q2_K", "Q3_K", "Q4_K", "Q5_K", "Q6_K", "IQ2_XXS", "IQ2_XS",
- "IQ2_S", "IQ3_XXS", "IQ3_S", "IQ4_XS", "TQK6", "TQK7", "TQK5X", "TQK4X",
+ "IQ2_S", "IQ3_XXS", "IQ3_S", "IQ4_XS", "IQ1_M", "TQK6", "TQK7", "TQK5X", "TQK4X",
"TQK5T", "TQK4T"};
return ok.count(t) != 0;
}
@@ -127,8 +127,13 @@ int pack_info(const std::string& dir, std::FILE* out) {
++missing;
continue;
}
+#if defined(_WIN32)
+ _fseeki64(f, 0, SEEK_END); // (long is 32 bits there: the weights are far past 2 GB)
+ const long long sz = _ftelli64(f);
+#else
std::fseek(f, 0, SEEK_END);
const long long sz = std::ftell(f);
+#endif
std::fclose(f);
if ((double) sz + 0.5 < n) {
std::fprintf(out, " SHORT %s: %.2f GB there, %.2f GB wanted\n", path.c_str(), (double) sz / 1e9, n / 1e9);
diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.py
index 4fe27ec94..3d39d4346 100644
--- a/tools/dsv41/pack_gguf.py
+++ b/tools/dsv41/pack_gguf.py
@@ -40,7 +40,7 @@ TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 30: ("BF16", 1
# the low-bit expert types (the Maya quants' experts; the engine's ds4_iq kernels)
10: ("Q2_K", 256, 84), 11: ("Q3_K", 256, 110), 12: ("Q4_K", 256, 144), 13: ("Q5_K", 256, 176),
14: ("Q6_K", 256, 210), 16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98),
- 20: ("IQ4_NL", 32, 18), 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136),
+ 19: ("IQ1_S", 256, 50), 29: ("IQ1_M", 256, 56), 20: ("IQ4_NL", 32, 18), 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136),
# the trellis types (tools/trellis/tqenc.hpp; 145 / 146 are the agentionai fork's ggml ids, 224-227 Strata's own)
145: ("TQK6", 128, 26), 146: ("TQK7", 128, 30), 224: ("TQK5X", 128, 22), 225: ("TQK4X", 128, 18),
226: ("TQK5T", 128, 22), 227: ("TQK4T", 128, 18)}
@@ -268,8 +268,12 @@ def int_tensors_to_engram(gs, kv: dict):
with g.path.open("rb") as f:
f.seek(g.data_start + off)
kv["%s.engram.%s" % (a, key)] = list(struct.unpack("<%d%s" % (n, fmt), f.read(n * sz)))
- if "%s.engram.pad_token_id" % a not in kv and "%s.engram_pad_token_id" % a in kv:
- kv["%s.engram.pad_token_id" % a] = kv["%s.engram_pad_token_id" % a]
+ # the IST-DAS spelling of the Engram's scalar keys -> the dotted ones this pack reads
+ for dotted, theirs in (("engram.layers", "engram.layer_ids"), ("engram.max_ngram_size", "engram_max_ngram_size"),
+ ("engram.head_count", "engram_n_heads"), ("engram.head_dim", "engram_head_dim"),
+ ("engram.pad_token_id", "engram_pad_token_id")):
+ if "%s.%s" % (a, dotted) not in kv and "%s.%s" % (a, theirs) in kv:
+ kv["%s.%s" % (a, dotted)] = kv["%s.%s" % (a, theirs)]
def split_ple(g: Gguf, kv: dict):
@@ -444,8 +448,8 @@ def main() -> int:
# Looked for only when the quant's shards leave some out.
have = {t[0] for g in gs for t in g.tensors}
arch = kv["general.architecture"]
- layers = kv.get("%s.engram.layers" % arch, [])
int_tensors_to_engram(gs, kv)
+ layers = kv.get("%s.engram.layers" % arch, [])
def has_part(il, part, names):
if "blk.%d.%s.weight" % (il, part) in names:
diff --git a/tools/dsv41/win/cachesim.py b/tools/dsv41/win/cachesim.py
new file mode 100644
index 000000000..e89027823
--- /dev/null
+++ b/tools/dsv41/win/cachesim.py
@@ -0,0 +1,342 @@
+"""cachesim.py - offline expert-cache policies on routing traces (DS4_TRACE files of strata-dsv41).
+
+A trace is a run of records: u8 layer, u8 T (bit 7: guesses follow), T x k int16 expert ids [, u8 na1, u8 na2, T x (na1 + na2)
+int16 guesses for the next layer and the one after]. One decode step (or verify window) is the records of layers 0..nl-1 in order.
+
+Sizes are in bytes (an expert's real size in its layer: sizes.json), capacities in GB. Policies:
+ lru one global LRU over (layer, expert)
+ lfu frequency with exponential decay (LRFU / CRF), evicts the lowest score
+ static the N hottest by training-trace frequency (a shipped per-set profile), no adaptation
+ static+lru half the capacity pinned from the profile, half LRU
+ belady the offline optimum (farthest next use): the ceiling for any predictor of reuse
+ lru+ahead LRU plus a prefetch of the guesses for the next layer (top m): counts the wasted bytes
+Reports: hit rate (by accesses and by bytes), SSD bytes per token (misses + wasted prefetch), the share of misses a guess covered.
+"""
+import argparse
+import collections
+import heapq
+import json
+import math
+import os
+import struct
+import sys
+
+
+def read_trace(path, k=6):
+ """-> list of steps; a step = list of (layer, [rows of expert ids], guesses or None)."""
+ data = open(path, 'rb').read()
+ pos, n = 0, len(data)
+ steps, cur, last_layer = [], [], -1
+ while pos + 2 <= n:
+ layer, t = data[pos], data[pos + 1]
+ pos += 2
+ has_ahead = bool(t & 0x80)
+ t &= 0x7f
+ rows = []
+ for _ in range(t):
+ rows.append(list(struct.unpack_from('<%dh' % k, data, pos)))
+ pos += 2 * k
+ guesses = None
+ if has_ahead:
+ na1, na2 = data[pos], data[pos + 1]
+ pos += 2
+ guesses = []
+ for _ in range(t):
+ g1 = list(struct.unpack_from('<%dh' % na1, data, pos)) if na1 else []
+ pos += 2 * na1
+ g2 = list(struct.unpack_from('<%dh' % na2, data, pos)) if na2 else []
+ pos += 2 * na2
+ guesses.append((g1, g2))
+ if layer <= last_layer and cur:
+ steps.append(cur)
+ cur = []
+ last_layer = layer
+ cur.append((layer, rows, guesses))
+ if cur:
+ steps.append(cur)
+ return steps
+
+
+def accesses(steps):
+ """the flat access stream: (step, layer, expert) with the union over a window's rows, in layer order"""
+ out = []
+ for si, st in enumerate(steps):
+ for layer, rows, _ in st:
+ seen = []
+ for r in rows:
+ for e in r:
+ if e not in seen:
+ seen.append(e)
+ for e in seen:
+ out.append((si, layer, e))
+ return out
+
+
+class Sizes:
+ def __init__(self, path):
+ d = json.load(open(path))
+ el = d['exp_layer']
+ self.s = {int(k): float(v) for k, v in el.items()}
+
+ def __call__(self, layer):
+ return self.s.get(layer, 8.6e6)
+
+
+def sim_lru(acc, size, cap):
+ od = collections.OrderedDict()
+ used = hits = hitb = tot = totb = 0
+ for _, l, e in acc:
+ k = (l, e)
+ sz = size(l)
+ tot += 1
+ totb += sz
+ if k in od:
+ od.move_to_end(k)
+ hits += 1
+ hitb += sz
+ continue
+ od[k] = sz
+ used += sz
+ while used > cap:
+ _, s0 = od.popitem(last=False)
+ used -= s0
+ return hits, hitb, tot, totb
+
+
+def sim_lfu(acc, size, cap, half_life_acc=20000):
+ lam = math.log(2) / half_life_acc
+ score, last, res = {}, {}, {}
+ heap = []
+ used = hits = hitb = tot = totb = 0
+ for t, (_, l, e) in enumerate(acc):
+ k = (l, e)
+ sz = size(l)
+ tot += 1
+ totb += sz
+ # CRF: decayed count, kept in log space: v = log(count) + lam * t
+ if k in res:
+ hits += 1
+ hitb += sz
+ old = score.get(k)
+ v = lam * t
+ score[k] = v if old is None else v + math.log1p(math.exp(old - v)) if old - v < 50 else old
+ if k not in res:
+ res[k] = sz
+ used += sz
+ heapq.heappush(heap, (score[k], k))
+ while used > cap and heap:
+ sc, kk = heapq.heappop(heap)
+ if kk not in res or score[kk] != sc:
+ continue
+ if kk == k and len(res) == 1:
+ break
+ used -= res.pop(kk)
+ return hits, hitb, tot, totb
+
+
+def freq_table(acc):
+ c = collections.Counter((l, e) for _, l, e in acc)
+ return c
+
+
+def pick_static(freq, size, cap):
+ keys = sorted(freq, key=lambda k: -freq[k] / size(k[0]))
+ chosen, used = set(), 0
+ for k in keys:
+ s = size(k[0])
+ if used + s > cap:
+ continue
+ chosen.add(k)
+ used += s
+ return chosen
+
+
+def sim_static(acc, size, chosen):
+ hits = hitb = tot = totb = 0
+ for _, l, e in acc:
+ sz = size(l)
+ tot += 1
+ totb += sz
+ if (l, e) in chosen:
+ hits += 1
+ hitb += sz
+ return hits, hitb, tot, totb
+
+
+def sim_static_lru(acc, size, cap, chosen, pinned_bytes):
+ od = collections.OrderedDict()
+ used = hits = hitb = tot = totb = 0
+ room = cap - pinned_bytes
+ for _, l, e in acc:
+ k = (l, e)
+ sz = size(l)
+ tot += 1
+ totb += sz
+ if k in chosen:
+ hits += 1
+ hitb += sz
+ continue
+ if k in od:
+ od.move_to_end(k)
+ hits += 1
+ hitb += sz
+ continue
+ od[k] = sz
+ used += sz
+ while used > room:
+ _, s0 = od.popitem(last=False)
+ used -= s0
+ return hits, hitb, tot, totb
+
+
+def sim_belady(acc, size, cap):
+ n = len(acc)
+ nxt = [0] * n
+ last = {}
+ for i in range(n - 1, -1, -1):
+ k = (acc[i][1], acc[i][2])
+ nxt[i] = last.get(k, n + 1)
+ last[k] = i
+ res = {} # k -> next use
+ heap = [] # (-next use, k)
+ used = hits = hitb = tot = totb = 0
+ for i, (_, l, e) in enumerate(acc):
+ k = (l, e)
+ sz = size(l)
+ tot += 1
+ totb += sz
+ if k in res:
+ hits += 1
+ hitb += sz
+ res[k] = nxt[i]
+ heapq.heappush(heap, (-nxt[i], k))
+ continue
+ if nxt[i] > n: # never used again: do not keep
+ continue
+ res[k] = nxt[i]
+ used += sz
+ heapq.heappush(heap, (-nxt[i], k))
+ while used > cap and heap:
+ nu, kk = heapq.heappop(heap)
+ if kk not in res or res[kk] != -nu:
+ continue
+ used -= size(kk[0])
+ del res[kk]
+ return hits, hitb, tot, totb
+
+
+def sim_lru_ahead(steps, size, cap, m=6, depth=1):
+ """LRU + prefetch of the top-m guesses (from the record of layer L-1 for layer L): a miss that a guess covered is a 'hidden'
+ miss (its read started a layer early), a guess nobody used is wasted bytes."""
+ od = collections.OrderedDict()
+ used = 0
+ hits = tot = miss = hidden = 0
+ ssd = 0.0
+ wasted = 0.0
+ pend = {} # (layer, expert) -> bytes prefetched, not yet used this step
+ nsteps = len(steps)
+
+ def put(k, sz):
+ nonlocal used
+ if k in od:
+ od.move_to_end(k)
+ return
+ od[k] = sz
+ used += sz
+ while used > cap:
+ _, s0 = od.popitem(last=False)
+ used -= s0
+
+ for st in steps:
+ pend.clear()
+ for layer, rows, guesses in st:
+ sz = size(layer)
+ seen = []
+ for r in rows:
+ for e in r:
+ if e not in seen:
+ seen.append(e)
+ for e in seen:
+ k = (layer, e)
+ tot += 1
+ if k in pend:
+ hidden += 1
+ del pend[k]
+ miss += 1
+ put(k, sz)
+ elif k in od:
+ hits += 1
+ od.move_to_end(k)
+ else:
+ miss += 1
+ ssd += sz
+ put(k, sz)
+ for k, b in list(pend.items()):
+ if k[0] <= layer:
+ wasted += b
+ del pend[k]
+ if guesses:
+ nl = layer + 1
+ gs = []
+ for g1, g2 in guesses:
+ for e in g1[:m]:
+ if e not in gs:
+ gs.append(e)
+ for e in gs:
+ k = (nl, e)
+ if k in od or k in pend:
+ continue
+ pend[k] = size(nl)
+ ssd += size(nl)
+ for b in pend.values():
+ wasted += b
+ return dict(hit=hits / max(1, tot), hidden=hidden / max(1, tot), ssd_per_step_MB=ssd / max(1, nsteps) / 1e6,
+ wasted_per_step_MB=wasted / max(1, nsteps) / 1e6)
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument('traces', nargs='+', help='trace files: all are test sets; --train-on names the profile source')
+ ap.add_argument('--sizes', default='C:/ds-ista-work/sizes.json')
+ ap.add_argument('--caps', default='20,30,40,44', help='cache sizes in GB (VRAM hot set + pinned RAM)')
+ ap.add_argument('--train', nargs='*', default=[], help='traces the static profile is built from (others than the test)')
+ ap.add_argument('--skip', default='', help='policies to skip (comma): lru,lfu,static,static+lru,belady,ahead')
+ a = ap.parse_args()
+ size = Sizes(a.sizes)
+ skip = set(a.skip.split(','))
+ train_acc = []
+ for t in a.train:
+ train_acc += accesses(read_trace(t))
+ for path in a.traces:
+ steps = read_trace(path)
+ acc = accesses(steps)
+ nsteps = len(steps)
+ print('== %s: %d steps, %d expert accesses (%.1f a step)' % (os.path.basename(path), nsteps, len(acc), len(acc) / max(1, nsteps)))
+ tb = sum(size(l) for _, l, _ in acc) / nsteps
+ print(' bytes a step (union per layer): %.2f GB' % (tb / 1e9))
+ print(' %-14s %6s %9s %9s %12s' % ('policy', 'capGB', 'hit(acc)', 'hit(bytes)', 'SSD MB/step'))
+ for cg in [float(x) for x in a.caps.split(',')]:
+ cap = cg * 1e9
+ rows = []
+ if 'lru' not in skip:
+ rows.append(('lru', sim_lru(acc, size, cap)))
+ if 'lfu' not in skip:
+ rows.append(('lfu(decay)', sim_lfu(acc, size, cap)))
+ if train_acc and 'static' not in skip:
+ ch = pick_static(freq_table(train_acc), size, cap)
+ rows.append(('static', sim_static(acc, size, ch)))
+ ch2 = pick_static(freq_table(train_acc), size, cap / 2)
+ rows.append(('static+lru', sim_static_lru(acc, size, cap, ch2, cap / 2)))
+ if 'belady' not in skip:
+ rows.append(('belady', sim_belady(acc, size, cap)))
+ for name, (h, hb, t, tb_) in rows:
+ print(' %-14s %6.0f %8.1f%% %8.1f%% %12.1f' % (name, cg, 100 * h / t, 100 * hb / tb_, (tb_ - hb) / nsteps / 1e6))
+ if 'ahead' not in skip and any(g for st in steps for _, _, g in st):
+ for m in (6, 8, 12):
+ r = sim_lru_ahead(steps, size, cap, m=m)
+ print(' lru+ahead(top%-2d) %4.0f hit %.1f%% (+%.1f%% hidden) SSD %.0f MB/step (wasted %.0f)' % (
+ m, cg, 100 * r['hit'], 100 * r['hidden'], r['ssd_per_step_MB'], r['wasted_per_step_MB']))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/dsv41/win/hwbench.cu b/tools/dsv41/win/hwbench.cu
index 6903ffc0d..0d03d3bf3 100644
--- a/tools/dsv41/win/hwbench.cu
+++ b/tools/dsv41/win/hwbench.cu
@@ -61,7 +61,7 @@ static void bench_pcie() {
}
static void bench_nvme(const char* path, size_t block, int qd, double secs, bool seq) {
- HANDLE f = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, nullptr);
+ HANDLE f = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, nullptr);
if (f == INVALID_HANDLE_VALUE) { printf("open failed %lu\n", GetLastError()); return; }
LARGE_INTEGER sz; GetFileSizeEx(f, &sz);
HANDLE port = CreateIoCompletionPort(f, nullptr, 0, 1);