Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

tools/dsv41/engram_fold.py --gate-kv-from: make the ple table the only variable, and the answer it gave

Niko1221226d73b45bb4
diff --git a/include/strata/dsv41/hip_or_cuda.h b/include/strata/dsv41/hip_or_cuda.h
index c839ce882..b0b835f95 100644
--- a/include/strata/dsv41/hip_or_cuda.h
+++ b/include/strata/dsv41/hip_or_cuda.h
@@ -6,6 +6,9 @@
// are plain .cpp files that launch kernels and copy buffers, and they used to include <hip/hip_runtime.h> unconditionally,
// which no CUDA toolkit ships: every CUDA box had to pass a shim on -I to build them (found on l620 and z440, 2026-10-23).
// This header is that shim, in the tree: HIP names on a HIP build, mapped onto the CUDA runtime otherwise.
+//
+// Two details that a first version got wrong (z440, 2026-10-23): HIP's hipMalloc is a *template* taking T**, so a plain
+// void** overload does not bind `hipMalloc(&uint8_ptr)`; and hipMemset has no CUDA counterpart under that name.
#pragma once
#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
@@ -21,10 +24,17 @@ constexpr int hipMemcpyHostToDevice = cudaMemcpyHostToDevice;
constexpr int hipMemcpyDeviceToHost = cudaMemcpyDeviceToHost;
inline hipError_t hipMalloc(void** p, size_t n) { return cudaMalloc(p, n); }
+template <typename T>
+inline hipError_t hipMalloc(T** p, size_t n) { return cudaMalloc(reinterpret_cast<void**>(p), n); }
inline hipError_t hipFree(void* p) { return cudaFree(p); }
inline hipError_t hipMemcpy(void* dst, const void* src, size_t n, int kind) {
return cudaMemcpy(dst, src, n, static_cast<cudaMemcpyKind>(kind));
}
+template <typename T>
+inline hipError_t hipMemcpy(T* dst, const T* src, size_t n, int kind) {
+ return cudaMemcpy(static_cast<void*>(dst), static_cast<const void*>(src), n, static_cast<cudaMemcpyKind>(kind));
+}
+inline hipError_t hipMemset(void* p, int v, size_t n) { return cudaMemset(p, v, n); }
inline hipError_t hipDeviceSynchronize() { return cudaDeviceSynchronize(); }
inline const char* hipGetErrorString(hipError_t e) { return cudaGetErrorString(e); }
inline hipError_t hipEventCreate(hipEvent_t* e) { return cudaEventCreate(e); }
diff --git a/tools/dsv41/engram_fold.py b/tools/dsv41/engram_fold.py
index a647e7ed5..58850f352 100644
--- a/tools/dsv41/engram_fold.py
+++ b/tools/dsv41/engram_fold.py
@@ -23,6 +23,7 @@ import argparse
import json
import os
import pathlib
+import re
import sys
import numpy as np
@@ -112,6 +113,7 @@ def main() -> int:
ap.add_argument("--model", required=True, help="the folder of the quant's GGUF shards")
ap.add_argument("--out", required=True, help="the override folder to write (dense.json + dense.bin)")
ap.add_argument("--layers", default="", help="comma list of Engram layers; default: every layer with an engram_k")
+ ap.add_argument("--gate-kv-from", default="", help="a known-good pack to copy engram.gate/wkv from (the ple table stays the only difference)")
a = ap.parse_args()
model = pathlib.Path(a.model).expanduser().resolve()
out = pathlib.Path(a.out).expanduser().resolve()
@@ -138,6 +140,30 @@ def main() -> int:
idx[name] = [tname, "dense.bin", off, len(data), shape]
off += len(data)
+ # --gate-kv-from PACK: take the gate and the kv from a pack that is known good (the engine's own layout, already in the
+ # right orientation) instead of deriving them from their k/q/wkv. That leaves the ple table as the only difference
+ # between the two packs, which is the only way to score their IQ4_NL Engram against our MXFP8 one.
+ if a.gate_kv_from:
+ src = pathlib.Path(a.gate_kv_from).expanduser().resolve()
+ md = [l.split(None, 1)[1] for l in (src / "ds4.txt").read_text().splitlines() if l.startswith("model_dir ")][0]
+ files, want = {}, {}
+ for line in (src / "tensors.txt").read_text().splitlines():
+ f = line.split()
+ if not f:
+ continue
+ if f[0] == "file":
+ files[int(f[1])] = " ".join(f[2:])
+ elif f[0] == "t" and re.match(r"layers\.\d+\.engram\.(gate|wkv)\.weight$", f[1]):
+ want[f[1]] = (f[2], int(f[3]), int(f[4]), int(f[5]), [int(x) for x in f[6:]])
+ for name, (tname, fi, o, nb, shape) in sorted(want.items()):
+ p = (pathlib.Path(md) / files[fi]).resolve()
+ add(name, tname, np.fromfile(p, dtype=np.uint8, count=nb, offset=o).tobytes(), shape)
+ print("%s: copied from %s (%s, %d bytes)" % (name, src.name, tname, nb))
+ (out / "dense.bin").write_bytes(b"".join(blobs))
+ json.dump(idx, open(out / "dense.json", "w"), indent=1)
+ print("wrote %s (%.2f MB), dense.json with %d tensors" % (out / "dense.bin", os.path.getsize(out / "dense.bin") / 1e6, len(idx)))
+ return 0
+
for il in layers:
kg, kt = find(gs, "blk.%d.engram_k.weight" % il)
qg, qt = find(gs, "blk.%d.engram_q.weight" % il)