Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

trellis: corpus builder, original-weight fetch, swap builder for Gyro-S layer variants

Niko12212ec3d2e381fe
diff --git a/tools/trellis/build_swap.py b/tools/trellis/build_swap.py
new file mode 100644
index 000000000..2df3be544
--- /dev/null
+++ b/tools/trellis/build_swap.py
@@ -0,0 +1,191 @@
+"""build_swap.py - Gyro-S with a few layers' experts swapped, without copying the 58 GB file.
+
+Writes <var>/swap.bin (the new gate / up / down tensors of the chosen layers) and a pack <var>/pack whose native_experts.txt
+points those layers at swap.bin (the shard column), everything else being the shipped Gyro-S pack and file: <var>/ holds a
+symlink to the shipped GGUF, so `--native <var>/<gguf name>` finds swap.bin beside it.
+
+Modes (all from the ORIGINAL Qwen weights, rotated into the file's basis with the file's own sign vectors):
+ q8 Q8_0 (types 8/8): the near-lossless reference for the swapped layers
+ plain TQK6 gate/up + TQK7 down, plain-MSE trellis encode (tools/tqenc.py)
+ diag the same with per-input-column importance weights h = E[x^2]^power from the calibration dump (STRATA_ACT_DUMP)
+
+ python3 tools/trellis/build_swap.py --gguf SHIPPED --pack PACK --orig DIR --layers 4,16,28,40 --mode plain --var DIR [--gpu]
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import pathlib
+import sys
+import time
+
+import numpy as np
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+sys.path.insert(0, str(HERE))
+import gguf_reader # noqa: E402
+import tqenc # noqa: E402
+
+N_EXP, N_EMBD, N_FF = 512, 2560, 640
+ALIGN = 4096
+
+
+def q8_0_quantize(w: np.ndarray) -> np.ndarray:
+ """float32 [..., n] (n % 32 == 0) -> uint8 [..., n/32 * 34] block_q8_0 (fp16 scale, 32 int8)."""
+ shp = w.shape
+ b = w.reshape(-1, 32).astype(np.float32)
+ amax = np.abs(b).max(axis=1)
+ d = amax / 127.0
+ d16 = d.astype(np.float16)
+ dq = d16.astype(np.float32)
+ with np.errstate(divide="ignore", invalid="ignore"):
+ q = np.where(dq[:, None] > 0, b / dq[:, None], 0.0)
+ q = np.clip(np.sign(q) * np.floor(np.abs(q) + 0.5), -127, 127).astype(np.int8)
+ out = np.empty((b.shape[0], 34), dtype=np.uint8)
+ out[:, :2] = d16.view(np.uint8).reshape(-1, 2)
+ out[:, 2:] = q.view(np.uint8)
+ return out.reshape(*shp[:-1], shp[-1] // 32 * 34)
+
+
+def bf16(raw: np.ndarray) -> np.ndarray:
+ return (raw.astype(np.uint32) << 16).view(np.float32)
+
+
+def load_h(dump_dir, layer, kind, power):
+ p = pathlib.Path(dump_dir) / ("act_L%d_%s.bin" % (layer, kind))
+ raw = p.read_bytes()
+ n, count = np.frombuffer(raw[:16], dtype=np.int64)
+ sq = np.frombuffer(raw[16:16 + 8 * n], dtype=np.float64)
+ h = (sq / max(count, 1)) ** power
+ h = np.maximum(h, h.mean() * 1e-4) # no weight below 1e-4 of the mean
+ return (h / h.mean()).astype(np.float32), int(count)
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--gguf", required=True)
+ ap.add_argument("--pack", required=True)
+ ap.add_argument("--orig", required=True)
+ ap.add_argument("--layers", required=True)
+ ap.add_argument("--mode", required=True, choices=["q8", "plain", "diag"])
+ ap.add_argument("--var", required=True)
+ ap.add_argument("--dump-dir", default=None)
+ ap.add_argument("--power", type=float, default=1.0)
+ ap.add_argument("--gpu", action="store_true")
+ ap.add_argument("--threads", type=int, default=16)
+ ap.add_argument("--iters", type=int, default=2)
+ ap.add_argument("--ncand", type=int, default=2)
+ ap.add_argument("--chunk", type=int, default=16)
+ a = ap.parse_args()
+ layers = [int(x) for x in a.layers.split(",")]
+ var = pathlib.Path(a.var)
+ var.mkdir(parents=True, exist_ok=True)
+ gg = pathlib.Path(a.gguf)
+ link = var / gg.name
+ if not link.exists():
+ link.symlink_to(gg)
+ g = gguf_reader.GGUFFile(a.gguf)
+ sw = [int(x) for x in g.metadata["prism.hadamard.sign_widths"]]
+ sv = np.array(g.metadata["prism.hadamard.sign_values"], dtype=np.float32)
+ signs, at = {}, 0
+ for w in sw:
+ signs[w] = sv[at:at + w]
+ at += w
+ if a.mode == "q8":
+ gu_t = d_t = 8
+ row_gu, row_d = N_EMBD // 32 * 34, N_FF // 32 * 34
+ else:
+ gu_t, d_t = tqenc.TQK6, tqenc.TQK7
+ row_gu, row_d = N_EMBD // 128 * tqenc.block_bytes(gu_t), N_FF // 128 * tqenc.block_bytes(d_t)
+ t_gate = N_EXP * N_FF * row_gu # bytes of the gate (and up) tensor of a layer
+ t_down = N_EXP * N_EMBD * row_d
+ swap = open(var / "swap.bin", "wb")
+ offs = {}
+ pos = 0
+ for l in layers:
+ offs[l] = {}
+ for role, nbytes in (("gate", t_gate), ("up", t_gate), ("down", t_down)):
+ offs[l][role] = pos
+ pos += (nbytes + ALIGN - 1) // ALIGN * ALIGN
+ swap.truncate(pos)
+
+ def enc(w2d, t, h):
+ kw = dict(iters=a.iters, ncand=a.ncand, gpu=a.gpu)
+ if not a.gpu:
+ kw["threads"] = a.threads
+ hh = None
+ if h is not None:
+ hh = np.broadcast_to(h, w2d.shape).astype(np.float32)
+ packed, err, tot, st = tqenc.encode(w2d, t, h=hh, **kw)
+ return packed, tot
+
+ stats = {}
+ t_start = time.time()
+ for l in layers:
+ gu_raw = np.memmap(pathlib.Path(a.orig) / ("L%d_gate_up.bf16" % l), dtype=np.uint16, mode="r").reshape(N_EXP, 2 * N_FF, N_EMBD)
+ dn_raw = np.memmap(pathlib.Path(a.orig) / ("L%d_down.bf16" % l), dtype=np.uint16, mode="r").reshape(N_EXP, N_EMBD, N_FF)
+ h_gu = h_dn = None
+ if a.mode == "diag":
+ h_gu, c1 = load_h(a.dump_dir, l, "gu", a.power)
+ h_dn, c2 = load_h(a.dump_dir, l, "dn", a.power)
+ print("layer %d: calibration counts gu %d dn %d; h cv gu %.2f dn %.2f" % (l, c1, c2, h_gu.std() / h_gu.mean(), h_dn.std() / h_dn.mean()))
+ err_sum = {"gate": 0.0, "up": 0.0, "down": 0.0}
+ en_sum = {"gate": 0.0, "up": 0.0, "down": 0.0}
+ for e0 in range(0, N_EXP, a.chunk):
+ e1 = min(N_EXP, e0 + a.chunk)
+ gu = bf16(np.asarray(gu_raw[e0:e1]))
+ dn = bf16(np.asarray(dn_raw[e0:e1]))
+ mats = {"gate": (gu[:, :N_FF], signs[N_EMBD], h_gu, row_gu, N_FF, t_gate, gu_t),
+ "up": (gu[:, N_FF:], signs[N_EMBD], h_gu, row_gu, N_FF, t_gate, gu_t),
+ "down": (dn, signs[N_FF], h_dn, row_d, N_EMBD, t_down, d_t)}
+ for role, (w, sg, h, row, n_out, tb, ty) in mats.items():
+ wr = tqenc.rotate(np.ascontiguousarray(w), sg) # [chunk, n_out, n_in]
+ n_in = wr.shape[-1]
+ if a.mode == "q8":
+ data = q8_0_quantize(wr)
+ dec = None
+ else:
+ packed, tot = enc(wr.reshape(-1, n_in), ty, h if a.mode == "diag" else None)
+ data = packed
+ err_sum[role] += tot
+ en_sum[role] += float(np.sum(wr.astype(np.float64) ** 2))
+ swap.seek(offs[l][role] + e0 * n_out * row)
+ swap.write(np.ascontiguousarray(data).tobytes())
+ if e0 % 128 == 0:
+ print("layer %d experts %d-%d done (%.0f s)" % (l, e0, e1, time.time() - t_start), flush=True)
+ stats[l] = {r: (10 * np.log10(en_sum[r] / err_sum[r]) if err_sum[r] > 0 else None) for r in err_sum}
+ print("layer %d weight SQNR (dB, unweighted): %s" % (l, {k: (round(v, 3) if v else v) for k, v in stats[l].items()}), flush=True)
+ swap.close()
+ # the pack: symlinks to the shipped pack's files, a new native_experts.txt
+ pk = var / "pack"
+ pk.mkdir(exist_ok=True)
+ src = pathlib.Path(a.pack)
+ for f in src.iterdir():
+ if f.name == "native_experts.txt":
+ continue
+ d = pk / f.name
+ if not d.exists():
+ d.symlink_to(f.resolve())
+ lines = (src / "native_experts.txt").read_text().splitlines()
+ head, body = lines[0], lines[1:]
+ out_lines, arena = [], 0
+ for ln in body:
+ f = ln.split()
+ l = int(f[0])
+ if l in offs:
+ blob = row_gu * N_FF * 2 + row_d * N_EMBD
+ f = [str(l), str(gu_t), str(d_t), str(arena), str(blob), str(offs[l]["gate"]), str(offs[l]["up"]), str(offs[l]["down"]), "swap.bin"]
+ else:
+ f[3] = str(arena)
+ out_lines.append(" ".join(f))
+ arena += N_EXP * int(f[4])
+ import re
+ head = re.sub(r"total \d+", "total %d" % arena, head)
+ (pk / "native_experts.txt").write_text(head + "\n" + "\n".join(out_lines) + "\n")
+ print("wrote", var, "swap.bin %.2f GB, pack %s" % (pos / 1e9, pk))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/trellis/fetch_orig.py b/tools/trellis/fetch_orig.py
new file mode 100644
index 000000000..0483964cb
--- /dev/null
+++ b/tools/trellis/fetch_orig.py
@@ -0,0 +1,67 @@
+"""fetch_orig.py - copy the routed-expert tensors of chosen layers of the ORIGINAL Qwen3.8-Flash-Next (BF16 safetensors) to
+local raw files, by HTTP range requests (resumable, per chunk of experts). Files: <out>/L<l>_gate_up.bf16 and
+L<l>_down.bf16 (raw bf16, shape [512, 1280, 2560] and [512, 2560, 640]).
+
+ python3 tools/trellis/fetch_orig.py --layers 4,16,28,40 --out /data/dsv41-enc/orig [--workers 48]
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import pathlib
+import sys
+import time
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import hf_slice # noqa: E402
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--repo", default="Qwen/Qwen3.8-Flash-Next")
+ ap.add_argument("--layers", required=True)
+ ap.add_argument("--out", required=True)
+ ap.add_argument("--workers", type=int, default=48)
+ ap.add_argument("--chunk", type=int, default=16, help="experts per range")
+ a = ap.parse_args()
+ out = pathlib.Path(a.out)
+ out.mkdir(parents=True, exist_ok=True)
+ repo = hf_slice.Repo(a.repo)
+ for layer in [int(x) for x in a.layers.split(",")]:
+ for tname, fname in (("gate_up_proj", "gate_up"), ("down_proj", "down")):
+ name = "model.language_model.layers.%d.mlp.experts.%s" % (layer, tname)
+ shard, t, base = repo.info(name)
+ n_exp = t["shape"][0]
+ inner = t["data_offsets"][1] - t["data_offsets"][0]
+ inner //= n_exp
+ path = out / ("L%d_%s.bf16" % (layer, fname))
+ done = out / ("L%d_%s.done" % (layer, fname))
+ if done.exists():
+ continue
+ marks = out / ("L%d_%s.chunks" % (layer, fname))
+ have = set(int(x) for x in marks.read_text().split()) if marks.exists() else set()
+ mode = "r+b" if path.exists() else "w+b"
+ with open(path, mode) as f:
+ f.truncate(inner * n_exp)
+ t0 = time.time()
+ for e0 in range(0, n_exp, a.chunk):
+ if e0 in have:
+ continue
+ e1 = min(n_exp, e0 + a.chunk)
+ s = base + t["data_offsets"][0] + e0 * inner
+ raw = hf_slice.get_parallel(repo.base + shard, s, s + (e1 - e0) * inner - 1, a.workers)
+ f.seek(e0 * inner)
+ f.write(raw)
+ f.flush()
+ with open(marks, "a") as m:
+ m.write("%d\n" % e0)
+ print("L%d %s experts %d-%d %.1f MB/s" % (layer, fname, e0, e1, len(raw) / 1e6 / (time.time() - t0 + 1e-9)),
+ flush=True)
+ t0 = time.time()
+ done.write_text("ok")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/trellis/hf_slice.py b/tools/trellis/hf_slice.py
index b93d55914..6e6186601 100644
--- a/tools/trellis/hf_slice.py
+++ b/tools/trellis/hf_slice.py
@@ -8,6 +8,7 @@ the environment for gated ones - it is sent as a header and never printed).
"""
from __future__ import annotations
+import concurrent.futures as cf
import json
import os
import struct
@@ -30,6 +31,25 @@ def _get(url: str, start: int | None = None, end: int | None = None) -> bytes:
return r.read()
+def get_parallel(url: str, start: int, end: int, workers: int = 16, piece: int = 8 << 20) -> bytes:
+ """bytes start..end (inclusive) of url, in `piece`-byte range requests on `workers` threads (one stream of a CDN
+ object is slow, a dozen are not)."""
+ spans = [(a, min(end, a + piece - 1)) for a in range(start, end + 1, piece)]
+ if len(spans) == 1:
+ return _get(url, start, end)
+
+ def one(sp):
+ for attempt in range(6):
+ try:
+ return _get(url, sp[0], sp[1])
+ except Exception:
+ if attempt == 5:
+ raise
+ return b""
+ with cf.ThreadPoolExecutor(workers) as ex:
+ return b"".join(ex.map(one, spans))
+
+
def bf16_to_f32(raw: bytes | np.ndarray) -> np.ndarray:
u = np.frombuffer(raw, dtype=np.uint16).astype(np.uint32) << 16
return u.view(np.float32)
@@ -54,7 +74,7 @@ class Repo:
h, base = self.header(shard)
return shard, h[name], base
- def tensor(self, name: str, lead: slice | int | None = None) -> np.ndarray:
+ def tensor(self, name: str, lead: slice | int | None = None, workers: int = 16) -> np.ndarray:
"""The tensor `name` as float32. lead: an index or slice on the first axis (e.g. an expert), fetched alone."""
shard, t, base = self.info(name)
shape = t["shape"]
@@ -67,7 +87,7 @@ class Repo:
i0, i1 = lead, lead + 1
else:
i0, i1 = lead.start or 0, lead.stop
- raw = _get(self.base + shard, base + s + i0 * inner, base + s + i1 * inner - 1)
+ raw = get_parallel(self.base + shard, base + s + i0 * inner, base + s + i1 * inner - 1, workers)
out_shape = [i1 - i0] + shape[1:]
if t["dtype"] == "BF16":
return bf16_to_f32(raw).reshape(out_shape)
diff --git a/tools/trellis/make_corpus.py b/tools/trellis/make_corpus.py
new file mode 100644
index 000000000..44f584363
--- /dev/null
+++ b/tools/trellis/make_corpus.py
@@ -0,0 +1,178 @@
+"""make_corpus.py - the calibration set and the held-out evaluation set for the Gyro swap experiment.
+
+Writes two ids files in the format of tools/quant_quality.py (run / compare): `cal.ids.json` and `eval.ids.json`, every
+segment SEG tokens. The two sets share no document: Wikipedia articles come from two different seeds with their titles
+checked against each other, code from two disjoint directory lists, agent transcripts from two seeds of a generator.
+
+ python3 tools/trellis/make_corpus.py --tokenizer packs/gyro-s/tokenizer --out DIR [--seg 512] [--cal 320] [--eval 128]
+"""
+from __future__ import annotations
+
+import argparse
+import concurrent.futures as cf
+import json
+import pathlib
+import random
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+import strata_tokenizer as ST # noqa: E402
+
+LANGS = ["en", "de", "fr", "es", "ru", "zh", "ja", "pt", "it", "ar", "ko", "tr", "pl", "nl", "hi", "vi"]
+
+
+def load_tokenizer(path):
+ tp = pathlib.Path(path)
+ vocab = json.loads((tp / "vocab.json").read_text(encoding="utf-8"))
+ toks = [None] * len(vocab)
+ for t, i in vocab.items():
+ toks[i] = t
+ return ST.Tokenizer(toks, (tp / "merges.txt").read_text(encoding="utf-8").split("\n"),
+ json.loads((tp / "token_type.json").read_text()))
+
+
+def wiki_article(lang: str, seed: int):
+ url = ("https://%s.wikipedia.org/w/api.php?action=query&format=json&generator=random&grnnamespace=0&grnlimit=1"
+ "&prop=extracts&explaintext=1&redirects=1&grnfilterredir=nonredirects" % lang)
+ req = urllib.request.Request(url + "&_=%d" % seed, headers={"User-Agent": "strata-research/1.0 (research corpus; contact via GitHub Niko1221)"})
+ d = {}
+ for attempt in range(8):
+ try:
+ with urllib.request.urlopen(req, timeout=60) as r:
+ d = json.loads(r.read())
+ break
+ except urllib.error.HTTPError as e:
+ if e.code != 429:
+ raise
+ time.sleep(3 * (attempt + 1))
+ for p in d.get("query", {}).get("pages", {}).values():
+ return p.get("title", ""), p.get("extract", "")
+ return "", ""
+
+
+def wiki_docs(langs, n_each, seed):
+ jobs = [(l, seed * 100000 + i * 17 + k) for k, l in enumerate(langs) for i in range(n_each)]
+ out = {}
+ with cf.ThreadPoolExecutor(2) as ex:
+ for (l, s), res in zip(jobs, ex.map(lambda j: wiki_article(*j), jobs)):
+ title, text = res
+ if len(text) > 1500:
+ out.setdefault(l, []).append((title, text))
+ return out
+
+
+def code_files(dirs, exts, limit_bytes=40000):
+ files = []
+ for d in dirs:
+ for p in sorted(pathlib.Path(d).rglob("*")):
+ if p.is_file() and p.suffix in exts and 800 < p.stat().st_size < 400000:
+ files.append(p)
+ return files
+
+
+def agent_transcript(rnd: random.Random) -> str:
+ tools = [("read_file", {"path": "src/%s.py"}), ("run_shell", {"command": "grep -rn %s ."}),
+ ("search", {"query": "%s", "limit": 5}), ("write_file", {"path": "notes/%s.md", "content": "..."}),
+ ("http_get", {"url": "https://example.org/%s"}), ("list_dir", {"path": "%s"})]
+ words = ["parser", "cache", "router", "scheduler", "tokenizer", "config", "retry", "metrics", "auth", "index",
+ "queue", "loader", "report", "build", "deploy", "schema"]
+ turns = ["<|im_start|>system\nYou are a coding agent. Tools: " +
+ json.dumps([{"name": t[0], "parameters": t[1]} for t in tools]) + "<|im_end|>"]
+ turns.append("<|im_start|>user\n%s<|im_end|>" % rnd.choice([
+ "Find where the %s is configured and fix the retry bug." % rnd.choice(words),
+ "Summarize how %s works in this repository and list the files involved." % rnd.choice(words),
+ "Add a unit test for the %s module and run it." % rnd.choice(words)]))
+ for _ in range(rnd.randint(3, 6)):
+ name, args = rnd.choice(tools)
+ w = rnd.choice(words)
+ a = {k: (v % w if isinstance(v, str) and "%s" in v else v) for k, v in args.items()}
+ turns.append("<|im_start|>assistant\nI will check the %s first.\n<tool_call>\n%s\n</tool_call><|im_end|>" % (
+ w, json.dumps({"name": name, "arguments": a})))
+ turns.append("<|im_start|>user\n<tool_response>\n%s\n</tool_response><|im_end|>" % json.dumps(
+ {"ok": True, "result": "%s: %d matches in %d files" % (w, rnd.randint(1, 40), rnd.randint(1, 9))}))
+ turns.append("<|im_start|>assistant\nDone. The %s change is in place and the tests pass.<|im_end|>" % rnd.choice(words))
+ return "\n".join(turns)
+
+
+def windows(tok, text, seg, parse_special=False):
+ ids = tok.encode(text, parse_special=parse_special)
+ return [ids[i:i + seg] for i in range(0, len(ids) - seg + 1, seg)]
+
+
+def build(tok, split: str, n: int, seg: int, rnd: random.Random, seed: int, code_dirs, md_dirs):
+ quota = {"english": 0.25, "multilingual": 0.25, "code": 0.25, "agent": 0.125, "docs": 0.125}
+ segs = []
+ # prose / multilingual from Wikipedia (the seed decides the articles; the two splits use different seeds)
+ wl = wiki_docs(["en"], int(n * 0.25 / 2) + 6, seed)
+ ml = wiki_docs([l for l in LANGS if l != "en"], max(2, int(n * 0.25 / 15 / 2) + 2), seed + 1)
+ titles = set()
+ for kind, docs in (("english", [d for v in wl.values() for d in v]),
+ ("multilingual", [d for v in ml.values() for d in v])):
+ pool = []
+ for title, text in docs:
+ if title in titles:
+ continue
+ titles.add(title)
+ pool += windows(tok, text, seg)
+ rnd.shuffle(pool)
+ segs += [{"name": "%s_%03d" % (kind, i), "ids": w} for i, w in enumerate(pool[:int(n * quota[kind])])]
+ # code
+ pool = []
+ for p in code_files(code_dirs, {".py", ".cpp", ".h", ".hpp", ".c", ".cu", ".js", ".sh", ".cmake", ".rs", ".go"}):
+ try:
+ pool += windows(tok, p.read_text(encoding="utf-8", errors="ignore")[:60000], seg)[:2]
+ except Exception:
+ pass
+ rnd.shuffle(pool)
+ segs += [{"name": "code_%03d" % i, "ids": w} for i, w in enumerate(pool[:int(n * quota["code"])])]
+ # docs (markdown / text of the repositories)
+ pool = []
+ for d in md_dirs:
+ for p in sorted(pathlib.Path(d).rglob("*.md")):
+ if p.stat().st_size > 3000:
+ pool += windows(tok, p.read_text(encoding="utf-8", errors="ignore")[:50000], seg)[:2]
+ rnd.shuffle(pool)
+ segs += [{"name": "docs_%03d" % i, "ids": w} for i, w in enumerate(pool[:int(n * quota["docs"])])]
+ # agent transcripts (tool calls)
+ pool = []
+ while len(pool) < int(n * quota["agent"]) + 4:
+ t = "\n".join(agent_transcript(rnd) for _ in range(3))
+ pool += windows(tok, t, seg, parse_special=True)
+ segs += [{"name": "agent_%03d" % i, "ids": w} for i, w in enumerate(pool[:int(n * quota["agent"])])]
+ print(split, {k: sum(1 for s in segs if s["name"].startswith(k)) for k in quota}, "total", len(segs))
+ return {"version": "gyro-swap-" + split, "seg_tokens": seg, "segments": segs}
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--tokenizer", required=True)
+ ap.add_argument("--out", required=True)
+ ap.add_argument("--seg", type=int, default=512)
+ ap.add_argument("--cal", type=int, default=320)
+ ap.add_argument("--eval", type=int, default=128)
+ ap.add_argument("--cal-code", nargs="+", default=["/data/llama.cpp/src", "/data/llama.cpp/tools", "/data/llama.cpp/gguf-py"])
+ ap.add_argument("--eval-code", nargs="+", default=["/data/llama.cpp/ggml", "/data/llama.cpp/examples", "/data/llama.cpp/tests"])
+ ap.add_argument("--cal-md", nargs="+", default=["/data/llama.cpp/docs"])
+ ap.add_argument("--eval-md", nargs="+", default=["/data/llama.cpp/tools", "/data/llama.cpp/examples"])
+ a = ap.parse_args()
+ tok = load_tokenizer(a.tokenizer)
+ out = pathlib.Path(a.out)
+ out.mkdir(parents=True, exist_ok=True)
+ cal = build(tok, "cal", a.cal, a.seg, random.Random(11), 1, a.cal_code, a.cal_md)
+ ev = build(tok, "eval", a.eval, a.seg, random.Random(22), 2, a.eval_code, a.eval_md)
+ # no document in common: compare the first 64 tokens of every segment
+ heads = {tuple(s["ids"][:64]) for s in cal["segments"]}
+ ev["segments"] = [s for s in ev["segments"] if tuple(s["ids"][:64]) not in heads]
+ (out / "cal.ids.json").write_text(json.dumps(cal))
+ (out / "eval.ids.json").write_text(json.dumps(ev))
+ print("wrote", out)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())