Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
trellis: corpus builder, original-weight fetch, swap builder for Gyro-S layer variants
Niko12212ec3d2e381fe
diff --git a/tools/trellis/build_swap.py b/tools/trellis/build_swap.pynew file mode 100644index 000000000..2df3be544--- /dev/null+++ b/tools/trellis/build_swap.py@@ -0,0 +1,191 @@+"""build_swap.py - Gyro-S with a few layers' experts swapped, without copying the 58 GB file.++Writes <var>/swap.bin (the new gate / up / down tensors of the chosen layers) and a pack <var>/pack whose native_experts.txt+points those layers at swap.bin (the shard column), everything else being the shipped Gyro-S pack and file: <var>/ holds a+symlink to the shipped GGUF, so `--native <var>/<gguf name>` finds swap.bin beside it.++Modes (all from the ORIGINAL Qwen weights, rotated into the file's basis with the file's own sign vectors):+ q8 Q8_0 (types 8/8): the near-lossless reference for the swapped layers+ plain TQK6 gate/up + TQK7 down, plain-MSE trellis encode (tools/tqenc.py)+ diag the same with per-input-column importance weights h = E[x^2]^power from the calibration dump (STRATA_ACT_DUMP)++ python3 tools/trellis/build_swap.py --gguf SHIPPED --pack PACK --orig DIR --layers 4,16,28,40 --mode plain --var DIR [--gpu]+"""+from __future__ import annotations++import argparse+import os+import pathlib+import sys+import time++import numpy as np++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE.parent))+sys.path.insert(0, str(HERE))+import gguf_reader # noqa: E402+import tqenc # noqa: E402++N_EXP, N_EMBD, N_FF = 512, 2560, 640+ALIGN = 4096+++def q8_0_quantize(w: np.ndarray) -> np.ndarray:+ """float32 [..., n] (n % 32 == 0) -> uint8 [..., n/32 * 34] block_q8_0 (fp16 scale, 32 int8)."""+ shp = w.shape+ b = w.reshape(-1, 32).astype(np.float32)+ amax = np.abs(b).max(axis=1)+ d = amax / 127.0+ d16 = d.astype(np.float16)+ dq = d16.astype(np.float32)+ with np.errstate(divide="ignore", invalid="ignore"):+ q = np.where(dq[:, None] > 0, b / dq[:, None], 0.0)+ q = np.clip(np.sign(q) * np.floor(np.abs(q) + 0.5), -127, 127).astype(np.int8)+ out = np.empty((b.shape[0], 34), dtype=np.uint8)+ out[:, :2] = d16.view(np.uint8).reshape(-1, 2)+ out[:, 2:] = q.view(np.uint8)+ return out.reshape(*shp[:-1], shp[-1] // 32 * 34)+++def bf16(raw: np.ndarray) -> np.ndarray:+ return (raw.astype(np.uint32) << 16).view(np.float32)+++def load_h(dump_dir, layer, kind, power):+ p = pathlib.Path(dump_dir) / ("act_L%d_%s.bin" % (layer, kind))+ raw = p.read_bytes()+ n, count = np.frombuffer(raw[:16], dtype=np.int64)+ sq = np.frombuffer(raw[16:16 + 8 * n], dtype=np.float64)+ h = (sq / max(count, 1)) ** power+ h = np.maximum(h, h.mean() * 1e-4) # no weight below 1e-4 of the mean+ return (h / h.mean()).astype(np.float32), int(count)+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--gguf", required=True)+ ap.add_argument("--pack", required=True)+ ap.add_argument("--orig", required=True)+ ap.add_argument("--layers", required=True)+ ap.add_argument("--mode", required=True, choices=["q8", "plain", "diag"])+ ap.add_argument("--var", required=True)+ ap.add_argument("--dump-dir", default=None)+ ap.add_argument("--power", type=float, default=1.0)+ ap.add_argument("--gpu", action="store_true")+ ap.add_argument("--threads", type=int, default=16)+ ap.add_argument("--iters", type=int, default=2)+ ap.add_argument("--ncand", type=int, default=2)+ ap.add_argument("--chunk", type=int, default=16)+ a = ap.parse_args()+ layers = [int(x) for x in a.layers.split(",")]+ var = pathlib.Path(a.var)+ var.mkdir(parents=True, exist_ok=True)+ gg = pathlib.Path(a.gguf)+ link = var / gg.name+ if not link.exists():+ link.symlink_to(gg)+ g = gguf_reader.GGUFFile(a.gguf)+ sw = [int(x) for x in g.metadata["prism.hadamard.sign_widths"]]+ sv = np.array(g.metadata["prism.hadamard.sign_values"], dtype=np.float32)+ signs, at = {}, 0+ for w in sw:+ signs[w] = sv[at:at + w]+ at += w+ if a.mode == "q8":+ gu_t = d_t = 8+ row_gu, row_d = N_EMBD // 32 * 34, N_FF // 32 * 34+ else:+ gu_t, d_t = tqenc.TQK6, tqenc.TQK7+ row_gu, row_d = N_EMBD // 128 * tqenc.block_bytes(gu_t), N_FF // 128 * tqenc.block_bytes(d_t)+ t_gate = N_EXP * N_FF * row_gu # bytes of the gate (and up) tensor of a layer+ t_down = N_EXP * N_EMBD * row_d+ swap = open(var / "swap.bin", "wb")+ offs = {}+ pos = 0+ for l in layers:+ offs[l] = {}+ for role, nbytes in (("gate", t_gate), ("up", t_gate), ("down", t_down)):+ offs[l][role] = pos+ pos += (nbytes + ALIGN - 1) // ALIGN * ALIGN+ swap.truncate(pos)++ def enc(w2d, t, h):+ kw = dict(iters=a.iters, ncand=a.ncand, gpu=a.gpu)+ if not a.gpu:+ kw["threads"] = a.threads+ hh = None+ if h is not None:+ hh = np.broadcast_to(h, w2d.shape).astype(np.float32)+ packed, err, tot, st = tqenc.encode(w2d, t, h=hh, **kw)+ return packed, tot++ stats = {}+ t_start = time.time()+ for l in layers:+ gu_raw = np.memmap(pathlib.Path(a.orig) / ("L%d_gate_up.bf16" % l), dtype=np.uint16, mode="r").reshape(N_EXP, 2 * N_FF, N_EMBD)+ dn_raw = np.memmap(pathlib.Path(a.orig) / ("L%d_down.bf16" % l), dtype=np.uint16, mode="r").reshape(N_EXP, N_EMBD, N_FF)+ h_gu = h_dn = None+ if a.mode == "diag":+ h_gu, c1 = load_h(a.dump_dir, l, "gu", a.power)+ h_dn, c2 = load_h(a.dump_dir, l, "dn", a.power)+ print("layer %d: calibration counts gu %d dn %d; h cv gu %.2f dn %.2f" % (l, c1, c2, h_gu.std() / h_gu.mean(), h_dn.std() / h_dn.mean()))+ err_sum = {"gate": 0.0, "up": 0.0, "down": 0.0}+ en_sum = {"gate": 0.0, "up": 0.0, "down": 0.0}+ for e0 in range(0, N_EXP, a.chunk):+ e1 = min(N_EXP, e0 + a.chunk)+ gu = bf16(np.asarray(gu_raw[e0:e1]))+ dn = bf16(np.asarray(dn_raw[e0:e1]))+ mats = {"gate": (gu[:, :N_FF], signs[N_EMBD], h_gu, row_gu, N_FF, t_gate, gu_t),+ "up": (gu[:, N_FF:], signs[N_EMBD], h_gu, row_gu, N_FF, t_gate, gu_t),+ "down": (dn, signs[N_FF], h_dn, row_d, N_EMBD, t_down, d_t)}+ for role, (w, sg, h, row, n_out, tb, ty) in mats.items():+ wr = tqenc.rotate(np.ascontiguousarray(w), sg) # [chunk, n_out, n_in]+ n_in = wr.shape[-1]+ if a.mode == "q8":+ data = q8_0_quantize(wr)+ dec = None+ else:+ packed, tot = enc(wr.reshape(-1, n_in), ty, h if a.mode == "diag" else None)+ data = packed+ err_sum[role] += tot+ en_sum[role] += float(np.sum(wr.astype(np.float64) ** 2))+ swap.seek(offs[l][role] + e0 * n_out * row)+ swap.write(np.ascontiguousarray(data).tobytes())+ if e0 % 128 == 0:+ print("layer %d experts %d-%d done (%.0f s)" % (l, e0, e1, time.time() - t_start), flush=True)+ stats[l] = {r: (10 * np.log10(en_sum[r] / err_sum[r]) if err_sum[r] > 0 else None) for r in err_sum}+ print("layer %d weight SQNR (dB, unweighted): %s" % (l, {k: (round(v, 3) if v else v) for k, v in stats[l].items()}), flush=True)+ swap.close()+ # the pack: symlinks to the shipped pack's files, a new native_experts.txt+ pk = var / "pack"+ pk.mkdir(exist_ok=True)+ src = pathlib.Path(a.pack)+ for f in src.iterdir():+ if f.name == "native_experts.txt":+ continue+ d = pk / f.name+ if not d.exists():+ d.symlink_to(f.resolve())+ lines = (src / "native_experts.txt").read_text().splitlines()+ head, body = lines[0], lines[1:]+ out_lines, arena = [], 0+ for ln in body:+ f = ln.split()+ l = int(f[0])+ if l in offs:+ blob = row_gu * N_FF * 2 + row_d * N_EMBD+ f = [str(l), str(gu_t), str(d_t), str(arena), str(blob), str(offs[l]["gate"]), str(offs[l]["up"]), str(offs[l]["down"]), "swap.bin"]+ else:+ f[3] = str(arena)+ out_lines.append(" ".join(f))+ arena += N_EXP * int(f[4])+ import re+ head = re.sub(r"total \d+", "total %d" % arena, head)+ (pk / "native_experts.txt").write_text(head + "\n" + "\n".join(out_lines) + "\n")+ print("wrote", var, "swap.bin %.2f GB, pack %s" % (pos / 1e9, pk))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/trellis/fetch_orig.py b/tools/trellis/fetch_orig.pynew file mode 100644index 000000000..0483964cb--- /dev/null+++ b/tools/trellis/fetch_orig.py@@ -0,0 +1,67 @@+"""fetch_orig.py - copy the routed-expert tensors of chosen layers of the ORIGINAL Qwen3.8-Flash-Next (BF16 safetensors) to+local raw files, by HTTP range requests (resumable, per chunk of experts). Files: <out>/L<l>_gate_up.bf16 and+L<l>_down.bf16 (raw bf16, shape [512, 1280, 2560] and [512, 2560, 640]).++ python3 tools/trellis/fetch_orig.py --layers 4,16,28,40 --out /data/dsv41-enc/orig [--workers 48]+"""+from __future__ import annotations++import argparse+import os+import pathlib+import sys+import time++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE))+import hf_slice # noqa: E402+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--repo", default="Qwen/Qwen3.8-Flash-Next")+ ap.add_argument("--layers", required=True)+ ap.add_argument("--out", required=True)+ ap.add_argument("--workers", type=int, default=48)+ ap.add_argument("--chunk", type=int, default=16, help="experts per range")+ a = ap.parse_args()+ out = pathlib.Path(a.out)+ out.mkdir(parents=True, exist_ok=True)+ repo = hf_slice.Repo(a.repo)+ for layer in [int(x) for x in a.layers.split(",")]:+ for tname, fname in (("gate_up_proj", "gate_up"), ("down_proj", "down")):+ name = "model.language_model.layers.%d.mlp.experts.%s" % (layer, tname)+ shard, t, base = repo.info(name)+ n_exp = t["shape"][0]+ inner = t["data_offsets"][1] - t["data_offsets"][0]+ inner //= n_exp+ path = out / ("L%d_%s.bf16" % (layer, fname))+ done = out / ("L%d_%s.done" % (layer, fname))+ if done.exists():+ continue+ marks = out / ("L%d_%s.chunks" % (layer, fname))+ have = set(int(x) for x in marks.read_text().split()) if marks.exists() else set()+ mode = "r+b" if path.exists() else "w+b"+ with open(path, mode) as f:+ f.truncate(inner * n_exp)+ t0 = time.time()+ for e0 in range(0, n_exp, a.chunk):+ if e0 in have:+ continue+ e1 = min(n_exp, e0 + a.chunk)+ s = base + t["data_offsets"][0] + e0 * inner+ raw = hf_slice.get_parallel(repo.base + shard, s, s + (e1 - e0) * inner - 1, a.workers)+ f.seek(e0 * inner)+ f.write(raw)+ f.flush()+ with open(marks, "a") as m:+ m.write("%d\n" % e0)+ print("L%d %s experts %d-%d %.1f MB/s" % (layer, fname, e0, e1, len(raw) / 1e6 / (time.time() - t0 + 1e-9)),+ flush=True)+ t0 = time.time()+ done.write_text("ok")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/trellis/hf_slice.py b/tools/trellis/hf_slice.pyindex b93d55914..6e6186601 100644--- a/tools/trellis/hf_slice.py+++ b/tools/trellis/hf_slice.py@@ -8,6 +8,7 @@ the environment for gated ones - it is sent as a header and never printed)."""from __future__ import annotations+import concurrent.futures as cfimport jsonimport osimport struct@@ -30,6 +31,25 @@ def _get(url: str, start: int | None = None, end: int | None = None) -> bytes:return r.read()+def get_parallel(url: str, start: int, end: int, workers: int = 16, piece: int = 8 << 20) -> bytes:+ """bytes start..end (inclusive) of url, in `piece`-byte range requests on `workers` threads (one stream of a CDN+ object is slow, a dozen are not)."""+ spans = [(a, min(end, a + piece - 1)) for a in range(start, end + 1, piece)]+ if len(spans) == 1:+ return _get(url, start, end)++ def one(sp):+ for attempt in range(6):+ try:+ return _get(url, sp[0], sp[1])+ except Exception:+ if attempt == 5:+ raise+ return b""+ with cf.ThreadPoolExecutor(workers) as ex:+ return b"".join(ex.map(one, spans))++def bf16_to_f32(raw: bytes | np.ndarray) -> np.ndarray:u = np.frombuffer(raw, dtype=np.uint16).astype(np.uint32) << 16return u.view(np.float32)@@ -54,7 +74,7 @@ class Repo:h, base = self.header(shard)return shard, h[name], base- def tensor(self, name: str, lead: slice | int | None = None) -> np.ndarray:+ def tensor(self, name: str, lead: slice | int | None = None, workers: int = 16) -> np.ndarray:"""The tensor `name` as float32. lead: an index or slice on the first axis (e.g. an expert), fetched alone."""shard, t, base = self.info(name)shape = t["shape"]@@ -67,7 +87,7 @@ class Repo:i0, i1 = lead, lead + 1else:i0, i1 = lead.start or 0, lead.stop- raw = _get(self.base + shard, base + s + i0 * inner, base + s + i1 * inner - 1)+ raw = get_parallel(self.base + shard, base + s + i0 * inner, base + s + i1 * inner - 1, workers)out_shape = [i1 - i0] + shape[1:]if t["dtype"] == "BF16":return bf16_to_f32(raw).reshape(out_shape)diff --git a/tools/trellis/make_corpus.py b/tools/trellis/make_corpus.pynew file mode 100644index 000000000..44f584363--- /dev/null+++ b/tools/trellis/make_corpus.py@@ -0,0 +1,178 @@+"""make_corpus.py - the calibration set and the held-out evaluation set for the Gyro swap experiment.++Writes two ids files in the format of tools/quant_quality.py (run / compare): `cal.ids.json` and `eval.ids.json`, every+segment SEG tokens. The two sets share no document: Wikipedia articles come from two different seeds with their titles+checked against each other, code from two disjoint directory lists, agent transcripts from two seeds of a generator.++ python3 tools/trellis/make_corpus.py --tokenizer packs/gyro-s/tokenizer --out DIR [--seg 512] [--cal 320] [--eval 128]+"""+from __future__ import annotations++import argparse+import concurrent.futures as cf+import json+import pathlib+import random+import sys+import time+import urllib.error+import urllib.parse+import urllib.request++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE.parent))+import strata_tokenizer as ST # noqa: E402++LANGS = ["en", "de", "fr", "es", "ru", "zh", "ja", "pt", "it", "ar", "ko", "tr", "pl", "nl", "hi", "vi"]+++def load_tokenizer(path):+ tp = pathlib.Path(path)+ vocab = json.loads((tp / "vocab.json").read_text(encoding="utf-8"))+ toks = [None] * len(vocab)+ for t, i in vocab.items():+ toks[i] = t+ return ST.Tokenizer(toks, (tp / "merges.txt").read_text(encoding="utf-8").split("\n"),+ json.loads((tp / "token_type.json").read_text()))+++def wiki_article(lang: str, seed: int):+ url = ("https://%s.wikipedia.org/w/api.php?action=query&format=json&generator=random&grnnamespace=0&grnlimit=1"+ "&prop=extracts&explaintext=1&redirects=1&grnfilterredir=nonredirects" % lang)+ req = urllib.request.Request(url + "&_=%d" % seed, headers={"User-Agent": "strata-research/1.0 (research corpus; contact via GitHub Niko1221)"})+ d = {}+ for attempt in range(8):+ try:+ with urllib.request.urlopen(req, timeout=60) as r:+ d = json.loads(r.read())+ break+ except urllib.error.HTTPError as e:+ if e.code != 429:+ raise+ time.sleep(3 * (attempt + 1))+ for p in d.get("query", {}).get("pages", {}).values():+ return p.get("title", ""), p.get("extract", "")+ return "", ""+++def wiki_docs(langs, n_each, seed):+ jobs = [(l, seed * 100000 + i * 17 + k) for k, l in enumerate(langs) for i in range(n_each)]+ out = {}+ with cf.ThreadPoolExecutor(2) as ex:+ for (l, s), res in zip(jobs, ex.map(lambda j: wiki_article(*j), jobs)):+ title, text = res+ if len(text) > 1500:+ out.setdefault(l, []).append((title, text))+ return out+++def code_files(dirs, exts, limit_bytes=40000):+ files = []+ for d in dirs:+ for p in sorted(pathlib.Path(d).rglob("*")):+ if p.is_file() and p.suffix in exts and 800 < p.stat().st_size < 400000:+ files.append(p)+ return files+++def agent_transcript(rnd: random.Random) -> str:+ tools = [("read_file", {"path": "src/%s.py"}), ("run_shell", {"command": "grep -rn %s ."}),+ ("search", {"query": "%s", "limit": 5}), ("write_file", {"path": "notes/%s.md", "content": "..."}),+ ("http_get", {"url": "https://example.org/%s"}), ("list_dir", {"path": "%s"})]+ words = ["parser", "cache", "router", "scheduler", "tokenizer", "config", "retry", "metrics", "auth", "index",+ "queue", "loader", "report", "build", "deploy", "schema"]+ turns = ["<|im_start|>system\nYou are a coding agent. Tools: " ++ json.dumps([{"name": t[0], "parameters": t[1]} for t in tools]) + "<|im_end|>"]+ turns.append("<|im_start|>user\n%s<|im_end|>" % rnd.choice([+ "Find where the %s is configured and fix the retry bug." % rnd.choice(words),+ "Summarize how %s works in this repository and list the files involved." % rnd.choice(words),+ "Add a unit test for the %s module and run it." % rnd.choice(words)]))+ for _ in range(rnd.randint(3, 6)):+ name, args = rnd.choice(tools)+ w = rnd.choice(words)+ a = {k: (v % w if isinstance(v, str) and "%s" in v else v) for k, v in args.items()}+ turns.append("<|im_start|>assistant\nI will check the %s first.\n<tool_call>\n%s\n</tool_call><|im_end|>" % (+ w, json.dumps({"name": name, "arguments": a})))+ turns.append("<|im_start|>user\n<tool_response>\n%s\n</tool_response><|im_end|>" % json.dumps(+ {"ok": True, "result": "%s: %d matches in %d files" % (w, rnd.randint(1, 40), rnd.randint(1, 9))}))+ turns.append("<|im_start|>assistant\nDone. The %s change is in place and the tests pass.<|im_end|>" % rnd.choice(words))+ return "\n".join(turns)+++def windows(tok, text, seg, parse_special=False):+ ids = tok.encode(text, parse_special=parse_special)+ return [ids[i:i + seg] for i in range(0, len(ids) - seg + 1, seg)]+++def build(tok, split: str, n: int, seg: int, rnd: random.Random, seed: int, code_dirs, md_dirs):+ quota = {"english": 0.25, "multilingual": 0.25, "code": 0.25, "agent": 0.125, "docs": 0.125}+ segs = []+ # prose / multilingual from Wikipedia (the seed decides the articles; the two splits use different seeds)+ wl = wiki_docs(["en"], int(n * 0.25 / 2) + 6, seed)+ ml = wiki_docs([l for l in LANGS if l != "en"], max(2, int(n * 0.25 / 15 / 2) + 2), seed + 1)+ titles = set()+ for kind, docs in (("english", [d for v in wl.values() for d in v]),+ ("multilingual", [d for v in ml.values() for d in v])):+ pool = []+ for title, text in docs:+ if title in titles:+ continue+ titles.add(title)+ pool += windows(tok, text, seg)+ rnd.shuffle(pool)+ segs += [{"name": "%s_%03d" % (kind, i), "ids": w} for i, w in enumerate(pool[:int(n * quota[kind])])]+ # code+ pool = []+ for p in code_files(code_dirs, {".py", ".cpp", ".h", ".hpp", ".c", ".cu", ".js", ".sh", ".cmake", ".rs", ".go"}):+ try:+ pool += windows(tok, p.read_text(encoding="utf-8", errors="ignore")[:60000], seg)[:2]+ except Exception:+ pass+ rnd.shuffle(pool)+ segs += [{"name": "code_%03d" % i, "ids": w} for i, w in enumerate(pool[:int(n * quota["code"])])]+ # docs (markdown / text of the repositories)+ pool = []+ for d in md_dirs:+ for p in sorted(pathlib.Path(d).rglob("*.md")):+ if p.stat().st_size > 3000:+ pool += windows(tok, p.read_text(encoding="utf-8", errors="ignore")[:50000], seg)[:2]+ rnd.shuffle(pool)+ segs += [{"name": "docs_%03d" % i, "ids": w} for i, w in enumerate(pool[:int(n * quota["docs"])])]+ # agent transcripts (tool calls)+ pool = []+ while len(pool) < int(n * quota["agent"]) + 4:+ t = "\n".join(agent_transcript(rnd) for _ in range(3))+ pool += windows(tok, t, seg, parse_special=True)+ segs += [{"name": "agent_%03d" % i, "ids": w} for i, w in enumerate(pool[:int(n * quota["agent"])])]+ print(split, {k: sum(1 for s in segs if s["name"].startswith(k)) for k in quota}, "total", len(segs))+ return {"version": "gyro-swap-" + split, "seg_tokens": seg, "segments": segs}+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--tokenizer", required=True)+ ap.add_argument("--out", required=True)+ ap.add_argument("--seg", type=int, default=512)+ ap.add_argument("--cal", type=int, default=320)+ ap.add_argument("--eval", type=int, default=128)+ ap.add_argument("--cal-code", nargs="+", default=["/data/llama.cpp/src", "/data/llama.cpp/tools", "/data/llama.cpp/gguf-py"])+ ap.add_argument("--eval-code", nargs="+", default=["/data/llama.cpp/ggml", "/data/llama.cpp/examples", "/data/llama.cpp/tests"])+ ap.add_argument("--cal-md", nargs="+", default=["/data/llama.cpp/docs"])+ ap.add_argument("--eval-md", nargs="+", default=["/data/llama.cpp/tools", "/data/llama.cpp/examples"])+ a = ap.parse_args()+ tok = load_tokenizer(a.tokenizer)+ out = pathlib.Path(a.out)+ out.mkdir(parents=True, exist_ok=True)+ cal = build(tok, "cal", a.cal, a.seg, random.Random(11), 1, a.cal_code, a.cal_md)+ ev = build(tok, "eval", a.eval, a.seg, random.Random(22), 2, a.eval_code, a.eval_md)+ # no document in common: compare the first 64 tokens of every segment+ heads = {tuple(s["ids"][:64]) for s in cal["segments"]}+ ev["segments"] = [s for s in ev["segments"] if tuple(s["ids"][:64]) not in heads]+ (out / "cal.ids.json").write_text(json.dumps(cal))+ (out / "eval.ids.json").write_text(json.dumps(ev))+ print("wrote", out)+ return 0+++if __name__ == "__main__":+ sys.exit(main())