Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
bench: survive transient hub errors β retry with backoff, per-task isolation, atomic writes (#304)
bokikof86fc6860d7b
diff --git a/c/coli b/c/coliindex efc8526f..e38badc7 100755--- a/c/coli+++ b/c/coli@@ -631,6 +631,16 @@ def cmd_bench(a):print(f" {C.dim}downloading missing datasets: {', '.join(missing)}{C.r}")subprocess.call([py, os.path.join(TOOLS,"fetch_benchmarks.py"),"--out", a.data, "--tasks", ",".join(missing), "--limit", str(max(a.limit,200))])+ # il fetch riprova da solo (#304), ma se l'hub resta giu' il bench gira sui task+ # disponibili invece di passare a eval file inesistenti.+ # EN: the fetch retries on its own (#304), but if the hub stays down the bench+ # runs on the available tasks instead of handing eval nonexistent files.+ still=[t for t in tasks.split(",") if not os.path.exists(os.path.join(a.data,f"{t}.jsonl"))]+ if still:+ tasks=",".join(t for t in tasks.split(",") if t not in still)+ print(f" {C.yel}skipping (download failed, rerun later): {', '.join(still)}{C.r}")+ if not tasks:+ print(f" {C.yel}no datasets available β nothing to bench{C.r}"); sys.exit(1)cmd=[py, os.path.join(TOOLS,"eval_glm.py"), "--glm", GLM, "--snap",a.model,"--tasks", tasks, "--limit", str(a.limit), "--data", a.data]if a.ram: cmd+=["--ram",str(a.ram)]diff --git a/c/tools/fetch_benchmarks.py b/c/tools/fetch_benchmarks.pyindex 6d2abd20..05dd9e7c 100644--- a/c/tools/fetch_benchmarks.py+++ b/c/tools/fetch_benchmarks.py@@ -8,7 +8,7 @@ USO:Poi:python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --limit 40 --ram 15"""-import os, json, argparse, random+import os, sys, json, time, argparse, randomdef f_hellaswag(d):ctx = (d["activity_label"] + ": " + d["ctx_a"] + " " + d["ctx_b"].capitalize()).strip()@@ -38,19 +38,44 @@ TASKS = { # task: (path, config, split, formatter)"openbookqa": ("allenai/openbookqa", "main", "validation", f_openbookqa),}+def load_retry(path, cfg, split, tries=5):+ """L'hub restituisce 5xx/timeout transitori (#304): riprova con backoff invece di+ morire al primo HEAD fallito. hf_hub riprende i download parziali dalla cache,+ quindi il retry riparte da dove si era fermato, non da zero.+ EN: the hub throws transient 5xx/timeouts (#304): retry with backoff instead of+ dying on the first failed HEAD. hf_hub resumes partial downloads from its cache,+ so a retry continues where it stopped rather than starting over."""+ from datasets import load_dataset+ for k in range(tries):+ try:+ return load_dataset(path, cfg, split=split)+ except (KeyboardInterrupt, SystemExit):+ raise+ except Exception as e:+ if k == tries - 1: raise+ wait = 2 ** (k + 1)+ print(f" {path}: {type(e).__name__}: {e} β retry {k+1}/{tries-1} in {wait}s")+ time.sleep(wait)+def main():ap = argparse.ArgumentParser()ap.add_argument("--out", default="./bench")ap.add_argument("--tasks", default="hellaswag,arc_challenge,mmlu")ap.add_argument("--limit", type=int, default=300)ap.add_argument("--seed", type=int, default=1234)+ ap.add_argument("--tries", type=int, default=5)a = ap.parse_args()- from datasets import load_datasetos.makedirs(a.out, exist_ok=True)+ failed = []for t in [x.strip() for x in a.tasks.split(",") if x.strip()]:- if t not in TASKS: print("unknown task:", t); continue+ if t not in TASKS: print("unknown task:", t); failed.append(t); continuepath, cfg, split, fn = TASKS[t]- ds = load_dataset(path, cfg, split=split)+ try:+ ds = load_retry(path, cfg, split, tries=max(a.tries, 1))+ except Exception as e:+ # un task fallito non deve uccidere gli altri / one failed task must not kill the rest+ print(f"{t}: FAILED after {a.tries} tries ({type(e).__name__}: {e}) β skipping")+ failed.append(t); continueidx = list(range(len(ds))); random.Random(a.seed).shuffle(idx)rows, n = [], 0for i in idx:@@ -61,9 +86,18 @@ def main():except Exception: continueif n >= a.limit: breakoutp = os.path.join(a.out, t + ".jsonl")- with open(outp, "w") as f:+ # scrittura atomica: coli controlla solo l'ESISTENZA del file, quindi un jsonl+ # troncato da un run interrotto bloccherebbe il re-download per sempre.+ # EN: atomic write: coli only checks the file EXISTS, so a truncated jsonl from+ # an interrupted run would block re-download forever.+ tmp = outp + ".part"+ with open(tmp, "w") as f:for r in rows: f.write(json.dumps(r) + "\n")+ os.replace(tmp, outp)print(f"{t}: {len(rows)} -> {outp}")+ if failed:+ print(f"incomplete: {', '.join(failed)} β rerun when the hub recovers (cached progress is kept)")+ sys.exit(1)if __name__ == "__main__":main()