Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
windows: fix issue #128 — converter path robustness, ASCII-safe output, engine version probe, dependency preflight
Alex Zhou8b0ea9c00d11
diff --git a/windows/app/src-tauri/src/doctor.rs b/windows/app/src-tauri/src/doctor.rsindex 811ff15..8ac2fb5 100644--- a/windows/app/src-tauri/src/doctor.rs+++ b/windows/app/src-tauri/src/doctor.rs@@ -78,10 +78,11 @@ pub fn run(state: &engine::EngineState) -> Value {check("memory", "fail", format!("{mem} GB: below the 8b floor"), Some("E-MEM-LOAD"), None)});- // engine binary presence + version probe+ // engine binary presence + version probe (always-measure: doctor reports live facts,+ // and a cached negative from before the binary landed must not haunt it — issue #128)let exe = paths::bin_dir().join("llama-server.exe");checks.push(if exe.exists() {- match engine::server_version() {+ match engine::server_version_probe() {Some(v) => check("engine", "pass", v, None, None),None => check("engine", "warn", format!("{} exists but version probe failed", exe.display()), None, None),}diff --git a/windows/app/src-tauri/src/engine.rs b/windows/app/src-tauri/src/engine.rsindex ed95452..d08365a 100644--- a/windows/app/src-tauri/src/engine.rs+++ b/windows/app/src-tauri/src/engine.rs@@ -228,24 +228,38 @@ pub fn status(state: &EngineState) -> Value {}}-/// Engine binary version: the actual `llama-server --version` line, cached in a-/// OnceLock; the UI shows the measured value and falls back to null rather than guessing.+/// Engine binary version: the actual `llama-server --version` line.+/// A *successful* probe is cached for the session; failures are never cached+/// (issue #128: the old OnceLock<Option> shape let one early miss — e.g. a status+/// poll before the engine had landed — poison a whole fresh-install session with+/// "exists but version probe failed"). Capture is stdout+stderr with a tolerant+/// `version:` line match: the fork logs its banner to the log stream, and the old+/// stdout-only / stderr-discarded / strict-prefix probe never saw it.+static VER_CACHE: OnceLock<String> = OnceLock::new();+pub fn server_version() -> Option<String> {- static VER: OnceLock<Option<String>> = OnceLock::new();- VER.get_or_init(|| {- let exe = paths::bin_dir().join("llama-server.exe");- let mut cmd = Command::new(&exe);- crate::no_window(&mut cmd)- .arg("--version")- .current_dir(paths::bin_dir())- .stdout(Stdio::piped())- .stderr(Stdio::null());- cmd.spawn()- .and_then(|c| c.wait_with_output())- .ok()- .map(|o| String::from_utf8_lossy(&o.stdout).lines().find(|l| l.starts_with("version:")).map(|l| l.trim().to_string()).unwrap_or_default())- .filter(|s| !s.is_empty())- }).clone()+ VER_CACHE.get().cloned().or_else(server_version_probe)+}++/// Always-measure probe (doctor uses this: a manual check must read live facts,+/// never a stale negative). Only a success writes the cache.+pub fn server_version_probe() -> Option<String> {+ let exe = paths::bin_dir().join("llama-server.exe");+ let mut cmd = Command::new(&exe);+ crate::no_window(&mut cmd)+ .arg("--version")+ .current_dir(paths::bin_dir())+ .stdout(Stdio::piped())+ .stderr(Stdio::piped());+ let out = cmd.spawn().and_then(|c| c.wait_with_output()).ok()?;+ let text = format!("{}\n{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));+ let line = text.lines().map(str::trim_start)+ .find(|l| l.starts_with("version:"))+ .or_else(|| text.lines().find(|l| l.contains("version:")))?+ .trim().to_string();+ if line.is_empty() { return None; }+ let _ = VER_CACHE.set(line.clone());+ Some(line)}/// Read the POOL2 init telemetry line from a log (used by doctor; a missing line =diff --git a/windows/app/src-tauri/tests/doctor.rs b/windows/app/src-tauri/tests/doctor.rsindex dcfc6d5..d4ac83b 100644--- a/windows/app/src-tauri/tests/doctor.rs+++ b/windows/app/src-tauri/tests/doctor.rs@@ -1,6 +1,6 @@// tests/doctor.rs — doctor check contract + model_delete protection lines (isolated home;// tests serialized via a process lock to avoid env races).-use edge0_app_lib::{doctor, paths};+use edge0_app_lib::{doctor, engine, paths};use serde_json::json;use std::sync::Once;@@ -45,6 +45,38 @@ fn doctor_contract_shape() {assert_eq!(find("model-35b").unwrap()["verdict"], "warn", "not installed = warn, not fail");}+#[test]+fn engine_probe_recovers_when_binary_lands_late() {+ // Issue #128 fresh-install sequence: an early failed probe must NOT poison the+ // session cache, and a version line written to STDERR (fork logging behavior)+ // must still be captured. Serialized on LOCK; own bin dir; env restored.+ let _g = LOCK.lock().unwrap_or_else(|e| e.into_inner());+ let h = iso_home();+ let fake = h.join("bin-late");+ std::fs::create_dir_all(&fake).unwrap();+ std::env::set_var("EDGE0_BIN_DIR", &fake);+ // leg 1: exe absent ⇒ probe fails, nothing may be cached+ assert!(engine::server_version_probe().is_none(), "early probe (no binary) must fail cleanly");+ // leg 2: binary lands afterwards; emits its version line to stderr only.+ // POSIX shebang stand-in — not spawnable on Windows, so the body is unix-gated+ // (same exemption style as the mklink leg above; Windows coverage via manual runs).+ #[cfg(unix)]+ {+ use std::os::unix::fs::PermissionsExt;+ let exe = fake.join("llama-server.exe");+ std::fs::write(&exe, "#!/bin/sh\necho 'load_backend: cpu' 1>&2\necho 'version: 0.0.0-itest (build 1, commit deadbeef)' 1>&2\n").unwrap();+ std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap();+ let v = engine::server_version_probe().expect("late binary must probe OK (stderr capture)");+ assert!(v.contains("0.0.0-itest"), "{v}");+ // leg 3: doctor surfaces the recovered value as engine=pass+ let r = doctor::run(&std::sync::Mutex::new(None));+ let eng = r["checks"].as_array().unwrap().iter()+ .find(|c| c["id"] == "engine").cloned().unwrap();+ assert_eq!(eng["verdict"], "pass", "{:?}", eng["detail"]);+ }+ std::env::set_var("EDGE0_BIN_DIR", &h); // restore the E-ENGINE-MISSING anchor+}+#[test]fn delete_protection_and_idempotent() {let _g = LOCK.lock().unwrap_or_else(|e| e.into_inner());diff --git a/windows/tools/convert_mlx_to_gguf.py b/windows/tools/convert_mlx_to_gguf.pyindex 94e7041..32706a4 100644--- a/windows/tools/convert_mlx_to_gguf.py+++ b/windows/tools/convert_mlx_to_gguf.py@@ -22,6 +22,18 @@ Usage: python tools/convert_mlx_to_gguf.py --dir models/edge0-8b [--out models/e"""import argparse, hashlib, json, os, re, subprocess, sys, time+# Dependency preflight + stdout hardening (issue #128): fail with a clear E-DEP line+# instead of a bare ModuleNotFoundError traceback, and force UTF-8 on stdout so the+# DONE/progress lines never crash under a redirected/ANSI-codepage stdout+# (the app pipes us; cp1252-family consoles used to end a *successful* conversion in+# a UnicodeEncodeError on the final line).+try:+ import numpy # noqa: F401 (the whole chain needs it; report the miss here, with guidance)+except ImportError:+ sys.exit("E-DEP: the on-device converter needs python 3.10+ with numpy; fix: pip install numpy")+if hasattr(sys.stdout, "reconfigure"):+ sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")+TOOLS = os.path.dirname(os.path.abspath(__file__))CONVERTER_REV = subprocess.run(["git", "-C", os.path.dirname(TOOLS), "rev-parse", "--short", "HEAD"],capture_output=True, text=True).stdout.strip() or "unknown"@@ -97,8 +109,6 @@ def main():m = re.match(r"edge0[-_](35b|8b)", stem)assert m, f"cannot detect tier from directory name: {stem}"tier = m.group(1)- if a.out and os.path.normpath(a.out) != os.path.normpath(d + "-gguf"):- print("WARNING: --out deviates from the <dir>-gguf convention ⇒ the lora cross-check gate still reads the base from the conventional path (use the default layout on-device)", flush=True)os.makedirs(out, exist_ok=True)log = open(os.path.join(out, "convert.log"), "ab")arts = {"gguf": os.path.join(out, f"edge0-{tier}.gguf"),@@ -112,7 +122,7 @@ def main():old = json.load(open(manifest_p, encoding="utf8"))ok = all(os.path.isfile(p) and sha256(p) == old["artifacts"][k] for k, p in arts.items())if ok:- print("manifest hit and all artifact shas match ⇒ already converted, skipping (idempotent).", flush=True)+ print("manifest hit and all artifact shas match => already converted, skipping (idempotent).", flush=True)return 0except Exception:pass@@ -137,7 +147,10 @@ def main():"r3-8b", log)n_min = 500# ---- step 2: adapter ----- run_step([sys.executable, os.path.join(TOOLS, "lora_mlx_to_gguf.py"), "--dir", d], "lora", log)+ # --base pins the cross-check gate to the base GGUF THIS run produced (issue #128:+ # the gate used to hardcode <repo>/windows/models/... and missed the app's ~/.edge0 layout).+ run_step([sys.executable, os.path.join(TOOLS, "lora_mlx_to_gguf.py"), "--dir", d,+ "--base", arts["gguf"]], "lora", log)assert os.path.isfile(src_lora), f"adapter not produced: {src_lora}"if os.path.abspath(src_lora) != os.path.abspath(arts["lora"]):import shutil; shutil.copyfile(src_lora, arts["lora"])@@ -163,10 +176,10 @@ def main():det = {k: (k in ref_base and ref_base[k][1] == man["artifacts"][k]) for k in arts}man["determinism_vs"] = {"baselines": {k: ref_base[k][0] for k in ref_base}, **det}assert all(det.get(k) for k in arts), f"determinism mismatch (artifact sha differs from baseline): {det}"- print(f"[determinism] vs baselines: ALL MATCH ✔", flush=True)+ print(f"[determinism] vs baselines: ALL MATCH OK", flush=True)json.dump(man, open(manifest_p, "w", encoding="utf8"), ensure_ascii=False, indent=1)print(f"DONE {tier}: gguf={man['sizes']['gguf']/2**30:.2f}GiB lora={man['sizes']['lora']/2**20:.1f}MiB "- f"took {man['elapsed_s']}s → {manifest_p}", flush=True)+ f"took {man['elapsed_s']}s -> {manifest_p}", flush=True)if __name__ == "__main__":diff --git a/windows/tools/lora_mlx_to_gguf.py b/windows/tools/lora_mlx_to_gguf.pyindex 226febf..0bb52a9 100644--- a/windows/tools/lora_mlx_to_gguf.py+++ b/windows/tools/lora_mlx_to_gguf.py@@ -22,6 +22,8 @@ Basis (verified against fork @7ab4ee7):"""import argparse, os, re, struct, sysimport numpy as np+if hasattr(sys.stdout, "reconfigure"): # issue #128: Windows ANSI-codepage stdout hardening+ sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))from repack_mlx_to_gguf import StIndex, GgufWriter, KV, T_F16, bf16_to_f32from repack_r3 import ssm_ba_perm@@ -72,6 +74,8 @@ def main():ap.add_argument("--alpha", type=float, default=32.0) # = alpha baked in the source artifact; plain --lora (default scale 1.0) then reproduces the same expressionap.add_argument("--only", default=None, help="generate only families whose base name matches this regex (single-variable experiments)")ap.add_argument("--out-suffix", default="-gguf", help="infix in the output name")+ ap.add_argument("--base", default=None,+ help="base GGUF for the cross-check gate (default: the conventional <dir>-gguf/edge0-<tier>.gguf sibling)")a = ap.parse_args()stem = os.path.basename(a.dir.rstrip("\\/"))is8 = "8b" in stem@@ -153,16 +157,21 @@ def main():w.write(np.ascontiguousarray(A, np.float16).tobytes()); w.align_pad()w.write(np.ascontiguousarray(B, np.float16).tobytes()); w.align_pad()w.close(); idx.close()- print(f"adapter: {out} tensors={2*len(entries)} pairs={len(entries)} (incl. ssm_ba merge ×{comp} compensation)")- verify_against_base(out, "edge0-8b" if is8 else "edge0-35b")+ print(f"adapter: {out} tensors={2*len(entries)} pairs={len(entries)} (incl. ssm_ba merge x{comp} compensation)")+ base_stem = "edge0-8b" if is8 else "edge0-35b"+ base = a.base or os.path.join(os.path.dirname(os.path.normpath(a.dir)), f"{stem}-gguf", f"{base_stem}.gguf")+ verify_against_base(out, base)-def verify_against_base(adapter_path, base_stem):+def verify_against_base(adapter_path, base_path):"""Cross-check gate: right after writing, verify (1) adapter names ⊆ base names(2) the upstream shape three-law (llama-adapter.cpp:358-371: a.ne0=base.ne0 ∧b.ne1=base.ne1 ∧ a.ne1=b.ne0; token_embd flip is a special case).Any miss/shape violation = hard throw, preventing the 'loads without error but- never enters the graph' failure mode from recurring."""+ never enters the graph' failure mode from recurring.+ The base GGUF path arrives as an argument (converter passes its own --out;+ issue #128: it used to be hardcoded to <repo>/models/... and missed the app's+ ~/.edge0/models layout on end-user machines)."""# gguf-py comes from the pinned vendored upstream (monorepo layout or bootstrap depot);# standalone checkouts fall back to an installed `gguf` package on sys.path._here = os.path.dirname(os.path.abspath(__file__))@@ -173,9 +182,12 @@ def verify_against_base(adapter_path, base_stem):if os.path.isdir(_p):sys.path.insert(0, _p)break- from gguf import GGUFReader- root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))- base_path = os.path.join(root, "models", f"{base_stem}-gguf", f"{base_stem}.gguf")+ try:+ from gguf import GGUFReader+ except ImportError:+ raise SystemExit("E-DEP: cross-check gate needs gguf-py (vendored llama.cpp/gguf-py on the search path, or: pip install gguf)")+ if not os.path.isfile(base_path):+ raise SystemExit(f"E-BASE-MISSING: base GGUF not found at {base_path} — pass --base <path> (the converter derives it from --out)")bt = {str(t.name): [int(x) for x in t.shape]for t in GGUFReader(base_path).tensors}ad = GGUFReader(adapter_path)diff --git a/windows/tools/repack_mlx_to_gguf.py b/windows/tools/repack_mlx_to_gguf.pyindex f513c88..1c96e52 100644--- a/windows/tools/repack_mlx_to_gguf.py+++ b/windows/tools/repack_mlx_to_gguf.py@@ -33,6 +33,8 @@ Usage: --dir models/edge0-35b --out models/edge0-35b-gguf [--dry|--selftest] [--"""import argparse, json, os, re, struct, sys, time, hashlibimport numpy as np+if hasattr(sys.stdout, "reconfigure"): # issue #128: Windows ANSI-codepage stdout hardening+ sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")T_F32, T_F16, T_Q4_1 = 0, 1, 3 # measured against fork ggml.hKV = dict(u32=4, i32=5, f32=6, bool=7, str=8, arr=9, u64=10, f64=12)@@ -272,7 +274,7 @@ def run(args):agg.setdefault((k, idx.t[n][3], idx.t[n][4]), 0)agg[(k, idx.t[n][3], idx.t[n][4])] += 1for (k, dt, shp), c in sorted(agg.items(), key=lambda x: str(x[0])):- print(f"[{k:6s}] src {dt:4s} {str(list(shp)):22s} ×{c}")+ print(f"[{k:6s}] src {dt:4s} {str(list(shp)):22s} x{c}")for n, why in skipped:print(f"[skip] {n}")print(f"plan={len(plan)} skip={len(skipped)}")@@ -342,7 +344,7 @@ def run(args):q8 = W[sl].view(np.uint8).reshape(sl.stop - sl.start, words * 4)w.write(deq_mlx_side(q8, S[sl], Bs[sl]).astype(np.float32, copy=False).tobytes())w.align_pad()- print(f" [{kind:6s}] {name:64s} →{str(dims):20s} {w._f.tell()/1e9:6.2f}GB {time.time()-t0:5.0f}s", flush=True)+ print(f" [{kind:6s}] {name:64s} ->{str(dims):20s} {w._f.tell()/1e9:6.2f}GB {time.time()-t0:5.0f}s", flush=True)audit.update(fp16_gate=dict(_gate), n_tensors=len(regs), elapsed_s=round(time.time() - t0, 1),exempt_groups=len(audit_exempt), exempt_list=sorted(set(map(tuple, audit_exempt)))[:500]@@ -359,8 +361,8 @@ def run(args):json.dump(dict(audit=audit, verdict=tag, gates=ng, gguf_size=os.path.getsize(gp)),open(os.path.join(args.out, f"{stem}-r1-audit.json"), "w"), ensure_ascii=False, indent=1)print(f"\nGGUF {gp} {os.path.getsize(gp)/1e9:.2f}GB tensors={len(regs)} {time.time()-t0:.0f}s")- print(f"parity gates {ng} (mismatches outside exemptions: {nb}) · fp16 gate elems={_gate['elems']:,} "- f"benign={_gate['breach_benign']:,} malignant-exempt={_gate['breach']:,} ⇒ {tag}")+ print(f"parity gates {ng} (mismatches outside exemptions: {nb}) - fp16 gate elems={_gate['elems']:,} "+ f"benign={_gate['breach_benign']:,} malignant-exempt={_gate['breach']:,} => {tag}")return 0 if verdict == "GREEN" else 1@@ -401,7 +403,7 @@ def selftest(args):d0, m0 = float(blocks["d"][2, 0]), float(blocks["m"][2, 0])for e in (0, 5, 16, 21, 31):assert abs(float(Bq[2, e]) - (code(2, e) * d0 + m0)) < 1e-9, f"element {e} nibble-order pin failed"- print("selftest pack/deq sha equality PASS · extreme codes 0/15 + cross-word nibble order PASS")+ print("selftest pack/deq sha equality PASS - extreme codes 0/15 + cross-word nibble order PASS")p = os.path.join(args.out or ".", "_selftest.gguf")os.makedirs(p and (args.out or "."), exist_ok=True)diff --git a/windows/tools/repack_r3.py b/windows/tools/repack_r3.pyindex c00907c..8731148 100644--- a/windows/tools/repack_r3.py+++ b/windows/tools/repack_r3.py@@ -24,6 +24,8 @@ reversible; Q4_1 sampled pack two-sided sha. Output: <stem>.gguf."""import argparse, json, math, os, re, struct, sys, timeimport numpy as np+if hasattr(sys.stdout, "reconfigure"): # issue #128: Windows ANSI-codepage stdout hardening+ sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))from repack_mlx_to_gguf import (StIndex, GgufWriter, KV, T_F32, T_F16, T_Q4_1, ALIGN, QK41,DT_Q41, mlx_q_bytes, pack_q41_rows, deq_mlx_side, deq_gguf_side,@@ -280,7 +282,7 @@ def main():v = "GREEN" if not bad and not audit["errors"] and n_done == len(table) else "RED"json.dump(dict(verdict=f"R3-artifact {v}", audit=audit, size=os.path.getsize(gp)),open(os.path.join(a.out, f"{stem}-audit.json"), "w"), ensure_ascii=False, indent=1)- print(f"\nGGUF {gp} {os.path.getsize(gp)/1e9:.2f}GB tensors={n_done}/{len(table)} ⇒ R3-artifact {v}"+ print(f"\nGGUF {gp} {os.path.getsize(gp)/1e9:.2f}GB tensors={n_done}/{len(table)} => R3-artifact {v}"f" (errors={audit['errors']})")return 0 if v == "GREEN" else 1diff --git a/windows/tools/repack_r3_8b.py b/windows/tools/repack_r3_8b.pyindex 0a373ea..0b41e1b 100644--- a/windows/tools/repack_r3_8b.py+++ b/windows/tools/repack_r3_8b.py@@ -34,6 +34,8 @@ identity / sentinel sha / kv_b transform reversibility spot-check / exp dual-pat"""import argparse, json, math, os, re, struct, sys, timeimport numpy as np+if hasattr(sys.stdout, "reconfigure"): # issue #128: Windows ANSI-codepage stdout hardening+ sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))from repack_mlx_to_gguf import (StIndex, GgufWriter, KV, T_F32, T_F16, T_Q4_1, QK41, DT_Q41,mlx_q_bytes, pack_q41_rows, deq_mlx_side, deq_gguf_side, sha16)