Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
// engine.rs — supervisor for the llama-server child process (the daemon's duties
// folded into the shell). The spawn arg table is the process form of the runtime
// contract: -ngl 99 -cmoe --ctx-size 8192 --flash-attn on --pool-mb <tier-clamped>
// --no-webui --port <random free>. Readiness is decided by polling /health, not by
// a fixed sleep. A crash is surfaced and not auto-restarted within the session.
use crate::catalog;
use crate::paths;
use crate::sink::Sink;
use serde_json::{json, Value};
use std::net::TcpListener;
use std::process::{Child, Command, Stdio};
use std::sync::{Arc, Mutex, OnceLock};
#[derive(Debug)]
pub struct Engine {
pub child: Child,
pub port: u16,
pub tier: String,
pub pool_mb: u32,
pub started_at: u64, // UNIX seconds (source of truth for the service panel's uptime)
pub log_path: String,
pub job: bool, // true = attached to a KILL_ON_JOB_CLOSE job; false = only the ExitRequested fallback (recorded honestly)
}
pub type EngineState = Mutex<Option<Engine>>;
// —— direct kernel32 read (no windows-sys here: this specific binding tripped link/
// resolution issues before, and a plain extern declaration is all we need) ——
#[repr(C)]
struct MemoryStatusEx {
dw_length: u32,
memory_load: u32,
total_phys: u64,
avail_phys: u64,
total_pagefile: u64,
avail_pagefile: u64,
total_virtual: u64,
avail_virtual: u64,
avail_ext_virtual: u64,
}
extern "system" {
fn GlobalMemoryStatusEx(buf: *mut MemoryStatusEx) -> i32;
}
/// Physical RAM in GB: env override first (tests pin it), else a real
/// GlobalMemoryStatusEx read; on read failure fall back conservatively to 48 (the
/// reference machine — clamping stays safe without this value).
pub fn phys_mem_gb() -> u64 {
if let Ok(v) = std::env::var("EDGE0_PHYS_MEM_GB") {
if let Ok(n) = v.parse::<u64>() {
return n;
}
}
unsafe {
let mut m: MemoryStatusEx = std::mem::zeroed();
m.dw_length = std::mem::size_of::<MemoryStatusEx>() as u32;
if GlobalMemoryStatusEx(&mut m) != 0 && m.total_phys > 0 {
return (m.total_phys / 1_000_000_000).max(1);
}
}
48
}
/// Pool size = catalog value clamped by physical RAM (safe band ~2-4 GB; no XL tier).
pub fn pool_for(tier: &str) -> u32 {
let base = catalog::catalog().get(tier).map(|t| t.pool_mb).unwrap_or(2048);
let gb = phys_mem_gb();
let cap = match () {
_ if gb < 10 => 1024,
_ if gb < 16 => 2048,
_ => base,
};
base.min(cap).max(512)
}
// Job Object safety net: llama-server is assigned to a job created with
// KILL_ON_JOB_CLOSE, so whether the shell exits cleanly, crashes, or is force-killed
// (taskkill /f), the OS closes the process handle -> closes the job -> kills the
// child. No orphans. This uses the official windows-sys JobObjects bindings (a
// standard kernel32 import; the struct layout also rules out the hand-rolled-FFI
// ERROR_BAD_LENGTH class of bugs).
use std::os::windows::io::AsRawHandle;
use windows_sys::Win32::Foundation::CloseHandle;
use windows_sys::Win32::System::JobObjects::{
AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation,
SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
};
struct JobHandle(Option<*mut core::ffi::c_void>); // windows-sys 0.59: HANDLE = *mut c_void
// The job object is thread-agnostic, so sharing the raw handle across threads is
// safe; we intentionally never close it during process lifetime — the OS reclaims
// handles at exit, and that very close is what kills the child.
unsafe impl Sync for JobHandle {}
unsafe impl Send for JobHandle {}
static KILL_JOB: OnceLock<JobHandle> = OnceLock::new();
fn kill_job() -> Option<*mut core::ffi::c_void> {
KILL_JOB.get_or_init(|| unsafe {
let h = CreateJobObjectW(std::ptr::null(), std::ptr::null());
if h.is_null() {
return JobHandle(None);
}
let mut li: JOBOBJECT_EXTENDED_LIMIT_INFORMATION = std::mem::zeroed();
li.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
let ok = SetInformationJobObject(
h,
JobObjectExtendedLimitInformation,
&li as *const _ as *const core::ffi::c_void,
std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32,
);
if ok == 0 {
CloseHandle(h);
return JobHandle(None);
}
JobHandle(Some(h))
})
.0
}
/// Attach the child to the kill-on-close job (best effort: assign fails if the
/// process already belongs to another job; the caller records false honestly, and
/// the normal path keeps the ExitRequested→stop fallback as double insurance).
pub fn assign_kill_job(child: &Child) -> bool {
match kill_job() {
Some(job) => unsafe { AssignProcessToJobObject(job, child.as_raw_handle()) != 0 },
None => false,
}
}
fn free_port() -> Result<u16, String> {
TcpListener::bind("127.0.0.1:0").map(|l| l.local_addr().map(|a| a.port()).unwrap_or(0)).map_err(|e| e.to_string())
}
pub fn start(sink: &Arc<dyn Sink>, state: &EngineState, tier: &str) -> Result<Value, String> {
stop(state);
let gguf = paths::gguf_dir(tier).join(format!("edge0-{tier}.gguf"));
let adapter = paths::files_dir(tier).join(format!("lora_edge0_{tier}-gguf.gguf"));
if !gguf.exists() {
return Err(format!("E-MODEL-MISSING not converted yet (run the download first): {gguf:?}"));
}
let bin = paths::bin_dir();
let exe = bin.join("llama-server.exe");
if !exe.exists() {
return Err(format!("E-ENGINE-MISSING {exe:?} (set EDGE0_BIN_DIR)"));
}
let port = free_port()?;
let pool = pool_for(tier);
let log_p = paths::logs_dir().join(format!("engine-{tier}-{}.log", std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0)));
let logfile = std::fs::File::create(&log_p).map_err(|e| e.to_string())?;
let log2 = logfile.try_clone().map_err(|e| e.to_string())?;
let mut cmd = Command::new(&exe);
crate::no_window(&mut cmd)
.arg("-m").arg(&gguf)
.arg("--lora").arg(&adapter)
.args(["-ngl", "99", "-cmoe", "--ctx-size", "8192", "--flash-attn", "on", "--no-webui"])
.arg("--pool-mb").arg(pool.to_string())
.arg("--port").arg(port.to_string())
.current_dir(&bin) // resolve llama/ggml DLLs from the engine's own directory
.stdout(Stdio::from(logfile))
.stderr(Stdio::from(log2));
let child = cmd.spawn()
.map_err(|e| format!("spawn {exe:?} failed: {e}"))?;
let job = assign_kill_job(&child); // attach to the job before recording state (failure never blocks; normal path has double insurance)
let e = Engine {
child,
port,
tier: tier.into(),
pool_mb: pool,
started_at: std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0),
log_path: log_p.to_string_lossy().to_string(),
job,
};
let base = format!("http://127.0.0.1:{}", e.port);
*state.lock().unwrap() = Some(e);
// Readiness = the /health endpoint actually answering (700 ms tick, 300 s budget)
let t0 = std::time::Instant::now();
let client = reqwest::blocking::Client::builder().timeout(std::time::Duration::from_secs(2)).build().unwrap();
loop {
if client.get(format!("{base}/health")).send().map(|r| r.status().is_success()).unwrap_or(false) {
break;
}
let dead = state.lock().unwrap().as_mut().map(|en| en.child.try_wait().ok().flatten().is_some()).unwrap_or(true);
if dead {
*state.lock().unwrap() = None;
return Err(format!("E-ENGINE-DIED see {log_p:?}"));
}
if t0.elapsed().as_secs() > 300 {
return Err("E-ENGINE-READY-TIMEOUT".into());
}
std::thread::sleep(std::time::Duration::from_millis(700));
}
// Pool telemetry line from the engine log (absent line = pool not active; surfaced, not hidden)
let pool_line = std::fs::read_to_string(&log_p).ok().and_then(|s| {
s.lines().rev().find(|l| l.contains("POOL2 init")).map(|l| l.to_string())
});
let v = json!({ "running": true, "tier": tier, "base_url": base, "bound": "127.0.0.1",
"port": port, "pool_mb": pool, "uptime_s": t0.elapsed().as_secs(),
"log": log_p.to_string_lossy(), "version": server_version(), "job": job,
"phys_mem_gb": phys_mem_gb(), "pool_telemetry": pool_line });
sink.emit("engine.status", &v);
Ok(v)
}
pub fn stop(state: &EngineState) {
if let Some(mut e) = state.lock().unwrap().take() {
let _ = e.child.kill();
let _ = e.child.wait();
}
}
pub fn status(state: &EngineState) -> Value {
let mut ts = state.lock().unwrap();
match ts.as_mut() {
Some(e) => {
let dead = e.child.try_wait().ok().flatten().is_some();
let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0);
json!({ "running": !dead, "tier": e.tier,
"base_url": format!("http://127.0.0.1:{}", e.port),
"bound": "127.0.0.1", "port": e.port, "pool_mb": e.pool_mb,
"pid": e.child.id(), "uptime_s": now.saturating_sub(e.started_at),
"log": e.log_path, "version": server_version(), "job": e.job,
"phys_mem_gb": phys_mem_gb() })
}
None => json!({ "running": false }),
}
}
/// Engine binary version: the actual `llama-server --version` line.
/// A *successful* probe is cached for the session; failures are never cached
/// (issue #128: the old OnceLock<Option> shape let one early miss — e.g. a status
/// poll before the engine had landed — poison a whole fresh-install session with
/// "exists but version probe failed"). Capture is stdout+stderr with a tolerant
/// `version:` line match: the fork logs its banner to the log stream, and the old
/// stdout-only / stderr-discarded / strict-prefix probe never saw it.
static VER_CACHE: OnceLock<String> = OnceLock::new();
pub fn server_version() -> Option<String> {
VER_CACHE.get().cloned().or_else(server_version_probe)
}
/// Always-measure probe (doctor uses this: a manual check must read live facts,
/// never a stale negative). Only a success writes the cache.
pub fn server_version_probe() -> Option<String> {
let exe = paths::bin_dir().join("llama-server.exe");
let mut cmd = Command::new(&exe);
crate::no_window(&mut cmd)
.arg("--version")
.current_dir(paths::bin_dir())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let out = cmd.spawn().and_then(|c| c.wait_with_output()).ok()?;
let text = format!("{}\n{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));
let line = text.lines().map(str::trim_start)
.find(|l| l.starts_with("version:"))
.or_else(|| text.lines().find(|l| l.contains("version:")))?
.trim().to_string();
if line.is_empty() { return None; }
let _ = VER_CACHE.set(line.clone());
Some(line)
}
/// Read the POOL2 init telemetry line from a log (used by doctor; a missing line =
/// pool-not-active red flag, surfaced honestly).
pub fn pool_telemetry(log_path: &str) -> Option<String> {
std::fs::read_to_string(log_path).ok().and_then(|s| {
s.lines().rev().find(|l| l.contains("POOL2 init")).map(|l| l.trim().to_string())
})
}