Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
11.4 KB
Raw Blame
// engine.rs — supervisor for the llama-server child process (the daemon's duties
// folded into the shell). The spawn arg table is the process form of the runtime
// contract: -ngl 99 -cmoe --ctx-size 8192 --flash-attn on --pool-mb <tier-clamped>
// --no-webui --port <random free>. Readiness is decided by polling /health, not by
// a fixed sleep. A crash is surfaced and not auto-restarted within the session.
use crate::catalog;
use crate::paths;
use crate::sink::Sink;
use serde_json::{json, Value};
use std::net::TcpListener;
use std::process::{Child, Command, Stdio};
use std::sync::{Arc, Mutex, OnceLock};

#[derive(Debug)]
pub struct Engine {
    pub child: Child,
    pub port: u16,
    pub tier: String,
    pub pool_mb: u32,
    pub started_at: u64, // UNIX seconds (source of truth for the service panel's uptime)
    pub log_path: String,
    pub job: bool, // true = attached to a KILL_ON_JOB_CLOSE job; false = only the ExitRequested fallback (recorded honestly)
}

pub type EngineState = Mutex<Option<Engine>>;

// —— direct kernel32 read (no windows-sys here: this specific binding tripped link/
//    resolution issues before, and a plain extern declaration is all we need) ——
#[repr(C)]
struct MemoryStatusEx {
    dw_length: u32,
    memory_load: u32,
    total_phys: u64,
    avail_phys: u64,
    total_pagefile: u64,
    avail_pagefile: u64,
    total_virtual: u64,
    avail_virtual: u64,
    avail_ext_virtual: u64,
}

extern "system" {
    fn GlobalMemoryStatusEx(buf: *mut MemoryStatusEx) -> i32;
}

/// Physical RAM in GB: env override first (tests pin it), else a real
/// GlobalMemoryStatusEx read; on read failure fall back conservatively to 48 (the
/// reference machine — clamping stays safe without this value).
pub fn phys_mem_gb() -> u64 {
    if let Ok(v) = std::env::var("EDGE0_PHYS_MEM_GB") {
        if let Ok(n) = v.parse::<u64>() {
            return n;
        }
    }
    unsafe {
        let mut m: MemoryStatusEx = std::mem::zeroed();
        m.dw_length = std::mem::size_of::<MemoryStatusEx>() as u32;
        if GlobalMemoryStatusEx(&mut m) != 0 && m.total_phys > 0 {
            return (m.total_phys / 1_000_000_000).max(1);
        }
    }
    48
}

/// Pool size = catalog value clamped by physical RAM (safe band ~2-4 GB; no XL tier).
pub fn pool_for(tier: &str) -> u32 {
    let base = catalog::catalog().get(tier).map(|t| t.pool_mb).unwrap_or(2048);
    let gb = phys_mem_gb();
    let cap = match () {
        _ if gb < 10 => 1024,
        _ if gb < 16 => 2048,
        _ => base,
    };
    base.min(cap).max(512)
}

// Job Object safety net: llama-server is assigned to a job created with
// KILL_ON_JOB_CLOSE, so whether the shell exits cleanly, crashes, or is force-killed
// (taskkill /f), the OS closes the process handle -> closes the job -> kills the
// child. No orphans. This uses the official windows-sys JobObjects bindings (a
// standard kernel32 import; the struct layout also rules out the hand-rolled-FFI
// ERROR_BAD_LENGTH class of bugs).
use std::os::windows::io::AsRawHandle;
use windows_sys::Win32::Foundation::CloseHandle;
use windows_sys::Win32::System::JobObjects::{
    AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation,
    SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
};

struct JobHandle(Option<*mut core::ffi::c_void>); // windows-sys 0.59: HANDLE = *mut c_void
// The job object is thread-agnostic, so sharing the raw handle across threads is
// safe; we intentionally never close it during process lifetime — the OS reclaims
// handles at exit, and that very close is what kills the child.
unsafe impl Sync for JobHandle {}
unsafe impl Send for JobHandle {}

static KILL_JOB: OnceLock<JobHandle> = OnceLock::new();

fn kill_job() -> Option<*mut core::ffi::c_void> {
    KILL_JOB.get_or_init(|| unsafe {
        let h = CreateJobObjectW(std::ptr::null(), std::ptr::null());
        if h.is_null() {
            return JobHandle(None);
        }
        let mut li: JOBOBJECT_EXTENDED_LIMIT_INFORMATION = std::mem::zeroed();
        li.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
        let ok = SetInformationJobObject(
            h,
            JobObjectExtendedLimitInformation,
            &li as *const _ as *const core::ffi::c_void,
            std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32,
        );
        if ok == 0 {
            CloseHandle(h);
            return JobHandle(None);
        }
        JobHandle(Some(h))
    })
    .0
}

/// Attach the child to the kill-on-close job (best effort: assign fails if the
/// process already belongs to another job; the caller records false honestly, and
/// the normal path keeps the ExitRequested→stop fallback as double insurance).
pub fn assign_kill_job(child: &Child) -> bool {
    match kill_job() {
        Some(job) => unsafe { AssignProcessToJobObject(job, child.as_raw_handle()) != 0 },
        None => false,
    }
}

fn free_port() -> Result<u16, String> {
    TcpListener::bind("127.0.0.1:0").map(|l| l.local_addr().map(|a| a.port()).unwrap_or(0)).map_err(|e| e.to_string())
}

pub fn start(sink: &Arc<dyn Sink>, state: &EngineState, tier: &str) -> Result<Value, String> {
    stop(state);
    let gguf = paths::gguf_dir(tier).join(format!("edge0-{tier}.gguf"));
    let adapter = paths::files_dir(tier).join(format!("lora_edge0_{tier}-gguf.gguf"));
    if !gguf.exists() {
        return Err(format!("E-MODEL-MISSING not converted yet (run the download first): {gguf:?}"));
    }
    let bin = paths::bin_dir();
    let exe = bin.join("llama-server.exe");
    if !exe.exists() {
        return Err(format!("E-ENGINE-MISSING {exe:?} (set EDGE0_BIN_DIR)"));
    }
    let port = free_port()?;
    let pool = pool_for(tier);
    let log_p = paths::logs_dir().join(format!("engine-{tier}-{}.log", std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0)));
    let logfile = std::fs::File::create(&log_p).map_err(|e| e.to_string())?;
    let log2 = logfile.try_clone().map_err(|e| e.to_string())?;
    let mut cmd = Command::new(&exe);
    crate::no_window(&mut cmd)
        .arg("-m").arg(&gguf)
        .arg("--lora").arg(&adapter)
        .args(["-ngl", "99", "-cmoe", "--ctx-size", "8192", "--flash-attn", "on", "--no-webui"])
        .arg("--pool-mb").arg(pool.to_string())
        .arg("--port").arg(port.to_string())
        .current_dir(&bin) // resolve llama/ggml DLLs from the engine's own directory
        .stdout(Stdio::from(logfile))
        .stderr(Stdio::from(log2));
    let child = cmd.spawn()
        .map_err(|e| format!("spawn {exe:?} failed: {e}"))?;
    let job = assign_kill_job(&child); // attach to the job before recording state (failure never blocks; normal path has double insurance)
    let e = Engine {
        child,
        port,
        tier: tier.into(),
        pool_mb: pool,
        started_at: std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0),
        log_path: log_p.to_string_lossy().to_string(),
        job,
    };
    let base = format!("http://127.0.0.1:{}", e.port);
    *state.lock().unwrap() = Some(e);

    // Readiness = the /health endpoint actually answering (700 ms tick, 300 s budget)
    let t0 = std::time::Instant::now();
    let client = reqwest::blocking::Client::builder().timeout(std::time::Duration::from_secs(2)).build().unwrap();
    loop {
        if client.get(format!("{base}/health")).send().map(|r| r.status().is_success()).unwrap_or(false) {
            break;
        }
        let dead = state.lock().unwrap().as_mut().map(|en| en.child.try_wait().ok().flatten().is_some()).unwrap_or(true);
        if dead {
            *state.lock().unwrap() = None;
            return Err(format!("E-ENGINE-DIED see {log_p:?}"));
        }
        if t0.elapsed().as_secs() > 300 {
            return Err("E-ENGINE-READY-TIMEOUT".into());
        }
        std::thread::sleep(std::time::Duration::from_millis(700));
    }
    // Pool telemetry line from the engine log (absent line = pool not active; surfaced, not hidden)
    let pool_line = std::fs::read_to_string(&log_p).ok().and_then(|s| {
        s.lines().rev().find(|l| l.contains("POOL2 init")).map(|l| l.to_string())
    });
    let v = json!({ "running": true, "tier": tier, "base_url": base, "bound": "127.0.0.1",
                    "port": port, "pool_mb": pool, "uptime_s": t0.elapsed().as_secs(),
                    "log": log_p.to_string_lossy(), "version": server_version(), "job": job,
                    "phys_mem_gb": phys_mem_gb(), "pool_telemetry": pool_line });
    sink.emit("engine.status", &v);
    Ok(v)
}

pub fn stop(state: &EngineState) {
    if let Some(mut e) = state.lock().unwrap().take() {
        let _ = e.child.kill();
        let _ = e.child.wait();
    }
}

pub fn status(state: &EngineState) -> Value {
    let mut ts = state.lock().unwrap();
    match ts.as_mut() {
        Some(e) => {
            let dead = e.child.try_wait().ok().flatten().is_some();
            let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0);
            json!({ "running": !dead, "tier": e.tier,
                    "base_url": format!("http://127.0.0.1:{}", e.port),
                    "bound": "127.0.0.1", "port": e.port, "pool_mb": e.pool_mb,
                    "pid": e.child.id(), "uptime_s": now.saturating_sub(e.started_at),
                    "log": e.log_path, "version": server_version(), "job": e.job,
                    "phys_mem_gb": phys_mem_gb() })
        }
        None => json!({ "running": false }),
    }
}

/// Engine binary version: the actual `llama-server --version` line.
/// A *successful* probe is cached for the session; failures are never cached
/// (issue #128: the old OnceLock<Option> shape let one early miss — e.g. a status
/// poll before the engine had landed — poison a whole fresh-install session with
/// "exists but version probe failed"). Capture is stdout+stderr with a tolerant
/// `version:` line match: the fork logs its banner to the log stream, and the old
/// stdout-only / stderr-discarded / strict-prefix probe never saw it.
static VER_CACHE: OnceLock<String> = OnceLock::new();

pub fn server_version() -> Option<String> {
    VER_CACHE.get().cloned().or_else(server_version_probe)
}

/// Always-measure probe (doctor uses this: a manual check must read live facts,
/// never a stale negative). Only a success writes the cache.
pub fn server_version_probe() -> Option<String> {
    let exe = paths::bin_dir().join("llama-server.exe");
    let mut cmd = Command::new(&exe);
    crate::no_window(&mut cmd)
        .arg("--version")
        .current_dir(paths::bin_dir())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    let out = cmd.spawn().and_then(|c| c.wait_with_output()).ok()?;
    let text = format!("{}\n{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));
    let line = text.lines().map(str::trim_start)
        .find(|l| l.starts_with("version:"))
        .or_else(|| text.lines().find(|l| l.contains("version:")))?
        .trim().to_string();
    if line.is_empty() { return None; }
    let _ = VER_CACHE.set(line.clone());
    Some(line)
}

/// Read the POOL2 init telemetry line from a log (used by doctor; a missing line =
/// pool-not-active red flag, surfaced honestly).
pub fn pool_telemetry(log_path: &str) -> Option<String> {
    std::fs::read_to_string(log_path).ok().and_then(|s| {
        s.lines().rev().find(|l| l.contains("POOL2 init")).map(|l| l.trim().to_string())
    })
}