Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
6.7 KB
Raw Blame
//! Shared `dlopen` + symbol resolve + ABI handshake for daemon and worker.

use std::path::Path;

use libloading::{Library, Symbol};

use crate::version::E0_ABI_VERSION;

#[repr(C)]
#[derive(Clone, Copy)]
pub struct E0TokenOutC {
    pub size: u32,
    pub token: i32,
    pub flags: u32,
}

pub type FnAbiVersion = extern "C" fn() -> u32;
pub type FnEngineCreate = unsafe extern "C" fn(*const i8, *const i8, *const i8) -> *mut c_void;
pub type FnGenerateBegin =
    unsafe extern "C" fn(*mut c_void, *const i32, u32, *const i8) -> *mut c_void;
pub type FnNext = unsafe extern "C" fn(*mut c_void, *mut c_void, *mut E0TokenOutC) -> i32;
pub type FnCancel = unsafe extern "C" fn(*mut c_void, *mut c_void) -> i32;
pub type FnStats = unsafe extern "C" fn(*mut c_void, *mut i8, usize, *mut usize) -> i32;
pub type FnTurnReset = unsafe extern "C" fn(*mut c_void) -> i32;
pub type FnLastError = unsafe extern "C" fn(*mut c_void, *mut i8, usize, *mut usize) -> i32;
pub type FnEngineDestroy = unsafe extern "C" fn(*mut c_void);
pub type FnTokCreate = unsafe extern "C" fn(*const i8) -> *mut c_void;
pub type FnTokCall =
    unsafe extern "C" fn(*mut c_void, *const i8, *mut i8, usize, *mut usize) -> i32;
pub type FnTokDestroy = unsafe extern "C" fn(*mut c_void);

pub use std::ffi::c_void;

pub const E0_OK: i32 = 0;
pub const E0_NOTIFIED: i32 = 1;
pub const E0_WOULD_BLOCK: i32 = 2;

/// Report what was actually loaded; unknown basenames are passed through, not classified.
pub fn engine_backend(path: &Path) -> String {
    let base = path
        .file_name()
        .map(|s| s.to_string_lossy().into_owned())
        .unwrap_or_else(|| path.display().to_string());
    if base.contains("native") {
        "native".into()
    } else if base.contains("replay") {
        "replay".into()
    } else {
        base
    }
}

/// Version mismatch refuses to load; missing symbols (including `e0_tok_decode`) fail load.
pub struct EngineLib {
    _lib: Library,
    pub path: std::path::PathBuf,
    pub abi_version: FnAbiVersion,
    pub engine_create: FnEngineCreate,
    pub generate_begin: FnGenerateBegin,
    pub next: FnNext,
    pub cancel: FnCancel,
    pub stats: FnStats,
    pub turn_reset: FnTurnReset,
    pub last_error: FnLastError,
    pub engine_destroy: FnEngineDestroy,
    pub tok_create: FnTokCreate,
    pub tok_tokenize: FnTokCall,
    pub tok_apply_template: FnTokCall,
    pub tok_decode: FnTokCall,
    pub tok_destroy: FnTokDestroy,
}

// dyld is thread-safe; function pointers are read-only addresses.
unsafe impl Send for EngineLib {}
unsafe impl Sync for EngineLib {}

macro_rules! sym {
    ($lib:expr, $ty:ty, $name:literal) => {{
        let s: Symbol<$ty> = (unsafe { $lib.get($name.as_bytes()) })
            .map_err(|e| format!("symbol {} missing or wrong type: {e}", $name))?;
        *s
    }};
}

impl EngineLib {
    pub fn backend(&self) -> String {
        engine_backend(&self.path)
    }

    pub fn load(path: &Path) -> Result<Self, String> {
        let lib = unsafe { Library::new(path) }
            .map_err(|e| format!("dlopen {} failed: {e}", path.display()))?;
        let abi_version: FnAbiVersion = sym!(lib, FnAbiVersion, "e0_abi_version");
        let got = abi_version();
        if got != E0_ABI_VERSION {
            return Err(format!(
                "ABI handshake failed: library reports {got}, daemon compiled with E0_ABI_VERSION={E0_ABI_VERSION} — refusing to load"
            ));
        }
        let e = Self {
            path: path.to_path_buf(),
            abi_version,
            engine_create: sym!(lib, FnEngineCreate, "e0_engine_create"),
            generate_begin: sym!(lib, FnGenerateBegin, "e0_generate_begin"),
            next: sym!(lib, FnNext, "e0_next"),
            cancel: sym!(lib, FnCancel, "e0_cancel"),
            stats: sym!(lib, FnStats, "e0_stats"),
            turn_reset: sym!(lib, FnTurnReset, "e0_turn_reset"),
            last_error: sym!(lib, FnLastError, "e0_last_error"),
            engine_destroy: sym!(lib, FnEngineDestroy, "e0_engine_destroy"),
            tok_create: sym!(lib, FnTokCreate, "e0_tok_create"),
            tok_tokenize: sym!(lib, FnTokCall, "e0_tok_tokenize"),
            tok_apply_template: sym!(lib, FnTokCall, "e0_tok_apply_template"),
            tok_decode: sym!(lib, FnTokCall, "e0_tok_decode"),
            tok_destroy: sym!(lib, FnTokDestroy, "e0_tok_destroy"),
            _lib: lib,
        };
        Ok(e)
    }

    /// Two-probe JSON out-param wrapper (returns an owned String).
    ///
    /// # Safety
    /// `f` must report the required length when `buf==NULL` and write exactly `*written` bytes
    /// into a sufficient buffer, capturing only state valid for this call.
    pub unsafe fn call_json(
        &self,
        f: impl Fn(*mut i8, usize, *mut usize) -> i32,
    ) -> Result<String, i32> {
        let mut written = 0usize;
        let st = f(std::ptr::null_mut(), 0, &mut written);
        if st != 0 || written == 0 {
            return Err(st);
        }
        let mut buf = vec![0u8; written];
        let st = f(buf.as_mut_ptr() as *mut i8, written, &mut written);
        if st != 0 {
            return Err(st);
        }
        String::from_utf8(buf).map_err(|_| -1)
    }
}

/// Per-tier tokenizer handle on the daemon; independent of worker keep_alive.
/// Pointers are only touched inside this wrapper, under a Mutex at the call site.
pub struct TokHandle {
    pub lib: std::sync::Arc<EngineLib>,
    ptr: *mut c_void,
}
unsafe impl Send for TokHandle {}

impl TokHandle {
    pub fn open(lib: std::sync::Arc<EngineLib>, model_dir: &Path) -> Result<Self, String> {
        let c =
            std::ffi::CString::new(model_dir.display().to_string()).map_err(|e| e.to_string())?;
        let ptr = unsafe { (lib.tok_create)(c.as_ptr()) };
        if ptr.is_null() {
            return Err(format!(
                "e0_tok_create({}) returned NULL (tokenizer files missing or unreadable)",
                model_dir.display()
            ));
        }
        Ok(Self { lib, ptr })
    }
    fn call(&self, f: FnTokCall, input: &str) -> Result<String, i32> {
        let c = match std::ffi::CString::new(input) {
            Ok(c) => c,
            Err(_) => return Err(-1),
        };
        let lib = &self.lib;
        let p = self.ptr;
        unsafe { lib.call_json(|b, l, w| f(p, c.as_ptr(), b, l, w)) }
    }
    pub fn tokenize(&self, text_json: &str) -> Result<String, i32> {
        self.call(self.lib.tok_tokenize, text_json)
    }
    pub fn apply_template(&self, messages_json: &str) -> Result<String, i32> {
        self.call(self.lib.tok_apply_template, messages_json)
    }
    pub fn decode(&self, tokens_json: &str) -> Result<String, i32> {
        self.call(self.lib.tok_decode, tokens_json)
    }
}

impl Drop for TokHandle {
    fn drop(&mut self) {
        unsafe { (self.lib.tok_destroy)(self.ptr) };
    }
}