Skip to content

The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.

Rust
10.8 KB
Raw Blame
//! Prepare a GGUF for Coachwhip: rewrite the expert tensors for speed on a 16 GB Mac and copy
//! everything else untouched, streaming, so the whole model never sits in memory.
//!
//!   coachwhip-prepare <in.gguf> <out.gguf>

use anyhow::{bail, Context, Result};
use byteorder::{LittleEndian, WriteBytesExt};
use candle::quantized::gguf_file::{Content, Value, ValueType};
use candle::quantized::{GgmlDType, QStorage, QTensor};
use candle::Device;
use coachwhip_engine::requant::{Requant, SrcKind, Q2K_BLOCK_BYTES, QK_K};
use std::borrow::Cow;
use std::io::{BufWriter, Read, Seek, SeekFrom, Write};
use std::time::Instant;

/// What each tensor becomes. The experts carry the bytes per word; the rest stays as it came.
fn target(name: &str, dtype: GgmlDType) -> GgmlDType {
    if name.ends_with(".ffn_gate_exps.weight") || name.ends_with(".ffn_up_exps.weight") {
        GgmlDType::Q2K
    } else if name.ends_with(".ffn_down_exps.weight") {
        // MXFP4 experts (gpt-oss) take down to Q2_K too: measured on the task-pool exam, the 120b
        // squeezed this way scores as the original. K-quant experts (Qwen) keep down at Q3_K.
        if dtype == GgmlDType::Mxfp4 || std::env::var("COACHWHIP_DOWN_Q2K").is_ok() { GgmlDType::Q2K } else { GgmlDType::Q3K }
    } else {
        dtype
    }
}

fn value_type_code(t: ValueType) -> u32 {
    match t {
        ValueType::U8 => 0,
        ValueType::I8 => 1,
        ValueType::U16 => 2,
        ValueType::I16 => 3,
        ValueType::U32 => 4,
        ValueType::I32 => 5,
        ValueType::F32 => 6,
        ValueType::Bool => 7,
        ValueType::String => 8,
        ValueType::Array => 9,
        ValueType::U64 => 10,
        ValueType::I64 => 11,
        ValueType::F64 => 12,
    }
}

fn write_string<W: Write>(w: &mut W, s: &str) -> Result<()> {
    w.write_u64::<LittleEndian>(s.len() as u64)?;
    w.write_all(s.as_bytes())?;
    Ok(())
}

fn write_value<W: Write>(w: &mut W, v: &Value) -> Result<()> {
    match v {
        Value::U8(x) => w.write_u8(*x)?,
        Value::I8(x) => w.write_i8(*x)?,
        Value::U16(x) => w.write_u16::<LittleEndian>(*x)?,
        Value::I16(x) => w.write_i16::<LittleEndian>(*x)?,
        Value::U32(x) => w.write_u32::<LittleEndian>(*x)?,
        Value::I32(x) => w.write_i32::<LittleEndian>(*x)?,
        Value::U64(x) => w.write_u64::<LittleEndian>(*x)?,
        Value::I64(x) => w.write_i64::<LittleEndian>(*x)?,
        Value::F32(x) => w.write_f32::<LittleEndian>(*x)?,
        Value::F64(x) => w.write_f64::<LittleEndian>(*x)?,
        Value::Bool(x) => w.write_u8(u8::from(*x))?,
        Value::String(s) => write_string(w, s)?,
        Value::Array(items) => {
            let t = items.first().map(|i| i.value_type()).unwrap_or(ValueType::U32);
            w.write_u32::<LittleEndian>(value_type_code(t))?;
            w.write_u64::<LittleEndian>(items.len() as u64)?;
            for item in items {
                write_value(w, item)?;
            }
        }
    }
    Ok(())
}

/// GGUF's type codes (Candle keeps its own mapping crate-private).
fn dtype_code(d: GgmlDType) -> u32 {
    match d {
        GgmlDType::F32 => 0,
        GgmlDType::F16 => 1,
        GgmlDType::Q4_0 => 2,
        GgmlDType::Q4_1 => 3,
        GgmlDType::Q5_0 => 6,
        GgmlDType::Q5_1 => 7,
        GgmlDType::Q8_0 => 8,
        GgmlDType::Q8_1 => 9,
        GgmlDType::Q2K => 10,
        GgmlDType::Q3K => 11,
        GgmlDType::Q4K => 12,
        GgmlDType::Q5K => 13,
        GgmlDType::Q6K => 14,
        GgmlDType::Q8K => 15,
        GgmlDType::BF16 => 30,
        GgmlDType::Mxfp4 => 39,
        GgmlDType::Iq2Xxs => 16,
        GgmlDType::Iq3Xxs => 18,
        GgmlDType::Iq4Xs => 23,
    }
}

/// A K-quant reads rows in whole 256-weight blocks; a row that is not a multiple of 256 (gpt-oss:
/// 2880) is padded with zeros to the next one. The engine pads its input to match.
fn out_dims(dims: &[usize], from: GgmlDType, to: GgmlDType) -> Vec<usize> {
    let mut d = dims.to_vec();
    if to != from {
        let last = d.len() - 1;
        d[last] = d[last].div_ceil(to.block_size()) * to.block_size();
    }
    d
}

fn bytes_for(dtype: GgmlDType, elems: usize) -> usize {
    elems / dtype.block_size() * dtype.type_size()
}

fn pad_to(w: &mut impl Write, pos: u64, align: u64) -> Result<u64> {
    let padding = (align - pos % align) % align;
    if padding > 0 {
        w.write_all(&vec![0u8; padding as usize])?;
    }
    Ok(pos + padding)
}

fn main() -> Result<()> {
    let args: Vec<String> = std::env::args().collect();
    if args.len() != 3 {
        bail!("usage: coachwhip-prepare <in.gguf> <out.gguf>");
    }
    let (src_path, dst_path) = (&args[1], &args[2]);
    let mut src = std::fs::File::open(src_path).with_context(|| format!("opening {src_path}"))?;
    let content = Content::read(&mut src).context("reading the GGUF header")?;
    let align = match content.metadata.get("general.alignment") {
        Some(Value::U32(a)) => *a as u64,
        _ => 32,
    };

    // Tensors in the order they sit in the source file.
    let mut infos: Vec<(&String, &candle::quantized::gguf_file::TensorInfo)> = content.tensor_infos.iter().collect();
    infos.sort_by_key(|(_, i)| i.offset);

    // New offsets, from the new sizes.
    let mut new_offsets = Vec::with_capacity(infos.len());
    let mut offset = 0u64;
    let mut changed = 0usize;
    for (name, info) in &infos {
        let to = target(name, info.ggml_dtype);
        if to != info.ggml_dtype {
            changed += 1;
        }
        new_offsets.push(offset);
        offset += bytes_for(to, out_dims(info.shape.dims(), info.ggml_dtype, to).iter().product()) as u64;
        offset = (offset + align - 1) / align * align;
    }
    eprintln!("coachwhip-prepare: {} tensors, {changed} to rewrite, {:.1} GB out", infos.len(), offset as f64 / 1e9);

    // The GPU shrink for gate and up (Q3_K/Q4_K/Q5_K -> Q2_K); down goes the CPU way to Q3_K.
    let gpu = match Device::new_metal(0) {
        Ok(Device::Metal(m)) => match Requant::new(&m) {
            Ok(rq) => {
                eprintln!("coachwhip-prepare: squeezing gate and up on the GPU");
                Some((m, rq))
            }
            Err(e) => {
                eprintln!("coachwhip-prepare: GPU shrink unavailable ({e}); using the CPU");
                None
            }
        },
        _ => None,
    };
    let out = std::fs::File::create(dst_path).with_context(|| format!("creating {dst_path}"))?;
    let mut w = BufWriter::with_capacity(16 << 20, out);
    // Header.
    w.write_u32::<LittleEndian>(0x4655_4747)?;
    w.write_u32::<LittleEndian>(3)?;
    w.write_u64::<LittleEndian>(infos.len() as u64)?;
    w.write_u64::<LittleEndian>(content.metadata.len() as u64)?;
    let mut keys: Vec<&String> = content.metadata.keys().collect();
    keys.sort();
    for k in keys {
        let v = &content.metadata[k];
        write_string(&mut w, k)?;
        w.write_u32::<LittleEndian>(value_type_code(v.value_type()))?;
        write_value(&mut w, v)?;
    }
    for ((name, info), new_off) in infos.iter().zip(&new_offsets) {
        write_string(&mut w, name)?;
        let dims = out_dims(info.shape.dims(), info.ggml_dtype, target(name, info.ggml_dtype));
        w.write_u32::<LittleEndian>(dims.len() as u32)?;
        for &d in dims.iter().rev() {
            w.write_u64::<LittleEndian>(d as u64)?;
        }
        w.write_u32::<LittleEndian>(dtype_code(target(name, info.ggml_dtype)))?;
        w.write_u64::<LittleEndian>(*new_off)?;
    }
    w.flush()?;
    let pos = w.get_ref().metadata()?.len();
    let data_start = pad_to(&mut w, pos, align)?;

    // Data, tensor by tensor.
    let started = Instant::now();
    let mut written = data_start;
    let total = infos.len();
    for (i, ((name, info), new_off)) in infos.iter().zip(&new_offsets).enumerate() {
        let expect = data_start + new_off;
        if written != expect {
            bail!("offset drift at {name}: at {written}, expected {expect}");
        }
        let elems = info.shape.elem_count();
        let src_len = bytes_for(info.ggml_dtype, elems);
        src.seek(SeekFrom::Start(content.tensor_data_offset + info.offset))?;
        let mut buf = vec![0u8; src_len];
        src.read_exact(&mut buf).with_context(|| format!("reading {name}"))?;
        let to = target(name, info.ggml_dtype);
        if to == info.ggml_dtype {
            w.write_all(&buf)?;
            written += src_len as u64;
        } else if let (GgmlDType::Q2K, Some((metal, rq)), Some(kind)) = (to, gpu.as_ref(), SrcKind::of(info.ggml_dtype)) {
            // On the GPU: the whole tensor in one dispatch, one thread per 256-weight block.
            let n_blocks = elems / QK_K;
            let src_buf = metal.new_buffer_builder().with_size(src_len).with_label("prepare_src").build()?;
            unsafe { std::ptr::copy_nonoverlapping(buf.as_ptr(), src_buf.contents() as *mut u8, src_len) };
            let out_len = n_blocks * Q2K_BLOCK_BYTES;
            let dst_buf = metal.new_buffer_builder().with_size(out_len).with_label("prepare_dst").build()?;
            rq.run(kind, &src_buf, 0, &dst_buf, 0, n_blocks)?;
            let data = unsafe { std::slice::from_raw_parts(dst_buf.contents() as *const u8, out_len) };
            w.write_all(data)?;
            written += out_len as u64;
        } else if out_dims(info.shape.dims(), info.ggml_dtype, to) != info.shape.dims() {
            // Padded rows: one expert at a time, so a big tensor never sits in memory as f32.
            let dims = info.shape.dims();
            let (n, rows, cols) = (dims[0], dims[1], dims[2]);
            let padded = out_dims(dims, info.ggml_dtype, to)[2];
            let per = bytes_for(info.ggml_dtype, rows * cols);
            for e in 0..n {
                let storage = QStorage::from_data(Cow::Owned(buf[e * per..(e + 1) * per].to_vec()), &Device::Cpu, info.ggml_dtype)?;
                let f = QTensor::new(storage, (rows, cols))?.dequantize(&Device::Cpu)?;
                let f = f.pad_with_zeros(1, 0, padded - cols)?;
                let q2 = QTensor::quantize(&f, to)?;
                let data = q2.data()?;
                w.write_all(&data)?;
                written += data.len() as u64;
            }
        } else {
            let storage = QStorage::from_data(Cow::Owned(buf), &Device::Cpu, info.ggml_dtype)?;
            let q = QTensor::new(storage, info.shape.clone())?;
            let f = q.dequantize(&Device::Cpu)?;
            let q2 = QTensor::quantize(&f, to)?;
            let data = q2.data()?;
            w.write_all(&data)?;
            written += data.len() as u64;
        }
        written = pad_to(&mut w, written, align)?;
        if i % 10 == 0 || i + 1 == total {
            eprintln!("coachwhip-prepare: {}/{} tensors, {:.1} GB written, {:.0} s", i + 1, total, (written as f64) / 1e9, started.elapsed().as_secs_f64());
        }
    }
    w.flush()?;
    eprintln!("coachwhip-prepare: done, {:.1} GB in {:.0} s", written as f64 / 1e9, started.elapsed().as_secs_f64());
    Ok(())
}