Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
954 B
Raw Blame
"""MLX backend: quantized gather kernels (bit-compatible with mlx-lm)."""

from __future__ import annotations

import mlx.core as mx


def gather_qmm(x, w, scales, biases, rhs_indices, transpose=True,
               group_size=64, bits=4, mode="affine",
               sorted_indices=False):
    """Quantized matmul over a gathered subset of expert rows.

    ``w`` is a stacked [num_experts, out, in] weight tensor; rows are
    gathered by ``rhs_indices`` before dequantizing.  Same kernel and
    defaults as the resident (non-streaming) MoE path, so outputs are
    bit-identical to a resident ``SwitchGLU``.
    """
    return mx.gather_qmm(
        x, w, scales, biases, rhs_indices=rhs_indices, transpose=transpose,
        group_size=group_size, bits=bits, mode=mode,
        sorted_indices=sorted_indices)


def swiglu(up: mx.array, gate: mx.array) -> mx.array:
    """SiLU gated activation: silu(gate) * up."""
    return mx.nn.silu(gate) * up