Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

2.1 KB
Raw Blame
#!/usr/bin/env bash
set -euo pipefail

PROJECT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
SOURCE_DIR="$PROJECT_DIR/deps/ml-drift"
"$PROJECT_DIR/bootstrap.sh"

MODEL=""
PRECISION="q4_0"
while [[ $# -gt 0 ]]; do
    case "$1" in
        -m|--model)
            if [[ $# -lt 2 || "$2" == -* ]]; then
                echo "$1 requires a model directory or Hugging Face repository" >&2
                exit 1
            fi
            MODEL="$2"
            shift 2
            ;;
        --precision)
            if [[ $# -lt 2 || ! "$2" =~ ^(q4_0|f16)$ ]]; then
                echo "$1 requires q4_0 or f16" >&2
                exit 1
            fi
            PRECISION="$2"
            shift 2
            ;;
        *)
            echo "Unknown argument: $1" >&2
            exit 1
            ;;
    esac
done
if [[ -z "$MODEL" ]]; then
    echo "--model is required" >&2
    exit 1
fi

MODEL_PATH="$MODEL"
if [[ ! -d "$MODEL_PATH" ]]; then
    if ! command -v hf >/dev/null; then
        echo "Install the Hugging Face CLI to download $MODEL: uv tool install huggingface_hub" >&2
        exit 1
    fi
    MODEL_PATH="$(hf download --quiet "$MODEL")"
fi
MODEL_PATH="$(cd -- "$MODEL_PATH" && pwd -P)"

WEIGHTS_DIR="$PROJECT_DIR/workspace/$(git -C "$SOURCE_DIR" rev-parse --short=12 HEAD)/$PRECISION"
WEIGHTS_DIR="$WEIGHTS_DIR/$(/usr/bin/stat -L -f '%N %z %m' "$MODEL_PATH"/*.safetensors | shasum -a 256 | cut -c1-16)"
if [[ ! -d "$WEIGHTS_DIR" ]]; then
    rm -rf "$WEIGHTS_DIR.partial"
    mkdir -p "$WEIGHTS_DIR.partial"
    QUANTIZATION=()
    if [[ "$PRECISION" == q4_0 ]]; then
        QUANTIZATION=(--quantize --embedding_quant_bits=4 --attention_quant_bits=4 --feedforward_quant_bits=4 --block_size=32)
    fi
    (cd "$SOURCE_DIR" && bazelisk run -c opt //ml_drift/samples/llm:extract_weights_hf -- \
        --model_path="$MODEL_PATH" \
        --output_dir="$WEIGHTS_DIR.partial" \
        ${QUANTIZATION[@]+"${QUANTIZATION[@]}"}) >&2
    mv "$WEIGHTS_DIR.partial" "$WEIGHTS_DIR"
fi

exec "$SOURCE_DIR/bazel-bin/benchmarks/engine-mldrift" \
    --checkpoint="$MODEL_PATH" \
    --weights="$WEIGHTS_DIR"