Skip to content

web search done right

option to perform simple quantization of some of the accelerated alice layers

Mikkel Denker479444dfb85a
diff --git a/configs/alice.toml b/configs/alice.toml
index ae1857d9..c6b0bf00 100644
--- a/configs/alice.toml
+++ b/configs/alice.toml
@@ -8,6 +8,7 @@ encryption_key = "URyJQTjwUjTq6FSRoZGdbUdTIvqs/QxkPacQio8Lhxc=" # don't use this
[accelerator]
layer_fraction = 1.0
+quantize_fraction = 0.0 # percentage of layers on accelerator to quantize
[accelerator.device]
type = "mps"
diff --git a/core/examples/alice.rs b/core/examples/alice.rs
index c4d03012..d6ad78cc 100644
--- a/core/examples/alice.rs
+++ b/core/examples/alice.rs
@@ -19,6 +19,7 @@ async fn main() {
"data/summarizer/dual_encoder",
Some(AcceleratorConfig {
layer_fraction: 1.0,
+ quantize_fraction: 0.0,
device: tch::Device::Mps,
kind: tch::Kind::Float,
}),
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rs
index 3f38a12e..df58767c 100644
--- a/core/src/alice/mod.rs
+++ b/core/src/alice/mod.rs
@@ -235,6 +235,7 @@ unsafe impl Sync for Alice {}
pub struct AcceleratorConfig {
pub layer_fraction: f64,
+ pub quantize_fraction: f64,
pub device: tch::Device,
pub kind: tch::Kind,
}
@@ -262,6 +263,7 @@ impl From<AliceAcceleratorConfig> for AcceleratorConfig {
fn from(value: AliceAcceleratorConfig) -> Self {
Self {
layer_fraction: value.layer_fraction,
+ quantize_fraction: value.quantize_fraction,
device: value.device.into(),
kind: value.dtype.into(),
}
@@ -281,6 +283,7 @@ impl Alice {
if let Some(accelerator) = accelerator {
model.load_to_device(
accelerator.layer_fraction,
+ accelerator.quantize_fraction,
accelerator.device,
accelerator.kind,
);
diff --git a/core/src/alice/raw_model.rs b/core/src/alice/raw_model.rs
index 44d96404..1256448f 100644
--- a/core/src/alice/raw_model.rs
+++ b/core/src/alice/raw_model.rs
@@ -18,16 +18,160 @@ use std::{fs::OpenOptions, path::Path};
use crate::alice::{Error, Result};
use safetensors::SafeTensors;
use tch::{
- nn::{embedding, layer_norm, Embedding, LayerNorm, LayerNormConfig, Linear, ModuleT, VarStore},
+ nn::{embedding, layer_norm, Embedding, LayerNorm, LayerNormConfig, ModuleT, VarStore},
IndexOp, Kind, Tensor,
};
const NUM_TOKENS: i64 = 50277;
+enum Linear {
+ Normal(tch::nn::Linear),
+ Quantized {
+ ws: Tensor,
+ bs: Option<Tensor>,
+ scale: Tensor,
+ zero_point: Tensor,
+ },
+}
+
+fn quantize_tensor_per_channel(tensor: &Tensor) -> (Tensor, Tensor, Tensor) {
+ let min_vals = tensor.amin(1, true);
+ let max_vals = tensor.amax(1, true);
+ let qmin: f64 = -127.0; // for qint8
+ let qmax: f64 = 127.0; // for qint8
+
+ // calculate scale and zero_point
+ let scale = (&max_vals - &min_vals) / (qmax - qmin);
+ let initial_zero_point = qmin - &min_vals / &scale;
+
+ // clamp zero_point to qint8 range
+ let zero_point = initial_zero_point
+ .clamp(qmin, qmax)
+ .round()
+ .to_kind(Kind::Float);
+
+ let scale = scale.to_kind(Kind::Float);
+
+ let quantized_weights = ((tensor / &scale) + &zero_point)
+ .clamp(qmin, qmax)
+ .round()
+ .to_kind(Kind::Int8);
+
+ (quantized_weights, scale, zero_point)
+}
+
+fn dequantize(tensor: &Tensor, scale: &Tensor, zero_point: &Tensor) -> Tensor {
+ (tensor.to_kind(Kind::Float) - zero_point) * scale
+}
+
+impl Linear {
+ fn forward_t(&self, x: &Tensor, train: bool) -> Tensor {
+ match self {
+ Linear::Normal(linear) => linear.forward_t(x, train),
+ Linear::Quantized {
+ ws,
+ bs,
+ scale,
+ zero_point,
+ } => {
+ let ws = dequantize(ws, scale, zero_point);
+ let mut output = x.matmul(&ws.transpose(-2, -1));
+ if let Some(bias) = bs {
+ output += bias;
+ }
+ output
+ }
+ }
+ }
+
+ fn ws(&self) -> &Tensor {
+ match self {
+ Linear::Normal(linear) => &linear.ws,
+ Linear::Quantized { ws, .. } => ws,
+ }
+ }
+
+ fn set_ws(&mut self, ws: Tensor) {
+ match self {
+ Linear::Normal(linear) => linear.ws = ws,
+ Linear::Quantized { .. } => {
+ unimplemented!();
+ }
+ }
+ }
+
+ fn bs(&self) -> Option<&Tensor> {
+ match self {
+ Linear::Normal(linear) => linear.bs.as_ref(),
+ Linear::Quantized { bs, .. } => bs.as_ref(),
+ }
+ }
+
+ fn set_bs(&mut self, bs: Option<Tensor>) {
+ if let Some(bs) = bs {
+ match self {
+ Linear::Normal(linear) => linear.bs = Some(bs),
+ Linear::Quantized { bs: bs_, .. } => *bs_ = Some(bs),
+ }
+ }
+ }
+
+ fn quantize(&self) -> Self {
+ match self {
+ Self::Quantized {
+ ws,
+ bs,
+ scale,
+ zero_point,
+ } => {
+ let new_ws = Tensor::empty(ws.size(), (ws.kind(), ws.device()));
+ let _ = ws.clone(&new_ws);
+
+ let mut new_bs = None;
+
+ if let Some(bs) = bs {
+ new_bs = Some(Tensor::empty(bs.size(), (bs.kind(), bs.device())));
+ let _ = bs.clone(new_bs.as_mut().unwrap());
+ }
+
+ let new_scale = Tensor::empty(scale.size(), (scale.kind(), scale.device()));
+ let _ = scale.clone(&new_scale);
+
+ let new_zeropoint =
+ Tensor::empty(zero_point.size(), (zero_point.kind(), zero_point.device()));
+ let _ = zero_point.clone(&new_zeropoint);
+
+ Self::Quantized {
+ ws: new_ws,
+ bs: new_bs,
+ scale: new_scale,
+ zero_point: new_zeropoint,
+ }
+ }
+ Self::Normal(linear) => {
+ let (ws, scale, zero_point) = quantize_tensor_per_channel(&linear.ws);
+
+ let mut new_bs = None;
+ if let Some(bs) = &linear.bs {
+ new_bs = Some(Tensor::empty(bs.size(), (bs.kind(), bs.device())));
+ let _ = bs.clone(new_bs.as_mut().unwrap());
+ }
+
+ Self::Quantized {
+ ws,
+ bs: new_bs,
+ scale,
+ zero_point,
+ }
+ }
+ }
+ }
+}
+
fn load_linear(weights: &SafeTensors, prefix: &str) -> Result<Linear> {
let ws = weights.tensor(&format!("{prefix}.weight"))?.try_into()?;
- let mut linear = Linear { ws, bs: None };
+ let mut linear = tch::nn::Linear { ws, bs: None };
if let Ok(bias_tensor) = weights.tensor(&format!("{prefix}.bias")) {
linear.bs = Some(bias_tensor.try_into()?);
@@ -39,7 +183,7 @@ fn load_linear(weights: &SafeTensors, prefix: &str) -> Result<Linear> {
linear.ws = linear.ws.to_kind(Kind::Float);
- Ok(linear)
+ Ok(Linear::Normal(linear))
}
fn load_emb(weights: &SafeTensors, prefix: &str, emb_size: i64) -> Result<Embedding> {
@@ -109,10 +253,10 @@ struct TimeMix {
time_mix_v: Tensor,
time_mix_r: Tensor,
- key: Linear,
- value: Linear,
- receptance: Linear,
- output: Linear,
+ pub key: Linear,
+ pub value: Linear,
+ pub receptance: Linear,
+ pub output: Linear,
block_idx: i64,
}
@@ -165,21 +309,23 @@ impl TimeMix {
self.time_mix_v = self.time_mix_v.to_kind(kind).to(device);
self.time_mix_r = self.time_mix_r.to_kind(kind).to(device);
- self.key.ws = self.key.ws.to_kind(kind).to(device);
- self.key.bs = self.key.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+ self.key.set_ws(self.key.ws().to_kind(kind).to(device));
+ self.key
+ .set_bs(self.key.bs().map(|t| t.to_kind(kind).to(device)));
- self.value.ws = self.value.ws.to_kind(kind).to(device);
- self.value.bs = self.value.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+ self.value.set_ws(self.value.ws().to_kind(kind).to(device));
+ self.value
+ .set_bs(self.value.bs().map(|t| t.to_kind(kind).to(device)));
- self.receptance.ws = self.receptance.ws.to_kind(kind).to(device);
- self.receptance.bs = self
- .receptance
- .bs
- .as_ref()
- .map(|t| t.to_kind(kind).to(device));
+ self.receptance
+ .set_ws(self.receptance.ws().to_kind(kind).to(device));
+ self.receptance
+ .set_bs(self.receptance.bs().map(|t| t.to_kind(kind).to(device)));
- self.output.ws = self.output.ws.to_kind(kind).to(device);
- self.output.bs = self.output.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+ self.output
+ .set_ws(self.output.ws().to_kind(kind).to(device));
+ self.output
+ .set_bs(self.output.bs().map(|t| t.to_kind(kind).to(device)));
}
}
@@ -187,10 +333,9 @@ struct ChannelMix {
time_mix_k: Tensor,
time_mix_r: Tensor,
- key: Linear,
- value: Linear,
-
- receptance: Linear,
+ pub key: Linear,
+ pub value: Linear,
+ pub receptance: Linear,
block_idx: i64,
}
@@ -212,18 +357,18 @@ impl ChannelMix {
self.time_mix_k = self.time_mix_k.to_kind(kind).to(device);
self.time_mix_r = self.time_mix_r.to_kind(kind).to(device);
- self.key.ws = self.key.ws.to_kind(kind).to(device);
- self.key.bs = self.key.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+ self.key.set_ws(self.key.ws().to_kind(kind).to(device));
+ self.key
+ .set_bs(self.key.bs().map(|t| t.to_kind(kind).to(device)));
- self.value.ws = self.value.ws.to_kind(kind).to(device);
- self.value.bs = self.value.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+ self.value.set_ws(self.value.ws().to_kind(kind).to(device));
+ self.value
+ .set_bs(self.value.bs().map(|t| t.to_kind(kind).to(device)));
- self.receptance.ws = self.receptance.ws.to_kind(kind).to(device);
- self.receptance.bs = self
- .receptance
- .bs
- .as_ref()
- .map(|t| t.to_kind(kind).to(device));
+ self.receptance
+ .set_ws(self.receptance.ws().to_kind(kind).to(device));
+ self.receptance
+ .set_bs(self.receptance.bs().map(|t| t.to_kind(kind).to(device)));
}
}
@@ -232,8 +377,8 @@ struct Block {
ln1: LayerNorm,
ln2: LayerNorm,
- att: TimeMix,
- ffn: ChannelMix,
+ pub att: TimeMix,
+ pub ffn: ChannelMix,
device: tch::Device,
}
@@ -340,6 +485,16 @@ impl Block {
self.ln2.ws = self.ln2.ws.as_ref().map(|t| t.to_kind(kind).to(device));
self.ln2.bs = self.ln2.bs.as_ref().map(|t| t.to_kind(kind).to(device));
}
+
+ fn quantize(&mut self) {
+ self.att.key = self.att.key.quantize();
+ self.att.value = self.att.value.quantize();
+ self.att.receptance = self.att.receptance.quantize();
+
+ self.ffn.key = self.ffn.key.quantize();
+ self.ffn.value = self.ffn.value.quantize();
+ self.ffn.receptance = self.ffn.receptance.quantize();
+ }
}
pub struct RawModel {
@@ -396,23 +551,37 @@ impl RawModel {
})
}
- pub fn load_to_device(&mut self, layer_fraction: f64, device: tch::Device, kind: tch::Kind) {
+ pub fn load_to_device(
+ &mut self,
+ layer_fraction: f64,
+ quantized_fraction: f64,
+ device: tch::Device,
+ kind: tch::Kind,
+ ) {
let layer_fraction = layer_fraction.max(0.0).min(1.0);
+ let quantized_fraction = quantized_fraction.max(0.0).min(1.0);
self.emb.ws = self.emb.ws.to_kind(kind).to(device);
let layers_to_move = (self.blocks.len() as f64 * layer_fraction).ceil() as usize;
+ let layers_to_quantize = ((layers_to_move as f64) * quantized_fraction).ceil() as usize;
- for block in self.blocks.iter_mut().take(layers_to_move) {
+ for (idx, block) in self.blocks.iter_mut().take(layers_to_move).enumerate() {
block.load_to_device(device, kind);
+
+ // quantize the last layers
+ if idx > layers_to_move - layers_to_quantize {
+ block.quantize();
+ }
}
if layer_fraction == 1.0 {
self.ln_out.ws = self.ln_out.ws.as_ref().map(|t| t.to_kind(kind).to(device));
self.ln_out.bs = self.ln_out.bs.as_ref().map(|t| t.to_kind(kind).to(device));
- self.head.ws = self.head.ws.to_kind(kind).to(device);
- self.head.bs = self.head.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+ self.head.set_ws(self.head.ws().to_kind(kind).to(device));
+ self.head
+ .set_bs(self.head.bs().map(|t| t.to_kind(kind).to(device)));
}
}
@@ -457,3 +626,17 @@ impl RawModel {
(x, state)
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn quantize() {
+ let t = tch::Tensor::randn([10, 10], (tch::Kind::Float, tch::Device::Cpu));
+ let (quantized, scale, zp) = quantize_tensor_per_channel(&t);
+ let dequantized = dequantize(&quantized, &scale, &zp);
+
+ assert!((t - &dequantized).abs().max().double_value(&[]) < 0.03);
+ }
+}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 3f133e65..f9ff3beb 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -237,6 +237,8 @@ pub enum AcceleratorDtype {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct AliceAcceleratorConfig {
pub layer_fraction: f64,
+ /// percentage of layers on accelerator to quantize
+ pub quantize_fraction: f64,
pub device: AcceleratorDevice,
pub dtype: AcceleratorDtype,
}