Skip to content

web search done right

Quantize crossencoder

Mikkel Denker72fa54a945fd
diff --git a/core/src/ranking/models/cross_encoder.rs b/core/src/ranking/models/cross_encoder.rs
index ba6a6dad..8ba3d688 100644
--- a/core/src/ranking/models/cross_encoder.rs
+++ b/core/src/ranking/models/cross_encoder.rs
@@ -22,7 +22,7 @@ use onnxruntime::{ndarray::ArrayBase, tensor::OrtOwnedTensor, GraphOptimizationL
use crate::Result;
use super::ONNX_ENVIRONMENT;
-const TRUNCATE_INPUT: usize = 128;
+const TRUNCATE_INPUT: usize = 256;
pub struct CrossEncoderModel {
tokenizer: tokenizers::Tokenizer,
@@ -38,7 +38,7 @@ impl CrossEncoderModel {
.new_session_builder()?
.with_optimization_level(GraphOptimizationLevel::All)?
.with_number_threads(1)?
- .with_model_from_file(folder.as_ref().join("model.onnx"))?,
+ .with_model_from_file(folder.as_ref().join("model_quantized.onnx"))?,
);
Ok(Self { tokenizer, session })
@@ -141,9 +141,9 @@ mod tests {
)
);
}
- Err(_err) => {
- // dbg!(_err);
- // panic!();
+ Err(err) => {
+ dbg!(err);
+ panic!();
}
}
}
diff --git a/justfile b/justfile
index 742129dc..005fed6b 100644
--- a/justfile
+++ b/justfile
@@ -18,6 +18,7 @@
cd frontend; npm run dev
@configure *ARGS:
+ pip3 install -r scripts/requirements.txt
./scripts/export_crossencoder
cd frontend; npm install; npm run build
cargo run --release --all-features -- configure {{ARGS}}
diff --git a/scripts/export_crossencoder b/scripts/export_crossencoder
index 2555584e..8b02b183 100755
--- a/scripts/export_crossencoder
+++ b/scripts/export_crossencoder
@@ -1,13 +1,23 @@
#!/usr/bin/env python3
import os
+from optimum.onnxruntime import ORTModelForSequenceClassification, ORTQuantizer
+from optimum.onnxruntime.configuration import AutoQuantizationConfig, QuantizationMode
import transformers
+from functools import partial
MODEL = "cross-encoder/ms-marco-TinyBERT-L-2-v2"
-os.system("pip3 install transformers[onnx] torch")
os.system("mkdir data/cross_encoder")
-os.system(
- f"python3 -m transformers.onnx --feature sequence-classification --model={MODEL} data/cross_encoder")
+
+model = ORTModelForSequenceClassification.from_pretrained(
+ MODEL, from_transformers=True)
tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)
tokenizer.save_pretrained("data/cross_encoder")
+
+qconfig = AutoQuantizationConfig.avx512(is_static=False)
+quantizer = ORTQuantizer.from_pretrained(model)
+
+quantizer.quantize(
+ quantization_config=qconfig, save_dir="data/cross_encoder")
+# model.save_pretrained("data/cross_encoder")
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
new file mode 100644
index 00000000..1991a5aa
--- /dev/null
+++ b/scripts/requirements.txt
@@ -0,0 +1,4 @@
+transformers[onnx]
+torch
+optimum
+datasets
\ No newline at end of file