web search done right
Quantize crossencoder
Mikkel Denker72fa54a945fd
diff --git a/core/src/ranking/models/cross_encoder.rs b/core/src/ranking/models/cross_encoder.rsindex ba6a6dad..8ba3d688 100644--- a/core/src/ranking/models/cross_encoder.rs+++ b/core/src/ranking/models/cross_encoder.rs@@ -22,7 +22,7 @@ use onnxruntime::{ndarray::ArrayBase, tensor::OrtOwnedTensor, GraphOptimizationLuse crate::Result;use super::ONNX_ENVIRONMENT;-const TRUNCATE_INPUT: usize = 128;+const TRUNCATE_INPUT: usize = 256;pub struct CrossEncoderModel {tokenizer: tokenizers::Tokenizer,@@ -38,7 +38,7 @@ impl CrossEncoderModel {.new_session_builder()?.with_optimization_level(GraphOptimizationLevel::All)?.with_number_threads(1)?- .with_model_from_file(folder.as_ref().join("model.onnx"))?,+ .with_model_from_file(folder.as_ref().join("model_quantized.onnx"))?,);Ok(Self { tokenizer, session })@@ -141,9 +141,9 @@ mod tests {));}- Err(_err) => {- // dbg!(_err);- // panic!();+ Err(err) => {+ dbg!(err);+ panic!();}}}diff --git a/justfile b/justfileindex 742129dc..005fed6b 100644--- a/justfile+++ b/justfile@@ -18,6 +18,7 @@cd frontend; npm run dev@configure *ARGS:+ pip3 install -r scripts/requirements.txt./scripts/export_crossencodercd frontend; npm install; npm run buildcargo run --release --all-features -- configure {{ARGS}}diff --git a/scripts/export_crossencoder b/scripts/export_crossencoderindex 2555584e..8b02b183 100755--- a/scripts/export_crossencoder+++ b/scripts/export_crossencoder@@ -1,13 +1,23 @@#!/usr/bin/env python3import os+from optimum.onnxruntime import ORTModelForSequenceClassification, ORTQuantizer+from optimum.onnxruntime.configuration import AutoQuantizationConfig, QuantizationModeimport transformers+from functools import partialMODEL = "cross-encoder/ms-marco-TinyBERT-L-2-v2"-os.system("pip3 install transformers[onnx] torch")os.system("mkdir data/cross_encoder")-os.system(- f"python3 -m transformers.onnx --feature sequence-classification --model={MODEL} data/cross_encoder")++model = ORTModelForSequenceClassification.from_pretrained(+ MODEL, from_transformers=True)tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)tokenizer.save_pretrained("data/cross_encoder")++qconfig = AutoQuantizationConfig.avx512(is_static=False)+quantizer = ORTQuantizer.from_pretrained(model)++quantizer.quantize(+ quantization_config=qconfig, save_dir="data/cross_encoder")+# model.save_pretrained("data/cross_encoder")diff --git a/scripts/requirements.txt b/scripts/requirements.txtnew file mode 100644index 00000000..1991a5aa--- /dev/null+++ b/scripts/requirements.txt@@ -0,0 +1,4 @@+transformers[onnx]+torch+optimum+datasets\ No newline at end of file