Skip to content

web search done right

Move from onnx to libtorch bindings for ML inference. Fuck onnx. It was an enormous hassle to get onnx to play ball with more advanced models and execute the onnx models on GPU since onnx is only compiled to older cuda versions. This commit removes our dependency to onnx and replaces it with direct bindings to libtorch which gives us more flexibility and still allows us to easily deploy simple models with tracing. Time will tell if this is sufficiently performant or if we may want to develop some kind of JIT that can fuse matrix operations to increase performance.

Mikkel Denkerfe713a873701
diff --git a/.cargo/config.toml b/.cargo/config.toml
index d5135e98..d47df8ab 100644
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@@ -1,2 +1,6 @@
[build]
-rustflags = ["-C", "target-cpu=native"]
\ No newline at end of file
+rustflags = ["-C", "target-cpu=native"]
+
+[env]
+LIBTORCH = {value = ".venv/lib/python3.11/site-packages/torch", force=true, relative=true}
+DYLD_LIBRARY_PATH = {value = ".venv/lib/python3.11/site-packages/torch/lib", force=true, relative=true}
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 9f7c053e..0f2ab876 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,4 @@ notes
*.tar.gz
*.csv
ltr_scores.json
+.venv
\ No newline at end of file
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 1c6cc8d1..e642d593 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -12,4 +12,4 @@
"Cargo.toml",
"optics-lsp/Cargo.toml"
]
-}
+}
\ No newline at end of file
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 2f13d730..adcc7437 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -27,8 +27,8 @@ The process is simple and fast. Upon your first pull request, you will be prompt
* Install rust by following the steps outlined [here](https://www.rust-lang.org/tools/install)
* Install clang and npm
* Update ulimit. RocksDB tends to exceed the max number of allowed open files, so you will have to run `ulimit -n 10240` to increase the allowed max number of open files.
-* Build the frontend `cd frontend && npm install && npm run build`
+* Install just by running `cargo install just`. This allows you to run the scripts in the `justfile` file. A justfile is basically a simple Makefile, so this is just for convenience.
+* Run the command `just configure` which should automatically configures the rest of your dev environment. The script creates a python virtual environment, installs relevant dependencies, traces and exports the ML models and creates a small local index which you can use for development.
* (Optional) Install cargo-watch by running `cargo install cargo-watch`. This makes frontend development easier.
-* (Optional) Install just by running `cargo install just`. This allows you to run the scripts in the Justfile. A Justfile is basically a simple Makefile, so this is just for convenience.
-After the non optional steps you can now run `cargo test` and should see all tests passing. If you have followed the optional steps, you should be able to run `just configure` which will setup a small local index. You can run `just frontend` to start the search server and launch the frontend at `0.0.0.0:3000`.
+After the non optional steps you can now run `cargo test` and should see all tests passing. If you have followed the optional steps, you should be able to run `just frontend` to start the search server and launch the frontend at `0.0.0.0:3000`.
diff --git a/Cargo.lock b/Cargo.lock
index 4fbe9a6b..3abc3237 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -526,7 +526,7 @@ dependencies = [
"tar",
"tempfile",
"thiserror",
- "zip 0.6.4",
+ "zip",
]
[[package]]
@@ -2571,29 +2571,6 @@ dependencies = [
"pkg-config",
]
-[[package]]
-name = "onnxruntime"
-version = "0.0.14"
-source = "git+https://github.com/StractOrg/onnxruntime-rs#b8d281e0e6f1e180700e8df437f6a07a36fec931"
-dependencies = [
- "lazy_static",
- "ndarray",
- "onnxruntime-sys",
- "thiserror",
- "tracing",
-]
-
-[[package]]
-name = "onnxruntime-sys"
-version = "0.0.14"
-source = "git+https://github.com/StractOrg/onnxruntime-rs#b8d281e0e6f1e180700e8df437f6a07a36fec931"
-dependencies = [
- "flate2",
- "tar",
- "ureq",
- "zip 0.5.13",
-]
-
[[package]]
name = "oorandom"
version = "11.1.3"
@@ -3783,6 +3760,7 @@ dependencies = [
"kuchiki",
"lalrpop",
"lalrpop-util",
+ "libc",
"logos",
"maplit",
"md5",
@@ -3791,7 +3769,6 @@ dependencies = [
"min-max-heap",
"num_cpus",
"once_cell",
- "onnxruntime",
"optics",
"parse_wiki_text",
"quick-xml",
@@ -3807,11 +3784,13 @@ dependencies = [
"serde_json",
"serde_urlencoded",
"tantivy",
+ "tch",
"thiserror",
"tokenizers",
"tokio",
"tokio-stream",
"toml",
+ "torch-sys",
"tower-http",
"tracing",
"tracing-subscriber",
@@ -4012,6 +3991,22 @@ dependencies = [
"xattr",
]
+[[package]]
+name = "tch"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3394fea57e43ef3708196025beb3445dc13998b162f972233f11b9fdf091401"
+dependencies = [
+ "half 2.2.1",
+ "lazy_static",
+ "libc",
+ "ndarray",
+ "rand 0.8.5",
+ "thiserror",
+ "torch-sys",
+ "zip",
+]
+
[[package]]
name = "tempfile"
version = "3.5.0"
@@ -4318,6 +4313,19 @@ dependencies = [
"winnow",
]
+[[package]]
+name = "torch-sys"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ff928d3e632acda675e388d8e0efe526136350620800cc770e3727c9abc6ea1"
+dependencies = [
+ "anyhow",
+ "cc",
+ "libc",
+ "ureq",
+ "zip",
+]
+
[[package]]
name = "tower"
version = "0.4.13"
@@ -4996,20 +5004,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "zip"
-version = "0.5.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815"
-dependencies = [
- "byteorder",
- "bzip2",
- "crc32fast",
- "flate2",
- "thiserror",
- "time 0.1.45",
-]
-
[[package]]
name = "zip"
version = "0.6.4"
diff --git a/Cargo.toml b/Cargo.toml
index 5edd700c..fab85809 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -67,7 +67,6 @@ maplit = "1.0.2"
rand = "0.8.5"
crossbeam-channel = "0.5.6"
num_cpus = "1.15.0"
-onnxruntime = { git = "https://github.com/StractOrg/onnxruntime-rs" }
tokenizers = "0.13.2"
byteorder = "1.4.3"
async-stream = "0.3.3"
@@ -77,3 +76,6 @@ chitchat = "0.5.0"
memmap2 = "0.5.10"
rkyv = "0.7.41"
dashmap = "5.4.0"
+tch = "0.11.0"
+torch-sys = "0.11.0"
+libc = "0.2.142"
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 786412de..84cf65d2 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -11,17 +11,13 @@ RUN apt-get update -y \
libssl-dev \
nodejs
-RUN wget https://github.com/microsoft/onnxruntime/releases/download/v1.13.1/onnxruntime-linux-x64-1.13.1.tgz -P /tmp \
- && tar -xf /tmp/onnxruntime-linux-x64-1.13.1.tgz -C /tmp \
- && mv /tmp/onnxruntime-linux-x64-1.13.1/lib/* /usr/lib
-
RUN rustup component add rustfmt
COPY . .
RUN cd frontend && npm install && npm run build
-RUN ORT_STRATEGY=system ORT_LIB_LOCATION=/usr/lib/libonnxruntime.so.1.13.1 cargo build --release \
+RUN cargo build --release \
&& mkdir /stract/bin \
&& find target/release -maxdepth 1 -perm /a+x -type f -exec mv {} /stract/bin \;
@@ -40,7 +36,5 @@ WORKDIR /stract
COPY --from=builder /stract/bin/stract /usr/local/bin/stract
COPY --from=builder /stract/frontend/dist frontend/dist
-COPY --from=builder /usr/lib/libonnxruntime.so.1.13.1 /usr/lib
-COPY --from=builder /usr/lib/libonnxruntime.so /usr/lib
ENTRYPOINT ["/usr/local/bin/stract"]
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 5d85127f..f57564f6 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -67,7 +67,6 @@ once_cell = { workspace = true }
optics = { path = "../optics" }
crossbeam-channel = { workspace = true }
num_cpus = { workspace = true }
-onnxruntime = { workspace = true }
tokenizers = { workspace = true }
lalrpop-util = { workspace = true }
byteorder = { workspace = true }
@@ -78,6 +77,9 @@ chitchat = { workspace = true }
memmap2 = { workspace = true }
rkyv = { workspace = true }
dashmap = { workspace = true }
+tch = { workspace = true }
+torch-sys = { workspace = true }
+libc = { workspace = true }
[build-dependencies]
lalrpop = { workspace = true }
diff --git a/core/src/api/summarize.rs b/core/src/api/summarize.rs
index a4377e44..ddb8a80d 100644
--- a/core/src/api/summarize.rs
+++ b/core/src/api/summarize.rs
@@ -14,20 +14,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::convert::Infallible;
use std::sync::Arc;
-use axum::response::sse::KeepAlive;
-use axum::response::{sse::Event, Sse};
use axum::{extract, Extension};
-use futures::stream::Stream;
use http::StatusCode;
use serde::Deserialize;
-use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
-use tokio_stream::StreamExt as _;
use super::State;
-use crate::Result;
#[derive(Deserialize)]
pub struct Params {
@@ -35,44 +28,18 @@ pub struct Params {
pub query: String,
}
-fn summarize_blocking(iter: impl Iterator<Item = String>, tx: UnboundedSender<String>) {
- for tok in iter {
- tx.send(tok).unwrap();
- }
-}
-
-async fn summarize(
- params: Params,
- state: Arc<State>,
-) -> Result<Sse<impl Stream<Item = std::result::Result<Event, Infallible>>>> {
- let webpage = state.searcher.get_webpage(&params.url).await?;
- let (tx, mut rx) = unbounded_channel();
-
- let summarizer = Arc::clone(&state.summarizer);
- tokio::task::spawn_blocking(move || {
- summarize_blocking(summarizer.summarize_iter(&params.query, &webpage.body), tx)
- });
-
- let stream = async_stream::stream! {
- while let Some(item) = rx.recv().await {
- yield item;
- }
- };
-
- Ok(
- Sse::new(stream.map(|term| Event::default().data(term)).map(Ok))
- .keep_alive(KeepAlive::default()),
- )
-}
-
#[allow(clippy::unused_async)]
pub async fn route(
extract::Query(params): extract::Query<Params>,
Extension(state): Extension<Arc<State>>,
-) -> std::result::Result<Sse<impl Stream<Item = std::result::Result<Event, Infallible>>>, StatusCode>
-{
- match summarize(params, state).await {
- Ok(stream) => Ok(stream),
- Err(_) => Err(StatusCode::NO_CONTENT),
- }
+) -> std::result::Result<String, StatusCode> {
+ let webpage = state
+ .searcher
+ .get_webpage(&params.url)
+ .await
+ .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
+
+ Ok(state
+ .summarizer
+ .abstractive_summary(&params.query, &webpage.body))
}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index d67e1a78..aebe0b56 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -271,15 +271,6 @@ pub enum Error {
#[error("Internal error")]
InternalError(String),
- #[error("Huggingface tokenizer")]
- HuggingfaceTokenizer(#[from] tokenizers::Error),
-
- #[error("Onnx")]
- Onnx(#[from] onnxruntime::OrtError),
-
- #[error("Word2Vec")]
- Word2Vec(#[from] spell::word2vec::Error),
-
#[error("Distributed searcher")]
DistributedSearcher(#[from] distributed::Error),
@@ -288,6 +279,9 @@ pub enum Error {
#[error("Lambdamart")]
LambdaMART(#[from] ranking::models::lambdamart::Error),
+
+ #[error("Summarizer")]
+ Summarizer(#[from] crate::summarizer::Error),
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
@@ -358,19 +352,3 @@ fn floor_char_boundary(str: &str, index: usize) -> usize {
res
}
-
-fn softmax(vec: &mut [f32]) {
- let s: f32 = vec.iter().map(|z| z.exp()).sum();
-
- for z in vec {
- *z = z.exp() / s
- }
-}
-
-pub static ONNX_ENVIRONMENT: once_cell::sync::Lazy<onnxruntime::environment::Environment> =
- once_cell::sync::Lazy::new(|| {
- onnxruntime::environment::Environment::builder()
- .with_log_level(onnxruntime::LoggingLevel::Info)
- .build()
- .unwrap()
- });
diff --git a/core/src/qa_model.rs b/core/src/qa_model.rs
index e5b79938..c95daf2c 100644
--- a/core/src/qa_model.rs
+++ b/core/src/qa_model.rs
@@ -14,23 +14,28 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{cmp::Ordering, ops::Range, path::Path, sync::Mutex};
+use std::{cmp::Ordering, ops::Range, path::Path};
-use crate::{softmax, Result, ONNX_ENVIRONMENT};
use itertools::Itertools;
-use onnxruntime::{
- ndarray::{Array2, Axis},
- tensor::OrtOwnedTensor,
- GraphOptimizationLevel, TypedArray,
-};
+use tch::{IValue, Tensor};
use tokenizers::{PaddingParams, TruncationParams};
const TRUNCATE_INPUT: usize = 128;
const DEFAULT_SCORE_THRESHOLD: f64 = 0.7;
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+ #[error("Tokenizer error")]
+ Tokenizer(#[from] tokenizers::Error),
+ #[error("Torch error")]
+ Torch(#[from] tch::TchError),
+}
+
+type Result<T> = std::result::Result<T, Error>;
+
pub struct QaModel {
tokenizer: tokenizers::Tokenizer,
- session: Mutex<onnxruntime::session::Session<'static>>,
+ model: tch::CModule,
score_threshold: f64,
}
@@ -56,17 +61,10 @@ impl QaModel {
tokenizer.with_truncation(Some(truncation));
tokenizer.with_padding(Some(padding));
- let session = Mutex::new(
- ONNX_ENVIRONMENT
- .new_session_builder()?
- .with_optimization_level(GraphOptimizationLevel::All)?
- .with_number_threads(1)?
- .with_model_from_file(folder.as_ref().join("model_quantized.onnx"))?,
- );
-
+ let model = tch::CModule::load(folder.as_ref().join("model.pt"))?;
Ok(Self {
tokenizer,
- session,
+ model,
score_threshold: DEFAULT_SCORE_THRESHOLD,
})
}
@@ -76,71 +74,76 @@ impl QaModel {
return None;
}
- let mut batches = Vec::with_capacity(contexts.len());
+ let bs = contexts.len();
+ let mut batches = Vec::with_capacity(bs);
for context in contexts {
batches.push((question.to_string(), context.to_string()));
}
let encoded = self.tokenizer.encode_batch(batches, true).unwrap();
- let encoded_ids: Vec<_> = encoded
+ let num_tokens = encoded
.iter()
- .map(|encoding| encoding.get_ids().to_vec())
- .collect();
+ .map(|enc| enc.get_ids().len())
+ .max()
+ .unwrap_or(0);
- let mut ids = Array2::zeros((encoded_ids.len(), encoded_ids.first().unwrap().len()));
- for i in 0..encoded_ids.len() {
- for j in 0..encoded_ids[i].len() {
- ids[[i, j]] = encoded_ids[i][j] as i64;
- }
- }
+ let ids = encoded
+ .iter()
+ .flat_map(|enc| enc.get_ids().iter().map(|i| *i as i64).take(TRUNCATE_INPUT))
+ .collect_vec();
- let encoded_masks: Vec<_> = encoded
+ let attention_mask = encoded
.iter()
- .map(|encoding| encoding.get_attention_mask().to_vec())
- .collect();
+ .flat_map(|enc| {
+ enc.get_attention_mask()
+ .iter()
+ .map(|i| *i as i64)
+ .take(TRUNCATE_INPUT)
+ })
+ .collect_vec();
+
+ let ids = Tensor::of_slice(&ids).reshape(&[bs as i64, num_tokens as i64]);
+ let attention_mask =
+ Tensor::of_slice(&attention_mask).reshape(&[bs as i64, num_tokens as i64]);
+
+ let output = self
+ .model
+ .forward_is(&[IValue::Tensor(ids), IValue::Tensor(attention_mask)])
+ .unwrap();
- let mut masks = Array2::zeros((encoded_masks.len(), encoded_masks.first().unwrap().len()));
- for i in 0..encoded_masks.len() {
- for j in 0..encoded_masks[i].len() {
- masks[[i, j]] = encoded_masks[i][j] as i64;
+ let (start_logits, end_logits) = if let IValue::Tuple(mut tup) = output {
+ if tup.len() != 2 {
+ return None;
}
- }
- let mut sess = self.session.lock().unwrap();
+ let end = tup.pop().unwrap();
+ let start = tup.pop().unwrap();
- let mut res: Vec<OrtOwnedTensor<f32, _>> = sess
- .run(vec![TypedArray::I64(ids), TypedArray::I64(masks)])
- .unwrap();
-
- if res.len() != 2 {
+ if let (IValue::Tensor(start), IValue::Tensor(end)) = (start, end) {
+ (start, end)
+ } else {
+ return None;
+ }
+ } else {
return None;
- }
-
- let start_logits = res.remove(0);
- let end_logits = res.remove(0);
+ };
- if start_logits.shape()[0] != start_logits.shape()[0] {
- return None;
- }
+ debug_assert_eq!(start_logits.size(), end_logits.size());
let mut best_answer: Option<Answer> = None;
#[allow(clippy::needless_range_loop)]
- for idx in 0..start_logits.shape()[0] {
- let mut start: Vec<f32> = start_logits
- .index_axis(Axis(0), idx)
- .into_iter()
- .copied()
+ for idx in 0..bs {
+ let start: Vec<f64> = start_logits
+ .get(idx as i64)
+ .iter::<f64>()
+ .unwrap()
.collect();
- softmax(&mut start);
+ // softmax(&mut start);
- let mut end: Vec<f32> = end_logits
- .index_axis(Axis(0), idx)
- .into_iter()
- .copied()
- .collect();
- softmax(&mut end);
+ let end: Vec<f64> = end_logits.get(idx as i64).iter::<f64>().unwrap().collect();
+ // softmax(&mut end);
let best_start = start
.iter()
@@ -155,7 +158,7 @@ impl QaModel {
continue;
}
- let score = (start[best_start] * end[best_end]) as f64;
+ let score = start[best_start] * end[best_end];
if score < self.score_threshold {
continue;
diff --git a/core/src/ranking/models/cross_encoder.rs b/core/src/ranking/models/cross_encoder.rs
index 0265f45e..6dc5f9f1 100644
--- a/core/src/ranking/models/cross_encoder.rs
+++ b/core/src/ranking/models/cross_encoder.rs
@@ -14,21 +14,28 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{path::Path, sync::Mutex};
+use std::path::Path;
use itertools::Itertools;
-use onnxruntime::{ndarray::ArrayBase, tensor::OrtOwnedTensor, GraphOptimizationLevel, TypedArray};
+use tch::Tensor;
use tokenizers::PaddingParams;
use tokenizers::TruncationParams;
-use crate::Result;
-
-use crate::ONNX_ENVIRONMENT;
const TRUNCATE_INPUT: usize = 128;
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+ #[error("Tokenizer error")]
+ Tokenizer(#[from] tokenizers::Error),
+ #[error("Torch error")]
+ Torch(#[from] tch::TchError),
+}
+
+type Result<T> = std::result::Result<T, Error>;
+
pub struct CrossEncoderModel {
tokenizer: tokenizers::Tokenizer,
- session: Mutex<onnxruntime::session::Session<'static>>,
+ model: tch::CModule,
}
impl CrossEncoderModel {
@@ -48,15 +55,9 @@ impl CrossEncoderModel {
tokenizer.with_truncation(Some(truncation));
tokenizer.with_padding(Some(padding));
- let session = Mutex::new(
- ONNX_ENVIRONMENT
- .new_session_builder()?
- .with_optimization_level(GraphOptimizationLevel::All)?
- .with_number_threads(5)?
- .with_model_from_file(folder.as_ref().join("model_quantized.onnx"))?,
- );
+ let model = tch::CModule::load(folder.as_ref().join("model.pt"))?;
- Ok(Self { tokenizer, session })
+ Ok(Self { tokenizer, model })
}
}
@@ -125,32 +126,20 @@ impl CrossEncoder for CrossEncoderModel {
})
.collect_vec();
- let mut sess = self.session.lock().unwrap();
-
- let output: Vec<OrtOwnedTensor<f32, _>> = sess
- .run(vec![
- TypedArray::I64(
- ArrayBase::from_vec(ids)
- .into_shape((bs, num_tokens))
- .unwrap(),
- ),
- TypedArray::I64(
- ArrayBase::from_vec(attention_mask)
- .into_shape((bs, num_tokens))
- .unwrap(),
- ),
- TypedArray::I64(
- ArrayBase::from_vec(type_ids)
- .into_shape((bs, num_tokens))
- .unwrap(),
- ),
- ])
+ let ids = Tensor::of_slice(&ids).reshape(&[bs as i64, num_tokens as i64]);
+ let attention_mask =
+ Tensor::of_slice(&attention_mask).reshape(&[bs as i64, num_tokens as i64]);
+ let type_ids = Tensor::of_slice(&type_ids).reshape(&[bs as i64, num_tokens as i64]);
+
+ let output = self
+ .model
+ .forward_ts(&[ids, attention_mask, type_ids])
.unwrap();
let mut res = Vec::with_capacity(bs);
for i in 0..bs {
- let s = output[0][[i, 0]] as f64;
+ let s = output.double_value(&[i as i64, 0]);
let s = s.exp();
let s = s / (s + 1.0);
res.push(s);
diff --git a/core/src/summarizer.rs b/core/src/summarizer.rs
index f7a8ed89..a3db1499 100644
--- a/core/src/summarizer.rs
+++ b/core/src/summarizer.rs
@@ -19,19 +19,34 @@ use std::{
collections::{BinaryHeap, VecDeque},
ops::Range,
path::Path,
- sync::{Arc, Mutex},
+ sync::Arc,
};
use itertools::{intersperse, Itertools};
-use onnxruntime::{
- ndarray::{ArrayBase, Axis, Dim, IxDynImpl, OwnedRepr},
- tensor::OrtOwnedTensor,
- GraphOptimizationLevel, TypedArray,
-};
+use tch::{IValue, Kind, Tensor};
use tokenizers::{PaddingParams, TruncationParams};
-use crate::{ceil_char_boundary, softmax, spell::word2vec::Word2Vec};
-use crate::{Result, ONNX_ENVIRONMENT};
+use crate::{ceil_char_boundary, spell::word2vec::Word2Vec};
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+ #[error("Torch")]
+ Tch(#[from] tch::TchError),
+
+ #[error("Tokenizer")]
+ Tokenizer(#[from] tokenizers::Error),
+
+ #[error("IO")]
+ Io(#[from] std::io::Error),
+
+ #[error("Word2vec")]
+ Word2Vec(#[from] crate::spell::word2vec::Error),
+
+ #[error("Unexpected output type")]
+ UnexpectedOutputType,
+}
+
+type Result<T> = std::result::Result<T, Error>;
#[derive(Clone)]
struct CandidatePassage<'a> {
@@ -268,19 +283,13 @@ impl Summarizer {
let summary = self.extractive_summary(query, text);
self.abstractive_summary(query, &summary)
}
-
- pub fn summarize_iter(&self, query: &str, text: &str) -> impl Iterator<Item = String> {
- let summary = self.extractive_summary(query, text);
- self.abstractive_summarizer
- .summarize_iter(&summary, GenerationConfig::default())
- }
}
const TRUNCATE_INPUT: usize = 1024;
pub struct AbstractiveModel {
- encoder: Mutex<onnxruntime::session::Session<'static>>,
- decoder: Mutex<onnxruntime::session::Session<'static>>,
- decoder_with_past: Mutex<onnxruntime::session::Session<'static>>,
+ encoder: tch::CModule,
+ decoder: tch::CModule,
+ decoder_with_past: tch::CModule,
tokenizer: tokenizers::Tokenizer,
bos_token_id: usize,
@@ -305,29 +314,9 @@ impl AbstractiveModel {
tokenizer.with_truncation(Some(truncation));
tokenizer.with_padding(Some(padding));
- let encoder = Mutex::new(
- ONNX_ENVIRONMENT
- .new_session_builder()?
- .with_optimization_level(GraphOptimizationLevel::All)?
- .with_number_threads(5)?
- .with_model_from_file(folder.as_ref().join("encoder_model.onnx"))?,
- );
-
- let decoder = Mutex::new(
- ONNX_ENVIRONMENT
- .new_session_builder()?
- .with_optimization_level(GraphOptimizationLevel::All)?
- .with_number_threads(5)?
- .with_model_from_file(folder.as_ref().join("decoder_model.onnx"))?,
- );
-
- let decoder_with_past = Mutex::new(
- ONNX_ENVIRONMENT
- .new_session_builder()?
- .with_optimization_level(GraphOptimizationLevel::All)?
- .with_number_threads(5)?
- .with_model_from_file(folder.as_ref().join("decoder_with_past_model.onnx"))?,
- );
+ let encoder = tch::CModule::load(folder.as_ref().join("traced_encoder.pt"))?;
+ let decoder = tch::CModule::load(folder.as_ref().join("traced_decoder.pt"))?;
+ let decoder_with_past = tch::CModule::load(folder.as_ref().join("traced_decoder_wp.pt"))?;
Ok(Self {
tokenizer,
@@ -340,36 +329,6 @@ impl AbstractiveModel {
begin_decoder_token: 2,
})
}
-
- fn encode(
- &self,
- ids: Vec<i64>,
- attention_mask: Vec<i64>,
- ) -> Result<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>> {
- let mut encoder = self
- .encoder
- .lock()
- .expect("Failed to get lock. Maybe another thread crashed?");
-
- let num_tokens = ids.len();
-
- let mut encoder_output: Vec<OrtOwnedTensor<f32, _>> = encoder.run(vec![
- TypedArray::I64(
- ArrayBase::from_vec(ids)
- .into_shape((1, num_tokens))
- .unwrap(),
- ),
- TypedArray::I64(
- ArrayBase::from_vec(attention_mask)
- .into_shape((1, num_tokens))
- .unwrap(),
- ),
- ])?;
-
- let res = encoder_output.pop().unwrap().clone().into_owned();
-
- Ok(res)
- }
}
pub struct AbstractiveSummarizer {
@@ -382,721 +341,407 @@ impl AbstractiveSummarizer {
model: Arc::new(model),
}
}
- pub fn summarize(&self, text: &str, config: GenerationConfig) -> String {
- BeamGenerator {
- model: Arc::clone(&self.model),
- config,
- state: Some(BeamState::Encoding {
- text: text.to_string(),
- }),
+ pub fn summarize(&self, text: &str, mut config: GenerationConfig) -> String {
+ let ids = self
+ .model
+ .tokenizer
+ .encode(text, false)
+ .unwrap()
+ .get_ids()
+ .iter()
+ .map(|id| *id as i64)
+ .collect_vec();
+
+ let ids = Tensor::of_slice(&ids).reshape(&[1, -1]);
+
+ if !config
+ .banned_tokens
+ .contains(&(self.model.bos_token_id as i64))
+ {
+ config.banned_tokens.push(self.model.bos_token_id as i64);
}
- .skip(2) // the first 2 tokens are BeginDecoder and BOS
- .collect()
- }
- pub fn summarize_iter(
- &self,
- text: &str,
- config: GenerationConfig,
- ) -> impl Iterator<Item = String> {
- BeamGenerator {
- model: Arc::clone(&self.model),
+ if !config
+ .end_tokens
+ .contains(&(self.model.eos_token_id as i64))
+ {
+ config.end_tokens.push(self.model.eos_token_id as i64);
+ }
+
+ match self.model.beam_search(
+ &ids,
+ &Tensor::of_slice(&[
+ self.model.begin_decoder_token as i64,
+ self.model.bos_token_id as i64,
+ ])
+ .reshape(&[1, -1]),
config,
- state: Some(BeamState::Encoding {
- text: text.to_string(),
- }),
+ ) {
+ Ok(res) => {
+ let tokens = res.tokens.into_iter().map(|tok| tok as u32).collect_vec();
+ self.model
+ .tokenizer
+ .decode(tokens, true)
+ .unwrap_or_default()
+ }
+ Err(err) => {
+ tracing::error!("Error while summarizing: {:?}", err);
+ String::new()
+ }
}
- .skip(2) // the first 2 tokens are BeginDecoder and BOS
}
}
-struct BeamGenerator {
- model: Arc<AbstractiveModel>,
- config: GenerationConfig,
- state: Option<BeamState>,
-}
-
-impl Iterator for BeamGenerator {
- type Item = String;
+struct ClonableTensor(Tensor);
- fn next(&mut self) -> Option<Self::Item> {
- self.state
- .take()
- .and_then(|state| match state.step(&self.model, &self.config) {
- Ok((state, res)) => {
- self.state = Some(state);
- res
- }
- Err(err) => {
- tracing::error!("Encountered an error while generating summary: {err}");
- None
- }
- })
+impl Clone for ClonableTensor {
+ fn clone(&self) -> Self {
+ let out = Tensor::empty(&self.0.size(), (Kind::Float, self.0.device()));
+ ClonableTensor(self.0.clone(&out))
}
}
-enum BeamState {
- Encoding {
- text: String,
- },
- Decoding {
- encoder_hidden_states: ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>,
- attention_mask: ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>>,
- },
- DecodingWithPast {
- encoder_hidden_states: ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>,
- attention_mask: ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>>,
- caches: Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>,
- beams: Vec<Beam>,
- generated_tokens: u32,
- /// all beams agree on the tokens up to this index
- beam_token_agreement_idx: usize,
- },
- Remaining {
- terms: VecDeque<String>,
- },
-}
+#[derive(Clone)]
+struct BartMemory {
+ encoder_hidden_states: ClonableTensor,
+ past_key_values: Vec<Vec<ClonableTensor>>,
+}
+
+fn parse_decoder_output(output: IValue) -> Result<(Tensor, Vec<Vec<ClonableTensor>>)> {
+ let mut output = if let IValue::Tuple(tup) = output {
+ Ok(tup)
+ } else {
+ Err(Error::UnexpectedOutputType)
+ }?;
+
+ if output.len() != 2 {
+ return Err(Error::UnexpectedOutputType);
+ }
+
+ let logits = if let IValue::Tensor(t) = output.remove(0) {
+ Ok(t)
+ } else {
+ Err(Error::UnexpectedOutputType)
+ }?;
+
+ let caches = if let IValue::Tuple(caches) = output.remove(0) {
+ Ok(caches)
+ } else {
+ Err(Error::UnexpectedOutputType)
+ }?;
+
+ let past_key_values = if caches.len() == 12 {
+ let mut new_caches = Vec::with_capacity(caches.len());
+
+ for cache in caches {
+ if let IValue::Tuple(cache) = cache {
+ let mut c = Vec::with_capacity(2);
+ for cache in cache.into_iter().take(2) {
+ if let IValue::Tensor(cache) = cache {
+ c.push(ClonableTensor(cache));
+ } else {
+ return Err(Error::UnexpectedOutputType);
+ }
+ }
-impl BeamState {
- fn encode_step(model: &AbstractiveModel, text: String) -> Result<Self> {
- let encoded_text = model.tokenizer.encode(text.as_str(), true)?;
+ new_caches.push(c);
+ } else {
+ return Err(Error::UnexpectedOutputType);
+ }
+ }
- let ids = encoded_text
- .get_ids()
- .iter()
- .map(|i| *i as i64)
- .take(TRUNCATE_INPUT)
- .collect_vec();
+ Ok(new_caches)
+ } else {
+ Err(Error::UnexpectedOutputType)
+ }?;
- let attention_mask = encoded_text
- .get_attention_mask()
- .iter()
- .take(ids.len())
- .map(|i| *i as i64)
- .collect_vec();
+ Ok((logits, past_key_values))
+}
+
+impl BeamSearch for AbstractiveModel {
+ type Memory = BartMemory;
- let encoder_num_tokens = ids.len();
+ fn initial(
+ &self,
+ encoder_ids: &Tensor,
+ decoder_ids: &Tensor,
+ ) -> Result<DecoderOutput<Self::Memory>> {
+ let encoder_hidden_states = self.encoder.forward_ts(&[encoder_ids])?;
+
+ let decoder_output = self.decoder.forward_is(&[
+ IValue::Tensor(decoder_ids.shallow_clone()),
+ IValue::Tensor(encoder_hidden_states.shallow_clone()),
+ ])?;
- let encoder_hidden_states = model.encode(ids, attention_mask.clone())?;
+ let (logits, past_key_values) = parse_decoder_output(decoder_output)?;
- let attention_mask: ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>> =
- ArrayBase::from_vec(attention_mask)
- .into_shape((1, encoder_num_tokens))
- .unwrap()
- .into_dyn();
+ // logits is [batch_size, seq_len, vocab_size]
+ // get last token logits
+ let next_token_logits = logits.select(1, logits.size()[1] - 1);
- Ok(BeamState::Decoding {
- encoder_hidden_states,
- attention_mask,
+ Ok(DecoderOutput {
+ next_token_logits,
+ memory: BartMemory {
+ encoder_hidden_states: ClonableTensor(encoder_hidden_states),
+ past_key_values,
+ },
})
}
- fn run_decoder_model(
- model: &AbstractiveModel,
- ids: &[i64],
- encoder_hidden_states: ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>,
- attention_mask: ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>>,
- ) -> Result<Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>> {
- let mut decoder = model
- .decoder
- .lock()
- .expect("Failed to get lock. Maybe another thread crashed?");
-
- let num_tokens = ids.len();
- let arr_ids = ArrayBase::from_vec(ids.to_vec())
- .into_shape((1, num_tokens))
- .unwrap()
- .into_dyn();
+ fn step(
+ &self,
+ prev_output: &DecoderInput<Self::Memory>,
+ ) -> Result<DecoderOutput<Self::Memory>> {
+ let ids = Tensor::of_slice(&[prev_output.last_token]).reshape(&[1, 1]);
+
+ let past_key_value: IValue = IValue::Tuple(
+ prev_output
+ .memory
+ .past_key_values
+ .iter()
+ .map(|c| {
+ IValue::Tuple(
+ c.iter()
+ .map(|c| IValue::Tensor(c.0.shallow_clone()))
+ .collect_vec(),
+ )
+ })
+ .collect_vec(),
+ );
+
+ let decoder_output = self.decoder_with_past.forward_is(&[
+ IValue::Tensor(ids),
+ IValue::Tensor(prev_output.memory.encoder_hidden_states.clone().0),
+ past_key_value,
+ ])?;
- let decoder_input = vec![
- TypedArray::I64(attention_mask),
- TypedArray::I64(arr_ids),
- TypedArray::F32(encoder_hidden_states),
- ];
+ let (logits, past_key_values) = parse_decoder_output(decoder_output)?;
- let output = decoder.run(decoder_input)?;
+ // logits is [batch_size, seq_len, vocab_size]
+ // get last token logits
+ let next_token_logits = logits.select(1, logits.size()[1] - 1).squeeze();
- Ok(output.into_iter().map(|t| t.clone().into_owned()).collect())
+ Ok(DecoderOutput {
+ next_token_logits,
+ memory: BartMemory {
+ encoder_hidden_states: prev_output.memory.encoder_hidden_states.clone(),
+ past_key_values,
+ },
+ })
}
+}
- fn create_beams(
- decoder_output: &[ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>],
- ids: Vec<i64>,
- num_beams: u32,
- bos_token: usize,
- ) -> Vec<Beam> {
- let mut next_scores = BinaryHeap::with_capacity(num_beams as usize + 1);
-
- let all_logits = &decoder_output[0];
- let vocab_size = all_logits.shape()[2];
- let mut scores = Vec::with_capacity(vocab_size);
-
- for token_id in 0..vocab_size {
- let score: f32 = all_logits[[
- all_logits.shape()[0] - 1,
- all_logits.shape()[1] - 1,
- token_id,
- ]];
- scores.push(score);
- }
+struct DecoderOutput<M: Clone> {
+ next_token_logits: Tensor,
+ memory: M,
+}
- softmax(&mut scores);
+struct DecoderInput<M: Clone> {
+ last_token: i64,
+ memory: M,
+}
- for (token_id, score) in scores.into_iter().enumerate() {
- if token_id == bos_token {
- continue;
+#[derive(Clone)]
+struct Beam<M: Clone> {
+ log_score: f64,
+ length_penalty: f64,
+ tokens: Vec<i64>,
+ end_tokens: Vec<i64>,
+ memory: M,
+}
+
+impl<M: Clone> Beam<M> {
+ fn is_finished(&self, config: &GenerationConfig) -> bool {
+ if let Some(forced) = &config.force_min_tokens {
+ if self.tokens.len() < *forced as usize {
+ return false;
}
+ }
- let scored_token = Reverse(ScoredToken {
- token: token_id as i64,
- score: score.log2(),
- });
-
- if next_scores.len() < num_beams as usize {
- next_scores.push(scored_token);
- } else if let Some(mut worst) = next_scores.peek_mut() {
- if worst.0.score < scored_token.0.score {
- *worst = scored_token
+ if let Some(early_stopping) = &config.early_stopping {
+ match early_stopping {
+ EarlyStopping::MaxTokens { max_new_tokens } => {
+ if self.tokens.len() >= *max_new_tokens as usize {
+ return true;
+ }
}
}
}
- let mut beams = Vec::new();
- for scored_token in next_scores.into_iter().map(|t| t.0) {
- let mut ids = ids.clone();
- ids.push(scored_token.token);
-
- beams.push(Beam {
- input_ids: ids,
- score: scored_token.score,
- })
+ if let Some(tok) = self.tokens.last() {
+ if config.end_tokens.contains(tok) {
+ return true;
+ }
}
- beams
+ false
}
- fn create_caches(
- decoder_output: Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>,
- num_beams: u32,
- ) -> Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>> {
- let num_outputs = decoder_output.len();
-
- decoder_output
- .into_iter()
- .map(|t| t.into_owned())
- .map(|mut t| {
- let orig = t.clone();
-
- for _ in 0..num_beams - 1 {
- t.append(Axis(0), orig.view()).unwrap();
+ fn score(&self) -> f64 {
+ let length = match self.tokens.last() {
+ Some(tok) => {
+ if self.end_tokens.contains(tok) {
+ self.tokens.len() - 1
+ } else {
+ self.tokens.len()
}
-
- t
- })
- .skip(1)
- .take(num_outputs - 2) // skip 'encoder_hidden_states' since this is not changed by the decoder model
- .collect::<Vec<_>>()
- }
-
- fn run_decoder_model_with_past(
- model: &AbstractiveModel,
- beams: &[Beam],
- encoder_hidden_states: &ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>,
- attention_mask: &ArrayBase<OwnedRepr<i64>, Dim<IxDynImpl>>,
- mut caches: Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>,
- ) -> Result<Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>>> {
- let mut decoder_with_past = model
- .decoder_with_past
- .lock()
- .expect("Failed to get lock. Maybe another thread crashed?");
-
- let mut combined_attention_mask = Vec::new();
- let mut combined_encoder_hidden_states = Vec::with_capacity(beams.len());
- let mut combined_ids = Vec::new();
-
- // TODO: Only advance non-finished beams
-
- for beam in beams {
- combined_ids.push(*beam.input_ids.last().unwrap());
-
- for mask in attention_mask {
- combined_attention_mask.push(*mask);
}
+ None => 0,
+ };
- for state in encoder_hidden_states {
- combined_encoder_hidden_states.push(*state);
- }
- }
-
- let combined_ids = ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_ids)
- .into_shape((beams.len(), 1))
- .unwrap()
- .into_dyn();
-
- let mut attention_shape = attention_mask.shape().to_vec();
- attention_shape[0] = beams.len();
-
- let combined_attention_mask =
- ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_attention_mask)
- .into_shape(attention_shape.as_slice())
- .unwrap();
-
- let mut state_shape = encoder_hidden_states.shape().to_vec();
- state_shape[0] = beams.len();
-
- let combined_encoder_hidden_states =
- ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_encoder_hidden_states)
- .into_shape(state_shape.as_slice())
- .unwrap();
-
- let mut decoder_input = vec![
- TypedArray::I64(combined_attention_mask),
- TypedArray::I64(combined_ids),
- TypedArray::F32(combined_encoder_hidden_states),
- ];
-
- for tensor in caches.drain(0..) {
- decoder_input.push(TypedArray::F32(tensor));
- }
-
- let res = decoder_with_past.run(decoder_input)?;
+ self.log_score / ((length as f64).powf(self.length_penalty))
+ }
+}
- Ok(res.into_iter().map(|t| t.clone().into_owned()).collect())
+impl<M: Clone> PartialOrd for Beam<M> {
+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+ self.score().partial_cmp(&other.score())
}
+}
- fn new_possible_beams(
- decoder_output: &[ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>],
- beams: &[Beam],
- force_min_tokens: Option<u32>,
- generated_tokens: u32,
- eos_token_id: usize,
- length_penalty: f32,
- ) -> Vec<PossibleBeam> {
- let all_logits = &decoder_output[0];
- let vocab_size = all_logits.shape()[2];
-
- let mut beam_tok_scores = Vec::with_capacity(beams.len());
- for beam_idx in 0..beams.len() {
- let mut scores = Vec::with_capacity(vocab_size);
-
- for token_id in 0..vocab_size {
- scores.push(all_logits[[beam_idx, 0, token_id]]);
- }
+impl<M: Clone> PartialEq for Beam<M> {
+ fn eq(&self, other: &Self) -> bool {
+ self.score() == other.score()
+ }
+}
- softmax(&mut scores);
- beam_tok_scores.push(scores)
- }
+impl<M: Clone> Ord for Beam<M> {
+ fn cmp(&self, other: &Self) -> Ordering {
+ self.partial_cmp(other).unwrap_or(Ordering::Equal)
+ }
+}
- let mut new_beams = BinaryHeap::with_capacity(beams.len());
+impl<M: Clone> Eq for Beam<M> {}
- let mut skip_eos = false;
+trait BeamSearch {
+ type Memory: Clone;
- if let Some(min_tokens) = force_min_tokens {
- if generated_tokens < min_tokens {
- skip_eos = true;
- }
- }
+ fn initial(
+ &self,
+ encoder_ids: &Tensor,
+ decoder_ids: &Tensor,
+ ) -> Result<DecoderOutput<Self::Memory>>;
- for beam_idx in 0..beams.len() {
- for new_token in 0..vocab_size {
- if skip_eos && new_token == eos_token_id {
- continue;
- }
+ fn step(&self, prev_output: &DecoderInput<Self::Memory>)
+ -> Result<DecoderOutput<Self::Memory>>;
- let tok_score = beam_tok_scores[beam_idx][new_token];
+ fn beam_search(
+ &self,
+ encoder_ids: &Tensor,
+ decoder_start_tokens: &Tensor,
+ config: GenerationConfig,
+ ) -> Result<Beam<Self::Memory>> {
+ let encoded = self.initial(encoder_ids, decoder_start_tokens)?;
+ let decoder_start_tokens = decoder_start_tokens.squeeze().iter::<i64>()?.collect_vec();
- let tok_score = tok_score.log2();
+ let scores = encoded.next_token_logits.softmax(-1, Kind::Float).squeeze();
- let new_score = beams[beam_idx].score + tok_score;
+ let (next_scores, next_tokens) = scores.topk(config.num_beams as i64, -1, true, true);
- let possible_beam = PossibleBeam {
- beam_idx,
- beam_length: beams[beam_idx].len(),
- new_token: new_token as i64,
- length_penalty,
- eos_token: eos_token_id as i64,
- new_score,
- };
+ let mut beams = BinaryHeap::with_capacity(config.num_beams as usize);
- if new_beams.len() < beams.len() {
- new_beams.push(Reverse(possible_beam));
- } else if let Some(mut worst) = new_beams.peek_mut() {
- if possible_beam > worst.0 {
- *worst = Reverse(possible_beam);
- }
- }
+ for (score, token) in next_scores.iter::<f64>()?.zip(next_tokens.iter::<i64>()?) {
+ if config.banned_tokens.contains(&token) {
+ continue;
}
- }
-
- new_beams.into_iter().map(|r| r.0).collect()
- }
-
- fn new_caches(
- decoder_output: &[ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>],
- new_beams: &[PossibleBeam],
- ) -> Vec<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>> {
- let num_outputs = decoder_output.len();
- // skip(num_outputs-2) to skip 'encoder_hidden_states' since this is not changed by the decoder model
- let output_caches: Vec<_> = decoder_output
- .iter()
- .skip(1)
- .take(num_outputs - 2)
- .collect();
- let mut new_caches = Vec::with_capacity(output_caches.len());
+ let mut toks = decoder_start_tokens.clone();
+ toks.push(token);
- for _ in &output_caches {
- new_caches.push(Vec::with_capacity(100));
- }
+ let beam = Reverse(Beam {
+ log_score: score.log2(),
+ tokens: toks,
+ memory: encoded.memory.clone(),
+ end_tokens: config.end_tokens.clone(),
+ length_penalty: config.length_penalty,
+ });
- for possible_beam in new_beams {
- for (new_cache, old_cache) in new_caches.iter_mut().zip_eq(output_caches.iter()) {
- let shape = old_cache.shape();
-
- new_cache.push(
- old_cache
- .index_axis(Axis(0), possible_beam.beam_idx)
- .into_shape((1, shape[1], shape[2], shape[3]))
- .unwrap()
- .into_dyn(),
- );
+ if beams.len() < config.num_beams as usize {
+ beams.push(beam);
+ } else {
+ let mut worst = beams.peek_mut().unwrap();
+ if beam.0 > worst.0 {
+ *worst = beam;
+ }
}
}
- new_caches
- .into_iter()
- .map(|cache| onnxruntime::ndarray::concatenate(Axis(0), cache.as_slice()).unwrap())
- .collect()
- }
+ loop {
+ let mut next_beams = BinaryHeap::with_capacity(config.num_beams as usize);
- fn cement_beams(
- beams: Vec<Beam>,
- possible_beams: Vec<PossibleBeam>,
- eos_token_id: i64,
- ) -> Vec<Beam> {
- possible_beams
- .into_iter()
- .map(|possible_beam| {
- if beams[possible_beam.beam_idx].is_finished(eos_token_id) {
- beams[possible_beam.beam_idx].clone()
- } else {
- let mut input_ids = beams[possible_beam.beam_idx].input_ids.clone();
- input_ids.push(possible_beam.new_token);
- Beam {
- input_ids,
- score: possible_beam.new_score,
+ for beam in beams.iter() {
+ if beam.0.is_finished(&config) {
+ if next_beams.len() < config.num_beams as usize {
+ next_beams.push(beam.clone());
+ } else {
+ let mut worst = next_beams.peek_mut().unwrap();
+ if beam.0 > worst.0 {
+ *worst = beam.clone();
+ }
}
- }
- })
- .collect()
- }
-
- fn output_from_best(
- model: &AbstractiveModel,
- new_beams: Vec<Beam>,
- beam_agreement_index: usize,
- ) -> BeamState {
- let best_beam = new_beams
- .into_iter()
- .max_by(|a, b| a.score.total_cmp(&b.score))
- .unwrap();
-
- let terms: VecDeque<_> = best_beam
- .input_ids
- .into_iter()
- .skip(beam_agreement_index)
- .map(|i| i as u32)
- .map(|tok| model.tokenizer.decode(vec![tok], true).unwrap())
- .collect();
- BeamState::Remaining { terms }
- }
-
- fn check_early_stopping(early_stopping: Option<&EarlyStopping>, generated_tokens: u32) -> bool {
- early_stopping
- .map(|early_stopping| match early_stopping {
- EarlyStopping::MaxTokens { max_new_tokens } => generated_tokens >= *max_new_tokens,
- })
- .unwrap_or(false)
- }
+ continue;
+ }
- fn all_beams_agree(beams: &[Beam], beam_token_agreement_idx: usize) -> bool {
- if beams
- .iter()
- .any(|beam| beam_token_agreement_idx >= beam.len())
- {
- return false;
- }
+ let prev_output = DecoderInput {
+ last_token: *beam.0.tokens.last().unwrap(),
+ memory: beam.0.memory.clone(),
+ };
- beams
- .iter()
- .map(|beam| beam.input_ids[beam_token_agreement_idx])
- .all_equal()
- }
+ let decoded = self.step(&prev_output)?;
- fn step(
- mut self,
- model: &AbstractiveModel,
- config: &GenerationConfig,
- ) -> Result<(Self, Option<<BeamGenerator as Iterator>::Item>)> {
- loop {
- match self {
- BeamState::Encoding { text } => {
- self = Self::encode_step(model, text)?;
- }
- BeamState::Decoding {
- encoder_hidden_states,
- attention_mask,
- } => {
- let ids = vec![model.begin_decoder_token as i64, model.bos_token_id as i64];
- let decoder_output = Self::run_decoder_model(
- model,
- &ids,
- encoder_hidden_states.clone(),
- attention_mask.clone(),
- )?;
-
- let beams = Self::create_beams(
- &decoder_output,
- ids,
- config.num_beams,
- model.bos_token_id,
- );
-
- let caches = Self::create_caches(decoder_output, config.num_beams);
-
- self = BeamState::DecodingWithPast {
- encoder_hidden_states,
- attention_mask,
- caches,
- beams,
- generated_tokens: 1,
- beam_token_agreement_idx: 0,
- };
- }
+ let scores = decoded.next_token_logits.softmax(-1, Kind::Float);
- BeamState::DecodingWithPast {
- encoder_hidden_states,
- attention_mask,
- caches,
- mut beams,
- generated_tokens,
- beam_token_agreement_idx,
- } => {
- let shortest_beam = beams
- .iter()
- .min_by(|a, b| a.input_ids.len().cmp(&b.input_ids.len()))
- .unwrap();
-
- if (shortest_beam.input_ids.len() - beam_token_agreement_idx) as u32
- > config.max_beam_divergence_tokens
- {
- let best_beam = beams
- .iter()
- .max_by(|a, b| a.score.partial_cmp(&b.score).unwrap_or(Ordering::Equal))
- .unwrap()
- .clone();
-
- let num_beams = beams.len();
- beams = vec![best_beam; num_beams];
- }
+ let (next_scores, next_tokens) =
+ scores.topk(config.num_beams as i64, -1, true, true);
- if Self::all_beams_agree(&beams, beam_token_agreement_idx) {
- let tok = beams[0].input_ids[beam_token_agreement_idx];
- // for some reason, [17, 27] decodes to "'" or something like that.
- // If the tokens are decoded separately, they correspond to some characters
- // that shows up as errors in a string.
- // This hack skips the token 17 and simply decodes 27 as "'".
- // While this might not be correct decoding in all scenarios, it seems to work
- // most of the time.
- if tok == 17 {
- self = BeamState::DecodingWithPast {
- encoder_hidden_states,
- attention_mask,
- caches,
- beams,
- generated_tokens,
- beam_token_agreement_idx: beam_token_agreement_idx + 1,
- };
- continue;
- } else if tok == 27 {
- self = BeamState::DecodingWithPast {
- encoder_hidden_states,
- attention_mask,
- caches,
- beams,
- generated_tokens,
- beam_token_agreement_idx: beam_token_agreement_idx + 1,
- };
-
- return Ok((self, Some("'".to_string())));
- } else {
- self = BeamState::DecodingWithPast {
- encoder_hidden_states,
- attention_mask,
- caches,
- beams,
- generated_tokens,
- beam_token_agreement_idx: beam_token_agreement_idx + 1,
- };
-
- let decoded = model.tokenizer.decode(vec![tok as u32], true)?;
-
- return Ok((self, Some(decoded)));
- }
- }
+ for (score, token) in next_scores.iter::<f64>()?.zip(next_tokens.iter::<i64>()?) {
+ let mut new_beam = beam.clone().0;
+ new_beam.log_score += score.log2();
+ new_beam.tokens.push(token);
+ new_beam.memory = decoded.memory.clone();
- let decoder_output = Self::run_decoder_model_with_past(
- model,
- &beams,
- &encoder_hidden_states,
- &attention_mask,
- caches,
- )?;
-
- let possible_beams = Self::new_possible_beams(
- &decoder_output,
- &beams,
- config.force_min_tokens,
- generated_tokens,
- model.eos_token_id,
- config.length_penalty,
- );
-
- let new_caches = Self::new_caches(&decoder_output, &possible_beams);
-
- let new_beams =
- Self::cement_beams(beams, possible_beams, model.eos_token_id as i64);
-
- if Self::check_early_stopping(config.early_stopping.as_ref(), generated_tokens)
- || new_beams
- .iter()
- .all(|beam| beam.is_finished(model.eos_token_id as i64))
- {
- self = Self::output_from_best(model, new_beams, beam_token_agreement_idx);
+ if next_beams.len() < config.num_beams as usize {
+ next_beams.push(Reverse(new_beam));
} else {
- self = BeamState::DecodingWithPast {
- encoder_hidden_states,
- attention_mask,
- caches: new_caches,
- beams: new_beams,
- generated_tokens: generated_tokens + 1,
- beam_token_agreement_idx,
- };
+ let mut worst = next_beams.peek_mut().unwrap();
+ if new_beam > worst.0 {
+ *worst = Reverse(new_beam);
+ }
}
}
- BeamState::Remaining { mut terms } => {
- let term = terms.pop_front();
-
- return Ok((BeamState::Remaining { terms }, term));
- }
}
- }
- }
-}
-
-struct ScoredToken {
- token: i64,
- score: f32,
-}
-
-impl PartialOrd for ScoredToken {
- fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
- self.score.partial_cmp(&other.score)
- }
-}
-
-impl Ord for ScoredToken {
- fn cmp(&self, other: &Self) -> Ordering {
- self.partial_cmp(other).unwrap_or(Ordering::Equal)
- }
-}
-
-impl PartialEq for ScoredToken {
- fn eq(&self, other: &Self) -> bool {
- self.score == other.score
- }
-}
-
-impl Eq for ScoredToken {}
-
-#[derive(Clone, Debug)]
-struct Beam {
- input_ids: Vec<i64>,
- score: f32,
-}
-impl Beam {
- fn is_finished(&self, eos_tok: i64) -> bool {
- self.input_ids
- .last()
- .map(|tok| *tok == eos_tok)
- .unwrap_or(false)
- }
-
- fn len(&self) -> usize {
- self.input_ids.len()
- }
-}
-
-#[derive(Debug)]
-struct PossibleBeam {
- beam_idx: usize,
- beam_length: usize,
- new_token: i64,
- length_penalty: f32,
- new_score: f32,
-
- eos_token: i64,
-}
-
-impl PossibleBeam {
- fn normalized_score(&self) -> f32 {
- if self.length_penalty == 1.0 {
- self.new_score
- } else {
- let length = if self.new_token == self.eos_token {
- self.beam_length as f32
- } else {
- (self.beam_length + 1) as f32
- };
+ if next_beams.iter().all(|b| b.0.is_finished(&config)) {
+ break;
+ }
- self.new_score / length.powf(self.length_penalty)
+ beams = next_beams;
}
- }
-}
-impl PartialOrd for PossibleBeam {
- fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
- self.normalized_score()
- .partial_cmp(&other.normalized_score())
- }
-}
+ let mut beams = beams
+ .into_sorted_vec()
+ .into_iter()
+ .map(|b| b.0)
+ .collect_vec();
-impl Ord for PossibleBeam {
- fn cmp(&self, other: &Self) -> Ordering {
- self.partial_cmp(other).unwrap_or(Ordering::Equal)
- }
-}
+ beams.sort();
-impl PartialEq for PossibleBeam {
- fn eq(&self, other: &Self) -> bool {
- self.normalized_score() == other.normalized_score()
+ Ok(beams.pop().unwrap())
}
}
-impl Eq for PossibleBeam {}
-
pub struct GenerationConfig {
pub early_stopping: Option<EarlyStopping>,
pub force_min_tokens: Option<u32>,
+ pub end_tokens: Vec<i64>,
+ pub banned_tokens: Vec<i64>,
/// Exponential penalty to the length that is used with beam-based generation.
/// It is applied as an exponent to the sequence length, which in turn is used to divide the score of the sequence.
/// Since the score is the log likelihood of the sequence (i.e. negative),
/// length_penalty > 0.0 promotes longer sequences, while length_penalty < 0.0 encourages shorter sequences
- pub length_penalty: f32,
+ pub length_penalty: f64,
pub num_beams: u32,
- pub max_beam_divergence_tokens: u32,
}
impl Default for GenerationConfig {
@@ -1105,10 +750,11 @@ impl Default for GenerationConfig {
early_stopping: Some(EarlyStopping::MaxTokens {
max_new_tokens: 128,
}),
+ end_tokens: vec![],
+ banned_tokens: vec![],
force_min_tokens: None,
length_penalty: 1.0,
num_beams: 10,
- max_beam_divergence_tokens: 20,
}
}
}
@@ -1178,8 +824,8 @@ mod tests {
let config = GenerationConfig {
num_beams: 2,
- length_penalty: 1.0,
early_stopping: Some(EarlyStopping::MaxTokens { max_new_tokens: 16 }),
+ length_penalty: 1.0,
..Default::default()
};
diff --git a/justfile b/justfile
index a89006af..caa81dd0 100644
--- a/justfile
+++ b/justfile
@@ -18,7 +18,7 @@
cd frontend; npm run dev
@configure *ARGS:
- pip install -r scripts/requirements.txt
+ just setup_python_env
./scripts/export_crossencoder
./scripts/export_qa_model
./scripts/export_abstractive_summary_model
@@ -32,3 +32,7 @@
@entity:
rm -rf data/entity
cargo run --release -- indexer entity data/enwiki_subset.xml.bz2 data/entity
+
+@setup_python_env:
+ python3 -m venv .venv
+ .venv/bin/pip install -r scripts/requirements.txt
\ No newline at end of file
diff --git a/scripts/blogroll_optic.py b/scripts/blogroll_optic.py
index 572a1440..9c1036f4 100644
--- a/scripts/blogroll_optic.py
+++ b/scripts/blogroll_optic.py
@@ -1,3 +1,5 @@
+#!.venv/bin/python3
+
# blogs from https://blogroll.org/
blogroll_org = [
'50watts.com',
diff --git a/scripts/export_abstractive_summary_model b/scripts/export_abstractive_summary_model
index 19840963..339c87bc 100755
--- a/scripts/export_abstractive_summary_model
+++ b/scripts/export_abstractive_summary_model
@@ -1,34 +1,68 @@
-#!/usr/bin/env python
+#!.venv/bin/python3
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
-from optimum.onnxruntime import ORTQuantizer, ORTModelForSeq2SeqLM
-from optimum.onnxruntime.configuration import AutoQuantizationConfig
import urllib.request
+import torch
+import os
+
+os.makedirs("data/summarizer/abstractive", exist_ok=True)
MODEL = "lidiya/bart-large-xsum-samsum"
-model = ORTModelForSeq2SeqLM.from_pretrained(MODEL, from_transformers=True)
+model = AutoModelForSeq2SeqLM.from_pretrained(MODEL, torchscript=True)
+# model.half()
+model.eval()
tokenizer = AutoTokenizer.from_pretrained(MODEL)
+tokenizer.save_pretrained("data/summarizer/abstractive")
+
+
+for p in model.parameters():
+ p.requires_grad_(False)
+
+
+def run_decoder_wp(input_ids, encoder_hidden_states, past_key_values):
+ (logits, past) = model.model.decoder(input_ids=input_ids,
+ encoder_hidden_states=encoder_hidden_states,
+ past_key_values=past_key_values)
+
+ return (model.lm_head(logits), past)
+
+
+def run_decoder(input_ids, encoder_hidden_states):
+ (logits, past) = model.model.decoder(input_ids=input_ids,
+ encoder_hidden_states=encoder_hidden_states)
+ out = model.lm_head(logits)
+ return (out, past)
+
+
+def run_encoder(input_ids):
+ res = model.model.encoder(input_ids=input_ids)[0]
+
+ return res
-model.save_pretrained("data/summarizer/abstractive")
-model_dir = model.model_save_dir
+embed_size_per_head = model.config.d_model // model.config.decoder_attention_heads
+input_ids = torch.tensor([[19]*1], dtype=torch.long)
+keys = torch.ones(model.config.decoder_layers, 1,
+ model.config.decoder_attention_heads, 1, embed_size_per_head)
+past_key_values = tuple((key, key) for key in keys)
+encoder_outputs = torch.rand(1, 1024, 1024, dtype=torch.float32)
+encoder_hidden_states = torch.rand(1, 1024, 1024, dtype=torch.float32)
-encoder_quantizer = ORTQuantizer.from_pretrained(
- model_dir, file_name="encoder_model.onnx")
-decoder_quantizer = ORTQuantizer.from_pretrained(
- model_dir, file_name="decoder_model.onnx")
-decoder_wp_quantizer = ORTQuantizer.from_pretrained(
- model_dir, file_name="decoder_with_past_model.onnx")
+# trace models
+traced_decoder_wp = torch.jit.trace(
+ run_decoder_wp, (input_ids, encoder_hidden_states, past_key_values))
-quantizer = [encoder_quantizer, decoder_quantizer, decoder_wp_quantizer]
+input_ids = torch.tensor([[19]*1024], dtype=torch.long)
-dqconfig = AutoQuantizationConfig.avx512_vnni(
- is_static=False, per_channel=False)
+traced_decoder = torch.jit.trace(
+ run_decoder, (input_ids, encoder_hidden_states))
-for q in quantizer:
- q.quantize(save_dir="data/summarizer/abstractive",
- quantization_config=dqconfig)
+traced_encoder = torch.jit.trace(
+ run_encoder, input_ids)
+traced_decoder_wp.save("data/summarizer/abstractive/traced_decoder_wp.pt")
+traced_decoder.save("data/summarizer/abstractive/traced_decoder.pt")
+traced_encoder.save("data/summarizer/abstractive/traced_encoder.pt")
urllib.request.urlretrieve("https://s3.trystract.com/public/truncated_word2vec.bin.gz",
"data/summarizer/word2vec.bin.gz")
diff --git a/scripts/export_crossencoder b/scripts/export_crossencoder
index 8e08053d..15f5db3b 100755
--- a/scripts/export_crossencoder
+++ b/scripts/export_crossencoder
@@ -1,23 +1,45 @@
-#!/usr/bin/env python
+#!.venv/bin/python3
import os
-from optimum.onnxruntime import ORTModelForSequenceClassification, ORTQuantizer
-from optimum.onnxruntime.configuration import AutoQuantizationConfig
import transformers
-from functools import partial
+import torch
MODEL = "cross-encoder/ms-marco-TinyBERT-L-2-v2"
os.system("mkdir data/cross_encoder")
-model = ORTModelForSequenceClassification.from_pretrained(
- MODEL, from_transformers=True)
+model = transformers.AutoModelForSequenceClassification.from_pretrained(MODEL)
+
+model.eval()
+for p in model.parameters():
+ p.requires_grad_(False)
tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)
+features = tokenizer(
+ ["How many people live in Berlin?", "How many people live in Berlin?"],
+ [
+ "Berlin has a population of 3,520,031 registered inhabitants in an area of 891.82 square kilometers.",
+ "New York City is famous for the Metropolitan Museum of Art.",
+ ],
+ padding="max_length",
+ truncation=True,
+ return_tensors="pt",
+)
+input_ids = features["input_ids"]
+attention_mask = features["attention_mask"]
+token_type_ids = features["token_type_ids"]
tokenizer.save_pretrained("data/cross_encoder")
-qconfig = AutoQuantizationConfig.avx512(is_static=False)
-quantizer = ORTQuantizer.from_pretrained(model)
-quantizer.quantize(
- quantization_config=qconfig, save_dir="data/cross_encoder")
-# model.save_pretrained("data/cross_encoder")
+def run_model(input_ids, attention_mask, token_type_ids):
+ res = model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ token_type_ids=token_type_ids,
+ return_dict=False,
+ )[0]
+ return res
+
+
+traced_model = torch.jit.trace(run_model, ((input_ids, attention_mask, token_type_ids)))
+
+traced_model.save("data/cross_encoder/model.pt")
diff --git a/scripts/export_qa_model b/scripts/export_qa_model
index 39f057dc..cfafbddf 100755
--- a/scripts/export_qa_model
+++ b/scripts/export_qa_model
@@ -1,22 +1,45 @@
-#!/usr/bin/env python
+#!.venv/bin/python3
import os
-from optimum.onnxruntime import ORTModelForQuestionAnswering, ORTQuantizer
-from optimum.onnxruntime.configuration import AutoQuantizationConfig
import transformers
-from functools import partial
+import torch
MODEL = "deepset/tinyroberta-squad2"
+question = "Where is amazon rain forest located"
+context = """The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of
+which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations.
+The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with
+minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain
+"Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract
+of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."""
+
os.system("mkdir data/qa_model")
-model = ORTModelForQuestionAnswering.from_pretrained(
- MODEL, from_transformers=True)
+model = transformers.AutoModelForQuestionAnswering.from_pretrained(MODEL)
+
+model.eval()
+for p in model.parameters():
+ p.requires_grad_(False)
tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)
tokenizer.save_pretrained("data/qa_model")
-qconfig = AutoQuantizationConfig.avx512(is_static=False)
-quantizer = ORTQuantizer.from_pretrained(model)
+features = tokenizer(
+ question,
+ context,
+ padding="max_length",
+ truncation=True,
+ return_tensors="pt",
+)
+input_ids = features["input_ids"]
+attention_mask = features["attention_mask"]
+
+
+def run_model(input_ids, attention_mask):
+ res = model(input_ids=input_ids, attention_mask=attention_mask, return_dict=False)
+ return res
+
+
+traced_model = torch.jit.trace(run_model, ((input_ids, attention_mask)))
-quantizer.quantize(
- quantization_config=qconfig, save_dir="data/qa_model")
+traced_model.save("data/qa_model/model.pt")
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
index 01730271..896e4cc2 100644
--- a/scripts/requirements.txt
+++ b/scripts/requirements.txt
@@ -1,6 +1,5 @@
-transformers[onnx]
+transformers
tensorboard>1.15.0
-torch
-optimum>=1.6.3
+torch>=2.0.0
datasets
gensim
\ No newline at end of file
diff --git a/scripts/truncate_word2vec b/scripts/truncate_word2vec
index 1fe02ec6..805fb329 100755
--- a/scripts/truncate_word2vec
+++ b/scripts/truncate_word2vec
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!.venv/bin/python3
import gensim
model = gensim.models.KeyedVectors.load_word2vec_format(
diff --git a/scripts/x_short_optic.py b/scripts/x_short_optic.py
index 6518dac9..658d1a17 100644
--- a/scripts/x_short_optic.py
+++ b/scripts/x_short_optic.py
@@ -1,3 +1,4 @@
+#!.venv/bin/python3
import argparse
parser = argparse.ArgumentParser()