Skip to content

web search done right

abstractive summarization model with beam search

Mikkel Denker2a1fa6109ac8
diff --git a/Cargo.lock b/Cargo.lock
index ba9999e0..599b2c5d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2348,7 +2348,7 @@ dependencies = [
[[package]]
name = "onnxruntime"
version = "0.0.14"
-source = "git+https://github.com/Cuely/onnxruntime-rs#11cf02029e385e9034f53cb60d6fb5d20570ec7a"
+source = "git+https://github.com/Cuely/onnxruntime-rs#b7b48c9a6d4a553572caf8f1a595c3721a923a4d"
dependencies = [
"lazy_static",
"ndarray",
@@ -2360,7 +2360,7 @@ dependencies = [
[[package]]
name = "onnxruntime-sys"
version = "0.0.14"
-source = "git+https://github.com/Cuely/onnxruntime-rs#11cf02029e385e9034f53cb60d6fb5d20570ec7a"
+source = "git+https://github.com/Cuely/onnxruntime-rs#b7b48c9a6d4a553572caf8f1a595c3721a923a4d"
dependencies = [
"flate2",
"tar",
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 06b1e910..87234b43 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -96,4 +96,8 @@ harness = false
[[bench]]
name = "online-harmonic"
+harness = false
+
+[[bench]]
+name = "abstractive-summarizer"
harness = false
\ No newline at end of file
diff --git a/core/benches/abstractive-summarizer.rs b/core/benches/abstractive-summarizer.rs
new file mode 100644
index 00000000..b4f6e492
--- /dev/null
+++ b/core/benches/abstractive-summarizer.rs
@@ -0,0 +1,26 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use cuely::summarizer::{AbstractiveModel, GenerationConfig};
+
+pub fn criterion_benchmark(c: &mut Criterion) {
+ let model = AbstractiveModel::open("data/abstractive_summary").unwrap();
+
+ let text = r#"Aristotle (/ˈærɪstɒtəl/;[1] Greek: Ἀριστοτέλης Aristotélēs, pronounced [aristotélɛːs]; 384–322 BC) was an Ancient Greek philosopher and polymath. His writings cover a broad range of subjects including physics, biology, zoology, metaphysics, logic, ethics, aesthetics, poetry, drama, music, rhetoric, psychology, linguistics, economics, politics, meteorology, geology, and government. As the founder of the Peripatetic school of philosophy in the Lyceum in Athens, he began the wider Aristotelian tradition that followed, which set the groundwork for the development of modern science.
+ Little is known about Aristotle's life. He was born in the city of Stagira in Northern Greece during the Classical period. His father, Nicomachus, died when Aristotle was a child, and he was brought up by a guardian. At seventeen or eighteen years of age he joined Plato's Academy in Athens and remained there until the age of thirty-seven (c. 347 BC). Shortly after Plato died, Aristotle left Athens and, at the request of Philip II of Macedon, tutored his son Alexander the Great beginning in 343 BC. He established a library in the Lyceum which helped him to produce many of his hundreds of books on papyrus scrolls.
+ Though Aristotle wrote many elegant treatises and dialogues for publication, only around a third of his original output has survived, none of it intended for publication. Aristotle provided a complex synthesis of the various philosophies existing prior to him. It was above all from his teachings that the West inherited its intellectual lexicon, as well as problems and methods of inquiry. As a result, his philosophy has exerted a unique influence on almost every form of knowledge in the West and it continues to be a subject of contemporary philosophical discussion.
+ Aristotle's views profoundly shaped medieval scholarship. The influence of physical science extended from Late Antiquity and the Early Middle Ages into the Renaissance, and were not replaced systematically until the Enlightenment and theories such as classical mechanics were developed. Some of Aristotle's zoological observations found in his biology, such as on the hectocotyl (reproductive) arm of the octopus, were disbelieved until the 19th century. He also influenced Judeo-Islamic philosophies during the Middle Ages, as well as Christian theology, especially the Neoplatonism of the Early Church and the scholastic tradition of the Catholic Church. Aristotle was revered among medieval Muslim scholars as "The First Teacher", and among medieval Christians like Thomas Aquinas as simply "The Philosopher", while the poet Dante called him "the master of those who know". His works contain the earliest known formal study of logic, and were studied by medieval scholars such as Peter Abelard and John Buridan. Aristotle's influence on logic continued well into the 19th century. In addition, his ethics, though always influential, gained renewed interest with the modern advent of virtue ethics."#;
+
+ c.bench_function("Summarize Aristotles wiki text", |b| {
+ b.iter(|| {
+ model.summarize(
+ text,
+ GenerationConfig {
+ num_beams: 10,
+ ..Default::default()
+ },
+ )
+ })
+ });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/core/src/lib.rs b/core/src/lib.rs
index fdf56824..c4eae5ef 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -60,7 +60,7 @@ mod snippet;
mod sonic;
mod spell;
mod subdomain_count;
-mod summarizer;
+pub mod summarizer;
mod tokenizer;
#[allow(unused)]
mod ttl_cache;
@@ -326,3 +326,19 @@ fn floor_char_boundary(str: &str, index: usize) -> usize {
res
}
+
+fn softmax(vec: &mut [f32]) {
+ let s: f32 = vec.iter().map(|z| z.exp()).sum();
+
+ for z in vec {
+ *z = z.exp() / s
+ }
+}
+
+pub static ONNX_ENVIRONMENT: once_cell::sync::Lazy<onnxruntime::environment::Environment> =
+ once_cell::sync::Lazy::new(|| {
+ onnxruntime::environment::Environment::builder()
+ .with_log_level(onnxruntime::LoggingLevel::Info)
+ .build()
+ .unwrap()
+ });
diff --git a/core/src/qa_model.rs b/core/src/qa_model.rs
index 95652b5c..ac7c12c1 100644
--- a/core/src/qa_model.rs
+++ b/core/src/qa_model.rs
@@ -16,25 +16,15 @@
use std::{cmp::Ordering, ops::Range, path::Path, sync::Mutex};
-use crate::Result;
+use crate::{softmax, Result, ONNX_ENVIRONMENT};
use itertools::Itertools;
use onnxruntime::{
- environment::Environment,
ndarray::{Array2, Axis},
tensor::OrtOwnedTensor,
- GraphOptimizationLevel, LoggingLevel,
+ GraphOptimizationLevel, TypedArray,
};
use tokenizers::{PaddingParams, TruncationParams};
-pub static ONNX_ENVIRONMENT: once_cell::sync::Lazy<Environment> =
- once_cell::sync::Lazy::new(|| {
- Environment::builder()
- .with_name("qa")
- .with_log_level(LoggingLevel::Info)
- .build()
- .unwrap()
- });
-
const TRUNCATE_INPUT: usize = 128;
const SCORE_THRESHOLD: f64 = 0.0;
@@ -114,7 +104,9 @@ impl QaModel {
let mut sess = self.session.lock().unwrap();
- let mut res: Vec<OrtOwnedTensor<f32, _>> = sess.run(vec![ids, masks]).unwrap();
+ let mut res: Vec<OrtOwnedTensor<f32, _>> = sess
+ .run(vec![TypedArray::I64(ids), TypedArray::I64(masks)])
+ .unwrap();
if res.len() != 2 {
return None;
@@ -193,14 +185,6 @@ impl QaModel {
}
}
-fn softmax(vec: &mut [f32]) {
- let s: f32 = vec.iter().map(|z| z.exp()).sum();
-
- for z in vec {
- *z = z.exp() / s
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
@@ -217,20 +201,14 @@ minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana
of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."#,
];
- match QaModel::open("../data/qa_model") {
- Ok(model) => {
- let answer = model.run(question, &contexts).unwrap();
+ let model = QaModel::open("../data/qa_model").expect("Failed to find QA model");
+ let answer = model.run(question, &contexts).unwrap();
- assert!(answer.score > 0.6);
- assert_eq!(
- &contexts[answer.context_idx][answer.offset],
- "South America"
- );
- }
- Err(err) => {
- panic!("{err:?}");
- }
- }
+ assert!(answer.score > 0.6);
+ assert_eq!(
+ &contexts[answer.context_idx][answer.offset],
+ "South America"
+ );
}
#[test]
@@ -258,20 +236,14 @@ of tropical rainforest in the world, with an estimated 390 billion individual tr
"#,
];
- match QaModel::open("../data/qa_model") {
- Ok(model) => {
- let answer = model.run(question, &contexts).unwrap();
+ let model = QaModel::open("../data/qa_model").expect("Failed to find QA model");
+ let answer = model.run(question, &contexts).unwrap();
- assert!(answer.score > 0.6);
- assert_eq!(
- &contexts[answer.context_idx][answer.offset],
- "South America"
- );
- }
- Err(err) => {
- panic!("{err:?}");
- }
- }
+ assert!(answer.score > 0.6);
+ assert_eq!(
+ &contexts[answer.context_idx][answer.offset],
+ "South America"
+ );
}
#[test]
@@ -299,20 +271,14 @@ minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana
of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."#,
];
- match QaModel::open("../data/qa_model") {
- Ok(model) => {
- let answer = model.run(question, &contexts).unwrap();
+ let model = QaModel::open("../data/qa_model").expect("Failed to find QA model");
+ let answer = model.run(question, &contexts).unwrap();
- assert!(answer.score > 0.6);
- assert_eq!(
- &contexts[answer.context_idx][answer.offset],
- "South America"
- );
- }
- Err(err) => {
- panic!("{err:?}");
- }
- }
+ assert!(answer.score > 0.6);
+ assert_eq!(
+ &contexts[answer.context_idx][answer.offset],
+ "South America"
+ );
}
#[test]
@@ -339,25 +305,19 @@ minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana
of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."#,
];
- match QaModel::open("../data/qa_model") {
- Ok(model) => {
- let answer = model.run(question, &contexts);
+ let model = QaModel::open("../data/qa_model").expect("Failed to find QA model");
- assert!(answer.is_none());
- }
- Err(err) => {
- panic!("{err:?}");
- }
- }
+ let answer = model.run(question, &contexts);
+ assert!(answer.is_none());
}
#[test]
fn empty() {
- match QaModel::open("../data/qa_model") {
- Ok(model) => {
- let question = "";
- let contexts = [
- r#"The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of
+ let model = QaModel::open("../data/qa_model").expect("Failed to find QA model");
+
+ let question = "";
+ let contexts = [
+ r#"The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of
which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations.
The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with
minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain
@@ -375,24 +335,19 @@ of tropical rainforest in the world, with an estimated 390 billion individual tr
minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain
"Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract
of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."#,
- ];
- let answer = model.run(question, &contexts);
+ ];
+ let answer = model.run(question, &contexts);
- assert!(answer.is_none());
+ assert!(answer.is_none());
- let question = "test";
- let contexts = [r#""#];
- let answer = model.run(question, &contexts);
- assert!(answer.is_none());
+ let question = "test";
+ let contexts = [r#""#];
+ let answer = model.run(question, &contexts);
+ assert!(answer.is_none());
- let question = "";
- let contexts = [r#""#];
- let answer = model.run(question, &contexts);
- assert!(answer.is_none());
- }
- Err(err) => {
- panic!("{err:?}");
- }
- }
+ let question = "";
+ let contexts = [r#""#];
+ let answer = model.run(question, &contexts);
+ assert!(answer.is_none());
}
}
diff --git a/core/src/ranking/models/cross_encoder.rs b/core/src/ranking/models/cross_encoder.rs
index 8546b324..6e57acc1 100644
--- a/core/src/ranking/models/cross_encoder.rs
+++ b/core/src/ranking/models/cross_encoder.rs
@@ -17,11 +17,13 @@
use std::{path::Path, sync::Mutex};
use itertools::Itertools;
-use onnxruntime::{ndarray::ArrayBase, tensor::OrtOwnedTensor, GraphOptimizationLevel};
+use onnxruntime::{ndarray::ArrayBase, tensor::OrtOwnedTensor, GraphOptimizationLevel, TypedArray};
+use tokenizers::PaddingParams;
+use tokenizers::TruncationParams;
use crate::Result;
-use super::ONNX_ENVIRONMENT;
+use crate::ONNX_ENVIRONMENT;
const TRUNCATE_INPUT: usize = 256;
pub struct CrossEncoderModel {
@@ -31,7 +33,20 @@ pub struct CrossEncoderModel {
impl CrossEncoderModel {
pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {
- let tokenizer = tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))?;
+ let truncation = TruncationParams {
+ max_length: TRUNCATE_INPUT,
+ ..Default::default()
+ };
+
+ let padding = PaddingParams {
+ ..Default::default()
+ };
+
+ let mut tokenizer =
+ tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))?;
+
+ tokenizer.with_truncation(Some(truncation));
+ tokenizer.with_padding(Some(padding));
let session = Mutex::new(
ONNX_ENVIRONMENT
@@ -89,15 +104,21 @@ impl CrossEncoder for CrossEncoderModel {
let res: Vec<OrtOwnedTensor<f32, _>> = sess
.run(vec![
- ArrayBase::from_vec(ids)
- .into_shape((1, num_tokens))
- .unwrap(),
- ArrayBase::from_vec(attention_mask)
- .into_shape((1, num_tokens))
- .unwrap(),
- ArrayBase::from_vec(type_ids)
- .into_shape((1, num_tokens))
- .unwrap(),
+ TypedArray::I64(
+ ArrayBase::from_vec(ids)
+ .into_shape((1, num_tokens))
+ .unwrap(),
+ ),
+ TypedArray::I64(
+ ArrayBase::from_vec(attention_mask)
+ .into_shape((1, num_tokens))
+ .unwrap(),
+ ),
+ TypedArray::I64(
+ ArrayBase::from_vec(type_ids)
+ .into_shape((1, num_tokens))
+ .unwrap(),
+ ),
])
.unwrap();
@@ -114,36 +135,32 @@ mod tests {
#[test]
fn sanity_check() {
- match CrossEncoderModel::open("../data/cross_encoder") {
- Ok(model) => {
- let s = model.run(
+ let model = CrossEncoderModel::open("../data/cross_encoder")
+ .expect("Failed to find cross-encoder model");
+
+ let s = model.run(
+ "how many people live in paris",
+ "there are currently 1234 people living in paris",
+ );
+
+ for _ in 0..10 {
+ assert_eq!(
+ s,
+ model.run(
"how many people live in paris",
- "there are currently 1234 people living in paris",
- );
-
- for _ in 0..10 {
- assert_eq!(
- s,
- model.run(
- "how many people live in paris",
- "there are currently 1234 people living in paris"
- )
- );
- }
-
- assert!(
- model.run(
- "how many people live in paris",
- "there are currently 1234 people living in paris"
- ) > model.run(
- "how many people live in paris",
- "I really like cake and my favorite cake is probably brownie"
- )
- );
- }
- Err(err) => {
- panic!("{err:?}");
- }
+ "there are currently 1234 people living in paris"
+ )
+ );
}
+
+ assert!(
+ model.run(
+ "how many people live in paris",
+ "there are currently 1234 people living in paris"
+ ) > model.run(
+ "how many people live in paris",
+ "I really like cake and my favorite cake is probably brownie"
+ )
+ );
}
}
diff --git a/core/src/ranking/models/mod.rs b/core/src/ranking/models/mod.rs
index 13e94d77..5a80c249 100644
--- a/core/src/ranking/models/mod.rs
+++ b/core/src/ranking/models/mod.rs
@@ -14,15 +14,4 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use onnxruntime::{environment::Environment, LoggingLevel};
-
pub mod cross_encoder;
-
-pub static ONNX_ENVIRONMENT: once_cell::sync::Lazy<Environment> =
- once_cell::sync::Lazy::new(|| {
- Environment::builder()
- .with_name("ranking")
- .with_log_level(LoggingLevel::Info)
- .build()
- .unwrap()
- });
diff --git a/core/src/summarizer.rs b/core/src/summarizer.rs
index 0b33e6b3..4eececc9 100644
--- a/core/src/summarizer.rs
+++ b/core/src/summarizer.rs
@@ -16,17 +16,21 @@
use std::{
cmp::{Ordering, Reverse},
- collections::BinaryHeap,
+ collections::{BinaryHeap, VecDeque},
+ path::Path,
+ sync::Mutex,
};
-use itertools::intersperse;
+use itertools::{intersperse, Itertools};
+use onnxruntime::{
+ ndarray::{Array, ArrayBase, Axis, Dim, IxDynImpl, OwnedRepr},
+ tensor::OrtOwnedTensor,
+ GraphOptimizationLevel, TypedArray,
+};
+use tokenizers::{PaddingParams, TruncationParams};
-use crate::spell::word2vec::Word2Vec;
-
-pub struct Summarizer {
- word2vec: Word2Vec,
- top_n_passages: usize,
-}
+use crate::{softmax, spell::word2vec::Word2Vec};
+use crate::{Result, ONNX_ENVIRONMENT};
struct CandidatePassage<'a> {
passage: &'a str,
@@ -54,8 +58,97 @@ impl<'a> PartialEq for CandidatePassage<'a> {
impl<'a> Eq for CandidatePassage<'a> {}
+struct OverlappingSents<'a> {
+ text: &'a str,
+ window_size: usize,
+ next_start: VecDeque<usize>,
+ overlap: usize,
+}
+
+impl<'a> OverlappingSents<'a> {
+ fn new(text: &'a str, window_size: usize, overlap: usize) -> Self {
+ assert!(
+ overlap < window_size,
+ "overlap needs to be smaller than window size"
+ );
+
+ let next_start = VecDeque::with_capacity(overlap + 1);
+
+ Self {
+ text,
+ window_size,
+ overlap,
+ next_start,
+ }
+ }
+}
+
+impl<'a> Iterator for OverlappingSents<'a> {
+ type Item = &'a str;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ if self.text.is_empty() {
+ return None;
+ }
+
+ let mut end = self.text.len();
+ let mut missing_words = self.window_size;
+ self.next_start.clear();
+
+ for (i, (idx, _)) in self
+ .text
+ .char_indices()
+ .filter(|(_, c)| c.is_whitespace())
+ .enumerate()
+ {
+ if i >= self.window_size {
+ break;
+ }
+
+ missing_words -= 1;
+
+ if self.next_start.len() > self.overlap {
+ self.next_start.pop_front();
+ }
+
+ self.next_start.push_back(idx);
+ end = idx;
+ }
+
+ if missing_words > 0 {
+ end = self.text.len();
+ for _ in 0..missing_words {
+ self.next_start.pop_front();
+ }
+ }
+
+ let res = &self.text[..end];
+
+ if let Some(next_start) = self.next_start.pop_front() {
+ if next_start == 0 {
+ self.text = "";
+ } else {
+ self.text = &self.text[next_start + 1..];
+ }
+ } else {
+ self.text = "";
+ }
+
+ Some(res)
+ }
+}
+pub struct Summarizer {
+ word2vec: Word2Vec,
+ top_n_passages: usize,
+ abstractive_model: AbstractiveModel,
+}
+
impl Summarizer {
- pub fn summarize(&self, query: &str, text: &str) -> Option<String> {
+ // pub fn new() -> Self {
+ // Self {}
+ // }
+
+ pub fn extractive_summary(&self, query: &str, text: &str) -> Option<String> {
let query_vectors: Vec<_> = query
.split_whitespace()
.filter_map(|word| self.word2vec.get(word))
@@ -68,8 +161,9 @@ impl Summarizer {
let mut best_passages: BinaryHeap<Reverse<CandidatePassage<'_>>> =
BinaryHeap::with_capacity(self.top_n_passages);
- // for (index, passage) in text.split_whitespace().windows(100).enumerate() {
- for (index, passage) in text.split_whitespace().enumerate() {
+ let overlap_sents = OverlappingSents::new(text, 100, 10);
+
+ for (index, passage) in overlap_sents.enumerate() {
let mut score = 0.0;
let mut count = 0;
@@ -111,9 +205,566 @@ impl Summarizer {
let mut best_passages: Vec<_> = best_passages.into_iter().map(|r| r.0).collect();
best_passages.sort_by_key(|a| a.index);
- let extractive_summary: String =
- intersperse(best_passages.into_iter().map(|p| p.passage), ". ").collect();
+ Some(intersperse(best_passages.into_iter().map(|p| p.passage), ". ").collect())
+ }
+
+ pub fn abstractive_summary(&self, _query: &str, text: &str) -> String {
+ todo!("use onnx to generate stuff")
+ }
+
+ pub fn summarize(&self, query: &str, text: &str) -> Option<String> {
+ self.extractive_summary(query, text)
+ .map(|summary| self.abstractive_summary(query, &summary))
+ }
+}
+
+const TRUNCATE_INPUT: usize = 1024;
+pub struct AbstractiveModel {
+ encoder: Mutex<onnxruntime::session::Session<'static>>,
+ decoder: Mutex<onnxruntime::session::Session<'static>>,
+ decoder_with_past: Mutex<onnxruntime::session::Session<'static>>,
+ tokenizer: tokenizers::Tokenizer,
+
+ bos_token_id: usize,
+ pad_token_id: usize,
+ eos_token_id: usize,
+ begin_decoder_token: usize,
+}
+
+impl AbstractiveModel {
+ pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {
+ let truncation = TruncationParams {
+ max_length: TRUNCATE_INPUT,
+ ..Default::default()
+ };
+
+ let padding = PaddingParams {
+ ..Default::default()
+ };
+
+ let mut tokenizer =
+ tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))?;
+
+ tokenizer.with_truncation(Some(truncation));
+ tokenizer.with_padding(Some(padding));
+
+ let encoder = Mutex::new(
+ ONNX_ENVIRONMENT
+ .new_session_builder()?
+ .with_optimization_level(GraphOptimizationLevel::All)?
+ .with_number_threads(1)?
+ .with_model_from_file(folder.as_ref().join("encoder_model.onnx"))?,
+ );
+
+ let decoder = Mutex::new(
+ ONNX_ENVIRONMENT
+ .new_session_builder()?
+ .with_optimization_level(GraphOptimizationLevel::All)?
+ .with_number_threads(1)?
+ .with_model_from_file(folder.as_ref().join("decoder_model.onnx"))?,
+ );
+
+ let decoder_with_past = Mutex::new(
+ ONNX_ENVIRONMENT
+ .new_session_builder()?
+ .with_optimization_level(GraphOptimizationLevel::All)?
+ .with_number_threads(1)?
+ .with_model_from_file(folder.as_ref().join("decoder_with_past_model.onnx"))?,
+ );
+
+ Ok(Self {
+ tokenizer,
+ encoder,
+ decoder,
+ decoder_with_past,
+
+ bos_token_id: 0,
+ pad_token_id: 1,
+ eos_token_id: 2,
+ begin_decoder_token: 2,
+ })
+ }
+
+ fn encode(
+ &self,
+ ids: Vec<i64>,
+ attention_mask: Vec<i64>,
+ ) -> Result<ArrayBase<OwnedRepr<f32>, Dim<IxDynImpl>>> {
+ let mut encoder = self
+ .encoder
+ .lock()
+ .expect("Failed to get lock. Maybe another thread crashed?");
+
+ let num_tokens = ids.len();
+
+ let mut encoder_output: Vec<OrtOwnedTensor<f32, _>> = encoder.run(vec![
+ TypedArray::I64(
+ ArrayBase::from_vec(ids)
+ .into_shape((1, num_tokens))
+ .unwrap(),
+ ),
+ TypedArray::I64(
+ ArrayBase::from_vec(attention_mask)
+ .into_shape((1, num_tokens))
+ .unwrap(),
+ ),
+ ])?;
+
+ let res = encoder_output.pop().unwrap().clone().into_owned();
+
+ Ok(res)
+ }
+
+ pub fn summarize(&self, text: &str, config: GenerationConfig) -> Result<String> {
+ let encoded_text = self.tokenizer.encode(text, true)?;
+
+ let mut ids = encoded_text
+ .get_ids()
+ .iter()
+ .map(|i| *i as i64)
+ .take(TRUNCATE_INPUT)
+ .collect_vec();
+
+ let attention_mask = encoded_text
+ .get_attention_mask()
+ .iter()
+ .take(ids.len())
+ .map(|i| *i as i64)
+ .collect_vec();
+
+ let encoder_num_tokens = ids.len();
+
+ let mut decoder = self
+ .decoder
+ .lock()
+ .expect("Failed to get lock. Maybe another thread crashed?");
+
+ let mut decoder_with_past = self
+ .decoder_with_past
+ .lock()
+ .expect("Failed to get lock. Maybe another thread crashed?");
+
+ let arr_encoder_hidden_states = self.encode(ids.clone(), attention_mask.clone())?;
+
+ let arr_attention_mask = ArrayBase::from_vec(attention_mask.clone())
+ .into_shape((1, encoder_num_tokens))
+ .unwrap()
+ .into_dyn();
+
+ let mut beams = Vec::new();
+
+ ids = vec![self.begin_decoder_token as i64, self.bos_token_id as i64];
+ let mut generated_tokens = 0;
+
+ let mut caches = Vec::new();
+
+ loop {
+ generated_tokens += 1;
+
+ if beams.is_empty() {
+ let num_tokens = ids.len();
+ let arr_ids = ArrayBase::from_vec(ids.clone())
+ .into_shape((1, num_tokens))
+ .unwrap()
+ .into_dyn();
+
+ let decoder_input = vec![
+ TypedArray::I64(arr_attention_mask.clone()),
+ TypedArray::I64(arr_ids),
+ TypedArray::F32(arr_encoder_hidden_states.clone()),
+ ];
+
+ let decoder_output = decoder.run(decoder_input)?;
+ let mut next_scores = BinaryHeap::with_capacity(config.num_beams as usize + 1);
+
+ let all_logits = &decoder_output[0];
+ let vocab_size = all_logits.shape()[2];
+ let mut scores = Vec::with_capacity(vocab_size);
+
+ for token_id in 0..vocab_size {
+ let score: f32 = all_logits[[0, num_tokens - 1, token_id]];
+ scores.push(score);
+ }
+
+ softmax(&mut scores);
+
+ for (token_id, score) in scores.into_iter().enumerate() {
+ if token_id == self.bos_token_id {
+ continue;
+ }
+
+ let scored_token = Reverse(ScoredToken {
+ token: token_id as i64,
+ score: score.log2(),
+ });
+
+ if next_scores.len() < config.num_beams as usize {
+ next_scores.push(scored_token);
+ } else if let Some(mut worst) = next_scores.peek_mut() {
+ if worst.0.score < scored_token.0.score {
+ *worst = scored_token
+ }
+ }
+ }
+
+ let num_outputs = decoder_output.len();
+
+ caches = decoder_output
+ .into_iter()
+ .map(|t| t.clone().into_owned())
+ .map(|mut t| {
+ let orig = t.clone();
+
+ for _ in 0..config.num_beams - 1 {
+ t.append(Axis(0), orig.view()).unwrap();
+ }
+
+ t
+ })
+ .skip(1)
+ .take(num_outputs - 2) // skip 'encoder_hidden_states' since this is not changed by the decoder model
+ .collect::<Vec<_>>();
+
+ for scored_token in next_scores.into_iter().map(|t| t.0) {
+ let mut ids = ids.clone();
+ ids.push(scored_token.token);
+
+ beams.push(Beam {
+ input_ids: ids,
+ score: scored_token.score,
+ })
+ }
+ } else {
+ let mut combined_attention_mask = Vec::new();
+ let mut combined_encoder_hidden_states = Vec::with_capacity(beams.len());
+ let mut combined_ids = Vec::new();
+
+ // TODO: Only advance non-finished beams
+
+ for beam in &beams {
+ combined_ids.push(*beam.input_ids.last().unwrap());
+
+ for mask in &attention_mask {
+ combined_attention_mask.push(*mask);
+ }
+
+ for state in &arr_encoder_hidden_states {
+ combined_encoder_hidden_states.push(*state);
+ }
+ }
+
+ let combined_ids = ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_ids)
+ .into_shape((beams.len(), 1))
+ .unwrap()
+ .into_dyn();
+
+ let mut attention_shape = arr_attention_mask.shape().to_vec();
+ attention_shape[0] = beams.len();
+
+ let combined_attention_mask =
+ ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_attention_mask)
+ .into_shape(attention_shape.as_slice())
+ .unwrap();
+
+ let mut state_shape = arr_encoder_hidden_states.shape().to_vec();
+ state_shape[0] = beams.len();
+
+ let combined_encoder_hidden_states =
+ ArrayBase::<OwnedRepr<_>, _>::from_vec(combined_encoder_hidden_states)
+ .into_shape(state_shape.as_slice())
+ .unwrap();
+
+ let mut decoder_input = vec![
+ TypedArray::I64(combined_attention_mask),
+ TypedArray::I64(combined_ids),
+ TypedArray::F32(combined_encoder_hidden_states),
+ ];
+
+ for tensor in caches.drain(0..) {
+ decoder_input.push(TypedArray::F32(tensor));
+ }
+
+ let decoder_output: Vec<OrtOwnedTensor<f32, Dim<IxDynImpl>>> =
+ decoder_with_past.run(decoder_input)?;
+
+ let all_logits = &decoder_output[0];
+ let vocab_size = all_logits.shape()[2];
+
+ let mut beam_tok_scores = Vec::with_capacity(beams.len());
+ for beam_idx in 0..beams.len() {
+ let mut scores = Vec::with_capacity(vocab_size);
+
+ for token_id in 0..vocab_size {
+ scores.push(all_logits[[beam_idx, 0, token_id]]);
+ }
+
+ softmax(&mut scores);
+ beam_tok_scores.push(scores)
+ }
+
+ let num_outputs = decoder_output.len();
+
+ // skip(num_outputs-2) to skip 'encoder_hidden_states' since this is not changed by the decoder model
+ let output_caches: Vec<_> = decoder_output
+ .iter()
+ .skip(1)
+ .take(num_outputs - 2)
+ .collect();
+
+ let mut new_beams = BinaryHeap::with_capacity(beams.len());
+
+ for beam_idx in 0..beams.len() {
+ for new_token in 0..vocab_size {
+ let tok_score = beam_tok_scores[beam_idx][new_token];
+
+ let possible_beam = PossibleBeam {
+ beam_idx,
+ new_token: new_token as i64,
+ new_score: beams[beam_idx].score + tok_score.log2(),
+ };
+
+ if new_beams.len() < beams.len() {
+ new_beams.push(Reverse(possible_beam));
+ } else if let Some(mut worst) = new_beams.peek_mut() {
+ if possible_beam.new_score > worst.0.new_score {
+ *worst = Reverse(possible_beam);
+ }
+ }
+ }
+ }
+
+ let mut new_caches = Vec::with_capacity(output_caches.len());
+
+ for c in &output_caches {
+ let shape = c.shape();
+ new_caches.push(Array::zeros((0, shape[1], shape[2], shape[3])).into_dyn());
+ }
+
+ let new_beams: Vec<_> = new_beams
+ .into_iter()
+ .map(|r| r.0)
+ .map(|possible_beam| {
+ let input_ids = if beams[possible_beam.beam_idx]
+ .is_finished(self.eos_token_id as i64)
+ {
+ beams[possible_beam.beam_idx].input_ids.clone()
+ } else {
+ let mut ids = beams[possible_beam.beam_idx].input_ids.clone();
+ ids.push(possible_beam.new_token);
+ ids
+ };
+
+ for (new_cache, old_cache) in
+ new_caches.iter_mut().zip_eq(output_caches.iter())
+ {
+ let shape = old_cache.shape();
+
+ new_cache
+ .append(
+ Axis(0),
+ old_cache
+ .index_axis(Axis(0), possible_beam.beam_idx)
+ .into_shape((1, shape[1], shape[2], shape[3]))
+ .unwrap()
+ .into_dyn(),
+ )
+ .unwrap();
+ }
+ Beam {
+ input_ids,
+ score: possible_beam.new_score,
+ }
+ })
+ .collect();
+
+ beams = new_beams;
+ caches = new_caches;
+
+ if beams
+ .iter()
+ .all(|beam| beam.is_finished(self.eos_token_id as i64))
+ {
+ break;
+ }
+ }
+
+ if let Some(min_tokens) = config.force_min_tokens {
+ if generated_tokens < min_tokens {
+ continue;
+ }
+ }
+
+ if let Some(early_stopping) = &config.early_stopping {
+ match early_stopping {
+ EarlyStopping::MaxTokens { max_new_tokens } => {
+ if generated_tokens >= *max_new_tokens {
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ let best_beam = beams
+ .into_iter()
+ .max_by(|a, b| a.score.total_cmp(&b.score))
+ .unwrap();
+
+ let res: Vec<_> = best_beam.input_ids.into_iter().map(|i| i as u32).collect();
+ Ok(self.tokenizer.decode(res, true).unwrap())
+ }
+}
+
+struct ScoredToken {
+ token: i64,
+ score: f32,
+}
+
+impl PartialOrd for ScoredToken {
+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+ self.score.partial_cmp(&other.score)
+ }
+}
+
+impl Ord for ScoredToken {
+ fn cmp(&self, other: &Self) -> Ordering {
+ self.partial_cmp(other).unwrap_or(Ordering::Equal)
+ }
+}
+
+impl PartialEq for ScoredToken {
+ fn eq(&self, other: &Self) -> bool {
+ self.score == other.score
+ }
+}
+
+impl Eq for ScoredToken {}
+
+struct Beam {
+ input_ids: Vec<i64>,
+ score: f32,
+}
+
+impl Beam {
+ fn is_finished(&self, eos_tok: i64) -> bool {
+ self.input_ids
+ .last()
+ .map(|tok| *tok == eos_tok)
+ .unwrap_or(false)
+ }
+}
+
+struct PossibleBeam {
+ beam_idx: usize,
+ new_token: i64,
+ new_score: f32,
+}
+
+impl PartialOrd for PossibleBeam {
+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+ self.new_score.partial_cmp(&other.new_score)
+ }
+}
+
+impl Ord for PossibleBeam {
+ fn cmp(&self, other: &Self) -> Ordering {
+ self.partial_cmp(other).unwrap_or(Ordering::Equal)
+ }
+}
+
+impl PartialEq for PossibleBeam {
+ fn eq(&self, other: &Self) -> bool {
+ self.new_score == other.new_score
+ }
+}
+
+impl Eq for PossibleBeam {}
+
+pub struct GenerationConfig {
+ pub early_stopping: Option<EarlyStopping>,
+ pub force_min_tokens: Option<u32>,
+ pub num_beams: u32,
+ pub length_penalty: f32,
+}
+
+impl Default for GenerationConfig {
+ fn default() -> Self {
+ Self {
+ early_stopping: Some(EarlyStopping::MaxTokens {
+ max_new_tokens: 256,
+ }),
+ force_min_tokens: None,
+ num_beams: 5,
+ length_penalty: 1.5,
+ }
+ }
+}
+
+pub enum EarlyStopping {
+ MaxTokens { max_new_tokens: u32 },
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn overlapping_sentences() {
+ let mut it = OverlappingSents::new("this is a test sentence", 3, 1);
+
+ assert_eq!(it.next(), Some("this is a"));
+ assert_eq!(it.next(), Some("a test sentence"));
+ assert_eq!(it.next(), Some("sentence"));
+ assert_eq!(it.next(), None);
+
+ let mut it = OverlappingSents::new("this is a test sentence", 3, 0);
+
+ assert_eq!(it.next(), Some("this is a"));
+ assert_eq!(it.next(), Some("test sentence"));
+ assert_eq!(it.next(), None);
+
+ let mut it = OverlappingSents::new("this is a test sentence", 3, 2);
+
+ assert_eq!(it.next(), Some("this is a"));
+ assert_eq!(it.next(), Some("is a test"));
+ assert_eq!(it.next(), Some("a test sentence"));
+ assert_eq!(it.next(), Some("sentence"));
+ assert_eq!(it.next(), None);
+
+ let mut it = OverlappingSents::new("this", 3, 1);
+
+ assert_eq!(it.next(), Some("this"));
+ assert_eq!(it.next(), None);
+
+ let mut it = OverlappingSents::new("this ", 3, 0);
+
+ assert_eq!(it.next(), Some("this ")); // this is not really great, but close enough. At least no panic
+ assert_eq!(it.next(), None);
+ }
+
+ #[test]
+ fn abstractive_summary() {
+ let model = AbstractiveModel::open("../data/abstractive_summary")
+ .expect("abstractive summary model not found");
+
+ let text = r#"Aristotle (/ˈærɪstɒtəl/;[1] Greek: Ἀριστοτέλης Aristotélēs, pronounced [aristotélɛːs]; 384–322 BC) was an Ancient Greek philosopher and polymath. His writings cover a broad range of subjects including physics, biology, zoology, metaphysics, logic, ethics, aesthetics, poetry, drama, music, rhetoric, psychology, linguistics, economics, politics, meteorology, geology, and government. As the founder of the Peripatetic school of philosophy in the Lyceum in Athens, he began the wider Aristotelian tradition that followed, which set the groundwork for the development of modern science.
+ Little is known about Aristotle's life. He was born in the city of Stagira in Northern Greece during the Classical period. His father, Nicomachus, died when Aristotle was a child, and he was brought up by a guardian. At seventeen or eighteen years of age he joined Plato's Academy in Athens and remained there until the age of thirty-seven (c. 347 BC). Shortly after Plato died, Aristotle left Athens and, at the request of Philip II of Macedon, tutored his son Alexander the Great beginning in 343 BC. He established a library in the Lyceum which helped him to produce many of his hundreds of books on papyrus scrolls.
+ Though Aristotle wrote many elegant treatises and dialogues for publication, only around a third of his original output has survived, none of it intended for publication. Aristotle provided a complex synthesis of the various philosophies existing prior to him. It was above all from his teachings that the West inherited its intellectual lexicon, as well as problems and methods of inquiry. As a result, his philosophy has exerted a unique influence on almost every form of knowledge in the West and it continues to be a subject of contemporary philosophical discussion.
+ Aristotle's views profoundly shaped medieval scholarship. The influence of physical science extended from Late Antiquity and the Early Middle Ages into the Renaissance, and were not replaced systematically until the Enlightenment and theories such as classical mechanics were developed. Some of Aristotle's zoological observations found in his biology, such as on the hectocotyl (reproductive) arm of the octopus, were disbelieved until the 19th century. He also influenced Judeo-Islamic philosophies during the Middle Ages, as well as Christian theology, especially the Neoplatonism of the Early Church and the scholastic tradition of the Catholic Church. Aristotle was revered among medieval Muslim scholars as "The First Teacher", and among medieval Christians like Thomas Aquinas as simply "The Philosopher", while the poet Dante called him "the master of those who know". His works contain the earliest known formal study of logic, and were studied by medieval scholars such as Peter Abelard and John Buridan. Aristotle's influence on logic continued well into the 19th century. In addition, his ethics, though always influential, gained renewed interest with the modern advent of virtue ethics."#;
+
+ let start = std::time::Instant::now();
+ let config = GenerationConfig {
+ num_beams: 1,
+ ..Default::default()
+ };
+
+ let summary = model.summarize(text, config).unwrap();
+ dbg!(start.elapsed());
+
+ assert!(summary.len() > 50);
+ assert!(summary.contains("Aristotle"));
+ dbg!(&summary);
- todo!("run abstractive summary model");
+ // todo!()
}
}
diff --git a/justfile b/justfile
index 88efc4dc..3a137168 100644
--- a/justfile
+++ b/justfile
@@ -21,6 +21,7 @@
pip3 install -r scripts/requirements.txt
./scripts/export_crossencoder
./scripts/export_qa_model
+ ./scripts/export_abstractive_summary_model
cd frontend; npm install; npm run build
cargo run --release --all-features -- configure {{ARGS}}
diff --git a/scripts/export_abstractive_summary_model b/scripts/export_abstractive_summary_model
new file mode 100755
index 00000000..4b1395e3
--- /dev/null
+++ b/scripts/export_abstractive_summary_model
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+from optimum.onnxruntime import ORTQuantizer, ORTModelForSeq2SeqLM
+from optimum.onnxruntime.configuration import AutoQuantizationConfig
+
+MODEL = "lidiya/bart-large-xsum-samsum"
+
+model = ORTModelForSeq2SeqLM.from_pretrained(MODEL, from_transformers=True)
+tokenizer = AutoTokenizer.from_pretrained(MODEL)
+
+model.save_pretrained("data/abstractive_summary")
+
+model_dir = model.model_save_dir
+
+encoder_quantizer = ORTQuantizer.from_pretrained(model_dir, file_name="encoder_model.onnx")
+decoder_quantizer = ORTQuantizer.from_pretrained(model_dir, file_name="decoder_model.onnx")
+decoder_wp_quantizer = ORTQuantizer.from_pretrained(model_dir, file_name="decoder_with_past_model.onnx")
+
+quantizer = [encoder_quantizer, decoder_quantizer, decoder_wp_quantizer]
+
+dqconfig = AutoQuantizationConfig.avx512_vnni(is_static=False, per_channel=False)
+
+for q in quantizer:
+ q.quantize(save_dir="data/abstractive_summary",quantization_config=dqconfig)
+
+
+
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
index 5200fc7e..73edbe68 100644
--- a/scripts/requirements.txt
+++ b/scripts/requirements.txt
@@ -1,5 +1,5 @@
transformers[onnx]
tensorboard>1.15.0
torch
-optimum
+optimum>=1.6.3
datasets
\ No newline at end of file