Skip to content

web search done right

Dual-encoder as passage scorer for extractive summarization

Mikkel Denker72d10866724c
diff --git a/core/src/api/improvement.rs b/core/src/api/improvement.rs
index 5521884f..776d3332 100644
--- a/core/src/api/improvement.rs
+++ b/core/src/api/improvement.rs
@@ -17,7 +17,7 @@
use std::sync::Arc;
use askama::Template;
-use axum::{extract, response::IntoResponse, Extension};
+use axum::{extract, response::IntoResponse};
use serde::Deserialize;
use uuid::Uuid;
@@ -40,7 +40,7 @@ pub struct StoreParams {
#[allow(clippy::unused_async)]
pub async fn click(
extract::Query(params): extract::Query<ClickParams>,
- Extension(state): Extension<Arc<State>>,
+ extract::State(state): extract::State<Arc<State>>,
) {
if let Some(q) = state.improvement_queue.as_ref() {
q.lock().await.push(ImprovementEvent::Click {
@@ -61,7 +61,7 @@ impl From<StoreParams> for StoredQuery {
#[allow(clippy::unused_async)]
pub async fn store(
- Extension(state): Extension<Arc<State>>,
+ extract::State(state): extract::State<Arc<State>>,
extract::Json(params): extract::Json<StoreParams>,
) -> impl IntoResponse {
match state.improvement_queue.as_ref() {
diff --git a/core/src/api/summarize.rs b/core/src/api/summarize.rs
index ddb8a80d..b32532db 100644
--- a/core/src/api/summarize.rs
+++ b/core/src/api/summarize.rs
@@ -16,7 +16,7 @@
use std::sync::Arc;
-use axum::{extract, Extension};
+use axum::extract;
use http::StatusCode;
use serde::Deserialize;
@@ -31,7 +31,7 @@ pub struct Params {
#[allow(clippy::unused_async)]
pub async fn route(
extract::Query(params): extract::Query<Params>,
- Extension(state): Extension<Arc<State>>,
+ extract::State(state): extract::State<Arc<State>>,
) -> std::result::Result<String, StatusCode> {
let webpage = state
.searcher
@@ -39,7 +39,5 @@ pub async fn route(
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
- Ok(state
- .summarizer
- .abstractive_summary(&params.query, &webpage.body))
+ Ok(state.summarizer.summarize(&params.query, &webpage.body))
}
diff --git a/core/src/spell/word2vec.rs b/core/src/spell/word2vec.rs
index 162f4717..20404738 100644
--- a/core/src/spell/word2vec.rs
+++ b/core/src/spell/word2vec.rs
@@ -115,7 +115,7 @@ impl<R: BufRead> Iterator for WordVectorReader<R> {
}
}
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct WordVec(Vec<f32>);
impl WordVec {
fn magnitude(&self) -> f32 {
diff --git a/core/src/summarizer.rs b/core/src/summarizer.rs
index a3db1499..f7735918 100644
--- a/core/src/summarizer.rs
+++ b/core/src/summarizer.rs
@@ -26,7 +26,10 @@ use itertools::{intersperse, Itertools};
use tch::{IValue, Kind, Tensor};
use tokenizers::{PaddingParams, TruncationParams};
-use crate::{ceil_char_boundary, spell::word2vec::Word2Vec};
+use crate::{
+ ceil_char_boundary,
+ spell::word2vec::{Word2Vec, WordVec},
+};
#[derive(thiserror::Error, Debug)]
pub enum Error {
@@ -163,8 +166,64 @@ impl<'a> Iterator for OverlappingSents<'a> {
Some((res, range))
}
}
+
+trait PassageScorer {
+ type QueryEmbedding;
+ type PassageEmbedding;
+
+ fn embed_query(&self, query: &str) -> Option<Self::QueryEmbedding>;
+ fn embed_passage(&self, passage: &str) -> Option<Self::PassageEmbedding>;
+
+ fn score(&self, query: &Self::QueryEmbedding, passage: &Self::PassageEmbedding) -> f32;
+}
+
+impl PassageScorer for Word2Vec {
+ type QueryEmbedding = Vec<WordVec>;
+
+ type PassageEmbedding = Self::QueryEmbedding;
+
+ fn embed_query(&self, query: &str) -> Option<Self::QueryEmbedding> {
+ let res: Self::QueryEmbedding = query
+ .split_whitespace()
+ .filter_map(|word| self.get(word).cloned())
+ .collect();
+
+ if res.is_empty() {
+ return None;
+ }
+
+ Some(res)
+ }
+
+ fn embed_passage(&self, passage: &str) -> Option<Self::PassageEmbedding> {
+ let res: Self::PassageEmbedding = passage
+ .split_whitespace()
+ .filter_map(|word| self.get(word).cloned())
+ .collect();
+
+ if res.is_empty() {
+ return None;
+ }
+
+ Some(res)
+ }
+
+ fn score(&self, query: &Self::QueryEmbedding, passage: &Self::PassageEmbedding) -> f32 {
+ let mut score = 0.0;
+ let mut count = 0;
+
+ for passage_vec in passage {
+ score += query.iter().map(|vec| vec.sim(passage_vec)).sum::<f32>();
+
+ count += 1;
+ }
+
+ score / count as f32
+ }
+}
+
pub struct Summarizer {
- word2vec: Word2Vec,
+ extractive_summarizer: DualEncoder,
top_n_extractive_passages: usize,
abstractive_summarizer: AbstractiveSummarizer,
}
@@ -172,7 +231,7 @@ pub struct Summarizer {
impl Summarizer {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
Ok(Self {
- word2vec: Word2Vec::open(path.as_ref().join("word2vec.bin.gz").as_path())?,
+ extractive_summarizer: DualEncoder::open(path.as_ref().join("dual_encoder").as_path())?,
top_n_extractive_passages: 10,
abstractive_summarizer: AbstractiveSummarizer {
model: Arc::new(AbstractiveModel::open(
@@ -183,14 +242,7 @@ impl Summarizer {
}
fn extractive_query_specific(&self, query: &str, text: &str) -> Option<String> {
- let query_vectors: Vec<_> = query
- .split_whitespace()
- .filter_map(|word| self.word2vec.get(word))
- .collect();
-
- if query_vectors.is_empty() {
- return None;
- }
+ let query_vectors = self.extractive_summarizer.embed_query(query)?;
let mut best_passages: BinaryHeap<Reverse<CandidatePassage<'_>>> =
BinaryHeap::with_capacity(self.top_n_extractive_passages);
@@ -198,38 +250,27 @@ impl Summarizer {
let overlap_sents = OverlappingSents::new(text, 100, 10);
for (index, (passage, range)) in overlap_sents.enumerate() {
- let mut score = 0.0;
- let mut count = 0;
-
- for passage_vec in passage
- .split_whitespace()
- .filter_map(|word| self.word2vec.get(word))
- {
- score += query_vectors
- .iter()
- .map(|vec| vec.sim(passage_vec))
- .sum::<f32>();
-
- count += 1;
- }
-
- score /= count as f32;
-
- let candidate = CandidatePassage {
- passage,
- index,
- score,
- range,
- };
+ if let Some(passage_vec) = self.extractive_summarizer.embed_passage(passage) {
+ let score = self
+ .extractive_summarizer
+ .score(&query_vectors, &passage_vec);
+
+ let candidate = CandidatePassage {
+ passage,
+ index,
+ score,
+ range,
+ };
- if best_passages.len() >= self.top_n_extractive_passages {
- if let Some(mut worst) = best_passages.peek_mut() {
- if worst.0.score < candidate.score {
- *worst = Reverse(candidate);
+ if best_passages.len() >= self.top_n_extractive_passages {
+ if let Some(mut worst) = best_passages.peek_mut() {
+ if worst.0.score < candidate.score {
+ *worst = Reverse(candidate);
+ }
}
+ } else {
+ best_passages.push(Reverse(candidate));
}
- } else {
- best_passages.push(Reverse(candidate));
}
}
@@ -285,7 +326,82 @@ impl Summarizer {
}
}
-const TRUNCATE_INPUT: usize = 1024;
+pub struct DualEncoder {
+ model: tch::CModule,
+ tokenizer: tokenizers::Tokenizer,
+}
+
+impl DualEncoder {
+ pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {
+ let truncation = TruncationParams {
+ max_length: 256,
+ ..Default::default()
+ };
+
+ let padding = PaddingParams {
+ ..Default::default()
+ };
+
+ let mut tokenizer =
+ tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))?;
+
+ tokenizer.with_truncation(Some(truncation));
+ tokenizer.with_padding(Some(padding));
+
+ let model = tch::CModule::load(folder.as_ref().join("model.pt"))?;
+ Ok(Self { model, tokenizer })
+ }
+
+ fn embed(&self, text: &str) -> Result<Tensor> {
+ let query = self.tokenizer.encode(text, false).unwrap();
+
+ let ids = query
+ .get_ids()
+ .iter()
+ .map(|&id| id as i64)
+ .collect::<Vec<_>>();
+
+ let types = query
+ .get_type_ids()
+ .iter()
+ .map(|&id| id as i64)
+ .collect::<Vec<_>>();
+
+ let mask = query
+ .get_attention_mask()
+ .iter()
+ .map(|&id| id as i64)
+ .collect::<Vec<_>>();
+
+ let ids = Tensor::of_slice(&ids).reshape(&[1, -1]);
+ let types = Tensor::of_slice(&types).reshape(&[1, -1]);
+ let mask = Tensor::of_slice(&mask).reshape(&[1, -1]);
+
+ Ok(self.model.forward_ts(&[ids, types, mask])?.squeeze())
+ }
+}
+
+impl PassageScorer for DualEncoder {
+ type QueryEmbedding = Tensor;
+
+ type PassageEmbedding = Tensor;
+
+ fn embed_query(&self, query: &str) -> Option<Self::QueryEmbedding> {
+ self.embed(query).ok()
+ }
+
+ fn embed_passage(&self, passage: &str) -> Option<Self::PassageEmbedding> {
+ self.embed(passage).ok()
+ }
+
+ fn score(&self, query: &Self::QueryEmbedding, passage: &Self::PassageEmbedding) -> f32 {
+ let score = query.dot(&passage.transpose(0, 0));
+
+ score.double_value(&[]) as f32
+ }
+}
+
+const TRUNCATE_INPUT_ABSTRACTIVE: usize = 1024;
pub struct AbstractiveModel {
encoder: tch::CModule,
decoder: tch::CModule,
@@ -300,7 +416,7 @@ pub struct AbstractiveModel {
impl AbstractiveModel {
pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {
let truncation = TruncationParams {
- max_length: TRUNCATE_INPUT,
+ max_length: TRUNCATE_INPUT_ABSTRACTIVE,
..Default::default()
};
@@ -754,7 +870,7 @@ impl Default for GenerationConfig {
banned_tokens: vec![],
force_min_tokens: None,
length_penalty: 1.0,
- num_beams: 10,
+ num_beams: 3,
}
}
}
diff --git a/frontend/src/components/SearchModal.astro b/frontend/src/components/SearchModal.astro
index 53128d33..b07e6f29 100644
--- a/frontend/src/components/SearchModal.astro
+++ b/frontend/src/components/SearchModal.astro
@@ -127,6 +127,7 @@ import Hero from "./Hero.astro";
updateSelection();
updateSummaryBtn();
}
+
function updateSummaryBtn() {
var idx = parseInt(currentIdx);
var snip = snippets[idx];
@@ -207,82 +208,49 @@ import Hero from "./Hero.astro";
"summaryShown": false,
"summary": {
"text": "",
- "shownUpto": 0,
"hasStarted": false,
}};
}
}
- function displayPreviousStreamed(summary, elem) {
- let s = ""
-
- for (i = 0; i < summary["shownUpto"]; i++) {
- s += summary["text"][i];
- }
-
- elem.innerHTML = s;
- }
-
- var previousDisplayStreams = {};
-
// called from button
function summarizeBtn() {
if (currentIdx === null) {
return;
}
- var snippetElem = document.getElementById("snippet-text-" + currentIdx);
+ var snippetElem = document.getElementById("snippet-" + currentIdx);
if (snippetElem == null) {
return;
}
+ var snippetText = snippetElem.querySelector(".snippet-text");
+ var snippetBubble = snippetElem.querySelector(".chat-bubble");
var idx = parseInt(currentIdx);
var snip = snippets[idx];
snip["summaryShown"] = !snip["summaryShown"];
if (snip["summaryShown"]) {
- if (!btnSummarize.classList.contains("active")) {
- btnSummarize.classList.add("active");
- }
- summarize(idx, snippetElem);
+ btnSummarize.classList.add("active");
+ summarize(idx, snippetBubble, snippetText);
} else {
- if (snippetElem.classList.contains("blinking-cursor")) {
- snippetElem.classList.remove("blinking-cursor");
- }
- if (btnSummarize.classList.contains("active")) {
- btnSummarize.classList.remove("active");
- }
+ snippetBubble.classList.add("hidden");
+ btnSummarize.classList.remove("active");
- snippetElem.innerHTML = snip["origSnippet"];
+ snippetText.innerHTML = snip["origSnippet"];
}
}
- function summarize(idx, snippetElem) {
- if (previousDisplayStreams[idx] != null) {
- clearInterval(previousDisplayStreams[idx]);
- }
-
+ function summarize(idx, snippetBubble, snippetText) {
var snip = snippets[idx];
- displayPreviousStreamed(snip["summary"], snippetElem);
+ snippetText.innerHTML = snip['summary']["text"];
var searchParams = new URLSearchParams(window.location.search);
var query = searchParams.get("q");
- var displayChars = setInterval(() => {
- if (snip["summary"]["shownUpto"] < snip["summary"]["text"].length) {
- if (snip["summaryShown"]) {
- snippetElem.innerHTML += snip["summary"]["text"][snip["summary"]["shownUpto"]];
- }
-
- snip["summary"]["shownUpto"] += 1;
- }
- }, 20);
-
- previousDisplayStreams[idx] = displayChars;
-
var reqData = {
'query': query,
'url': currentUrl,
@@ -290,34 +258,29 @@ import Hero from "./Hero.astro";
var queryData = new URLSearchParams(reqData).toString();
+ if (snip["summary"]["text"] == "") {
+ snippetBubble.classList.remove("hidden");
+ }
+
if (!snip["summary"]["hasStarted"]) {
snip["summary"]["hasStarted"] = true;
- snippetElem.classList.add("blinking-cursor");
- var source = new EventSource("/summarize?" + queryData);
- source.onmessage = function(event) {
- if (event.data == "") {
- setTimeout(() => {
- clearInterval(displayChars);
- snip["summary"]["shownUpto"] = snip["summary"]["text"].length;
- if (snip["summaryShown"]) {
- displayPreviousStreamed(snip["summary"], snippetElem);
- }
- }, 10000);
- snippetElem.classList.remove("blinking-cursor");
- source.close();
+
+ fetch("/summarize?" + queryData, {
+ method: "GET",
+ }).then((e) => {
+ snippetBubble.classList.add("hidden");
+
+ if (e.status != 200) {
+ console.log("summarization error", e);
+ snippetText.innerHTML = snip["origSnippet"];
return;
}
- snip["summary"]["text"] += event.data;
- }
-
- source.onerror = function(e) {
- console.log("summarization error", e);
- clearInterval(displayChars);
- snippetElem.classList.remove("blinking-cursor");
- snippetElem.innerHTML = snip["origSnippet"];
- source.close();
- }
+ return e.text();
+ }).then((text) => {
+ snip["summary"]["text"] = text;
+ snippetText.innerHTML = snip["summary"]["text"];
+ });
}
}
</script>
diff --git a/frontend/src/components/Snippet.astro b/frontend/src/components/Snippet.astro
index cc5d72d4..94bff9e3 100644
--- a/frontend/src/components/Snippet.astro
+++ b/frontend/src/components/Snippet.astro
@@ -16,9 +16,18 @@ export interface Props {
{askama.if_("let Some(date) = date", () => (
<span class="text-gray-500">{askama`date $ 2. May 2022`}</span>
))}
- <span id={"snippet-text-" + askama`loop.index0`} class="[&:nth-child(2)]:before:content-['—'] snippet-text">
- {askama`text|safe $ {{lorem.lines}}`}
- </span>
+ <div id={"snippet-" + askama`loop.index0`}>
+ <span id="snippet-text" class="[&:nth-child(2)]:before:content-['—'] snippet-text">
+ {askama`text|safe $ {{lorem.lines}}`}
+ </span>
+ <div class="chat-bubble hidden">
+ <div class="typing">
+ <div class="dot"></div>
+ <div class="dot"></div>
+ <div class="dot"></div>
+ </div>
+ </div>
+ </div>
</div>
),
"Snippet::StackOverflowQA { question, answers }": (
@@ -84,18 +93,50 @@ export interface Props {
</div>
<style>
- @keyframes cursor-blink {
- 0% {
- opacity: 0;
- }
+ .chat-bubble {
+ padding-top: 5px;
+ padding-left: 10px;
}
- .blinking-cursor::after {
- content: "";
- width: 2px;
- height: 1rem;
- background: #292d30;
- display: inline-block;
- animation: cursor-blink 1s steps(2) infinite;
+ .typing {
+ align-items: center;
+ display: flex;
+ height: 17px;
+ }
+ .typing .dot {
+ animation: mercuryTypingAnimation 1.8s infinite ease-in-out;
+ background-color: rgba(20, 105, 69, .7);
+ border-radius: 50%;
+ height: 7px;
+ margin-right: 4px;
+ vertical-align: middle;
+ width: 7px;
+ }
+ .typing .dot:nth-child(1) {
+ animation-delay: 200ms;
+ }
+ .typing .dot:nth-child(2) {
+ animation-delay: 300ms;
+ }
+ .typing .dot:nth-child(3) {
+ animation-delay: 400ms;
+ }
+ .typing .dot:last-child {
+ margin-right: 0;
+ }
+
+ @keyframes mercuryTypingAnimation {
+ 0% {
+ transform: translateY(0px);
+ background-color:rgba(20, 105, 69, .7);
+ }
+ 28% {
+ transform: translateY(-7px);
+ background-color:rgba(20, 105, 69, .4);
+ }
+ 44% {
+ transform: translateY(0px);
+ background-color: rgba(20, 105, 69, .2);
+ }
}
</style>
\ No newline at end of file
diff --git a/scripts/export_dual_encoder b/scripts/export_dual_encoder
new file mode 100755
index 00000000..0462bca8
--- /dev/null
+++ b/scripts/export_dual_encoder
@@ -0,0 +1,65 @@
+#!.venv/bin/python3
+import os
+import transformers
+import torch
+import torch.nn.functional as F
+
+# MODEL = "nreimers/MiniLM-L6-H384-uncased"
+MODEL = "sentence-transformers/all-MiniLM-L6-v2"
+
+question = "Where is amazon rain forest located"
+context = """The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of
+which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations.
+The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with
+minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain
+"Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract
+of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."""
+
+os.system("mkdir data/summarizer/dual_encoder")
+
+model = transformers.AutoModel.from_pretrained(MODEL)
+
+model.eval()
+for p in model.parameters():
+ p.requires_grad_(False)
+
+tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)
+tokenizer.save_pretrained("data/summarizer/dual_encoder")
+
+features = tokenizer(
+ [question, context],
+ padding="max_length",
+ truncation=True,
+ return_tensors="pt",
+)
+input_ids = features["input_ids"]
+attention_mask = features["attention_mask"]
+token_type_ids = features["token_type_ids"]
+
+
+def mean_pooling(model_output, attention_mask):
+ token_embeddings = model_output[
+ 0
+ ] # First element of model_output contains all token embeddings
+ input_mask_expanded = (
+ attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+ )
+ return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(
+ input_mask_expanded.sum(1), min=1e-9
+ )
+
+
+def run_model(input_ids, token_type_ids, attention_mask):
+ res = model(
+ input_ids=input_ids,
+ token_type_ids=token_type_ids,
+ attention_mask=attention_mask,
+ return_dict=False,
+ )
+ embs = mean_pooling(res, attention_mask)
+ return F.normalize(embs, p=2, dim=1)
+
+
+traced_model = torch.jit.trace(run_model, ((input_ids, token_type_ids, attention_mask)))
+
+traced_model.save("data/summarizer/dual_encoder/model.pt")