web search done right
Dual-encoder as passage scorer for extractive summarization
Mikkel Denker72d10866724c
diff --git a/core/src/api/improvement.rs b/core/src/api/improvement.rsindex 5521884f..776d3332 100644--- a/core/src/api/improvement.rs+++ b/core/src/api/improvement.rs@@ -17,7 +17,7 @@use std::sync::Arc;use askama::Template;-use axum::{extract, response::IntoResponse, Extension};+use axum::{extract, response::IntoResponse};use serde::Deserialize;use uuid::Uuid;@@ -40,7 +40,7 @@ pub struct StoreParams {#[allow(clippy::unused_async)]pub async fn click(extract::Query(params): extract::Query<ClickParams>,- Extension(state): Extension<Arc<State>>,+ extract::State(state): extract::State<Arc<State>>,) {if let Some(q) = state.improvement_queue.as_ref() {q.lock().await.push(ImprovementEvent::Click {@@ -61,7 +61,7 @@ impl From<StoreParams> for StoredQuery {#[allow(clippy::unused_async)]pub async fn store(- Extension(state): Extension<Arc<State>>,+ extract::State(state): extract::State<Arc<State>>,extract::Json(params): extract::Json<StoreParams>,) -> impl IntoResponse {match state.improvement_queue.as_ref() {diff --git a/core/src/api/summarize.rs b/core/src/api/summarize.rsindex ddb8a80d..b32532db 100644--- a/core/src/api/summarize.rs+++ b/core/src/api/summarize.rs@@ -16,7 +16,7 @@use std::sync::Arc;-use axum::{extract, Extension};+use axum::extract;use http::StatusCode;use serde::Deserialize;@@ -31,7 +31,7 @@ pub struct Params {#[allow(clippy::unused_async)]pub async fn route(extract::Query(params): extract::Query<Params>,- Extension(state): Extension<Arc<State>>,+ extract::State(state): extract::State<Arc<State>>,) -> std::result::Result<String, StatusCode> {let webpage = state.searcher@@ -39,7 +39,5 @@ pub async fn route(.await.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;- Ok(state- .summarizer- .abstractive_summary(¶ms.query, &webpage.body))+ Ok(state.summarizer.summarize(¶ms.query, &webpage.body))}diff --git a/core/src/spell/word2vec.rs b/core/src/spell/word2vec.rsindex 162f4717..20404738 100644--- a/core/src/spell/word2vec.rs+++ b/core/src/spell/word2vec.rs@@ -115,7 +115,7 @@ impl<R: BufRead> Iterator for WordVectorReader<R> {}}-#[derive(Serialize, Deserialize, Debug)]+#[derive(Serialize, Deserialize, Debug, Clone)]pub struct WordVec(Vec<f32>);impl WordVec {fn magnitude(&self) -> f32 {diff --git a/core/src/summarizer.rs b/core/src/summarizer.rsindex a3db1499..f7735918 100644--- a/core/src/summarizer.rs+++ b/core/src/summarizer.rs@@ -26,7 +26,10 @@ use itertools::{intersperse, Itertools};use tch::{IValue, Kind, Tensor};use tokenizers::{PaddingParams, TruncationParams};-use crate::{ceil_char_boundary, spell::word2vec::Word2Vec};+use crate::{+ ceil_char_boundary,+ spell::word2vec::{Word2Vec, WordVec},+};#[derive(thiserror::Error, Debug)]pub enum Error {@@ -163,8 +166,64 @@ impl<'a> Iterator for OverlappingSents<'a> {Some((res, range))}}++trait PassageScorer {+ type QueryEmbedding;+ type PassageEmbedding;++ fn embed_query(&self, query: &str) -> Option<Self::QueryEmbedding>;+ fn embed_passage(&self, passage: &str) -> Option<Self::PassageEmbedding>;++ fn score(&self, query: &Self::QueryEmbedding, passage: &Self::PassageEmbedding) -> f32;+}++impl PassageScorer for Word2Vec {+ type QueryEmbedding = Vec<WordVec>;++ type PassageEmbedding = Self::QueryEmbedding;++ fn embed_query(&self, query: &str) -> Option<Self::QueryEmbedding> {+ let res: Self::QueryEmbedding = query+ .split_whitespace()+ .filter_map(|word| self.get(word).cloned())+ .collect();++ if res.is_empty() {+ return None;+ }++ Some(res)+ }++ fn embed_passage(&self, passage: &str) -> Option<Self::PassageEmbedding> {+ let res: Self::PassageEmbedding = passage+ .split_whitespace()+ .filter_map(|word| self.get(word).cloned())+ .collect();++ if res.is_empty() {+ return None;+ }++ Some(res)+ }++ fn score(&self, query: &Self::QueryEmbedding, passage: &Self::PassageEmbedding) -> f32 {+ let mut score = 0.0;+ let mut count = 0;++ for passage_vec in passage {+ score += query.iter().map(|vec| vec.sim(passage_vec)).sum::<f32>();++ count += 1;+ }++ score / count as f32+ }+}+pub struct Summarizer {- word2vec: Word2Vec,+ extractive_summarizer: DualEncoder,top_n_extractive_passages: usize,abstractive_summarizer: AbstractiveSummarizer,}@@ -172,7 +231,7 @@ pub struct Summarizer {impl Summarizer {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {Ok(Self {- word2vec: Word2Vec::open(path.as_ref().join("word2vec.bin.gz").as_path())?,+ extractive_summarizer: DualEncoder::open(path.as_ref().join("dual_encoder").as_path())?,top_n_extractive_passages: 10,abstractive_summarizer: AbstractiveSummarizer {model: Arc::new(AbstractiveModel::open(@@ -183,14 +242,7 @@ impl Summarizer {}fn extractive_query_specific(&self, query: &str, text: &str) -> Option<String> {- let query_vectors: Vec<_> = query- .split_whitespace()- .filter_map(|word| self.word2vec.get(word))- .collect();-- if query_vectors.is_empty() {- return None;- }+ let query_vectors = self.extractive_summarizer.embed_query(query)?;let mut best_passages: BinaryHeap<Reverse<CandidatePassage<'_>>> =BinaryHeap::with_capacity(self.top_n_extractive_passages);@@ -198,38 +250,27 @@ impl Summarizer {let overlap_sents = OverlappingSents::new(text, 100, 10);for (index, (passage, range)) in overlap_sents.enumerate() {- let mut score = 0.0;- let mut count = 0;-- for passage_vec in passage- .split_whitespace()- .filter_map(|word| self.word2vec.get(word))- {- score += query_vectors- .iter()- .map(|vec| vec.sim(passage_vec))- .sum::<f32>();-- count += 1;- }-- score /= count as f32;-- let candidate = CandidatePassage {- passage,- index,- score,- range,- };+ if let Some(passage_vec) = self.extractive_summarizer.embed_passage(passage) {+ let score = self+ .extractive_summarizer+ .score(&query_vectors, &passage_vec);++ let candidate = CandidatePassage {+ passage,+ index,+ score,+ range,+ };- if best_passages.len() >= self.top_n_extractive_passages {- if let Some(mut worst) = best_passages.peek_mut() {- if worst.0.score < candidate.score {- *worst = Reverse(candidate);+ if best_passages.len() >= self.top_n_extractive_passages {+ if let Some(mut worst) = best_passages.peek_mut() {+ if worst.0.score < candidate.score {+ *worst = Reverse(candidate);+ }}+ } else {+ best_passages.push(Reverse(candidate));}- } else {- best_passages.push(Reverse(candidate));}}@@ -285,7 +326,82 @@ impl Summarizer {}}-const TRUNCATE_INPUT: usize = 1024;+pub struct DualEncoder {+ model: tch::CModule,+ tokenizer: tokenizers::Tokenizer,+}++impl DualEncoder {+ pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {+ let truncation = TruncationParams {+ max_length: 256,+ ..Default::default()+ };++ let padding = PaddingParams {+ ..Default::default()+ };++ let mut tokenizer =+ tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))?;++ tokenizer.with_truncation(Some(truncation));+ tokenizer.with_padding(Some(padding));++ let model = tch::CModule::load(folder.as_ref().join("model.pt"))?;+ Ok(Self { model, tokenizer })+ }++ fn embed(&self, text: &str) -> Result<Tensor> {+ let query = self.tokenizer.encode(text, false).unwrap();++ let ids = query+ .get_ids()+ .iter()+ .map(|&id| id as i64)+ .collect::<Vec<_>>();++ let types = query+ .get_type_ids()+ .iter()+ .map(|&id| id as i64)+ .collect::<Vec<_>>();++ let mask = query+ .get_attention_mask()+ .iter()+ .map(|&id| id as i64)+ .collect::<Vec<_>>();++ let ids = Tensor::of_slice(&ids).reshape(&[1, -1]);+ let types = Tensor::of_slice(&types).reshape(&[1, -1]);+ let mask = Tensor::of_slice(&mask).reshape(&[1, -1]);++ Ok(self.model.forward_ts(&[ids, types, mask])?.squeeze())+ }+}++impl PassageScorer for DualEncoder {+ type QueryEmbedding = Tensor;++ type PassageEmbedding = Tensor;++ fn embed_query(&self, query: &str) -> Option<Self::QueryEmbedding> {+ self.embed(query).ok()+ }++ fn embed_passage(&self, passage: &str) -> Option<Self::PassageEmbedding> {+ self.embed(passage).ok()+ }++ fn score(&self, query: &Self::QueryEmbedding, passage: &Self::PassageEmbedding) -> f32 {+ let score = query.dot(&passage.transpose(0, 0));++ score.double_value(&[]) as f32+ }+}++const TRUNCATE_INPUT_ABSTRACTIVE: usize = 1024;pub struct AbstractiveModel {encoder: tch::CModule,decoder: tch::CModule,@@ -300,7 +416,7 @@ pub struct AbstractiveModel {impl AbstractiveModel {pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {let truncation = TruncationParams {- max_length: TRUNCATE_INPUT,+ max_length: TRUNCATE_INPUT_ABSTRACTIVE,..Default::default()};@@ -754,7 +870,7 @@ impl Default for GenerationConfig {banned_tokens: vec![],force_min_tokens: None,length_penalty: 1.0,- num_beams: 10,+ num_beams: 3,}}}diff --git a/frontend/src/components/SearchModal.astro b/frontend/src/components/SearchModal.astroindex 53128d33..b07e6f29 100644--- a/frontend/src/components/SearchModal.astro+++ b/frontend/src/components/SearchModal.astro@@ -127,6 +127,7 @@ import Hero from "./Hero.astro";updateSelection();updateSummaryBtn();}+function updateSummaryBtn() {var idx = parseInt(currentIdx);var snip = snippets[idx];@@ -207,82 +208,49 @@ import Hero from "./Hero.astro";"summaryShown": false,"summary": {"text": "",- "shownUpto": 0,"hasStarted": false,}};}}- function displayPreviousStreamed(summary, elem) {- let s = ""-- for (i = 0; i < summary["shownUpto"]; i++) {- s += summary["text"][i];- }-- elem.innerHTML = s;- }-- var previousDisplayStreams = {};-// called from buttonfunction summarizeBtn() {if (currentIdx === null) {return;}- var snippetElem = document.getElementById("snippet-text-" + currentIdx);+ var snippetElem = document.getElementById("snippet-" + currentIdx);if (snippetElem == null) {return;}+ var snippetText = snippetElem.querySelector(".snippet-text");+ var snippetBubble = snippetElem.querySelector(".chat-bubble");var idx = parseInt(currentIdx);var snip = snippets[idx];snip["summaryShown"] = !snip["summaryShown"];if (snip["summaryShown"]) {- if (!btnSummarize.classList.contains("active")) {- btnSummarize.classList.add("active");- }- summarize(idx, snippetElem);+ btnSummarize.classList.add("active");+ summarize(idx, snippetBubble, snippetText);} else {- if (snippetElem.classList.contains("blinking-cursor")) {- snippetElem.classList.remove("blinking-cursor");- }- if (btnSummarize.classList.contains("active")) {- btnSummarize.classList.remove("active");- }+ snippetBubble.classList.add("hidden");+ btnSummarize.classList.remove("active");- snippetElem.innerHTML = snip["origSnippet"];+ snippetText.innerHTML = snip["origSnippet"];}}- function summarize(idx, snippetElem) {- if (previousDisplayStreams[idx] != null) {- clearInterval(previousDisplayStreams[idx]);- }-+ function summarize(idx, snippetBubble, snippetText) {var snip = snippets[idx];- displayPreviousStreamed(snip["summary"], snippetElem);+ snippetText.innerHTML = snip['summary']["text"];var searchParams = new URLSearchParams(window.location.search);var query = searchParams.get("q");- var displayChars = setInterval(() => {- if (snip["summary"]["shownUpto"] < snip["summary"]["text"].length) {- if (snip["summaryShown"]) {- snippetElem.innerHTML += snip["summary"]["text"][snip["summary"]["shownUpto"]];- }-- snip["summary"]["shownUpto"] += 1;- }- }, 20);-- previousDisplayStreams[idx] = displayChars;-var reqData = {'query': query,'url': currentUrl,@@ -290,34 +258,29 @@ import Hero from "./Hero.astro";var queryData = new URLSearchParams(reqData).toString();+ if (snip["summary"]["text"] == "") {+ snippetBubble.classList.remove("hidden");+ }+if (!snip["summary"]["hasStarted"]) {snip["summary"]["hasStarted"] = true;- snippetElem.classList.add("blinking-cursor");- var source = new EventSource("/summarize?" + queryData);- source.onmessage = function(event) {- if (event.data == "") {- setTimeout(() => {- clearInterval(displayChars);- snip["summary"]["shownUpto"] = snip["summary"]["text"].length;- if (snip["summaryShown"]) {- displayPreviousStreamed(snip["summary"], snippetElem);- }- }, 10000);- snippetElem.classList.remove("blinking-cursor");- source.close();++ fetch("/summarize?" + queryData, {+ method: "GET",+ }).then((e) => {+ snippetBubble.classList.add("hidden");++ if (e.status != 200) {+ console.log("summarization error", e);+ snippetText.innerHTML = snip["origSnippet"];return;}- snip["summary"]["text"] += event.data;- }-- source.onerror = function(e) {- console.log("summarization error", e);- clearInterval(displayChars);- snippetElem.classList.remove("blinking-cursor");- snippetElem.innerHTML = snip["origSnippet"];- source.close();- }+ return e.text();+ }).then((text) => {+ snip["summary"]["text"] = text;+ snippetText.innerHTML = snip["summary"]["text"];+ });}}</script>diff --git a/frontend/src/components/Snippet.astro b/frontend/src/components/Snippet.astroindex cc5d72d4..94bff9e3 100644--- a/frontend/src/components/Snippet.astro+++ b/frontend/src/components/Snippet.astro@@ -16,9 +16,18 @@ export interface Props {{askama.if_("let Some(date) = date", () => (<span class="text-gray-500">{askama`date $ 2. May 2022`}</span>))}- <span id={"snippet-text-" + askama`loop.index0`} class="[&:nth-child(2)]:before:content-['—'] snippet-text">- {askama`text|safe $ {{lorem.lines}}`}- </span>+ <div id={"snippet-" + askama`loop.index0`}>+ <span id="snippet-text" class="[&:nth-child(2)]:before:content-['—'] snippet-text">+ {askama`text|safe $ {{lorem.lines}}`}+ </span>+ <div class="chat-bubble hidden">+ <div class="typing">+ <div class="dot"></div>+ <div class="dot"></div>+ <div class="dot"></div>+ </div>+ </div>+ </div></div>),"Snippet::StackOverflowQA { question, answers }": (@@ -84,18 +93,50 @@ export interface Props {</div><style>- @keyframes cursor-blink {- 0% {- opacity: 0;- }+ .chat-bubble {+ padding-top: 5px;+ padding-left: 10px;}- .blinking-cursor::after {- content: "";- width: 2px;- height: 1rem;- background: #292d30;- display: inline-block;- animation: cursor-blink 1s steps(2) infinite;+ .typing {+ align-items: center;+ display: flex;+ height: 17px;+ }+ .typing .dot {+ animation: mercuryTypingAnimation 1.8s infinite ease-in-out;+ background-color: rgba(20, 105, 69, .7);+ border-radius: 50%;+ height: 7px;+ margin-right: 4px;+ vertical-align: middle;+ width: 7px;+ }+ .typing .dot:nth-child(1) {+ animation-delay: 200ms;+ }+ .typing .dot:nth-child(2) {+ animation-delay: 300ms;+ }+ .typing .dot:nth-child(3) {+ animation-delay: 400ms;+ }+ .typing .dot:last-child {+ margin-right: 0;+ }++ @keyframes mercuryTypingAnimation {+ 0% {+ transform: translateY(0px);+ background-color:rgba(20, 105, 69, .7);+ }+ 28% {+ transform: translateY(-7px);+ background-color:rgba(20, 105, 69, .4);+ }+ 44% {+ transform: translateY(0px);+ background-color: rgba(20, 105, 69, .2);+ }}</style>\ No newline at end of filediff --git a/scripts/export_dual_encoder b/scripts/export_dual_encodernew file mode 100755index 00000000..0462bca8--- /dev/null+++ b/scripts/export_dual_encoder@@ -0,0 +1,65 @@+#!.venv/bin/python3+import os+import transformers+import torch+import torch.nn.functional as F++# MODEL = "nreimers/MiniLM-L6-H384-uncased"+MODEL = "sentence-transformers/all-MiniLM-L6-v2"++question = "Where is amazon rain forest located"+context = """The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of+which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations.+The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with+minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain+"Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract+of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."""++os.system("mkdir data/summarizer/dual_encoder")++model = transformers.AutoModel.from_pretrained(MODEL)++model.eval()+for p in model.parameters():+ p.requires_grad_(False)++tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)+tokenizer.save_pretrained("data/summarizer/dual_encoder")++features = tokenizer(+ [question, context],+ padding="max_length",+ truncation=True,+ return_tensors="pt",+)+input_ids = features["input_ids"]+attention_mask = features["attention_mask"]+token_type_ids = features["token_type_ids"]+++def mean_pooling(model_output, attention_mask):+ token_embeddings = model_output[+ 0+ ] # First element of model_output contains all token embeddings+ input_mask_expanded = (+ attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()+ )+ return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(+ input_mask_expanded.sum(1), min=1e-9+ )+++def run_model(input_ids, token_type_ids, attention_mask):+ res = model(+ input_ids=input_ids,+ token_type_ids=token_type_ids,+ attention_mask=attention_mask,+ return_dict=False,+ )+ embs = mean_pooling(res, attention_mask)+ return F.normalize(embs, p=2, dim=1)+++traced_model = torch.jit.trace(run_model, ((input_ids, token_type_ids, attention_mask)))++traced_model.save("data/summarizer/dual_encoder/model.pt")