web search done right
spell correction only highlight corrected part
Mikkel Denkere221eb1292b7
diff --git a/frontend/src/layouts/Layout.astro b/frontend/src/layouts/Layout.astroindex 360fca54..61488bdf 100644--- a/frontend/src/layouts/Layout.astro+++ b/frontend/src/layouts/Layout.astro@@ -65,10 +65,6 @@ const { title } = Astro.props;border-color: #666;}- b {- font-weight: 500 !important;- }-pre > code {font-size: 0.8rem;}diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astroindex daa2494c..a03eb0e5 100644--- a/frontend/src/pages/search.astro+++ b/frontend/src/pages/search.astro@@ -67,10 +67,10 @@ import { askama } from "../askama";<div>Did you mean:{" "}<a- class="title-link font-bold italic"- href="/search?q={{ correction|urlencode }}"+ class="title-link font-medium"+ href="/search?q={{ correction.raw|urlencode }}">- {askama`correction $ {{lorem.sentence}}`}+ {askama`correction.highlighted $ {{lorem.sentence}}`}</a></div>))diff --git a/src/frontend/autosuggest.rs b/src/frontend/autosuggest.rsindex 9d59e5c5..3ca4434e 100644--- a/src/frontend/autosuggest.rs+++ b/src/frontend/autosuggest.rs@@ -21,6 +21,9 @@ use serde::Serialize;use super::State;+const HIGHLIGHTED_PREFIX: &str = "<b style=\"font-weight: 500;\">";+const HIGHLIGHTED_POSTFIX: &str = "</b>";+fn highlight(query: &str, suggestion: &str) -> String {let idx = suggestion.chars()@@ -29,9 +32,9 @@ fn highlight(query: &str, suggestion: &str) -> String {.unwrap_or(query.len());let mut new_suggestion: String = suggestion.chars().take(idx).collect();- new_suggestion += "<b>";+ new_suggestion += HIGHLIGHTED_PREFIX;new_suggestion += suggestion.chars().skip(idx).collect::<String>().as_str();- new_suggestion += "</b>";+ new_suggestion += HIGHLIGHTED_POSTFIX;new_suggestion}@@ -81,10 +84,25 @@ mod tests {#[test]fn suffix_highlight() {- assert_eq!(&highlight("", "test"), "<b>test</b>");- assert_eq!(&highlight("t", "test"), "t<b>est</b>");- assert_eq!(&highlight("te", "test"), "te<b>st</b>");- assert_eq!(&highlight("tes", "test"), "tes<b>t</b>");- assert_eq!(&highlight("test", "test"), "test<b></b>");+ assert_eq!(+ highlight("", "test"),+ format!("{HIGHLIGHTED_PREFIX}test{HIGHLIGHTED_POSTFIX}")+ );+ assert_eq!(+ highlight("t", "test"),+ format!("t{HIGHLIGHTED_PREFIX}est{HIGHLIGHTED_POSTFIX}")+ );+ assert_eq!(+ highlight("te", "test"),+ format!("te{HIGHLIGHTED_PREFIX}st{HIGHLIGHTED_POSTFIX}")+ );+ assert_eq!(+ highlight("tes", "test"),+ format!("tes{HIGHLIGHTED_PREFIX}t{HIGHLIGHTED_POSTFIX}")+ );+ assert_eq!(+ highlight("test", "test"),+ format!("test{HIGHLIGHTED_PREFIX}{HIGHLIGHTED_POSTFIX}")+ );}}diff --git a/src/frontend/search.rs b/src/frontend/search.rsindex a2a95a2f..cebf5e35 100644--- a/src/frontend/search.rs+++ b/src/frontend/search.rs@@ -20,7 +20,9 @@ use axum::Extension;use crate::{ranking::site_rankings::SiteRankings,- search_prettifier::{thousand_sep_number, DisplayedEntity, DisplayedWebpage},+ search_prettifier::{+ thousand_sep_number, DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection,+ },searcher::{self, PrettifiedSearchResult, SearchQuery},webpage::region::{Region, ALL_REGIONS},};@@ -41,7 +43,7 @@ struct SearchTemplate {search_result: Vec<DisplayedWebpage>,query: String,entity: Option<DisplayedEntity>,- spell_correction: Option<String>,+ spell_correction: Option<HighlightedSpellCorrection>,num_matches: String,search_duration_sec: String,all_regions: Vec<RegionSelection>,diff --git a/src/index.rs b/src/index.rsindex 22b9a2ad..844a45b4 100644--- a/src/index.rs+++ b/src/index.rs@@ -20,7 +20,6 @@ use std::path::Path;use std::sync::Arc;use std::time::Duration;-use itertools::intersperse;use serde::{Deserialize, Serialize};use tantivy::collector::Collector;use tantivy::schema::Schema;@@ -32,7 +31,9 @@ use crate::image_downloader::{ImageDownloadJob, ImageDownloader};use crate::image_store::{FaviconStore, Image, ImageStore, PrimaryImageStore};use crate::inverted_index::{self, InitialSearchResult, InvertedIndex, SearchResult};use crate::query::Query;-use crate::spell::{Dictionary, LogarithmicEdit, SpellChecker, TermSplitter};+use crate::spell::{+ Correction, CorrectionTerm, Dictionary, LogarithmicEdit, SpellChecker, TermSplitter,+};use crate::subdomain_count::SubdomainCounter;use crate::webpage::region::{Region, RegionCount};use crate::webpage::{Url, Webpage};@@ -222,58 +223,68 @@ impl Index {.download(&mut self.primary_image_store);}- fn spell_check(&self, terms: &[String]) -> Option<String> {+ fn spell_check(&self, terms: &[String]) -> Option<Correction> {let spellchecker = SpellChecker::new(&self.spell_dictionary, LogarithmicEdit::new(4));- let mut corrections: Vec<String> = Vec::new();++ let mut original = String::new();++ for term in terms {+ original.push_str(term);+ original.push(' ');+ }+ original = original.trim_end().to_string();++ let mut possible_correction = Correction::empty(original);for term in terms {match spellchecker.correct(term.to_ascii_lowercase().as_str()) {- Some(correction) => corrections.push(correction.to_ascii_lowercase()),- None => corrections.push(term.to_ascii_lowercase()),+ Some(correction) => possible_correction+ .push(CorrectionTerm::Corrected(correction.to_ascii_lowercase())),+ None => possible_correction+ .push(CorrectionTerm::NotCorrected(term.to_ascii_lowercase())),}}- if corrections- .iter()- .cloned()- .zip(terms.iter().map(|term| term.to_ascii_lowercase()))- .all(|(correction, term)| correction == term)- {+ if possible_correction.is_all_orig() {None} else {- Some(intersperse(corrections.into_iter(), " ".to_string()).collect())+ Some(possible_correction)}}- fn split_words(&self, terms: &[String]) -> Option<String> {+ fn split_words(&self, terms: &[String]) -> Option<Correction> {let splitter = TermSplitter::new(&self.spell_dictionary);- let mut corrections: Vec<String> = Vec::new();++ let mut original = String::new();++ for term in terms {+ original.push_str(term);+ original.push(' ');+ }+ original = original.trim_end().to_string();++ let mut possible_correction = Correction::empty(original);for term in terms {let t = term.to_ascii_lowercase();let split = splitter.split(t.as_str());if split.is_empty() {- corrections.push(t);+ possible_correction.push(CorrectionTerm::NotCorrected(t));} else {for s in split {- corrections.push(s.to_string())+ possible_correction.push(CorrectionTerm::Corrected(s.to_string()))}}}- if corrections- .iter()- .map(|s| s.to_ascii_lowercase())- .zip(terms.iter().map(|term| term.to_ascii_lowercase()))- .all(|(correction, term)| correction == term)- {+ if possible_correction.is_all_orig() {None} else {- Some(intersperse(corrections.into_iter(), " ".to_string()).collect())+ Some(possible_correction)}}- pub fn spell_correction(&self, terms: &[String]) -> Option<String> {+ pub fn spell_correction(&self, terms: &[String]) -> Option<Correction> {self.spell_check(terms).or_else(|| self.split_words(terms))}@@ -380,8 +391,8 @@ mod tests {assert_eq!(result.documents[0].url, "https://www.example.com");assert_eq!(- index.spell_correction(&["thiss".to_string()]),- Some("this".to_string())+ String::from(index.spell_correction(&["thiss".to_string()]).unwrap()),+ "this".to_string());}@@ -410,8 +421,12 @@ mod tests {index.commit().unwrap();assert_eq!(- index.spell_correction(&["th".to_string(), "best".to_string()]),- Some("the best".to_string())+ String::from(+ index+ .spell_correction(&["th".to_string(), "best".to_string()])+ .unwrap()+ ),+ "the best".to_string());assert_eq!(index.spell_correction(&["the".to_string(), "best".to_string()]),diff --git a/src/kv/mod.rs b/src/kv/mod.rsindex 3844bc37..1311325e 100644--- a/src/kv/mod.rs+++ b/src/kv/mod.rs@@ -78,7 +78,7 @@ wherefn flush(&self) {}fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a> {- todo!();+ unimplemented!()}fn get(&self, key: &K) -> Option<V> {diff --git a/src/search_prettifier.rs b/src/search_prettifier.rsindex af287bc3..72525148 100644--- a/src/search_prettifier.rs+++ b/src/search_prettifier.rs@@ -24,6 +24,7 @@ use crate::{entity_index::{entity::Span, StoredEntity},inverted_index::{self, RetrievedWebpage},searcher::{self, LocalSearcher},+ spell::CorrectionTerm,webpage::Url,};@@ -35,8 +36,34 @@ pub fn initial(.entity.map(|entity| DisplayedEntity::from(entity, local_searcher));+ let spell_corrected_query = result.spell_corrected_query.map(|correction| {+ let mut highlighted = String::new();+ let mut raw = String::new();++ for term in correction.terms {+ match term {+ CorrectionTerm::Corrected(correction) => {+ highlighted.push_str(&("<b><i>".to_string() + &correction + "</i></b>"));+ raw.push_str(&correction);+ }+ CorrectionTerm::NotCorrected(orig) => {+ highlighted.push_str(&orig);+ raw.push_str(&orig);+ }+ }++ raw.push(' ');+ highlighted.push(' ');+ }++ raw = raw.trim_end().to_string();+ highlighted = highlighted.trim_end().to_string();++ HighlightedSpellCorrection { raw, highlighted }+ });+InitialWebsiteResult {- spell_corrected_query: result.spell_corrected_query,+ spell_corrected_query,websites: result.websites,entity,}@@ -65,9 +92,15 @@ pub fn retrieve(result: Vec<RetrievedWebpage>, searcher: &LocalSearcher) -> Vec<.collect()}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct HighlightedSpellCorrection {+ pub raw: String,+ pub highlighted: String,+}+#[derive(Debug, Serialize, Deserialize)]pub struct InitialWebsiteResult {- pub spell_corrected_query: Option<String>,+ pub spell_corrected_query: Option<HighlightedSpellCorrection>,pub websites: inverted_index::InitialSearchResult,pub entity: Option<DisplayedEntity>,}diff --git a/src/searcher/local.rs b/src/searcher/local.rsindex 191a5712..736d6b09 100644--- a/src/searcher/local.rs+++ b/src/searcher/local.rs@@ -28,6 +28,7 @@ use crate::query::Query;use crate::ranking::centrality_store::CentralityStore;use crate::ranking::goggles::{self, Goggle};use crate::ranking::{Ranker, SignalAggregator};+use crate::spell::Correction;use crate::webpage::region::Region;use crate::webpage::Url;use crate::{inverted_index, Error, Result};@@ -234,7 +235,7 @@ impl LocalSearcher {#[derive(Debug, Serialize, Deserialize)]pub struct InitialWebsiteResult {- pub spell_corrected_query: Option<String>,+ pub spell_corrected_query: Option<Correction>,pub websites: inverted_index::InitialSearchResult,pub entity: Option<StoredEntity>,}diff --git a/src/searcher/mod.rs b/src/searcher/mod.rsindex 5244134f..731be3bc 100644--- a/src/searcher/mod.rs+++ b/src/searcher/mod.rs@@ -26,7 +26,8 @@ use crate::{entity_index::StoredEntity,inverted_index,ranking::site_rankings::SiteRankings,- search_prettifier::{self, DisplayedEntity, DisplayedWebpage},+ search_prettifier::{self, DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection},+ spell::Correction,webpage::region::Region,};@@ -34,7 +35,7 @@ pub const NUM_RESULTS_PER_PAGE: usize = 20;#[derive(Debug, Serialize)]pub struct WebsitesResult {- pub spell_corrected_query: Option<String>,+ pub spell_corrected_query: Option<Correction>,pub webpages: inverted_index::SearchResult,pub entity: Option<StoredEntity>,pub search_duration_ms: u128,@@ -54,7 +55,7 @@ pub enum PrettifiedSearchResult {#[derive(Debug, Serialize)]pub struct PrettifiedWebsitesResult {- pub spell_corrected_query: Option<String>,+ pub spell_corrected_query: Option<HighlightedSpellCorrection>,pub webpages: Vec<DisplayedWebpage>,pub num_docs: usize,pub entity: Option<DisplayedEntity>,diff --git a/src/snippet.rs b/src/snippet.rsindex b197a821..0d04f1b4 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -79,7 +79,7 @@ struct Snippet {highlighted: Vec<Range<usize>>,}-const HIGHLIGHTEN_PREFIX: &str = "<b>";+const HIGHLIGHTEN_PREFIX: &str = "<b style=\"font-weight: 500;\">";const HIGHLIGHTEN_POSTFIX: &str = "</b>";impl Snippet {@@ -381,7 +381,7 @@ Survey in 2016, 2017, and 2018."#;assert_eq!(result.webpages.num_docs, 1);assert_eq!(result.webpages.documents.len(), 1);- assert_eq!(result.webpages.documents[0].snippet, "<b>Rust</b> is a systems programming <b>language</b> sponsored by Mozilla which describes it as a \"safe, concurrent, practical <b>language</b>\", supporting functional and imperative-procedural paradigms. <b>Rust</b> is syntactically similar to C++[according to whom?], but its designers intend it to provide...".to_string());+ assert_eq!(result.webpages.documents[0].snippet, format!("{HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is a systems programming {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX} sponsored by Mozilla which describes it as a \"safe, concurrent, practical {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX}\", supporting functional and imperative-procedural paradigms. {HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is syntactically similar to C++[according to whom?], but its designers intend it to provide..."));}#[test]@@ -424,7 +424,7 @@ Survey in 2016, 2017, and 2018."#;assert_eq!(result.webpages.num_docs, 1);assert_eq!(result.webpages.documents.len(), 1);- assert_eq!(result.webpages.documents[0].snippet, "Rust is a systems programming language sponsored by Mozilla which <b>describes</b> it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide...".to_string());+ assert_eq!(result.webpages.documents[0].snippet, format!("Rust is a systems programming language sponsored by Mozilla which {HIGHLIGHTEN_PREFIX}describes{HIGHLIGHTEN_POSTFIX} it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide..."));}#[test]@@ -448,8 +448,8 @@ Survey in 2016, 2017, and 2018."#;);assert_eq!(snippet.to_html(),- "<b>Rust</b> is a systems programming <b>language</b> sponsored by\nMozilla which \- describes it as a \"safe"+ format!("{HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is a systems programming {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX} sponsored by\nMozilla which \+ describes it as a \"safe"))}@@ -467,7 +467,10 @@ Survey in 2016, 2017, and 2018."#;assert_eq!(first.stop_offset, 17);}let snippet = select_best_fragment_combination(&fragments[..], TEST_TEXT);- assert_eq!(snippet.to_html(), "<b>Rust</b> is a systems")+ assert_eq!(+ snippet.to_html(),+ format!("{HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is a systems")+ )}{let terms = btreemap! {@@ -482,7 +485,10 @@ Survey in 2016, 2017, and 2018."#;assert_eq!(first.stop_offset, 17);}let snippet = select_best_fragment_combination(&fragments[..], TEST_TEXT);- assert_eq!(snippet.to_html(), "programming <b>language</b>")+ assert_eq!(+ snippet.to_html(),+ format!("programming {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX}")+ )}}@@ -505,7 +511,10 @@ Survey in 2016, 2017, and 2018."#;let snippet = select_best_fragment_combination(&fragments[..], text);assert_eq!(snippet.fragment, "c d");- assert_eq!(snippet.to_html(), "<b>c</b> d");+ assert_eq!(+ snippet.to_html(),+ format!("{HIGHLIGHTEN_PREFIX}c{HIGHLIGHTEN_POSTFIX} d")+ );}#[test]@@ -527,7 +536,10 @@ Survey in 2016, 2017, and 2018."#;let snippet = select_best_fragment_combination(&fragments[..], text);assert_eq!(snippet.fragment, "e f");- assert_eq!(snippet.to_html(), "e <b>f</b>");+ assert_eq!(+ snippet.to_html(),+ format!("e {HIGHLIGHTEN_PREFIX}f{HIGHLIGHTEN_POSTFIX}")+ );}#[test]@@ -550,7 +562,10 @@ Survey in 2016, 2017, and 2018."#;let snippet = select_best_fragment_combination(&fragments[..], text);assert_eq!(snippet.fragment, "e f g");- assert_eq!(snippet.to_html(), "e <b>f</b> g");+ assert_eq!(+ snippet.to_html(),+ format!("e {HIGHLIGHTEN_PREFIX}f{HIGHLIGHTEN_POSTFIX} g")+ );}#[test]@@ -624,7 +639,7 @@ Survey in 2016, 2017, and 2018."#;assert_eq!(result.webpages.documents.len(), 1);assert_eq!(result.webpages.documents[0].snippet,- "Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural <b>paradigms</b>. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide...".to_string()+ format!("Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural {HIGHLIGHTEN_PREFIX}paradigms{HIGHLIGHTEN_POSTFIX}. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide..."));}}diff --git a/src/spell/mod.rs b/src/spell/mod.rsindex 3cdf2aec..b0c47857 100644--- a/src/spell/mod.rs+++ b/src/spell/mod.rs@@ -18,6 +18,53 @@ pub mod distance;pub mod spell_checker;pub mod splitter;+use itertools::intersperse;+use serde::{Deserialize, Serialize};+pub use self::dictionary::{Dictionary, DictionaryResult, EditStrategy, LogarithmicEdit};pub use self::spell_checker::SpellChecker;pub use self::splitter::TermSplitter;++#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]+pub struct Correction {+ original: String,+ pub terms: Vec<CorrectionTerm>,+}++#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]+pub enum CorrectionTerm {+ Corrected(String),+ NotCorrected(String),+}++impl From<Correction> for String {+ fn from(correction: Correction) -> Self {+ intersperse(+ correction.terms.into_iter().map(|term| match term {+ CorrectionTerm::Corrected(correction) => correction,+ CorrectionTerm::NotCorrected(orig) => orig,+ }),+ " ".to_string(),+ )+ .collect()+ }+}++impl Correction {+ pub fn empty(original: String) -> Self {+ Self {+ original,+ terms: Vec::new(),+ }+ }++ pub fn push(&mut self, term: CorrectionTerm) {+ self.terms.push(term);+ }++ pub fn is_all_orig(&self) -> bool {+ self.terms+ .iter()+ .all(|term| matches!(term, CorrectionTerm::NotCorrected(_)))+ }+}