Skip to content

web search done right

spell correction only highlight corrected part

Mikkel Denkere221eb1292b7
diff --git a/frontend/src/layouts/Layout.astro b/frontend/src/layouts/Layout.astro
index 360fca54..61488bdf 100644
--- a/frontend/src/layouts/Layout.astro
+++ b/frontend/src/layouts/Layout.astro
@@ -65,10 +65,6 @@ const { title } = Astro.props;
border-color: #666;
}
- b {
- font-weight: 500 !important;
- }
-
pre > code {
font-size: 0.8rem;
}
diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astro
index daa2494c..a03eb0e5 100644
--- a/frontend/src/pages/search.astro
+++ b/frontend/src/pages/search.astro
@@ -67,10 +67,10 @@ import { askama } from "../askama";
<div>
Did you mean:{" "}
<a
- class="title-link font-bold italic"
- href="/search?q={{ correction|urlencode }}"
+ class="title-link font-medium"
+ href="/search?q={{ correction.raw|urlencode }}"
>
- {askama`correction $ {{lorem.sentence}}`}
+ {askama`correction.highlighted $ {{lorem.sentence}}`}
</a>
</div>
))
diff --git a/src/frontend/autosuggest.rs b/src/frontend/autosuggest.rs
index 9d59e5c5..3ca4434e 100644
--- a/src/frontend/autosuggest.rs
+++ b/src/frontend/autosuggest.rs
@@ -21,6 +21,9 @@ use serde::Serialize;
use super::State;
+const HIGHLIGHTED_PREFIX: &str = "<b style=\"font-weight: 500;\">";
+const HIGHLIGHTED_POSTFIX: &str = "</b>";
+
fn highlight(query: &str, suggestion: &str) -> String {
let idx = suggestion
.chars()
@@ -29,9 +32,9 @@ fn highlight(query: &str, suggestion: &str) -> String {
.unwrap_or(query.len());
let mut new_suggestion: String = suggestion.chars().take(idx).collect();
- new_suggestion += "<b>";
+ new_suggestion += HIGHLIGHTED_PREFIX;
new_suggestion += suggestion.chars().skip(idx).collect::<String>().as_str();
- new_suggestion += "</b>";
+ new_suggestion += HIGHLIGHTED_POSTFIX;
new_suggestion
}
@@ -81,10 +84,25 @@ mod tests {
#[test]
fn suffix_highlight() {
- assert_eq!(&highlight("", "test"), "<b>test</b>");
- assert_eq!(&highlight("t", "test"), "t<b>est</b>");
- assert_eq!(&highlight("te", "test"), "te<b>st</b>");
- assert_eq!(&highlight("tes", "test"), "tes<b>t</b>");
- assert_eq!(&highlight("test", "test"), "test<b></b>");
+ assert_eq!(
+ highlight("", "test"),
+ format!("{HIGHLIGHTED_PREFIX}test{HIGHLIGHTED_POSTFIX}")
+ );
+ assert_eq!(
+ highlight("t", "test"),
+ format!("t{HIGHLIGHTED_PREFIX}est{HIGHLIGHTED_POSTFIX}")
+ );
+ assert_eq!(
+ highlight("te", "test"),
+ format!("te{HIGHLIGHTED_PREFIX}st{HIGHLIGHTED_POSTFIX}")
+ );
+ assert_eq!(
+ highlight("tes", "test"),
+ format!("tes{HIGHLIGHTED_PREFIX}t{HIGHLIGHTED_POSTFIX}")
+ );
+ assert_eq!(
+ highlight("test", "test"),
+ format!("test{HIGHLIGHTED_PREFIX}{HIGHLIGHTED_POSTFIX}")
+ );
}
}
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index a2a95a2f..cebf5e35 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -20,7 +20,9 @@ use axum::Extension;
use crate::{
ranking::site_rankings::SiteRankings,
- search_prettifier::{thousand_sep_number, DisplayedEntity, DisplayedWebpage},
+ search_prettifier::{
+ thousand_sep_number, DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection,
+ },
searcher::{self, PrettifiedSearchResult, SearchQuery},
webpage::region::{Region, ALL_REGIONS},
};
@@ -41,7 +43,7 @@ struct SearchTemplate {
search_result: Vec<DisplayedWebpage>,
query: String,
entity: Option<DisplayedEntity>,
- spell_correction: Option<String>,
+ spell_correction: Option<HighlightedSpellCorrection>,
num_matches: String,
search_duration_sec: String,
all_regions: Vec<RegionSelection>,
diff --git a/src/index.rs b/src/index.rs
index 22b9a2ad..844a45b4 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -20,7 +20,6 @@ use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
-use itertools::intersperse;
use serde::{Deserialize, Serialize};
use tantivy::collector::Collector;
use tantivy::schema::Schema;
@@ -32,7 +31,9 @@ use crate::image_downloader::{ImageDownloadJob, ImageDownloader};
use crate::image_store::{FaviconStore, Image, ImageStore, PrimaryImageStore};
use crate::inverted_index::{self, InitialSearchResult, InvertedIndex, SearchResult};
use crate::query::Query;
-use crate::spell::{Dictionary, LogarithmicEdit, SpellChecker, TermSplitter};
+use crate::spell::{
+ Correction, CorrectionTerm, Dictionary, LogarithmicEdit, SpellChecker, TermSplitter,
+};
use crate::subdomain_count::SubdomainCounter;
use crate::webpage::region::{Region, RegionCount};
use crate::webpage::{Url, Webpage};
@@ -222,58 +223,68 @@ impl Index {
.download(&mut self.primary_image_store);
}
- fn spell_check(&self, terms: &[String]) -> Option<String> {
+ fn spell_check(&self, terms: &[String]) -> Option<Correction> {
let spellchecker = SpellChecker::new(&self.spell_dictionary, LogarithmicEdit::new(4));
- let mut corrections: Vec<String> = Vec::new();
+
+ let mut original = String::new();
+
+ for term in terms {
+ original.push_str(term);
+ original.push(' ');
+ }
+ original = original.trim_end().to_string();
+
+ let mut possible_correction = Correction::empty(original);
for term in terms {
match spellchecker.correct(term.to_ascii_lowercase().as_str()) {
- Some(correction) => corrections.push(correction.to_ascii_lowercase()),
- None => corrections.push(term.to_ascii_lowercase()),
+ Some(correction) => possible_correction
+ .push(CorrectionTerm::Corrected(correction.to_ascii_lowercase())),
+ None => possible_correction
+ .push(CorrectionTerm::NotCorrected(term.to_ascii_lowercase())),
}
}
- if corrections
- .iter()
- .cloned()
- .zip(terms.iter().map(|term| term.to_ascii_lowercase()))
- .all(|(correction, term)| correction == term)
- {
+ if possible_correction.is_all_orig() {
None
} else {
- Some(intersperse(corrections.into_iter(), " ".to_string()).collect())
+ Some(possible_correction)
}
}
- fn split_words(&self, terms: &[String]) -> Option<String> {
+ fn split_words(&self, terms: &[String]) -> Option<Correction> {
let splitter = TermSplitter::new(&self.spell_dictionary);
- let mut corrections: Vec<String> = Vec::new();
+
+ let mut original = String::new();
+
+ for term in terms {
+ original.push_str(term);
+ original.push(' ');
+ }
+ original = original.trim_end().to_string();
+
+ let mut possible_correction = Correction::empty(original);
for term in terms {
let t = term.to_ascii_lowercase();
let split = splitter.split(t.as_str());
if split.is_empty() {
- corrections.push(t);
+ possible_correction.push(CorrectionTerm::NotCorrected(t));
} else {
for s in split {
- corrections.push(s.to_string())
+ possible_correction.push(CorrectionTerm::Corrected(s.to_string()))
}
}
}
- if corrections
- .iter()
- .map(|s| s.to_ascii_lowercase())
- .zip(terms.iter().map(|term| term.to_ascii_lowercase()))
- .all(|(correction, term)| correction == term)
- {
+ if possible_correction.is_all_orig() {
None
} else {
- Some(intersperse(corrections.into_iter(), " ".to_string()).collect())
+ Some(possible_correction)
}
}
- pub fn spell_correction(&self, terms: &[String]) -> Option<String> {
+ pub fn spell_correction(&self, terms: &[String]) -> Option<Correction> {
self.spell_check(terms).or_else(|| self.split_words(terms))
}
@@ -380,8 +391,8 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.example.com");
assert_eq!(
- index.spell_correction(&["thiss".to_string()]),
- Some("this".to_string())
+ String::from(index.spell_correction(&["thiss".to_string()]).unwrap()),
+ "this".to_string()
);
}
@@ -410,8 +421,12 @@ mod tests {
index.commit().unwrap();
assert_eq!(
- index.spell_correction(&["th".to_string(), "best".to_string()]),
- Some("the best".to_string())
+ String::from(
+ index
+ .spell_correction(&["th".to_string(), "best".to_string()])
+ .unwrap()
+ ),
+ "the best".to_string()
);
assert_eq!(
index.spell_correction(&["the".to_string(), "best".to_string()]),
diff --git a/src/kv/mod.rs b/src/kv/mod.rs
index 3844bc37..1311325e 100644
--- a/src/kv/mod.rs
+++ b/src/kv/mod.rs
@@ -78,7 +78,7 @@ where
fn flush(&self) {}
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a> {
- todo!();
+ unimplemented!()
}
fn get(&self, key: &K) -> Option<V> {
diff --git a/src/search_prettifier.rs b/src/search_prettifier.rs
index af287bc3..72525148 100644
--- a/src/search_prettifier.rs
+++ b/src/search_prettifier.rs
@@ -24,6 +24,7 @@ use crate::{
entity_index::{entity::Span, StoredEntity},
inverted_index::{self, RetrievedWebpage},
searcher::{self, LocalSearcher},
+ spell::CorrectionTerm,
webpage::Url,
};
@@ -35,8 +36,34 @@ pub fn initial(
.entity
.map(|entity| DisplayedEntity::from(entity, local_searcher));
+ let spell_corrected_query = result.spell_corrected_query.map(|correction| {
+ let mut highlighted = String::new();
+ let mut raw = String::new();
+
+ for term in correction.terms {
+ match term {
+ CorrectionTerm::Corrected(correction) => {
+ highlighted.push_str(&("<b><i>".to_string() + &correction + "</i></b>"));
+ raw.push_str(&correction);
+ }
+ CorrectionTerm::NotCorrected(orig) => {
+ highlighted.push_str(&orig);
+ raw.push_str(&orig);
+ }
+ }
+
+ raw.push(' ');
+ highlighted.push(' ');
+ }
+
+ raw = raw.trim_end().to_string();
+ highlighted = highlighted.trim_end().to_string();
+
+ HighlightedSpellCorrection { raw, highlighted }
+ });
+
InitialWebsiteResult {
- spell_corrected_query: result.spell_corrected_query,
+ spell_corrected_query,
websites: result.websites,
entity,
}
@@ -65,9 +92,15 @@ pub fn retrieve(result: Vec<RetrievedWebpage>, searcher: &LocalSearcher) -> Vec<
.collect()
}
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct HighlightedSpellCorrection {
+ pub raw: String,
+ pub highlighted: String,
+}
+
#[derive(Debug, Serialize, Deserialize)]
pub struct InitialWebsiteResult {
- pub spell_corrected_query: Option<String>,
+ pub spell_corrected_query: Option<HighlightedSpellCorrection>,
pub websites: inverted_index::InitialSearchResult,
pub entity: Option<DisplayedEntity>,
}
diff --git a/src/searcher/local.rs b/src/searcher/local.rs
index 191a5712..736d6b09 100644
--- a/src/searcher/local.rs
+++ b/src/searcher/local.rs
@@ -28,6 +28,7 @@ use crate::query::Query;
use crate::ranking::centrality_store::CentralityStore;
use crate::ranking::goggles::{self, Goggle};
use crate::ranking::{Ranker, SignalAggregator};
+use crate::spell::Correction;
use crate::webpage::region::Region;
use crate::webpage::Url;
use crate::{inverted_index, Error, Result};
@@ -234,7 +235,7 @@ impl LocalSearcher {
#[derive(Debug, Serialize, Deserialize)]
pub struct InitialWebsiteResult {
- pub spell_corrected_query: Option<String>,
+ pub spell_corrected_query: Option<Correction>,
pub websites: inverted_index::InitialSearchResult,
pub entity: Option<StoredEntity>,
}
diff --git a/src/searcher/mod.rs b/src/searcher/mod.rs
index 5244134f..731be3bc 100644
--- a/src/searcher/mod.rs
+++ b/src/searcher/mod.rs
@@ -26,7 +26,8 @@ use crate::{
entity_index::StoredEntity,
inverted_index,
ranking::site_rankings::SiteRankings,
- search_prettifier::{self, DisplayedEntity, DisplayedWebpage},
+ search_prettifier::{self, DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection},
+ spell::Correction,
webpage::region::Region,
};
@@ -34,7 +35,7 @@ pub const NUM_RESULTS_PER_PAGE: usize = 20;
#[derive(Debug, Serialize)]
pub struct WebsitesResult {
- pub spell_corrected_query: Option<String>,
+ pub spell_corrected_query: Option<Correction>,
pub webpages: inverted_index::SearchResult,
pub entity: Option<StoredEntity>,
pub search_duration_ms: u128,
@@ -54,7 +55,7 @@ pub enum PrettifiedSearchResult {
#[derive(Debug, Serialize)]
pub struct PrettifiedWebsitesResult {
- pub spell_corrected_query: Option<String>,
+ pub spell_corrected_query: Option<HighlightedSpellCorrection>,
pub webpages: Vec<DisplayedWebpage>,
pub num_docs: usize,
pub entity: Option<DisplayedEntity>,
diff --git a/src/snippet.rs b/src/snippet.rs
index b197a821..0d04f1b4 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -79,7 +79,7 @@ struct Snippet {
highlighted: Vec<Range<usize>>,
}
-const HIGHLIGHTEN_PREFIX: &str = "<b>";
+const HIGHLIGHTEN_PREFIX: &str = "<b style=\"font-weight: 500;\">";
const HIGHLIGHTEN_POSTFIX: &str = "</b>";
impl Snippet {
@@ -381,7 +381,7 @@ Survey in 2016, 2017, and 2018."#;
assert_eq!(result.webpages.num_docs, 1);
assert_eq!(result.webpages.documents.len(), 1);
- assert_eq!(result.webpages.documents[0].snippet, "<b>Rust</b> is a systems programming <b>language</b> sponsored by Mozilla which describes it as a \"safe, concurrent, practical <b>language</b>\", supporting functional and imperative-procedural paradigms. <b>Rust</b> is syntactically similar to C++[according to whom?], but its designers intend it to provide...".to_string());
+ assert_eq!(result.webpages.documents[0].snippet, format!("{HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is a systems programming {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX} sponsored by Mozilla which describes it as a \"safe, concurrent, practical {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX}\", supporting functional and imperative-procedural paradigms. {HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is syntactically similar to C++[according to whom?], but its designers intend it to provide..."));
}
#[test]
@@ -424,7 +424,7 @@ Survey in 2016, 2017, and 2018."#;
assert_eq!(result.webpages.num_docs, 1);
assert_eq!(result.webpages.documents.len(), 1);
- assert_eq!(result.webpages.documents[0].snippet, "Rust is a systems programming language sponsored by Mozilla which <b>describes</b> it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide...".to_string());
+ assert_eq!(result.webpages.documents[0].snippet, format!("Rust is a systems programming language sponsored by Mozilla which {HIGHLIGHTEN_PREFIX}describes{HIGHLIGHTEN_POSTFIX} it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide..."));
}
#[test]
@@ -448,8 +448,8 @@ Survey in 2016, 2017, and 2018."#;
);
assert_eq!(
snippet.to_html(),
- "<b>Rust</b> is a systems programming <b>language</b> sponsored by\nMozilla which \
- describes it as a \"safe"
+ format!("{HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is a systems programming {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX} sponsored by\nMozilla which \
+ describes it as a \"safe")
)
}
@@ -467,7 +467,10 @@ Survey in 2016, 2017, and 2018."#;
assert_eq!(first.stop_offset, 17);
}
let snippet = select_best_fragment_combination(&fragments[..], TEST_TEXT);
- assert_eq!(snippet.to_html(), "<b>Rust</b> is a systems")
+ assert_eq!(
+ snippet.to_html(),
+ format!("{HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is a systems")
+ )
}
{
let terms = btreemap! {
@@ -482,7 +485,10 @@ Survey in 2016, 2017, and 2018."#;
assert_eq!(first.stop_offset, 17);
}
let snippet = select_best_fragment_combination(&fragments[..], TEST_TEXT);
- assert_eq!(snippet.to_html(), "programming <b>language</b>")
+ assert_eq!(
+ snippet.to_html(),
+ format!("programming {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX}")
+ )
}
}
@@ -505,7 +511,10 @@ Survey in 2016, 2017, and 2018."#;
let snippet = select_best_fragment_combination(&fragments[..], text);
assert_eq!(snippet.fragment, "c d");
- assert_eq!(snippet.to_html(), "<b>c</b> d");
+ assert_eq!(
+ snippet.to_html(),
+ format!("{HIGHLIGHTEN_PREFIX}c{HIGHLIGHTEN_POSTFIX} d")
+ );
}
#[test]
@@ -527,7 +536,10 @@ Survey in 2016, 2017, and 2018."#;
let snippet = select_best_fragment_combination(&fragments[..], text);
assert_eq!(snippet.fragment, "e f");
- assert_eq!(snippet.to_html(), "e <b>f</b>");
+ assert_eq!(
+ snippet.to_html(),
+ format!("e {HIGHLIGHTEN_PREFIX}f{HIGHLIGHTEN_POSTFIX}")
+ );
}
#[test]
@@ -550,7 +562,10 @@ Survey in 2016, 2017, and 2018."#;
let snippet = select_best_fragment_combination(&fragments[..], text);
assert_eq!(snippet.fragment, "e f g");
- assert_eq!(snippet.to_html(), "e <b>f</b> g");
+ assert_eq!(
+ snippet.to_html(),
+ format!("e {HIGHLIGHTEN_PREFIX}f{HIGHLIGHTEN_POSTFIX} g")
+ );
}
#[test]
@@ -624,7 +639,7 @@ Survey in 2016, 2017, and 2018."#;
assert_eq!(result.webpages.documents.len(), 1);
assert_eq!(
result.webpages.documents[0].snippet,
- "Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural <b>paradigms</b>. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide...".to_string()
+ format!("Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural {HIGHLIGHTEN_PREFIX}paradigms{HIGHLIGHTEN_POSTFIX}. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide...")
);
}
}
diff --git a/src/spell/mod.rs b/src/spell/mod.rs
index 3cdf2aec..b0c47857 100644
--- a/src/spell/mod.rs
+++ b/src/spell/mod.rs
@@ -18,6 +18,53 @@ pub mod distance;
pub mod spell_checker;
pub mod splitter;
+use itertools::intersperse;
+use serde::{Deserialize, Serialize};
+
pub use self::dictionary::{Dictionary, DictionaryResult, EditStrategy, LogarithmicEdit};
pub use self::spell_checker::SpellChecker;
pub use self::splitter::TermSplitter;
+
+#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]
+pub struct Correction {
+ original: String,
+ pub terms: Vec<CorrectionTerm>,
+}
+
+#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]
+pub enum CorrectionTerm {
+ Corrected(String),
+ NotCorrected(String),
+}
+
+impl From<Correction> for String {
+ fn from(correction: Correction) -> Self {
+ intersperse(
+ correction.terms.into_iter().map(|term| match term {
+ CorrectionTerm::Corrected(correction) => correction,
+ CorrectionTerm::NotCorrected(orig) => orig,
+ }),
+ " ".to_string(),
+ )
+ .collect()
+ }
+}
+
+impl Correction {
+ pub fn empty(original: String) -> Self {
+ Self {
+ original,
+ terms: Vec::new(),
+ }
+ }
+
+ pub fn push(&mut self, term: CorrectionTerm) {
+ self.terms.push(term);
+ }
+
+ pub fn is_all_orig(&self) -> bool {
+ self.terms
+ .iter()
+ .all(|term| matches!(term, CorrectionTerm::NotCorrected(_)))
+ }
+}