web search done right
take bigrams and trigrams into account during spell correction and as separate bm25 fields
Mikkel Denker4b7b3043064c
diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 4b3fc5de..c0d6ff3e 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -108,4 +108,8 @@ harness = false[[bench]]name = "build-similarity"+harness = false++[[bench]]+name = "spell-correction"harness = false\ No newline at end of filediff --git a/core/benches/spell-correction.rs b/core/benches/spell-correction.rsnew file mode 100644index 00000000..e2b6a65b--- /dev/null+++ b/core/benches/spell-correction.rs@@ -0,0 +1,34 @@+use criterion::{criterion_group, criterion_main, Criterion};+use stract::{index::Index, searcher::SearchQuery, spell::Spell};++const INDEX_PATH: &str = "data/index";++macro_rules! bench {+ ($query:tt, $spell:ident, $c:ident) => {+ let mut desc = "correct '".to_string();+ desc.push_str($query);+ desc.push('\'');+ $c.bench_function(desc.as_str(), |b| {+ b.iter(|| {+ $spell+ .correction(&SearchQuery {+ query: $query.to_string(),+ ..Default::default()+ })+ .unwrap()+ })+ });+ };+}++pub fn criterion_benchmark(c: &mut Criterion) {+ let index = Index::open(INDEX_PATH).unwrap();+ let spell = Spell::for_index(&index);++ for _ in 0..100 {+ bench!("asdf", spell, c);+ }+}++criterion_group!(benches, criterion_benchmark);+criterion_main!(benches);diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rsindex e919f8a0..c61ddecf 100644--- a/core/examples/print_inbound_links.rs+++ b/core/examples/print_inbound_links.rs@@ -23,10 +23,10 @@ pub fn main() {let graph = WebgraphBuilder::new("data/webgraph").open();let inbound = InboundSimilarity::open("data/centrality/inbound_similarity").unwrap();- for host in ["archwaypublishing.com", "wordpressfoundation.org"] {+ for host in ["apache.org", "yyhelyhalvin.weebly.com"] {println!("{host}:");- let node = Node::from(host);+ let node = Node::from(host).into_host();let node_id = graph.node2id(&node).unwrap();let inbound_vec = inbound.get(node_id).unwrap();println!("{:?}", inbound_vec);diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex 9bfc4207..c37e2b5a 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -16,6 +16,8 @@use std::net::SocketAddr;+use tracing::info;+use crate::{cluster::{member::{Member, Service},@@ -32,7 +34,6 @@ use crate::{pub async fn run(config: SearchServerConfig) -> Result<()> {let addr: SocketAddr = config.host;let server = sonic::Server::bind(addr).await.unwrap();- tracing::info!("listening on {}", addr);let entity_index = config.entity_index_path@@ -66,6 +67,8 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {).await?;+ info!("search server is ready to accept requests on {}", addr);+loop {if let Ok(req) = server.accept::<searcher::distributed::Request>().await {match &req.body {diff --git a/core/src/index.rs b/core/src/index.rsindex b9d734ce..835c740f 100644--- a/core/src/index.rs+++ b/core/src/index.rs@@ -33,9 +33,6 @@ use crate::image_store::{FaviconStore, Image, ImageStore, PrimaryImageStore};use crate::inverted_index::{self, InvertedIndex};use crate::query::Query;use crate::search_ctx::Ctx;-use crate::spell::{- Correction, CorrectionTerm, Dictionary, LogarithmicEdit, SpellChecker, TermSplitter,-};use crate::subdomain_count::SubdomainCounter;use crate::webgraph::NodeID;use crate::webpage::region::{Region, RegionCount};@@ -45,7 +42,6 @@ use crate::Result;const INVERTED_INDEX_SUBFOLDER_NAME: &str = "inverted_index";const FAVICON_STORE_SUBFOLDER_NAME: &str = "favicon_store";const PRIMARY_IMAGE_STORE_SUBFOLDER_NAME: &str = "primary_image_store";-const SPELL_SUBFOLDER_NAME: &str = "primary_image_store";const REGION_COUNT_FILE_NAME: &str = "region_count.json";const SUBDOMAIN_COUNT_SUBFOLDER_NAME: &str = "subdomain_count";const IMAGE_WEBPAGE_CENTRALITY_THRESHOLD: f64 = 0.0;@@ -56,7 +52,6 @@ pub struct Index {primary_image_store: PrimaryImageStore,favicon_downloader: ImageDownloader<String>,primary_image_downloader: ImageDownloader<Uuid>,- spell_dictionary: Dictionary<100_000>,pub region_count: RegionCount,pub subdomain_counter: SubdomainCounter,pub path: String,@@ -84,7 +79,6 @@ impl Index {region_count,primary_image_downloader: ImageDownloader::new(),favicon_downloader: ImageDownloader::new(),- spell_dictionary: Dictionary::open(Some(path.as_ref().join(SPELL_SUBFOLDER_NAME)))?,subdomain_counter: SubdomainCounter::open(path.as_ref().join(SUBDOMAIN_COUNT_SUBFOLDER_NAME),),@@ -105,7 +99,6 @@ impl Index {pub fn insert(&mut self, mut webpage: Webpage) -> Result<()> {self.maybe_insert_favicon(&webpage);self.maybe_insert_primary_image(&mut webpage);- self.spell_dictionary.insert_page(&webpage);self.subdomain_counter.increment(webpage.html.url().clone());if let Ok(region) = Region::guess_from(&webpage) {@@ -116,7 +109,6 @@ impl Index {}pub fn commit(&mut self) -> Result<()> {- self.spell_dictionary.commit()?;self.inverted_index.commit()?;self.region_count.commit();self.subdomain_counter.commit();@@ -170,8 +162,6 @@ impl Index {self.primary_image_store.merge(other.primary_image_store);drop(self.primary_image_store);- self.spell_dictionary.merge(other.spell_dictionary);-self.region_count.merge(other.region_count);self.subdomain_counter.merge(other.subdomain_counter);@@ -242,71 +232,6 @@ impl Index {.download(&mut self.primary_image_store);}- fn spell_check(&self, terms: &[String]) -> Option<Correction> {- let spellchecker = SpellChecker::new(&self.spell_dictionary, LogarithmicEdit::new(4));-- let mut original = String::new();-- for term in terms {- original.push_str(term);- original.push(' ');- }- original = original.trim_end().to_string();-- let mut possible_correction = Correction::empty(original);-- for term in terms {- match spellchecker.correct(term.to_ascii_lowercase().as_str()) {- Some(correction) => possible_correction- .push(CorrectionTerm::Corrected(correction.to_ascii_lowercase())),- None => possible_correction- .push(CorrectionTerm::NotCorrected(term.to_ascii_lowercase())),- }- }-- if possible_correction.is_all_orig() {- None- } else {- Some(possible_correction)- }- }-- fn split_words(&self, terms: &[String]) -> Option<Correction> {- let splitter = TermSplitter::new(&self.spell_dictionary);-- let mut original = String::new();-- for term in terms {- original.push_str(term);- original.push(' ');- }- original = original.trim_end().to_string();-- let mut possible_correction = Correction::empty(original);-- for term in terms {- let t = term.to_ascii_lowercase();- let split = splitter.split(t.as_str());- if split.is_empty() {- possible_correction.push(CorrectionTerm::NotCorrected(t));- } else {- for s in split {- possible_correction.push(CorrectionTerm::Corrected(s.to_string()))- }- }- }-- if possible_correction.is_all_orig() {- None- } else {- Some(possible_correction)- }- }-- pub fn spell_correction(&self, terms: &[String]) -> Option<Correction> {- self.spell_check(terms).or_else(|| self.split_words(terms))- }-pub fn num_segments(&self) -> usize {self.inverted_index.num_segments()}@@ -406,42 +331,4 @@ mod tests {assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.example.com");}-- #[test]- fn sentence_spell_correction() {- let mut index = Index::temporary().expect("Unable to open index");-- index- .insert(Webpage::new(- &format!(- r#"- <html>- <head>- <title>Test website</title>- </head>- <body>- {CONTENT}- </body>- </html>- "#- ),- "https://www.example.com",- ))- .expect("failed to insert webpage");-- index.commit().unwrap();-- assert_eq!(- String::from(- index- .spell_correction(&["th".to_string(), "best".to_string()])- .unwrap()- ),- "the best".to_string()- );- assert_eq!(- index.spell_correction(&["the".to_string(), "best".to_string()]),- None- );- }}diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex d60acb4b..a42de483 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -34,7 +34,7 @@ use crate::ranking::SignalAggregator;use crate::schema::{FastField, Field, TextField, ALL_FIELDS};use crate::search_ctx::Ctx;use crate::snippet;-use crate::tokenizer::Identity;+use crate::tokenizer::{BigramTokenizer, Identity, TrigramTokenizer};use crate::webgraph::NodeID;use crate::webpage::region::Region;use crate::webpage::{schema_org, StoredPrimaryImage, Webpage};@@ -133,12 +133,22 @@ impl InvertedIndex {.tokenizers().register(tokenizer.as_str(), tokenizer);+ let tokenizer = Tokenizer::Bigram(BigramTokenizer::default());+ tantivy_index+ .tokenizers()+ .register(tokenizer.as_str(), tokenizer);++ let tokenizer = Tokenizer::Trigram(TrigramTokenizer::default());+ tantivy_index+ .tokenizers()+ .register(tokenizer.as_str(), tokenizer);+let writer = tantivy_index.writer_with_num_threads(1, 1_000_000_000)?;let merge_policy = NoMergePolicy::default();writer.set_merge_policy(Box::new(merge_policy));- let reader = tantivy_index.reader_builder().try_into()?;+ let reader: IndexReader = tantivy_index.reader_builder().try_into()?;Ok(InvertedIndex {writer,diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex e2487be7..541a4e96 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -33,6 +33,7 @@ use std::sync::Arc;use tantivy::fieldnorm::FieldNormReader;use tantivy::postings::SegmentPostings;use tantivy::query::Scorer;+use tantivy::tokenizer::Tokenizer;use thiserror::Error;use tantivy::DocSet;@@ -56,7 +57,11 @@ pub enum Error {pub enum Signal {Bm25,Bm25Title,+ Bm25TitleBigrams,+ Bm25TitleTrigrams,Bm25CleanBody,+ Bm25CleanBodyBigrams,+ Bm25CleanBodyTrigrams,Bm25StemmedTitle,Bm25StemmedCleanBody,Bm25AllBody,@@ -92,10 +97,14 @@ impl From<Signal> for usize {}}-pub const ALL_SIGNALS: [Signal; 30] = [+pub const ALL_SIGNALS: [Signal; 34] = [Signal::Bm25,Signal::Bm25Title,+ Signal::Bm25TitleBigrams,+ Signal::Bm25TitleTrigrams,Signal::Bm25CleanBody,+ Signal::Bm25CleanBodyBigrams,+ Signal::Bm25CleanBodyTrigrams,Signal::Bm25StemmedTitle,Signal::Bm25StemmedCleanBody,Signal::Bm25AllBody,@@ -169,7 +178,11 @@ impl Signal {match self {Signal::Bm25 => 1.0,Signal::Bm25Title => 15.0,+ Signal::Bm25TitleBigrams => 15.0,+ Signal::Bm25TitleTrigrams => 15.0,Signal::Bm25CleanBody => 4.0,+ Signal::Bm25CleanBodyBigrams => 4.0,+ Signal::Bm25CleanBodyTrigrams => 4.0,Signal::Bm25StemmedTitle => 0.5,Signal::Bm25StemmedCleanBody => 0.5,Signal::Bm25AllBody => 1.0,@@ -310,7 +323,11 @@ impl Signal {.sum()}),Signal::Bm25Title+ | Signal::Bm25TitleBigrams+ | Signal::Bm25TitleTrigrams| Signal::Bm25CleanBody+ | Signal::Bm25CleanBodyBigrams+ | Signal::Bm25CleanBodyTrigrams| Signal::Bm25StemmedTitle| Signal::Bm25StemmedCleanBody| Signal::Bm25AllBody@@ -382,7 +399,11 @@ impl Signal {}Signal::Bm25| Signal::Bm25Title+ | Signal::Bm25TitleBigrams+ | Signal::Bm25TitleTrigrams| Signal::Bm25CleanBody+ | Signal::Bm25CleanBodyBigrams+ | Signal::Bm25CleanBodyTrigrams| Signal::Bm25StemmedTitle| Signal::Bm25StemmedCleanBody| Signal::Bm25AllBody@@ -431,7 +452,11 @@ impl Signal {fn as_textfield(&self) -> Option<TextField> {match self {Signal::Bm25Title => Some(TextField::Title),+ Signal::Bm25TitleBigrams => Some(TextField::TitleBigrams),+ Signal::Bm25TitleTrigrams => Some(TextField::TitleTrigrams),Signal::Bm25CleanBody => Some(TextField::CleanBody),+ Signal::Bm25CleanBodyBigrams => Some(TextField::CleanBodyBigrams),+ Signal::Bm25CleanBodyTrigrams => Some(TextField::CleanBodyTrigrams),Signal::Bm25StemmedTitle => Some(TextField::StemmedTitle),Signal::Bm25StemmedCleanBody => Some(TextField::StemmedCleanBody),Signal::Bm25AllBody => Some(TextField::AllBody),@@ -459,7 +484,11 @@ impl FromStr for Signal {match name {"bm25" => Ok(Signal::Bm25),"bm25_title" => Ok(Signal::Bm25Title),+ "bm25_title_bigrams" => Ok(Signal::Bm25TitleBigrams),+ "bm25_title_trigrams" => Ok(Signal::Bm25TitleTrigrams),"bm25_clean_body" => Ok(Signal::Bm25CleanBody),+ "bm25_clean_body_bigrams" => Ok(Signal::Bm25CleanBodyBigrams),+ "bm25_clean_body_trigrams" => Ok(Signal::Bm25CleanBodyTrigrams),"bm25_stemmed_title" => Ok(Signal::Bm25StemmedTitle),"bm25_stemmed_clean_body" => Ok(Signal::Bm25StemmedCleanBody),"bm25_all_body" => Ok(Signal::Bm25AllBody),@@ -651,11 +680,23 @@ impl SignalAggregator {for signal in ALL_SIGNALS {if let Some(text_field) = signal.as_textfield() {let tv_field = schema.get_field(text_field.name()).unwrap();- let terms: Vec<_> = query- .simple_terms()- .iter()- .map(|text| tantivy::Term::from_field_text(tv_field, text))- .collect();+ let simple_query = itertools::intersperse(+ query.simple_terms().iter().map(|s| s.as_str()),+ " ",+ )+ .collect::<String>();++ let mut terms = Vec::new();+ let mut stream = text_field.tokenizer().token_stream(&simple_query);++ while let Some(token) = stream.next() {+ let term = tantivy::Term::from_field_text(tv_field, &token.text);+ terms.push(term);+ }++ if terms.is_empty() {+ continue;+ }let weight = Bm25Weight::for_terms(tv_searcher, &terms)?;diff --git a/core/src/schema.rs b/core/src/schema.rsindex f9eff207..815d41c7 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -18,7 +18,7 @@ use tantivy::schema::{BytesOptions, Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,};-use crate::tokenizer::{Identity, JsonField, Tokenizer};+use crate::tokenizer::{BigramTokenizer, Identity, JsonField, Tokenizer, TrigramTokenizer};pub const FLOAT_SCALING: u64 = 1_000_000_000;@@ -47,6 +47,10 @@ pub enum TextField {DmozDescription,SchemaOrgJson,FlattenedSchemaOrgJson,+ CleanBodyBigrams,+ TitleBigrams,+ CleanBodyTrigrams,+ TitleTrigrams,}impl From<TextField> for usize {@@ -78,6 +82,10 @@ impl TextField {TextField::DmozDescription => Tokenizer::default(),TextField::SchemaOrgJson => Tokenizer::Identity(Identity {}),TextField::FlattenedSchemaOrgJson => Tokenizer::Json(JsonField),+ TextField::CleanBodyBigrams => Tokenizer::Bigram(BigramTokenizer::default()),+ TextField::TitleBigrams => Tokenizer::Bigram(BigramTokenizer::default()),+ TextField::CleanBodyTrigrams => Tokenizer::Trigram(TrigramTokenizer::default()),+ TextField::TitleTrigrams => Tokenizer::Trigram(TrigramTokenizer::default()),}}@@ -111,6 +119,10 @@ impl TextField {TextField::DmozDescription => true,TextField::SchemaOrgJson => false,TextField::FlattenedSchemaOrgJson => true,+ TextField::CleanBodyBigrams => false,+ TextField::TitleBigrams => false,+ TextField::CleanBodyTrigrams => false,+ TextField::TitleTrigrams => false,}}@@ -136,6 +148,10 @@ impl TextField {TextField::DmozDescription => "dmoz_description",TextField::SchemaOrgJson => "schema_org_json",TextField::FlattenedSchemaOrgJson => "flattened_schema_org_json",+ TextField::CleanBodyBigrams => "clean_body_bigrams",+ TextField::TitleBigrams => "title_bigrams",+ TextField::CleanBodyTrigrams => "clean_body_trigrams",+ TextField::TitleTrigrams => "title_trigrams",}}}@@ -219,7 +235,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 48] = [+pub static ALL_FIELDS: [Field; 52] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -240,6 +256,10 @@ pub static ALL_FIELDS: [Field; 48] = [Field::Text(TextField::DmozDescription),Field::Text(TextField::SchemaOrgJson),Field::Text(TextField::FlattenedSchemaOrgJson),+ Field::Text(TextField::CleanBodyBigrams),+ Field::Text(TextField::TitleBigrams),+ Field::Text(TextField::CleanBodyTrigrams),+ Field::Text(TextField::TitleTrigrams),// FAST FIELDSField::Fast(FastField::IsHomepage),Field::Fast(FastField::HostCentrality),@@ -344,6 +364,18 @@ impl Field {Field::Text(TextField::FlattenedSchemaOrgJson) => {IndexingOption::Text(self.default_text_options())}+ Field::Text(TextField::CleanBodyBigrams) => {+ IndexingOption::Text(self.default_text_options())+ }+ Field::Text(TextField::TitleBigrams) => {+ IndexingOption::Text(self.default_text_options())+ }+ Field::Text(TextField::CleanBodyTrigrams) => {+ IndexingOption::Text(self.default_text_options())+ }+ Field::Text(TextField::TitleTrigrams) => {+ IndexingOption::Text(self.default_text_options())+ }Field::Fast(FastField::IsHomepage) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue)@@ -484,6 +516,10 @@ impl Field {| Field::Text(TextField::BacklinkText)| Field::Text(TextField::HostTopic)| Field::Text(TextField::SchemaOrgJson)+ | Field::Text(TextField::CleanBodyBigrams)+ | Field::Text(TextField::TitleBigrams)+ | Field::Text(TextField::CleanBodyTrigrams)+ | Field::Text(TextField::TitleTrigrams)) && !self.is_fast()}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex c8d117d8..da4dcc4f 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -22,15 +22,14 @@ use crate::entity_index::{EntityIndex, StoredEntity};use crate::image_store::Image;use crate::index::Index;use crate::inverted_index::RetrievedWebpage;-use crate::query::parser::Term;-use crate::query::{self, Query};+use crate::query::Query;use crate::ranking::centrality_store::SearchCentralityStore;use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};use crate::schema::TextField;use crate::search_ctx::Ctx;use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};-use crate::spell::Correction;+use crate::spell::Spell;use crate::webgraph::centrality::topic::TopicCentrality;use crate::webgraph::Node;use crate::webpage::region::Region;@@ -42,6 +41,7 @@ use super::{InitialWebsiteResult, SearchQuery};pub struct LocalSearcher {index: Index,+ spell: Spell,entity_index: Option<EntityIndex>,centrality_store: Option<SearchCentralityStore>,topic_centrality: Option<TopicCentrality>,@@ -61,8 +61,10 @@ struct InvertedIndexResult {impl LocalSearcher {pub fn new(index: Index) -> Self {+ let spell = Spell::for_index(&index);LocalSearcher {index,+ spell,entity_index: None,centrality_store: None,topic_centrality: None,@@ -226,18 +228,6 @@ impl LocalSearcher {})}- fn spell_correction(&self, query: &SearchQuery) -> Option<Correction> {- let terms: Vec<_> = query::parser::parse(&query.query)- .into_iter()- .filter_map(|term| match *term {- Term::Simple(s) => Some(s),- _ => None,- })- .collect();-- self.index.spell_correction(&terms)- }-fn entity_sidebar(&self, query: &SearchQuery) -> Option<StoredEntity> {self.entity_index.as_ref()@@ -251,7 +241,7 @@ impl LocalSearcher {) -> Result<InitialWebsiteResult> {let ctx = self.index.inverted_index.local_search_ctx();let inverted_index_result = self.search_inverted_index(&ctx, query, de_rank_similar)?;- let correction = self.spell_correction(query);+ let correction = self.spell.correction(query);let sidebar = self.entity_sidebar(query).map(|entity| DisplayedEntity::from(entity, self));@@ -438,4 +428,50 @@ mod tests {}}}++ #[test]+ fn sentence_spell_correction() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever+ </body>+ </html>+ "#+ ,+ "https://www.example.com",+ ))+ .expect("failed to insert webpage");++ index.commit().unwrap();++ let searcher = LocalSearcher::new(index);++ assert_eq!(+ String::from(+ searcher+ .spell+ .correction(&SearchQuery {+ query: "th best".to_string(),+ ..Default::default()+ })+ .unwrap()+ ),+ "the best".to_string()+ );+ assert_eq!(+ searcher.spell.correction(&SearchQuery {+ query: "the best".to_string(),+ ..Default::default()+ }),+ None+ );+ }}diff --git a/core/src/spell/dictionary.rs b/core/src/spell/dictionary.rsindex a035fd1e..b3e55b9e 100644--- a/core/src/spell/dictionary.rs+++ b/core/src/spell/dictionary.rs@@ -14,18 +14,12 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::spell::distance::LevenshteinDistance;-use crate::tokenizer::Normal;-use crate::webpage::Webpage;-use fst::map::Union;-use fst::{Automaton, IntoStreamer, Map, MapBuilder, Streamer};-use memmap::Mmap;-use std::collections::{BTreeMap, BinaryHeap, HashSet};-use std::fs::{self, File, OpenOptions};+use std::cmp::Reverse;+use std::collections::{BTreeMap, BinaryHeap};use std::hash::{Hash, Hasher};-use std::ops::AddAssign;-use std::path::Path;-use std::{cmp, io, mem};-use tantivy::tokenizer::Tokenizer;+use std::ops::Deref;+use std::sync::Arc;+use std::{cmp, io};use thiserror::Error;pub trait EditStrategy: Send + Sync {@@ -110,370 +104,331 @@ pub enum DictionaryError {pub type Result<T> = std::result::Result<T, DictionaryError>;-enum InnerMap {- Memory(Map<Vec<u8>>),- File(Map<Mmap>),-}+pub type TermId = u64;-impl InnerMap {- fn total_freq(&self) -> u64 {- let mut stream = match self {- InnerMap::Memory(map) => map.into_stream(),- InnerMap::File(map) => map.into_stream(),- };+#[derive(PartialEq, Eq, PartialOrd, Ord)]+struct Monogram(TermId);- let mut total_freq = 0;- while let Some((_, freq)) = stream.next() {- total_freq += freq;- }+#[derive(PartialEq, Eq, PartialOrd, Ord)]+struct Bigram(TermId, TermId);- total_freq- }-}+#[derive(PartialEq, Eq, PartialOrd, Ord)]+struct Trigram(TermId, TermId, TermId);-/// Dictionary that contains term frequency information-pub struct Dictionary<const TOP_N: usize> {- cache: BTreeMap<String, u64>,- map: InnerMap,- folder_path: Option<String>,- total_freq: u64,-}+type HeavyMonogram = String;+type HeavyBigram = (String, String);+type HeavyTrigram = (String, String, String);-#[cfg(test)]-impl Default for Dictionary<1_000> {- fn default() -> Self {- Dictionary::open::<&str>(None).unwrap()- }+pub struct DictionaryBuilder {+ monograms: BinaryHeap<Reverse<(u64, HeavyMonogram)>>,+ bigrams: BinaryHeap<Reverse<(u64, HeavyBigram)>>,+ trigrams: BinaryHeap<Reverse<(u64, HeavyTrigram)>>,+ top_n: usize,}-impl<const TOP_N: usize> Dictionary<TOP_N> {- pub fn open<P: AsRef<Path>>(folder_path: Option<P>) -> Result<Self> {- if let Some(path) = folder_path.as_ref() {- fs::create_dir_all(path)?;+impl DictionaryBuilder {+ pub fn new(top_n: usize) -> Self {+ Self {+ monograms: BinaryHeap::new(),+ bigrams: BinaryHeap::new(),+ trigrams: BinaryHeap::new(),+ top_n,}-- let folder_path = folder_path.map(|path| path.as_ref().to_str().unwrap().to_string());-- let map = match &folder_path {- Some(path) => {- let dictionary_path = Path::new(path).join("dictionary");-- if !dictionary_path.exists() {- let file = File::create(dictionary_path.clone())?;- let builder = MapBuilder::new(io::BufWriter::new(file))?;- builder.finish()?;- }-- let dictionary_file = OpenOptions::new().read(true).open(dictionary_path)?;-- let mmap = unsafe { Mmap::map(&dictionary_file)? };- InnerMap::File(Map::new(mmap)?)- }- None => InnerMap::Memory(Map::default()),- };-- Ok(Dictionary {- total_freq: map.total_freq(),- map,- folder_path,- cache: BTreeMap::default(),- })}- fn store_union(&mut self, mut union: Union) -> Result<()> {- match &self.folder_path {- Some(path) => {- let path = Path::new(path);- let wrt = io::BufWriter::new(File::create(path.join("new_dictionary"))?);- let mut builder = MapBuilder::new(wrt)?;-- let mut heap = BinaryHeap::with_capacity(TOP_N + 1);-- while let Some((key, values)) = union.next() {- let val: u64 = values.iter().map(|idx_val| idx_val.value).sum();-- heap.push((- std::cmp::Reverse(val),- String::from_utf8_lossy(key).to_string(),- ));-- if heap.len() > TOP_N {- heap.pop();- }- }-- let top_values: BTreeMap<_, _> =- heap.into_iter().map(|(val, key)| (key, val.0)).collect();-- for (key, value) in top_values {- builder.insert(key, value)?;- }-- let path = Path::new(path);- builder.finish()?;- std::fs::rename(path.join("new_dictionary"), path.join("dictionary"))?;- let mmap = unsafe { Mmap::map(&File::open(path.join("dictionary"))?)? };- self.map = InnerMap::File(Map::new(mmap)?);- }- None => {- let mut builder = MapBuilder::memory();-- let mut heap = BinaryHeap::with_capacity(TOP_N + 1);+ pub fn add_monogram(&mut self, term: HeavyMonogram, freq: u64) {+ if self.monograms.len() < self.top_n {+ self.monograms.push(Reverse((freq, term)));+ } else if self.monograms.peek().unwrap().0 .0 < freq {+ let mut worst = self.monograms.peek_mut().unwrap();+ *worst = Reverse((freq, term));+ }+ }- while let Some((key, values)) = union.next() {- let val: u64 = values.iter().map(|idx_val| idx_val.value).sum();+ pub fn add_bigram(&mut self, term: HeavyBigram, freq: u64) {+ if self.bigrams.len() < self.top_n {+ self.bigrams.push(Reverse((freq, term)));+ } else if self.bigrams.peek().unwrap().0 .0 < freq {+ let mut worst = self.bigrams.peek_mut().unwrap();+ *worst = Reverse((freq, term));+ }+ }- heap.push((- std::cmp::Reverse(val),- String::from_utf8_lossy(key).to_string(),- ));+ pub fn add_trigram(&mut self, term: HeavyTrigram, freq: u64) {+ if self.trigrams.len() < self.top_n {+ self.trigrams.push(Reverse((freq, term)));+ } else if self.trigrams.peek().unwrap().0 .0 < freq {+ let mut worst = self.trigrams.peek_mut().unwrap();+ *worst = Reverse((freq, term));+ }+ }- if heap.len() > TOP_N {- heap.pop();- }- }+ pub fn build(self) -> Dictionary {+ Dictionary::build(+ self.monograms+ .into_iter()+ .map(|Reverse((freq, term))| (term, freq)),+ self.bigrams+ .into_iter()+ .map(|Reverse((freq, term))| (term, freq)),+ self.trigrams+ .into_iter()+ .map(|Reverse((freq, term))| (term, freq)),+ )+ }+}- let top_values: BTreeMap<_, _> =- heap.into_iter().map(|(val, key)| (key, val.0)).collect();+#[derive(Clone)]+pub struct Dictionary {+ inner: Arc<InnerDictionary>,+}- for (key, value) in top_values {- builder.insert(key, value)?;- }+impl Dictionary {+ pub fn build(+ monograms: impl Iterator<Item = (String, u64)>,+ bigrams: impl Iterator<Item = ((String, String), u64)>,+ trigrams: impl Iterator<Item = ((String, String, String), u64)>,+ ) -> Self {+ let inner = InnerDictionary::build(monograms, bigrams, trigrams);+ Self {+ inner: Arc::new(inner),+ }+ }+}- let bytes = builder.into_inner().unwrap();- self.map = InnerMap::Memory(Map::new(bytes)?);- }- };+impl Deref for Dictionary {+ type Target = InnerDictionary;- Ok(())+ fn deref(&self) -> &Self::Target {+ &self.inner}+}- pub fn commit(&mut self) -> Result<()> {- let cache = mem::take(&mut self.cache);- let cache_map = Map::from_iter(cache.into_iter())?;- let map = mem::replace(&mut self.map, InnerMap::Memory(Map::default()));+/// Dictionary that contains term frequency information+pub struct InnerDictionary {+ total_freq: u64,+ monograms: BTreeMap<Monogram, u64>,+ bigrams: BTreeMap<Bigram, u64>,+ trigrams: BTreeMap<Trigram, u64>,+ terms: BTreeMap<String, TermId>,+ rev_terms: BTreeMap<TermId, String>,+}- let union = match &map {- InnerMap::File(map) => map.op().add(&cache_map).union(),- InnerMap::Memory(map) => map.op().add(&cache_map).union(),+impl InnerDictionary {+ fn build(+ monograms: impl Iterator<Item = (String, u64)>,+ bigrams: impl Iterator<Item = ((String, String), u64)>,+ trigrams: impl Iterator<Item = ((String, String, String), u64)>,+ ) -> Self {+ let mut dict = InnerDictionary {+ total_freq: 0,+ monograms: BTreeMap::new(),+ bigrams: BTreeMap::new(),+ trigrams: BTreeMap::new(),+ terms: BTreeMap::new(),+ rev_terms: BTreeMap::new(),};- self.store_union(union)?;- self.total_freq = self.map.total_freq();+ dict.load_monograms(monograms);+ dict.load_bigrams(bigrams);+ dict.load_trigrams(trigrams);- Ok(())+ dict}- pub fn insert(&mut self, term: &str) {- self.cache- .entry(- term.chars()- .map(|c| c.to_ascii_lowercase())- .filter(|c| !matches!(c, ',' | '.' | '\\' | '=' | '*' | '(' | ')'))- .collect(),- )- .or_insert(0)- .add_assign(1);- }-- #[inline]- pub fn all_probabilities(- &self,- term: &str,- edit_distance: usize,- ) -> Vec<HashSet<DictionaryResult>> {- let searcher = fst::automaton::Levenshtein::new(term, edit_distance as u32);- let mut res = Vec::with_capacity(edit_distance + 1);-- if searcher.is_err() {- return res;- }-- for _ in 0..edit_distance + 1 {- res.push(HashSet::new());+ fn load_monograms(&mut self, monograms: impl Iterator<Item = (String, u64)>) {+ for (term, freq) in monograms {+ let l = self.terms.len();+ let id = *self.terms.entry(term.clone()).or_insert_with(|| l as u64);+ self.rev_terms.insert(id, term);+ *self.monograms.entry(Monogram(id)).or_insert(0) += freq;+ self.total_freq += freq;}+ }- let searcher = searcher.unwrap();+ fn load_bigrams(&mut self, bigrams: impl Iterator<Item = ((String, String), u64)>) {+ for ((a, b), freq) in bigrams {+ let l = self.terms.len();+ let id_a = *self.terms.entry(a.clone()).or_insert_with(|| l as u64);+ if self.rev_terms.get(&id_a).is_none() {+ self.rev_terms.insert(id_a, a);+ }- let mut matches = self.perform_search(&searcher);+ let l = self.terms.len();+ let id_b = *self.terms.entry(b.clone()).or_insert_with(|| l as u64);+ if self.rev_terms.get(&id_b).is_none() {+ self.rev_terms.insert(id_b, b);+ }- while let Some((correction, freq)) = matches.next() {- let correction = std::str::from_utf8(correction).unwrap().to_owned();- let dist = LevenshteinDistance::compare(term, &correction);+ *self.bigrams.entry(Bigram(id_a, id_b)).or_insert(0) += freq;+ }+ }- let prob = freq as f64 / self.total_freq as f64;+ fn load_trigrams(&mut self, trigrams: impl Iterator<Item = ((String, String, String), u64)>) {+ for ((a, b, c), freq) in trigrams {+ let l = self.terms.len();+ let id_a = *self.terms.entry(a.clone()).or_insert_with(|| l as u64);+ if self.rev_terms.get(&id_a).is_none() {+ self.rev_terms.insert(id_a, a);+ }- res[dist].insert(DictionaryResult { correction, prob });- }+ let l = self.terms.len();+ let id_b = *self.terms.entry(b.clone()).or_insert_with(|| l as u64);+ if self.rev_terms.get(&id_b).is_none() {+ self.rev_terms.insert(id_b, b);+ }- res- }+ let l = self.terms.len();+ let id_c = *self.terms.entry(c.clone()).or_insert_with(|| l as u64);+ if self.rev_terms.get(&id_c).is_none() {+ self.rev_terms.insert(id_c, c);+ }- fn perform_search<A: Automaton>(&self, searcher: A) -> fst::map::Stream<'_, A> {- match &self.map {- InnerMap::File(map) => map.search(searcher).into_stream(),- InnerMap::Memory(map) => map.search(searcher).into_stream(),+ *self.trigrams.entry(Trigram(id_a, id_b, id_c)).or_insert(0) += freq;}}#[inline]- pub fn probability(&self, term: &str) -> Option<f64> {- let searcher = fst::automaton::Str::new(term);- let mut matches = self.perform_search(&searcher);+ pub fn score(&self, before: &[&str], term: &str, after: &[&str]) -> Option<f64> {+ self.terms.get(term).map(|id| {+ let mut d = 1;+ let mut total_score = self.monograms[&Monogram(*id)] as f64 / self.total_freq as f64;++ if !before.is_empty() {+ if let Some(id_before) = self.terms.get(before[before.len() - 1]) {+ if let Some(bigram_freq) = self.bigrams.get(&Bigram(*id_before, *id)) {+ total_score +=+ *bigram_freq as f64 / self.monograms[&Monogram(*id_before)] as f64;+ d += 1;+ }- matches- .next()- .map(|(_key, frequency)| frequency as f64 / self.total_freq as f64)- }+ if before.len() > 1 {+ if let Some(id_before_before) = self.terms.get(before[before.len() - 2]) {+ if let Some(trigram_freq) =+ self.trigrams+ .get(&Trigram(*id_before_before, *id_before, *id))+ {+ if let Some(c_freq) =+ self.bigrams.get(&Bigram(*id_before_before, *id_before))+ {+ total_score += *trigram_freq as f64 / *c_freq as f64;+ d += 1;+ }+ }+ }+ }+ }+ }- #[cfg(test)]- pub fn contains(&self, term: &str) -> bool {- self.probability(term).is_some()- }+ if !after.is_empty() {+ if let Some(id_after) = self.terms.get(after[0]) {+ if let Some(bigram_freq) = self.bigrams.get(&Bigram(*id, *id_after)) {+ total_score += *bigram_freq as f64 / self.monograms[&Monogram(*id)] as f64;+ d += 1;+ }- pub fn insert_page(&mut self, webpage: &Webpage) {- let text = webpage.html.clean_text().cloned().unwrap_or_default();+ if after.len() > 1 {+ if let Some(id_after_after) = self.terms.get(after[1]) {+ if let Some(trigram_freq) =+ self.trigrams.get(&Trigram(*id, *id_after, *id_after_after))+ {+ if let Some(c_freq) = self.bigrams.get(&Bigram(*id, *id_after)) {+ total_score += *trigram_freq as f64 / *c_freq as f64;+ d += 1;+ }+ }+ }+ }+ }+ }- let mut stream = Normal::default().token_stream(text.as_str());+ if !after.is_empty() && !before.is_empty() {+ if let Some(id_after) = self.terms.get(after[after.len() - 1]) {+ if let Some(id_before) = self.terms.get(before[before.len() - 1]) {+ if let Some(trigram_freq) =+ self.trigrams.get(&Trigram(*id_before, *id, *id_after))+ {+ if let Some(c_freq) = self.bigrams.get(&Bigram(*id_before, *id)) {+ total_score += *trigram_freq as f64 / *c_freq as f64;+ d += 1;+ }+ }+ }+ }+ }- while let Some(token) = stream.next() {- self.insert(&token.text);- }+ total_score / d as f64+ })}- pub fn merge(mut self, mut other: Dictionary<TOP_N>) -> Self {- self.commit().unwrap();- other.commit().unwrap();-- let map = mem::replace(&mut self.map, InnerMap::Memory(Map::default()));-- let union = match (&map, &other.map) {- (InnerMap::Memory(map), InnerMap::Memory(other_map)) => map.op().add(other_map).union(),- (InnerMap::Memory(map), InnerMap::File(other_map)) => map.op().add(other_map).union(),- (InnerMap::File(map), InnerMap::Memory(other_map)) => map.op().add(other_map).union(),- (InnerMap::File(map), InnerMap::File(other_map)) => map.op().add(other_map).union(),- };+ pub fn terms(&self) -> impl Iterator<Item = (&String, &TermId)> {+ self.terms.iter()+ }- self.store_union(union).unwrap();- self.total_freq = self.map.total_freq();+ pub fn term_id(&self, term: &str) -> Option<&TermId> {+ self.terms.get(term)+ }- self+ pub fn term(&self, term_id: &u64) -> Option<&str> {+ self.rev_terms.get(term_id).map(|s| s.as_str())}}#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn test_probability() {- let mut dict = Dictionary::default();-- dict.insert("this");- dict.insert("is");- dict.insert("a");- dict.insert("test");- dict.insert("test");-- dict.commit().unwrap();-- assert_eq!(dict.probability("this"), Some(0.2));- assert_eq!(dict.probability("is"), Some(0.2));- assert_eq!(dict.probability("a"), Some(0.2));- assert_eq!(dict.probability("test"), Some(0.4));+pub fn build_from_str(text: &str) -> Dictionary {+ use itertools::Itertools;+ use std::collections::HashMap;+ let mut mono_freqs = HashMap::new();+ let mut bi_freqs = HashMap::new();+ let mut tri_freqs = HashMap::new();++ for word in text.split_ascii_whitespace() {+ *mono_freqs.entry(word.to_string()).or_insert(0) += 1;}- #[test]- fn test_uncontained_term() {- let mut dict = Dictionary::default();-- dict.insert("this");- dict.insert("is");- dict.insert("a");- dict.insert("test");- dict.insert("test");-- dict.commit().unwrap();-- assert_eq!(dict.probability("the"), None);-- assert!(dict.contains("this"));- assert!(dict.contains("is"));- assert!(dict.contains("test"));- assert!(!dict.contains("the"));- }-- #[test]- fn test_probability_edit_3() {- let mut dict = Dictionary::default();-- dict.insert("test");- dict.insert("twst");- dict.insert("kage");-- dict.commit().unwrap();- let res = dict.all_probabilities("test", 2);-- let mut num_elements = 0;-- for set in res {- num_elements += set.len();- }-- assert_eq!(num_elements, 2);+ for (a, b) in text.split_ascii_whitespace().tuple_windows() {+ *bi_freqs.entry((a.to_string(), b.to_string())).or_insert(0) += 1;}- #[test]- fn merge() {- let mut dict1 = Dictionary::default();-- dict1.insert("test");- dict1.insert("kage");-- let mut dict2 = Dictionary::default();-- dict2.insert("yay");- dict2.insert("ay");-- let res = dict1.merge(dict2);-- assert!(res.contains("test"));- assert!(res.contains("kage"));- assert!(res.contains("yay"));- assert!(res.contains("ay"));-- assert_eq!(res.probability("test"), Some(1.0 / 4.0));- assert_eq!(res.probability("kage"), Some(1.0 / 4.0));- assert_eq!(res.probability("yay"), Some(1.0 / 4.0));- assert_eq!(res.probability("ay"), Some(1.0 / 4.0));+ for (a, b, c) in text.split_ascii_whitespace().tuple_windows() {+ *tri_freqs+ .entry((a.to_string(), b.to_string(), c.to_string()))+ .or_insert(0) += 1;}- #[test]- fn only_store_top_n() {- let mut dict: Dictionary<2> = Dictionary::open::<&str>(None).unwrap();-- dict.insert("test");- dict.insert("test");- dict.insert("hej");- dict.insert("kage");- dict.insert("kage");+ Dictionary::build(+ text.split_ascii_whitespace()+ .map(|word| (word.to_string(), mono_freqs[word])),+ text.split_ascii_whitespace().tuple_windows().map(|(a, b)| {+ (+ (a.to_string(), b.to_string()),+ bi_freqs[&(a.to_string(), b.to_string())],+ )+ }),+ text.split_ascii_whitespace()+ .tuple_windows()+ .map(|(a, b, c)| {+ (+ (a.to_string(), b.to_string(), c.to_string()),+ tri_freqs[&(a.to_string(), b.to_string(), c.to_string())],+ )+ }),+ )+}- dict.commit().unwrap();+#[cfg(test)]+mod tests {- assert!(dict.contains("test"));- assert!(dict.contains("kage"));- assert!(!dict.contains("hej"));- }+ use super::*;#[test]- fn weird_characters_ignored() {- let mut dict = Dictionary::default();-- dict.insert("lennon,");-- dict.commit().unwrap();+ fn test_probability() {+ let dict = build_from_str("this is a test test");- assert!(dict.contains("lennon"));- assert!(!dict.contains("lennon,"));+ assert_eq!(+ dict.score(&[], "this", &[]).unwrap(),+ dict.score(&[], "is", &[]).unwrap()+ );+ assert!(dict.score(&[], "test", &[]).unwrap() > dict.score(&[], "this", &[]).unwrap());}}diff --git a/core/src/spell/distance.rs b/core/src/spell/distance.rsindex 21857482..615b80b5 100644--- a/core/src/spell/distance.rs+++ b/core/src/spell/distance.rs@@ -26,6 +26,10 @@ impl LevenshteinDistance {LevenshteinDistance { dist: distance }}+ pub fn distance(&self) -> usize {+ self.dist+ }+/// Function taken from https://github.com/febeling/edit-distance/blob/5597816456e7153cf69092f6ab5d0b4edb5e3797/src/lib.rs#L31////// The [Levenshtein edit distance][wikipedia] between two strings isdiff --git a/core/src/spell/mod.rs b/core/src/spell/mod.rsindex f97f4d21..b6f03f02 100644--- a/core/src/spell/mod.rs+++ b/core/src/spell/mod.rs@@ -23,9 +23,17 @@ use std::ops::Range;use itertools::intersperse;use serde::{Deserialize, Serialize};+use tantivy::tokenizer::Tokenizer as _;+use tracing::info;-use crate::floor_char_boundary;+use crate::index::Index;+use crate::query::parser::Term;+use crate::schema::TextField;+use crate::searcher::SearchQuery;+use crate::tokenizer::Tokenizer;+use crate::{floor_char_boundary, query};+use self::dictionary::DictionaryBuilder;pub use self::dictionary::{Dictionary, DictionaryResult, EditStrategy, LogarithmicEdit};pub use self::spell_checker::SpellChecker;pub use self::splitter::TermSplitter;@@ -90,3 +98,212 @@ pub fn sentence_ranges(text: &str) -> Vec<Range<usize>> {res}++pub struct Spell {+ dict: Dictionary,+ spell_checker: SpellChecker<LogarithmicEdit>,+}++impl Spell {+ pub fn for_index(index: &Index) -> Self {+ let dict = Self::build_dict(index);+ let spell_checker = SpellChecker::new(dict.clone(), LogarithmicEdit::new(4));++ Self {+ dict,+ spell_checker,+ }+ }+ fn build_dict(index: &Index) -> Dictionary {+ info!("Building spell correction dictionary");+ let searcher = index.inverted_index.tv_searcher();+ let schema = searcher.schema();+ let mut dict = DictionaryBuilder::new(100_000);+ #[allow(unused_assignments)]+ let mut limit_terms: Option<usize> = None;+ #[cfg(debug_assertions)]+ {+ limit_terms = Some(100);+ }++ for segment in searcher.segment_readers() {+ let inv_index = segment+ .inverted_index(schema.get_field(TextField::CleanBody.name()).unwrap())+ .unwrap();+ let term_dict = inv_index.terms();+ let mut stream = term_dict.stream().unwrap();+ let mut count = 0;+ while let Some((term, info)) = stream.next() {+ if let Some(limit) = limit_terms {+ if count > limit {+ break;+ }+ }++ let term = std::str::from_utf8(term).unwrap();++ if !term.is_empty()+ && term+ .chars()+ .all(|c| c.is_ascii_alphabetic() || c.is_whitespace())+ {+ dict.add_monogram(term.to_ascii_lowercase(), info.doc_freq as u64);+ count += 1;+ }+ }+ }++ for segment in searcher.segment_readers() {+ let inv_index = segment+ .inverted_index(+ schema+ .get_field(TextField::CleanBodyBigrams.name())+ .unwrap(),+ )+ .unwrap();+ let term_dict = inv_index.terms();+ let mut stream = term_dict.stream().unwrap();+ let mut count = 0;++ while let Some((term, info)) = stream.next() {+ if let Some(limit) = limit_terms {+ if count > limit {+ break;+ }+ }++ let term = std::str::from_utf8(term).unwrap();+ if !term.is_empty()+ && term+ .chars()+ .all(|c| c.is_ascii_alphabetic() || c.is_whitespace())+ {+ let mut tokenizer = Tokenizer::default().token_stream(term);+ let first = tokenizer.next().unwrap().text.to_ascii_lowercase();+ let second = tokenizer.next().unwrap().text.to_ascii_lowercase();+ dict.add_bigram((first, second), info.doc_freq as u64);+ count += 1;+ }+ }+ }++ for segment in searcher.segment_readers() {+ let inv_index = segment+ .inverted_index(+ schema+ .get_field(TextField::CleanBodyTrigrams.name())+ .unwrap(),+ )+ .unwrap();+ let term_dict = inv_index.terms();+ let mut stream = term_dict.stream().unwrap();+ let mut count = 0;++ while let Some((term, info)) = stream.next() {+ if let Some(limit) = limit_terms {+ if count > limit {+ break;+ }+ }++ let term = std::str::from_utf8(term).unwrap();+ if !term.is_empty()+ && term+ .chars()+ .all(|c| c.is_ascii_alphabetic() || c.is_whitespace())+ {+ let mut tokenizer = Tokenizer::default().token_stream(term);+ let first = tokenizer.next().unwrap().text.to_ascii_lowercase();+ let second = tokenizer.next().unwrap().text.to_ascii_lowercase();+ let third = tokenizer.next().unwrap().text.to_ascii_lowercase();+ dict.add_trigram((first, second, third), info.doc_freq as u64);+ count += 1;+ }+ }+ }++ dict.build()+ }++ fn spell_check(&self, terms: &[String]) -> Option<Correction> {+ let mut original = String::new();+ let num_terms = terms.len();+ let terms = terms.iter().map(|s| s.as_str()).collect::<Vec<_>>();++ for term in &terms {+ original.push_str(term);+ original.push(' ');+ }+ original = original.trim_end().to_string();++ let mut possible_correction = Correction::empty(original);++ for i in 0..num_terms {+ let before = if i > 0 { &terms[..i] } else { &[] };+ let term = &terms[i];+ let after = if i < num_terms - 1 {+ &terms[i + 1..]+ } else {+ &[]+ };++ match self.spell_checker.correct(before, term, after) {+ Some(correction) => possible_correction+ .push(CorrectionTerm::Corrected(correction.to_ascii_lowercase())),+ None => possible_correction+ .push(CorrectionTerm::NotCorrected(term.to_ascii_lowercase())),+ }+ }++ if possible_correction.is_all_orig() {+ None+ } else {+ Some(possible_correction)+ }+ }++ fn split_words(&self, terms: &[String]) -> Option<Correction> {+ let splitter = TermSplitter::new(&self.dict);++ let mut original = String::new();++ for term in terms {+ original.push_str(term);+ original.push(' ');+ }+ original = original.trim_end().to_string();++ let mut possible_correction = Correction::empty(original);++ for term in terms {+ let split = splitter.split(term.as_str());+ if split.is_empty() {+ possible_correction.push(CorrectionTerm::NotCorrected(term.to_string()));+ } else {+ for s in split {+ possible_correction.push(CorrectionTerm::Corrected(s.to_string()))+ }+ }+ }++ if possible_correction.is_all_orig() {+ None+ } else {+ Some(possible_correction)+ }+ }++ pub fn correction(&self, query: &SearchQuery) -> Option<Correction> {+ let terms: Vec<_> = query::parser::parse(&query.query)+ .into_iter()+ .filter_map(|term| match *term {+ Term::Simple(s) => Some(s),+ _ => None,+ })+ .map(|s| s.to_ascii_lowercase())+ .collect();++ self.spell_check(&terms)+ .or_else(|| self.split_words(&terms))+ }+}diff --git a/core/src/spell/spell_checker.rs b/core/src/spell/spell_checker.rsindex b4abd0dc..9fb523f2 100644--- a/core/src/spell/spell_checker.rs+++ b/core/src/spell/spell_checker.rs@@ -14,174 +14,249 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::spell::dictionary::EditStrategy;-use crate::spell::{Dictionary, DictionaryResult};-use std::iter::FromIterator;+use crate::spell::Dictionary;+use std::cmp::Ordering;+use std::collections::{BTreeMap, HashSet, VecDeque};-pub struct SpellChecker<'a, T: EditStrategy, const DICT_N: usize> {- dict: &'a Dictionary<DICT_N>,+use super::dictionary::TermId;+use super::distance::LevenshteinDistance;++pub struct SpellChecker<T: EditStrategy> {+ dict: Dictionary,edit_strategy: T,+ deletes: BTreeMap<String, Vec<TermId>>,+}++fn deletes_rec(+ word: &str,+ edit_distance: usize,+ max_edit_distance: usize,+ delete_words: &mut HashSet<String>,+) {+ let edit_distance = edit_distance + 1;+ let word_len = word.chars().count();++ if word_len > 1 {+ for i in 0..word_len {+ let delete = word+ .char_indices()+ .filter(|(j, _)| *j != i)+ .map(|(_, c)| c)+ .collect::<String>();++ if !delete_words.contains(&delete) {+ delete_words.insert(delete.clone());++ if edit_distance < max_edit_distance {+ deletes_rec(&delete, edit_distance, max_edit_distance, delete_words);+ }+ }+ }+ }}-impl<'a, T: EditStrategy, const DICT_N: usize> SpellChecker<'a, T, DICT_N> {- pub fn new(dict: &'a Dictionary<DICT_N>, edit_strategy: T) -> Self {+fn deletes(term: &str, edit_strategy: &impl EditStrategy) -> Vec<String> {+ let mut res = HashSet::new();+ deletes_rec(term, 0, edit_strategy.dist().distance(), &mut res);++ res.into_iter().collect()+}++#[derive(Debug)]+struct Suggestion {+ term: String,+ distance: usize,+ score: f64,+}++impl Ord for Suggestion {+ fn cmp(&self, other: &Suggestion) -> Ordering {+ let distance_cmp = self.distance.cmp(&other.distance);+ if distance_cmp == Ordering::Equal {+ return self+ .score+ .partial_cmp(&other.score)+ .unwrap_or(Ordering::Equal)+ .reverse();+ }+ distance_cmp+ }+}++impl PartialOrd for Suggestion {+ fn partial_cmp(&self, other: &Suggestion) -> Option<Ordering> {+ Some(self.cmp(other))+ }+}++impl PartialEq for Suggestion {+ fn eq(&self, other: &Suggestion) -> bool {+ self.distance == other.distance && self.score == other.score+ }+}+impl Eq for Suggestion {}++impl<T: EditStrategy> SpellChecker<T> {+ pub fn new(dict: Dictionary, edit_strategy: T) -> Self {+ let mut deletes_map = BTreeMap::new();+ for (term, id) in dict.terms() {+ let term = term.to_ascii_lowercase();+ for delete in deletes(&term, &edit_strategy) {+ deletes_map.entry(delete).or_insert_with(Vec::new).push(*id);+ }+ }+SpellChecker {dict,edit_strategy,+ deletes: deletes_map,}}- pub fn correct_top(&self, term: &str, top_n: usize) -> Vec<String> {- let mut res = Vec::new();+ fn suggestions(&self, before: &[&str], term: &str, after: &[&str]) -> Vec<Suggestion> {+ let mut suggestions = Vec::new();+ let mut suggestion_terms = HashSet::new();- if term.chars().any(|c| !c.is_alphabetic()) {- return Vec::new();+ let mut candidates = VecDeque::new();+ candidates.push_back(term.to_ascii_lowercase());+ for delete in deletes(term, &self.edit_strategy) {+ candidates.push_back(delete.to_ascii_lowercase());}+ let max_dist = self.edit_strategy.distance_for_string(term);- // this is sorted by increasing edit distance- for corrections in self- .dict- .all_probabilities(term, self.edit_strategy.distance_for_string(term))- .iter_mut()- {- let mut corrections: Vec<&DictionaryResult> = Vec::from_iter(corrections.iter());- corrections.sort_by(|a, b| b.prob.partial_cmp(&a.prob).unwrap());-- res.extend(- corrections- .into_iter()- .map(|dict_result| dict_result.correction.clone())- .take(top_n - res.len()),- );-- if res.len() >= top_n {- return res;+ while let Some(candidate) = candidates.pop_front() {+ if suggestion_terms.contains(&candidate) {+ continue;+ }++ let distance = LevenshteinDistance::compare(term, &candidate);+ if distance <= max_dist {+ if let Some(score) = self.dict.score(before, &candidate, after) {+ suggestions.push(Suggestion {+ term: candidate.clone(),+ distance,+ score,+ });+ suggestion_terms.insert(candidate.clone());+ }++ if let Some(deletes) = self.deletes.get(&candidate) {+ for delete in deletes {+ candidates.push_back(self.dict.term(delete).unwrap().to_string());+ }+ }}}- res+ suggestions.sort();+ suggestions}- pub fn correct(&self, term: &str) -> Option<String> {- self.correct_top(term, 1)+ pub fn correct(&self, before: &[&str], term: &str, after: &[&str]) -> Option<String> {+ if self.dict.term_id(term).is_some() {+ return None;+ }++ if term.chars().any(|c| !c.is_ascii_alphabetic()) {+ return None;+ }++ self.suggestions(before, term, after).into_iter()+ .map(|suggestion| suggestion.term).next()- .filter(|correction| correction.to_ascii_lowercase() != term.to_ascii_lowercase())}}#[cfg(test)]mod tests {use super::*;- use crate::spell::{dictionary::MaxEdit, LogarithmicEdit};+ use crate::spell::{+ dictionary::{self, MaxEdit},+ LogarithmicEdit,+ };#[test]fn simple_corrections() {- let mut dict = Dictionary::default();+ let dict = dictionary::build_from_str("this is a test test pest");- dict.insert("this");- dict.insert("is");- dict.insert("a");- dict.insert("test");- dict.insert("test");- dict.insert("pest");+ let spell_checker = SpellChecker::new(dict, MaxEdit::new(1));- dict.commit().unwrap();-- let spell_checker = SpellChecker::new(&dict, MaxEdit::new(1));-- assert_eq!(spell_checker.correct("tst"), Some("test".to_string()));- assert_eq!(spell_checker.correct("ths"), Some("this".to_string()));- assert_eq!(spell_checker.correct("thes"), Some("this".to_string()));- assert_eq!(spell_checker.correct("is"), None);assert_eq!(- spell_checker.correct_top("thes", 3),- vec!["this".to_string()]+ spell_checker.correct(&[], "tst", &[]),+ Some("test".to_string())+ );+ assert_eq!(+ spell_checker.correct(&[], "ths", &[]),+ Some("this".to_string()));assert_eq!(- spell_checker.correct_top("dest", 3),- vec!["test".to_string(), "pest".to_string()]+ spell_checker.correct(&[], "thes", &[]),+ Some("this".to_string()));+ assert_eq!(spell_checker.correct(&[], "is", &[]), None);assert_eq!(- spell_checker.correct_top("dest", 1),- vec!["test".to_string()]+ spell_checker.correct(&[], "dest", &[]),+ Some("test".to_string()));}#[test]fn correct_uncontained_word() {- let mut dict = Dictionary::default();-- dict.insert("this");- dict.insert("is");- dict.insert("a");- dict.insert("test");- dict.insert("test");-- dict.commit().unwrap();+ let dict = dictionary::build_from_str("this is a test test");- let spell_checker = SpellChecker::new(&dict, MaxEdit::new(1));+ let spell_checker = SpellChecker::new(dict, MaxEdit::new(1));- assert_eq!(spell_checker.correct("what"), None);- assert!(spell_checker.correct_top("what", 2).is_empty());+ assert_eq!(spell_checker.correct(&[], "what", &[]), None);}#[test]fn prioritise_low_distance_words() {- let mut dict = Dictionary::default();+ let dict = dictionary::build_from_str("this is a test test contest contest");- dict.insert("this");- dict.insert("is");- dict.insert("a");- dict.insert("test");- dict.insert("contest");- dict.insert("contest");+ let spell_checker = SpellChecker::new(dict, MaxEdit::new(4));- dict.commit().unwrap();-- let spell_checker = SpellChecker::new(&dict, MaxEdit::new(4));-- assert_eq!(spell_checker.correct("dest"), Some("test".to_string()));assert_eq!(- spell_checker.correct_top("dest", 3),- vec!["test".to_string(), "is".to_string(), "contest".to_string()]+ spell_checker.correct(&[], "dest", &[]),+ Some("test".to_string()));}#[test]fn correct_sorting_multiple_hits() {- let mut dict = Dictionary::default();-- dict.insert("the");- dict.insert("the");- dict.insert("the");- dict.insert("he");+ let dict = dictionary::build_from_str("the the the he");- dict.commit().unwrap();+ let spell_checker = SpellChecker::new(dict, LogarithmicEdit::new(4));- let spell_checker = SpellChecker::new(&dict, LogarithmicEdit::new(4));-- assert_eq!(spell_checker.correct("fhe"), Some("the".to_string()));assert_eq!(- spell_checker.correct_top("fhe", 3),- vec!["the".to_string(), "he".to_string()]+ spell_checker.correct(&[], "fhe", &[]),+ Some("the".to_string()));}#[test]fn dont_correct_non_alphabet() {- let mut dict = Dictionary::default();+ let dict = dictionary::build_from_str("this is a test c");- dict.insert("this");- dict.insert("is");- dict.insert("a");- dict.insert("test");- dict.insert("c");+ let spell_checker = SpellChecker::new(dict, MaxEdit::new(1));- dict.commit().unwrap();+ assert_eq!(spell_checker.correct(&[], "c++", &[]), None);+ assert_eq!(spell_checker.correct(&[], "c#", &[]), None);+ }- let spell_checker = SpellChecker::new(&dict, MaxEdit::new(1));+ #[test]+ fn context_correction() {+ let dict = dictionary::build_from_str("abraham lincoln was the 16th president of the united states. A wrong way to spell hist last name would be linculn linculn");- assert_eq!(spell_checker.correct("c++"), None);- assert_eq!(spell_checker.correct("c#"), None);+ let spell_checker = SpellChecker::new(dict, MaxEdit::new(1));++ assert_eq!(+ spell_checker.correct(&[], "lincln", &[]),+ Some("linculn".to_string())+ );+ assert_eq!(+ spell_checker.correct(&["abraham"], "lincln", &[]),+ Some("lincoln".to_string())+ );}}diff --git a/core/src/spell/splitter.rs b/core/src/spell/splitter.rsindex d77830a6..2d66409d 100644--- a/core/src/spell/splitter.rs+++ b/core/src/spell/splitter.rs@@ -15,12 +15,12 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::spell::Dictionary;-pub struct TermSplitter<'a, const DICT_N: usize> {- dict: &'a Dictionary<DICT_N>,+pub struct TermSplitter<'a> {+ dict: &'a Dictionary,}-impl<'a, const DICT_N: usize> TermSplitter<'a, DICT_N> {- pub fn new(dict: &'a Dictionary<DICT_N>) -> Self {+impl<'a> TermSplitter<'a> {+ pub fn new(dict: &'a Dictionary) -> Self {TermSplitter { dict }}pub fn split(&self, text: &'a str) -> Vec<&'a str> {@@ -36,7 +36,7 @@ impl<'a, const DICT_N: usize> TermSplitter<'a, DICT_N> {continue;}- if let Some(prob) = &self.dict.probability(&text[j..i]) {+ if let Some(prob) = &self.dict.score(&[], &text[j..i], &[]) {let new_prob = probs[j] * prob;if new_prob > best_prob_k {best_prob_k = new_prob;@@ -67,16 +67,14 @@ impl<'a, const DICT_N: usize> TermSplitter<'a, DICT_N> {#[cfg(test)]mod tests {++ use crate::spell::dictionary;+use super::*;#[test]fn test_split() {- let mut dictionary = Dictionary::default();-- dictionary.insert("wicked");- dictionary.insert("weather");-- dictionary.commit().unwrap();+ let dictionary = dictionary::build_from_str("wicked weather");assert_eq!(TermSplitter::new(&dictionary).split("wickedweather"),@@ -84,21 +82,12 @@ mod tests {);assert_eq!(TermSplitter::new(&dictionary).split("wicked").len(), 0);-assert_eq!(TermSplitter::new(&dictionary).split("udlæg").len(), 0);}#[test]fn test_most_probable() {- let mut dictionary = Dictionary::default();-- dictionary.insert("wicked");- dictionary.insert("wicked");- dictionary.insert("weather");- dictionary.insert("weather");- dictionary.insert("eat"); // "eat" is a substring of "weather"-- dictionary.commit().unwrap();+ let dictionary = dictionary::build_from_str("wicked weather wicked weather eat");assert_eq!(TermSplitter::new(&dictionary).split("wickedweather"),diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rsindex 7063ae68..4929eb89 100644--- a/core/src/tokenizer.rs+++ b/core/src/tokenizer.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::str::CharIndices;+use std::{array, ops::Range, str::CharIndices};use logos::{Lexer, Logos};use tantivy::tokenizer::{@@ -55,6 +55,8 @@ pub enum Tokenizer {Normal(Normal),Identity(Identity),Stemmed(Stemmed),+ Bigram(BigramTokenizer),+ Trigram(TrigramTokenizer),Json(JsonField),}@@ -68,6 +70,8 @@ impl Tokenizer {Tokenizer::Normal(_) => Normal::as_str(),Tokenizer::Stemmed(_) => Stemmed::as_str(),Tokenizer::Identity(_) => Identity::as_str(),+ Tokenizer::Bigram(_) => BigramTokenizer::as_str(),+ Tokenizer::Trigram(_) => TrigramTokenizer::as_str(),Tokenizer::Json(_) => JsonField::as_str(),}}@@ -96,6 +100,24 @@ impl Normal {}}+#[derive(Clone, Default)]+pub struct BigramTokenizer {}++impl BigramTokenizer {+ pub fn as_str() -> &'static str {+ "bigram_tokenizer"+ }+}++#[derive(Clone, Default)]+pub struct TrigramTokenizer {}++impl TrigramTokenizer {+ pub fn as_str() -> &'static str {+ "trigram_tokenizer"+ }+}+#[derive(Clone, Default)]pub struct Stemmed {force_language: Option<Lang>,@@ -128,6 +150,8 @@ impl tantivy::tokenizer::Tokenizer for Tokenizer {Tokenizer::Stemmed(tokenizer) => tokenizer.token_stream(text),Tokenizer::Identity(tokenizer) => tokenizer.token_stream(text),Tokenizer::Json(tokenizer) => tokenizer.token_stream(text),+ Tokenizer::Bigram(tokenizer) => tokenizer.token_stream(text),+ Tokenizer::Trigram(tokenizer) => tokenizer.token_stream(text),}}}@@ -168,6 +192,22 @@ impl tantivy::tokenizer::Tokenizer for Identity {}}+impl tantivy::tokenizer::Tokenizer for BigramTokenizer {+ fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {+ let inner = Normal::default().token_stream(text);+ let stream: NGramTokenStream<2> = NGramTokenStream::new(text, inner);+ BoxTokenStream::from(stream)+ }+}++impl tantivy::tokenizer::Tokenizer for TrigramTokenizer {+ fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {+ let inner = Normal::default().token_stream(text);+ let stream: NGramTokenStream<3> = NGramTokenStream::new(text, inner);+ BoxTokenStream::from(stream)+ }+}+pub struct IdentityTokenStream {num_advances: usize,token: Option<tantivy::tokenizer::Token>,@@ -266,6 +306,161 @@ impl<'a> tantivy::tokenizer::TokenStream for SimpleTokenStream<'a> {}}+#[derive(Default)]+struct TokenRef {+ range: Range<usize>,+ position: usize,+}++impl From<&tantivy::tokenizer::Token> for TokenRef {+ fn from(token: &tantivy::tokenizer::Token) -> Self {+ Self {+ range: token.offset_from..token.offset_to,+ position: token.position,+ }+ }+}++struct NGramTokenStream<'a, const N: usize> {+ inner: BoxTokenStream<'a>,+ text: &'a str,+ token: tantivy::tokenizer::Token,+ token_refs: [TokenRef; N],+ is_first: bool,+ next_pos: usize,+}++impl<'a, const N: usize> NGramTokenStream<'a, N> {+ fn new(text: &'a str, inner: BoxTokenStream<'a>) -> Self {+ Self {+ inner,+ text,+ token: tantivy::tokenizer::Token::default(),+ token_refs: array::from_fn(|_| TokenRef::default()),+ is_first: true,+ next_pos: 0,+ }+ }+}++impl<'a> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, 1> {+ fn advance(&mut self) -> bool {+ self.is_first = false;+ let res = self.inner.advance();++ if res {+ self.token_refs[0].range = self.inner.token().offset_from..self.inner.token().offset_to;+ self.token_refs[0].position = self.next_pos;+ self.next_pos += 1;+ }++ self.token = tantivy::tokenizer::Token {+ offset_from: self.token_refs[0].range.start,+ offset_to: self.token_refs[0].range.end,+ position: self.token_refs[0].position,+ text: self.text[self.token_refs[0].range.clone()].to_string(),+ position_length: 1,+ };++ res+ }++ fn token(&self) -> &tantivy::tokenizer::Token {+ &self.token+ }++ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {+ &mut self.token+ }+}++impl<'a> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, 2> {+ fn advance(&mut self) -> bool {+ if self.is_first {+ if !self.inner.advance() {+ return false;+ }++ self.token_refs[0] = self.inner.token().into();++ if !self.inner.advance() {+ return false;+ }+ self.token_refs[1] = self.inner.token().into();+ } else {+ if !self.inner.advance() {+ return false;+ }++ self.token_refs.rotate_left(1);+ self.token_refs[1] = self.inner.token().into();+ }+ self.is_first = false;++ self.next_pos += 1;++ self.token.position = self.next_pos;+ self.token.text =+ self.text[self.token_refs[0].range.start..self.token_refs[1].range.end].to_string();+ self.token.position_length = 2;+ true+ }++ fn token(&self) -> &tantivy::tokenizer::Token {+ &self.token+ }++ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {+ &mut self.token+ }+}++impl<'a> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, 3> {+ fn advance(&mut self) -> bool {+ if self.is_first {+ if !self.inner.advance() {+ return false;+ }++ self.token_refs[0] = self.inner.token().into();++ if !self.inner.advance() {+ return false;+ }+ self.token_refs[1] = self.inner.token().into();++ if !self.inner.advance() {+ return false;+ }+ self.token_refs[2] = self.inner.token().into();+ } else {+ if !self.inner.advance() {+ return false;+ }++ self.token_refs.rotate_left(1);+ self.token_refs[2] = self.inner.token().into();+ }+ self.is_first = false;++ self.next_pos += 1;++ self.token.position = self.next_pos;+ self.token.text =+ self.text[self.token_refs[0].range.start..self.token_refs[2].range.end].to_string();+ self.token.position_length = 3;+ true+ }++ fn token(&self) -> &tantivy::tokenizer::Token {+ &self.token+ }++ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {+ &mut self.token+ }+}+pub struct FlattenedJson {flattened_json: String,}@@ -443,7 +638,7 @@ impl<'a> tantivy::tokenizer::TokenStream for JsonFieldTokenStream<'a> {#[cfg(test)]mod tests {- use tantivy::tokenizer::Tokenizer;+ use tantivy::tokenizer::Tokenizer as _;use super::*;@@ -469,6 +664,28 @@ mod tests {res}+ fn tokenize_bigram(s: &str) -> Vec<String> {+ let mut res = Vec::new();+ let mut stream = Tokenizer::Bigram(BigramTokenizer::default()).token_stream(s);++ while let Some(token) = stream.next() {+ res.push(token.text.clone());+ }++ res+ }++ fn tokenize_trigram(s: &str) -> Vec<String> {+ let mut res = Vec::new();+ let mut stream = Tokenizer::Trigram(TrigramTokenizer::default()).token_stream(s);++ while let Some(token) = stream.next() {+ res.push(token.text.clone());+ }++ res+ }+#[test]fn simple_tokenization() {assert_eq!(@@ -672,6 +889,39 @@ key1.key2="this\" is @ a # test""#;flattened_json_helper(json, expected);}+ #[test]+ fn bigram_tokenizer() {+ assert!(tokenize_bigram("").is_empty());+ assert!(tokenize_bigram("test").is_empty());++ assert_eq!(tokenize_bigram("this is"), vec!["this is".to_string()]);+ assert_eq!(+ tokenize_bigram("this is a"),+ vec!["this is".to_string(), "is a".to_string()]+ );+ assert_eq!(+ tokenize_bigram("this is a test"),+ vec![+ "this is".to_string(),+ "is a".to_string(),+ "a test".to_string()+ ]+ );+ }++ #[test]+ fn trigram_tokenizer() {+ assert!(tokenize_trigram("").is_empty());+ assert!(tokenize_trigram("test").is_empty());+ assert!(tokenize_trigram("this is").is_empty());++ assert_eq!(tokenize_trigram("this is a"), vec!["this is a".to_string()]);+ assert_eq!(+ tokenize_trigram("this is a test"),+ vec!["this is a".to_string(), "is a test".to_string(),]+ );+ }+#[test]fn han() {assert_eq!(diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex 4c5e54ea..ceb1b90d 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -231,7 +231,6 @@ mod tests {#[test]fn harmonic_centrality() {let graph = test_graph();-let centrality = HarmonicCentrality::calculate(&graph);assert!(@@ -281,4 +280,28 @@ mod tests {assert_eq!(centrality.host, centrality_extra.host);}++ #[test]+ fn same_centrality_after_segment_merge() {+ let mut graph = WebgraphBuilder::new_memory().open();++ graph.insert(Node::from("A"), Node::from("B"), String::new());+ graph.commit();+ graph.insert(Node::from("B"), Node::from("C"), String::new());+ graph.commit();+ graph.insert(Node::from("A"), Node::from("C"), String::new());+ graph.commit();+ graph.insert(Node::from("C"), Node::from("A"), String::new());+ graph.commit();+ graph.insert(Node::from("D"), Node::from("C"), String::new());+ graph.commit();++ graph.merge_segments(1);+ let centrality = HarmonicCentrality::calculate(&graph);++ let orig_graph = test_graph();+ let orig_centrality = HarmonicCentrality::calculate(&orig_graph);++ assert_eq!(centrality.host, orig_centrality.host);+ }}diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex f0446546..02ae0ba4 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -211,19 +211,19 @@ impl StoredSegment {j += 1;}- if !packet.new_id_mapping.contains_key(node_id) {- let r = packet- .new_id_mapping- .insert(*node_id, packet.next_segment_node_id);- debug_assert!(r.is_none());- let r = packet- .new_rev_id_mapping- .insert(packet.next_segment_node_id, *node_id);- debug_assert!(r.is_none());- packet.next_segment_node_id.0 += 1;- }-if !node_seen_before {+ if !packet.new_id_mapping.contains_key(node_id) {+ let r = packet+ .new_id_mapping+ .insert(*node_id, packet.next_segment_node_id);+ debug_assert!(r.is_none());+ let r = packet+ .new_rev_id_mapping+ .insert(packet.next_segment_node_id, *node_id);+ debug_assert!(r.is_none());+ packet.next_segment_node_id.0 += 1;+ }+let mut edges = HashSet::new();let mut j = i;diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 5c196e67..131415d2 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -775,6 +775,24 @@ impl Html {},);}+ Field::Text(TextField::CleanBodyBigrams) => {+ doc.add_text(+ tantivy_field,+ self.clean_text().cloned().unwrap_or_default(),+ );+ }+ Field::Text(TextField::CleanBodyTrigrams) => {+ doc.add_text(+ tantivy_field,+ self.clean_text().cloned().unwrap_or_default(),+ );+ }+ Field::Text(TextField::TitleBigrams) => {+ doc.add_text(tantivy_field, title.text.clone());+ }+ Field::Text(TextField::TitleTrigrams) => {+ doc.add_text(tantivy_field, title.text.clone());+ }Field::Text(TextField::Description) => {doc.add_pre_tokenized_text(tantivy_field, description.clone());}