web search done right
tokenizer module
Mikkel Denker86b07e3f87a1
diff --git a/Cargo.toml b/Cargo.tomlindex f39cd7e7..acfdd1a6 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -28,3 +28,4 @@ tantivy = "0.18.0"thiserror = "1.0.31"tokio = { version = "1.18.2", features = ["full"] }toml = "0.5.9"+whatlang = "0.16.0"diff --git a/src/index.rs b/src/index.rsindex 6111c4a1..479fb97f 100644--- a/src/index.rs+++ b/src/index.rs@@ -19,9 +19,10 @@ use tantivy::{DocAddress, Document, LeasedItem};use crate::query::Query;use crate::schema::{create_schema, Field, ALL_FIELDS};use crate::searcher::SearchResult;+use crate::snippet;+use crate::tokenizer::Tokenizer;use crate::webpage::Webpage;use crate::Result;-use crate::{ranking, snippet};use std::path::Path;pub struct Index {@@ -40,6 +41,10 @@ impl Index {let schema = create_schema();let tantivy_index = tantivy::Index::create_in_ram(schema);+ tantivy_index+ .tokenizers()+ .register("tokenizer", Tokenizer::default());+Ok(Self {writer: tantivy_index.writer(100_000_000)?,reader: tantivy_index.reader()?,@@ -57,6 +62,7 @@ impl Index {pub fn commit(&mut self) -> Result<()> {self.writer.commit()?;self.reader.reload()?;+Ok(())}diff --git a/src/lib.rs b/src/lib.rsindex 3efff74d..d7cf790a 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -26,6 +26,7 @@ mod ranking;mod schema;mod searcher;mod snippet;+mod tokenizer;mod warc;mod webgraph;mod webpage;diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex e496abc9..62d74848 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -43,7 +43,6 @@ impl Ranker {mod tests {use crate::{index::Index,- query::Query,searcher::Searcher,webpage::{Link, Webpage},};diff --git a/src/schema.rs b/src/schema.rsindex a008e926..44d54af1 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -40,7 +40,7 @@ impl Field {fn default_text_options(&self) -> tantivy::schema::TextOptions {TextOptions::default().set_indexing_options(TextFieldIndexing::default()- .set_tokenizer("en_stem")+ .set_tokenizer("tokenizer").set_index_option(IndexRecordOption::WithFreqsAndPositions),)}diff --git a/src/snippet.rs b/src/snippet.rsindex da1ac0b2..f83ede79 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -42,8 +42,6 @@ pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Resumod tests {use crate::{index::Index, searcher::Searcher, webpage::Webpage};- use super::*;-const TEST_TEXT: &str = r#"Rust is a systems programming language sponsored byMozilla which describes it as a "safe, concurrent, practical language", supporting functional andimperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?],diff --git a/src/tokenizer.rs b/src/tokenizer.rsnew file mode 100644index 00000000..a3da085b--- /dev/null+++ b/src/tokenizer.rs@@ -0,0 +1,37 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::tokenizer::{Language, LowerCaser, SimpleTokenizer, Stemmer, TextAnalyzer};+#[derive(Clone, Default)]+pub struct Tokenizer {}++impl tantivy::tokenizer::Tokenizer for Tokenizer {+ fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {+ let mut analyzer = TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser);++ // TODO:+ // Use whatlang to determine which stemmer to use.+ // We will somehow need to search for the non-stemmed version+ // of the string, as whatlang might be wrong which will result in+ // wrong stemming. I tried merging multiple token streams but it+ // caused tantivy to crash. A solution might be to create a separate index+ // for stem and non-stemmed and do merging in the searcher. This+ // might cause too much space.+ analyzer = analyzer.filter(Stemmer::new(Language::English));++ tantivy::tokenizer::BoxTokenStream::from(analyzer.clone().token_stream(text.clone()))+ }+}