Skip to content

web search done right

tokenizer module

Mikkel Denker86b07e3f87a1
diff --git a/Cargo.toml b/Cargo.toml
index f39cd7e7..acfdd1a6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -28,3 +28,4 @@ tantivy = "0.18.0"
thiserror = "1.0.31"
tokio = { version = "1.18.2", features = ["full"] }
toml = "0.5.9"
+whatlang = "0.16.0"
diff --git a/src/index.rs b/src/index.rs
index 6111c4a1..479fb97f 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -19,9 +19,10 @@ use tantivy::{DocAddress, Document, LeasedItem};
use crate::query::Query;
use crate::schema::{create_schema, Field, ALL_FIELDS};
use crate::searcher::SearchResult;
+use crate::snippet;
+use crate::tokenizer::Tokenizer;
use crate::webpage::Webpage;
use crate::Result;
-use crate::{ranking, snippet};
use std::path::Path;
pub struct Index {
@@ -40,6 +41,10 @@ impl Index {
let schema = create_schema();
let tantivy_index = tantivy::Index::create_in_ram(schema);
+ tantivy_index
+ .tokenizers()
+ .register("tokenizer", Tokenizer::default());
+
Ok(Self {
writer: tantivy_index.writer(100_000_000)?,
reader: tantivy_index.reader()?,
@@ -57,6 +62,7 @@ impl Index {
pub fn commit(&mut self) -> Result<()> {
self.writer.commit()?;
self.reader.reload()?;
+
Ok(())
}
diff --git a/src/lib.rs b/src/lib.rs
index 3efff74d..d7cf790a 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -26,6 +26,7 @@ mod ranking;
mod schema;
mod searcher;
mod snippet;
+mod tokenizer;
mod warc;
mod webgraph;
mod webpage;
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index e496abc9..62d74848 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -43,7 +43,6 @@ impl Ranker {
mod tests {
use crate::{
index::Index,
- query::Query,
searcher::Searcher,
webpage::{Link, Webpage},
};
diff --git a/src/schema.rs b/src/schema.rs
index a008e926..44d54af1 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -40,7 +40,7 @@ impl Field {
fn default_text_options(&self) -> tantivy::schema::TextOptions {
TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
- .set_tokenizer("en_stem")
+ .set_tokenizer("tokenizer")
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
)
}
diff --git a/src/snippet.rs b/src/snippet.rs
index da1ac0b2..f83ede79 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -42,8 +42,6 @@ pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Resu
mod tests {
use crate::{index::Index, searcher::Searcher, webpage::Webpage};
- use super::*;
-
const TEST_TEXT: &str = r#"Rust is a systems programming language sponsored by
Mozilla which describes it as a "safe, concurrent, practical language", supporting functional and
imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?],
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
new file mode 100644
index 00000000..a3da085b
--- /dev/null
+++ b/src/tokenizer.rs
@@ -0,0 +1,37 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::tokenizer::{Language, LowerCaser, SimpleTokenizer, Stemmer, TextAnalyzer};
+#[derive(Clone, Default)]
+pub struct Tokenizer {}
+
+impl tantivy::tokenizer::Tokenizer for Tokenizer {
+ fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
+ let mut analyzer = TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser);
+
+ // TODO:
+ // Use whatlang to determine which stemmer to use.
+ // We will somehow need to search for the non-stemmed version
+ // of the string, as whatlang might be wrong which will result in
+ // wrong stemming. I tried merging multiple token streams but it
+ // caused tantivy to crash. A solution might be to create a separate index
+ // for stem and non-stemmed and do merging in the searcher. This
+ // might cause too much space.
+ analyzer = analyzer.filter(Stemmer::new(Language::English));
+
+ tantivy::tokenizer::BoxTokenStream::from(analyzer.clone().token_stream(text.clone()))
+ }
+}