Skip to content

web search done right

Separate stemmed and non-stemmed fields. This allows us to prioritize non-stemmed matches over stemmed matches.

Mikkel Denker0632110a8d84
diff --git a/src/index.rs b/src/index.rs
index a2d43497..3c881986 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -54,9 +54,15 @@ impl Index {
tantivy::Index::create_in_dir(path.as_ref(), schema.clone())?
};
+ let tokenizer = Tokenizer::default();
tantivy_index
.tokenizers()
- .register("tokenizer", Tokenizer::default());
+ .register(tokenizer.as_str(), tokenizer);
+
+ let tokenizer = Tokenizer::new_stemmed();
+ tantivy_index
+ .tokenizers()
+ .register(tokenizer.as_str(), tokenizer);
let writer = tantivy_index.writer(10_000_000_000)?;
@@ -218,7 +224,11 @@ impl From<Document> for RetrievedWebpage {
.expect("Url field should be text")
.to_string()
}
- Field::BacklinkText | Field::Centrality | Field::Host => {}
+ Field::BacklinkText
+ | Field::Centrality
+ | Field::Host
+ | Field::StemmedTitle
+ | Field::StemmedBody => {}
}
}
diff --git a/src/schema.rs b/src/schema.rs
index 2241b736..6428418b 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -18,20 +18,26 @@ use tantivy::schema::{
Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,
};
+use crate::tokenizer::{NormalTokenizer, StemmedTokenizer};
+
pub const CENTRALITY_SCALING: u64 = 1_000_000_000;
#[derive(Clone)]
pub enum Field {
Title,
Body,
+ StemmedTitle,
+ StemmedBody,
Url,
Host,
BacklinkText,
Centrality,
}
-pub static ALL_FIELDS: [Field; 6] = [
+pub static ALL_FIELDS: [Field; 8] = [
Field::Title,
Field::Body,
+ Field::StemmedTitle,
+ Field::StemmedBody,
Field::Url,
Field::Host,
Field::BacklinkText,
@@ -39,14 +45,21 @@ pub static ALL_FIELDS: [Field; 6] = [
];
impl Field {
- fn default_text_options(&self) -> tantivy::schema::TextOptions {
+ fn default_text_options_with_tokenizer(
+ &self,
+ tokenizer_name: &str,
+ ) -> tantivy::schema::TextOptions {
TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
- .set_tokenizer("tokenizer")
+ .set_tokenizer(tokenizer_name)
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
)
}
+ fn default_text_options(&self) -> tantivy::schema::TextOptions {
+ self.default_text_options_with_tokenizer(NormalTokenizer::as_str())
+ }
+
fn options(&self) -> IndexingOption {
match self {
Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),
@@ -59,6 +72,12 @@ impl Field {
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
+ Field::StemmedTitle => IndexingOption::Text(
+ self.default_text_options_with_tokenizer(StemmedTokenizer::as_str()),
+ ),
+ Field::StemmedBody => IndexingOption::Text(
+ self.default_text_options_with_tokenizer(StemmedTokenizer::as_str()),
+ ),
}
}
@@ -70,12 +89,15 @@ impl Field {
Field::Host => "host",
Field::BacklinkText => "backlink_text",
Field::Centrality => "centrality",
+ Field::StemmedTitle => "stemmed_title",
+ Field::StemmedBody => "stemmed_body",
}
}
pub fn boost(&self) -> Option<f32> {
match self {
Field::Host => Some(5.0),
+ Field::StemmedBody | Field::StemmedTitle => Some(0.5),
Field::Title | Field::Body | Field::BacklinkText | Field::Centrality | Field::Url => {
None
}
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 9cda808c..3dc2aba5 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -14,52 +14,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use tantivy::tokenizer::{
- Language, LowerCaser, PreTokenizedStream, PreTokenizedString, SimpleTokenizer, Stemmer,
- TextAnalyzer, TokenStream,
-};
+use tantivy::tokenizer::{Language, LowerCaser, SimpleTokenizer, Stemmer, TextAnalyzer};
use whatlang::Lang;
-struct TokenStreamMerger<'a> {
- streams: Vec<tantivy::tokenizer::BoxTokenStream<'a>>,
- current_stream: usize,
-}
-
-impl<'a> TokenStreamMerger<'a> {
- fn merge(streams: Vec<tantivy::tokenizer::BoxTokenStream<'a>>) -> Self {
- Self {
- streams,
- current_stream: 0,
- }
- }
-}
-
-impl<'a> tantivy::tokenizer::TokenStream for TokenStreamMerger<'a> {
- fn advance(&mut self) -> bool {
- if let Some(stream) = self.streams.get_mut(self.current_stream) {
- let mut has_more = stream.advance();
-
- if !has_more {
- self.current_stream += 1;
- has_more = self.current_stream < self.streams.len()
- && self.streams[self.current_stream].advance();
- }
-
- has_more
- } else {
- false
- }
- }
-
- fn token(&self) -> &tantivy::tokenizer::Token {
- self.streams[self.current_stream].token()
- }
-
- fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {
- self.streams[self.current_stream].token_mut()
- }
-}
-
struct MyStemmer(Stemmer);
impl From<Lang> for MyStemmer {
@@ -86,36 +43,72 @@ impl From<Lang> for MyStemmer {
}
}
-#[derive(Clone, Default)]
-pub struct Tokenizer {}
+#[derive(Clone)]
+pub enum Tokenizer {
+ NormalTokenizer(NormalTokenizer),
+ StemmedTokenizer(StemmedTokenizer),
+}
-impl tantivy::tokenizer::Tokenizer for Tokenizer {
- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
- let analyzer = TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser);
+impl Tokenizer {
+ pub fn new_stemmed() -> Self {
+ Self::StemmedTokenizer(Default::default())
+ }
+ pub fn as_str(&self) -> &'static str {
+ match self {
+ Tokenizer::NormalTokenizer(_) => NormalTokenizer::as_str(),
+ Tokenizer::StemmedTokenizer(_) => StemmedTokenizer::as_str(),
+ }
+ }
+}
- let mut streams = vec![analyzer.token_stream(text)];
+impl Default for Tokenizer {
+ fn default() -> Self {
+ Self::NormalTokenizer(Default::default())
+ }
+}
- if let Some(lang) = whatlang::detect_lang(text) {
- let stemmed_analyser = analyzer.clone().filter(MyStemmer::from(lang).0);
- streams.push(stemmed_analyser.token_stream(text));
- }
+#[derive(Clone, Default)]
+pub struct NormalTokenizer {}
- let mut merger = TokenStreamMerger::merge(streams);
+impl NormalTokenizer {
+ pub fn as_str() -> &'static str {
+ "tokenizer"
+ }
+}
+
+#[derive(Clone, Default)]
+pub struct StemmedTokenizer {}
+
+impl StemmedTokenizer {
+ pub fn as_str() -> &'static str {
+ "stemmed_tokenizer"
+ }
+}
- let mut tokens = Vec::new();
- while let Some(token) = merger.next() {
- tokens.push(token.clone());
+impl tantivy::tokenizer::Tokenizer for Tokenizer {
+ fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
+ match self {
+ Tokenizer::NormalTokenizer(tokenizer) => tokenizer.token_stream(text),
+ Tokenizer::StemmedTokenizer(tokenizer) => tokenizer.token_stream(text),
}
+ }
+}
- tokens.sort_by(|a, b| {
- a.offset_from
- .partial_cmp(&b.offset_from)
- .unwrap_or_else(|| a.offset_to.cmp(&b.offset_to))
- });
+impl tantivy::tokenizer::Tokenizer for NormalTokenizer {
+ fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
+ TextAnalyzer::from(SimpleTokenizer)
+ .filter(LowerCaser)
+ .token_stream(text)
+ }
+}
- tantivy::tokenizer::BoxTokenStream::from(PreTokenizedStream::from(PreTokenizedString {
- text: text.to_string(),
- tokens,
- }))
+impl tantivy::tokenizer::Tokenizer for StemmedTokenizer {
+ fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
+ let analyzer = TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser);
+ if let Some(lang) = whatlang::detect_lang(text) {
+ analyzer.filter(MyStemmer::from(lang).0).token_stream(text)
+ } else {
+ analyzer.token_stream(text)
+ }
}
}
diff --git a/src/webpage.rs b/src/webpage.rs
index 64d41255..85a7d70b 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -383,18 +383,22 @@ impl<'a> Html<'a> {
.unwrap_or_else(|| panic!("Unknown field: {}", field.as_str()));
match field {
- Field::Title => {
- if self.title().is_none() {
+ Field::Title | Field::StemmedTitle => {
+ let title = self.title();
+
+ if title.is_none() {
return Err(Error::EmptyField("title"));
}
- doc.add_text(tantivy_field, self.title().unwrap())
+ doc.add_text(tantivy_field, title.unwrap())
}
- Field::Body => {
+ Field::Body | Field::StemmedBody => {
let text = self.text();
+
if text.is_none() {
return Err(Error::EmptyField("body"));
}
+
doc.add_text(tantivy_field, text.unwrap())
}
Field::Url => doc.add_text(tantivy_field, self.url()),