web search done right
Separate stemmed and non-stemmed fields. This allows us to prioritize non-stemmed matches over stemmed matches.
Mikkel Denker0632110a8d84
diff --git a/src/index.rs b/src/index.rsindex a2d43497..3c881986 100644--- a/src/index.rs+++ b/src/index.rs@@ -54,9 +54,15 @@ impl Index {tantivy::Index::create_in_dir(path.as_ref(), schema.clone())?};+ let tokenizer = Tokenizer::default();tantivy_index.tokenizers()- .register("tokenizer", Tokenizer::default());+ .register(tokenizer.as_str(), tokenizer);++ let tokenizer = Tokenizer::new_stemmed();+ tantivy_index+ .tokenizers()+ .register(tokenizer.as_str(), tokenizer);let writer = tantivy_index.writer(10_000_000_000)?;@@ -218,7 +224,11 @@ impl From<Document> for RetrievedWebpage {.expect("Url field should be text").to_string()}- Field::BacklinkText | Field::Centrality | Field::Host => {}+ Field::BacklinkText+ | Field::Centrality+ | Field::Host+ | Field::StemmedTitle+ | Field::StemmedBody => {}}}diff --git a/src/schema.rs b/src/schema.rsindex 2241b736..6428418b 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -18,20 +18,26 @@ use tantivy::schema::{Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,};+use crate::tokenizer::{NormalTokenizer, StemmedTokenizer};+pub const CENTRALITY_SCALING: u64 = 1_000_000_000;#[derive(Clone)]pub enum Field {Title,Body,+ StemmedTitle,+ StemmedBody,Url,Host,BacklinkText,Centrality,}-pub static ALL_FIELDS: [Field; 6] = [+pub static ALL_FIELDS: [Field; 8] = [Field::Title,Field::Body,+ Field::StemmedTitle,+ Field::StemmedBody,Field::Url,Field::Host,Field::BacklinkText,@@ -39,14 +45,21 @@ pub static ALL_FIELDS: [Field; 6] = [];impl Field {- fn default_text_options(&self) -> tantivy::schema::TextOptions {+ fn default_text_options_with_tokenizer(+ &self,+ tokenizer_name: &str,+ ) -> tantivy::schema::TextOptions {TextOptions::default().set_indexing_options(TextFieldIndexing::default()- .set_tokenizer("tokenizer")+ .set_tokenizer(tokenizer_name).set_index_option(IndexRecordOption::WithFreqsAndPositions),)}+ fn default_text_options(&self) -> tantivy::schema::TextOptions {+ self.default_text_options_with_tokenizer(NormalTokenizer::as_str())+ }+fn options(&self) -> IndexingOption {match self {Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),@@ -59,6 +72,12 @@ impl Field {.set_fast(Cardinality::SingleValue).set_indexed(),),+ Field::StemmedTitle => IndexingOption::Text(+ self.default_text_options_with_tokenizer(StemmedTokenizer::as_str()),+ ),+ Field::StemmedBody => IndexingOption::Text(+ self.default_text_options_with_tokenizer(StemmedTokenizer::as_str()),+ ),}}@@ -70,12 +89,15 @@ impl Field {Field::Host => "host",Field::BacklinkText => "backlink_text",Field::Centrality => "centrality",+ Field::StemmedTitle => "stemmed_title",+ Field::StemmedBody => "stemmed_body",}}pub fn boost(&self) -> Option<f32> {match self {Field::Host => Some(5.0),+ Field::StemmedBody | Field::StemmedTitle => Some(0.5),Field::Title | Field::Body | Field::BacklinkText | Field::Centrality | Field::Url => {None}diff --git a/src/tokenizer.rs b/src/tokenizer.rsindex 9cda808c..3dc2aba5 100644--- a/src/tokenizer.rs+++ b/src/tokenizer.rs@@ -14,52 +14,9 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use tantivy::tokenizer::{- Language, LowerCaser, PreTokenizedStream, PreTokenizedString, SimpleTokenizer, Stemmer,- TextAnalyzer, TokenStream,-};+use tantivy::tokenizer::{Language, LowerCaser, SimpleTokenizer, Stemmer, TextAnalyzer};use whatlang::Lang;-struct TokenStreamMerger<'a> {- streams: Vec<tantivy::tokenizer::BoxTokenStream<'a>>,- current_stream: usize,-}--impl<'a> TokenStreamMerger<'a> {- fn merge(streams: Vec<tantivy::tokenizer::BoxTokenStream<'a>>) -> Self {- Self {- streams,- current_stream: 0,- }- }-}--impl<'a> tantivy::tokenizer::TokenStream for TokenStreamMerger<'a> {- fn advance(&mut self) -> bool {- if let Some(stream) = self.streams.get_mut(self.current_stream) {- let mut has_more = stream.advance();-- if !has_more {- self.current_stream += 1;- has_more = self.current_stream < self.streams.len()- && self.streams[self.current_stream].advance();- }-- has_more- } else {- false- }- }-- fn token(&self) -> &tantivy::tokenizer::Token {- self.streams[self.current_stream].token()- }-- fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {- self.streams[self.current_stream].token_mut()- }-}-struct MyStemmer(Stemmer);impl From<Lang> for MyStemmer {@@ -86,36 +43,72 @@ impl From<Lang> for MyStemmer {}}-#[derive(Clone, Default)]-pub struct Tokenizer {}+#[derive(Clone)]+pub enum Tokenizer {+ NormalTokenizer(NormalTokenizer),+ StemmedTokenizer(StemmedTokenizer),+}-impl tantivy::tokenizer::Tokenizer for Tokenizer {- fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {- let analyzer = TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser);+impl Tokenizer {+ pub fn new_stemmed() -> Self {+ Self::StemmedTokenizer(Default::default())+ }+ pub fn as_str(&self) -> &'static str {+ match self {+ Tokenizer::NormalTokenizer(_) => NormalTokenizer::as_str(),+ Tokenizer::StemmedTokenizer(_) => StemmedTokenizer::as_str(),+ }+ }+}- let mut streams = vec![analyzer.token_stream(text)];+impl Default for Tokenizer {+ fn default() -> Self {+ Self::NormalTokenizer(Default::default())+ }+}- if let Some(lang) = whatlang::detect_lang(text) {- let stemmed_analyser = analyzer.clone().filter(MyStemmer::from(lang).0);- streams.push(stemmed_analyser.token_stream(text));- }+#[derive(Clone, Default)]+pub struct NormalTokenizer {}- let mut merger = TokenStreamMerger::merge(streams);+impl NormalTokenizer {+ pub fn as_str() -> &'static str {+ "tokenizer"+ }+}++#[derive(Clone, Default)]+pub struct StemmedTokenizer {}++impl StemmedTokenizer {+ pub fn as_str() -> &'static str {+ "stemmed_tokenizer"+ }+}- let mut tokens = Vec::new();- while let Some(token) = merger.next() {- tokens.push(token.clone());+impl tantivy::tokenizer::Tokenizer for Tokenizer {+ fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {+ match self {+ Tokenizer::NormalTokenizer(tokenizer) => tokenizer.token_stream(text),+ Tokenizer::StemmedTokenizer(tokenizer) => tokenizer.token_stream(text),}+ }+}- tokens.sort_by(|a, b| {- a.offset_from- .partial_cmp(&b.offset_from)- .unwrap_or_else(|| a.offset_to.cmp(&b.offset_to))- });+impl tantivy::tokenizer::Tokenizer for NormalTokenizer {+ fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {+ TextAnalyzer::from(SimpleTokenizer)+ .filter(LowerCaser)+ .token_stream(text)+ }+}- tantivy::tokenizer::BoxTokenStream::from(PreTokenizedStream::from(PreTokenizedString {- text: text.to_string(),- tokens,- }))+impl tantivy::tokenizer::Tokenizer for StemmedTokenizer {+ fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {+ let analyzer = TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser);+ if let Some(lang) = whatlang::detect_lang(text) {+ analyzer.filter(MyStemmer::from(lang).0).token_stream(text)+ } else {+ analyzer.token_stream(text)+ }}}diff --git a/src/webpage.rs b/src/webpage.rsindex 64d41255..85a7d70b 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -383,18 +383,22 @@ impl<'a> Html<'a> {.unwrap_or_else(|| panic!("Unknown field: {}", field.as_str()));match field {- Field::Title => {- if self.title().is_none() {+ Field::Title | Field::StemmedTitle => {+ let title = self.title();++ if title.is_none() {return Err(Error::EmptyField("title"));}- doc.add_text(tantivy_field, self.title().unwrap())+ doc.add_text(tantivy_field, title.unwrap())}- Field::Body => {+ Field::Body | Field::StemmedBody => {let text = self.text();+if text.is_none() {return Err(Error::EmptyField("body"));}+doc.add_text(tantivy_field, text.unwrap())}Field::Url => doc.add_text(tantivy_field, self.url()),