web search done right
Make index smaller by not storing positions for unecesarry fields
Mikkel Denkera2fcf0221868
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rsindex 9f01d033..b76354f8 100644--- a/core/src/query/mod.rs+++ b/core/src/query/mod.rs@@ -685,7 +685,7 @@ mod tests {let searcher = LocalSearcher::from(index);let query = SearchQuery {- query: "\"This is a test website\"".to_string(),+ query: "\"Test website\"".to_string(),..Default::default()};let result = searcher.search(&query).expect("Search failed");@@ -694,7 +694,7 @@ mod tests {assert_eq!(result.webpages[0].url, "https://www.first.com");let query = SearchQuery {- query: "\"This is a test website\" site:www.second.com".to_string(),+ query: "\"Test website\" site:www.second.com".to_string(),..Default::default()};let result = searcher.search(&query).expect("Search failed");diff --git a/core/src/query/parser.rs b/core/src/query/parser.rsindex 26410db1..5dd55ec3 100644--- a/core/src/query/parser.rs+++ b/core/src/query/parser.rs@@ -16,7 +16,6 @@use tantivy::{query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, TermQuery},- schema::IndexRecordOption,tokenizer::{TextAnalyzer, TokenizerManager},};@@ -107,6 +106,7 @@ impl Term {for (field, entry) in fields.iter().filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].is_searchable())+ .filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].has_pos()){phrases.push((Occur::Should,@@ -250,15 +250,18 @@ impl Term {let analyzer = Term::get_tantivy_analyzer(entry, tokenizer_manager);let mut processed_terms = Term::process_tantivy_term(term, analyzer, *field);- let processed_query = if processed_terms.len() > 1 {- Box::new(PhraseQuery::new(processed_terms)) as Box<dyn tantivy::query::Query>- } else {- let term = processed_terms.pop().unwrap();- Box::new(TermQuery::new(- term,- IndexRecordOption::WithFreqsAndPositions,- ))- };+ let processed_query =+ if processed_terms.len() > 1 && ALL_FIELDS[field.field_id() as usize].has_pos() {+ Box::new(PhraseQuery::new(processed_terms)) as Box<dyn tantivy::query::Query>+ } else {+ let option = ALL_FIELDS[field.field_id() as usize]+ .as_text()+ .unwrap()+ .index_option();++ let term = processed_terms.remove(0);+ Box::new(TermQuery::new(term, option))+ };let boost =field_boost.get(&ALL_FIELDS[field.field_id() as usize].as_text().unwrap()) as f32;diff --git a/core/src/schema.rs b/core/src/schema.rsindex 152c6377..2b0ef4de 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -74,6 +74,39 @@ impl TextField {TextField::FlattenedSchemaOrgJson => Tokenizer::Json(JsonField),}}++ pub fn index_option(&self) -> IndexRecordOption {+ if self.has_pos() {+ IndexRecordOption::WithFreqsAndPositions+ } else {+ IndexRecordOption::WithFreqs+ }+ }++ pub fn has_pos(&self) -> bool {+ match self {+ TextField::Title => true,+ TextField::CleanBody => true,+ TextField::StemmedTitle => false,+ TextField::StemmedCleanBody => false,+ TextField::AllBody => false,+ TextField::Url => true,+ TextField::Site => true,+ TextField::Domain => true,+ TextField::SiteNoTokenizer => false,+ TextField::DomainNoTokenizer => false,+ TextField::DomainIfHomepage => false,+ TextField::DomainNameIfHomepageNoTokenizer => false,+ TextField::TitleIfHomepage => false,+ TextField::BacklinkText => false,+ TextField::PrimaryImage => false,+ TextField::Description => true,+ TextField::HostTopic => false,+ TextField::DmozDescription => true,+ TextField::SchemaOrgJson => false,+ TextField::FlattenedSchemaOrgJson => true,+ }+ }}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -169,13 +202,22 @@ pub static ALL_FIELDS: [Field; 48] = [impl Field {fn default_text_options(&self) -> tantivy::schema::TextOptions {let tokenizer = self.as_text().unwrap().tokenizer();+ let option = self.as_text().unwrap().index_option();+TextOptions::default().set_indexing_options(TextFieldIndexing::default().set_tokenizer(tokenizer.as_str())- .set_index_option(IndexRecordOption::WithFreqsAndPositions),+ .set_index_option(option),)}+ pub fn has_pos(&self) -> bool {+ match self {+ Field::Fast(_) => false,+ Field::Text(text) => text.has_pos(),+ }+ }+pub fn options(&self) -> IndexingOption {match self {Field::Text(TextField::Title) => {@@ -193,9 +235,7 @@ impl Field {Field::Text(TextField::DomainNoTokenizer) => {IndexingOption::Text(self.default_text_options())}- Field::Text(TextField::AllBody) => {- IndexingOption::Text(self.default_text_options().set_stored())- }+ Field::Text(TextField::AllBody) => IndexingOption::Text(self.default_text_options()),Field::Text(TextField::DomainIfHomepage) => {IndexingOption::Text(self.default_text_options())}@@ -230,7 +270,7 @@ impl Field {IndexingOption::Text(self.default_text_options().set_stored())}Field::Text(TextField::FlattenedSchemaOrgJson) => {- IndexingOption::Text(self.default_text_options().set_stored())+ IndexingOption::Text(self.default_text_options())}Field::Fast(FastField::IsHomepage) => IndexingOption::Integer(NumericOptions::default()diff --git a/core/src/summarizer.rs b/core/src/summarizer.rsindex cb417f6a..6ec1c565 100644--- a/core/src/summarizer.rs+++ b/core/src/summarizer.rs@@ -903,6 +903,12 @@ impl BeamState {if Self::all_beams_agree(&beams, beam_token_agreement_idx) {let tok = beams[0].input_ids[beam_token_agreement_idx];+ // for some reason, [17, 27] decodes to "'" or something like that.+ // If the tokens are decoded separately, they correspond to some characters+ // that shows up as errors in a string.+ // This hack skips the token 17 and simply decodes 27 as "'".+ // While this might not be correct decoding in all scenarios, it seems to work+ // most of the time.if tok == 17 {self = BeamState::DecodingWithPast {encoder_hidden_states,@@ -933,6 +939,7 @@ impl BeamState {generated_tokens,beam_token_agreement_idx: beam_token_agreement_idx + 1,};+let decoded = model.tokenizer.decode(vec![tok as u32], true)?;return Ok((self, Some(decoded)));