Skip to content

web search done right

Make index smaller by not storing positions for unecesarry fields

Mikkel Denkera2fcf0221868
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index 9f01d033..b76354f8 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -685,7 +685,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let query = SearchQuery {
- query: "\"This is a test website\"".to_string(),
+ query: "\"Test website\"".to_string(),
..Default::default()
};
let result = searcher.search(&query).expect("Search failed");
@@ -694,7 +694,7 @@ mod tests {
assert_eq!(result.webpages[0].url, "https://www.first.com");
let query = SearchQuery {
- query: "\"This is a test website\" site:www.second.com".to_string(),
+ query: "\"Test website\" site:www.second.com".to_string(),
..Default::default()
};
let result = searcher.search(&query).expect("Search failed");
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index 26410db1..5dd55ec3 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -16,7 +16,6 @@
use tantivy::{
query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, TermQuery},
- schema::IndexRecordOption,
tokenizer::{TextAnalyzer, TokenizerManager},
};
@@ -107,6 +106,7 @@ impl Term {
for (field, entry) in fields
.iter()
.filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].is_searchable())
+ .filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].has_pos())
{
phrases.push((
Occur::Should,
@@ -250,15 +250,18 @@ impl Term {
let analyzer = Term::get_tantivy_analyzer(entry, tokenizer_manager);
let mut processed_terms = Term::process_tantivy_term(term, analyzer, *field);
- let processed_query = if processed_terms.len() > 1 {
- Box::new(PhraseQuery::new(processed_terms)) as Box<dyn tantivy::query::Query>
- } else {
- let term = processed_terms.pop().unwrap();
- Box::new(TermQuery::new(
- term,
- IndexRecordOption::WithFreqsAndPositions,
- ))
- };
+ let processed_query =
+ if processed_terms.len() > 1 && ALL_FIELDS[field.field_id() as usize].has_pos() {
+ Box::new(PhraseQuery::new(processed_terms)) as Box<dyn tantivy::query::Query>
+ } else {
+ let option = ALL_FIELDS[field.field_id() as usize]
+ .as_text()
+ .unwrap()
+ .index_option();
+
+ let term = processed_terms.remove(0);
+ Box::new(TermQuery::new(term, option))
+ };
let boost =
field_boost.get(&ALL_FIELDS[field.field_id() as usize].as_text().unwrap()) as f32;
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 152c6377..2b0ef4de 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -74,6 +74,39 @@ impl TextField {
TextField::FlattenedSchemaOrgJson => Tokenizer::Json(JsonField),
}
}
+
+ pub fn index_option(&self) -> IndexRecordOption {
+ if self.has_pos() {
+ IndexRecordOption::WithFreqsAndPositions
+ } else {
+ IndexRecordOption::WithFreqs
+ }
+ }
+
+ pub fn has_pos(&self) -> bool {
+ match self {
+ TextField::Title => true,
+ TextField::CleanBody => true,
+ TextField::StemmedTitle => false,
+ TextField::StemmedCleanBody => false,
+ TextField::AllBody => false,
+ TextField::Url => true,
+ TextField::Site => true,
+ TextField::Domain => true,
+ TextField::SiteNoTokenizer => false,
+ TextField::DomainNoTokenizer => false,
+ TextField::DomainIfHomepage => false,
+ TextField::DomainNameIfHomepageNoTokenizer => false,
+ TextField::TitleIfHomepage => false,
+ TextField::BacklinkText => false,
+ TextField::PrimaryImage => false,
+ TextField::Description => true,
+ TextField::HostTopic => false,
+ TextField::DmozDescription => true,
+ TextField::SchemaOrgJson => false,
+ TextField::FlattenedSchemaOrgJson => true,
+ }
+ }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -169,13 +202,22 @@ pub static ALL_FIELDS: [Field; 48] = [
impl Field {
fn default_text_options(&self) -> tantivy::schema::TextOptions {
let tokenizer = self.as_text().unwrap().tokenizer();
+ let option = self.as_text().unwrap().index_option();
+
TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer(tokenizer.as_str())
- .set_index_option(IndexRecordOption::WithFreqsAndPositions),
+ .set_index_option(option),
)
}
+ pub fn has_pos(&self) -> bool {
+ match self {
+ Field::Fast(_) => false,
+ Field::Text(text) => text.has_pos(),
+ }
+ }
+
pub fn options(&self) -> IndexingOption {
match self {
Field::Text(TextField::Title) => {
@@ -193,9 +235,7 @@ impl Field {
Field::Text(TextField::DomainNoTokenizer) => {
IndexingOption::Text(self.default_text_options())
}
- Field::Text(TextField::AllBody) => {
- IndexingOption::Text(self.default_text_options().set_stored())
- }
+ Field::Text(TextField::AllBody) => IndexingOption::Text(self.default_text_options()),
Field::Text(TextField::DomainIfHomepage) => {
IndexingOption::Text(self.default_text_options())
}
@@ -230,7 +270,7 @@ impl Field {
IndexingOption::Text(self.default_text_options().set_stored())
}
Field::Text(TextField::FlattenedSchemaOrgJson) => {
- IndexingOption::Text(self.default_text_options().set_stored())
+ IndexingOption::Text(self.default_text_options())
}
Field::Fast(FastField::IsHomepage) => IndexingOption::Integer(
NumericOptions::default()
diff --git a/core/src/summarizer.rs b/core/src/summarizer.rs
index cb417f6a..6ec1c565 100644
--- a/core/src/summarizer.rs
+++ b/core/src/summarizer.rs
@@ -903,6 +903,12 @@ impl BeamState {
if Self::all_beams_agree(&beams, beam_token_agreement_idx) {
let tok = beams[0].input_ids[beam_token_agreement_idx];
+ // for some reason, [17, 27] decodes to "'" or something like that.
+ // If the tokens are decoded separately, they correspond to some characters
+ // that shows up as errors in a string.
+ // This hack skips the token 17 and simply decodes 27 as "'".
+ // While this might not be correct decoding in all scenarios, it seems to work
+ // most of the time.
if tok == 17 {
self = BeamState::DecodingWithPast {
encoder_hidden_states,
@@ -933,6 +939,7 @@ impl BeamState {
generated_tokens,
beam_token_agreement_idx: beam_token_agreement_idx + 1,
};
+
let decoded = model.tokenizer.decode(vec![tok as u32], true)?;
return Ok((self, Some(decoded)));