web search done right
Optimizations to index schema and indexing. There seems to be a problem with large fast-fields, both in terms of crashes and performance. We therefore now check whether a query-document match is navigational by indexing the host string and boosting the field during search. In the future, we may want to add a boolean fast-field to indicate if the given webpage is the homepage and only do navigational searches on those. The commit also increases indexing ressources in the hopes that this reduces the number of merges during indexing.
Mikkel Denker383d84266f28
diff --git a/src/index.rs b/src/index.rsindex 6442f38f..20fce9ee 100644--- a/src/index.rs+++ b/src/index.rs@@ -15,6 +15,7 @@ use serde::{Deserialize, Serialize};// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use tantivy::collector::{Collector, Count};+use tantivy::merge_policy::LogMergePolicy;use tantivy::{DocAddress, Document, LeasedItem};use crate::directory::{self, DirEntry};@@ -57,8 +58,15 @@ impl Index {.tokenizers().register("tokenizer", Tokenizer::default());+ let writer = tantivy_index.writer(10_000_000_000)?;++ let mut merge_policy = LogMergePolicy::default();+ merge_policy.set_max_docs_before_merge(100_000_000);++ writer.set_merge_policy(Box::new(merge_policy));+Ok(Index {- writer: tantivy_index.writer(100_000_000)?,+ writer,reader: tantivy_index.reader()?,schema,path: path.as_ref().to_str().unwrap().to_string(),@@ -198,7 +206,7 @@ impl From<Document> for RetrievedWebpage {.expect("Url field should be text").to_string()}- Field::BacklinkText | Field::Centrality | Field::FastUrl => {}+ Field::BacklinkText | Field::Centrality | Field::Host => {}}}diff --git a/src/query.rs b/src/query.rsindex 9be68219..69b16205 100644--- a/src/query.rs+++ b/src/query.rs@@ -13,7 +13,7 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{Error, Result};+use crate::{schema::ALL_FIELDS, Error, Result};use nom::{bytes::complete::{tag, take_while},character::complete::multispace0,@@ -23,7 +23,7 @@ use nom::{IResult,};use tantivy::{- query::{BooleanQuery, Occur, TermQuery},+ query::{BooleanQuery, BoostQuery, Occur, TermQuery},schema::{Field, FieldEntry, IndexRecordOption, Schema},tokenizer::{TextAnalyzer, TokenizerManager},Term,@@ -130,11 +130,14 @@ impl Query {)}).collect();+ let boost = ALL_FIELDS[field.field_id() as usize].boost();(Occur::Should,- Box::new(BooleanQuery::new(processed_queries))- as Box<dyn tantivy::query::Query>,+ Box::new(BoostQuery::new(+ Box::new(BooleanQuery::new(processed_queries)),+ boost.unwrap_or(1.0),+ )) as Box<dyn tantivy::query::Query>,)}).collect();diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex dc3f7513..bd0dcc92 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -14,41 +14,16 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::query::Query;-use crate::schema::Field;-use crate::webpage;-use std::sync::Arc;+use crate::schema::{Field, CENTRALITY_SCALING};use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};-use tantivy::fastfield::{BytesFastFieldReader, DynamicFastFieldReader, FastFieldReader};+use tantivy::fastfield::{DynamicFastFieldReader, FastFieldReader};use tantivy::{DocId, Score, SegmentReader};-pub(crate) struct InitialScoreTweaker {- query: Arc<Query>,-}--impl InitialScoreTweaker {- pub fn new(query: Arc<Query>) -> Self {- Self { query }- }-}+#[derive(Default)]+pub(crate) struct InitialScoreTweaker {}pub(crate) struct InitialSegmentScoreTweaker {- centrality_reader: DynamicFastFieldReader<f64>,- fast_url_reader: BytesFastFieldReader,- sorted_dedupped_terms: Vec<String>,-}--impl InitialSegmentScoreTweaker {- fn navigational_score(&self, sorted_dedupped_terms: &[String], doc: DocId) -> f64 {- let bytes = self.fast_url_reader.get_bytes(doc);- let url = bincode::deserialize(bytes).expect("Failed to deserialize domain");-- if is_navigational(sorted_dedupped_terms, url) {- 1.0- } else {- 0.0- }- }+ centrality_reader: DynamicFastFieldReader<u64>,}impl ScoreTweaker<f64> for InitialScoreTweaker {@@ -61,87 +36,16 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {.expect("Faild to load centrality field");let centrality_reader = segment_reader.fast_fields()- .f64(centrality_field)+ .u64(centrality_field).expect("Failed to get centrality fast-field reader");- let fast_url_field = segment_reader- .schema()- .get_field(Field::FastUrl.as_str())- .expect("Faild to load domain field");- let fast_url_reader = segment_reader- .fast_fields()- .bytes(fast_url_field)- .expect("Failed to get domain fast-field reader");-- let mut sorted_dedupped_terms = self.query.terms.clone();-- sorted_dedupped_terms.sort();- sorted_dedupped_terms.dedup();-- Ok(InitialSegmentScoreTweaker {- centrality_reader,- fast_url_reader,- sorted_dedupped_terms,- })+ Ok(InitialSegmentScoreTweaker { centrality_reader })}}impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {fn score(&mut self, doc: DocId, score: Score) -> f64 {- let centrality = self.centrality_reader.get(doc);- let navigational = self.navigational_score(&self.sorted_dedupped_terms, doc);- score as f64 + 10_000.0 * centrality + 100.0 * navigational- }-}--fn jaccard_sim(sorted_dedupped_terms: &[String], domain_parts: &[&str]) -> f64 {- debug_assert_eq!(domain_parts.len(), 2);-- let intersection_size: f64 = sorted_dedupped_terms- .iter()- .map(|term| {- if domain_parts.contains(&term.as_str()) {- 1.0- } else {- 0.0- }- })- .sum();-- intersection_size- / (sorted_dedupped_terms.len() as f64 + domain_parts.len() as f64 - intersection_size)-}--fn is_navigational(sorted_dedupped_terms: &[String], url: &str) -> bool {- if !webpage::is_homepage(url) {- return false;- }- let domain = webpage::domain(url);-- let (name, remaining) = domain.split_once('.').unwrap();- jaccard_sim(sorted_dedupped_terms, &[name, remaining]) >= 0.5-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn test_jaccard_sim() {- assert_eq!(- jaccard_sim(&["dr".to_string(), "dk".to_string()], &["dr", "dk"]),- 1.0- );-- assert_eq!(jaccard_sim(&["dr".to_string()], &["dr", "dk"]), 0.5);- }-- #[test]- fn navigational() {- assert!(is_navigational(&["dr".to_string()], "dr.dk"));- assert!(is_navigational(- &["dk".to_string(), "dr".to_string()],- "dr.dk"- ));+ let centrality: f64 = self.centrality_reader.get(doc) as f64 / CENTRALITY_SCALING as f64;+ score as f64 + 10_000.0 * centrality}}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 8bdef605..b1229e3f 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -17,25 +17,19 @@pub mod centrality_store;mod initial;-use std::sync::Arc;-use crate::query::Query;use initial::InitialScoreTweaker;use tantivy::collector::{Collector, TopDocs};-pub struct Ranker {- query: Arc<Query>,-}+pub struct Ranker {}impl Ranker {- pub fn new(query: Query) -> Self {- Ranker {- query: Arc::new(query),- }+ pub fn new(_query: Query) -> Self {+ Ranker {}}pub fn collector(&self) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {- let score_tweaker = InitialScoreTweaker::new(self.query.clone());+ let score_tweaker = InitialScoreTweaker::default();TopDocs::with_limit(20).tweak_score(score_tweaker)}}@@ -159,7 +153,7 @@ mod tests {assert_eq!(result.documents.len(), 3);assert_eq!(result.documents[0].url, "https://www.dr.dk");- assert_eq!(result.documents[1].url, "https://www.b.com");- assert_eq!(result.documents[2].url, "https://www.dr.dk/whatever");+ assert_eq!(result.documents[1].url, "https://www.dr.dk/whatever");+ assert_eq!(result.documents[2].url, "https://www.b.com");}}diff --git a/src/schema.rs b/src/schema.rsindex 44d54af1..286de329 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -15,15 +15,17 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use tantivy::schema::{- BytesOptions, Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,+ Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,};+pub const CENTRALITY_SCALING: u64 = 1_000_000_000;+#[derive(Clone)]pub enum Field {Title,Body,Url,- FastUrl,+ Host,BacklinkText,Centrality,}@@ -31,7 +33,7 @@ pub static ALL_FIELDS: [Field; 6] = [Field::Title,Field::Body,Field::Url,- Field::FastUrl,+ Field::Host,Field::BacklinkText,Field::Centrality,];@@ -50,9 +52,7 @@ impl Field {Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),Field::Body => IndexingOption::Text(self.default_text_options().set_stored()),Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),- Field::FastUrl => {- IndexingOption::Bytes(BytesOptions::default().set_fast().set_indexed())- }+ Field::Host => IndexingOption::Text(self.default_text_options()),Field::BacklinkText => IndexingOption::Text(self.default_text_options()),Field::Centrality => IndexingOption::Numeric(NumericOptions::default()@@ -67,11 +67,20 @@ impl Field {Field::Title => "title",Field::Body => "body",Field::Url => "url",- Field::FastUrl => "fast_url",+ Field::Host => "host",Field::BacklinkText => "backlink_text",Field::Centrality => "centrality",}}++ pub fn boost(&self) -> Option<f32> {+ match self {+ Field::Host => Some(30.0),+ Field::Title | Field::Body | Field::BacklinkText | Field::Centrality | Field::Url => {+ None+ }+ }+ }}pub fn create_schema() -> tantivy::schema::Schema {@@ -80,8 +89,7 @@ pub fn create_schema() -> tantivy::schema::Schema {for field in &ALL_FIELDS {match field.options() {IndexingOption::Text(options) => builder.add_text_field(field.as_str(), options),- IndexingOption::Numeric(options) => builder.add_f64_field(field.as_str(), options),- IndexingOption::Bytes(options) => builder.add_bytes_field(field.as_str(), options),+ IndexingOption::Numeric(options) => builder.add_u64_field(field.as_str(), options),};}@@ -91,5 +99,4 @@ pub fn create_schema() -> tantivy::schema::Schema {enum IndexingOption {Text(tantivy::schema::TextOptions),Numeric(tantivy::schema::NumericOptions),- Bytes(tantivy::schema::BytesOptions),}diff --git a/src/webpage.rs b/src/webpage.rsindex 12bc5536..39554f30 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -20,7 +20,7 @@ use scraper::{Node, Selector};use std::collections::BTreeMap;use tl::Children;-use crate::schema::{Field, ALL_FIELDS};+use crate::schema::{Field, ALL_FIELDS, CENTRALITY_SCALING};pub fn strip_protocol(url: &str) -> &'_ str {let mut start_host = 0;@@ -112,11 +112,11 @@ impl<'a> Webpage<'a> {backlink_text,);- doc.add_f64(+ doc.add_u64(schema.get_field(Field::Centrality.as_str()).expect("Failed to get centrality field"),- self.centrality,+ (self.centrality * CENTRALITY_SCALING as f64) as u64,);Ok(doc)@@ -397,10 +397,7 @@ impl<'a> Html<'a> {doc.add_text(tantivy_field, text.unwrap())}Field::Url => doc.add_text(tantivy_field, self.url()),- Field::FastUrl => doc.add_bytes(- tantivy_field,- bincode::serialize(self.url()).expect("Failed to serialize bytes"),- ),+ Field::Host => doc.add_text(tantivy_field, self.host()),Field::BacklinkText | Field::Centrality => {}}}