web search done right
add additional fields to the schema that might be used in the future to reduce 'duplicates feeling' in search results
Mikkel Denkerf4446dcd1109
diff --git a/src/index.rs b/src/index.rsindex 81fa492d..19dd506c 100644--- a/src/index.rs+++ b/src/index.rs@@ -50,7 +50,7 @@ pub struct Index {primary_image_store: PrimaryImageStore,favicon_downloader: ImageDownloader<String>,primary_image_downloader: ImageDownloader<Uuid>,- spell_dictionary: Dictionary<1_000_000>,+ spell_dictionary: Dictionary<100_000>,pub region_count: RegionCount,pub path: String,}diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex e3c991b9..b9e29cc1 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -388,9 +388,12 @@ impl From<Document> for RetrievedWebpage {| Field::StemmedTitle| Field::CleanBody| Field::DomainIfHomepage+ | Field::TitleIfHomepage| Field::IsHomepage| Field::SiteHash| Field::UrlWithoutQueryHash+ | Field::TitleHash+ | Field::UrlHash| Field::NumTrackers| Field::PreComputedScore| Field::NumCleanBodyTokensdiff --git a/src/schema.rs b/src/schema.rsindex 8ef7d10f..2ae525e8 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -38,6 +38,8 @@ pub enum Field {DomainIfHomepage,/// this field is only set if the webpage is the homepage for the site. Allows us to boostDomainNameIfHomepageNoTokenizer,+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost+ TitleIfHomepage,IsHomepage,BacklinkText,HostCentrality,@@ -54,10 +56,12 @@ pub enum Field {NumDescriptionTokens,SiteHash,UrlWithoutQueryHash,+ TitleHash,+ UrlHash,PreComputedScore,}-pub static ALL_FIELDS: [Field; 29] = [+pub static ALL_FIELDS: [Field; 32] = [Field::Title,Field::CleanBody,Field::StemmedTitle,@@ -70,6 +74,7 @@ pub static ALL_FIELDS: [Field; 29] = [Field::DomainNoTokenizer,Field::DomainIfHomepage,Field::DomainNameIfHomepageNoTokenizer,+ Field::TitleIfHomepage,Field::IsHomepage,Field::BacklinkText,Field::HostCentrality,@@ -86,6 +91,8 @@ pub static ALL_FIELDS: [Field; 29] = [Field::NumDescriptionTokens,Field::SiteHash,Field::UrlWithoutQueryHash,+ Field::TitleHash,+ Field::UrlHash,Field::PreComputedScore,];@@ -126,6 +133,7 @@ impl Field {),Field::AllBody => IndexingOption::Text(self.default_text_options().set_stored()),Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),+ Field::TitleIfHomepage => IndexingOption::Text(self.default_text_options()),Field::DomainNameIfHomepageNoTokenizer => IndexingOption::Text(TextOptions::default().set_indexing_options(TextFieldIndexing::default()@@ -205,6 +213,12 @@ impl Field {Field::UrlWithoutQueryHash => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::MultiValues),),+ Field::UrlHash => IndexingOption::Integer(+ NumericOptions::default().set_fast(Cardinality::MultiValues),+ ),+ Field::TitleHash => IndexingOption::Integer(+ NumericOptions::default().set_fast(Cardinality::MultiValues),+ ),Field::PreComputedScore => IndexingOption::Float(NumericOptions::default().set_fast(Cardinality::SingleValue)@@ -245,12 +259,16 @@ impl Field {Field::SiteHash => "site_hash",Field::UrlWithoutQueryHash => "url_without_query_hash",Field::PreComputedScore => "pre_computed_score",+ Field::TitleIfHomepage => "title_if_homepage",+ Field::TitleHash => "title_hash",+ Field::UrlHash => "url_hash",}}pub fn boost(&self) -> Option<f32> {match self {Field::Site => Some(3.0),+ Field::TitleIfHomepage => Some(2.0),Field::DomainIfHomepage => Some(10.0),Field::DomainNameIfHomepageNoTokenizer => Some(30.0),Field::StemmedCleanBody | Field::StemmedTitle => Some(0.1),@@ -263,6 +281,8 @@ impl Field {Field::HostCentrality| Field::PageCentrality| Field::SiteHash+ | Field::TitleHash+ | Field::UrlHash| Field::UrlWithoutQueryHash| Field::SiteNoTokenizer| Field::DomainNoTokenizer@@ -300,6 +320,8 @@ impl Field {| Field::NumDescriptionTokens| Field::SiteHash| Field::UrlWithoutQueryHash+ | Field::TitleHash+ | Field::UrlHash| Field::PreComputedScore)}@@ -329,6 +351,9 @@ impl Field {"url_without_query_hash" => Some(Field::UrlWithoutQueryHash),"domain_name_if_homepage_no_tokenizer" => Some(Field::DomainNameIfHomepageNoTokenizer),"pre_computed_score" => Some(Field::PreComputedScore),+ "title_if_homepage" => Some(Field::TitleIfHomepage),+ "url_hash" => Some(Field::UrlHash),+ "title_hash" => Some(Field::TitleHash),_ => None,}}diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 62bfe623..200d7da4 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -593,6 +593,13 @@ impl Html {}],},),+ Field::TitleIfHomepage => {+ if self.url().is_homepage() {+ doc.add_pre_tokenized_text(tantivy_field, title.clone());+ } else {+ doc.add_text(tantivy_field, "");+ }+ }Field::DomainIfHomepage => {if self.url().is_homepage() {doc.add_text(tantivy_field, self.url().domain());@@ -649,6 +656,18 @@ impl Html {doc.add_u64(tantivy_field, u64s[0]);doc.add_u64(tantivy_field, u64s[1]);}+ Field::UrlHash => {+ let hash = hash(self.url().full()).0;+ let u64s = split_u128(hash);+ doc.add_u64(tantivy_field, u64s[0]);+ doc.add_u64(tantivy_field, u64s[1]);+ }+ Field::TitleHash => {+ let hash = hash(self.title().unwrap_or_default()).0;+ let u64s = split_u128(hash);+ doc.add_u64(tantivy_field, u64s[0]);+ doc.add_u64(tantivy_field, u64s[1]);+ }Field::BacklinkText| Field::HostCentrality| Field::PageCentrality