web search done right
added domain_name_if_hompage field for better navigation queries
Mikkel Denker0da016f67823
diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex 132e30b8..fe8b7bae 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -369,6 +369,7 @@ impl From<Document> for RetrievedWebpage {| Field::Domain| Field::SiteNoTokenizer| Field::DomainNoTokenizer+ | Field::DomainNameIfHomepageNoTokenizer| Field::StemmedTitle| Field::CleanBody| Field::DomainIfHomepagediff --git a/src/schema.rs b/src/schema.rsindex ceabaf9d..1a377c57 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -36,6 +36,8 @@ pub enum Field {DomainNoTokenizer,/// this field is only set if the webpage is the homepage for the site. Allows us to boostDomainIfHomepage,+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost+ DomainNameIfHomepageNoTokenizer,IsHomepage,BacklinkText,HostCentrality,@@ -53,7 +55,7 @@ pub enum Field {SiteHash,}-pub static ALL_FIELDS: [Field; 26] = [+pub static ALL_FIELDS: [Field; 27] = [Field::Title,Field::CleanBody,Field::StemmedTitle,@@ -65,6 +67,7 @@ pub static ALL_FIELDS: [Field; 26] = [Field::SiteNoTokenizer,Field::DomainNoTokenizer,Field::DomainIfHomepage,+ Field::DomainNameIfHomepageNoTokenizer,Field::IsHomepage,Field::BacklinkText,Field::HostCentrality,@@ -119,6 +122,12 @@ impl Field {),Field::AllBody => IndexingOption::Text(self.default_text_options().set_stored()),Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),+ Field::DomainNameIfHomepageNoTokenizer => IndexingOption::Text(+ TextOptions::default().set_indexing_options(+ TextFieldIndexing::default()+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),+ ),+ ),Field::IsHomepage => IndexingOption::Numeric(NumericOptions::default().set_fast(Cardinality::SingleValue)@@ -207,6 +216,7 @@ impl Field {Field::StemmedTitle => "stemmed_title",Field::StemmedCleanBody => "stemmed_body",Field::DomainIfHomepage => "domain_if_homepage",+ Field::DomainNameIfHomepageNoTokenizer => "domain_name_if_homepage_no_tokenizer",Field::IsHomepage => "is_homepage",Field::FetchTimeMs => "fetch_time_ms",Field::PrimaryImage => "primary_image_uuid",@@ -226,7 +236,8 @@ impl Field {pub fn boost(&self) -> Option<f32> {match self {Field::Site => Some(6.0),- Field::DomainIfHomepage => Some(250.0),+ Field::DomainIfHomepage => Some(50.0),+ Field::DomainNameIfHomepageNoTokenizer => Some(50.0),Field::StemmedCleanBody | Field::StemmedTitle => Some(0.1),Field::CleanBody => Some(4.0),Field::Title => Some(10.0),@@ -296,6 +307,7 @@ impl Field {"num_trackers" => Some(Field::NumTrackers),"region" => Some(Field::Region),"site_hash" => Some(Field::SiteHash),+ "domain_name_if_homepage_no_tokenizer" => Some(Field::DomainNameIfHomepageNoTokenizer),_ => None,}}diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex a669e41c..e701270b 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -591,6 +591,25 @@ impl Html {doc.add_text(tantivy_field, "");}}+ Field::DomainNameIfHomepageNoTokenizer => {+ if self.url().is_homepage() {+ doc.add_pre_tokenized_text(+ tantivy_field,+ PreTokenizedString {+ text: self.url().domain_name().to_string(),+ tokens: vec![tantivy::tokenizer::Token {+ offset_from: 0,+ offset_to: self.url().domain_name().len(),+ position: 0,+ text: self.url().domain_name().to_string(),+ position_length: 1,+ }],+ },+ );+ } else {+ doc.add_text(tantivy_field, "");+ }+ }Field::IsHomepage => {doc.add_u64(tantivy_field, if self.url().is_homepage() { 1 } else { 0 });}diff --git a/src/webpage/url.rs b/src/webpage/url.rsindex 02780160..3c7ad612 100644--- a/src/webpage/url.rs+++ b/src/webpage/url.rs@@ -90,6 +90,16 @@ impl Url {}}+ pub fn domain_name(&self) -> &str {+ let domain = self.domain();++ if let Some(tld_start) = domain.find('.') {+ &domain[..tld_start]+ } else {+ domain+ }+ }+pub fn subdomain(&self) -> Option<&str> {if let Some(subdomain) = self.site().strip_suffix(self.domain()) {if subdomain.is_empty() || subdomain == "." {@@ -194,6 +204,7 @@ mod tests {let url: Url = "//scripts.dailymail.co.uk".to_string().into();assert_eq!(url.domain(), "dailymail.co.uk");+ assert_eq!(url.domain_name(), "dailymail");assert_eq!(url.site(), "scripts.dailymail.co.uk");}