Skip to content

web search done right

added domain_name_if_hompage field for better navigation queries

Mikkel Denker0da016f67823
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 132e30b8..fe8b7bae 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -369,6 +369,7 @@ impl From<Document> for RetrievedWebpage {
| Field::Domain
| Field::SiteNoTokenizer
| Field::DomainNoTokenizer
+ | Field::DomainNameIfHomepageNoTokenizer
| Field::StemmedTitle
| Field::CleanBody
| Field::DomainIfHomepage
diff --git a/src/schema.rs b/src/schema.rs
index ceabaf9d..1a377c57 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -36,6 +36,8 @@ pub enum Field {
DomainNoTokenizer,
/// this field is only set if the webpage is the homepage for the site. Allows us to boost
DomainIfHomepage,
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ DomainNameIfHomepageNoTokenizer,
IsHomepage,
BacklinkText,
HostCentrality,
@@ -53,7 +55,7 @@ pub enum Field {
SiteHash,
}
-pub static ALL_FIELDS: [Field; 26] = [
+pub static ALL_FIELDS: [Field; 27] = [
Field::Title,
Field::CleanBody,
Field::StemmedTitle,
@@ -65,6 +67,7 @@ pub static ALL_FIELDS: [Field; 26] = [
Field::SiteNoTokenizer,
Field::DomainNoTokenizer,
Field::DomainIfHomepage,
+ Field::DomainNameIfHomepageNoTokenizer,
Field::IsHomepage,
Field::BacklinkText,
Field::HostCentrality,
@@ -119,6 +122,12 @@ impl Field {
),
Field::AllBody => IndexingOption::Text(self.default_text_options().set_stored()),
Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),
+ Field::DomainNameIfHomepageNoTokenizer => IndexingOption::Text(
+ TextOptions::default().set_indexing_options(
+ TextFieldIndexing::default()
+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),
+ ),
+ ),
Field::IsHomepage => IndexingOption::Numeric(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
@@ -207,6 +216,7 @@ impl Field {
Field::StemmedTitle => "stemmed_title",
Field::StemmedCleanBody => "stemmed_body",
Field::DomainIfHomepage => "domain_if_homepage",
+ Field::DomainNameIfHomepageNoTokenizer => "domain_name_if_homepage_no_tokenizer",
Field::IsHomepage => "is_homepage",
Field::FetchTimeMs => "fetch_time_ms",
Field::PrimaryImage => "primary_image_uuid",
@@ -226,7 +236,8 @@ impl Field {
pub fn boost(&self) -> Option<f32> {
match self {
Field::Site => Some(6.0),
- Field::DomainIfHomepage => Some(250.0),
+ Field::DomainIfHomepage => Some(50.0),
+ Field::DomainNameIfHomepageNoTokenizer => Some(50.0),
Field::StemmedCleanBody | Field::StemmedTitle => Some(0.1),
Field::CleanBody => Some(4.0),
Field::Title => Some(10.0),
@@ -296,6 +307,7 @@ impl Field {
"num_trackers" => Some(Field::NumTrackers),
"region" => Some(Field::Region),
"site_hash" => Some(Field::SiteHash),
+ "domain_name_if_homepage_no_tokenizer" => Some(Field::DomainNameIfHomepageNoTokenizer),
_ => None,
}
}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index a669e41c..e701270b 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -591,6 +591,25 @@ impl Html {
doc.add_text(tantivy_field, "");
}
}
+ Field::DomainNameIfHomepageNoTokenizer => {
+ if self.url().is_homepage() {
+ doc.add_pre_tokenized_text(
+ tantivy_field,
+ PreTokenizedString {
+ text: self.url().domain_name().to_string(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: self.url().domain_name().len(),
+ position: 0,
+ text: self.url().domain_name().to_string(),
+ position_length: 1,
+ }],
+ },
+ );
+ } else {
+ doc.add_text(tantivy_field, "");
+ }
+ }
Field::IsHomepage => {
doc.add_u64(tantivy_field, if self.url().is_homepage() { 1 } else { 0 });
}
diff --git a/src/webpage/url.rs b/src/webpage/url.rs
index 02780160..3c7ad612 100644
--- a/src/webpage/url.rs
+++ b/src/webpage/url.rs
@@ -90,6 +90,16 @@ impl Url {
}
}
+ pub fn domain_name(&self) -> &str {
+ let domain = self.domain();
+
+ if let Some(tld_start) = domain.find('.') {
+ &domain[..tld_start]
+ } else {
+ domain
+ }
+ }
+
pub fn subdomain(&self) -> Option<&str> {
if let Some(subdomain) = self.site().strip_suffix(self.domain()) {
if subdomain.is_empty() || subdomain == "." {
@@ -194,6 +204,7 @@ mod tests {
let url: Url = "//scripts.dailymail.co.uk".to_string().into();
assert_eq!(url.domain(), "dailymail.co.uk");
+ assert_eq!(url.domain_name(), "dailymail");
assert_eq!(url.site(), "scripts.dailymail.co.uk");
}