web search done right
Better navigational ranking
Mikkel Denker97583078a60d
diff --git a/src/index.rs b/src/index.rsindex 3c881986..2f914f58 100644--- a/src/index.rs+++ b/src/index.rs@@ -228,7 +228,10 @@ impl From<Document> for RetrievedWebpage {| Field::Centrality| Field::Host| Field::StemmedTitle- | Field::StemmedBody => {}+ | Field::StemmedBody+ | Field::Domain+ | Field::DomainIfHomepage+ | Field::IsHomepage => {}}}diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex bd0dcc92..d11dc09e 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -24,6 +24,7 @@ pub(crate) struct InitialScoreTweaker {}pub(crate) struct InitialSegmentScoreTweaker {centrality_reader: DynamicFastFieldReader<u64>,+ is_homepage_reader: DynamicFastFieldReader<u64>,}impl ScoreTweaker<f64> for InitialScoreTweaker {@@ -39,13 +40,28 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {.u64(centrality_field).expect("Failed to get centrality fast-field reader");- Ok(InitialSegmentScoreTweaker { centrality_reader })+ let is_homepage_field = segment_reader+ .schema()+ .get_field(Field::IsHomepage.as_str())+ .expect("Faild to load is_homepage field");+ let is_homepage_reader = segment_reader+ .fast_fields()+ .u64(is_homepage_field)+ .expect("Failed to get is_homepage fast-field reader");++ Ok(InitialSegmentScoreTweaker {+ centrality_reader,+ is_homepage_reader,+ })}}impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {fn score(&mut self, doc: DocId, score: Score) -> f64 {+ let score = score as f64;let centrality: f64 = self.centrality_reader.get(doc) as f64 / CENTRALITY_SCALING as f64;- score as f64 + 10_000.0 * centrality+ let is_homepage = self.is_homepage_reader.get(doc) as f64;++ score + 10_000.0 * centrality + 50.0 * is_homepage}}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 2e0d9fe0..039db554 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -153,7 +153,7 @@ mod tests {assert_eq!(result.documents.len(), 3);assert_eq!(result.documents[0].url, "https://www.dr.dk");- assert_eq!(result.documents[1].url, "https://www.dr.dk/whatever");- assert_eq!(result.documents[2].url, "https://www.b.com");+ assert_eq!(result.documents[1].url, "https://www.b.com");+ assert_eq!(result.documents[2].url, "https://www.dr.dk/whatever");}}diff --git a/src/schema.rs b/src/schema.rsindex 6428418b..98d36b0f 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -30,16 +30,22 @@ pub enum Field {StemmedBody,Url,Host,+ Domain,+ DomainIfHomepage, // this field is only set if the webpage is the homepage for the site. Allows us to boost+ IsHomepage,BacklinkText,Centrality,}-pub static ALL_FIELDS: [Field; 8] = [+pub static ALL_FIELDS: [Field; 11] = [Field::Title,Field::Body,Field::StemmedTitle,Field::StemmedBody,Field::Url,Field::Host,+ Field::Domain,+ Field::DomainIfHomepage,+ Field::IsHomepage,Field::BacklinkText,Field::Centrality,];@@ -66,6 +72,13 @@ impl Field {Field::Body => IndexingOption::Text(self.default_text_options().set_stored()),Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),Field::Host => IndexingOption::Text(self.default_text_options()),+ Field::Domain => IndexingOption::Text(self.default_text_options()),+ Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),+ Field::IsHomepage => IndexingOption::Numeric(+ NumericOptions::default()+ .set_fast(Cardinality::SingleValue)+ .set_indexed(),+ ),Field::BacklinkText => IndexingOption::Text(self.default_text_options()),Field::Centrality => IndexingOption::Numeric(NumericOptions::default()@@ -91,16 +104,24 @@ impl Field {Field::Centrality => "centrality",Field::StemmedTitle => "stemmed_title",Field::StemmedBody => "stemmed_body",+ Field::Domain => "domain",+ Field::DomainIfHomepage => "domain_if_homepage",+ Field::IsHomepage => "is_homepage",}}pub fn boost(&self) -> Option<f32> {match self {- Field::Host => Some(5.0),+ Field::Host => Some(2.0),+ Field::DomainIfHomepage => Some(50.0),Field::StemmedBody | Field::StemmedTitle => Some(0.5),- Field::Title | Field::Body | Field::BacklinkText | Field::Centrality | Field::Url => {- None- }+ Field::Title+ | Field::Body+ | Field::BacklinkText+ | Field::Centrality+ | Field::Url+ | Field::Domain+ | Field::IsHomepage => None,}}}diff --git a/src/warc.rs b/src/warc.rsindex 118850db..dc5f0cb8 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -166,7 +166,6 @@ struct RawWarcRecord {content: Vec<u8>,}-#[derive(Debug)]pub(crate) struct WarcRecord {pub(crate) request: Request,pub(crate) response: Response,@@ -210,7 +209,6 @@ impl Response {}}-#[derive(Debug)]pub(crate) struct Metadata {// fetchTimeMspub(crate) fetch_time_ms: usize,diff --git a/src/webpage.rs b/src/webpage.rsindex 85a7d70b..c92b3983 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -403,6 +403,17 @@ impl<'a> Html<'a> {}Field::Url => doc.add_text(tantivy_field, self.url()),Field::Host => doc.add_text(tantivy_field, self.host()),+ Field::Domain => doc.add_text(tantivy_field, self.domain()),+ Field::DomainIfHomepage => {+ if self.is_homepage() {+ doc.add_text(tantivy_field, self.domain())+ } else {+ doc.add_text(tantivy_field, "")+ }+ }+ Field::IsHomepage => {+ doc.add_u64(tantivy_field, self.is_homepage().then(|| 1).unwrap_or(0))+ }Field::BacklinkText | Field::Centrality => {}}}