Skip to content

web search done right

Better navigational ranking

Mikkel Denker97583078a60d
diff --git a/src/index.rs b/src/index.rs
index 3c881986..2f914f58 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -228,7 +228,10 @@ impl From<Document> for RetrievedWebpage {
| Field::Centrality
| Field::Host
| Field::StemmedTitle
- | Field::StemmedBody => {}
+ | Field::StemmedBody
+ | Field::Domain
+ | Field::DomainIfHomepage
+ | Field::IsHomepage => {}
}
}
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rs
index bd0dcc92..d11dc09e 100644
--- a/src/ranking/initial.rs
+++ b/src/ranking/initial.rs
@@ -24,6 +24,7 @@ pub(crate) struct InitialScoreTweaker {}
pub(crate) struct InitialSegmentScoreTweaker {
centrality_reader: DynamicFastFieldReader<u64>,
+ is_homepage_reader: DynamicFastFieldReader<u64>,
}
impl ScoreTweaker<f64> for InitialScoreTweaker {
@@ -39,13 +40,28 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {
.u64(centrality_field)
.expect("Failed to get centrality fast-field reader");
- Ok(InitialSegmentScoreTweaker { centrality_reader })
+ let is_homepage_field = segment_reader
+ .schema()
+ .get_field(Field::IsHomepage.as_str())
+ .expect("Faild to load is_homepage field");
+ let is_homepage_reader = segment_reader
+ .fast_fields()
+ .u64(is_homepage_field)
+ .expect("Failed to get is_homepage fast-field reader");
+
+ Ok(InitialSegmentScoreTweaker {
+ centrality_reader,
+ is_homepage_reader,
+ })
}
}
impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {
fn score(&mut self, doc: DocId, score: Score) -> f64 {
+ let score = score as f64;
let centrality: f64 = self.centrality_reader.get(doc) as f64 / CENTRALITY_SCALING as f64;
- score as f64 + 10_000.0 * centrality
+ let is_homepage = self.is_homepage_reader.get(doc) as f64;
+
+ score + 10_000.0 * centrality + 50.0 * is_homepage
}
}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 2e0d9fe0..039db554 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -153,7 +153,7 @@ mod tests {
assert_eq!(result.documents.len(), 3);
assert_eq!(result.documents[0].url, "https://www.dr.dk");
- assert_eq!(result.documents[1].url, "https://www.dr.dk/whatever");
- assert_eq!(result.documents[2].url, "https://www.b.com");
+ assert_eq!(result.documents[1].url, "https://www.b.com");
+ assert_eq!(result.documents[2].url, "https://www.dr.dk/whatever");
}
}
diff --git a/src/schema.rs b/src/schema.rs
index 6428418b..98d36b0f 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -30,16 +30,22 @@ pub enum Field {
StemmedBody,
Url,
Host,
+ Domain,
+ DomainIfHomepage, // this field is only set if the webpage is the homepage for the site. Allows us to boost
+ IsHomepage,
BacklinkText,
Centrality,
}
-pub static ALL_FIELDS: [Field; 8] = [
+pub static ALL_FIELDS: [Field; 11] = [
Field::Title,
Field::Body,
Field::StemmedTitle,
Field::StemmedBody,
Field::Url,
Field::Host,
+ Field::Domain,
+ Field::DomainIfHomepage,
+ Field::IsHomepage,
Field::BacklinkText,
Field::Centrality,
];
@@ -66,6 +72,13 @@ impl Field {
Field::Body => IndexingOption::Text(self.default_text_options().set_stored()),
Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),
Field::Host => IndexingOption::Text(self.default_text_options()),
+ Field::Domain => IndexingOption::Text(self.default_text_options()),
+ Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),
+ Field::IsHomepage => IndexingOption::Numeric(
+ NumericOptions::default()
+ .set_fast(Cardinality::SingleValue)
+ .set_indexed(),
+ ),
Field::BacklinkText => IndexingOption::Text(self.default_text_options()),
Field::Centrality => IndexingOption::Numeric(
NumericOptions::default()
@@ -91,16 +104,24 @@ impl Field {
Field::Centrality => "centrality",
Field::StemmedTitle => "stemmed_title",
Field::StemmedBody => "stemmed_body",
+ Field::Domain => "domain",
+ Field::DomainIfHomepage => "domain_if_homepage",
+ Field::IsHomepage => "is_homepage",
}
}
pub fn boost(&self) -> Option<f32> {
match self {
- Field::Host => Some(5.0),
+ Field::Host => Some(2.0),
+ Field::DomainIfHomepage => Some(50.0),
Field::StemmedBody | Field::StemmedTitle => Some(0.5),
- Field::Title | Field::Body | Field::BacklinkText | Field::Centrality | Field::Url => {
- None
- }
+ Field::Title
+ | Field::Body
+ | Field::BacklinkText
+ | Field::Centrality
+ | Field::Url
+ | Field::Domain
+ | Field::IsHomepage => None,
}
}
}
diff --git a/src/warc.rs b/src/warc.rs
index 118850db..dc5f0cb8 100644
--- a/src/warc.rs
+++ b/src/warc.rs
@@ -166,7 +166,6 @@ struct RawWarcRecord {
content: Vec<u8>,
}
-#[derive(Debug)]
pub(crate) struct WarcRecord {
pub(crate) request: Request,
pub(crate) response: Response,
@@ -210,7 +209,6 @@ impl Response {
}
}
-#[derive(Debug)]
pub(crate) struct Metadata {
// fetchTimeMs
pub(crate) fetch_time_ms: usize,
diff --git a/src/webpage.rs b/src/webpage.rs
index 85a7d70b..c92b3983 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -403,6 +403,17 @@ impl<'a> Html<'a> {
}
Field::Url => doc.add_text(tantivy_field, self.url()),
Field::Host => doc.add_text(tantivy_field, self.host()),
+ Field::Domain => doc.add_text(tantivy_field, self.domain()),
+ Field::DomainIfHomepage => {
+ if self.is_homepage() {
+ doc.add_text(tantivy_field, self.domain())
+ } else {
+ doc.add_text(tantivy_field, "")
+ }
+ }
+ Field::IsHomepage => {
+ doc.add_u64(tantivy_field, self.is_homepage().then(|| 1).unwrap_or(0))
+ }
Field::BacklinkText | Field::Centrality => {}
}
}