Skip to content

web search done right

navigational ranking only applies to homepage

Mikkel Denker19916da491a3
diff --git a/src/index.rs b/src/index.rs
index a4c759d6..6111c4a1 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -130,7 +130,7 @@ impl From<Document> for RetrievedWebpage {
.expect("Url field should be text")
.to_string()
}
- Field::BacklinkText | Field::Centrality | Field::Domain => {}
+ Field::BacklinkText | Field::Centrality | Field::FastUrl => {}
}
}
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rs
index 433c2ee7..2c1279d2 100644
--- a/src/ranking/initial.rs
+++ b/src/ranking/initial.rs
@@ -16,6 +16,7 @@
use crate::query::Query;
use crate::schema::Field;
+use crate::webpage;
use std::sync::Arc;
use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};
use tantivy::fastfield::{BytesFastFieldReader, DynamicFastFieldReader, FastFieldReader};
@@ -33,16 +34,16 @@ impl InitialScoreTweaker {
pub(crate) struct InitialSegmentScoreTweaker {
centrality_reader: DynamicFastFieldReader<f64>,
- domain_reader: BytesFastFieldReader,
+ fast_url_reader: BytesFastFieldReader,
sorted_dedupped_terms: Vec<String>,
}
impl InitialSegmentScoreTweaker {
fn navigational_score(&self, sorted_dedupped_terms: &[String], doc: DocId) -> f64 {
- let bytes = self.domain_reader.get_bytes(doc);
- let domain = bincode::deserialize(bytes).expect("Failed to deserialize domain");
+ let bytes = self.fast_url_reader.get_bytes(doc);
+ let url = bincode::deserialize(bytes).expect("Failed to deserialize domain");
- if is_navigational(sorted_dedupped_terms, domain) {
+ if is_navigational(sorted_dedupped_terms, url) {
1.0
} else {
0.0
@@ -63,13 +64,13 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {
.f64(centrality_field)
.expect("Failed to get centrality fast-field reader");
- let domain_field = segment_reader
+ let fast_url_field = segment_reader
.schema()
- .get_field(Field::Domain.as_str())
+ .get_field(Field::FastUrl.as_str())
.expect("Faild to load domain field");
- let domain_reader = segment_reader
+ let fast_url_reader = segment_reader
.fast_fields()
- .bytes(domain_field)
+ .bytes(fast_url_field)
.expect("Failed to get domain fast-field reader");
let mut sorted_dedupped_terms = self.query.terms.clone();
@@ -79,7 +80,7 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {
Ok(InitialSegmentScoreTweaker {
centrality_reader,
- domain_reader,
+ fast_url_reader,
sorted_dedupped_terms,
})
}
@@ -111,7 +112,12 @@ fn jaccard_sim(sorted_dedupped_terms: &[String], domain_parts: &[&str]) -> f64 {
/ (sorted_dedupped_terms.len() as f64 + domain_parts.len() as f64 - intersection_size)
}
-fn is_navigational(sorted_dedupped_terms: &[String], domain: &str) -> bool {
+fn is_navigational(sorted_dedupped_terms: &[String], url: &str) -> bool {
+ if !webpage::is_homepage(url) {
+ return false;
+ }
+ let domain = webpage::domain(url);
+
let (name, remaining) = domain.split_once('.').unwrap();
jaccard_sim(sorted_dedupped_terms, &[name, remaining]) >= 0.5
}
@@ -119,11 +125,6 @@ fn is_navigational(sorted_dedupped_terms: &[String], domain: &str) -> bool {
#[cfg(test)]
mod tests {
use super::*;
- use crate::{
- index::Index,
- query::Query,
- webpage::{Link, Webpage},
- };
#[test]
fn test_jaccard_sim() {
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index c660a38c..e496abc9 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -105,7 +105,7 @@ mod tests {
r#"
<html>
<head>
- <title>Website A</title>
+ <title>DR Homepage</title>
</head>
</html>
"#,
@@ -114,6 +114,20 @@ mod tests {
0.0,
))
.expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Subsite dr</title>
+ </head>
+ </html>
+ "#,
+ "https://www.dr.dk/whatever",
+ vec![],
+ 0.0,
+ ))
+ .expect("failed to parse webpage");
index
.insert(Webpage::new(
r#"
@@ -126,15 +140,16 @@ mod tests {
"#,
"https://www.b.com",
vec![],
- 0.0003,
+ 0.003,
))
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
let result = searcher.search("dr dk").expect("Search failed");
- assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents.len(), 3);
assert_eq!(result.documents[0].url, "https://www.dr.dk");
assert_eq!(result.documents[1].url, "https://www.b.com");
+ assert_eq!(result.documents[2].url, "https://www.dr.dk/whatever");
}
}
diff --git a/src/schema.rs b/src/schema.rs
index 232f8bde..a008e926 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -23,7 +23,7 @@ pub enum Field {
Title,
Body,
Url,
- Domain,
+ FastUrl,
BacklinkText,
Centrality,
}
@@ -31,7 +31,7 @@ pub static ALL_FIELDS: [Field; 6] = [
Field::Title,
Field::Body,
Field::Url,
- Field::Domain,
+ Field::FastUrl,
Field::BacklinkText,
Field::Centrality,
];
@@ -50,7 +50,7 @@ impl Field {
Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),
Field::Body => IndexingOption::Text(self.default_text_options().set_stored()),
Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),
- Field::Domain => {
+ Field::FastUrl => {
IndexingOption::Bytes(BytesOptions::default().set_fast().set_indexed())
}
Field::BacklinkText => IndexingOption::Text(self.default_text_options()),
@@ -67,7 +67,7 @@ impl Field {
Field::Title => "title",
Field::Body => "body",
Field::Url => "url",
- Field::Domain => "host",
+ Field::FastUrl => "fast_url",
Field::BacklinkText => "backlink_text",
Field::Centrality => "centrality",
}
diff --git a/src/webpage.rs b/src/webpage.rs
index ffbe4562..85032cc6 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -21,6 +21,54 @@ use std::collections::BTreeMap;
use crate::schema::{Field, ALL_FIELDS};
+fn strip_protocol<'a>(url: &'a str) -> &'a str {
+ let mut start_host = 0;
+ if url.starts_with("http://") || url.starts_with("https://") {
+ start_host = url
+ .find('/')
+ .expect("It was checked that url starts with protocol");
+ start_host += 2; // skip the two '/'
+ }
+
+ &url[start_host..]
+}
+
+pub(crate) fn host<'a>(url: &'a str) -> &'a str {
+ let url = strip_protocol(url);
+
+ let mut end_host = url.len();
+ if url.contains('/') {
+ end_host = url.find('/').expect("The url contains atleast 1 '/'");
+ }
+
+ &url[..end_host]
+}
+
+pub(crate) fn is_homepage<'a>(url: &'a str) -> bool {
+ let url = strip_protocol(url);
+ match url.find('/') {
+ Some(idx) => idx == url.len() - 1,
+ None => true,
+ }
+}
+
+pub(crate) fn domain<'a>(url: &'a str) -> &'a str {
+ let host = host(url);
+ let num_punctuations: usize = host.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();
+ if num_punctuations > 1 {
+ let domain_index = host.rfind('.').unwrap();
+ let mut start_index = host[..domain_index].rfind('.').unwrap();
+
+ if &host[start_index + 1..] == "co.uk" {
+ start_index = host[start_index..].rfind('.').unwrap();
+ }
+
+ &host[start_index + 1..]
+ } else {
+ host
+ }
+}
+
pub struct Webpage {
pub html: Html,
pub backlinks: Vec<Link>,
@@ -156,41 +204,15 @@ impl Html {
}
pub fn host(&self) -> &str {
- let mut start_host = 0;
- if self.url().starts_with("http://") || self.url().starts_with("https://") {
- start_host = self
- .url()
- .find('/')
- .expect("It was checked that url starts with protocol");
- start_host += 2; // skip the two '/'
- }
-
- let mut end_host = self.url.len();
- if self.url()[start_host..].contains('/') {
- end_host = self.url()[start_host..]
- .find('/')
- .expect("The url contains atleast 1 '/'");
- end_host += start_host; // offset the start position
- }
-
- &self.url()[start_host..end_host]
+ host(self.url())
}
pub fn domain(&self) -> &str {
- let host = self.host();
- let num_punctuations: usize = host.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();
- if num_punctuations > 1 {
- let domain_index = host.rfind('.').unwrap();
- let mut start_index = host[..domain_index].rfind('.').unwrap();
-
- if &host[start_index + 1..] == "co.uk" {
- start_index = host[start_index..].rfind('.').unwrap();
- }
+ domain(self.url())
+ }
- &host[start_index + 1..]
- } else {
- host
- }
+ pub fn is_homepage(&self) -> bool {
+ is_homepage(self.url())
}
pub fn metadata(&self) -> Vec<Meta> {
@@ -225,9 +247,9 @@ impl Html {
}
Field::Body => doc.add_text(tantivy_field, self.text()),
Field::Url => doc.add_text(tantivy_field, self.url()),
- Field::Domain => doc.add_bytes(
+ Field::FastUrl => doc.add_bytes(
tantivy_field,
- bincode::serialize(self.domain()).expect("Failed to serialize bytes"),
+ bincode::serialize(self.url()).expect("Failed to serialize bytes"),
),
Field::BacklinkText | Field::Centrality => {}
}
@@ -367,4 +389,28 @@ mod tests {
let webpage = Html::parse(raw, "https://www.domain.co.uk");
assert_eq!(webpage.domain(), "domain.co.uk");
}
+
+ #[test]
+ fn is_homepage() {
+ let webpage = Html::parse("", "https://www.example.com");
+ assert!(webpage.is_homepage());
+
+ let webpage = Html::parse("", "https://www.example.com/");
+ assert!(webpage.is_homepage());
+
+ let webpage = Html::parse("", "https://www.example.com/test");
+ assert!(!webpage.is_homepage());
+
+ let webpage = Html::parse("", "https://example.com/test");
+ assert!(!webpage.is_homepage());
+
+ let webpage = Html::parse("", "https://example.com/");
+ assert!(webpage.is_homepage());
+
+ let webpage = Html::parse("", "https://example.com");
+ assert!(webpage.is_homepage());
+
+ let webpage = Html::parse("", "http://example.com");
+ assert!(webpage.is_homepage());
+ }
}