web search done right
navigational ranking only applies to homepage
Mikkel Denker19916da491a3
diff --git a/src/index.rs b/src/index.rsindex a4c759d6..6111c4a1 100644--- a/src/index.rs+++ b/src/index.rs@@ -130,7 +130,7 @@ impl From<Document> for RetrievedWebpage {.expect("Url field should be text").to_string()}- Field::BacklinkText | Field::Centrality | Field::Domain => {}+ Field::BacklinkText | Field::Centrality | Field::FastUrl => {}}}diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex 433c2ee7..2c1279d2 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -16,6 +16,7 @@use crate::query::Query;use crate::schema::Field;+use crate::webpage;use std::sync::Arc;use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};use tantivy::fastfield::{BytesFastFieldReader, DynamicFastFieldReader, FastFieldReader};@@ -33,16 +34,16 @@ impl InitialScoreTweaker {pub(crate) struct InitialSegmentScoreTweaker {centrality_reader: DynamicFastFieldReader<f64>,- domain_reader: BytesFastFieldReader,+ fast_url_reader: BytesFastFieldReader,sorted_dedupped_terms: Vec<String>,}impl InitialSegmentScoreTweaker {fn navigational_score(&self, sorted_dedupped_terms: &[String], doc: DocId) -> f64 {- let bytes = self.domain_reader.get_bytes(doc);- let domain = bincode::deserialize(bytes).expect("Failed to deserialize domain");+ let bytes = self.fast_url_reader.get_bytes(doc);+ let url = bincode::deserialize(bytes).expect("Failed to deserialize domain");- if is_navigational(sorted_dedupped_terms, domain) {+ if is_navigational(sorted_dedupped_terms, url) {1.0} else {0.0@@ -63,13 +64,13 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {.f64(centrality_field).expect("Failed to get centrality fast-field reader");- let domain_field = segment_reader+ let fast_url_field = segment_reader.schema()- .get_field(Field::Domain.as_str())+ .get_field(Field::FastUrl.as_str()).expect("Faild to load domain field");- let domain_reader = segment_reader+ let fast_url_reader = segment_reader.fast_fields()- .bytes(domain_field)+ .bytes(fast_url_field).expect("Failed to get domain fast-field reader");let mut sorted_dedupped_terms = self.query.terms.clone();@@ -79,7 +80,7 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {Ok(InitialSegmentScoreTweaker {centrality_reader,- domain_reader,+ fast_url_reader,sorted_dedupped_terms,})}@@ -111,7 +112,12 @@ fn jaccard_sim(sorted_dedupped_terms: &[String], domain_parts: &[&str]) -> f64 {/ (sorted_dedupped_terms.len() as f64 + domain_parts.len() as f64 - intersection_size)}-fn is_navigational(sorted_dedupped_terms: &[String], domain: &str) -> bool {+fn is_navigational(sorted_dedupped_terms: &[String], url: &str) -> bool {+ if !webpage::is_homepage(url) {+ return false;+ }+ let domain = webpage::domain(url);+let (name, remaining) = domain.split_once('.').unwrap();jaccard_sim(sorted_dedupped_terms, &[name, remaining]) >= 0.5}@@ -119,11 +125,6 @@ fn is_navigational(sorted_dedupped_terms: &[String], domain: &str) -> bool {#[cfg(test)]mod tests {use super::*;- use crate::{- index::Index,- query::Query,- webpage::{Link, Webpage},- };#[test]fn test_jaccard_sim() {diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex c660a38c..e496abc9 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -105,7 +105,7 @@ mod tests {r#"<html><head>- <title>Website A</title>+ <title>DR Homepage</title></head></html>"#,@@ -114,6 +114,20 @@ mod tests {0.0,)).expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Subsite dr</title>+ </head>+ </html>+ "#,+ "https://www.dr.dk/whatever",+ vec![],+ 0.0,+ ))+ .expect("failed to parse webpage");index.insert(Webpage::new(r#"@@ -126,15 +140,16 @@ mod tests {"#,"https://www.b.com",vec![],- 0.0003,+ 0.003,)).expect("failed to parse webpage");index.commit().expect("failed to commit index");let searcher = Searcher::from(index);let result = searcher.search("dr dk").expect("Search failed");- assert_eq!(result.documents.len(), 2);+ assert_eq!(result.documents.len(), 3);assert_eq!(result.documents[0].url, "https://www.dr.dk");assert_eq!(result.documents[1].url, "https://www.b.com");+ assert_eq!(result.documents[2].url, "https://www.dr.dk/whatever");}}diff --git a/src/schema.rs b/src/schema.rsindex 232f8bde..a008e926 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -23,7 +23,7 @@ pub enum Field {Title,Body,Url,- Domain,+ FastUrl,BacklinkText,Centrality,}@@ -31,7 +31,7 @@ pub static ALL_FIELDS: [Field; 6] = [Field::Title,Field::Body,Field::Url,- Field::Domain,+ Field::FastUrl,Field::BacklinkText,Field::Centrality,];@@ -50,7 +50,7 @@ impl Field {Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),Field::Body => IndexingOption::Text(self.default_text_options().set_stored()),Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),- Field::Domain => {+ Field::FastUrl => {IndexingOption::Bytes(BytesOptions::default().set_fast().set_indexed())}Field::BacklinkText => IndexingOption::Text(self.default_text_options()),@@ -67,7 +67,7 @@ impl Field {Field::Title => "title",Field::Body => "body",Field::Url => "url",- Field::Domain => "host",+ Field::FastUrl => "fast_url",Field::BacklinkText => "backlink_text",Field::Centrality => "centrality",}diff --git a/src/webpage.rs b/src/webpage.rsindex ffbe4562..85032cc6 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -21,6 +21,54 @@ use std::collections::BTreeMap;use crate::schema::{Field, ALL_FIELDS};+fn strip_protocol<'a>(url: &'a str) -> &'a str {+ let mut start_host = 0;+ if url.starts_with("http://") || url.starts_with("https://") {+ start_host = url+ .find('/')+ .expect("It was checked that url starts with protocol");+ start_host += 2; // skip the two '/'+ }++ &url[start_host..]+}++pub(crate) fn host<'a>(url: &'a str) -> &'a str {+ let url = strip_protocol(url);++ let mut end_host = url.len();+ if url.contains('/') {+ end_host = url.find('/').expect("The url contains atleast 1 '/'");+ }++ &url[..end_host]+}++pub(crate) fn is_homepage<'a>(url: &'a str) -> bool {+ let url = strip_protocol(url);+ match url.find('/') {+ Some(idx) => idx == url.len() - 1,+ None => true,+ }+}++pub(crate) fn domain<'a>(url: &'a str) -> &'a str {+ let host = host(url);+ let num_punctuations: usize = host.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();+ if num_punctuations > 1 {+ let domain_index = host.rfind('.').unwrap();+ let mut start_index = host[..domain_index].rfind('.').unwrap();++ if &host[start_index + 1..] == "co.uk" {+ start_index = host[start_index..].rfind('.').unwrap();+ }++ &host[start_index + 1..]+ } else {+ host+ }+}+pub struct Webpage {pub html: Html,pub backlinks: Vec<Link>,@@ -156,41 +204,15 @@ impl Html {}pub fn host(&self) -> &str {- let mut start_host = 0;- if self.url().starts_with("http://") || self.url().starts_with("https://") {- start_host = self- .url()- .find('/')- .expect("It was checked that url starts with protocol");- start_host += 2; // skip the two '/'- }-- let mut end_host = self.url.len();- if self.url()[start_host..].contains('/') {- end_host = self.url()[start_host..]- .find('/')- .expect("The url contains atleast 1 '/'");- end_host += start_host; // offset the start position- }-- &self.url()[start_host..end_host]+ host(self.url())}pub fn domain(&self) -> &str {- let host = self.host();- let num_punctuations: usize = host.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();- if num_punctuations > 1 {- let domain_index = host.rfind('.').unwrap();- let mut start_index = host[..domain_index].rfind('.').unwrap();-- if &host[start_index + 1..] == "co.uk" {- start_index = host[start_index..].rfind('.').unwrap();- }+ domain(self.url())+ }- &host[start_index + 1..]- } else {- host- }+ pub fn is_homepage(&self) -> bool {+ is_homepage(self.url())}pub fn metadata(&self) -> Vec<Meta> {@@ -225,9 +247,9 @@ impl Html {}Field::Body => doc.add_text(tantivy_field, self.text()),Field::Url => doc.add_text(tantivy_field, self.url()),- Field::Domain => doc.add_bytes(+ Field::FastUrl => doc.add_bytes(tantivy_field,- bincode::serialize(self.domain()).expect("Failed to serialize bytes"),+ bincode::serialize(self.url()).expect("Failed to serialize bytes"),),Field::BacklinkText | Field::Centrality => {}}@@ -367,4 +389,28 @@ mod tests {let webpage = Html::parse(raw, "https://www.domain.co.uk");assert_eq!(webpage.domain(), "domain.co.uk");}++ #[test]+ fn is_homepage() {+ let webpage = Html::parse("", "https://www.example.com");+ assert!(webpage.is_homepage());++ let webpage = Html::parse("", "https://www.example.com/");+ assert!(webpage.is_homepage());++ let webpage = Html::parse("", "https://www.example.com/test");+ assert!(!webpage.is_homepage());++ let webpage = Html::parse("", "https://example.com/test");+ assert!(!webpage.is_homepage());++ let webpage = Html::parse("", "https://example.com/");+ assert!(webpage.is_homepage());++ let webpage = Html::parse("", "https://example.com");+ assert!(webpage.is_homepage());++ let webpage = Html::parse("", "http://example.com");+ assert!(webpage.is_homepage());+ }}