web search done right
navigational query
Mikkel Denker8f98ebd0dee5
diff --git a/src/query.rs b/src/query.rsindex e8e7e0a4..c0a169d7 100644--- a/src/query.rs+++ b/src/query.rs@@ -43,8 +43,9 @@ wherefn term(term: &str) -> IResult<&str, &str> {ws(take_while(|c: char| !c.is_whitespace()))(term)}+#[derive(Clone)]pub struct Query {- terms: Vec<String>,+ pub terms: Vec<String>,}impl Query {diff --git a/src/ranking.rs b/src/ranking.rsindex 1325e41c..80dee0dd 100644--- a/src/ranking.rs+++ b/src/ranking.rs@@ -14,15 +14,52 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use crate::query::Query;use crate::schema::Field;-use tantivy::fastfield::FastFieldReader;+use crate::search_index::FastWebpage;+use std::sync::Arc;+use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};+use tantivy::fastfield::{BytesFastFieldReader, DynamicFastFieldReader, FastFieldReader};use tantivy::{collector::{Collector, TopDocs},DocId, Score, SegmentReader,};-pub(crate) fn initial_collector() -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {- TopDocs::with_limit(20).tweak_score(move |segment_reader: &SegmentReader| {+struct InitialScoreTweaker {+ query: Arc<Query>,+}++impl InitialScoreTweaker {+ pub fn new(query: Query) -> Self {+ Self {+ query: Arc::new(query),+ }+ }+}++struct InitialSegmentScoreTweaker {+ query: Arc<Query>,+ centrality_reader: DynamicFastFieldReader<f64>,+ domain_reader: BytesFastFieldReader,+}++impl InitialSegmentScoreTweaker {+ fn navigational_score(&self, terms: Vec<String>, doc: DocId) -> f64 {+ let bytes = self.domain_reader.get_bytes(doc);+ let domain = bincode::deserialize(bytes).expect("Failed to deserialize domain");++ if is_navigational(terms, domain) {+ 1.0+ } else {+ 0.0+ }+ }+}++impl ScoreTweaker<f64> for InitialScoreTweaker {+ type Child = InitialSegmentScoreTweaker;++ fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {let centrality_field = segment_reader.schema().get_field(Field::Centrality.as_str())@@ -32,22 +69,95 @@ pub(crate) fn initial_collector() -> impl Collector<Fruit = Vec<(f64, tantivy::D.f64(centrality_field).expect("Failed to get centrality fast-field reader");- move |doc: DocId, original_score: Score| {- let centrality = centrality_reader.get(doc);- original_score as f64 + 100.0 * centrality- }- })+ let domain_field = segment_reader+ .schema()+ .get_field(Field::Domain.as_str())+ .expect("Faild to load domain field");+ let domain_reader = segment_reader+ .fast_fields()+ .bytes(domain_field)+ .expect("Failed to get domain fast-field reader");++ Ok(InitialSegmentScoreTweaker {+ query: self.query.clone(),+ centrality_reader,+ domain_reader,+ })+ }+}++impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {+ fn score(&mut self, doc: DocId, score: Score) -> f64 {+ let centrality = self.centrality_reader.get(doc);+ let navigational = self.navigational_score(self.query.terms.clone(), doc);+ score as f64 + 1000.0 * centrality + 100.0 * navigational+ }+}++fn jaccard_sim(mut terms: Vec<String>, domain_parts: &[&str]) -> f64 {+ debug_assert_eq!(domain_parts.len(), 2);++ terms.sort();+ terms.dedup();++ let intersection_size: f64 = terms+ .iter()+ .map(|term| {+ if domain_parts.contains(&term.as_str()) {+ 1.0+ } else {+ 0.0+ }+ })+ .sum();++ intersection_size / (terms.len() as f64 + domain_parts.len() as f64 - intersection_size)+}++fn is_navigational(mut terms: Vec<String>, domain: &str) -> bool {+ let (name, remaining) = domain.split_once(".").unwrap();+ jaccard_sim(terms, &[name, remaining]) >= 0.5+}++pub(crate) fn initial_collector(+ query: Query,+) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {+ let score_tweaker = InitialScoreTweaker::new(query);+ TopDocs::with_limit(20).tweak_score(score_tweaker)}#[cfg(test)]mod tests {+ use super::*;use crate::{query::Query,search_index::Index,webpage::{Link, Webpage},};+ #[test]+ fn test_jaccard_sim() {+ assert_eq!(+ jaccard_sim(vec!["dr".to_string(), "dk".to_string()], &["dr", "dk"]),+ 1.0+ );++ assert_eq!(+ jaccard_sim(+ vec![+ "dr".to_string(),+ "dk".to_string(),+ "dr".to_string(),+ "dr".to_string()+ ],+ &["dr", "dk"]+ ),+ 1.0+ );+ assert_eq!(jaccard_sim(vec!["dr".to_string()], &["dr", "dk"]), 0.5);+ }+#[test]fn harmonic_ranking() {let query = Query::parse("great site").expect("Failed to parse query");@@ -96,4 +206,56 @@ mod tests {assert_eq!(result.documents[1].url, "https://www.a.com");}}++ #[test]+ fn navigational() {+ assert!(is_navigational(vec!["dr".to_string()], "dr.dk"));+ assert!(is_navigational(+ vec!["dr".to_string(), "dk".to_string()],+ "dr.dk"+ ));+ }++ #[test]+ fn navigational_search() {+ let query = Query::parse("dr dk").expect("Failed to parse query");++ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Website A</title>+ </head>+ </html>+ "#,+ "https://www.dr.dk",+ vec![],+ 0.0,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Website B</title>+ dr dk dr dk dr dk dr dk+ </head>+ </html>+ "#,+ "https://www.b.com",+ vec![],+ 0.0003,+ ))+ .expect("failed to parse webpage");++ index.commit().expect("failed to commit index");+ let result = index.search(&query).expect("Search failed");+ assert_eq!(result.documents.len(), 2);+ assert_eq!(result.documents[0].url, "https://www.dr.dk");+ assert_eq!(result.documents[1].url, "https://www.b.com");+ }}diff --git a/src/schema.rs b/src/schema.rsindex 5807028d..232f8bde 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use tantivy::schema::{- Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,+ BytesOptions, Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,};#[derive(Clone)]@@ -23,13 +23,15 @@ pub enum Field {Title,Body,Url,+ Domain,BacklinkText,Centrality,}-pub static ALL_FIELDS: [Field; 5] = [+pub static ALL_FIELDS: [Field; 6] = [Field::Title,Field::Body,Field::Url,+ Field::Domain,Field::BacklinkText,Field::Centrality,];@@ -48,6 +50,9 @@ impl Field {Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),Field::Body => IndexingOption::Text(self.default_text_options().set_stored()),Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),+ Field::Domain => {+ IndexingOption::Bytes(BytesOptions::default().set_fast().set_indexed())+ }Field::BacklinkText => IndexingOption::Text(self.default_text_options()),Field::Centrality => IndexingOption::Numeric(NumericOptions::default()@@ -62,6 +67,7 @@ impl Field {Field::Title => "title",Field::Body => "body",Field::Url => "url",+ Field::Domain => "host",Field::BacklinkText => "backlink_text",Field::Centrality => "centrality",}@@ -75,6 +81,7 @@ pub fn create_schema() -> tantivy::schema::Schema {match field.options() {IndexingOption::Text(options) => builder.add_text_field(field.as_str(), options),IndexingOption::Numeric(options) => builder.add_f64_field(field.as_str(), options),+ IndexingOption::Bytes(options) => builder.add_bytes_field(field.as_str(), options),};}@@ -84,4 +91,5 @@ pub fn create_schema() -> tantivy::schema::Schema {enum IndexingOption {Text(tantivy::schema::TextOptions),Numeric(tantivy::schema::NumericOptions),+ Bytes(tantivy::schema::BytesOptions),}diff --git a/src/search_index.rs b/src/search_index.rsindex ed23d903..6cf2b806 100644--- a/src/search_index.rs+++ b/src/search_index.rs@@ -63,8 +63,10 @@ impl Index {let tantivy_query = query.tantivy(&self.schema, self.tantivy_index.tokenizers());let searcher = self.reader.searcher();- let (count, docs) =- searcher.search(&tantivy_query, &(Count, ranking::initial_collector()))?;+ let (count, docs) = searcher.search(+ &tantivy_query,+ &(Count, ranking::initial_collector(query.clone())),+ )?;let mut webpages: Vec<RetrievedWebpage> = docs.into_iter()@@ -97,6 +99,11 @@ pub struct SearchResult {pub documents: Vec<RetrievedWebpage>,}+#[derive(Default)]+pub struct FastWebpage {+ pub host: String,+}+#[derive(Default)]pub struct RetrievedWebpage {pub title: String,@@ -132,7 +139,7 @@ impl From<Document> for RetrievedWebpage {.expect("Url field should be text").to_string()}- Field::BacklinkText | Field::Centrality => {}+ Field::BacklinkText | Field::Centrality | Field::Domain => {}}}@@ -280,4 +287,58 @@ mod tests {assert_eq!(result.documents[0].url, "https://www.a.com");assert_eq!(result.documents[1].url, "https://www.b.com");}++ #[test]+ fn limited_top_docs() {+ let mut index = Index::temporary().expect("Unable to open index");+ let query = Query::parse("runner").expect("Failed to parse query");++ for _ in 0..100 {+ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Website for runners</title>+ </head>+ </html>+ "#,+ "https://www.example.com",+ vec![],+ 1.0,+ ))+ .expect("failed to parse webpage");+ }++ index.commit().expect("failed to commit index");++ let result = index.search(&query).expect("Search failed");+ assert_eq!(result.documents.len(), 20);+ }++ #[test]+ fn host_search() {+ let mut index = Index::temporary().expect("Unable to open index");+ let query = Query::parse("dr").expect("Failed to parse query");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>News website</title>+ </head>+ </html>+ "#,+ "https://www.dr.dk",+ vec![],+ 1.0,+ ))+ .expect("failed to parse webpage");+ index.commit().expect("failed to commit index");++ let result = index.search(&query).expect("Search failed");+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.dr.dk");+ }}diff --git a/src/webpage.rs b/src/webpage.rsindex d9c00444..739dfcd7 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -176,6 +176,23 @@ impl Html {&self.url()[start_host..end_host]}+ pub fn domain(&self) -> &str {+ let host = self.host();+ let num_punctuations: usize = host.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();+ if num_punctuations > 1 {+ let domain_index = host.rfind(".").unwrap();+ let mut start_index = host[..domain_index].rfind(".").unwrap();++ if &host[start_index + 1..] == "co.uk" {+ start_index = host[start_index..].rfind(".").unwrap();+ }++ &host[start_index + 1..]+ } else {+ host+ }+ }+pub fn metadata(&self) -> Vec<Meta> {let selector = Selector::parse("meta").expect("Failed to parse selector");self.dom@@ -208,6 +225,10 @@ impl Html {}Field::Body => doc.add_text(tantivy_field, self.text()),Field::Url => doc.add_text(tantivy_field, self.url()),+ Field::Domain => doc.add_bytes(+ tantivy_field,+ bincode::serialize(self.domain()).expect("Failed to serialize bytes"),+ ),Field::BacklinkText | Field::Centrality => {}}}@@ -263,6 +284,7 @@ mod tests {assert_eq!(webpage.metadata(), vec![expected_meta]);assert_eq!(webpage.url(), "https://www.example.com/whatever");assert_eq!(webpage.host(), "www.example.com");+ assert_eq!(webpage.domain(), "example.com");}#[test]@@ -337,4 +359,12 @@ mod tests {"This text should be the first text extracted\nThis text should be the second text extracted");}++ #[test]+ fn co_uk_domain() {+ let raw = "";++ let webpage = Html::parse(raw, "https://www.domain.co.uk");+ assert_eq!(webpage.domain(), "domain.co.uk");+ }}