Skip to content

web search done right

navigational query

Mikkel Denker8f98ebd0dee5
diff --git a/src/query.rs b/src/query.rs
index e8e7e0a4..c0a169d7 100644
--- a/src/query.rs
+++ b/src/query.rs
@@ -43,8 +43,9 @@ where
fn term(term: &str) -> IResult<&str, &str> {
ws(take_while(|c: char| !c.is_whitespace()))(term)
}
+#[derive(Clone)]
pub struct Query {
- terms: Vec<String>,
+ pub terms: Vec<String>,
}
impl Query {
diff --git a/src/ranking.rs b/src/ranking.rs
index 1325e41c..80dee0dd 100644
--- a/src/ranking.rs
+++ b/src/ranking.rs
@@ -14,15 +14,52 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::query::Query;
use crate::schema::Field;
-use tantivy::fastfield::FastFieldReader;
+use crate::search_index::FastWebpage;
+use std::sync::Arc;
+use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};
+use tantivy::fastfield::{BytesFastFieldReader, DynamicFastFieldReader, FastFieldReader};
use tantivy::{
collector::{Collector, TopDocs},
DocId, Score, SegmentReader,
};
-pub(crate) fn initial_collector() -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {
- TopDocs::with_limit(20).tweak_score(move |segment_reader: &SegmentReader| {
+struct InitialScoreTweaker {
+ query: Arc<Query>,
+}
+
+impl InitialScoreTweaker {
+ pub fn new(query: Query) -> Self {
+ Self {
+ query: Arc::new(query),
+ }
+ }
+}
+
+struct InitialSegmentScoreTweaker {
+ query: Arc<Query>,
+ centrality_reader: DynamicFastFieldReader<f64>,
+ domain_reader: BytesFastFieldReader,
+}
+
+impl InitialSegmentScoreTweaker {
+ fn navigational_score(&self, terms: Vec<String>, doc: DocId) -> f64 {
+ let bytes = self.domain_reader.get_bytes(doc);
+ let domain = bincode::deserialize(bytes).expect("Failed to deserialize domain");
+
+ if is_navigational(terms, domain) {
+ 1.0
+ } else {
+ 0.0
+ }
+ }
+}
+
+impl ScoreTweaker<f64> for InitialScoreTweaker {
+ type Child = InitialSegmentScoreTweaker;
+
+ fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {
let centrality_field = segment_reader
.schema()
.get_field(Field::Centrality.as_str())
@@ -32,22 +69,95 @@ pub(crate) fn initial_collector() -> impl Collector<Fruit = Vec<(f64, tantivy::D
.f64(centrality_field)
.expect("Failed to get centrality fast-field reader");
- move |doc: DocId, original_score: Score| {
- let centrality = centrality_reader.get(doc);
- original_score as f64 + 100.0 * centrality
- }
- })
+ let domain_field = segment_reader
+ .schema()
+ .get_field(Field::Domain.as_str())
+ .expect("Faild to load domain field");
+ let domain_reader = segment_reader
+ .fast_fields()
+ .bytes(domain_field)
+ .expect("Failed to get domain fast-field reader");
+
+ Ok(InitialSegmentScoreTweaker {
+ query: self.query.clone(),
+ centrality_reader,
+ domain_reader,
+ })
+ }
+}
+
+impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {
+ fn score(&mut self, doc: DocId, score: Score) -> f64 {
+ let centrality = self.centrality_reader.get(doc);
+ let navigational = self.navigational_score(self.query.terms.clone(), doc);
+ score as f64 + 1000.0 * centrality + 100.0 * navigational
+ }
+}
+
+fn jaccard_sim(mut terms: Vec<String>, domain_parts: &[&str]) -> f64 {
+ debug_assert_eq!(domain_parts.len(), 2);
+
+ terms.sort();
+ terms.dedup();
+
+ let intersection_size: f64 = terms
+ .iter()
+ .map(|term| {
+ if domain_parts.contains(&term.as_str()) {
+ 1.0
+ } else {
+ 0.0
+ }
+ })
+ .sum();
+
+ intersection_size / (terms.len() as f64 + domain_parts.len() as f64 - intersection_size)
+}
+
+fn is_navigational(mut terms: Vec<String>, domain: &str) -> bool {
+ let (name, remaining) = domain.split_once(".").unwrap();
+ jaccard_sim(terms, &[name, remaining]) >= 0.5
+}
+
+pub(crate) fn initial_collector(
+ query: Query,
+) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {
+ let score_tweaker = InitialScoreTweaker::new(query);
+ TopDocs::with_limit(20).tweak_score(score_tweaker)
}
#[cfg(test)]
mod tests {
+ use super::*;
use crate::{
query::Query,
search_index::Index,
webpage::{Link, Webpage},
};
+ #[test]
+ fn test_jaccard_sim() {
+ assert_eq!(
+ jaccard_sim(vec!["dr".to_string(), "dk".to_string()], &["dr", "dk"]),
+ 1.0
+ );
+
+ assert_eq!(
+ jaccard_sim(
+ vec![
+ "dr".to_string(),
+ "dk".to_string(),
+ "dr".to_string(),
+ "dr".to_string()
+ ],
+ &["dr", "dk"]
+ ),
+ 1.0
+ );
+ assert_eq!(jaccard_sim(vec!["dr".to_string()], &["dr", "dk"]), 0.5);
+ }
+
#[test]
fn harmonic_ranking() {
let query = Query::parse("great site").expect("Failed to parse query");
@@ -96,4 +206,56 @@ mod tests {
assert_eq!(result.documents[1].url, "https://www.a.com");
}
}
+
+ #[test]
+ fn navigational() {
+ assert!(is_navigational(vec!["dr".to_string()], "dr.dk"));
+ assert!(is_navigational(
+ vec!["dr".to_string(), "dk".to_string()],
+ "dr.dk"
+ ));
+ }
+
+ #[test]
+ fn navigational_search() {
+ let query = Query::parse("dr dk").expect("Failed to parse query");
+
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Website A</title>
+ </head>
+ </html>
+ "#,
+ "https://www.dr.dk",
+ vec![],
+ 0.0,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Website B</title>
+ dr dk dr dk dr dk dr dk
+ </head>
+ </html>
+ "#,
+ "https://www.b.com",
+ vec![],
+ 0.0003,
+ ))
+ .expect("failed to parse webpage");
+
+ index.commit().expect("failed to commit index");
+ let result = index.search(&query).expect("Search failed");
+ assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents[0].url, "https://www.dr.dk");
+ assert_eq!(result.documents[1].url, "https://www.b.com");
+ }
}
diff --git a/src/schema.rs b/src/schema.rs
index 5807028d..232f8bde 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use tantivy::schema::{
- Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,
+ BytesOptions, Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,
};
#[derive(Clone)]
@@ -23,13 +23,15 @@ pub enum Field {
Title,
Body,
Url,
+ Domain,
BacklinkText,
Centrality,
}
-pub static ALL_FIELDS: [Field; 5] = [
+pub static ALL_FIELDS: [Field; 6] = [
Field::Title,
Field::Body,
Field::Url,
+ Field::Domain,
Field::BacklinkText,
Field::Centrality,
];
@@ -48,6 +50,9 @@ impl Field {
Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),
Field::Body => IndexingOption::Text(self.default_text_options().set_stored()),
Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),
+ Field::Domain => {
+ IndexingOption::Bytes(BytesOptions::default().set_fast().set_indexed())
+ }
Field::BacklinkText => IndexingOption::Text(self.default_text_options()),
Field::Centrality => IndexingOption::Numeric(
NumericOptions::default()
@@ -62,6 +67,7 @@ impl Field {
Field::Title => "title",
Field::Body => "body",
Field::Url => "url",
+ Field::Domain => "host",
Field::BacklinkText => "backlink_text",
Field::Centrality => "centrality",
}
@@ -75,6 +81,7 @@ pub fn create_schema() -> tantivy::schema::Schema {
match field.options() {
IndexingOption::Text(options) => builder.add_text_field(field.as_str(), options),
IndexingOption::Numeric(options) => builder.add_f64_field(field.as_str(), options),
+ IndexingOption::Bytes(options) => builder.add_bytes_field(field.as_str(), options),
};
}
@@ -84,4 +91,5 @@ pub fn create_schema() -> tantivy::schema::Schema {
enum IndexingOption {
Text(tantivy::schema::TextOptions),
Numeric(tantivy::schema::NumericOptions),
+ Bytes(tantivy::schema::BytesOptions),
}
diff --git a/src/search_index.rs b/src/search_index.rs
index ed23d903..6cf2b806 100644
--- a/src/search_index.rs
+++ b/src/search_index.rs
@@ -63,8 +63,10 @@ impl Index {
let tantivy_query = query.tantivy(&self.schema, self.tantivy_index.tokenizers());
let searcher = self.reader.searcher();
- let (count, docs) =
- searcher.search(&tantivy_query, &(Count, ranking::initial_collector()))?;
+ let (count, docs) = searcher.search(
+ &tantivy_query,
+ &(Count, ranking::initial_collector(query.clone())),
+ )?;
let mut webpages: Vec<RetrievedWebpage> = docs
.into_iter()
@@ -97,6 +99,11 @@ pub struct SearchResult {
pub documents: Vec<RetrievedWebpage>,
}
+#[derive(Default)]
+pub struct FastWebpage {
+ pub host: String,
+}
+
#[derive(Default)]
pub struct RetrievedWebpage {
pub title: String,
@@ -132,7 +139,7 @@ impl From<Document> for RetrievedWebpage {
.expect("Url field should be text")
.to_string()
}
- Field::BacklinkText | Field::Centrality => {}
+ Field::BacklinkText | Field::Centrality | Field::Domain => {}
}
}
@@ -280,4 +287,58 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.a.com");
assert_eq!(result.documents[1].url, "https://www.b.com");
}
+
+ #[test]
+ fn limited_top_docs() {
+ let mut index = Index::temporary().expect("Unable to open index");
+ let query = Query::parse("runner").expect("Failed to parse query");
+
+ for _ in 0..100 {
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Website for runners</title>
+ </head>
+ </html>
+ "#,
+ "https://www.example.com",
+ vec![],
+ 1.0,
+ ))
+ .expect("failed to parse webpage");
+ }
+
+ index.commit().expect("failed to commit index");
+
+ let result = index.search(&query).expect("Search failed");
+ assert_eq!(result.documents.len(), 20);
+ }
+
+ #[test]
+ fn host_search() {
+ let mut index = Index::temporary().expect("Unable to open index");
+ let query = Query::parse("dr").expect("Failed to parse query");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>News website</title>
+ </head>
+ </html>
+ "#,
+ "https://www.dr.dk",
+ vec![],
+ 1.0,
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let result = index.search(&query).expect("Search failed");
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.dr.dk");
+ }
}
diff --git a/src/webpage.rs b/src/webpage.rs
index d9c00444..739dfcd7 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -176,6 +176,23 @@ impl Html {
&self.url()[start_host..end_host]
}
+ pub fn domain(&self) -> &str {
+ let host = self.host();
+ let num_punctuations: usize = host.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();
+ if num_punctuations > 1 {
+ let domain_index = host.rfind(".").unwrap();
+ let mut start_index = host[..domain_index].rfind(".").unwrap();
+
+ if &host[start_index + 1..] == "co.uk" {
+ start_index = host[start_index..].rfind(".").unwrap();
+ }
+
+ &host[start_index + 1..]
+ } else {
+ host
+ }
+ }
+
pub fn metadata(&self) -> Vec<Meta> {
let selector = Selector::parse("meta").expect("Failed to parse selector");
self.dom
@@ -208,6 +225,10 @@ impl Html {
}
Field::Body => doc.add_text(tantivy_field, self.text()),
Field::Url => doc.add_text(tantivy_field, self.url()),
+ Field::Domain => doc.add_bytes(
+ tantivy_field,
+ bincode::serialize(self.domain()).expect("Failed to serialize bytes"),
+ ),
Field::BacklinkText | Field::Centrality => {}
}
}
@@ -263,6 +284,7 @@ mod tests {
assert_eq!(webpage.metadata(), vec![expected_meta]);
assert_eq!(webpage.url(), "https://www.example.com/whatever");
assert_eq!(webpage.host(), "www.example.com");
+ assert_eq!(webpage.domain(), "example.com");
}
#[test]
@@ -337,4 +359,12 @@ mod tests {
"This text should be the first text extracted\nThis text should be the second text extracted"
);
}
+
+ #[test]
+ fn co_uk_domain() {
+ let raw = "";
+
+ let webpage = Html::parse(raw, "https://www.domain.co.uk");
+ assert_eq!(webpage.domain(), "domain.co.uk");
+ }
}