web search done right
use harmonic centrality during ranking
Mikkel Denker82f82229991e
diff --git a/src/ranking.rs b/src/ranking.rsindex bf819a2b..ec48ac89 100644--- a/src/ranking.rs+++ b/src/ranking.rs@@ -1,13 +1,83 @@-use tantivy::collector::TopDocs;+use crate::schema::Field;+use tantivy::fastfield::FastFieldReader;+use tantivy::{+ collector::{Collector, TopDocs},+ DocId, Score, SegmentReader,+};-pub(crate) fn initial_collector() -> TopDocs {- TopDocs::with_limit(20) // TODO: take harmonic centrality into account initially+pub(crate) fn initial_collector() -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {+ TopDocs::with_limit(20).tweak_score(move |segment_reader: &SegmentReader| {+ let centrality_field = segment_reader+ .schema()+ .get_field(Field::Centrality.as_str())+ .expect("Faild to load centrality field");+ let centrality_reader = segment_reader+ .fast_fields()+ .f64(centrality_field)+ .expect("Failed to get centrality fast-field reader");++ move |doc: DocId, original_score: Score| {+ let centrality = centrality_reader.get(doc);+ original_score as f64 + 100.0 * centrality+ }+ })}#[cfg(test)]mod tests {+ use super::*;+ use crate::{+ query::Query,+ search_index::Index,+ webpage::{Link, Webpage},+ };+#[test]fn harmonic_ranking() {- todo!();+ let query = Query::parse("great site").expect("Failed to parse query");++ for _ in 0..10 {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Website A</title>+ </head>+ <a href="https://www.b.com">B site is great</a>+ </html>+ "#,+ "https://www.a.com",+ vec![],+ 0.0,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Website B</title>+ </head>+ </html>+ "#,+ "https://www.b.com",+ vec![Link {+ source: "https://www.a.com".to_string(),+ destination: "https://www.b.com".to_string(),+ text: "B site is great".to_string(),+ }],+ 5.0,+ ))+ .expect("failed to parse webpage");++ index.commit().expect("failed to commit index");+ let result = index.search(&query).expect("Search failed");+ assert_eq!(result.documents.len(), 2);+ assert_eq!(result.documents[0].url, "https://www.b.com");+ assert_eq!(result.documents[1].url, "https://www.a.com");+ }}}diff --git a/src/schema.rs b/src/schema.rsindex 82fedef6..df31d75f 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -1,4 +1,6 @@-use tantivy::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};+use tantivy::schema::{+ Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,+};#[derive(Clone)]pub enum Field {@@ -17,7 +19,7 @@ pub static ALL_FIELDS: [Field; 5] = [];impl Field {- fn default_options(&self) -> tantivy::schema::TextOptions {+ fn default_text_options(&self) -> tantivy::schema::TextOptions {TextOptions::default().set_indexing_options(TextFieldIndexing::default().set_tokenizer("en_stem")@@ -25,13 +27,17 @@ impl Field {)}- pub fn options(&self) -> tantivy::schema::TextOptions {+ fn options(&self) -> IndexingOption {match self {- Field::Title => self.default_options().set_stored(),- Field::Body => self.default_options().set_stored(),- Field::Url => self.default_options().set_stored(),- Field::BacklinkText => self.default_options(),- Field::Centrality => self.default_options().set_fast(),+ Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),+ Field::Body => IndexingOption::Text(self.default_text_options().set_stored()),+ Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),+ Field::BacklinkText => IndexingOption::Text(self.default_text_options()),+ Field::Centrality => IndexingOption::Numeric(+ NumericOptions::default()+ .set_fast(Cardinality::SingleValue)+ .set_indexed(),+ ),}}@@ -50,8 +56,16 @@ pub fn create_schema() -> tantivy::schema::Schema {let mut builder = tantivy::schema::Schema::builder();for field in &ALL_FIELDS {- builder.add_text_field(field.as_str(), field.options());+ match field.options() {+ IndexingOption::Text(options) => builder.add_text_field(field.as_str(), options),+ IndexingOption::Numeric(options) => builder.add_f64_field(field.as_str(), options),+ };}builder.build()}++enum IndexingOption {+ Text(tantivy::schema::TextOptions),+ Numeric(tantivy::schema::NumericOptions),+}diff --git a/src/search_index.rs b/src/search_index.rsindex f54428bc..2d7b0378 100644--- a/src/search_index.rs+++ b/src/search_index.rs@@ -20,7 +20,7 @@ impl Index {todo!();}- fn temporary() -> Result<Self> {+ pub(crate) fn temporary() -> Result<Self> {let schema = create_schema();let tantivy_index = tantivy::Index::create_in_ram(schema);@@ -74,16 +74,16 @@ impl Index {}pub struct SearchResult {- num_docs: usize,- documents: Vec<RetrievedWebpage>,+ pub num_docs: usize,+ pub documents: Vec<RetrievedWebpage>,}#[derive(Default)]pub struct RetrievedWebpage {- title: String,- url: String,- snippet: String,- body: String,+ pub title: String,+ pub url: String,+ pub snippet: String,+ pub body: String,}impl From<Document> for RetrievedWebpage {