Skip to content

web search done right

use harmonic centrality during ranking

Mikkel Denker82f82229991e
diff --git a/src/ranking.rs b/src/ranking.rs
index bf819a2b..ec48ac89 100644
--- a/src/ranking.rs
+++ b/src/ranking.rs
@@ -1,13 +1,83 @@
-use tantivy::collector::TopDocs;
+use crate::schema::Field;
+use tantivy::fastfield::FastFieldReader;
+use tantivy::{
+ collector::{Collector, TopDocs},
+ DocId, Score, SegmentReader,
+};
-pub(crate) fn initial_collector() -> TopDocs {
- TopDocs::with_limit(20) // TODO: take harmonic centrality into account initially
+pub(crate) fn initial_collector() -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {
+ TopDocs::with_limit(20).tweak_score(move |segment_reader: &SegmentReader| {
+ let centrality_field = segment_reader
+ .schema()
+ .get_field(Field::Centrality.as_str())
+ .expect("Faild to load centrality field");
+ let centrality_reader = segment_reader
+ .fast_fields()
+ .f64(centrality_field)
+ .expect("Failed to get centrality fast-field reader");
+
+ move |doc: DocId, original_score: Score| {
+ let centrality = centrality_reader.get(doc);
+ original_score as f64 + 100.0 * centrality
+ }
+ })
}
#[cfg(test)]
mod tests {
+ use super::*;
+ use crate::{
+ query::Query,
+ search_index::Index,
+ webpage::{Link, Webpage},
+ };
+
#[test]
fn harmonic_ranking() {
- todo!();
+ let query = Query::parse("great site").expect("Failed to parse query");
+
+ for _ in 0..10 {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Website A</title>
+ </head>
+ <a href="https://www.b.com">B site is great</a>
+ </html>
+ "#,
+ "https://www.a.com",
+ vec![],
+ 0.0,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Website B</title>
+ </head>
+ </html>
+ "#,
+ "https://www.b.com",
+ vec![Link {
+ source: "https://www.a.com".to_string(),
+ destination: "https://www.b.com".to_string(),
+ text: "B site is great".to_string(),
+ }],
+ 5.0,
+ ))
+ .expect("failed to parse webpage");
+
+ index.commit().expect("failed to commit index");
+ let result = index.search(&query).expect("Search failed");
+ assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents[0].url, "https://www.b.com");
+ assert_eq!(result.documents[1].url, "https://www.a.com");
+ }
}
}
diff --git a/src/schema.rs b/src/schema.rs
index 82fedef6..df31d75f 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -1,4 +1,6 @@
-use tantivy::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};
+use tantivy::schema::{
+ Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,
+};
#[derive(Clone)]
pub enum Field {
@@ -17,7 +19,7 @@ pub static ALL_FIELDS: [Field; 5] = [
];
impl Field {
- fn default_options(&self) -> tantivy::schema::TextOptions {
+ fn default_text_options(&self) -> tantivy::schema::TextOptions {
TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer("en_stem")
@@ -25,13 +27,17 @@ impl Field {
)
}
- pub fn options(&self) -> tantivy::schema::TextOptions {
+ fn options(&self) -> IndexingOption {
match self {
- Field::Title => self.default_options().set_stored(),
- Field::Body => self.default_options().set_stored(),
- Field::Url => self.default_options().set_stored(),
- Field::BacklinkText => self.default_options(),
- Field::Centrality => self.default_options().set_fast(),
+ Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),
+ Field::Body => IndexingOption::Text(self.default_text_options().set_stored()),
+ Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),
+ Field::BacklinkText => IndexingOption::Text(self.default_text_options()),
+ Field::Centrality => IndexingOption::Numeric(
+ NumericOptions::default()
+ .set_fast(Cardinality::SingleValue)
+ .set_indexed(),
+ ),
}
}
@@ -50,8 +56,16 @@ pub fn create_schema() -> tantivy::schema::Schema {
let mut builder = tantivy::schema::Schema::builder();
for field in &ALL_FIELDS {
- builder.add_text_field(field.as_str(), field.options());
+ match field.options() {
+ IndexingOption::Text(options) => builder.add_text_field(field.as_str(), options),
+ IndexingOption::Numeric(options) => builder.add_f64_field(field.as_str(), options),
+ };
}
builder.build()
}
+
+enum IndexingOption {
+ Text(tantivy::schema::TextOptions),
+ Numeric(tantivy::schema::NumericOptions),
+}
diff --git a/src/search_index.rs b/src/search_index.rs
index f54428bc..2d7b0378 100644
--- a/src/search_index.rs
+++ b/src/search_index.rs
@@ -20,7 +20,7 @@ impl Index {
todo!();
}
- fn temporary() -> Result<Self> {
+ pub(crate) fn temporary() -> Result<Self> {
let schema = create_schema();
let tantivy_index = tantivy::Index::create_in_ram(schema);
@@ -74,16 +74,16 @@ impl Index {
}
pub struct SearchResult {
- num_docs: usize,
- documents: Vec<RetrievedWebpage>,
+ pub num_docs: usize,
+ pub documents: Vec<RetrievedWebpage>,
}
#[derive(Default)]
pub struct RetrievedWebpage {
- title: String,
- url: String,
- snippet: String,
- body: String,
+ pub title: String,
+ pub url: String,
+ pub snippet: String,
+ pub body: String,
}
impl From<Document> for RetrievedWebpage {