web search done right
use backlink text during ranking
Mikkel Denker847cf07e842e
diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex 86b50d96..e0b3376c 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -484,7 +484,7 @@ mod tests {}#[test]- fn searchable_backlinks() {+ fn not_searchable_backlinks() {let mut index = InvertedIndex::temporary().expect("Unable to open index");let query = Query::parse("great site", index.schema(), index.tokenizers()).expect("Failed to parse query");@@ -544,9 +544,8 @@ mod tests {.documents.sort_by(|a, b| a.url.partial_cmp(&b.url).unwrap());- assert_eq!(result.documents.len(), 2);+ assert_eq!(result.documents.len(), 1);assert_eq!(result.documents[0].url, "https://www.a.com");- assert_eq!(result.documents[1].url, "https://www.b.com");}#[test]diff --git a/src/query/mod.rs b/src/query/mod.rsindex f91c22da..ade301a4 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -63,8 +63,7 @@ impl Query {let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms.iter()- .map(|term| term.as_tantivy_query(&fields, tokenizer_manager))- // .map(|term_query| (Occur::Must, term_query))+ .flat_map(|term| term.as_tantivy_query(&fields, tokenizer_manager)).collect();queries.push((Occur::Should, Box::new(AllQuery)));diff --git a/src/query/parser.rs b/src/query/parser.rsindex ae49bea8..30a71063 100644--- a/src/query/parser.rs+++ b/src/query/parser.rs@@ -37,30 +37,50 @@ impl Term {&self,fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],tokenizer_manager: &TokenizerManager,- ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {match self {- Term::Simple(term) => (- Occur::Must,- Box::new(BooleanQuery::new(Term::into_tantivy_simple(- term,- fields,- tokenizer_manager,- ))),- ),- Term::Not(subterm) => (+ Term::Simple(term) => {+ let (backlink_field, backlink_field_entry) = fields+ .iter()+ .find(|(field, _)| {+ matches!(ALL_FIELDS[field.field_id() as usize], Field::BacklinkText)+ })+ .unwrap();++ vec![+ (+ Occur::Must,+ Box::new(BooleanQuery::new(Term::into_tantivy_simple(+ term,+ fields,+ tokenizer_manager,+ ))),+ ),+ (+ Occur::Should,+ Box::new(Term::tantivy_term_query(+ backlink_field,+ backlink_field_entry,+ tokenizer_manager,+ term,+ )),+ ),+ ]+ }+ Term::Not(subterm) => vec![(Occur::MustNot,- Box::new(BooleanQuery::new(vec![- subterm.as_tantivy_query(fields, tokenizer_manager)- ])),- ),- Term::Site(site) => (+ Box::new(BooleanQuery::new(+ subterm.as_tantivy_query(fields, tokenizer_manager),+ )),+ )],+ Term::Site(site) => vec![(Occur::Must,Box::new(BooleanQuery::new(Term::into_tantivy_site(site,fields,tokenizer_manager,))),- ),+ )],Term::Title(title) => {let (field, entry) = fields.iter()@@ -68,10 +88,10 @@ impl Term {matches!(ALL_FIELDS[field.field_id() as usize], Field::Title)}).unwrap();- (+ vec![(Occur::Must,Term::tantivy_term_query(field, entry, tokenizer_manager, title),- )+ )]}Term::Body(body) => {let (field, entry) = fields@@ -80,20 +100,20 @@ impl Term {matches!(ALL_FIELDS[field.field_id() as usize], Field::AllBody)}).unwrap();- (+ vec![(Occur::Must,Term::tantivy_term_query(field, entry, tokenizer_manager, body),- )+ )]}Term::Url(url) => {let (field, entry) = fields.iter().find(|(field, _)| matches!(ALL_FIELDS[field.field_id() as usize], Field::Url)).unwrap();- (+ vec![(Occur::Must,Term::tantivy_term_query(field, entry, tokenizer_manager, url),- )+ )]}}}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex ef2ba76f..e03f0f49 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -71,8 +71,10 @@ mod tests {<head><title>Website A</title></head>- {CONTENT}- <a href="https://www.b.com">B site is great</a>+ <body>+ {CONTENT}+ example example example+ </body></html>"#),@@ -97,11 +99,7 @@ mod tests {"#),"https://www.b.com",- vec![Link {- source: "https://www.a.com".to_string().into(),- destination: "https://www.b.com".to_string().into(),- text: "B site is great".to_string(),- }],+ vec![],5.0,500,))@@ -109,7 +107,7 @@ mod tests {index.commit().expect("failed to commit index");let searcher = Searcher::from(index);- let result = searcher.search("great site", None).expect("Search failed");+ let result = searcher.search("example", None).expect("Search failed");assert_eq!(result.webpages.documents.len(), 2);assert_eq!(result.webpages.documents[0].url, "https://www.b.com");assert_eq!(result.webpages.documents[1].url, "https://www.a.com");@@ -302,4 +300,57 @@ mod tests {assert_eq!(result.webpages.documents.len(), 2);assert_eq!(result.webpages.documents[0].url, "https://www.first.com");}++ #[test]+ fn backlink_text() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test site</title>+ </head>+ <body>+ test+ </body>+ </html>+ "#,+ "https://www.first.com",+ vec![Link {+ source: "https://www.second.com".to_string().into(),+ destination: "https://www.first.com".to_string().into(),+ text: "test this is the best test site".to_string(),+ }],+ 0.0,+ 500,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Second test site</title>+ </head>+ <body>+ test test test test test test test+ </body>+ </html>+ "#,+ "https://www.second.com",+ vec![],+ 0.003,+ 500,+ ))+ .expect("failed to parse webpage");++ index.commit().expect("failed to commit index");+ let searcher = Searcher::from(index);+ let result = searcher.search("test", None).expect("Search failed");++ assert_eq!(result.webpages.documents.len(), 2);+ assert_eq!(result.webpages.documents[0].url, "https://www.first.com");+ }}diff --git a/src/schema.rs b/src/schema.rsindex 5d761e70..f76d8acd 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -176,8 +176,8 @@ impl Field {Field::Domain => Some(4.0),Field::AllBody => Some(0.01),Field::StemmedAllBody => Some(0.001),- Field::BacklinkText- | Field::Centrality+ Field::BacklinkText => Some(4.0),+ Field::Centrality| Field::IsHomepage| Field::PrimaryImageUuid| Field::FetchTimeMs@@ -189,7 +189,7 @@ impl Field {}pub fn is_searchable(&self) -> bool {- !matches!(self, Field::PrimaryImageUuid)+ !matches!(self, Field::PrimaryImageUuid | Field::BacklinkText)}}