web search done right
url digits and slashes are computable before search
Mikkel Denkerd9bf123572e8
diff --git a/core/src/index.rs b/core/src/index.rsindex 35fe1cda..9a07c07f 100644--- a/core/src/index.rs+++ b/core/src/index.rs@@ -237,4 +237,86 @@ mod tests {assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.example.com");}++ #[test]+ fn bm25_all_docs() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT} {}+ </body>+ </html>+ "#,+ crate::rand_words(100)+ ),+ "https://www.first.com",+ ))+ .expect("failed to insert webpage");+ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT} {}+ </body>+ </html>+ "#,+ crate::rand_words(100)+ ),+ "https://www.second.com",+ ))+ .expect("failed to insert webpage");+ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT} {}+ </body>+ </html>+ "#,+ crate::rand_words(100)+ ),+ "https://www.third.com",+ ))+ .expect("failed to insert webpage");++ index.commit().unwrap();++ let searcher = LocalSearcher::from(index);+ let res = searcher+ .search(&SearchQuery {+ query: "test".to_string(),+ return_ranking_signals: true,+ ..Default::default()+ })+ .unwrap();++ assert!(res+ .webpages+ .iter()+ .map(|d| d+ .ranking_signals+ .as_ref()+ .unwrap()+ .get(&crate::ranking::Signal::Bm25)+ .unwrap())+ .all(|&v| v > 0.0));+ }}diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 7d4e6492..a8362ea0 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -503,6 +503,26 @@ impl Signal {let region = Region::guess_from(webpage).unwrap_or(Region::All);Some(score_region(region, signal_aggregator))}+ Signal::UrlDigits => {+ let num_digits = webpage+ .html+ .url()+ .path_and_query()+ .chars()+ .filter(|c| c.is_ascii_digit())+ .count() as f64;+ Some(score_digits(num_digits))+ }+ Signal::UrlSlashes => {+ let num_slashes = webpage+ .html+ .url()+ .path_and_query()+ .chars()+ .filter(|c| c == &'/')+ .count() as f64;+ Some(score_slashes(num_slashes))+ }Signal::Bm25| Signal::Bm25Title| Signal::Bm25TitleBigrams@@ -528,8 +548,6 @@ impl Signal {| Signal::ProximitySlop2| Signal::ProximitySlop4| Signal::ProximitySlop8- | Signal::UrlDigits- | Signal::UrlSlashes| Signal::CrossEncoder| Signal::InboundSimilarity| Signal::LambdaMART