Skip to content

web search done right

url digits and slashes are computable before search

Mikkel Denkerd9bf123572e8
diff --git a/core/src/index.rs b/core/src/index.rs
index 35fe1cda..9a07c07f 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -237,4 +237,86 @@ mod tests {
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.example.com");
}
+
+ #[test]
+ fn bm25_all_docs() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.first.com",
+ ))
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.second.com",
+ ))
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.third.com",
+ ))
+ .expect("failed to insert webpage");
+
+ index.commit().unwrap();
+
+ let searcher = LocalSearcher::from(index);
+ let res = searcher
+ .search(&SearchQuery {
+ query: "test".to_string(),
+ return_ranking_signals: true,
+ ..Default::default()
+ })
+ .unwrap();
+
+ assert!(res
+ .webpages
+ .iter()
+ .map(|d| d
+ .ranking_signals
+ .as_ref()
+ .unwrap()
+ .get(&crate::ranking::Signal::Bm25)
+ .unwrap())
+ .all(|&v| v > 0.0));
+ }
}
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 7d4e6492..a8362ea0 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -503,6 +503,26 @@ impl Signal {
let region = Region::guess_from(webpage).unwrap_or(Region::All);
Some(score_region(region, signal_aggregator))
}
+ Signal::UrlDigits => {
+ let num_digits = webpage
+ .html
+ .url()
+ .path_and_query()
+ .chars()
+ .filter(|c| c.is_ascii_digit())
+ .count() as f64;
+ Some(score_digits(num_digits))
+ }
+ Signal::UrlSlashes => {
+ let num_slashes = webpage
+ .html
+ .url()
+ .path_and_query()
+ .chars()
+ .filter(|c| c == &'/')
+ .count() as f64;
+ Some(score_slashes(num_slashes))
+ }
Signal::Bm25
| Signal::Bm25Title
| Signal::Bm25TitleBigrams
@@ -528,8 +548,6 @@ impl Signal {
| Signal::ProximitySlop2
| Signal::ProximitySlop4
| Signal::ProximitySlop8
- | Signal::UrlDigits
- | Signal::UrlSlashes
| Signal::CrossEncoder
| Signal::InboundSimilarity
| Signal::LambdaMART