web search done right
tests to ensure proximity values are calculated correctly
Mikkel Denker7a19797e2c04
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex d2683f2d..de795b97 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -835,15 +835,20 @@ impl SignalAggregator {let mut queries = Vec::new();for field in proximity_fields {let tv_field = schema.get_field(field.name()).unwrap();+ let tokenizer = field.tokenizer();let mut terms = Vec::with_capacity(simple_terms.len());for term in &simple_terms {- let term = tantivy::Term::from_field_text(tv_field, term);- terms.push(term);+ let mut stream = tokenizer.token_stream(term);+ while let Some(term) = stream.next() {+ let term = tantivy::Term::from_field_text(tv_field, &term.text);+ terms.push(term);+ }}let mut phrase_query = PhraseQuery::new(terms);phrase_query.set_slop(slop);+queries.push((tantivy::query::Occur::Should,Box::new(phrase_query) as Box<dyn tantivy::query::Query>,@@ -998,3 +1003,99 @@ pub struct ComputedSignal {pub coefficient: f64,pub value: f64,}++#[cfg(test)]+mod tests {+ use std::collections::HashMap;++ use crate::{+ index::Index,+ searcher::{LocalSearcher, SearchQuery},+ };++ use super::*;++ fn signals_for_query(searcher: &LocalSearcher, q: &str) -> HashMap<Signal, f64> {+ let res = searcher+ .search(&SearchQuery {+ query: q.to_string(),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ res[0].ranking_signals.clone().unwrap()+ }+ #[test]+ fn test_proximity() {+ let mut index = Index::temporary().expect("Unable to open index");++ let mut webpage = Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {}+ </body>+ </html>+ "#,+ crate::rand_words(1_000)+ ),+ "https://www.example.com",+ );++ webpage+ .html+ .set_clean_text("a b c d e f g h i j k l m n o p q r s t u v w x y z".to_string());++ index.insert(webpage).expect("failed to insert webpage");++ index.commit().unwrap();++ let searcher = LocalSearcher::new(index);++ let signals = signals_for_query(&searcher, "a b");+ assert!(signals[&Signal::ProximitySlop8] > 0.0);+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop2]);+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop1]);+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop0]);++ let signals = signals_for_query(&searcher, "a c");+ assert!(signals[&Signal::ProximitySlop8] > 0.0);+ assert_eq!(signals[&Signal::ProximitySlop0], 0.0);+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop2]);+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop1]);++ let signals = signals_for_query(&searcher, "a d");+ assert!(signals[&Signal::ProximitySlop8] > 0.0);+ assert_eq!(signals[&Signal::ProximitySlop0], 0.0);+ assert_eq!(signals[&Signal::ProximitySlop1], 0.0);+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop2]);++ let signals = signals_for_query(&searcher, "a e");+ assert!(signals[&Signal::ProximitySlop8] > 0.0);+ assert_eq!(signals[&Signal::ProximitySlop0], 0.0);+ assert_eq!(signals[&Signal::ProximitySlop1], 0.0);+ assert_eq!(signals[&Signal::ProximitySlop2], 0.0);+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);++ let signals = signals_for_query(&searcher, "a g");+ assert!(signals[&Signal::ProximitySlop8] > 0.0);+ assert_eq!(signals[&Signal::ProximitySlop0], 0.0);+ assert_eq!(signals[&Signal::ProximitySlop1], 0.0);+ assert_eq!(signals[&Signal::ProximitySlop2], 0.0);+ assert_eq!(signals[&Signal::ProximitySlop4], 0.0);++ let signals = signals_for_query(&searcher, "a k");+ assert_eq!(signals[&Signal::ProximitySlop0], 0.0);+ assert_eq!(signals[&Signal::ProximitySlop1], 0.0);+ assert_eq!(signals[&Signal::ProximitySlop2], 0.0);+ assert_eq!(signals[&Signal::ProximitySlop4], 0.0);+ assert_eq!(signals[&Signal::ProximitySlop8], 0.0);+ }+}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex df22bdfa..5fc7ab4f 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -14,6 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::collections::HashMap;use std::str::FromStr;use uuid::Uuid;@@ -25,7 +26,7 @@ use crate::inverted_index::RetrievedWebpage;use crate::query::Query;use crate::ranking::centrality_store::SearchCentralityStore;use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};-use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};+use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator, ALL_SIGNALS};use crate::schema::TextField;use crate::search_ctx::Ctx;use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};@@ -307,21 +308,33 @@ impl LocalSearcher {let has_more_results = search_len != top_websites.len();let pointers: Vec<_> = top_websites- .into_iter()- .map(|website| website.pointer)+ .iter()+ .map(|website| website.pointer.clone()).collect();let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;+ let mut webpages: Vec<_> = retrieved_sites+ .into_iter()+ .map(DisplayedWebpage::from)+ .collect();++ for (webpage, ranking) in webpages.iter_mut().zip(top_websites.into_iter()) {+ let mut ranking_signals = HashMap::new();++ for signal in ALL_SIGNALS {+ ranking_signals.insert(signal, *ranking.signals.get(signal).unwrap_or(&0.0));+ }++ webpage.ranking_signals = Some(ranking_signals);+ }+Ok(WebsitesResult {spell_corrected_query: search_result.spell_corrected_query.map(HighlightedSpellCorrection::from),num_hits: search_result.num_websites,- webpages: retrieved_sites- .into_iter()- .map(DisplayedWebpage::from)- .collect(),+ webpages,discussions: None,widget: None,direct_answer: None,diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 131415d2..33506e40 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -331,6 +331,11 @@ impl Html {html}+ #[cfg(test)]+ pub fn set_clean_text(&mut self, text: String) {+ self.clean_text = Some(text);+ }+pub fn parse_without_text(html: &str, url: &str) -> Self {let root = kuchiki::parse_html().one(html);let mut url: Url = url.to_string().into();diff --git a/ltr/generate_data.py b/ltr/generate_data.pyindex 89f42af2..5b425411 100644--- a/ltr/generate_data.py+++ b/ltr/generate_data.py@@ -7,8 +7,8 @@ import pandas as pdfrom tqdm import tqdmSTRACT_ENDPOINT = "localhost:3000"-NUM_QUERIES = 10-TOP_N_RESULTS = 10+NUM_QUERIES = 1+TOP_N_RESULTS = 2CENTRALITY_BOOST_OPTIC = 'Ranking(Signal("host_centrality"), 100000000);'