Skip to content

web search done right

tests to ensure proximity values are calculated correctly

Mikkel Denker7a19797e2c04
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index d2683f2d..de795b97 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -835,15 +835,20 @@ impl SignalAggregator {
let mut queries = Vec::new();
for field in proximity_fields {
let tv_field = schema.get_field(field.name()).unwrap();
+ let tokenizer = field.tokenizer();
let mut terms = Vec::with_capacity(simple_terms.len());
for term in &simple_terms {
- let term = tantivy::Term::from_field_text(tv_field, term);
- terms.push(term);
+ let mut stream = tokenizer.token_stream(term);
+ while let Some(term) = stream.next() {
+ let term = tantivy::Term::from_field_text(tv_field, &term.text);
+ terms.push(term);
+ }
}
let mut phrase_query = PhraseQuery::new(terms);
phrase_query.set_slop(slop);
+
queries.push((
tantivy::query::Occur::Should,
Box::new(phrase_query) as Box<dyn tantivy::query::Query>,
@@ -998,3 +1003,99 @@ pub struct ComputedSignal {
pub coefficient: f64,
pub value: f64,
}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::HashMap;
+
+ use crate::{
+ index::Index,
+ searcher::{LocalSearcher, SearchQuery},
+ };
+
+ use super::*;
+
+ fn signals_for_query(searcher: &LocalSearcher, q: &str) -> HashMap<Signal, f64> {
+ let res = searcher
+ .search(&SearchQuery {
+ query: q.to_string(),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ res[0].ranking_signals.clone().unwrap()
+ }
+ #[test]
+ fn test_proximity() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ let mut webpage = Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {}
+ </body>
+ </html>
+ "#,
+ crate::rand_words(1_000)
+ ),
+ "https://www.example.com",
+ );
+
+ webpage
+ .html
+ .set_clean_text("a b c d e f g h i j k l m n o p q r s t u v w x y z".to_string());
+
+ index.insert(webpage).expect("failed to insert webpage");
+
+ index.commit().unwrap();
+
+ let searcher = LocalSearcher::new(index);
+
+ let signals = signals_for_query(&searcher, "a b");
+ assert!(signals[&Signal::ProximitySlop8] > 0.0);
+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);
+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop2]);
+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop1]);
+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop0]);
+
+ let signals = signals_for_query(&searcher, "a c");
+ assert!(signals[&Signal::ProximitySlop8] > 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop0], 0.0);
+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);
+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop2]);
+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop1]);
+
+ let signals = signals_for_query(&searcher, "a d");
+ assert!(signals[&Signal::ProximitySlop8] > 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop0], 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop1], 0.0);
+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);
+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop2]);
+
+ let signals = signals_for_query(&searcher, "a e");
+ assert!(signals[&Signal::ProximitySlop8] > 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop0], 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop1], 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop2], 0.0);
+ assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);
+
+ let signals = signals_for_query(&searcher, "a g");
+ assert!(signals[&Signal::ProximitySlop8] > 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop0], 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop1], 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop2], 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop4], 0.0);
+
+ let signals = signals_for_query(&searcher, "a k");
+ assert_eq!(signals[&Signal::ProximitySlop0], 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop1], 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop2], 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop4], 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop8], 0.0);
+ }
+}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index df22bdfa..5fc7ab4f 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::collections::HashMap;
use std::str::FromStr;
use uuid::Uuid;
@@ -25,7 +26,7 @@ use crate::inverted_index::RetrievedWebpage;
use crate::query::Query;
use crate::ranking::centrality_store::SearchCentralityStore;
use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};
-use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};
+use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator, ALL_SIGNALS};
use crate::schema::TextField;
use crate::search_ctx::Ctx;
use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};
@@ -307,21 +308,33 @@ impl LocalSearcher {
let has_more_results = search_len != top_websites.len();
let pointers: Vec<_> = top_websites
- .into_iter()
- .map(|website| website.pointer)
+ .iter()
+ .map(|website| website.pointer.clone())
.collect();
let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;
+ let mut webpages: Vec<_> = retrieved_sites
+ .into_iter()
+ .map(DisplayedWebpage::from)
+ .collect();
+
+ for (webpage, ranking) in webpages.iter_mut().zip(top_websites.into_iter()) {
+ let mut ranking_signals = HashMap::new();
+
+ for signal in ALL_SIGNALS {
+ ranking_signals.insert(signal, *ranking.signals.get(signal).unwrap_or(&0.0));
+ }
+
+ webpage.ranking_signals = Some(ranking_signals);
+ }
+
Ok(WebsitesResult {
spell_corrected_query: search_result
.spell_corrected_query
.map(HighlightedSpellCorrection::from),
num_hits: search_result.num_websites,
- webpages: retrieved_sites
- .into_iter()
- .map(DisplayedWebpage::from)
- .collect(),
+ webpages,
discussions: None,
widget: None,
direct_answer: None,
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 131415d2..33506e40 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -331,6 +331,11 @@ impl Html {
html
}
+ #[cfg(test)]
+ pub fn set_clean_text(&mut self, text: String) {
+ self.clean_text = Some(text);
+ }
+
pub fn parse_without_text(html: &str, url: &str) -> Self {
let root = kuchiki::parse_html().one(html);
let mut url: Url = url.to_string().into();
diff --git a/ltr/generate_data.py b/ltr/generate_data.py
index 89f42af2..5b425411 100644
--- a/ltr/generate_data.py
+++ b/ltr/generate_data.py
@@ -7,8 +7,8 @@ import pandas as pd
from tqdm import tqdm
STRACT_ENDPOINT = "localhost:3000"
-NUM_QUERIES = 10
-TOP_N_RESULTS = 10
+NUM_QUERIES = 1
+TOP_N_RESULTS = 2
CENTRALITY_BOOST_OPTIC = 'Ranking(Signal("host_centrality"), 100000000);'