web search done right
Ftr/term proximity ranking (#49)
Mikkel Denker8446c38c6d8d
diff --git a/src/query/mod.rs b/src/query/mod.rsindex 004d5b98..3096b257 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -16,28 +16,83 @@use crate::{ranking::goggles::{Goggle, SignalAggregator},+ schema::Field,Result,};use std::{collections::HashMap, sync::Arc};use tantivy::{- query::{BooleanQuery, Occur, QueryClone},+ query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, QueryClone},schema::Schema,tokenizer::TokenizerManager,};pub mod intersection;+mod ngram;pub mod parser;use parser::Term;+use self::ngram::NGram;+const MAX_SIMILAR_TERMS: usize = 10;#[derive(Clone, Debug)]pub struct Query {#[allow(clippy::vec_box)]terms: Vec<Box<Term>>,+ simple_terms_text: Vec<String>,tantivy_query: Box<BooleanQuery>,}+fn proximity_queries(+ simple_terms_text: Vec<String>,+ schema: &Arc<Schema>,+ tokenizer_manager: &TokenizerManager,+) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {+ let mut proximity_queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = Vec::new();++ let ngrams: NGram<3, _> = NGram::from_iter(simple_terms_text.into_iter());+ let proxmity_fields = [Field::Title, Field::CleanBody];++ for ngram in ngrams {+ for field in &proxmity_fields {+ let tantivy_field = schema.get_field(field.as_str()).unwrap();+ let tantivy_entry = schema.get_field_entry(tantivy_field);++ for (boost, slop) in [(3, 0), (2, 4), (1, 16)] {+ let mut terms = Vec::new();++ let mut num_terms = 0;+ for term in ngram.iter().flatten() {+ let analyzer = Term::get_tantivy_analyzer(tantivy_entry, tokenizer_manager);+ num_terms += 1;+ terms.append(&mut Term::process_tantivy_term(+ term,+ analyzer,+ tantivy_field,+ ));+ }++ if num_terms < 2 {+ continue;+ }++ let terms = terms.into_iter().enumerate().collect();++ proximity_queries.push((+ Occur::Should,+ BoostQuery::new(+ PhraseQuery::new_with_offset_and_slop(terms, slop).box_clone(),+ boost as f32,+ )+ .box_clone(),+ ))+ }+ }+ }++ proximity_queries+}+impl Query {pub fn parse(query: &str,@@ -65,15 +120,33 @@ impl Query {let field_boost = aggregator.field_boosts();- let queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms+ let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms.iter().flat_map(|term| term.as_tantivy_query(&fields, tokenizer_manager, field_boost)).collect();+ let simple_terms_text: Vec<String> = terms+ .iter()+ .filter_map(|term| {+ if let Term::Simple(term) = term.as_ref() {+ Some(term.clone())+ } else {+ None+ }+ })+ .collect();++ queries.append(&mut proximity_queries(+ simple_terms_text.clone(),+ &schema,+ tokenizer_manager,+ ));+let tantivy_query = Box::new(BooleanQuery::new(queries));Ok(Query {terms,+ simple_terms_text,tantivy_query,})}@@ -87,16 +160,7 @@ impl Query {}pub fn simple_terms(&self) -> Vec<String> {- self.terms- .iter()- .filter_map(|term| {- if let Term::Simple(term) = term.as_ref() {- Some(term.clone())- } else {- None- }- })- .collect()+ self.simple_terms_text.clone()}pub fn terms(&self) -> &[Box<Term>] {diff --git a/src/query/ngram.rs b/src/query/ngram.rsnew file mode 100644index 00000000..a6465c56--- /dev/null+++ b/src/query/ngram.rs@@ -0,0 +1,74 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::array;++pub struct NGram<const N: usize, T: Clone> {+ res: [Option<T>; N],+ inner: Box<dyn Iterator<Item = T>>,+}++impl<const N: usize, T: Clone> NGram<N, T> {+ pub fn from_iter(it: impl Iterator<Item = T> + 'static) -> Self {+ Self {+ res: array::from_fn(|_| None),+ inner: Box::new(it),+ }+ }+}++impl<const N: usize, T: Clone> Iterator for NGram<N, T> {+ type Item = [Option<T>; N];++ fn next(&mut self) -> Option<Self::Item> {+ self.res.rotate_left(1);+ self.res[N - 1] = self.inner.next();++ if self.res.iter().all(Option::is_none) {+ None+ } else {+ Some(self.res.clone())+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn simple() {+ let mut it: NGram<3, _> = NGram::from_iter(vec![1, 2, 3, 4].into_iter());++ assert_eq!(it.next(), Some([None, None, Some(1)]));+ assert_eq!(it.next(), Some([None, Some(1), Some(2)]));+ assert_eq!(it.next(), Some([Some(1), Some(2), Some(3)]));+ assert_eq!(it.next(), Some([Some(2), Some(3), Some(4)]));+ assert_eq!(it.next(), Some([Some(3), Some(4), None]));+ assert_eq!(it.next(), Some([Some(4), None, None]));+ assert_eq!(it.next(), None);+ }++ #[test]+ fn small_iter() {+ let mut it: NGram<3, _> = NGram::from_iter(vec![1].into_iter());++ assert_eq!(it.next(), Some([None, None, Some(1)]));+ assert_eq!(it.next(), Some([None, Some(1), None]));+ assert_eq!(it.next(), Some([Some(1), None, None]));+ assert_eq!(it.next(), None);+ }+}diff --git a/src/query/parser.rs b/src/query/parser.rsindex 03730c5c..2cbccfa9 100644--- a/src/query/parser.rs+++ b/src/query/parser.rs@@ -223,7 +223,7 @@ impl Term {Box::new(BoostQuery::new(processed_query, boost))}- fn get_tantivy_analyzer(+ pub fn get_tantivy_analyzer(entry: &tantivy::schema::FieldEntry,tokenizer_manager: &tantivy::tokenizer::TokenizerManager,) -> Option<TextAnalyzer> {@@ -238,7 +238,7 @@ impl Term {}}- fn process_tantivy_term(+ pub fn process_tantivy_term(term: &str,analyzer: Option<TextAnalyzer>,tantivy_field: tantivy::schema::Field,diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 3663c7c9..ef508c02 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -195,7 +195,7 @@ mod tests {<head><title>Website B</title></head>- dr dk dr dk and some other text {CONTENT}+ dr and some other text {CONTENT} dk</html>"#),@@ -519,4 +519,84 @@ mod tests {assert_eq!(res.webpages.num_docs, 3);assert_eq!(&res.webpages.documents[0].url, "https://www.centrality.com");}++ #[test]+ fn term_proximity_ranking() {+ let mut index = Index::temporary().expect("Unable to open index");+ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT} termA termB d d d d d d d d d+ </body>+ </html>+ "#+ ),+ "https://www.first.com",+ vec![],+ 1.0,+ 500,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT} termA d d d d d d d d d termB+ </body>+ </html>+ "#+ ),+ "https://www.third.com",+ vec![],+ 1.0,+ 500,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT} termA d d d d termB d d d d d+ </body>+ </html>+ "#+ ),+ "https://www.second.com",+ vec![],+ 1.0,+ 500,+ ))+ .expect("failed to parse webpage");+ index.commit().expect("failed to commit index");+ let searcher = Searcher::new(index, None, None);++ let result = searcher+ .search("termA termB", None, None, None)+ .expect("Search failed")+ .into_websites()+ .unwrap()+ .webpages;++ assert_eq!(result.num_docs, 3);+ assert_eq!(result.documents.len(), 3);+ assert_eq!(result.documents[0].url, "https://www.first.com");+ assert_eq!(result.documents[1].url, "https://www.second.com");+ assert_eq!(result.documents[2].url, "https://www.third.com");+ }}