Skip to content

web search done right

Ftr/term proximity ranking (#49)

Mikkel Denker8446c38c6d8d
diff --git a/src/query/mod.rs b/src/query/mod.rs
index 004d5b98..3096b257 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -16,28 +16,83 @@
use crate::{
ranking::goggles::{Goggle, SignalAggregator},
+ schema::Field,
Result,
};
use std::{collections::HashMap, sync::Arc};
use tantivy::{
- query::{BooleanQuery, Occur, QueryClone},
+ query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, QueryClone},
schema::Schema,
tokenizer::TokenizerManager,
};
pub mod intersection;
+mod ngram;
pub mod parser;
use parser::Term;
+use self::ngram::NGram;
+
const MAX_SIMILAR_TERMS: usize = 10;
#[derive(Clone, Debug)]
pub struct Query {
#[allow(clippy::vec_box)]
terms: Vec<Box<Term>>,
+ simple_terms_text: Vec<String>,
tantivy_query: Box<BooleanQuery>,
}
+fn proximity_queries(
+ simple_terms_text: Vec<String>,
+ schema: &Arc<Schema>,
+ tokenizer_manager: &TokenizerManager,
+) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
+ let mut proximity_queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = Vec::new();
+
+ let ngrams: NGram<3, _> = NGram::from_iter(simple_terms_text.into_iter());
+ let proxmity_fields = [Field::Title, Field::CleanBody];
+
+ for ngram in ngrams {
+ for field in &proxmity_fields {
+ let tantivy_field = schema.get_field(field.as_str()).unwrap();
+ let tantivy_entry = schema.get_field_entry(tantivy_field);
+
+ for (boost, slop) in [(3, 0), (2, 4), (1, 16)] {
+ let mut terms = Vec::new();
+
+ let mut num_terms = 0;
+ for term in ngram.iter().flatten() {
+ let analyzer = Term::get_tantivy_analyzer(tantivy_entry, tokenizer_manager);
+ num_terms += 1;
+ terms.append(&mut Term::process_tantivy_term(
+ term,
+ analyzer,
+ tantivy_field,
+ ));
+ }
+
+ if num_terms < 2 {
+ continue;
+ }
+
+ let terms = terms.into_iter().enumerate().collect();
+
+ proximity_queries.push((
+ Occur::Should,
+ BoostQuery::new(
+ PhraseQuery::new_with_offset_and_slop(terms, slop).box_clone(),
+ boost as f32,
+ )
+ .box_clone(),
+ ))
+ }
+ }
+ }
+
+ proximity_queries
+}
+
impl Query {
pub fn parse(
query: &str,
@@ -65,15 +120,33 @@ impl Query {
let field_boost = aggregator.field_boosts();
- let queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms
+ let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms
.iter()
.flat_map(|term| term.as_tantivy_query(&fields, tokenizer_manager, field_boost))
.collect();
+ let simple_terms_text: Vec<String> = terms
+ .iter()
+ .filter_map(|term| {
+ if let Term::Simple(term) = term.as_ref() {
+ Some(term.clone())
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ queries.append(&mut proximity_queries(
+ simple_terms_text.clone(),
+ &schema,
+ tokenizer_manager,
+ ));
+
let tantivy_query = Box::new(BooleanQuery::new(queries));
Ok(Query {
terms,
+ simple_terms_text,
tantivy_query,
})
}
@@ -87,16 +160,7 @@ impl Query {
}
pub fn simple_terms(&self) -> Vec<String> {
- self.terms
- .iter()
- .filter_map(|term| {
- if let Term::Simple(term) = term.as_ref() {
- Some(term.clone())
- } else {
- None
- }
- })
- .collect()
+ self.simple_terms_text.clone()
}
pub fn terms(&self) -> &[Box<Term>] {
diff --git a/src/query/ngram.rs b/src/query/ngram.rs
new file mode 100644
index 00000000..a6465c56
--- /dev/null
+++ b/src/query/ngram.rs
@@ -0,0 +1,74 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::array;
+
+pub struct NGram<const N: usize, T: Clone> {
+ res: [Option<T>; N],
+ inner: Box<dyn Iterator<Item = T>>,
+}
+
+impl<const N: usize, T: Clone> NGram<N, T> {
+ pub fn from_iter(it: impl Iterator<Item = T> + 'static) -> Self {
+ Self {
+ res: array::from_fn(|_| None),
+ inner: Box::new(it),
+ }
+ }
+}
+
+impl<const N: usize, T: Clone> Iterator for NGram<N, T> {
+ type Item = [Option<T>; N];
+
+ fn next(&mut self) -> Option<Self::Item> {
+ self.res.rotate_left(1);
+ self.res[N - 1] = self.inner.next();
+
+ if self.res.iter().all(Option::is_none) {
+ None
+ } else {
+ Some(self.res.clone())
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn simple() {
+ let mut it: NGram<3, _> = NGram::from_iter(vec![1, 2, 3, 4].into_iter());
+
+ assert_eq!(it.next(), Some([None, None, Some(1)]));
+ assert_eq!(it.next(), Some([None, Some(1), Some(2)]));
+ assert_eq!(it.next(), Some([Some(1), Some(2), Some(3)]));
+ assert_eq!(it.next(), Some([Some(2), Some(3), Some(4)]));
+ assert_eq!(it.next(), Some([Some(3), Some(4), None]));
+ assert_eq!(it.next(), Some([Some(4), None, None]));
+ assert_eq!(it.next(), None);
+ }
+
+ #[test]
+ fn small_iter() {
+ let mut it: NGram<3, _> = NGram::from_iter(vec![1].into_iter());
+
+ assert_eq!(it.next(), Some([None, None, Some(1)]));
+ assert_eq!(it.next(), Some([None, Some(1), None]));
+ assert_eq!(it.next(), Some([Some(1), None, None]));
+ assert_eq!(it.next(), None);
+ }
+}
diff --git a/src/query/parser.rs b/src/query/parser.rs
index 03730c5c..2cbccfa9 100644
--- a/src/query/parser.rs
+++ b/src/query/parser.rs
@@ -223,7 +223,7 @@ impl Term {
Box::new(BoostQuery::new(processed_query, boost))
}
- fn get_tantivy_analyzer(
+ pub fn get_tantivy_analyzer(
entry: &tantivy::schema::FieldEntry,
tokenizer_manager: &tantivy::tokenizer::TokenizerManager,
) -> Option<TextAnalyzer> {
@@ -238,7 +238,7 @@ impl Term {
}
}
- fn process_tantivy_term(
+ pub fn process_tantivy_term(
term: &str,
analyzer: Option<TextAnalyzer>,
tantivy_field: tantivy::schema::Field,
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 3663c7c9..ef508c02 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -195,7 +195,7 @@ mod tests {
<head>
<title>Website B</title>
</head>
- dr dk dr dk and some other text {CONTENT}
+ dr and some other text {CONTENT} dk
</html>
"#
),
@@ -519,4 +519,84 @@ mod tests {
assert_eq!(res.webpages.num_docs, 3);
assert_eq!(&res.webpages.documents[0].url, "https://www.centrality.com");
}
+
+ #[test]
+ fn term_proximity_ranking() {
+ let mut index = Index::temporary().expect("Unable to open index");
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} termA termB d d d d d d d d d
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.first.com",
+ vec![],
+ 1.0,
+ 500,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} termA d d d d d d d d d termB
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.third.com",
+ vec![],
+ 1.0,
+ 500,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} termA d d d d termB d d d d d
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.second.com",
+ vec![],
+ 1.0,
+ 500,
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+ let searcher = Searcher::new(index, None, None);
+
+ let result = searcher
+ .search("termA termB", None, None, None)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
+
+ assert_eq!(result.num_docs, 3);
+ assert_eq!(result.documents.len(), 3);
+ assert_eq!(result.documents[0].url, "https://www.first.com");
+ assert_eq!(result.documents[1].url, "https://www.second.com");
+ assert_eq!(result.documents[2].url, "https://www.third.com");
+ }
}