Skip to content

web search done right

simplify term proximity by not using ngrams, only slop

Mikkel Denker06e5a733484c
diff --git a/src/query/mod.rs b/src/query/mod.rs
index 3096b257..da9c07fb 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -27,12 +27,9 @@ use tantivy::{
};
pub mod intersection;
-mod ngram;
pub mod parser;
use parser::Term;
-use self::ngram::NGram;
-
const MAX_SIMILAR_TERMS: usize = 10;
#[derive(Clone, Debug)]
@@ -50,43 +47,40 @@ fn proximity_queries(
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
let mut proximity_queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = Vec::new();
- let ngrams: NGram<3, _> = NGram::from_iter(simple_terms_text.into_iter());
let proxmity_fields = [Field::Title, Field::CleanBody];
- for ngram in ngrams {
- for field in &proxmity_fields {
- let tantivy_field = schema.get_field(field.as_str()).unwrap();
- let tantivy_entry = schema.get_field_entry(tantivy_field);
-
- for (boost, slop) in [(3, 0), (2, 4), (1, 16)] {
- let mut terms = Vec::new();
-
- let mut num_terms = 0;
- for term in ngram.iter().flatten() {
- let analyzer = Term::get_tantivy_analyzer(tantivy_entry, tokenizer_manager);
- num_terms += 1;
- terms.append(&mut Term::process_tantivy_term(
- term,
- analyzer,
- tantivy_field,
- ));
- }
+ for field in &proxmity_fields {
+ let tantivy_field = schema.get_field(field.as_str()).unwrap();
+ let tantivy_entry = schema.get_field_entry(tantivy_field);
+
+ for (boost, slop) in [(4, 0), (3, 4), (2, 16), (1, 32)] {
+ let mut terms = Vec::new();
+
+ let mut num_terms = 0;
+ for term in &simple_terms_text {
+ let analyzer = Term::get_tantivy_analyzer(tantivy_entry, tokenizer_manager);
+ num_terms += 1;
+ terms.append(&mut Term::process_tantivy_term(
+ term,
+ analyzer,
+ tantivy_field,
+ ));
+ }
- if num_terms < 2 {
- continue;
- }
+ if num_terms < 2 {
+ continue;
+ }
- let terms = terms.into_iter().enumerate().collect();
+ let terms = terms.into_iter().enumerate().collect();
- proximity_queries.push((
- Occur::Should,
- BoostQuery::new(
- PhraseQuery::new_with_offset_and_slop(terms, slop).box_clone(),
- boost as f32,
- )
- .box_clone(),
- ))
- }
+ proximity_queries.push((
+ Occur::Should,
+ BoostQuery::new(
+ PhraseQuery::new_with_offset_and_slop(terms, slop).box_clone(),
+ boost as f32,
+ )
+ .box_clone(),
+ ))
}
}
diff --git a/src/query/ngram.rs b/src/query/ngram.rs
deleted file mode 100644
index a6465c56..00000000
--- a/src/query/ngram.rs
+++ /dev/null
@@ -1,74 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::array;
-
-pub struct NGram<const N: usize, T: Clone> {
- res: [Option<T>; N],
- inner: Box<dyn Iterator<Item = T>>,
-}
-
-impl<const N: usize, T: Clone> NGram<N, T> {
- pub fn from_iter(it: impl Iterator<Item = T> + 'static) -> Self {
- Self {
- res: array::from_fn(|_| None),
- inner: Box::new(it),
- }
- }
-}
-
-impl<const N: usize, T: Clone> Iterator for NGram<N, T> {
- type Item = [Option<T>; N];
-
- fn next(&mut self) -> Option<Self::Item> {
- self.res.rotate_left(1);
- self.res[N - 1] = self.inner.next();
-
- if self.res.iter().all(Option::is_none) {
- None
- } else {
- Some(self.res.clone())
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn simple() {
- let mut it: NGram<3, _> = NGram::from_iter(vec![1, 2, 3, 4].into_iter());
-
- assert_eq!(it.next(), Some([None, None, Some(1)]));
- assert_eq!(it.next(), Some([None, Some(1), Some(2)]));
- assert_eq!(it.next(), Some([Some(1), Some(2), Some(3)]));
- assert_eq!(it.next(), Some([Some(2), Some(3), Some(4)]));
- assert_eq!(it.next(), Some([Some(3), Some(4), None]));
- assert_eq!(it.next(), Some([Some(4), None, None]));
- assert_eq!(it.next(), None);
- }
-
- #[test]
- fn small_iter() {
- let mut it: NGram<3, _> = NGram::from_iter(vec![1].into_iter());
-
- assert_eq!(it.next(), Some([None, None, Some(1)]));
- assert_eq!(it.next(), Some([None, Some(1), None]));
- assert_eq!(it.next(), Some([Some(1), None, None]));
- assert_eq!(it.next(), None);
- }
-}