web search done right
simplify term proximity by not using ngrams, only slop
Mikkel Denker06e5a733484c
diff --git a/src/query/mod.rs b/src/query/mod.rsindex 3096b257..da9c07fb 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -27,12 +27,9 @@ use tantivy::{};pub mod intersection;-mod ngram;pub mod parser;use parser::Term;-use self::ngram::NGram;-const MAX_SIMILAR_TERMS: usize = 10;#[derive(Clone, Debug)]@@ -50,43 +47,40 @@ fn proximity_queries() -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {let mut proximity_queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = Vec::new();- let ngrams: NGram<3, _> = NGram::from_iter(simple_terms_text.into_iter());let proxmity_fields = [Field::Title, Field::CleanBody];- for ngram in ngrams {- for field in &proxmity_fields {- let tantivy_field = schema.get_field(field.as_str()).unwrap();- let tantivy_entry = schema.get_field_entry(tantivy_field);-- for (boost, slop) in [(3, 0), (2, 4), (1, 16)] {- let mut terms = Vec::new();-- let mut num_terms = 0;- for term in ngram.iter().flatten() {- let analyzer = Term::get_tantivy_analyzer(tantivy_entry, tokenizer_manager);- num_terms += 1;- terms.append(&mut Term::process_tantivy_term(- term,- analyzer,- tantivy_field,- ));- }+ for field in &proxmity_fields {+ let tantivy_field = schema.get_field(field.as_str()).unwrap();+ let tantivy_entry = schema.get_field_entry(tantivy_field);++ for (boost, slop) in [(4, 0), (3, 4), (2, 16), (1, 32)] {+ let mut terms = Vec::new();++ let mut num_terms = 0;+ for term in &simple_terms_text {+ let analyzer = Term::get_tantivy_analyzer(tantivy_entry, tokenizer_manager);+ num_terms += 1;+ terms.append(&mut Term::process_tantivy_term(+ term,+ analyzer,+ tantivy_field,+ ));+ }- if num_terms < 2 {- continue;- }+ if num_terms < 2 {+ continue;+ }- let terms = terms.into_iter().enumerate().collect();+ let terms = terms.into_iter().enumerate().collect();- proximity_queries.push((- Occur::Should,- BoostQuery::new(- PhraseQuery::new_with_offset_and_slop(terms, slop).box_clone(),- boost as f32,- )- .box_clone(),- ))- }+ proximity_queries.push((+ Occur::Should,+ BoostQuery::new(+ PhraseQuery::new_with_offset_and_slop(terms, slop).box_clone(),+ boost as f32,+ )+ .box_clone(),+ ))}}diff --git a/src/query/ngram.rs b/src/query/ngram.rsdeleted file mode 100644index a6465c56..00000000--- a/src/query/ngram.rs+++ /dev/null@@ -1,74 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::array;--pub struct NGram<const N: usize, T: Clone> {- res: [Option<T>; N],- inner: Box<dyn Iterator<Item = T>>,-}--impl<const N: usize, T: Clone> NGram<N, T> {- pub fn from_iter(it: impl Iterator<Item = T> + 'static) -> Self {- Self {- res: array::from_fn(|_| None),- inner: Box::new(it),- }- }-}--impl<const N: usize, T: Clone> Iterator for NGram<N, T> {- type Item = [Option<T>; N];-- fn next(&mut self) -> Option<Self::Item> {- self.res.rotate_left(1);- self.res[N - 1] = self.inner.next();-- if self.res.iter().all(Option::is_none) {- None- } else {- Some(self.res.clone())- }- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn simple() {- let mut it: NGram<3, _> = NGram::from_iter(vec![1, 2, 3, 4].into_iter());-- assert_eq!(it.next(), Some([None, None, Some(1)]));- assert_eq!(it.next(), Some([None, Some(1), Some(2)]));- assert_eq!(it.next(), Some([Some(1), Some(2), Some(3)]));- assert_eq!(it.next(), Some([Some(2), Some(3), Some(4)]));- assert_eq!(it.next(), Some([Some(3), Some(4), None]));- assert_eq!(it.next(), Some([Some(4), None, None]));- assert_eq!(it.next(), None);- }-- #[test]- fn small_iter() {- let mut it: NGram<3, _> = NGram::from_iter(vec![1].into_iter());-- assert_eq!(it.next(), Some([None, None, Some(1)]));- assert_eq!(it.next(), Some([None, Some(1), None]));- assert_eq!(it.next(), Some([Some(1), None, None]));- assert_eq!(it.next(), None);- }-}