web search done right
don't augment long queries with ngram lookups for performance
Mikkel Denker651ff30aa7d3
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex ec6e384c..992f40b7 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -38,9 +38,10 @@ mod plan;pub mod shortcircuit;pub mod union;+use self::{optic::AsMultipleTantivyQuery, parser::SimpleOrPhrase};use parser::Term;-use self::{optic::AsMultipleTantivyQuery, parser::SimpleOrPhrase};+pub const MAX_TERMS_FOR_NGRAM_LOOKUPS: usize = 16;#[derive(Debug)]pub struct Query {diff --git a/crates/core/src/query/plan/mod.rs b/crates/core/src/query/plan/mod.rsindex 79f7ee14..cf578f63 100644--- a/crates/core/src/query/plan/mod.rs+++ b/crates/core/src/query/plan/mod.rs@@ -22,7 +22,10 @@ pub use node::Node;use crate::schema::{self, text_field::TextField, TextFieldEnum};-use super::parser::{SimpleOrPhrase, SimpleTerm};+use super::{+ parser::{SimpleOrPhrase, SimpleTerm},+ MAX_TERMS_FOR_NGRAM_LOOKUPS,+};#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub struct Term {@@ -229,24 +232,29 @@ pub fn initial(terms: Vec<super::Term>) -> Option<Node> {let mut nodes = Vec::new();let terms_for_adjacent = terms.clone();+ let augment_with_adjacent = terms.len() <= MAX_TERMS_FOR_NGRAM_LOOKUPS;+for (i, term) in terms.into_iter().enumerate() {let mut adjacent = Vec::new();- if let super::Term::SimpleOrPhrase(_) = &term {- for window_size in 2..=3 {- for (start, end) in sliding_window(window_size, i) {- let mut compounds = Vec::new();-- for k in start..=end {- if let Some(super::Term::SimpleOrPhrase(super::SimpleOrPhrase::Simple(s))) =- terms_for_adjacent.get(k)- {- compounds.push(s.clone());+ if augment_with_adjacent {+ if let super::Term::SimpleOrPhrase(_) = &term {+ for window_size in 2..=3 {+ for (start, end) in sliding_window(window_size, i) {+ let mut compounds = Vec::new();++ for k in start..=end {+ if let Some(super::Term::SimpleOrPhrase(+ super::SimpleOrPhrase::Simple(s),+ )) = terms_for_adjacent.get(k)+ {+ compounds.push(s.clone());+ }}- }- if !compounds.is_empty() {- adjacent.push(super::TermCompound { terms: compounds });+ if !compounds.is_empty() {+ adjacent.push(super::TermCompound { terms: compounds });+ }}}}diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rsindex 84546d60..b42fba17 100644--- a/crates/core/src/ranking/signal/computer/mod.rs+++ b/crates/core/src/ranking/signal/computer/mod.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::query::optic::AsSearchableRule;-use crate::query::Query;+use crate::query::{Query, MAX_TERMS_FOR_NGRAM_LOOKUPS};use crate::schema::text_field::TextField;use crate::Result;use crate::{enum_map::EnumMap, fastfield_reader, schema::TextFieldEnum, webpage::Webpage};@@ -189,6 +189,12 @@ impl SignalComputer {.as_textfield().and_then(|f| (f.tantivy_field(schema).map(|tv_field| (f, tv_field)))){+ if text_field.ngram_size() > 1+ && query.simple_terms.len() > MAX_TERMS_FOR_NGRAM_LOOKUPS+ {+ continue;+ }+let simple_query = itertools::intersperse(query.simple_terms.iter().map(|s| s.as_str())," ",