Skip to content

web search done right

don't augment long queries with ngram lookups for performance

Mikkel Denker651ff30aa7d3
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index ec6e384c..992f40b7 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -38,9 +38,10 @@ mod plan;
pub mod shortcircuit;
pub mod union;
+use self::{optic::AsMultipleTantivyQuery, parser::SimpleOrPhrase};
use parser::Term;
-use self::{optic::AsMultipleTantivyQuery, parser::SimpleOrPhrase};
+pub const MAX_TERMS_FOR_NGRAM_LOOKUPS: usize = 16;
#[derive(Debug)]
pub struct Query {
diff --git a/crates/core/src/query/plan/mod.rs b/crates/core/src/query/plan/mod.rs
index 79f7ee14..cf578f63 100644
--- a/crates/core/src/query/plan/mod.rs
+++ b/crates/core/src/query/plan/mod.rs
@@ -22,7 +22,10 @@ pub use node::Node;
use crate::schema::{self, text_field::TextField, TextFieldEnum};
-use super::parser::{SimpleOrPhrase, SimpleTerm};
+use super::{
+ parser::{SimpleOrPhrase, SimpleTerm},
+ MAX_TERMS_FOR_NGRAM_LOOKUPS,
+};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Term {
@@ -229,24 +232,29 @@ pub fn initial(terms: Vec<super::Term>) -> Option<Node> {
let mut nodes = Vec::new();
let terms_for_adjacent = terms.clone();
+ let augment_with_adjacent = terms.len() <= MAX_TERMS_FOR_NGRAM_LOOKUPS;
+
for (i, term) in terms.into_iter().enumerate() {
let mut adjacent = Vec::new();
- if let super::Term::SimpleOrPhrase(_) = &term {
- for window_size in 2..=3 {
- for (start, end) in sliding_window(window_size, i) {
- let mut compounds = Vec::new();
-
- for k in start..=end {
- if let Some(super::Term::SimpleOrPhrase(super::SimpleOrPhrase::Simple(s))) =
- terms_for_adjacent.get(k)
- {
- compounds.push(s.clone());
+ if augment_with_adjacent {
+ if let super::Term::SimpleOrPhrase(_) = &term {
+ for window_size in 2..=3 {
+ for (start, end) in sliding_window(window_size, i) {
+ let mut compounds = Vec::new();
+
+ for k in start..=end {
+ if let Some(super::Term::SimpleOrPhrase(
+ super::SimpleOrPhrase::Simple(s),
+ )) = terms_for_adjacent.get(k)
+ {
+ compounds.push(s.clone());
+ }
}
- }
- if !compounds.is_empty() {
- adjacent.push(super::TermCompound { terms: compounds });
+ if !compounds.is_empty() {
+ adjacent.push(super::TermCompound { terms: compounds });
+ }
}
}
}
diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rs
index 84546d60..b42fba17 100644
--- a/crates/core/src/ranking/signal/computer/mod.rs
+++ b/crates/core/src/ranking/signal/computer/mod.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::query::optic::AsSearchableRule;
-use crate::query::Query;
+use crate::query::{Query, MAX_TERMS_FOR_NGRAM_LOOKUPS};
use crate::schema::text_field::TextField;
use crate::Result;
use crate::{enum_map::EnumMap, fastfield_reader, schema::TextFieldEnum, webpage::Webpage};
@@ -189,6 +189,12 @@ impl SignalComputer {
.as_textfield()
.and_then(|f| (f.tantivy_field(schema).map(|tv_field| (f, tv_field))))
{
+ if text_field.ngram_size() > 1
+ && query.simple_terms.len() > MAX_TERMS_FOR_NGRAM_LOOKUPS
+ {
+ continue;
+ }
+
let simple_query = itertools::intersperse(
query.simple_terms.iter().map(|s| s.as_str()),
" ",