web search done right
make sure distributive optimisation works with multiple common 'or' terms
Mikkel Denker3a8da50a4a77
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex 3d90bd79..860b11fa 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -813,7 +813,7 @@ mod tests {let b = plan::initial(b).unwrap();let b = b.into_query();- assert_eq!(a, b);+ assert_eq!(a.len(), b.len());}#[test]diff --git a/crates/core/src/query/pattern_query/mod.rs b/crates/core/src/query/pattern_query/mod.rsindex 9282fd14..cb7866d9 100644--- a/crates/core/src/query/pattern_query/mod.rs+++ b/crates/core/src/query/pattern_query/mod.rs@@ -96,7 +96,7 @@ impl PatternQuery {for pattern in &patterns {match pattern {PatternPart::Raw(text) => {- let mut tokenizer = field.as_text().unwrap().query_tokenizer();+ let mut tokenizer = field.as_text().unwrap().tokenizer();let mut stream = tokenizer.token_stream(text);while let Some(token) = stream.next() {diff --git a/crates/core/src/query/plan/mod.rs b/crates/core/src/query/plan/mod.rsindex f1359653..1d13dc7c 100644--- a/crates/core/src/query/plan/mod.rs+++ b/crates/core/src/query/plan/mod.rs@@ -72,6 +72,14 @@ pub enum Query {}impl Query {+ #[cfg(test)]+ pub fn len(&self) -> usize {+ match self {+ Query::Term(_) => 1,+ Query::Boolean { clauses } => clauses.iter().map(|(_, q)| q.len()).sum(),+ }+ }+fn compact(self) -> Query {match self {Query::Boolean { clauses } => {diff --git a/crates/core/src/query/plan/node.rs b/crates/core/src/query/plan/node.rsindex 567a5646..06f6f8fe 100644--- a/crates/core/src/query/plan/node.rs+++ b/crates/core/src/query/plan/node.rs@@ -14,13 +14,17 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/+use ahash::HashSetExt;+use crate::query::parser::{SimpleOrPhrase, Term as ParserTerm};use crate::schema::text_field::{self, TextField as _};use crate::{query::parser::SimpleTerm, schema::TextFieldEnum};+type HashSet<T> = std::collections::HashSet<T, ahash::RandomState>;+use super::{Occur, Term};-#[derive(Debug, Clone, PartialEq, Eq, Hash)]+#[derive(Debug, Clone)]pub enum Node {Term(Term),And(Box<Node>, Box<Node>),@@ -28,6 +32,37 @@ pub enum Node {Not(Box<Node>),}+impl PartialEq for Node {+ fn eq(&self, other: &Self) -> bool {+ match (self, other) {+ (Node::Term(a), Node::Term(b)) => a == b,+ (Node::And(a, b), Node::And(c, d)) => (a == c && b == d) || (a == d && b == c),+ (Node::Or(a, b), Node::Or(c, d)) => (a == c && b == d) || (a == d && b == c),+ (Node::Not(a), Node::Not(b)) => a == b,+ _ => false,+ }+ }+}++impl Eq for Node {}++impl std::hash::Hash for Node {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ match self {+ Node::Term(term) => term.hash(state),+ Node::And(left, right) => {+ left.hash(state);+ right.hash(state);+ }+ Node::Or(left, right) => {+ left.hash(state);+ right.hash(state);+ }+ Node::Not(inner) => inner.hash(state),+ }+ }+}+impl Node {pub fn and<T: Into<Node>>(self, other: T) -> Node {Node::And(Box::new(self), Box::new(other.into()))@@ -130,8 +165,50 @@ impl Node {}}+ fn or_children(self) -> HashSet<Node> {+ match self {+ Node::Or(left, right) => {+ let mut left = left.or_children();+ let right = right.or_children();++ left.extend(right);+ left+ }+ _ => {+ let mut set = HashSet::new();+ set.insert(self);+ set+ }+ }+ }++ fn prune(self, node: &Self) -> Option<Self> {+ if &self == node {+ None+ } else {+ match self {+ Node::Term(t) => Some(Node::Term(t)),+ Node::And(a, b) => match (a.prune(node), b.prune(node)) {+ (Some(a), Some(b)) => Some(Node::And(Box::new(a), Box::new(b))),+ (Some(a), None) => Some(a),+ (None, Some(b)) => Some(b),+ (None, None) => None,+ },+ Node::Or(a, b) => match (a.prune(node), b.prune(node)) {+ (Some(a), Some(b)) => Some(Node::Or(Box::new(a), Box::new(b))),+ (Some(a), None) => Some(a),+ (None, Some(b)) => Some(b),+ (None, None) => None,+ },+ Node::Not(t) => Some(Node::Not(Box::new(t.prune(node)?))),+ }+ }+ }+pub fn optimise(self) -> Self {- DistributiveLaw.optimise(self)+ let mut node = self;+ node = Deduplicate.optimise(node);+ DistributiveLaw.optimise(node)}}@@ -158,16 +235,42 @@ impl Optimisation for DistributiveLaw {match (left, right) {(Node::Or(left_left, left_right), Node::Or(right_left, right_right)) => {- if left_left == right_left {- left_left.and(left_right.or(*right_right))- } else if left_left == right_right {- left_left.and(left_right.or(*right_left))- } else if left_right == right_left {- left_right.and(left_left.or(*right_right))- } else if left_right == right_right {- left_right.and(left_left.or(*right_left))+ let left_children =+ Node::Or(left_left.clone(), left_right.clone()).or_children();+ let right_children =+ Node::Or(right_left.clone(), right_right.clone()).or_children();++ let common: Vec<_> = left_children+ .intersection(&right_children)+ .cloned()+ .collect();++ if common.is_empty() {+ Node::And(+ Box::new(Node::Or(left_left, left_right)),+ Box::new(Node::Or(right_left, right_right)),+ )} else {- (left_left.or(*left_right)).and(right_left.or(*right_right))+ let backup = Node::Or(left_left.clone(), left_right.clone());++ let mut res = Node::And(+ Box::new(Node::Or(left_left, left_right)),+ Box::new(Node::Or(right_left, right_right)),+ );++ for c in &common {+ match res.prune(c) {+ Some(new) => res = new,+ None => return backup,+ }+ }++ let common = common+ .into_iter()+ .reduce(|left, right| left.or(right))+ .unwrap();++ res.or(common)}}@@ -178,12 +281,43 @@ impl Optimisation for DistributiveLaw {}}+struct Deduplicate;++impl Optimisation for Deduplicate {+ fn optimise(&self, node: Node) -> Node {+ match node {+ Node::Term(term) => Node::Term(term),+ Node::Not(inner) => Node::Not(Box::new(self.optimise(*inner))),+ Node::Or(left, right) => {+ let left = self.optimise(*left);+ let right = self.optimise(*right);++ if left == right {+ left+ } else {+ Node::Or(Box::new(left), Box::new(right))+ }+ }+ Node::And(left, right) => {+ let left = self.optimise(*left);+ let right = self.optimise(*right);++ if left == right {+ left+ } else {+ Node::And(Box::new(left), Box::new(right))+ }+ }+ }+ }+}+#[cfg(test)]mod tests {use super::*;#[test]- fn test_distributive_law() {+ fn test_optimisation() {let a = Node::Term(Term {text: SimpleOrPhrase::Simple(SimpleTerm::from("a".to_string())),field: text_field::Title.into(),@@ -204,19 +338,45 @@ mod tests {field: text_field::Title.into(),});+ let e = Node::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from("e".to_string())),+ field: text_field::Title.into(),+ });++ let f = Node::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from("f".to_string())),+ field: text_field::Title.into(),+ });+let query = a.clone().or(b.clone()).and(a.clone().or(c.clone()));- let optimised = DistributiveLaw.optimise(query);+ let optimised = query.optimise();- assert_eq!(optimised, a.clone().and(b.clone().or(c.clone())));+ assert_eq!(optimised, a.clone().or(b.clone().and(c.clone())));let query = a.clone().or(b.clone()).and(c.clone().or(d.clone()));- let optimised = DistributiveLaw.optimise(query);+ let optimised = query.optimise();assert_eq!(optimised,a.clone().or(b.clone()).and(c.clone().or(d.clone())));++ let query = (a.clone().or(b.clone()).or(c.clone()).or(d.clone()))+ .and(e.clone().or(f.clone()).or(c.clone()).or(d.clone()));+ let optimised = query.optimise();++ assert_eq!(+ optimised,+ ((a.clone().or(b.clone())).and(e.clone().or(f.clone()))).or(c.clone().or(d.clone()))+ );++ let query = (a.clone().or(a.clone()).or(a.clone()).or(a.clone()))+ .and(a.clone().or(a.clone()).or(a.clone()).or(a.clone()));++ let optimised = query.optimise();++ assert_eq!(optimised, a.clone());}}diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rsindex 19683a81..7e4148af 100644--- a/crates/core/src/ranking/signal/computer/mod.rs+++ b/crates/core/src/ranking/signal/computer/mod.rs@@ -213,7 +213,7 @@ impl SignalComputer {.collect::<String>();let mut terms = Vec::new();- let mut tokenizer = text_field.query_tokenizer();+ let mut tokenizer = text_field.tokenizer();let mut stream = tokenizer.token_stream(&simple_query);while let Some(token) = stream.next() {diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rsindex 0407ef2d..132487a1 100644--- a/crates/core/src/schema/text_field.rs+++ b/crates/core/src/schema/text_field.rs@@ -60,12 +60,12 @@ pub trait TextField:Ok(())}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::default()}fn query_tokenizer(&self) -> Tokenizer {- self.indexing_tokenizer()+ self.tokenizer()}fn ngram_size(&self) -> usize {@@ -118,7 +118,7 @@ pub trait TextField:}fn indexing_option(&self) -> IndexingOption {- let tokenizer = self.indexing_tokenizer();+ let tokenizer = self.tokenizer();let option = self.record_option();let mut opt = TextOptions::default().set_indexing_options(@@ -362,7 +362,7 @@ impl TextField for StemmedTitle {"stemmed_title"}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::new_stemmed()}@@ -404,7 +404,7 @@ impl TextField for StemmedCleanBody {"stemmed_body"}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::new_stemmed()}@@ -517,7 +517,7 @@ impl TextField for UrlNoTokenizer {"url_no_tokenizer"}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -563,7 +563,7 @@ impl TextField for UrlForSiteOperator {true}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::SiteOperator(SiteOperatorUrlTokenizer)}@@ -640,7 +640,7 @@ impl TextField for SiteNoTokenizer {"site_no_tokenizer"}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -682,7 +682,7 @@ impl TextField for DomainNoTokenizer {"domain_no_tokenizer"}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -724,7 +724,7 @@ impl TextField for DomainNameNoTokenizer {"domain_name_no_tokenizer"}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -766,7 +766,7 @@ impl TextField for SiteIfHomepageNoTokenizer {"site_if_homepage_no_tokenizer"}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -833,7 +833,7 @@ impl TextField for DomainNameIfHomepageNoTokenizer {"domain_name_if_homepage_no_tokenizer"}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -875,7 +875,7 @@ impl TextField for DomainIfHomepageNoTokenizer {"domain_if_homepage_no_tokenizer"}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -1052,7 +1052,7 @@ impl TextField for SchemaOrgJson {"schema_org_json"}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -1084,7 +1084,7 @@ impl TextField for FlattenedSchemaOrgJson {true}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Json(JsonField)}@@ -1119,7 +1119,7 @@ impl TextField for CleanBodyBigrams {CleanBody.into()}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Bigram(BigramTokenizer::default())}@@ -1162,7 +1162,7 @@ impl TextField for TitleBigrams {Title.into()}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Bigram(BigramTokenizer::default())}@@ -1208,7 +1208,7 @@ impl TextField for CleanBodyTrigrams {CleanBody.into()}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Trigram(TrigramTokenizer::default())}@@ -1251,7 +1251,7 @@ impl TextField for TitleTrigrams {Title.into()}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Trigram(TrigramTokenizer::default())}@@ -1316,7 +1316,7 @@ impl TextField for SafetyClassification {"safety_classification"}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -1354,7 +1354,7 @@ impl TextField for InsertionTimestamp {"insertion_timestamp"}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -1396,7 +1396,7 @@ impl TextField for RecipeFirstIngredientTagId {"recipe_first_ingredient_tag_id"}- fn indexing_tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self) -> Tokenizer {Tokenizer::Identity(Identity {})}diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rsindex b38680c0..6bd3dd24 100644--- a/crates/core/src/webpage/html/into_tantivy.rs+++ b/crates/core/src/webpage/html/into_tantivy.rs@@ -108,7 +108,7 @@ impl Html {}fn pretokenize_string(&self, text: String, field: TextFieldEnum) -> PreTokenizedString {- self.pretokenize_string_with(text, field.indexing_tokenizer())+ self.pretokenize_string_with(text, field.tokenizer())}fn pretokenize_string_with(