Skip to content

web search done right

make sure distributive optimisation works with multiple common 'or' terms

Mikkel Denker3a8da50a4a77
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index 3d90bd79..860b11fa 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -813,7 +813,7 @@ mod tests {
let b = plan::initial(b).unwrap();
let b = b.into_query();
- assert_eq!(a, b);
+ assert_eq!(a.len(), b.len());
}
#[test]
diff --git a/crates/core/src/query/pattern_query/mod.rs b/crates/core/src/query/pattern_query/mod.rs
index 9282fd14..cb7866d9 100644
--- a/crates/core/src/query/pattern_query/mod.rs
+++ b/crates/core/src/query/pattern_query/mod.rs
@@ -96,7 +96,7 @@ impl PatternQuery {
for pattern in &patterns {
match pattern {
PatternPart::Raw(text) => {
- let mut tokenizer = field.as_text().unwrap().query_tokenizer();
+ let mut tokenizer = field.as_text().unwrap().tokenizer();
let mut stream = tokenizer.token_stream(text);
while let Some(token) = stream.next() {
diff --git a/crates/core/src/query/plan/mod.rs b/crates/core/src/query/plan/mod.rs
index f1359653..1d13dc7c 100644
--- a/crates/core/src/query/plan/mod.rs
+++ b/crates/core/src/query/plan/mod.rs
@@ -72,6 +72,14 @@ pub enum Query {
}
impl Query {
+ #[cfg(test)]
+ pub fn len(&self) -> usize {
+ match self {
+ Query::Term(_) => 1,
+ Query::Boolean { clauses } => clauses.iter().map(|(_, q)| q.len()).sum(),
+ }
+ }
+
fn compact(self) -> Query {
match self {
Query::Boolean { clauses } => {
diff --git a/crates/core/src/query/plan/node.rs b/crates/core/src/query/plan/node.rs
index 567a5646..06f6f8fe 100644
--- a/crates/core/src/query/plan/node.rs
+++ b/crates/core/src/query/plan/node.rs
@@ -14,13 +14,17 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/
+use ahash::HashSetExt;
+
use crate::query::parser::{SimpleOrPhrase, Term as ParserTerm};
use crate::schema::text_field::{self, TextField as _};
use crate::{query::parser::SimpleTerm, schema::TextFieldEnum};
+type HashSet<T> = std::collections::HashSet<T, ahash::RandomState>;
+
use super::{Occur, Term};
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone)]
pub enum Node {
Term(Term),
And(Box<Node>, Box<Node>),
@@ -28,6 +32,37 @@ pub enum Node {
Not(Box<Node>),
}
+impl PartialEq for Node {
+ fn eq(&self, other: &Self) -> bool {
+ match (self, other) {
+ (Node::Term(a), Node::Term(b)) => a == b,
+ (Node::And(a, b), Node::And(c, d)) => (a == c && b == d) || (a == d && b == c),
+ (Node::Or(a, b), Node::Or(c, d)) => (a == c && b == d) || (a == d && b == c),
+ (Node::Not(a), Node::Not(b)) => a == b,
+ _ => false,
+ }
+ }
+}
+
+impl Eq for Node {}
+
+impl std::hash::Hash for Node {
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+ match self {
+ Node::Term(term) => term.hash(state),
+ Node::And(left, right) => {
+ left.hash(state);
+ right.hash(state);
+ }
+ Node::Or(left, right) => {
+ left.hash(state);
+ right.hash(state);
+ }
+ Node::Not(inner) => inner.hash(state),
+ }
+ }
+}
+
impl Node {
pub fn and<T: Into<Node>>(self, other: T) -> Node {
Node::And(Box::new(self), Box::new(other.into()))
@@ -130,8 +165,50 @@ impl Node {
}
}
+ fn or_children(self) -> HashSet<Node> {
+ match self {
+ Node::Or(left, right) => {
+ let mut left = left.or_children();
+ let right = right.or_children();
+
+ left.extend(right);
+ left
+ }
+ _ => {
+ let mut set = HashSet::new();
+ set.insert(self);
+ set
+ }
+ }
+ }
+
+ fn prune(self, node: &Self) -> Option<Self> {
+ if &self == node {
+ None
+ } else {
+ match self {
+ Node::Term(t) => Some(Node::Term(t)),
+ Node::And(a, b) => match (a.prune(node), b.prune(node)) {
+ (Some(a), Some(b)) => Some(Node::And(Box::new(a), Box::new(b))),
+ (Some(a), None) => Some(a),
+ (None, Some(b)) => Some(b),
+ (None, None) => None,
+ },
+ Node::Or(a, b) => match (a.prune(node), b.prune(node)) {
+ (Some(a), Some(b)) => Some(Node::Or(Box::new(a), Box::new(b))),
+ (Some(a), None) => Some(a),
+ (None, Some(b)) => Some(b),
+ (None, None) => None,
+ },
+ Node::Not(t) => Some(Node::Not(Box::new(t.prune(node)?))),
+ }
+ }
+ }
+
pub fn optimise(self) -> Self {
- DistributiveLaw.optimise(self)
+ let mut node = self;
+ node = Deduplicate.optimise(node);
+ DistributiveLaw.optimise(node)
}
}
@@ -158,16 +235,42 @@ impl Optimisation for DistributiveLaw {
match (left, right) {
(Node::Or(left_left, left_right), Node::Or(right_left, right_right)) => {
- if left_left == right_left {
- left_left.and(left_right.or(*right_right))
- } else if left_left == right_right {
- left_left.and(left_right.or(*right_left))
- } else if left_right == right_left {
- left_right.and(left_left.or(*right_right))
- } else if left_right == right_right {
- left_right.and(left_left.or(*right_left))
+ let left_children =
+ Node::Or(left_left.clone(), left_right.clone()).or_children();
+ let right_children =
+ Node::Or(right_left.clone(), right_right.clone()).or_children();
+
+ let common: Vec<_> = left_children
+ .intersection(&right_children)
+ .cloned()
+ .collect();
+
+ if common.is_empty() {
+ Node::And(
+ Box::new(Node::Or(left_left, left_right)),
+ Box::new(Node::Or(right_left, right_right)),
+ )
} else {
- (left_left.or(*left_right)).and(right_left.or(*right_right))
+ let backup = Node::Or(left_left.clone(), left_right.clone());
+
+ let mut res = Node::And(
+ Box::new(Node::Or(left_left, left_right)),
+ Box::new(Node::Or(right_left, right_right)),
+ );
+
+ for c in &common {
+ match res.prune(c) {
+ Some(new) => res = new,
+ None => return backup,
+ }
+ }
+
+ let common = common
+ .into_iter()
+ .reduce(|left, right| left.or(right))
+ .unwrap();
+
+ res.or(common)
}
}
@@ -178,12 +281,43 @@ impl Optimisation for DistributiveLaw {
}
}
+struct Deduplicate;
+
+impl Optimisation for Deduplicate {
+ fn optimise(&self, node: Node) -> Node {
+ match node {
+ Node::Term(term) => Node::Term(term),
+ Node::Not(inner) => Node::Not(Box::new(self.optimise(*inner))),
+ Node::Or(left, right) => {
+ let left = self.optimise(*left);
+ let right = self.optimise(*right);
+
+ if left == right {
+ left
+ } else {
+ Node::Or(Box::new(left), Box::new(right))
+ }
+ }
+ Node::And(left, right) => {
+ let left = self.optimise(*left);
+ let right = self.optimise(*right);
+
+ if left == right {
+ left
+ } else {
+ Node::And(Box::new(left), Box::new(right))
+ }
+ }
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
#[test]
- fn test_distributive_law() {
+ fn test_optimisation() {
let a = Node::Term(Term {
text: SimpleOrPhrase::Simple(SimpleTerm::from("a".to_string())),
field: text_field::Title.into(),
@@ -204,19 +338,45 @@ mod tests {
field: text_field::Title.into(),
});
+ let e = Node::Term(Term {
+ text: SimpleOrPhrase::Simple(SimpleTerm::from("e".to_string())),
+ field: text_field::Title.into(),
+ });
+
+ let f = Node::Term(Term {
+ text: SimpleOrPhrase::Simple(SimpleTerm::from("f".to_string())),
+ field: text_field::Title.into(),
+ });
+
let query = a.clone().or(b.clone()).and(a.clone().or(c.clone()));
- let optimised = DistributiveLaw.optimise(query);
+ let optimised = query.optimise();
- assert_eq!(optimised, a.clone().and(b.clone().or(c.clone())));
+ assert_eq!(optimised, a.clone().or(b.clone().and(c.clone())));
let query = a.clone().or(b.clone()).and(c.clone().or(d.clone()));
- let optimised = DistributiveLaw.optimise(query);
+ let optimised = query.optimise();
assert_eq!(
optimised,
a.clone().or(b.clone()).and(c.clone().or(d.clone()))
);
+
+ let query = (a.clone().or(b.clone()).or(c.clone()).or(d.clone()))
+ .and(e.clone().or(f.clone()).or(c.clone()).or(d.clone()));
+ let optimised = query.optimise();
+
+ assert_eq!(
+ optimised,
+ ((a.clone().or(b.clone())).and(e.clone().or(f.clone()))).or(c.clone().or(d.clone()))
+ );
+
+ let query = (a.clone().or(a.clone()).or(a.clone()).or(a.clone()))
+ .and(a.clone().or(a.clone()).or(a.clone()).or(a.clone()));
+
+ let optimised = query.optimise();
+
+ assert_eq!(optimised, a.clone());
}
}
diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rs
index 19683a81..7e4148af 100644
--- a/crates/core/src/ranking/signal/computer/mod.rs
+++ b/crates/core/src/ranking/signal/computer/mod.rs
@@ -213,7 +213,7 @@ impl SignalComputer {
.collect::<String>();
let mut terms = Vec::new();
- let mut tokenizer = text_field.query_tokenizer();
+ let mut tokenizer = text_field.tokenizer();
let mut stream = tokenizer.token_stream(&simple_query);
while let Some(token) = stream.next() {
diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rs
index 0407ef2d..132487a1 100644
--- a/crates/core/src/schema/text_field.rs
+++ b/crates/core/src/schema/text_field.rs
@@ -60,12 +60,12 @@ pub trait TextField:
Ok(())
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::default()
}
fn query_tokenizer(&self) -> Tokenizer {
- self.indexing_tokenizer()
+ self.tokenizer()
}
fn ngram_size(&self) -> usize {
@@ -118,7 +118,7 @@ pub trait TextField:
}
fn indexing_option(&self) -> IndexingOption {
- let tokenizer = self.indexing_tokenizer();
+ let tokenizer = self.tokenizer();
let option = self.record_option();
let mut opt = TextOptions::default().set_indexing_options(
@@ -362,7 +362,7 @@ impl TextField for StemmedTitle {
"stemmed_title"
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::new_stemmed()
}
@@ -404,7 +404,7 @@ impl TextField for StemmedCleanBody {
"stemmed_body"
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::new_stemmed()
}
@@ -517,7 +517,7 @@ impl TextField for UrlNoTokenizer {
"url_no_tokenizer"
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -563,7 +563,7 @@ impl TextField for UrlForSiteOperator {
true
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::SiteOperator(SiteOperatorUrlTokenizer)
}
@@ -640,7 +640,7 @@ impl TextField for SiteNoTokenizer {
"site_no_tokenizer"
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -682,7 +682,7 @@ impl TextField for DomainNoTokenizer {
"domain_no_tokenizer"
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -724,7 +724,7 @@ impl TextField for DomainNameNoTokenizer {
"domain_name_no_tokenizer"
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -766,7 +766,7 @@ impl TextField for SiteIfHomepageNoTokenizer {
"site_if_homepage_no_tokenizer"
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -833,7 +833,7 @@ impl TextField for DomainNameIfHomepageNoTokenizer {
"domain_name_if_homepage_no_tokenizer"
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -875,7 +875,7 @@ impl TextField for DomainIfHomepageNoTokenizer {
"domain_if_homepage_no_tokenizer"
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -1052,7 +1052,7 @@ impl TextField for SchemaOrgJson {
"schema_org_json"
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -1084,7 +1084,7 @@ impl TextField for FlattenedSchemaOrgJson {
true
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Json(JsonField)
}
@@ -1119,7 +1119,7 @@ impl TextField for CleanBodyBigrams {
CleanBody.into()
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Bigram(BigramTokenizer::default())
}
@@ -1162,7 +1162,7 @@ impl TextField for TitleBigrams {
Title.into()
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Bigram(BigramTokenizer::default())
}
@@ -1208,7 +1208,7 @@ impl TextField for CleanBodyTrigrams {
CleanBody.into()
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Trigram(TrigramTokenizer::default())
}
@@ -1251,7 +1251,7 @@ impl TextField for TitleTrigrams {
Title.into()
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Trigram(TrigramTokenizer::default())
}
@@ -1316,7 +1316,7 @@ impl TextField for SafetyClassification {
"safety_classification"
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -1354,7 +1354,7 @@ impl TextField for InsertionTimestamp {
"insertion_timestamp"
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -1396,7 +1396,7 @@ impl TextField for RecipeFirstIngredientTagId {
"recipe_first_ingredient_tag_id"
}
- fn indexing_tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rs
index b38680c0..6bd3dd24 100644
--- a/crates/core/src/webpage/html/into_tantivy.rs
+++ b/crates/core/src/webpage/html/into_tantivy.rs
@@ -108,7 +108,7 @@ impl Html {
}
fn pretokenize_string(&self, text: String, field: TextFieldEnum) -> PreTokenizedString {
- self.pretokenize_string_with(text, field.indexing_tokenizer())
+ self.pretokenize_string_with(text, field.tokenizer())
}
fn pretokenize_string_with(