web search done right
use distributive law to re-write '(A | B) & (A | C)' queries to 'A | (B & C)' for speedup
Mikkel Denker8d315dbc95eb
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex 992a60df..3d90bd79 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -1012,6 +1012,7 @@ mod tests {}proptest! {+ #![proptest_config(ProptestConfig::with_cases(4096))]#[test]fn test_query_parse_non_panic(query in ".*") {fixture(&query)?;diff --git a/crates/core/src/query/pattern_query/mod.rs b/crates/core/src/query/pattern_query/mod.rsindex 6d9d1f7a..9282fd14 100644--- a/crates/core/src/query/pattern_query/mod.rs+++ b/crates/core/src/query/pattern_query/mod.rs@@ -96,7 +96,7 @@ impl PatternQuery {for pattern in &patterns {match pattern {PatternPart::Raw(text) => {- let mut tokenizer = field.as_text().unwrap().indexing_tokenizer();+ let mut tokenizer = field.as_text().unwrap().query_tokenizer();let mut stream = tokenizer.token_stream(text);while let Some(token) = stream.next() {diff --git a/crates/core/src/query/plan.rs b/crates/core/src/query/plan/mod.rssimilarity index 77%rename from crates/core/src/query/plan.rsrename to crates/core/src/query/plan/mod.rsindex 1c9407cf..f1359653 100644--- a/crates/core/src/query/plan.rs+++ b/crates/core/src/query/plan/mod.rs@@ -16,12 +16,11 @@use itertools::Itertools;use tantivy::tokenizer::Tokenizer as _;+mod node;-use crate::schema::{- self,- text_field::{self, TextField},- TextFieldEnum,-};+pub use node::Node;++use crate::schema::{self, text_field::TextField, TextFieldEnum};use super::parser::{SimpleOrPhrase, SimpleTerm};@@ -37,24 +36,6 @@ impl Term {}}-#[derive(Debug, Clone, PartialEq, Eq, Hash)]-pub enum Node {- Term(Term),- And(Box<Node>, Box<Node>),- Or(Box<Node>, Box<Node>),- Not(Box<Node>),-}--impl Node {- pub fn and<T: Into<Node>>(self, other: T) -> Node {- Node::And(Box::new(self), Box::new(other.into()))- }-- pub fn or<T: Into<Node>>(self, other: T) -> Node {- Node::Or(Box::new(self), Box::new(other.into()))- }-}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub enum Occur {Must,@@ -223,96 +204,6 @@ fn process_tantivy_term<T: TextField>(terms}-impl Node {- fn into_non_compacted_query(self) -> Query {- match self {- Node::Term(term) => Query::Term(term),- Node::And(left, right) => Query::Boolean {- clauses: vec![- (Occur::Must, left.into_non_compacted_query()),- (Occur::Must, right.into_non_compacted_query()),- ],- },- Node::Or(left, right) => Query::Boolean {- clauses: vec![- (Occur::Should, left.into_non_compacted_query()),- (Occur::Should, right.into_non_compacted_query()),- ],- },- Node::Not(inner) => Query::Boolean {- clauses: vec![(Occur::MustNot, inner.into_non_compacted_query())],- },- }- }-- pub fn into_query(self) -> Query {- self.into_non_compacted_query().compact().deduplicate()- }-- fn from_term(term: super::Term) -> Self {- match term {- super::Term::SimpleOrPhrase(s) => match s {- super::SimpleOrPhrase::Simple(term) => TextFieldEnum::all()- .filter(|f| f.is_searchable())- .map(|field| {- Node::Term(Term {- text: SimpleOrPhrase::Simple(term.clone()),- field,- })- })- .reduce(|left, right| left.or(right))- .expect("fields should not be empty"),- super::SimpleOrPhrase::Phrase(p) => TextFieldEnum::all()- .filter(|f| f.is_searchable())- .filter(|f| f.is_phrase_searchable())- .map(|field| {- Node::Term(Term {- text: SimpleOrPhrase::Phrase(p.clone()),- field,- })- })- .reduce(|left, right| left.or(right))- .expect("fields should not be empty"),- },- super::Term::Site(s) => Node::Term(Term {- text: SimpleOrPhrase::Simple(super::parser::SimpleTerm::from(s)),- field: text_field::UrlForSiteOperator.into(),- }),- super::Term::Title(t) => Node::Term(Term {- text: t,- field: text_field::Title.into(),- }),- super::Term::Body(b) => Node::Term(Term {- text: b,- field: text_field::AllBody.into(),- }),- super::Term::Url(u) => Node::Term(Term {- text: u,- field: text_field::Url.into(),- }),- super::Term::PossibleBang { prefix, bang } => {- let mut s = String::new();- s.push(prefix);- s.push_str(bang.as_str());-- let s = SimpleTerm::from(s);-- TextFieldEnum::all()- .filter(|f| f.is_searchable())- .map(|field| {- Node::Term(Term {- text: SimpleOrPhrase::Simple(s.clone()),- field,- })- })- .reduce(|left, right| left.or(right))- .expect("fields should not be empty")- }- super::Term::Not(n) => Node::Not(Box::new(Node::from_term(*n))),- }- }-}-fn sliding_window(window_size: usize, i: usize) -> impl Iterator<Item = (usize, usize)> {(0..=window_size).map(move |offset| {@@ -389,6 +280,8 @@ pub fn initial(terms: Vec<super::Term>) -> Option<Node> {#[cfg(test)]mod tests {+ use crate::schema::text_field;+use super::*;fn parse(query: &str, fields: &[TextFieldEnum]) -> Node {diff --git a/crates/core/src/query/plan/node.rs b/crates/core/src/query/plan/node.rsnew file mode 100644index 00000000..567a5646--- /dev/null+++ b/crates/core/src/query/plan/node.rs@@ -0,0 +1,222 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/++use crate::query::parser::{SimpleOrPhrase, Term as ParserTerm};+use crate::schema::text_field::{self, TextField as _};+use crate::{query::parser::SimpleTerm, schema::TextFieldEnum};++use super::{Occur, Term};++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub enum Node {+ Term(Term),+ And(Box<Node>, Box<Node>),+ Or(Box<Node>, Box<Node>),+ Not(Box<Node>),+}++impl Node {+ pub fn and<T: Into<Node>>(self, other: T) -> Node {+ Node::And(Box::new(self), Box::new(other.into()))+ }++ pub fn or<T: Into<Node>>(self, other: T) -> Node {+ Node::Or(Box::new(self), Box::new(other.into()))+ }+}++impl Node {+ fn into_non_compacted_query(self) -> super::Query {+ match self {+ Node::Term(term) => super::Query::Term(term),+ Node::And(left, right) => super::Query::Boolean {+ clauses: vec![+ (Occur::Must, left.into_non_compacted_query()),+ (Occur::Must, right.into_non_compacted_query()),+ ],+ },+ Node::Or(left, right) => super::Query::Boolean {+ clauses: vec![+ (Occur::Should, left.into_non_compacted_query()),+ (Occur::Should, right.into_non_compacted_query()),+ ],+ },+ Node::Not(inner) => super::Query::Boolean {+ clauses: vec![(Occur::MustNot, inner.into_non_compacted_query())],+ },+ }+ }++ pub fn into_query(self) -> super::Query {+ self.optimise()+ .into_non_compacted_query()+ .compact()+ .deduplicate()+ }++ pub fn from_term(term: ParserTerm) -> Self {+ match term {+ ParserTerm::SimpleOrPhrase(s) => match s {+ SimpleOrPhrase::Simple(term) => TextFieldEnum::all()+ .filter(|f| f.is_searchable())+ .map(|field| {+ Node::Term(Term {+ text: SimpleOrPhrase::Simple(term.clone()),+ field,+ })+ })+ .reduce(|left, right| left.or(right))+ .expect("fields should not be empty"),+ SimpleOrPhrase::Phrase(p) => TextFieldEnum::all()+ .filter(|f| f.is_searchable())+ .filter(|f| f.is_phrase_searchable())+ .map(|field| {+ Node::Term(Term {+ text: SimpleOrPhrase::Phrase(p.clone()),+ field,+ })+ })+ .reduce(|left, right| left.or(right))+ .expect("fields should not be empty"),+ },+ ParserTerm::Site(s) => Node::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from(s)),+ field: text_field::UrlForSiteOperator.into(),+ }),+ ParserTerm::Title(t) => Node::Term(Term {+ text: t,+ field: text_field::Title.into(),+ }),+ ParserTerm::Body(b) => Node::Term(Term {+ text: b,+ field: text_field::AllBody.into(),+ }),+ ParserTerm::Url(u) => Node::Term(Term {+ text: u,+ field: text_field::Url.into(),+ }),+ ParserTerm::PossibleBang { prefix, bang } => {+ let mut s = String::new();+ s.push(prefix);+ s.push_str(bang.as_str());++ let s = SimpleTerm::from(s);++ TextFieldEnum::all()+ .filter(|f| f.is_searchable())+ .map(|field| {+ Node::Term(Term {+ text: SimpleOrPhrase::Simple(s.clone()),+ field,+ })+ })+ .reduce(|left, right| left.or(right))+ .expect("fields should not be empty")+ }+ ParserTerm::Not(n) => Node::Not(Box::new(Node::from_term(*n))),+ }+ }++ pub fn optimise(self) -> Self {+ DistributiveLaw.optimise(self)+ }+}++trait Optimisation {+ fn optimise(&self, node: Node) -> Node;+}++/// Re-write queries on the form `(A | B) & (A | C)` to `A | (B & C)`.+/// This avoids having to evaluate the same term multiple times.+struct DistributiveLaw;++impl Optimisation for DistributiveLaw {+ fn optimise(&self, node: Node) -> Node {+ match node {+ Node::Term(term) => Node::Term(term),+ Node::Not(inner) => Node::Not(Box::new(self.optimise(*inner))),+ Node::Or(left, right) => Node::Or(+ Box::new(self.optimise(*left)),+ Box::new(self.optimise(*right)),+ ),+ Node::And(left, right) => {+ let left = self.optimise(*left);+ let right = self.optimise(*right);++ match (left, right) {+ (Node::Or(left_left, left_right), Node::Or(right_left, right_right)) => {+ if left_left == right_left {+ left_left.and(left_right.or(*right_right))+ } else if left_left == right_right {+ left_left.and(left_right.or(*right_left))+ } else if left_right == right_left {+ left_right.and(left_left.or(*right_right))+ } else if left_right == right_right {+ left_right.and(left_left.or(*right_left))+ } else {+ (left_left.or(*left_right)).and(right_left.or(*right_right))+ }+ }++ (left, right) => Node::And(Box::new(left), Box::new(right)),+ }+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_distributive_law() {+ let a = Node::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from("a".to_string())),+ field: text_field::Title.into(),+ });++ let b = Node::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from("b".to_string())),+ field: text_field::Title.into(),+ });++ let c = Node::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from("c".to_string())),+ field: text_field::Title.into(),+ });++ let d = Node::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from("d".to_string())),+ field: text_field::Title.into(),+ });++ let query = a.clone().or(b.clone()).and(a.clone().or(c.clone()));++ let optimised = DistributiveLaw.optimise(query);++ assert_eq!(optimised, a.clone().and(b.clone().or(c.clone())));++ let query = a.clone().or(b.clone()).and(c.clone().or(d.clone()));++ let optimised = DistributiveLaw.optimise(query);++ assert_eq!(+ optimised,+ a.clone().or(b.clone()).and(c.clone().or(d.clone()))+ );+ }+}diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rsindex faf73111..19683a81 100644--- a/crates/core/src/ranking/signal/computer/mod.rs+++ b/crates/core/src/ranking/signal/computer/mod.rs@@ -213,7 +213,7 @@ impl SignalComputer {.collect::<String>();let mut terms = Vec::new();- let mut tokenizer = text_field.indexing_tokenizer();+ let mut tokenizer = text_field.query_tokenizer();let mut stream = tokenizer.token_stream(&simple_query);while let Some(token) = stream.next() {