web search done right
introduce query plan as an internal representation. this allows us to potentially optimise the query before it is executed to avoid having to lookup the same term in the same field multiple times. the optimisations should ofcourse ensure that the results stay the same.
Mikkel Denker5f6bd0cb3e1a
diff --git a/crates/core/src/bangs.rs b/crates/core/src/bangs.rsindex 3d77b61c..05d9a76c 100644--- a/crates/core/src/bangs.rs+++ b/crates/core/src/bangs.rs@@ -141,8 +141,8 @@ impl Bangs {pub fn get(&self, terms: &[Term]) -> Option<BangHit> {for possible_bang in terms.iter().filter_map(|term| {- if let Term::PossibleBang(possible_bang) = term {- Some(possible_bang)+ if let Term::PossibleBang { prefix: _, bang } = term {+ Some(bang)} else {None}@@ -152,7 +152,7 @@ impl Bangs {terms.iter().filter(|term| {- if let Term::PossibleBang(bang) = term {+ if let Term::PossibleBang { prefix: _, bang } = term {bang != possible_bang} else {truediff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex 712f2b1e..c2dd5acc 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -18,38 +18,34 @@ use crate::{inverted_index::InvertedIndex,query::parser::TermCompound,ranking::SignalCoefficient,- schema::{text_field, Field},+ schema::text_field,search_ctx::Ctx,searcher::SearchQuery,webpage::{region::Region, safety_classifier},- Result,+ Error, Result,};+use optics::{HostRankings, Optic};-use std::collections::HashMap;-use tantivy::query::{BooleanQuery, Occur, QueryClone, TermQuery};++use tantivy::query::{BooleanQuery, Occur, QueryClone};mod const_query;pub mod intersection;pub mod optic;pub mod parser;mod pattern_query;+mod plan;pub mod shortcircuit;pub mod union;use parser::Term;-use self::{- optic::AsMultipleTantivyQuery,- parser::{CompoundAwareTerm, SimpleOrPhrase},-};--const MAX_SIMILAR_TERMS: usize = 10;+use self::{optic::AsMultipleTantivyQuery, parser::SimpleOrPhrase};-#[derive(Clone, Debug)]+#[derive(Debug)]pub struct Query {- terms: Vec<Term>,simple_terms_text: Vec<String>,- tantivy_query: Box<BooleanQuery>,+ tantivy_query: Box<dyn tantivy::query::Query>,host_rankings: HostRankings,offset: usize,region: Option<Region>,@@ -58,84 +54,40 @@ pub struct Query {count_results: bool,}-impl Query {- pub fn parse(ctx: &Ctx, query: &SearchQuery, index: &InvertedIndex) -> Result<Query> {- let parsed_terms = parser::parse(&query.query)?;- let mut term_count = HashMap::new();- let mut terms = Vec::new();-- for term in parsed_terms {- let count = term_count.entry(term.clone()).or_insert(0);-- if *count < MAX_SIMILAR_TERMS {- terms.push(term);- }-- *count += 1;+impl Clone for Query {+ fn clone(&self) -> Self {+ Self {+ simple_terms_text: self.simple_terms_text.clone(),+ tantivy_query: self.tantivy_query.box_clone(),+ host_rankings: self.host_rankings.clone(),+ offset: self.offset,+ region: self.region,+ optics: self.optics.clone(),+ top_n: self.top_n,+ count_results: self.count_results,}+ }+}- let mut compound_terms: Vec<_> = terms- .clone()- .into_iter()- .map(|term| CompoundAwareTerm {- term,- adjacent_terms: Vec::new(),- })- .collect();+impl Query {+ pub fn parse(ctx: &Ctx, query: &SearchQuery, index: &InvertedIndex) -> Result<Query> {+ let parsed_terms = parser::truncate(parser::parse(&query.query)?);- let term_ids: Vec<_> = compound_terms.iter().enumerate().map(|(i, _)| i).collect();-- for window_size in 2..=3 {- for window in term_ids.windows(window_size) {- let mut window_terms = Vec::new();- for i in window {- if let Term::SimpleOrPhrase(SimpleOrPhrase::Simple(t)) =- &compound_terms[*i].term- {- window_terms.push(t.clone());- }- }-- let compound = TermCompound {- terms: window_terms,- };-- for i in window {- compound_terms[*i].adjacent_terms.push(compound.clone());- }- }+ if parsed_terms.is_empty() {+ tracing::error!("No terms found in query");+ return Err(Error::EmptyQuery.into());}- let schema = index.schema();-- let fields: Vec<tantivy::schema::Field> = schema.fields().map(|(field, _)| field).collect();-- let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = compound_terms+ if parsed_terms.iter()- .map(|term| term.as_tantivy_query(&fields))- .collect();-- if query.safe_search {- let field = Field::Text(text_field::SafetyClassification.into());- let field = schema.get_field(field.name()).unwrap();-- queries.push((- Occur::MustNot,- Box::new(TermQuery::new(- tantivy::Term::from_field_text(- field,- safety_classifier::Label::NSFW.to_string().as_str(),- ),- tantivy::schema::IndexRecordOption::Basic,- )),- ));+ .all(|t| matches!(t, Term::PossibleBang { .. }))+ {+ tracing::error!("No non-bang terms found in query");+ return Err(Error::EmptyQuery.into());}- let mut tantivy_query = Box::new(BooleanQuery::new(queries));-- let simple_terms_text: Vec<String> = terms- .clone()- .into_iter()+ let simple_terms_text: Vec<String> = parsed_terms+ .iter().filter_map(|term| term.as_simple_text().map(|s| s.to_string())).flat_map(|term| {// term might be a phrase, so we split it into words@@ -145,6 +97,24 @@ impl Query {}).collect();+ let mut plan = plan::initial(parsed_terms).expect("terms are not empty and not all bangs");++ let schema = index.schema();++ if query.safe_search {+ plan = plan.and(plan::Node::Not(Box::new(plan::Node::Term(+ plan::Term::new(+ parser::SimpleTerm::from(safety_classifier::Label::NSFW.to_string()).into(),+ text_field::SafetyClassification.into(),+ ),+ ))));+ }++ let mut tantivy_query = plan+ .into_query()+ .as_tantivy(&schema)+ .expect("there should at least be one field in the index");+let mut optics = Vec::new();if let Some(site_rankigns_optic) = query.host_rankings.clone().map(|sr| sr.into_optic()) {optics.push(site_rankigns_optic);@@ -161,7 +131,6 @@ impl Query {}Ok(Query {- terms,host_rankings: optics.iter().fold(HostRankings::default(), |mut acc, el| {acc.merge_into(el.host_rankings.clone());acc@@ -184,18 +153,10 @@ impl Query {&self.simple_terms_text}- pub fn terms(&self) -> &[Term] {- &self.terms- }-pub fn optics(&self) -> &[Optic] {&self.optics}- pub fn is_empty(&self) -> bool {- self.terms.is_empty()- }-pub fn num_results(&self) -> usize {self.top_n}@@ -260,8 +221,7 @@ mod tests {let query = Query::parse(&ctx,&SearchQuery {- query: "this is a simple query the the the the the the the the the the the the the"- .to_string(),+ query: "this is a simple query".to_string(),..Default::default()},&index,@@ -276,16 +236,6 @@ mod tests {"a".to_string(),"simple".to_string(),"query".to_string(),- "the".to_string(),- "the".to_string(),- "the".to_string(),- "the".to_string(),- "the".to_string(),- "the".to_string(),- "the".to_string(),- "the".to_string(),- "the".to_string(),- "the".to_string(),]);}@@ -621,10 +571,13 @@ mod tests {..Default::default()},&index,- )- .expect("failed to parse query");+ );- assert!(query.is_empty())+ assert!(query.is_err());+ assert_eq!(+ query.err().unwrap().to_string(),+ anyhow::Error::from(Error::EmptyQuery).to_string()+ );}#[test]@@ -632,7 +585,7 @@ mod tests {let index = empty_index();let ctx = index.local_search_ctx();- let query = Query::parse(+ let _query = Query::parse(&ctx,&SearchQuery {query: "&".to_string(),@@ -641,8 +594,6 @@ mod tests {&index,).expect("Failed to parse query");-- assert!(!query.is_empty());}#[test]@@ -851,6 +802,19 @@ mod tests {assert_eq!(result.webpages.len(), 2);}+ #[test]+ fn deduplicate_terms() {+ let a = parser::parse("the the the the the").unwrap();+ let a = plan::initial(a).unwrap();+ let a = a.into_query();++ let b = parser::parse("the the the the the the the the the the the the").unwrap();+ let b = plan::initial(b).unwrap();+ let b = b.into_query();++ assert_eq!(a, b);+ }+#[test]fn safe_search() {let mut index = Index::temporary().expect("Unable to open index");diff --git a/crates/core/src/query/parser/as_tantivy.rs b/crates/core/src/query/parser/as_tantivy.rsdeleted file mode 100644index 9de8edc1..00000000--- a/crates/core/src/query/parser/as_tantivy.rs+++ /dev/null@@ -1,304 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2024 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use tantivy::{- query::{BooleanQuery, Occur, PhraseQuery, TermQuery},- tokenizer::Tokenizer as _,-};--use crate::{- bangs::BANG_PREFIXES,- schema::{text_field::TextField, Field, TextFieldEnum},-};--use super::{CompoundAwareTerm, SimpleOrPhrase, SimpleTerm, Term, TermCompound};--impl CompoundAwareTerm {- pub fn as_tantivy_query(- &self,- fields: &[tantivy::schema::Field],- ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {- if !self.adjacent_terms.is_empty() {- if let Term::SimpleOrPhrase(SimpleOrPhrase::Simple(simple_term)) = &self.term {- return simple_into_tantivy(simple_term, &self.adjacent_terms, fields);- }- }-- self.term.as_tantivy_query(fields)- }-}--fn simple_into_tantivy(- term: &SimpleTerm,- adjacent_terms: &[TermCompound],- fields: &[tantivy::schema::Field],-) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {- let mut queries = Term::into_tantivy_simple(term, fields);-- let fields = fields- .iter()- .filter(|field| {- matches!(- Field::get(field.field_id() as usize),- Some(Field::Text(TextFieldEnum::AllBody(_)))- | Some(Field::Text(TextFieldEnum::Title(_)))- | Some(Field::Text(TextFieldEnum::Url(_)))- )- })- .copied()- .collect::<Vec<_>>();-- for adjacent_term in adjacent_terms {- let combined = adjacent_term- .terms- .iter()- .map(|term| term.as_str())- .collect::<String>();-- for field in &fields {- queries.push((Occur::Should, Term::tantivy_text_query(field, &combined)))- }- }-- (Occur::Must, Box::new(BooleanQuery::new(queries)))-}--fn phrase_query(- terms: &[String],- fields: &[tantivy::schema::Field],-) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {- let mut phrases = Vec::with_capacity(fields.len());- let phrase = terms.join(" ");-- for (field, tv_field) in fields- .iter()- .filter_map(|tv_field| {- Field::get(tv_field.field_id() as usize).map(|mapped| (mapped, *tv_field))- })- .filter(|(field, _)| field.is_searchable())- .filter(|(field, _)| field.has_pos())- {- let mut processed_terms = Term::process_tantivy_term(&phrase, tv_field);-- if processed_terms.is_empty() {- continue;- }-- if processed_terms.len() == 1 {- let options = field.as_text().unwrap().record_option();-- phrases.push((- Occur::Should,- Box::new(TermQuery::new(processed_terms.pop().unwrap(), options))- as Box<dyn tantivy::query::Query>,- ));- } else {- phrases.push((- Occur::Should,- Box::new(PhraseQuery::new(processed_terms)) as Box<dyn tantivy::query::Query>,- ));- }- }-- (Occur::Must, Box::new(BooleanQuery::new(phrases)))-}--impl Term {- fn as_tantivy_query(- &self,- fields: &[tantivy::schema::Field],- ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {- match self {- Term::SimpleOrPhrase(SimpleOrPhrase::Simple(term)) => {- simple_into_tantivy(term, &[], fields)- }- Term::SimpleOrPhrase(SimpleOrPhrase::Phrase(phrase)) => phrase_query(phrase, fields),- Term::Not(subterm) => (- Occur::MustNot,- Box::new(BooleanQuery::new(vec![subterm.as_tantivy_query(fields)])),- ),- Term::Site(site) => (- Occur::Must,- Box::new(BooleanQuery::new(Term::into_tantivy_site(site, fields))),- ),- Term::Title(title) => {- let field = fields- .iter()- .find(|field| {- matches!(- Field::get(field.field_id() as usize),- Some(Field::Text(TextFieldEnum::Title(_)))- )- })- .unwrap();-- (- Occur::Must,- Term::tantivy_text_query(field, &title.as_string()),- )- }- Term::Body(body) => {- let field = fields- .iter()- .find(|field| {- matches!(- Field::get(field.field_id() as usize),- Some(Field::Text(TextFieldEnum::AllBody(_)))- )- })- .unwrap();-- (- Occur::Must,- Term::tantivy_text_query(field, &body.as_string()),- )- }- Term::Url(url) => {- let field = fields- .iter()- .find(|field| {- matches!(- Field::get(field.field_id() as usize),- Some(Field::Text(TextFieldEnum::Url(_)))- )- })- .unwrap();-- (- Occur::Must,- Term::tantivy_text_query(field, &url.as_string()),- )- }- Term::PossibleBang(text) => {- let mut term = String::new();-- term.push(BANG_PREFIXES[0]);- term.push_str(text);-- simple_into_tantivy(&term.into(), &[], fields)- }- }- }-- fn into_tantivy_simple(- term: &SimpleTerm,- fields: &[tantivy::schema::Field],- ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {- fields- .iter()- .filter_map(|tv_field| {- Field::get(tv_field.field_id() as usize)- .filter(|field| field.is_searchable())- .map(|_| tv_field)- })- .map(|field| {- (- Occur::Should,- Term::tantivy_text_query(field, term.as_str()),- )- })- .collect()- }-- fn into_tantivy_site(- term: &str,- fields: &[tantivy::schema::Field],- ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {- fields- .iter()- .filter(|field| {- matches!(- Field::get(field.field_id() as usize),- Some(Field::Text(TextFieldEnum::UrlForSiteOperator(_)))- )- })- .map(|field| {- let processed_terms = Term::process_tantivy_term(term, *field);-- if processed_terms.len() == 1 {- let term = processed_terms.first().unwrap().clone();- (- Occur::Should,- Box::new(TermQuery::new(- term,- tantivy::schema::IndexRecordOption::Basic,- )) as Box<dyn tantivy::query::Query>,- )- } else {- (- Occur::Should,- Box::new(PhraseQuery::new(processed_terms))- as Box<dyn tantivy::query::Query>,- )- }- })- .collect()- }-- fn tantivy_text_query(- field: &tantivy::schema::Field,- term: &str,- ) -> Box<dyn tantivy::query::Query + 'static> {- let mut processed_terms = Term::process_tantivy_term(term, *field);-- let option = Field::get(field.field_id() as usize)- .unwrap()- .as_text()- .unwrap()- .record_option();-- let processed_query = if processed_terms.len() == 1 {- let term = processed_terms.remove(0);- Box::new(TermQuery::new(term, option)) as Box<dyn tantivy::query::Query + 'static>- } else {- Box::new(BooleanQuery::new(- processed_terms- .into_iter()- .map(|term| {- (- Occur::Must,- Box::new(TermQuery::new(term, option))- as Box<dyn tantivy::query::Query + 'static>,- )- })- .collect(),- )) as Box<dyn tantivy::query::Query + 'static>- };-- Box::new(processed_query)- }-- fn process_tantivy_term(- term: &str,- tantivy_field: tantivy::schema::Field,- ) -> Vec<tantivy::Term> {- match Field::get(tantivy_field.field_id() as usize) {- Some(Field::Fast(_)) => vec![tantivy::Term::from_field_text(tantivy_field, term)],- Some(Field::Text(text_field)) => {- let mut terms: Vec<tantivy::Term> = Vec::new();- let mut tokenizer = text_field.query_tokenizer();- let mut token_stream = tokenizer.token_stream(term);- token_stream.process(&mut |token| {- let term = tantivy::Term::from_field_text(tantivy_field, &token.text);- terms.push(term);- });-- terms- }- None => vec![],- }- }-}diff --git a/crates/core/src/query/parser/mod.rs b/crates/core/src/query/parser/mod.rsindex 0e696cbc..f5bc56ca 100644--- a/crates/core/src/query/parser/mod.rs+++ b/crates/core/src/query/parser/mod.rs@@ -14,9 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+pub const MAX_TERMS_PER_QUERY: usize = 32;+use crate::bangs::BANG_PREFIXES;-mod as_tantivy;mod term;pub use term::*;@@ -88,7 +89,13 @@ fn simple_or_phrase(input: &str) -> nom::IResult<&str, SimpleOrPhrase> {fn single_bang(input: &str, pref: char) -> nom::IResult<&str, Term> {let (input, _) = nom::character::complete::char(pref)(input)?;let (input, output) = until_space_or_end(input)?;- Ok((input, Term::PossibleBang(output.to_string())))+ Ok((+ input,+ Term::PossibleBang {+ prefix: pref,+ bang: output.to_string(),+ },+ ))}fn bang(input: &str) -> nom::IResult<&str, Term> {@@ -194,6 +201,14 @@ pub fn parse(query: &str) -> anyhow::Result<Vec<Term>> {.map_err(|e| anyhow::anyhow!("Failed to parse query: {:?}", e))}+pub fn truncate(terms: Vec<Term>) -> Vec<Term> {+ terms+ .into_iter()+ .take(MAX_TERMS_PER_QUERY)+ .map(|t| t.truncate())+ .collect()+}+#[cfg(test)]mod tests {use proptest::prelude::*;@@ -201,7 +216,7 @@ mod tests {use super::{SimpleOrPhrase, Term};fn parse(input: &str) -> Vec<Term> {- super::parse(input).unwrap()+ super::truncate(super::parse(input).unwrap())}#[test]@@ -368,6 +383,18 @@ mod tests {assert_eq!(parse(query).len(), 1);}+ #[test]+ fn test_truncate() {+ let q = "test ";++ let terms = parse(q);+ assert_eq!(terms.len(), 1);++ let q = q.repeat(1024);+ let terms = parse(&q);+ assert!(terms.len() < 1020);+ }+proptest! {#[test]fn prop(query: String) {diff --git a/crates/core/src/query/parser/term.rs b/crates/core/src/query/parser/term.rsindex ac1b5a09..4f81fa61 100644--- a/crates/core/src/query/parser/term.rs+++ b/crates/core/src/query/parser/term.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -14,16 +14,17 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>-use crate::bangs::BANG_PREFIXES;+const MAX_PHRASE_LENGTH: usize = 32;+const MAX_TERM_LENGTH_CHARS: usize = 1024;-#[derive(Debug, Clone)]+#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub struct TermCompound {pub terms: Vec<SimpleTerm>,}-#[derive(Debug, Clone)]+#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub struct CompoundAwareTerm {- pub term: Term,+ pub term: SimpleTerm,pub adjacent_terms: Vec<TermCompound>,}@@ -33,6 +34,23 @@ pub enum SimpleOrPhrase {Phrase(Vec<String>),}+impl SimpleOrPhrase {+ fn truncate(self) -> SimpleOrPhrase {+ match self {+ SimpleOrPhrase::Simple(simple) => SimpleOrPhrase::Simple(SimpleTerm(+ simple.0.chars().take(MAX_TERM_LENGTH_CHARS).collect(),+ )),+ SimpleOrPhrase::Phrase(phrase) => SimpleOrPhrase::Phrase(+ phrase+ .into_iter()+ .take(MAX_PHRASE_LENGTH)+ .map(|s| s.chars().take(MAX_TERM_LENGTH_CHARS).collect())+ .collect(),+ ),+ }+ }+}+impl SimpleOrPhrase {pub fn as_string(&self) -> String {match self {@@ -42,6 +60,12 @@ impl SimpleOrPhrase {}}+impl From<SimpleTerm> for SimpleOrPhrase {+ fn from(value: SimpleTerm) -> Self {+ SimpleOrPhrase::Simple(value)+ }+}+#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub struct SimpleTerm(String);impl SimpleTerm {@@ -77,7 +101,7 @@ pub enum Term {Title(SimpleOrPhrase),Body(SimpleOrPhrase),Url(SimpleOrPhrase),- PossibleBang(String),+ PossibleBang { prefix: char, bang: String },Not(Box<Term>),}@@ -90,7 +114,7 @@ impl std::fmt::Display for Term {Term::Title(title) => write!(f, "intitle:{}", title),Term::Body(body) => write!(f, "inbody:{}", body),Term::Url(url) => write!(f, "inurl:{}", url),- Term::PossibleBang(bang) => write!(f, "{}{}", BANG_PREFIXES[0], bang),+ Term::PossibleBang { prefix, bang } => write!(f, "{}{}", prefix, bang),}}}@@ -102,4 +126,19 @@ impl Term {_ => None,}}++ pub fn truncate(self) -> Term {+ match self {+ Term::SimpleOrPhrase(s) => Term::SimpleOrPhrase(s.truncate()),+ Term::Site(s) => Term::Site(s),+ Term::Title(s) => Term::Title(s.truncate()),+ Term::Body(s) => Term::Body(s.truncate()),+ Term::Url(s) => Term::Url(s.truncate()),+ Term::Not(n) => Term::Not(Box::new(n.truncate())),+ Term::PossibleBang { prefix, bang } => Term::PossibleBang {+ prefix,+ bang: bang.chars().take(MAX_TERM_LENGTH_CHARS).collect(),+ },+ }+ }}diff --git a/crates/core/src/query/plan.rs b/crates/core/src/query/plan.rsnew file mode 100644index 00000000..1c9407cf--- /dev/null+++ b/crates/core/src/query/plan.rs@@ -0,0 +1,528 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/++use itertools::Itertools;+use tantivy::tokenizer::Tokenizer as _;++use crate::schema::{+ self,+ text_field::{self, TextField},+ TextFieldEnum,+};++use super::parser::{SimpleOrPhrase, SimpleTerm};++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub struct Term {+ text: SimpleOrPhrase,+ field: schema::TextFieldEnum,+}++impl Term {+ pub fn new(text: SimpleOrPhrase, field: TextFieldEnum) -> Self {+ Term { text, field }+ }+}++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub enum Node {+ Term(Term),+ And(Box<Node>, Box<Node>),+ Or(Box<Node>, Box<Node>),+ Not(Box<Node>),+}++impl Node {+ pub fn and<T: Into<Node>>(self, other: T) -> Node {+ Node::And(Box::new(self), Box::new(other.into()))+ }++ pub fn or<T: Into<Node>>(self, other: T) -> Node {+ Node::Or(Box::new(self), Box::new(other.into()))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum Occur {+ Must,+ Should,+ MustNot,+}++impl Occur {+ pub fn compose(left: Occur, right: Occur) -> Occur {+ match (left, right) {+ (Occur::Should, _) => right,+ (Occur::Must, Occur::MustNot) => Occur::MustNot,+ (Occur::Must, _) => Occur::Must,+ (Occur::MustNot, Occur::MustNot) => Occur::Must,+ (Occur::MustNot, _) => Occur::MustNot,+ }+ }+}++impl From<Occur> for tantivy::query::Occur {+ fn from(value: Occur) -> Self {+ match value {+ Occur::Must => tantivy::query::Occur::Must,+ Occur::Should => tantivy::query::Occur::Should,+ Occur::MustNot => tantivy::query::Occur::MustNot,+ }+ }+}++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub enum Query {+ Term(Term),+ Boolean { clauses: Vec<(Occur, Query)> },+}++impl Query {+ fn compact(self) -> Query {+ match self {+ Query::Boolean { clauses } => {+ let mut new_clauses = vec![];+ for (occur, query) in clauses {+ let query = query.compact();+ // if the query is a boolean query, and it has the same occur as the current+ // query, we can merge the clauses into the current query+ // otherwise, we add the clause as is+ match query {+ Query::Boolean {+ clauses: inner_clauses,+ } if inner_clauses+ .iter()+ .all(|(inner_occur, _)| occur == *inner_occur) =>+ {+ new_clauses.extend(inner_clauses);+ }++ Query::Boolean {+ clauses: inner_clauses,+ } if inner_clauses.len() == 1 => {+ let (inner_occur, q) = inner_clauses.into_iter().next().unwrap();++ new_clauses.push((Occur::compose(occur, inner_occur), q));+ }++ _ => new_clauses.push((occur, query)),+ }+ }+ Query::Boolean {+ clauses: new_clauses,+ }+ }+ Query::Term(term) => Query::Term(term),+ }+ }++ fn deduplicate(self) -> Query {+ match self {+ Query::Boolean { clauses } => Query::Boolean {+ clauses: clauses+ .into_iter()+ .map(|(occur, query)| (occur, query.deduplicate()))+ .unique()+ .collect(),+ },+ Query::Term(term) => Query::Term(term),+ }+ }++ pub fn as_tantivy(+ &self,+ schema: &tantivy::schema::Schema,+ ) -> Option<Box<dyn tantivy::query::Query>> {+ match self {+ Query::Term(Term { text, field }) => match text {+ SimpleOrPhrase::Simple(s) => {+ let mut terms = process_tantivy_term(s.as_str(), *field, schema);++ let option = field.record_option();+ if terms.len() == 1 {+ let term = terms.remove(0);+ Some(Box::new(tantivy::query::TermQuery::new(term, option)))+ } else {+ Some(Box::new(tantivy::query::BooleanQuery::new(+ terms+ .into_iter()+ .map(|term| {+ (+ tantivy::query::Occur::Must,+ Box::new(tantivy::query::TermQuery::new(term, option))+ as Box<dyn tantivy::query::Query + 'static>,+ )+ })+ .collect(),+ )))+ }+ }+ SimpleOrPhrase::Phrase(p) => {+ let phrase = p.join(" ");+ let mut processed_terms = process_tantivy_term(&phrase, *field, schema);++ if processed_terms.is_empty() {+ return None;+ }++ if processed_terms.len() == 1 {+ let options = field.record_option();++ Some(Box::new(tantivy::query::TermQuery::new(+ processed_terms.pop().unwrap(),+ options,+ )) as Box<dyn tantivy::query::Query>)+ } else {+ Some(Box::new(tantivy::query::PhraseQuery::new(processed_terms))+ as Box<dyn tantivy::query::Query>)+ }+ }+ },+ Query::Boolean { clauses } => {+ let mut t_clauses = Vec::new();+ for (occur, query) in clauses {+ if let Some(query) = query.as_tantivy(schema) {+ t_clauses.push(((*occur).into(), query));+ }+ }++ Some(Box::new(tantivy::query::BooleanQuery::new(t_clauses)))+ }+ }+ }+}++fn process_tantivy_term<T: TextField>(+ term: &str,+ field: T,+ schema: &tantivy::schema::Schema,+) -> Vec<tantivy::Term> {+ let mut terms: Vec<tantivy::Term> = Vec::new();+ let mut tokenizer = field.query_tokenizer();+ let mut token_stream = tokenizer.token_stream(term);+ let tantivy_field = field.tantivy_field(schema);++ token_stream.process(&mut |token| {+ let term = tantivy::Term::from_field_text(tantivy_field, &token.text);+ terms.push(term);+ });++ terms+}++impl Node {+ fn into_non_compacted_query(self) -> Query {+ match self {+ Node::Term(term) => Query::Term(term),+ Node::And(left, right) => Query::Boolean {+ clauses: vec![+ (Occur::Must, left.into_non_compacted_query()),+ (Occur::Must, right.into_non_compacted_query()),+ ],+ },+ Node::Or(left, right) => Query::Boolean {+ clauses: vec![+ (Occur::Should, left.into_non_compacted_query()),+ (Occur::Should, right.into_non_compacted_query()),+ ],+ },+ Node::Not(inner) => Query::Boolean {+ clauses: vec![(Occur::MustNot, inner.into_non_compacted_query())],+ },+ }+ }++ pub fn into_query(self) -> Query {+ self.into_non_compacted_query().compact().deduplicate()+ }++ fn from_term(term: super::Term) -> Self {+ match term {+ super::Term::SimpleOrPhrase(s) => match s {+ super::SimpleOrPhrase::Simple(term) => TextFieldEnum::all()+ .filter(|f| f.is_searchable())+ .map(|field| {+ Node::Term(Term {+ text: SimpleOrPhrase::Simple(term.clone()),+ field,+ })+ })+ .reduce(|left, right| left.or(right))+ .expect("fields should not be empty"),+ super::SimpleOrPhrase::Phrase(p) => TextFieldEnum::all()+ .filter(|f| f.is_searchable())+ .filter(|f| f.is_phrase_searchable())+ .map(|field| {+ Node::Term(Term {+ text: SimpleOrPhrase::Phrase(p.clone()),+ field,+ })+ })+ .reduce(|left, right| left.or(right))+ .expect("fields should not be empty"),+ },+ super::Term::Site(s) => Node::Term(Term {+ text: SimpleOrPhrase::Simple(super::parser::SimpleTerm::from(s)),+ field: text_field::UrlForSiteOperator.into(),+ }),+ super::Term::Title(t) => Node::Term(Term {+ text: t,+ field: text_field::Title.into(),+ }),+ super::Term::Body(b) => Node::Term(Term {+ text: b,+ field: text_field::AllBody.into(),+ }),+ super::Term::Url(u) => Node::Term(Term {+ text: u,+ field: text_field::Url.into(),+ }),+ super::Term::PossibleBang { prefix, bang } => {+ let mut s = String::new();+ s.push(prefix);+ s.push_str(bang.as_str());++ let s = SimpleTerm::from(s);++ TextFieldEnum::all()+ .filter(|f| f.is_searchable())+ .map(|field| {+ Node::Term(Term {+ text: SimpleOrPhrase::Simple(s.clone()),+ field,+ })+ })+ .reduce(|left, right| left.or(right))+ .expect("fields should not be empty")+ }+ super::Term::Not(n) => Node::Not(Box::new(Node::from_term(*n))),+ }+ }+}++fn sliding_window(window_size: usize, i: usize) -> impl Iterator<Item = (usize, usize)> {+ (0..=window_size)+ .map(move |offset| {+ let start = (i + offset).saturating_sub(window_size);+ let end = i + offset;++ (start, end)+ })+ .filter(|(start, end)| start < end)+ .filter(|(start, end)| end != start)+}++pub fn initial(terms: Vec<super::Term>) -> Option<Node> {+ let mut nodes = Vec::new();+ let terms_for_adjacent = terms.clone();++ for (i, term) in terms.into_iter().enumerate() {+ let mut adjacent = Vec::new();++ if let super::Term::SimpleOrPhrase(_) = &term {+ for window_size in 2..=3 {+ for (start, end) in sliding_window(window_size, i) {+ let mut compounds = Vec::new();++ for k in start..=end {+ if let Some(super::Term::SimpleOrPhrase(super::SimpleOrPhrase::Simple(s))) =+ terms_for_adjacent.get(k)+ {+ compounds.push(s.clone());+ }+ }++ if !compounds.is_empty() {+ adjacent.push(super::TermCompound { terms: compounds });+ }+ }+ }+ }++ let node = Node::from_term(term);++ if !adjacent.is_empty() {+ match adjacent+ .into_iter()+ .flat_map(|compound| {+ TextFieldEnum::all()+ .filter(|f| f.is_searchable())+ .filter(|f| f.is_compound_searchable())+ .map(move |field| {+ let compound_text: String = compound+ .terms+ .iter()+ .map(|s| s.as_str().to_string())+ .collect();++ Node::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from(compound_text)),+ field,+ })+ })+ })+ .reduce(|left, right| left.or(right))+ {+ Some(adj) => nodes.push(node.or(adj)),+ None => nodes.push(node),+ }+ } else {+ nodes.push(node);+ }+ }++ nodes.into_iter().reduce(|left, right| left.and(right))+}++#[cfg(test)]+mod tests {+ use super::*;++ fn parse(query: &str, fields: &[TextFieldEnum]) -> Node {+ let terms = query+ .split_whitespace()+ .map(|s| SimpleTerm::from(s.to_string()))+ .collect::<Vec<_>>();++ let mut queries = vec![];++ for term in terms {+ let nodes: Vec<_> = fields+ .iter()+ .copied()+ .map(|f| {+ Node::Term(Term {+ text: SimpleOrPhrase::Simple(term.clone()),+ field: f,+ })+ })+ .collect();++ let term_q = if nodes.len() == 1 {+ nodes[0].clone()+ } else {+ nodes+ .into_iter()+ .reduce(|left, right| left.or(right))+ .unwrap()+ };++ queries.push(term_q);+ }++ if queries.len() == 1 {+ queries[0].clone()+ } else {+ queries+ .into_iter()+ .reduce(|left, right| left.and(right))+ .unwrap()+ }+ }++ #[test]+ fn test_compact() {+ let query = Query::Boolean { clauses: vec![] };++ assert_eq!(query.clone().compact(), query);++ let query = parse(+ "foo bar",+ &[text_field::Title.into(), text_field::AllBody.into()],+ );++ let expected = Query::Boolean {+ clauses: vec![+ (+ Occur::Must,+ Query::Boolean {+ clauses: vec![+ (+ Occur::Should,+ Query::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from(+ "foo".to_string(),+ )),+ field: text_field::Title.into(),+ }),+ ),+ (+ Occur::Should,+ Query::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from(+ "foo".to_string(),+ )),+ field: text_field::AllBody.into(),+ }),+ ),+ ],+ },+ ),+ (+ Occur::Must,+ Query::Boolean {+ clauses: vec![+ (+ Occur::Should,+ Query::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from(+ "bar".to_string(),+ )),+ field: text_field::Title.into(),+ }),+ ),+ (+ Occur::Should,+ Query::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from(+ "bar".to_string(),+ )),+ field: text_field::AllBody.into(),+ }),+ ),+ ],+ },+ ),+ ],+ };++ assert_eq!(query.into_query().compact(), expected);+ }++ #[test]+ fn test_sliding_window() {+ let window_size = 3;+ let i = 3;++ let expected = vec![(0, 3), (1, 4), (2, 5), (3, 6)];++ assert_eq!(sliding_window(window_size, i).collect::<Vec<_>>(), expected);++ let window_size = 2;+ let i = 3;++ let expected = vec![(1, 3), (2, 4), (3, 5)];++ assert_eq!(sliding_window(window_size, i).collect::<Vec<_>>(), expected);++ let window_size = 2;+ let i = 0;++ let expected = vec![(0, 1), (0, 2)];++ assert_eq!(sliding_window(window_size, i).collect::<Vec<_>>(), expected);+ }+}diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rsindex 537828e8..23eba9bb 100644--- a/crates/core/src/ranking/optics.rs+++ b/crates/core/src/ranking/optics.rs@@ -212,30 +212,6 @@ mod tests {assert_eq!(result.webpages[0].url, "https://www.third.com/");assert_eq!(result.webpages[1].url, "https://www.second.com/");- let result = searcher- .search(&SearchQuery {- query: "test".to_string(),- host_rankings: Some(HostRankings {- liked: vec![],- disliked: vec![],- blocked: vec!["www.first.com".to_string()],- }),- optic: Some(Optic {- rankings: vec![RankingCoeff {- target: RankingTarget::Signal("inbound_similarity".to_string()),- value: 100_000_000.0,- }],- ..Default::default()- }),- return_ranking_signals: true,- ..Default::default()- })- .expect("Search failed");-- assert_eq!(result.webpages.len(), 2);- assert_eq!(result.webpages[0].url, "https://www.third.com/");- assert_eq!(result.webpages[1].url, "https://www.second.com/");-let result = searcher.search(&SearchQuery {query: "test".to_string(),diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rsindex 482dd1d1..0407ef2d 100644--- a/crates/core/src/schema/text_field.rs+++ b/crates/core/src/schema/text_field.rs@@ -90,10 +90,25 @@ pub trait TextField:false}+ fn is_phrase_searchable(&self) -> bool {+ self.is_searchable() && self.has_pos()+ }+fn is_stored(&self) -> bool {false}+ /// Whether or not we should use the field+ /// to lookup compounds. E.g. "new york times"+ /// should also match "newyorktimes".+ ///+ /// This should only be activated for very few fields+ /// as the number of queries we have to match against+ /// the index grows quite substantially.+ fn is_compound_searchable(&self) -> bool {+ false+ }+fn record_option(&self) -> IndexRecordOption {if self.has_pos() {IndexRecordOption::WithFreqsAndPositions@@ -285,6 +300,10 @@ impl TextField for Title {true}+ fn is_compound_searchable(&self) -> bool {+ true+ }+fn add_html_tantivy(&self,_html: &Html,@@ -431,6 +450,10 @@ impl TextField for AllBody {true}+ fn is_compound_searchable(&self) -> bool {+ true+ }+fn add_html_tantivy(&self,_html: &Html,@@ -469,6 +492,10 @@ impl TextField for Url {true}+ fn is_compound_searchable(&self) -> bool {+ true+ }+fn add_html_tantivy(&self,_html: &Html,diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex 22fa6506..079d6654 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -213,13 +213,13 @@ wherelet parsed_terms = query::parser::parse(&query.query)?;if parsed_terms.iter().any(|term| match term {- query::parser::Term::PossibleBang(t) => t.is_empty(),+ query::parser::Term::PossibleBang { prefix: _, bang } => bang.is_empty(),_ => false,}) {let q: String = intersperse(parsed_terms.iter()- .filter(|term| !matches!(term, query::parser::Term::PossibleBang(_)))+ .filter(|term| !matches!(term, query::parser::Term::PossibleBang { .. })).map(|term| term.to_string())," ".to_string(),)diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rsindex 44affdea..3e8bd9b9 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local.rs@@ -33,7 +33,7 @@ use crate::ranking::{self, query_centrality, Ranker, SignalComputer, SignalEnum}use crate::search_ctx::Ctx;use crate::search_prettifier::DisplayedWebpage;use crate::webgraph::Node;-use crate::{inverted_index, live_index, Error, Result};+use crate::{inverted_index, live_index, Result};use super::WebsitesResult;use super::{InitialWebsiteResult, SearchQuery};@@ -169,13 +169,7 @@ whereguard: &G,query: &SearchQuery,) -> Result<Query> {- let parsed_query = Query::parse(ctx, query, guard.inverted_index())?;-- if parsed_query.is_empty() {- Err(Error::EmptyQuery.into())- } else {- Ok(parsed_query)- }+ Query::parse(ctx, query, guard.inverted_index())}fn ranker<'a, G: SearchGuard<'a>>(@@ -339,10 +333,6 @@ where};let query = Query::parse(&ctx, &query, guard.inverted_index())?;- if query.is_empty() {- return Err(Error::EmptyQuery.into());- }-guard.inverted_index().retrieve_websites(websites, &query)}diff --git a/crates/core/src/speedy_kv/mod.rs b/crates/core/src/speedy_kv/mod.rsindex 69b744d0..dc982051 100644--- a/crates/core/src/speedy_kv/mod.rs+++ b/crates/core/src/speedy_kv/mod.rs@@ -580,9 +580,6 @@ mod tests {let mut db1 = Db::open_or_create(gen_temp_path()).unwrap();let mut db2 = Db::open_or_create(gen_temp_path()).unwrap();- dbg!(db1.folder());- dbg!(db2.folder());-db1.insert(1, 2).unwrap();db1.insert(2, 3).unwrap();diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rsindex 39352635..60c5934e 100644--- a/crates/core/src/webgraph/centrality/harmonic.rs+++ b/crates/core/src/webgraph/centrality/harmonic.rs@@ -460,7 +460,8 @@ mod tests {other.insert(Node::from("A"), Node::from("B"), "7".to_string());other.commit();- let graph = other.finalize();+ let mut graph = other.finalize();+ graph.optimize_read();let centrality_extra = HarmonicCentrality::calculate(&graph);