web search done right
Nom query parser (#184)
Mikkel Denkere47b49a01228
diff --git a/Cargo.lock b/Cargo.lockindex 35d4a147..b22de01f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4561,6 +4561,7 @@ dependencies = ["memmap2","mime","min-max-heap",+ "nom","num_cpus","once_cell","openraft",diff --git a/Cargo.toml b/Cargo.tomlindex 94d7587c..f7c975af 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -67,6 +67,7 @@ memmap = "0.7.0"memmap2 = "0.9.0"mime = "0.3.17"min-max-heap = "1.3.0"+nom = "7.1.3"num_cpus = "1.15.0"once_cell = "1.13.1"proptest = "1.2.0"diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 5c15455f..520c2940 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -68,6 +68,7 @@ memmap = { workspace = true }memmap2 = { workspace = true }mime = { workspace = true }min-max-heap = { workspace = true }+nom = { workspace = true }num_cpus = { workspace = true }once_cell = { workspace = true }openraft = { workspace = true }diff --git a/crates/core/src/bangs.rs b/crates/core/src/bangs.rsindex 70b3b57e..9ce07919 100644--- a/crates/core/src/bangs.rs+++ b/crates/core/src/bangs.rs@@ -109,9 +109,9 @@ impl Bangs {}}- pub fn get(&self, terms: &[Box<Term>]) -> Option<BangHit> {+ pub fn get(&self, terms: &[Term]) -> Option<BangHit> {for possible_bang in terms.iter().filter_map(|term| {- if let Term::PossibleBang(possible_bang) = term.as_ref() {+ if let Term::PossibleBang(possible_bang) = term {Some(possible_bang)} else {None@@ -122,13 +122,13 @@ impl Bangs {terms.iter().filter(|term| {- if let Term::PossibleBang(bang) = term.as_ref() {+ if let Term::PossibleBang(bang) = term {bang != possible_bang} else {true}})- .map(|term| term.as_ref().to_string()),+ .map(|term| term.to_string())," ".to_string(),).collect::<String>();@@ -172,11 +172,11 @@ mod tests {}]"#,);- assert_eq!(bangs.get(&parse("no bangs")), None);- assert_eq!(bangs.get(&parse("!no bangs")), None);+ assert_eq!(bangs.get(&parse("no bangs").unwrap()), None);+ assert_eq!(bangs.get(&parse("!no bangs").unwrap()), None);assert_eq!(- bangs.get(&parse("!ty bangs")),+ bangs.get(&parse("!ty bangs").unwrap()),Some(BangHit {bang: Bang {category: Some("Multimedia".to_string()),diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex d769ad09..abd71c50 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -38,14 +38,16 @@ pub mod union;use parser::Term;-use self::{optic::AsMultipleTantivyQuery, parser::CompoundAwareTerm};+use self::{+ optic::AsMultipleTantivyQuery,+ parser::{CompoundAwareTerm, SimpleOrPhrase},+};const MAX_SIMILAR_TERMS: usize = 10;#[derive(Clone, Debug)]pub struct Query {- #[allow(clippy::vec_box)]- terms: Vec<Box<Term>>,+ terms: Vec<Term>,simple_terms_text: Vec<String>,tantivy_query: Box<BooleanQuery>,host_rankings: HostRankings,@@ -58,7 +60,7 @@ pub struct Query {impl Query {pub fn parse(ctx: &Ctx, query: &SearchQuery, index: &InvertedIndex) -> Result<Query> {- let parsed_terms = parser::parse(&query.query);+ let parsed_terms = parser::parse(&query.query)?;let mut term_count = HashMap::new();let mut terms = Vec::new();@@ -76,7 +78,7 @@ impl Query {.clone().into_iter().map(|term| CompoundAwareTerm {- term: *term,+ term,adjacent_terms: Vec::new(),}).collect();@@ -87,7 +89,9 @@ impl Query {for window in term_ids.windows(window_size) {let mut window_terms = Vec::new();for i in window {- if let Term::Simple(t) = &compound_terms[*i].term {+ if let Term::SimpleOrPhrase(SimpleOrPhrase::Simple(t)) =+ &compound_terms[*i].term+ {window_terms.push(t.clone());}}@@ -180,7 +184,7 @@ impl Query {&self.simple_terms_text}- pub fn terms(&self) -> &[Box<Term>] {+ pub fn terms(&self) -> &[Term] {&self.terms}diff --git a/crates/core/src/query/parser.rs b/crates/core/src/query/parser.rsdeleted file mode 100644index 83cd8ad8..00000000--- a/crates/core/src/query/parser.rs+++ /dev/null@@ -1,581 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::fmt::Display;-use tantivy::{- query::{BooleanQuery, Occur, PhraseQuery, TermQuery},- tokenizer::Tokenizer,-};--use crate::{- bangs::BANG_PREFIXES,- floor_char_boundary,- schema::{Field, TextField},-};--#[derive(Debug, Clone)]-pub struct TermCompound {- pub terms: Vec<SimpleTerm>,-}--#[derive(Debug, Clone)]-pub struct CompoundAwareTerm {- pub term: Term,- pub adjacent_terms: Vec<TermCompound>,-}-impl CompoundAwareTerm {- pub fn as_tantivy_query(- &self,- fields: &[tantivy::schema::Field],- ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {- if !self.adjacent_terms.is_empty() {- if let Term::Simple(simple_term) = &self.term {- return simple_into_tantivy(simple_term, &self.adjacent_terms, fields);- }- }-- self.term.as_tantivy_query(fields)- }-}--#[derive(Debug, Clone, PartialEq, Eq, Hash)]-pub struct SimpleTerm(String);-impl SimpleTerm {- pub fn as_str(&self) -> &str {- self.0.as_str()- }-}--impl From<String> for SimpleTerm {- fn from(value: String) -> Self {- SimpleTerm(value)- }-}--impl From<SimpleTerm> for String {- fn from(value: SimpleTerm) -> Self {- value.0- }-}--#[derive(Debug, Clone, PartialEq, Eq, Hash)]-pub enum Term {- Simple(SimpleTerm),- Phrase(String),- Not(Box<Term>),- Site(String),- Title(String),- Body(String),- Url(String),- PossibleBang(String),-}--impl Display for Term {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- match self {- Term::Simple(term) => write!(f, "{}", term.0),- Term::Phrase(phrase) => write!(f, "\"{}\"", phrase),- Term::Not(term) => write!(f, "-{}", term),- Term::Site(site) => write!(f, "site:{}", site),- Term::Title(title) => write!(f, "intitle:{}", title),- Term::Body(body) => write!(f, "inbody:{}", body),- Term::Url(url) => write!(f, "inurl:{}", url),- Term::PossibleBang(bang) => write!(f, "{}{}", BANG_PREFIXES[0], bang),- }- }-}--fn simple_into_tantivy(- term: &SimpleTerm,- adjacent_terms: &[TermCompound],- fields: &[tantivy::schema::Field],-) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {- let mut queries = Term::into_tantivy_simple(term, fields);-- let fields = fields- .iter()- .filter(|field| {- matches!(- Field::get(field.field_id() as usize),- Some(Field::Text(TextField::AllBody))- | Some(Field::Text(TextField::Title))- | Some(Field::Text(TextField::Url))- )- })- .copied()- .collect::<Vec<_>>();-- for adjacent_term in adjacent_terms {- let combined = adjacent_term- .terms- .iter()- .map(|term| term.0.as_str())- .collect::<String>();-- for field in &fields {- queries.push((Occur::Should, Term::tantivy_text_query(field, &combined)))- }- }-- (Occur::Must, Box::new(BooleanQuery::new(queries)))-}--impl Term {- pub fn as_simple_text(&self) -> Option<&str> {- match self {- Term::Simple(term) => Some(&term.0),- Term::Phrase(p) => Some(p),- _ => None,- }- }-- fn as_tantivy_query(- &self,- fields: &[tantivy::schema::Field],- ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {- match self {- Term::Simple(term) => simple_into_tantivy(term, &[], fields),- Term::Phrase(phrase) => {- let mut phrases = Vec::with_capacity(fields.len());-- for (field, tv_field) in fields- .iter()- .filter_map(|tv_field| {- Field::get(tv_field.field_id() as usize).map(|mapped| (mapped, *tv_field))- })- .filter(|(field, _)| field.is_searchable())- .filter(|(field, _)| field.has_pos())- {- let mut processed_terms = Term::process_tantivy_term(phrase, tv_field);-- if processed_terms.is_empty() {- continue;- }-- if processed_terms.len() == 1 {- let options = field.as_text().unwrap().index_option();-- phrases.push((- Occur::Should,- Box::new(TermQuery::new(processed_terms.pop().unwrap(), options))- as Box<dyn tantivy::query::Query>,- ));- } else {- phrases.push((- Occur::Should,- Box::new(PhraseQuery::new(processed_terms))- as Box<dyn tantivy::query::Query>,- ));- }- }-- (Occur::Must, Box::new(BooleanQuery::new(phrases)))- }- Term::Not(subterm) => (- Occur::MustNot,- Box::new(BooleanQuery::new(vec![subterm.as_tantivy_query(fields)])),- ),- Term::Site(site) => (- Occur::Must,- Box::new(BooleanQuery::new(Term::into_tantivy_site(site, fields))),- ),- Term::Title(title) => {- let field = fields- .iter()- .find(|field| {- matches!(- Field::get(field.field_id() as usize),- Some(Field::Text(TextField::Title))- )- })- .unwrap();-- (Occur::Must, Term::tantivy_text_query(field, title))- }- Term::Body(body) => {- let field = fields- .iter()- .find(|field| {- matches!(- Field::get(field.field_id() as usize),- Some(Field::Text(TextField::AllBody))- )- })- .unwrap();-- (Occur::Must, Term::tantivy_text_query(field, body))- }- Term::Url(url) => {- let field = fields- .iter()- .find(|field| {- matches!(- Field::get(field.field_id() as usize),- Some(Field::Text(TextField::Url))- )- })- .unwrap();-- (Occur::Must, Term::tantivy_text_query(field, url))- }- Term::PossibleBang(text) => {- let mut term = String::new();-- term.push(BANG_PREFIXES[0]);- term.push_str(text);-- simple_into_tantivy(&term.into(), &[], fields)- }- }- }-- fn into_tantivy_simple(- term: &SimpleTerm,- fields: &[tantivy::schema::Field],- ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {- fields- .iter()- .filter_map(|tv_field| {- Field::get(tv_field.field_id() as usize)- .filter(|field| field.is_searchable())- .map(|_| tv_field)- })- .map(|field| (Occur::Should, Term::tantivy_text_query(field, &term.0)))- .collect()- }-- fn into_tantivy_site(- term: &str,- fields: &[tantivy::schema::Field],- ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {- fields- .iter()- .filter(|field| {- matches!(- Field::get(field.field_id() as usize),- Some(Field::Text(TextField::UrlForSiteOperator))- )- })- .map(|field| {- let processed_terms = Term::process_tantivy_term(term, *field);-- if processed_terms.len() == 1 {- let term = processed_terms.first().unwrap().clone();- (- Occur::Should,- Box::new(TermQuery::new(- term,- tantivy::schema::IndexRecordOption::Basic,- )) as Box<dyn tantivy::query::Query>,- )- } else {- (- Occur::Should,- Box::new(PhraseQuery::new(processed_terms))- as Box<dyn tantivy::query::Query>,- )- }- })- .collect()- }-- fn tantivy_text_query(- field: &tantivy::schema::Field,- term: &str,- ) -> Box<dyn tantivy::query::Query + 'static> {- let mut processed_terms = Term::process_tantivy_term(term, *field);-- let option = Field::get(field.field_id() as usize)- .unwrap()- .as_text()- .unwrap()- .index_option();-- let processed_query = if processed_terms.len() == 1 {- let term = processed_terms.remove(0);- Box::new(TermQuery::new(term, option)) as Box<dyn tantivy::query::Query + 'static>- } else {- Box::new(BooleanQuery::new(- processed_terms- .into_iter()- .map(|term| {- (- Occur::Must,- Box::new(TermQuery::new(term, option))- as Box<dyn tantivy::query::Query + 'static>,- )- })- .collect(),- )) as Box<dyn tantivy::query::Query + 'static>- };-- Box::new(processed_query)- }-- fn process_tantivy_term(- term: &str,- tantivy_field: tantivy::schema::Field,- ) -> Vec<tantivy::Term> {- match Field::get(tantivy_field.field_id() as usize) {- Some(Field::Fast(_)) => vec![tantivy::Term::from_field_text(tantivy_field, term)],- Some(Field::Text(text_field)) => {- let mut terms: Vec<tantivy::Term> = Vec::new();- let mut tokenizer = text_field.query_tokenizer();- let mut token_stream = tokenizer.token_stream(term);- token_stream.process(&mut |token| {- let term = tantivy::Term::from_field_text(tantivy_field, &token.text);- terms.push(term);- });-- terms- }- None => vec![],- }- }-}--fn parse_term(term: &str) -> Box<Term> {- // TODO: re-write this entire function once if-let chains become stable- if let Some(not_term) = term.strip_prefix('-') {- if !not_term.is_empty() && !not_term.starts_with('-') {- Box::new(Term::Not(parse_term(not_term)))- } else {- Box::new(Term::Simple(term.to_string().into()))- }- } else if let Some(site) = term.strip_prefix("site:") {- if !site.is_empty() {- Box::new(Term::Site(site.to_string()))- } else {- Box::new(Term::Simple(term.to_string().into()))- }- } else if let Some(title) = term.strip_prefix("intitle:") {- if !title.is_empty() {- Box::new(Term::Title(title.to_string()))- } else {- Box::new(Term::Simple(term.to_string().into()))- }- } else if let Some(body) = term.strip_prefix("inbody:") {- if !body.is_empty() {- Box::new(Term::Body(body.to_string()))- } else {- Box::new(Term::Simple(term.to_string().into()))- }- } else if let Some(url) = term.strip_prefix("inurl:") {- if !url.is_empty() {- Box::new(Term::Url(url.to_string()))- } else {- Box::new(Term::Simple(term.to_string().into()))- }- } else {- for bang_prefix in BANG_PREFIXES {- if let Some(bang) = term.strip_prefix(bang_prefix) {- return Box::new(Term::PossibleBang(bang.to_string()));- }- }-- Box::new(Term::Simple(term.to_string().into()))- }-}--#[allow(clippy::vec_box)]-pub fn parse(query: &str) -> Vec<Box<Term>> {- let query = query.to_lowercase().replace(['“', '”'], "\"");-- let mut res = Vec::new();-- let mut cur_term_begin = 0;-- for (offset, c) in query.char_indices() {- if cur_term_begin > offset {- continue;- }-- cur_term_begin = floor_char_boundary(&query, cur_term_begin);-- if query[cur_term_begin..].starts_with('"') {- if let Some(offset) = query[cur_term_begin + 1..].find('"') {- let offset = offset + cur_term_begin + 1;- res.push(Box::new(Term::Phrase(- query[cur_term_begin + 1..offset].to_string(),- )));-- cur_term_begin = offset + 1;- continue;- }- }- if c.is_whitespace() {- if offset - cur_term_begin == 0 {- cur_term_begin = offset + 1;- continue;- }-- res.push(parse_term(&query[cur_term_begin..offset]));- cur_term_begin = offset + 1;- }- }-- if cur_term_begin < query.len() {- res.push(parse_term(- &query[floor_char_boundary(&query, cur_term_begin)..query.len()],- ));- }-- res-}--#[cfg(test)]-mod tests {- use super::*;- use proptest::prelude::*;-- #[test]- fn parse_not() {- assert_eq!(- parse("this -that"),- vec![- Box::new(Term::Simple("this".to_string().into())),- Box::new(Term::Not(Box::new(Term::Simple("that".to_string().into()))))- ]- );-- assert_eq!(- parse("this -"),- vec![- Box::new(Term::Simple("this".to_string().into())),- Box::new(Term::Simple("-".to_string().into()))- ]- );- }-- #[test]- fn double_not() {- assert_eq!(- parse("this --that"),- vec![- Box::new(Term::Simple("this".to_string().into())),- Box::new(Term::Simple("--that".to_string().into()))- ]- );- }-- #[test]- fn site() {- assert_eq!(- parse("this site:test.com"),- vec![- Box::new(Term::Simple("this".to_string().into())),- Box::new(Term::Site("test.com".to_string()))- ]- );- }-- #[test]- fn title() {- assert_eq!(- parse("this intitle:test"),- vec![- Box::new(Term::Simple("this".to_string().into())),- Box::new(Term::Title("test".to_string()))- ]- );- }-- #[test]- fn body() {- assert_eq!(- parse("this inbody:test"),- vec![- Box::new(Term::Simple("this".to_string().into())),- Box::new(Term::Body("test".to_string()))- ]- );- }-- #[test]- fn url() {- assert_eq!(- parse("this inurl:test"),- vec![- Box::new(Term::Simple("this".to_string().into())),- Box::new(Term::Url("test".to_string()))- ]- );- }-- #[test]- fn empty() {- assert_eq!(parse(""), vec![]);- }-- #[test]- fn phrase() {- assert_eq!(- parse("\"this is a\" inurl:test"),- vec![- Box::new(Term::Phrase("this is a".to_string(),)),- Box::new(Term::Url("test".to_string()))- ]- );- assert_eq!(- parse("\"this is a inurl:test"),- vec![- Box::new(Term::Simple("\"this".to_string().into())),- Box::new(Term::Simple("is".to_string().into())),- Box::new(Term::Simple("a".to_string().into())),- Box::new(Term::Url("test".to_string()))- ]- );- assert_eq!(- parse("this is a\" inurl:test"),- vec![- Box::new(Term::Simple("this".to_string().into())),- Box::new(Term::Simple("is".to_string().into())),- Box::new(Term::Simple("a\"".to_string().into())),- Box::new(Term::Url("test".to_string()))- ]- );-- assert_eq!(- parse("\"this is a inurl:test\""),- vec![Box::new(Term::Phrase("this is a inurl:test".to_string(),)),]- );-- assert_eq!(- parse("\"\""),- vec![Box::new(Term::Phrase("".to_string(),)),]- );- assert_eq!(- parse("“this is a“ inurl:test"),- vec![- Box::new(Term::Phrase("this is a".to_string(),)),- Box::new(Term::Url("test".to_string()))- ]- );- }-- #[test]- fn unicode() {- let query = "\u{a0}";- assert_eq!(parse(query).len(), 1);- }-- proptest! {- #[test]- fn prop(query: String) {- parse(&query);- }- }-}diff --git a/crates/core/src/query/parser/as_tantivy.rs b/crates/core/src/query/parser/as_tantivy.rsnew file mode 100644index 00000000..efa90e91--- /dev/null+++ b/crates/core/src/query/parser/as_tantivy.rs@@ -0,0 +1,304 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::{+ query::{BooleanQuery, Occur, PhraseQuery, TermQuery},+ tokenizer::Tokenizer as _,+};++use crate::{+ bangs::BANG_PREFIXES,+ schema::{Field, TextField},+};++use super::{CompoundAwareTerm, SimpleOrPhrase, SimpleTerm, Term, TermCompound};++impl CompoundAwareTerm {+ pub fn as_tantivy_query(+ &self,+ fields: &[tantivy::schema::Field],+ ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {+ if !self.adjacent_terms.is_empty() {+ if let Term::SimpleOrPhrase(SimpleOrPhrase::Simple(simple_term)) = &self.term {+ return simple_into_tantivy(simple_term, &self.adjacent_terms, fields);+ }+ }++ self.term.as_tantivy_query(fields)+ }+}++fn simple_into_tantivy(+ term: &SimpleTerm,+ adjacent_terms: &[TermCompound],+ fields: &[tantivy::schema::Field],+) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {+ let mut queries = Term::into_tantivy_simple(term, fields);++ let fields = fields+ .iter()+ .filter(|field| {+ matches!(+ Field::get(field.field_id() as usize),+ Some(Field::Text(TextField::AllBody))+ | Some(Field::Text(TextField::Title))+ | Some(Field::Text(TextField::Url))+ )+ })+ .copied()+ .collect::<Vec<_>>();++ for adjacent_term in adjacent_terms {+ let combined = adjacent_term+ .terms+ .iter()+ .map(|term| term.as_str())+ .collect::<String>();++ for field in &fields {+ queries.push((Occur::Should, Term::tantivy_text_query(field, &combined)))+ }+ }++ (Occur::Must, Box::new(BooleanQuery::new(queries)))+}++fn phrase_query(+ terms: &[String],+ fields: &[tantivy::schema::Field],+) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {+ let mut phrases = Vec::with_capacity(fields.len());+ let phrase = terms.join(" ");++ for (field, tv_field) in fields+ .iter()+ .filter_map(|tv_field| {+ Field::get(tv_field.field_id() as usize).map(|mapped| (mapped, *tv_field))+ })+ .filter(|(field, _)| field.is_searchable())+ .filter(|(field, _)| field.has_pos())+ {+ let mut processed_terms = Term::process_tantivy_term(&phrase, tv_field);++ if processed_terms.is_empty() {+ continue;+ }++ if processed_terms.len() == 1 {+ let options = field.as_text().unwrap().index_option();++ phrases.push((+ Occur::Should,+ Box::new(TermQuery::new(processed_terms.pop().unwrap(), options))+ as Box<dyn tantivy::query::Query>,+ ));+ } else {+ phrases.push((+ Occur::Should,+ Box::new(PhraseQuery::new(processed_terms)) as Box<dyn tantivy::query::Query>,+ ));+ }+ }++ (Occur::Must, Box::new(BooleanQuery::new(phrases)))+}++impl Term {+ fn as_tantivy_query(+ &self,+ fields: &[tantivy::schema::Field],+ ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {+ match self {+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple(term)) => {+ simple_into_tantivy(term, &[], fields)+ }+ Term::SimpleOrPhrase(SimpleOrPhrase::Phrase(phrase)) => phrase_query(phrase, fields),+ Term::Not(subterm) => (+ Occur::MustNot,+ Box::new(BooleanQuery::new(vec![subterm.as_tantivy_query(fields)])),+ ),+ Term::Site(site) => (+ Occur::Must,+ Box::new(BooleanQuery::new(Term::into_tantivy_site(site, fields))),+ ),+ Term::Title(title) => {+ let field = fields+ .iter()+ .find(|field| {+ matches!(+ Field::get(field.field_id() as usize),+ Some(Field::Text(TextField::Title))+ )+ })+ .unwrap();++ (+ Occur::Must,+ Term::tantivy_text_query(field, &title.as_string()),+ )+ }+ Term::Body(body) => {+ let field = fields+ .iter()+ .find(|field| {+ matches!(+ Field::get(field.field_id() as usize),+ Some(Field::Text(TextField::AllBody))+ )+ })+ .unwrap();++ (+ Occur::Must,+ Term::tantivy_text_query(field, &body.as_string()),+ )+ }+ Term::Url(url) => {+ let field = fields+ .iter()+ .find(|field| {+ matches!(+ Field::get(field.field_id() as usize),+ Some(Field::Text(TextField::Url))+ )+ })+ .unwrap();++ (+ Occur::Must,+ Term::tantivy_text_query(field, &url.as_string()),+ )+ }+ Term::PossibleBang(text) => {+ let mut term = String::new();++ term.push(BANG_PREFIXES[0]);+ term.push_str(text);++ simple_into_tantivy(&term.into(), &[], fields)+ }+ }+ }++ fn into_tantivy_simple(+ term: &SimpleTerm,+ fields: &[tantivy::schema::Field],+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {+ fields+ .iter()+ .filter_map(|tv_field| {+ Field::get(tv_field.field_id() as usize)+ .filter(|field| field.is_searchable())+ .map(|_| tv_field)+ })+ .map(|field| {+ (+ Occur::Should,+ Term::tantivy_text_query(field, term.as_str()),+ )+ })+ .collect()+ }++ fn into_tantivy_site(+ term: &str,+ fields: &[tantivy::schema::Field],+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {+ fields+ .iter()+ .filter(|field| {+ matches!(+ Field::get(field.field_id() as usize),+ Some(Field::Text(TextField::UrlForSiteOperator))+ )+ })+ .map(|field| {+ let processed_terms = Term::process_tantivy_term(term, *field);++ if processed_terms.len() == 1 {+ let term = processed_terms.first().unwrap().clone();+ (+ Occur::Should,+ Box::new(TermQuery::new(+ term,+ tantivy::schema::IndexRecordOption::Basic,+ )) as Box<dyn tantivy::query::Query>,+ )+ } else {+ (+ Occur::Should,+ Box::new(PhraseQuery::new(processed_terms))+ as Box<dyn tantivy::query::Query>,+ )+ }+ })+ .collect()+ }++ fn tantivy_text_query(+ field: &tantivy::schema::Field,+ term: &str,+ ) -> Box<dyn tantivy::query::Query + 'static> {+ let mut processed_terms = Term::process_tantivy_term(term, *field);++ let option = Field::get(field.field_id() as usize)+ .unwrap()+ .as_text()+ .unwrap()+ .index_option();++ let processed_query = if processed_terms.len() == 1 {+ let term = processed_terms.remove(0);+ Box::new(TermQuery::new(term, option)) as Box<dyn tantivy::query::Query + 'static>+ } else {+ Box::new(BooleanQuery::new(+ processed_terms+ .into_iter()+ .map(|term| {+ (+ Occur::Must,+ Box::new(TermQuery::new(term, option))+ as Box<dyn tantivy::query::Query + 'static>,+ )+ })+ .collect(),+ )) as Box<dyn tantivy::query::Query + 'static>+ };++ Box::new(processed_query)+ }++ fn process_tantivy_term(+ term: &str,+ tantivy_field: tantivy::schema::Field,+ ) -> Vec<tantivy::Term> {+ match Field::get(tantivy_field.field_id() as usize) {+ Some(Field::Fast(_)) => vec![tantivy::Term::from_field_text(tantivy_field, term)],+ Some(Field::Text(text_field)) => {+ let mut terms: Vec<tantivy::Term> = Vec::new();+ let mut tokenizer = text_field.query_tokenizer();+ let mut token_stream = tokenizer.token_stream(term);+ token_stream.process(&mut |token| {+ let term = tantivy::Term::from_field_text(tantivy_field, &token.text);+ terms.push(term);+ });++ terms+ }+ None => vec![],+ }+ }+}diff --git a/crates/core/src/query/parser/mod.rs b/crates/core/src/query/parser/mod.rsnew file mode 100644index 00000000..8968ea94--- /dev/null+++ b/crates/core/src/query/parser/mod.rs@@ -0,0 +1,376 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::bangs::BANG_PREFIXES;++mod as_tantivy;+mod term;++pub use term::*;++fn trim_leading_whitespace(input: &str) -> nom::IResult<&str, &str> {+ let (input, _) = nom::character::complete::multispace0(input)?;+ Ok((input, input))+}++fn until_space_or_end(input: &str) -> nom::IResult<&str, &str> {+ let (input, output) = nom::bytes::complete::take_while(|c: char| c != ' ')(input)?;+ Ok((input, output))+}++fn simple_str(input: &str) -> nom::IResult<&str, &str> {+ // parse until we find a space or the end of the string+ if input.is_empty() {+ return Err(nom::Err::Error(nom::error::Error::new(+ input,+ nom::error::ErrorKind::Eof,+ )));+ }+ let (input, _) = trim_leading_whitespace(input)?;+ let (input, output) = until_space_or_end(input)?;+ Ok((input, output))+}++fn simple(input: &str) -> nom::IResult<&str, SimpleOrPhrase> {+ let (input, output) = simple_str(input)?;+ Ok((input, SimpleOrPhrase::Simple(output.to_string().into())))+}++fn simple_term(input: &str) -> nom::IResult<&str, Term> {+ let (input, output) = simple(input)?;+ Ok((input, Term::SimpleOrPhrase(output)))+}++fn phrase_helper(+ input: &str,+ start_quote: char,+ end_quote: char,+) -> nom::IResult<&str, SimpleOrPhrase> {+ let (input, _) = nom::character::complete::char(start_quote)(input)?;+ let (input, output) = nom::bytes::complete::take_until(end_quote.to_string().as_str())(input)?;+ let (input, _) = nom::character::complete::char(end_quote)(input)?;+ Ok((+ input,+ SimpleOrPhrase::Phrase(output.split_whitespace().map(|s| s.to_string()).collect()),+ ))+}++fn phrase(input: &str) -> nom::IResult<&str, SimpleOrPhrase> {+ let normal = |s| phrase_helper(s, '"', '"');+ let apple1 = |s| phrase_helper(s, '“', '”');+ let apple2 = |s| phrase_helper(s, '“', '“');++ nom::branch::alt((normal, apple1, apple2))(input)+}++fn phrase_term(input: &str) -> nom::IResult<&str, Term> {+ let (input, output) = phrase(input)?;+ Ok((input, Term::SimpleOrPhrase(output)))+}++fn simple_or_phrase(input: &str) -> nom::IResult<&str, SimpleOrPhrase> {+ nom::branch::alt((phrase, simple))(input)+}++fn single_bang(input: &str, pref: char) -> nom::IResult<&str, Term> {+ let (input, _) = nom::character::complete::char(pref)(input)?;+ let (input, output) = until_space_or_end(input)?;+ Ok((input, Term::PossibleBang(output.to_string())))+}++fn bang(input: &str) -> nom::IResult<&str, Term> {+ for pref in BANG_PREFIXES.iter() {+ let (input, output) = single_bang(input, *pref)?;+ return Ok((input, output));+ }++ Err(nom::Err::Error(nom::error::Error::new(+ input,+ nom::error::ErrorKind::Fail,+ )))+}++fn site_field(input: &str) -> nom::IResult<&str, Term> {+ // parse 'site:' and then a simple term+ let (input, _) = nom::bytes::complete::tag("site:")(input)?;+ let (input, output) = simple_str(input)?;++ Ok((input, Term::Site(output.to_string())))+}++fn title_field(input: &str) -> nom::IResult<&str, Term> {+ // parse 'title:' and then a simple term+ let (input, _) = nom::bytes::complete::tag("intitle:")(input)?;+ let (input, output) = simple_or_phrase(input)?;++ Ok((input, Term::Title(output)))+}++fn body_field(input: &str) -> nom::IResult<&str, Term> {+ // parse 'body:' and then a simple term+ let (input, _) = nom::bytes::complete::tag("inbody:")(input)?;+ let (input, output) = simple_or_phrase(input)?;++ Ok((input, Term::Body(output)))+}++fn url_field(input: &str) -> nom::IResult<&str, Term> {+ // parse 'url:' and then a simple term+ let (input, _) = nom::bytes::complete::tag("inurl:")(input)?;+ let (input, output) = simple_or_phrase(input)?;++ Ok((input, Term::Url(output)))+}++fn field_selector(input: &str) -> nom::IResult<&str, Term> {+ nom::branch::alt((site_field, title_field, body_field, url_field))(input)+}++fn not(input: &str) -> nom::IResult<&str, Term> {+ // ignore double negation+ if let Ok((_, _)) = nom::bytes::complete::tag::<_, _, nom::error::Error<&str>>("--")(input) {+ return Err(nom::Err::Error(nom::error::Error::new(+ input,+ nom::error::ErrorKind::Fail,+ )));+ }++ let (input, _) = nom::bytes::complete::tag("-")(input)?;+ let (input, output) = term(input)?;+ Ok((input, Term::Not(Box::new(output))))+}++fn ignore_quotes_helper(+ input: &str,+ start_quote: char,+ end_quote: char,+) -> nom::IResult<&str, &str> {+ let (input, _) = nom::character::complete::char(start_quote)(input)?;+ let (input, output) = nom::bytes::complete::take_until(end_quote.to_string().as_str())(input)?;+ let (input, _) = nom::character::complete::char(end_quote)(input)?;+ Ok((input, output))+}++fn ignore_weird_quotes(input: &str) -> nom::IResult<&str, &str> {+ let guillemet = |s| ignore_quotes_helper(s, '«', '»');+ let up_down_quotes = |s| ignore_quotes_helper(s, '„', '“');+ let rev_guillemet = |s| ignore_quotes_helper(s, '»', '«');+ let squares = |s| ignore_quotes_helper(s, '「', '」');++ nom::branch::alt((guillemet, up_down_quotes, rev_guillemet, squares))(input)+}++fn term(input: &str) -> nom::IResult<&str, Term> {+ let (mut input, _) = trim_leading_whitespace(input)?;++ if let Ok((_, new_input)) = ignore_weird_quotes(input) {+ input = new_input;+ }++ nom::branch::alt((phrase_term, bang, field_selector, not, simple_term))(input)+}++pub fn parse(query: &str) -> anyhow::Result<Vec<Term>> {+ if query.is_empty() || query.chars().all(char::is_whitespace) {+ return Ok(vec![]);+ }++ nom::multi::many1(term)(query)+ .map(|(_, res)| res)+ .map_err(|e| anyhow::anyhow!("Failed to parse query: {:?}", e))+}++#[cfg(test)]+mod tests {+ use proptest::prelude::*;++ use super::{SimpleOrPhrase, Term};++ fn parse(input: &str) -> Vec<Term> {+ super::parse(input).unwrap()+ }++ #[test]+ fn parse_not() {+ assert_eq!(+ parse("this -that"),+ vec![+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),+ Term::Not(Box::new(Term::SimpleOrPhrase(SimpleOrPhrase::Simple(+ "that".to_string().into()+ ))))+ ]+ );++ assert_eq!(+ parse("this -"),+ vec![+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("-".to_string().into()))+ ]+ );+ }++ #[test]+ fn double_not() {+ assert_eq!(+ parse("this --that"),+ vec![+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("--that".to_string().into()))+ ]+ );+ }++ #[test]+ fn site() {+ assert_eq!(+ parse("this site:test.com"),+ vec![+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),+ Term::Site("test.com".to_string())+ ]+ );+ }++ #[test]+ fn title() {+ assert_eq!(+ parse("this intitle:test"),+ vec![+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),+ Term::Title(SimpleOrPhrase::Simple("test".to_string().into()))+ ]+ );+ }++ #[test]+ fn body() {+ assert_eq!(+ parse("this inbody:test"),+ vec![+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),+ Term::Body(SimpleOrPhrase::Simple("test".to_string().into()))+ ]+ );+ }++ #[test]+ fn url() {+ assert_eq!(+ parse("this inurl:test"),+ vec![+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),+ Term::Url(SimpleOrPhrase::Simple("test".to_string().into()))+ ]+ );+ }++ #[test]+ fn empty() {+ assert_eq!(parse(""), vec![]);+ assert_eq!(parse(" "), vec![]);+ }++ #[test]+ fn phrase() {+ assert_eq!(+ parse("\"this is a\" inurl:test"),+ vec![+ Term::SimpleOrPhrase(SimpleOrPhrase::Phrase(vec![+ "this".to_string(),+ "is".to_string(),+ "a".to_string()+ ])),+ Term::Url(SimpleOrPhrase::Simple("test".to_string().into()))+ ]+ );+ assert_eq!(+ parse("\"this is a inurl:test"),+ vec![+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("\"this".to_string().into())),+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("is".to_string().into())),+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("a".to_string().into())),+ Term::Url(SimpleOrPhrase::Simple("test".to_string().into()))+ ]+ );+ assert_eq!(+ parse("this is a\" inurl:test"),+ vec![+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("is".to_string().into())),+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("a\"".to_string().into())),+ Term::Url(SimpleOrPhrase::Simple("test".to_string().into()))+ ]+ );++ assert_eq!(+ parse("\"this is a inurl:test\""),+ vec![Term::SimpleOrPhrase(SimpleOrPhrase::Phrase(vec![+ "this".to_string(),+ "is".to_string(),+ "a".to_string(),+ "inurl:test".to_string()+ ]))]+ );++ assert_eq!(+ parse("\"\""),+ vec![Term::SimpleOrPhrase(SimpleOrPhrase::Phrase(vec![]))]+ );+ assert_eq!(+ parse("“this is a“ inurl:test"),+ vec![+ Term::SimpleOrPhrase(SimpleOrPhrase::Phrase(vec![+ "this".to_string(),+ "is".to_string(),+ "a".to_string()+ ])),+ Term::Url(SimpleOrPhrase::Simple("test".to_string().into()))+ ]+ );++ assert_eq!(+ parse("that's not"),+ vec![+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("that's".to_string().into())),+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("not".to_string().into()))+ ]+ );++ assert_eq!(+ parse("inbody:\"this should work\""),+ vec![Term::Body(SimpleOrPhrase::Phrase(vec![+ "this".to_string(),+ "should".to_string(),+ "work".to_string()+ ]))]+ );+ }++ #[test]+ fn unicode() {+ let query = "🦀";+ assert_eq!(parse(query).len(), 1);+ }++ proptest! {+ #[test]+ fn prop(query: String) {+ parse(&query);+ }+ }+}diff --git a/crates/core/src/query/parser/term.rs b/crates/core/src/query/parser/term.rsnew file mode 100644index 00000000..ac1b5a09--- /dev/null+++ b/crates/core/src/query/parser/term.rs@@ -0,0 +1,105 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use crate::bangs::BANG_PREFIXES;++#[derive(Debug, Clone)]+pub struct TermCompound {+ pub terms: Vec<SimpleTerm>,+}++#[derive(Debug, Clone)]+pub struct CompoundAwareTerm {+ pub term: Term,+ pub adjacent_terms: Vec<TermCompound>,+}++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub enum SimpleOrPhrase {+ Simple(SimpleTerm),+ Phrase(Vec<String>),+}++impl SimpleOrPhrase {+ pub fn as_string(&self) -> String {+ match self {+ SimpleOrPhrase::Simple(simple) => simple.as_str().to_string(),+ SimpleOrPhrase::Phrase(phrase) => phrase.join(" "),+ }+ }+}++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub struct SimpleTerm(String);+impl SimpleTerm {+ pub fn as_str(&self) -> &str {+ self.0.as_str()+ }+}++impl From<String> for SimpleTerm {+ fn from(value: String) -> Self {+ SimpleTerm(value)+ }+}++impl From<SimpleTerm> for String {+ fn from(value: SimpleTerm) -> Self {+ value.0+ }+}+impl std::fmt::Display for SimpleOrPhrase {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ SimpleOrPhrase::Simple(s) => write!(f, "{}", s.as_str()),+ SimpleOrPhrase::Phrase(p) => write!(f, "\"{}\"", p.join(" ")),+ }+ }+}++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub enum Term {+ SimpleOrPhrase(SimpleOrPhrase),+ Site(String),+ Title(SimpleOrPhrase),+ Body(SimpleOrPhrase),+ Url(SimpleOrPhrase),+ PossibleBang(String),+ Not(Box<Term>),+}++impl std::fmt::Display for Term {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Term::SimpleOrPhrase(term) => write!(f, "{}", term),+ Term::Not(term) => write!(f, "-{}", term),+ Term::Site(site) => write!(f, "site:{}", site),+ Term::Title(title) => write!(f, "intitle:{}", title),+ Term::Body(body) => write!(f, "inbody:{}", body),+ Term::Url(url) => write!(f, "inurl:{}", url),+ Term::PossibleBang(bang) => write!(f, "{}{}", BANG_PREFIXES[0], bang),+ }+ }+}++impl Term {+ pub fn as_simple_text(&self) -> Option<String> {+ match self {+ Term::SimpleOrPhrase(term) => Some(term.as_string()),+ _ => None,+ }+ }+}diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex 9ad8846a..00e9e809 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -210,16 +210,16 @@ where}async fn check_bangs(&self, query: &SearchQuery) -> Result<Option<BangHit>> {- let parsed_terms = query::parser::parse(&query.query);+ let parsed_terms = query::parser::parse(&query.query)?;- if parsed_terms.iter().any(|term| match term.as_ref() {+ if parsed_terms.iter().any(|term| match term {query::parser::Term::PossibleBang(t) => t.is_empty(),_ => false,}) {let q: String = intersperse(parsed_terms.iter()- .filter(|term| !matches!(term.as_ref(), query::parser::Term::PossibleBang(_)))+ .filter(|term| !matches!(term, query::parser::Term::PossibleBang(_))).map(|term| term.to_string())," ".to_string(),)@@ -258,13 +258,15 @@ wherepub fn spell_check(&self, query: &str) -> Option<HighlightedSpellCorrection> {let query = query.to_lowercase();- let terms = query::parser::parse(&query);+ let terms = query::parser::parse(&query).ok()?;let simple_query = terms.clone().into_iter()- .filter_map(|term| match *term {- query::parser::Term::Simple(t) => Some(String::from(t)),+ .filter_map(|term| match term {+ query::parser::Term::SimpleOrPhrase(query::parser::SimpleOrPhrase::Simple(t)) => {+ Some(String::from(t))+ }_ => None,}).join(" ");@@ -288,8 +290,8 @@ wherelet mut correction = crate::web_spell::Correction::empty(query);for term in terms {- match *term {- query::parser::Term::Simple(t) => {+ match term {+ query::parser::Term::SimpleOrPhrase(query::parser::SimpleOrPhrase::Simple(t)) => {if let Some(term_correction) = correction_map.get(t.as_str()) {correction.push(crate::web_spell::CorrectionTerm::Corrected {orig: String::from(t),diff --git a/crates/core/src/searcher/api/widget.rs b/crates/core/src/searcher/api/widget.rsindex 4ee358a0..a7b2e58c 100644--- a/crates/core/src/searcher/api/widget.rs+++ b/crates/core/src/searcher/api/widget.rs@@ -29,13 +29,16 @@ impl WidgetManager {}pub async fn widget(&self, query: &str) -> Option<Widget> {- let parsed_terms = query::parser::parse(query);+ let parsed_terms = query::parser::parse(query).ok()?;self.widgets.widget(parsed_terms.into_iter().filter_map(|term| {- if let query::parser::Term::Simple(simple) = *term {+ if let query::parser::Term::SimpleOrPhrase(+ query::parser::SimpleOrPhrase::Simple(simple),+ ) = term+ {Some(String::from(simple))} else {None