Skip to content

web search done right

Nom query parser (#184)

Mikkel Denkere47b49a01228
diff --git a/Cargo.lock b/Cargo.lock
index 35d4a147..b22de01f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4561,6 +4561,7 @@ dependencies = [
"memmap2",
"mime",
"min-max-heap",
+ "nom",
"num_cpus",
"once_cell",
"openraft",
diff --git a/Cargo.toml b/Cargo.toml
index 94d7587c..f7c975af 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -67,6 +67,7 @@ memmap = "0.7.0"
memmap2 = "0.9.0"
mime = "0.3.17"
min-max-heap = "1.3.0"
+nom = "7.1.3"
num_cpus = "1.15.0"
once_cell = "1.13.1"
proptest = "1.2.0"
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 5c15455f..520c2940 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -68,6 +68,7 @@ memmap = { workspace = true }
memmap2 = { workspace = true }
mime = { workspace = true }
min-max-heap = { workspace = true }
+nom = { workspace = true }
num_cpus = { workspace = true }
once_cell = { workspace = true }
openraft = { workspace = true }
diff --git a/crates/core/src/bangs.rs b/crates/core/src/bangs.rs
index 70b3b57e..9ce07919 100644
--- a/crates/core/src/bangs.rs
+++ b/crates/core/src/bangs.rs
@@ -109,9 +109,9 @@ impl Bangs {
}
}
- pub fn get(&self, terms: &[Box<Term>]) -> Option<BangHit> {
+ pub fn get(&self, terms: &[Term]) -> Option<BangHit> {
for possible_bang in terms.iter().filter_map(|term| {
- if let Term::PossibleBang(possible_bang) = term.as_ref() {
+ if let Term::PossibleBang(possible_bang) = term {
Some(possible_bang)
} else {
None
@@ -122,13 +122,13 @@ impl Bangs {
terms
.iter()
.filter(|term| {
- if let Term::PossibleBang(bang) = term.as_ref() {
+ if let Term::PossibleBang(bang) = term {
bang != possible_bang
} else {
true
}
})
- .map(|term| term.as_ref().to_string()),
+ .map(|term| term.to_string()),
" ".to_string(),
)
.collect::<String>();
@@ -172,11 +172,11 @@ mod tests {
}]"#,
);
- assert_eq!(bangs.get(&parse("no bangs")), None);
- assert_eq!(bangs.get(&parse("!no bangs")), None);
+ assert_eq!(bangs.get(&parse("no bangs").unwrap()), None);
+ assert_eq!(bangs.get(&parse("!no bangs").unwrap()), None);
assert_eq!(
- bangs.get(&parse("!ty bangs")),
+ bangs.get(&parse("!ty bangs").unwrap()),
Some(BangHit {
bang: Bang {
category: Some("Multimedia".to_string()),
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index d769ad09..abd71c50 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -38,14 +38,16 @@ pub mod union;
use parser::Term;
-use self::{optic::AsMultipleTantivyQuery, parser::CompoundAwareTerm};
+use self::{
+ optic::AsMultipleTantivyQuery,
+ parser::{CompoundAwareTerm, SimpleOrPhrase},
+};
const MAX_SIMILAR_TERMS: usize = 10;
#[derive(Clone, Debug)]
pub struct Query {
- #[allow(clippy::vec_box)]
- terms: Vec<Box<Term>>,
+ terms: Vec<Term>,
simple_terms_text: Vec<String>,
tantivy_query: Box<BooleanQuery>,
host_rankings: HostRankings,
@@ -58,7 +60,7 @@ pub struct Query {
impl Query {
pub fn parse(ctx: &Ctx, query: &SearchQuery, index: &InvertedIndex) -> Result<Query> {
- let parsed_terms = parser::parse(&query.query);
+ let parsed_terms = parser::parse(&query.query)?;
let mut term_count = HashMap::new();
let mut terms = Vec::new();
@@ -76,7 +78,7 @@ impl Query {
.clone()
.into_iter()
.map(|term| CompoundAwareTerm {
- term: *term,
+ term,
adjacent_terms: Vec::new(),
})
.collect();
@@ -87,7 +89,9 @@ impl Query {
for window in term_ids.windows(window_size) {
let mut window_terms = Vec::new();
for i in window {
- if let Term::Simple(t) = &compound_terms[*i].term {
+ if let Term::SimpleOrPhrase(SimpleOrPhrase::Simple(t)) =
+ &compound_terms[*i].term
+ {
window_terms.push(t.clone());
}
}
@@ -180,7 +184,7 @@ impl Query {
&self.simple_terms_text
}
- pub fn terms(&self) -> &[Box<Term>] {
+ pub fn terms(&self) -> &[Term] {
&self.terms
}
diff --git a/crates/core/src/query/parser.rs b/crates/core/src/query/parser.rs
deleted file mode 100644
index 83cd8ad8..00000000
--- a/crates/core/src/query/parser.rs
+++ /dev/null
@@ -1,581 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::fmt::Display;
-use tantivy::{
- query::{BooleanQuery, Occur, PhraseQuery, TermQuery},
- tokenizer::Tokenizer,
-};
-
-use crate::{
- bangs::BANG_PREFIXES,
- floor_char_boundary,
- schema::{Field, TextField},
-};
-
-#[derive(Debug, Clone)]
-pub struct TermCompound {
- pub terms: Vec<SimpleTerm>,
-}
-
-#[derive(Debug, Clone)]
-pub struct CompoundAwareTerm {
- pub term: Term,
- pub adjacent_terms: Vec<TermCompound>,
-}
-impl CompoundAwareTerm {
- pub fn as_tantivy_query(
- &self,
- fields: &[tantivy::schema::Field],
- ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
- if !self.adjacent_terms.is_empty() {
- if let Term::Simple(simple_term) = &self.term {
- return simple_into_tantivy(simple_term, &self.adjacent_terms, fields);
- }
- }
-
- self.term.as_tantivy_query(fields)
- }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct SimpleTerm(String);
-impl SimpleTerm {
- pub fn as_str(&self) -> &str {
- self.0.as_str()
- }
-}
-
-impl From<String> for SimpleTerm {
- fn from(value: String) -> Self {
- SimpleTerm(value)
- }
-}
-
-impl From<SimpleTerm> for String {
- fn from(value: SimpleTerm) -> Self {
- value.0
- }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub enum Term {
- Simple(SimpleTerm),
- Phrase(String),
- Not(Box<Term>),
- Site(String),
- Title(String),
- Body(String),
- Url(String),
- PossibleBang(String),
-}
-
-impl Display for Term {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- match self {
- Term::Simple(term) => write!(f, "{}", term.0),
- Term::Phrase(phrase) => write!(f, "\"{}\"", phrase),
- Term::Not(term) => write!(f, "-{}", term),
- Term::Site(site) => write!(f, "site:{}", site),
- Term::Title(title) => write!(f, "intitle:{}", title),
- Term::Body(body) => write!(f, "inbody:{}", body),
- Term::Url(url) => write!(f, "inurl:{}", url),
- Term::PossibleBang(bang) => write!(f, "{}{}", BANG_PREFIXES[0], bang),
- }
- }
-}
-
-fn simple_into_tantivy(
- term: &SimpleTerm,
- adjacent_terms: &[TermCompound],
- fields: &[tantivy::schema::Field],
-) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
- let mut queries = Term::into_tantivy_simple(term, fields);
-
- let fields = fields
- .iter()
- .filter(|field| {
- matches!(
- Field::get(field.field_id() as usize),
- Some(Field::Text(TextField::AllBody))
- | Some(Field::Text(TextField::Title))
- | Some(Field::Text(TextField::Url))
- )
- })
- .copied()
- .collect::<Vec<_>>();
-
- for adjacent_term in adjacent_terms {
- let combined = adjacent_term
- .terms
- .iter()
- .map(|term| term.0.as_str())
- .collect::<String>();
-
- for field in &fields {
- queries.push((Occur::Should, Term::tantivy_text_query(field, &combined)))
- }
- }
-
- (Occur::Must, Box::new(BooleanQuery::new(queries)))
-}
-
-impl Term {
- pub fn as_simple_text(&self) -> Option<&str> {
- match self {
- Term::Simple(term) => Some(&term.0),
- Term::Phrase(p) => Some(p),
- _ => None,
- }
- }
-
- fn as_tantivy_query(
- &self,
- fields: &[tantivy::schema::Field],
- ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
- match self {
- Term::Simple(term) => simple_into_tantivy(term, &[], fields),
- Term::Phrase(phrase) => {
- let mut phrases = Vec::with_capacity(fields.len());
-
- for (field, tv_field) in fields
- .iter()
- .filter_map(|tv_field| {
- Field::get(tv_field.field_id() as usize).map(|mapped| (mapped, *tv_field))
- })
- .filter(|(field, _)| field.is_searchable())
- .filter(|(field, _)| field.has_pos())
- {
- let mut processed_terms = Term::process_tantivy_term(phrase, tv_field);
-
- if processed_terms.is_empty() {
- continue;
- }
-
- if processed_terms.len() == 1 {
- let options = field.as_text().unwrap().index_option();
-
- phrases.push((
- Occur::Should,
- Box::new(TermQuery::new(processed_terms.pop().unwrap(), options))
- as Box<dyn tantivy::query::Query>,
- ));
- } else {
- phrases.push((
- Occur::Should,
- Box::new(PhraseQuery::new(processed_terms))
- as Box<dyn tantivy::query::Query>,
- ));
- }
- }
-
- (Occur::Must, Box::new(BooleanQuery::new(phrases)))
- }
- Term::Not(subterm) => (
- Occur::MustNot,
- Box::new(BooleanQuery::new(vec![subterm.as_tantivy_query(fields)])),
- ),
- Term::Site(site) => (
- Occur::Must,
- Box::new(BooleanQuery::new(Term::into_tantivy_site(site, fields))),
- ),
- Term::Title(title) => {
- let field = fields
- .iter()
- .find(|field| {
- matches!(
- Field::get(field.field_id() as usize),
- Some(Field::Text(TextField::Title))
- )
- })
- .unwrap();
-
- (Occur::Must, Term::tantivy_text_query(field, title))
- }
- Term::Body(body) => {
- let field = fields
- .iter()
- .find(|field| {
- matches!(
- Field::get(field.field_id() as usize),
- Some(Field::Text(TextField::AllBody))
- )
- })
- .unwrap();
-
- (Occur::Must, Term::tantivy_text_query(field, body))
- }
- Term::Url(url) => {
- let field = fields
- .iter()
- .find(|field| {
- matches!(
- Field::get(field.field_id() as usize),
- Some(Field::Text(TextField::Url))
- )
- })
- .unwrap();
-
- (Occur::Must, Term::tantivy_text_query(field, url))
- }
- Term::PossibleBang(text) => {
- let mut term = String::new();
-
- term.push(BANG_PREFIXES[0]);
- term.push_str(text);
-
- simple_into_tantivy(&term.into(), &[], fields)
- }
- }
- }
-
- fn into_tantivy_simple(
- term: &SimpleTerm,
- fields: &[tantivy::schema::Field],
- ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
- fields
- .iter()
- .filter_map(|tv_field| {
- Field::get(tv_field.field_id() as usize)
- .filter(|field| field.is_searchable())
- .map(|_| tv_field)
- })
- .map(|field| (Occur::Should, Term::tantivy_text_query(field, &term.0)))
- .collect()
- }
-
- fn into_tantivy_site(
- term: &str,
- fields: &[tantivy::schema::Field],
- ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
- fields
- .iter()
- .filter(|field| {
- matches!(
- Field::get(field.field_id() as usize),
- Some(Field::Text(TextField::UrlForSiteOperator))
- )
- })
- .map(|field| {
- let processed_terms = Term::process_tantivy_term(term, *field);
-
- if processed_terms.len() == 1 {
- let term = processed_terms.first().unwrap().clone();
- (
- Occur::Should,
- Box::new(TermQuery::new(
- term,
- tantivy::schema::IndexRecordOption::Basic,
- )) as Box<dyn tantivy::query::Query>,
- )
- } else {
- (
- Occur::Should,
- Box::new(PhraseQuery::new(processed_terms))
- as Box<dyn tantivy::query::Query>,
- )
- }
- })
- .collect()
- }
-
- fn tantivy_text_query(
- field: &tantivy::schema::Field,
- term: &str,
- ) -> Box<dyn tantivy::query::Query + 'static> {
- let mut processed_terms = Term::process_tantivy_term(term, *field);
-
- let option = Field::get(field.field_id() as usize)
- .unwrap()
- .as_text()
- .unwrap()
- .index_option();
-
- let processed_query = if processed_terms.len() == 1 {
- let term = processed_terms.remove(0);
- Box::new(TermQuery::new(term, option)) as Box<dyn tantivy::query::Query + 'static>
- } else {
- Box::new(BooleanQuery::new(
- processed_terms
- .into_iter()
- .map(|term| {
- (
- Occur::Must,
- Box::new(TermQuery::new(term, option))
- as Box<dyn tantivy::query::Query + 'static>,
- )
- })
- .collect(),
- )) as Box<dyn tantivy::query::Query + 'static>
- };
-
- Box::new(processed_query)
- }
-
- fn process_tantivy_term(
- term: &str,
- tantivy_field: tantivy::schema::Field,
- ) -> Vec<tantivy::Term> {
- match Field::get(tantivy_field.field_id() as usize) {
- Some(Field::Fast(_)) => vec![tantivy::Term::from_field_text(tantivy_field, term)],
- Some(Field::Text(text_field)) => {
- let mut terms: Vec<tantivy::Term> = Vec::new();
- let mut tokenizer = text_field.query_tokenizer();
- let mut token_stream = tokenizer.token_stream(term);
- token_stream.process(&mut |token| {
- let term = tantivy::Term::from_field_text(tantivy_field, &token.text);
- terms.push(term);
- });
-
- terms
- }
- None => vec![],
- }
- }
-}
-
-fn parse_term(term: &str) -> Box<Term> {
- // TODO: re-write this entire function once if-let chains become stable
- if let Some(not_term) = term.strip_prefix('-') {
- if !not_term.is_empty() && !not_term.starts_with('-') {
- Box::new(Term::Not(parse_term(not_term)))
- } else {
- Box::new(Term::Simple(term.to_string().into()))
- }
- } else if let Some(site) = term.strip_prefix("site:") {
- if !site.is_empty() {
- Box::new(Term::Site(site.to_string()))
- } else {
- Box::new(Term::Simple(term.to_string().into()))
- }
- } else if let Some(title) = term.strip_prefix("intitle:") {
- if !title.is_empty() {
- Box::new(Term::Title(title.to_string()))
- } else {
- Box::new(Term::Simple(term.to_string().into()))
- }
- } else if let Some(body) = term.strip_prefix("inbody:") {
- if !body.is_empty() {
- Box::new(Term::Body(body.to_string()))
- } else {
- Box::new(Term::Simple(term.to_string().into()))
- }
- } else if let Some(url) = term.strip_prefix("inurl:") {
- if !url.is_empty() {
- Box::new(Term::Url(url.to_string()))
- } else {
- Box::new(Term::Simple(term.to_string().into()))
- }
- } else {
- for bang_prefix in BANG_PREFIXES {
- if let Some(bang) = term.strip_prefix(bang_prefix) {
- return Box::new(Term::PossibleBang(bang.to_string()));
- }
- }
-
- Box::new(Term::Simple(term.to_string().into()))
- }
-}
-
-#[allow(clippy::vec_box)]
-pub fn parse(query: &str) -> Vec<Box<Term>> {
- let query = query.to_lowercase().replace(['“', '”'], "\"");
-
- let mut res = Vec::new();
-
- let mut cur_term_begin = 0;
-
- for (offset, c) in query.char_indices() {
- if cur_term_begin > offset {
- continue;
- }
-
- cur_term_begin = floor_char_boundary(&query, cur_term_begin);
-
- if query[cur_term_begin..].starts_with('"') {
- if let Some(offset) = query[cur_term_begin + 1..].find('"') {
- let offset = offset + cur_term_begin + 1;
- res.push(Box::new(Term::Phrase(
- query[cur_term_begin + 1..offset].to_string(),
- )));
-
- cur_term_begin = offset + 1;
- continue;
- }
- }
- if c.is_whitespace() {
- if offset - cur_term_begin == 0 {
- cur_term_begin = offset + 1;
- continue;
- }
-
- res.push(parse_term(&query[cur_term_begin..offset]));
- cur_term_begin = offset + 1;
- }
- }
-
- if cur_term_begin < query.len() {
- res.push(parse_term(
- &query[floor_char_boundary(&query, cur_term_begin)..query.len()],
- ));
- }
-
- res
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use proptest::prelude::*;
-
- #[test]
- fn parse_not() {
- assert_eq!(
- parse("this -that"),
- vec![
- Box::new(Term::Simple("this".to_string().into())),
- Box::new(Term::Not(Box::new(Term::Simple("that".to_string().into()))))
- ]
- );
-
- assert_eq!(
- parse("this -"),
- vec![
- Box::new(Term::Simple("this".to_string().into())),
- Box::new(Term::Simple("-".to_string().into()))
- ]
- );
- }
-
- #[test]
- fn double_not() {
- assert_eq!(
- parse("this --that"),
- vec![
- Box::new(Term::Simple("this".to_string().into())),
- Box::new(Term::Simple("--that".to_string().into()))
- ]
- );
- }
-
- #[test]
- fn site() {
- assert_eq!(
- parse("this site:test.com"),
- vec![
- Box::new(Term::Simple("this".to_string().into())),
- Box::new(Term::Site("test.com".to_string()))
- ]
- );
- }
-
- #[test]
- fn title() {
- assert_eq!(
- parse("this intitle:test"),
- vec![
- Box::new(Term::Simple("this".to_string().into())),
- Box::new(Term::Title("test".to_string()))
- ]
- );
- }
-
- #[test]
- fn body() {
- assert_eq!(
- parse("this inbody:test"),
- vec![
- Box::new(Term::Simple("this".to_string().into())),
- Box::new(Term::Body("test".to_string()))
- ]
- );
- }
-
- #[test]
- fn url() {
- assert_eq!(
- parse("this inurl:test"),
- vec![
- Box::new(Term::Simple("this".to_string().into())),
- Box::new(Term::Url("test".to_string()))
- ]
- );
- }
-
- #[test]
- fn empty() {
- assert_eq!(parse(""), vec![]);
- }
-
- #[test]
- fn phrase() {
- assert_eq!(
- parse("\"this is a\" inurl:test"),
- vec![
- Box::new(Term::Phrase("this is a".to_string(),)),
- Box::new(Term::Url("test".to_string()))
- ]
- );
- assert_eq!(
- parse("\"this is a inurl:test"),
- vec![
- Box::new(Term::Simple("\"this".to_string().into())),
- Box::new(Term::Simple("is".to_string().into())),
- Box::new(Term::Simple("a".to_string().into())),
- Box::new(Term::Url("test".to_string()))
- ]
- );
- assert_eq!(
- parse("this is a\" inurl:test"),
- vec![
- Box::new(Term::Simple("this".to_string().into())),
- Box::new(Term::Simple("is".to_string().into())),
- Box::new(Term::Simple("a\"".to_string().into())),
- Box::new(Term::Url("test".to_string()))
- ]
- );
-
- assert_eq!(
- parse("\"this is a inurl:test\""),
- vec![Box::new(Term::Phrase("this is a inurl:test".to_string(),)),]
- );
-
- assert_eq!(
- parse("\"\""),
- vec![Box::new(Term::Phrase("".to_string(),)),]
- );
- assert_eq!(
- parse("“this is a“ inurl:test"),
- vec![
- Box::new(Term::Phrase("this is a".to_string(),)),
- Box::new(Term::Url("test".to_string()))
- ]
- );
- }
-
- #[test]
- fn unicode() {
- let query = "\u{a0}";
- assert_eq!(parse(query).len(), 1);
- }
-
- proptest! {
- #[test]
- fn prop(query: String) {
- parse(&query);
- }
- }
-}
diff --git a/crates/core/src/query/parser/as_tantivy.rs b/crates/core/src/query/parser/as_tantivy.rs
new file mode 100644
index 00000000..efa90e91
--- /dev/null
+++ b/crates/core/src/query/parser/as_tantivy.rs
@@ -0,0 +1,304 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::{
+ query::{BooleanQuery, Occur, PhraseQuery, TermQuery},
+ tokenizer::Tokenizer as _,
+};
+
+use crate::{
+ bangs::BANG_PREFIXES,
+ schema::{Field, TextField},
+};
+
+use super::{CompoundAwareTerm, SimpleOrPhrase, SimpleTerm, Term, TermCompound};
+
+impl CompoundAwareTerm {
+ pub fn as_tantivy_query(
+ &self,
+ fields: &[tantivy::schema::Field],
+ ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
+ if !self.adjacent_terms.is_empty() {
+ if let Term::SimpleOrPhrase(SimpleOrPhrase::Simple(simple_term)) = &self.term {
+ return simple_into_tantivy(simple_term, &self.adjacent_terms, fields);
+ }
+ }
+
+ self.term.as_tantivy_query(fields)
+ }
+}
+
+fn simple_into_tantivy(
+ term: &SimpleTerm,
+ adjacent_terms: &[TermCompound],
+ fields: &[tantivy::schema::Field],
+) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
+ let mut queries = Term::into_tantivy_simple(term, fields);
+
+ let fields = fields
+ .iter()
+ .filter(|field| {
+ matches!(
+ Field::get(field.field_id() as usize),
+ Some(Field::Text(TextField::AllBody))
+ | Some(Field::Text(TextField::Title))
+ | Some(Field::Text(TextField::Url))
+ )
+ })
+ .copied()
+ .collect::<Vec<_>>();
+
+ for adjacent_term in adjacent_terms {
+ let combined = adjacent_term
+ .terms
+ .iter()
+ .map(|term| term.as_str())
+ .collect::<String>();
+
+ for field in &fields {
+ queries.push((Occur::Should, Term::tantivy_text_query(field, &combined)))
+ }
+ }
+
+ (Occur::Must, Box::new(BooleanQuery::new(queries)))
+}
+
+fn phrase_query(
+ terms: &[String],
+ fields: &[tantivy::schema::Field],
+) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
+ let mut phrases = Vec::with_capacity(fields.len());
+ let phrase = terms.join(" ");
+
+ for (field, tv_field) in fields
+ .iter()
+ .filter_map(|tv_field| {
+ Field::get(tv_field.field_id() as usize).map(|mapped| (mapped, *tv_field))
+ })
+ .filter(|(field, _)| field.is_searchable())
+ .filter(|(field, _)| field.has_pos())
+ {
+ let mut processed_terms = Term::process_tantivy_term(&phrase, tv_field);
+
+ if processed_terms.is_empty() {
+ continue;
+ }
+
+ if processed_terms.len() == 1 {
+ let options = field.as_text().unwrap().index_option();
+
+ phrases.push((
+ Occur::Should,
+ Box::new(TermQuery::new(processed_terms.pop().unwrap(), options))
+ as Box<dyn tantivy::query::Query>,
+ ));
+ } else {
+ phrases.push((
+ Occur::Should,
+ Box::new(PhraseQuery::new(processed_terms)) as Box<dyn tantivy::query::Query>,
+ ));
+ }
+ }
+
+ (Occur::Must, Box::new(BooleanQuery::new(phrases)))
+}
+
+impl Term {
+ fn as_tantivy_query(
+ &self,
+ fields: &[tantivy::schema::Field],
+ ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
+ match self {
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple(term)) => {
+ simple_into_tantivy(term, &[], fields)
+ }
+ Term::SimpleOrPhrase(SimpleOrPhrase::Phrase(phrase)) => phrase_query(phrase, fields),
+ Term::Not(subterm) => (
+ Occur::MustNot,
+ Box::new(BooleanQuery::new(vec![subterm.as_tantivy_query(fields)])),
+ ),
+ Term::Site(site) => (
+ Occur::Must,
+ Box::new(BooleanQuery::new(Term::into_tantivy_site(site, fields))),
+ ),
+ Term::Title(title) => {
+ let field = fields
+ .iter()
+ .find(|field| {
+ matches!(
+ Field::get(field.field_id() as usize),
+ Some(Field::Text(TextField::Title))
+ )
+ })
+ .unwrap();
+
+ (
+ Occur::Must,
+ Term::tantivy_text_query(field, &title.as_string()),
+ )
+ }
+ Term::Body(body) => {
+ let field = fields
+ .iter()
+ .find(|field| {
+ matches!(
+ Field::get(field.field_id() as usize),
+ Some(Field::Text(TextField::AllBody))
+ )
+ })
+ .unwrap();
+
+ (
+ Occur::Must,
+ Term::tantivy_text_query(field, &body.as_string()),
+ )
+ }
+ Term::Url(url) => {
+ let field = fields
+ .iter()
+ .find(|field| {
+ matches!(
+ Field::get(field.field_id() as usize),
+ Some(Field::Text(TextField::Url))
+ )
+ })
+ .unwrap();
+
+ (
+ Occur::Must,
+ Term::tantivy_text_query(field, &url.as_string()),
+ )
+ }
+ Term::PossibleBang(text) => {
+ let mut term = String::new();
+
+ term.push(BANG_PREFIXES[0]);
+ term.push_str(text);
+
+ simple_into_tantivy(&term.into(), &[], fields)
+ }
+ }
+ }
+
+ fn into_tantivy_simple(
+ term: &SimpleTerm,
+ fields: &[tantivy::schema::Field],
+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
+ fields
+ .iter()
+ .filter_map(|tv_field| {
+ Field::get(tv_field.field_id() as usize)
+ .filter(|field| field.is_searchable())
+ .map(|_| tv_field)
+ })
+ .map(|field| {
+ (
+ Occur::Should,
+ Term::tantivy_text_query(field, term.as_str()),
+ )
+ })
+ .collect()
+ }
+
+ fn into_tantivy_site(
+ term: &str,
+ fields: &[tantivy::schema::Field],
+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
+ fields
+ .iter()
+ .filter(|field| {
+ matches!(
+ Field::get(field.field_id() as usize),
+ Some(Field::Text(TextField::UrlForSiteOperator))
+ )
+ })
+ .map(|field| {
+ let processed_terms = Term::process_tantivy_term(term, *field);
+
+ if processed_terms.len() == 1 {
+ let term = processed_terms.first().unwrap().clone();
+ (
+ Occur::Should,
+ Box::new(TermQuery::new(
+ term,
+ tantivy::schema::IndexRecordOption::Basic,
+ )) as Box<dyn tantivy::query::Query>,
+ )
+ } else {
+ (
+ Occur::Should,
+ Box::new(PhraseQuery::new(processed_terms))
+ as Box<dyn tantivy::query::Query>,
+ )
+ }
+ })
+ .collect()
+ }
+
+ fn tantivy_text_query(
+ field: &tantivy::schema::Field,
+ term: &str,
+ ) -> Box<dyn tantivy::query::Query + 'static> {
+ let mut processed_terms = Term::process_tantivy_term(term, *field);
+
+ let option = Field::get(field.field_id() as usize)
+ .unwrap()
+ .as_text()
+ .unwrap()
+ .index_option();
+
+ let processed_query = if processed_terms.len() == 1 {
+ let term = processed_terms.remove(0);
+ Box::new(TermQuery::new(term, option)) as Box<dyn tantivy::query::Query + 'static>
+ } else {
+ Box::new(BooleanQuery::new(
+ processed_terms
+ .into_iter()
+ .map(|term| {
+ (
+ Occur::Must,
+ Box::new(TermQuery::new(term, option))
+ as Box<dyn tantivy::query::Query + 'static>,
+ )
+ })
+ .collect(),
+ )) as Box<dyn tantivy::query::Query + 'static>
+ };
+
+ Box::new(processed_query)
+ }
+
+ fn process_tantivy_term(
+ term: &str,
+ tantivy_field: tantivy::schema::Field,
+ ) -> Vec<tantivy::Term> {
+ match Field::get(tantivy_field.field_id() as usize) {
+ Some(Field::Fast(_)) => vec![tantivy::Term::from_field_text(tantivy_field, term)],
+ Some(Field::Text(text_field)) => {
+ let mut terms: Vec<tantivy::Term> = Vec::new();
+ let mut tokenizer = text_field.query_tokenizer();
+ let mut token_stream = tokenizer.token_stream(term);
+ token_stream.process(&mut |token| {
+ let term = tantivy::Term::from_field_text(tantivy_field, &token.text);
+ terms.push(term);
+ });
+
+ terms
+ }
+ None => vec![],
+ }
+ }
+}
diff --git a/crates/core/src/query/parser/mod.rs b/crates/core/src/query/parser/mod.rs
new file mode 100644
index 00000000..8968ea94
--- /dev/null
+++ b/crates/core/src/query/parser/mod.rs
@@ -0,0 +1,376 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::bangs::BANG_PREFIXES;
+
+mod as_tantivy;
+mod term;
+
+pub use term::*;
+
+fn trim_leading_whitespace(input: &str) -> nom::IResult<&str, &str> {
+ let (input, _) = nom::character::complete::multispace0(input)?;
+ Ok((input, input))
+}
+
+fn until_space_or_end(input: &str) -> nom::IResult<&str, &str> {
+ let (input, output) = nom::bytes::complete::take_while(|c: char| c != ' ')(input)?;
+ Ok((input, output))
+}
+
+fn simple_str(input: &str) -> nom::IResult<&str, &str> {
+ // parse until we find a space or the end of the string
+ if input.is_empty() {
+ return Err(nom::Err::Error(nom::error::Error::new(
+ input,
+ nom::error::ErrorKind::Eof,
+ )));
+ }
+ let (input, _) = trim_leading_whitespace(input)?;
+ let (input, output) = until_space_or_end(input)?;
+ Ok((input, output))
+}
+
+fn simple(input: &str) -> nom::IResult<&str, SimpleOrPhrase> {
+ let (input, output) = simple_str(input)?;
+ Ok((input, SimpleOrPhrase::Simple(output.to_string().into())))
+}
+
+fn simple_term(input: &str) -> nom::IResult<&str, Term> {
+ let (input, output) = simple(input)?;
+ Ok((input, Term::SimpleOrPhrase(output)))
+}
+
+fn phrase_helper(
+ input: &str,
+ start_quote: char,
+ end_quote: char,
+) -> nom::IResult<&str, SimpleOrPhrase> {
+ let (input, _) = nom::character::complete::char(start_quote)(input)?;
+ let (input, output) = nom::bytes::complete::take_until(end_quote.to_string().as_str())(input)?;
+ let (input, _) = nom::character::complete::char(end_quote)(input)?;
+ Ok((
+ input,
+ SimpleOrPhrase::Phrase(output.split_whitespace().map(|s| s.to_string()).collect()),
+ ))
+}
+
+fn phrase(input: &str) -> nom::IResult<&str, SimpleOrPhrase> {
+ let normal = |s| phrase_helper(s, '"', '"');
+ let apple1 = |s| phrase_helper(s, '“', '”');
+ let apple2 = |s| phrase_helper(s, '“', '“');
+
+ nom::branch::alt((normal, apple1, apple2))(input)
+}
+
+fn phrase_term(input: &str) -> nom::IResult<&str, Term> {
+ let (input, output) = phrase(input)?;
+ Ok((input, Term::SimpleOrPhrase(output)))
+}
+
+fn simple_or_phrase(input: &str) -> nom::IResult<&str, SimpleOrPhrase> {
+ nom::branch::alt((phrase, simple))(input)
+}
+
+fn single_bang(input: &str, pref: char) -> nom::IResult<&str, Term> {
+ let (input, _) = nom::character::complete::char(pref)(input)?;
+ let (input, output) = until_space_or_end(input)?;
+ Ok((input, Term::PossibleBang(output.to_string())))
+}
+
+fn bang(input: &str) -> nom::IResult<&str, Term> {
+ for pref in BANG_PREFIXES.iter() {
+ let (input, output) = single_bang(input, *pref)?;
+ return Ok((input, output));
+ }
+
+ Err(nom::Err::Error(nom::error::Error::new(
+ input,
+ nom::error::ErrorKind::Fail,
+ )))
+}
+
+fn site_field(input: &str) -> nom::IResult<&str, Term> {
+ // parse 'site:' and then a simple term
+ let (input, _) = nom::bytes::complete::tag("site:")(input)?;
+ let (input, output) = simple_str(input)?;
+
+ Ok((input, Term::Site(output.to_string())))
+}
+
+fn title_field(input: &str) -> nom::IResult<&str, Term> {
+ // parse 'title:' and then a simple term
+ let (input, _) = nom::bytes::complete::tag("intitle:")(input)?;
+ let (input, output) = simple_or_phrase(input)?;
+
+ Ok((input, Term::Title(output)))
+}
+
+fn body_field(input: &str) -> nom::IResult<&str, Term> {
+ // parse 'body:' and then a simple term
+ let (input, _) = nom::bytes::complete::tag("inbody:")(input)?;
+ let (input, output) = simple_or_phrase(input)?;
+
+ Ok((input, Term::Body(output)))
+}
+
+fn url_field(input: &str) -> nom::IResult<&str, Term> {
+ // parse 'url:' and then a simple term
+ let (input, _) = nom::bytes::complete::tag("inurl:")(input)?;
+ let (input, output) = simple_or_phrase(input)?;
+
+ Ok((input, Term::Url(output)))
+}
+
+fn field_selector(input: &str) -> nom::IResult<&str, Term> {
+ nom::branch::alt((site_field, title_field, body_field, url_field))(input)
+}
+
+fn not(input: &str) -> nom::IResult<&str, Term> {
+ // ignore double negation
+ if let Ok((_, _)) = nom::bytes::complete::tag::<_, _, nom::error::Error<&str>>("--")(input) {
+ return Err(nom::Err::Error(nom::error::Error::new(
+ input,
+ nom::error::ErrorKind::Fail,
+ )));
+ }
+
+ let (input, _) = nom::bytes::complete::tag("-")(input)?;
+ let (input, output) = term(input)?;
+ Ok((input, Term::Not(Box::new(output))))
+}
+
+fn ignore_quotes_helper(
+ input: &str,
+ start_quote: char,
+ end_quote: char,
+) -> nom::IResult<&str, &str> {
+ let (input, _) = nom::character::complete::char(start_quote)(input)?;
+ let (input, output) = nom::bytes::complete::take_until(end_quote.to_string().as_str())(input)?;
+ let (input, _) = nom::character::complete::char(end_quote)(input)?;
+ Ok((input, output))
+}
+
+fn ignore_weird_quotes(input: &str) -> nom::IResult<&str, &str> {
+ let guillemet = |s| ignore_quotes_helper(s, '«', '»');
+ let up_down_quotes = |s| ignore_quotes_helper(s, '„', '“');
+ let rev_guillemet = |s| ignore_quotes_helper(s, '»', '«');
+ let squares = |s| ignore_quotes_helper(s, '「', '」');
+
+ nom::branch::alt((guillemet, up_down_quotes, rev_guillemet, squares))(input)
+}
+
+fn term(input: &str) -> nom::IResult<&str, Term> {
+ let (mut input, _) = trim_leading_whitespace(input)?;
+
+ if let Ok((_, new_input)) = ignore_weird_quotes(input) {
+ input = new_input;
+ }
+
+ nom::branch::alt((phrase_term, bang, field_selector, not, simple_term))(input)
+}
+
+pub fn parse(query: &str) -> anyhow::Result<Vec<Term>> {
+ if query.is_empty() || query.chars().all(char::is_whitespace) {
+ return Ok(vec![]);
+ }
+
+ nom::multi::many1(term)(query)
+ .map(|(_, res)| res)
+ .map_err(|e| anyhow::anyhow!("Failed to parse query: {:?}", e))
+}
+
+#[cfg(test)]
+mod tests {
+ use proptest::prelude::*;
+
+ use super::{SimpleOrPhrase, Term};
+
+ fn parse(input: &str) -> Vec<Term> {
+ super::parse(input).unwrap()
+ }
+
+ #[test]
+ fn parse_not() {
+ assert_eq!(
+ parse("this -that"),
+ vec![
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),
+ Term::Not(Box::new(Term::SimpleOrPhrase(SimpleOrPhrase::Simple(
+ "that".to_string().into()
+ ))))
+ ]
+ );
+
+ assert_eq!(
+ parse("this -"),
+ vec![
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("-".to_string().into()))
+ ]
+ );
+ }
+
+ #[test]
+ fn double_not() {
+ assert_eq!(
+ parse("this --that"),
+ vec![
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("--that".to_string().into()))
+ ]
+ );
+ }
+
+ #[test]
+ fn site() {
+ assert_eq!(
+ parse("this site:test.com"),
+ vec![
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),
+ Term::Site("test.com".to_string())
+ ]
+ );
+ }
+
+ #[test]
+ fn title() {
+ assert_eq!(
+ parse("this intitle:test"),
+ vec![
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),
+ Term::Title(SimpleOrPhrase::Simple("test".to_string().into()))
+ ]
+ );
+ }
+
+ #[test]
+ fn body() {
+ assert_eq!(
+ parse("this inbody:test"),
+ vec![
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),
+ Term::Body(SimpleOrPhrase::Simple("test".to_string().into()))
+ ]
+ );
+ }
+
+ #[test]
+ fn url() {
+ assert_eq!(
+ parse("this inurl:test"),
+ vec![
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),
+ Term::Url(SimpleOrPhrase::Simple("test".to_string().into()))
+ ]
+ );
+ }
+
+ #[test]
+ fn empty() {
+ assert_eq!(parse(""), vec![]);
+ assert_eq!(parse(" "), vec![]);
+ }
+
+ #[test]
+ fn phrase() {
+ assert_eq!(
+ parse("\"this is a\" inurl:test"),
+ vec![
+ Term::SimpleOrPhrase(SimpleOrPhrase::Phrase(vec![
+ "this".to_string(),
+ "is".to_string(),
+ "a".to_string()
+ ])),
+ Term::Url(SimpleOrPhrase::Simple("test".to_string().into()))
+ ]
+ );
+ assert_eq!(
+ parse("\"this is a inurl:test"),
+ vec![
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("\"this".to_string().into())),
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("is".to_string().into())),
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("a".to_string().into())),
+ Term::Url(SimpleOrPhrase::Simple("test".to_string().into()))
+ ]
+ );
+ assert_eq!(
+ parse("this is a\" inurl:test"),
+ vec![
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("this".to_string().into())),
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("is".to_string().into())),
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("a\"".to_string().into())),
+ Term::Url(SimpleOrPhrase::Simple("test".to_string().into()))
+ ]
+ );
+
+ assert_eq!(
+ parse("\"this is a inurl:test\""),
+ vec![Term::SimpleOrPhrase(SimpleOrPhrase::Phrase(vec![
+ "this".to_string(),
+ "is".to_string(),
+ "a".to_string(),
+ "inurl:test".to_string()
+ ]))]
+ );
+
+ assert_eq!(
+ parse("\"\""),
+ vec![Term::SimpleOrPhrase(SimpleOrPhrase::Phrase(vec![]))]
+ );
+ assert_eq!(
+ parse("“this is a“ inurl:test"),
+ vec![
+ Term::SimpleOrPhrase(SimpleOrPhrase::Phrase(vec![
+ "this".to_string(),
+ "is".to_string(),
+ "a".to_string()
+ ])),
+ Term::Url(SimpleOrPhrase::Simple("test".to_string().into()))
+ ]
+ );
+
+ assert_eq!(
+ parse("that's not"),
+ vec![
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("that's".to_string().into())),
+ Term::SimpleOrPhrase(SimpleOrPhrase::Simple("not".to_string().into()))
+ ]
+ );
+
+ assert_eq!(
+ parse("inbody:\"this should work\""),
+ vec![Term::Body(SimpleOrPhrase::Phrase(vec![
+ "this".to_string(),
+ "should".to_string(),
+ "work".to_string()
+ ]))]
+ );
+ }
+
+ #[test]
+ fn unicode() {
+ let query = "🦀";
+ assert_eq!(parse(query).len(), 1);
+ }
+
+ proptest! {
+ #[test]
+ fn prop(query: String) {
+ parse(&query);
+ }
+ }
+}
diff --git a/crates/core/src/query/parser/term.rs b/crates/core/src/query/parser/term.rs
new file mode 100644
index 00000000..ac1b5a09
--- /dev/null
+++ b/crates/core/src/query/parser/term.rs
@@ -0,0 +1,105 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use crate::bangs::BANG_PREFIXES;
+
+#[derive(Debug, Clone)]
+pub struct TermCompound {
+ pub terms: Vec<SimpleTerm>,
+}
+
+#[derive(Debug, Clone)]
+pub struct CompoundAwareTerm {
+ pub term: Term,
+ pub adjacent_terms: Vec<TermCompound>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum SimpleOrPhrase {
+ Simple(SimpleTerm),
+ Phrase(Vec<String>),
+}
+
+impl SimpleOrPhrase {
+ pub fn as_string(&self) -> String {
+ match self {
+ SimpleOrPhrase::Simple(simple) => simple.as_str().to_string(),
+ SimpleOrPhrase::Phrase(phrase) => phrase.join(" "),
+ }
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct SimpleTerm(String);
+impl SimpleTerm {
+ pub fn as_str(&self) -> &str {
+ self.0.as_str()
+ }
+}
+
+impl From<String> for SimpleTerm {
+ fn from(value: String) -> Self {
+ SimpleTerm(value)
+ }
+}
+
+impl From<SimpleTerm> for String {
+ fn from(value: SimpleTerm) -> Self {
+ value.0
+ }
+}
+impl std::fmt::Display for SimpleOrPhrase {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ SimpleOrPhrase::Simple(s) => write!(f, "{}", s.as_str()),
+ SimpleOrPhrase::Phrase(p) => write!(f, "\"{}\"", p.join(" ")),
+ }
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum Term {
+ SimpleOrPhrase(SimpleOrPhrase),
+ Site(String),
+ Title(SimpleOrPhrase),
+ Body(SimpleOrPhrase),
+ Url(SimpleOrPhrase),
+ PossibleBang(String),
+ Not(Box<Term>),
+}
+
+impl std::fmt::Display for Term {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Term::SimpleOrPhrase(term) => write!(f, "{}", term),
+ Term::Not(term) => write!(f, "-{}", term),
+ Term::Site(site) => write!(f, "site:{}", site),
+ Term::Title(title) => write!(f, "intitle:{}", title),
+ Term::Body(body) => write!(f, "inbody:{}", body),
+ Term::Url(url) => write!(f, "inurl:{}", url),
+ Term::PossibleBang(bang) => write!(f, "{}{}", BANG_PREFIXES[0], bang),
+ }
+ }
+}
+
+impl Term {
+ pub fn as_simple_text(&self) -> Option<String> {
+ match self {
+ Term::SimpleOrPhrase(term) => Some(term.as_string()),
+ _ => None,
+ }
+ }
+}
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index 9ad8846a..00e9e809 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -210,16 +210,16 @@ where
}
async fn check_bangs(&self, query: &SearchQuery) -> Result<Option<BangHit>> {
- let parsed_terms = query::parser::parse(&query.query);
+ let parsed_terms = query::parser::parse(&query.query)?;
- if parsed_terms.iter().any(|term| match term.as_ref() {
+ if parsed_terms.iter().any(|term| match term {
query::parser::Term::PossibleBang(t) => t.is_empty(),
_ => false,
}) {
let q: String = intersperse(
parsed_terms
.iter()
- .filter(|term| !matches!(term.as_ref(), query::parser::Term::PossibleBang(_)))
+ .filter(|term| !matches!(term, query::parser::Term::PossibleBang(_)))
.map(|term| term.to_string()),
" ".to_string(),
)
@@ -258,13 +258,15 @@ where
pub fn spell_check(&self, query: &str) -> Option<HighlightedSpellCorrection> {
let query = query.to_lowercase();
- let terms = query::parser::parse(&query);
+ let terms = query::parser::parse(&query).ok()?;
let simple_query = terms
.clone()
.into_iter()
- .filter_map(|term| match *term {
- query::parser::Term::Simple(t) => Some(String::from(t)),
+ .filter_map(|term| match term {
+ query::parser::Term::SimpleOrPhrase(query::parser::SimpleOrPhrase::Simple(t)) => {
+ Some(String::from(t))
+ }
_ => None,
})
.join(" ");
@@ -288,8 +290,8 @@ where
let mut correction = crate::web_spell::Correction::empty(query);
for term in terms {
- match *term {
- query::parser::Term::Simple(t) => {
+ match term {
+ query::parser::Term::SimpleOrPhrase(query::parser::SimpleOrPhrase::Simple(t)) => {
if let Some(term_correction) = correction_map.get(t.as_str()) {
correction.push(crate::web_spell::CorrectionTerm::Corrected {
orig: String::from(t),
diff --git a/crates/core/src/searcher/api/widget.rs b/crates/core/src/searcher/api/widget.rs
index 4ee358a0..a7b2e58c 100644
--- a/crates/core/src/searcher/api/widget.rs
+++ b/crates/core/src/searcher/api/widget.rs
@@ -29,13 +29,16 @@ impl WidgetManager {
}
pub async fn widget(&self, query: &str) -> Option<Widget> {
- let parsed_terms = query::parser::parse(query);
+ let parsed_terms = query::parser::parse(query).ok()?;
self.widgets.widget(
parsed_terms
.into_iter()
.filter_map(|term| {
- if let query::parser::Term::Simple(simple) = *term {
+ if let query::parser::Term::SimpleOrPhrase(
+ query::parser::SimpleOrPhrase::Simple(simple),
+ ) = term
+ {
Some(String::from(simple))
} else {
None