Skip to content

web search done right

Phrase query

Mikkel Denker7408ebf505db
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 34382625..2000b37f 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -91,4 +91,4 @@ harness = false
[[bench]]
name = "online-harmonic"
-harness = false
+harness = false
\ No newline at end of file
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 5349b34f..a688eb7d 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -220,6 +220,9 @@ pub enum Error {
#[error("Query cannot be completely empty")]
EmptyQuery,
+ #[error("Error parsing the query")]
+ QueryParser,
+
#[error("Unknown region")]
UnknownRegion,
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index 666d9363..b371235f 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -229,6 +229,7 @@ impl tantivy::query::Query for Query {
mod tests {
use crate::{
index::Index,
+ rand_words,
schema::create_schema,
searcher::{LocalSearcher, SearchQuery},
webpage::Webpage,
@@ -574,31 +575,37 @@ mod tests {
index
.insert(Webpage::new(
- r#"
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
- This is a test website
+ This is a test website {}
</body>
</html>
"#,
+ rand_words(1000)
+ ),
"https://www.the-first.com",
))
.expect("failed to insert webpage");
index
.insert(Webpage::new(
- r#"
+ &format!(
+ r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
- This test page does not contain the forbidden word
+ This test page does not contain the forbidden word {}
</body>
</html>
"#,
+ rand_words(1000)
+ ),
"https://www.second.com",
))
.expect("failed to insert webpage");
@@ -631,4 +638,65 @@ mod tests {
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.the-first.com");
}
+
+ #[test]
+ fn phrase_query() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ This is a test website {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "https://www.first.com",
+ ))
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test test</title>
+ </head>
+ <body>
+ This is a bad test website {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "https://www.second.com",
+ ))
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::from(index);
+
+ let query = SearchQuery {
+ query: "\"This is a test website\"".to_string(),
+ ..Default::default()
+ };
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.first.com");
+
+ let query = SearchQuery {
+ query: "\"This is a test website\" site:www.second.com".to_string(),
+ ..Default::default()
+ };
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 0);
+ assert_eq!(result.webpages.len(), 0);
+ }
}
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index f2d7c4a3..7a539645 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -29,6 +29,7 @@ use crate::{
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Term {
Simple(String),
+ Phrase(String),
Not(Box<Term>),
Site(String),
Title(String),
@@ -41,6 +42,7 @@ impl ToString for Term {
fn to_string(&self) -> String {
match self {
Term::Simple(term) => term.clone(),
+ Term::Phrase(phrase) => "\"".to_string() + &phrase + "\"",
Term::Not(term) => "-".to_string() + term.to_string().as_str(),
Term::Site(site) => "site:".to_string() + site.as_str(),
Term::Title(title) => "intitle:".to_string() + title.as_str(),
@@ -79,7 +81,7 @@ fn simple_into_tantivy(
),
(
Occur::Should,
- Box::new(Term::tantivy_term_query(
+ Box::new(Term::tantivy_text_query(
backlink_field,
backlink_field_entry,
tokenizer_manager,
@@ -99,6 +101,27 @@ impl Term {
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
match self {
Term::Simple(term) => simple_into_tantivy(term, fields, tokenizer_manager, field_boost),
+ Term::Phrase(phrase) => {
+ let mut phrases = Vec::with_capacity(fields.len());
+
+ for (field, entry) in fields
+ .iter()
+ .filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].is_searchable())
+ {
+ phrases.push((
+ Occur::Should,
+ Term::tantivy_text_query(
+ field,
+ entry,
+ tokenizer_manager,
+ field_boost,
+ phrase,
+ ),
+ ));
+ }
+
+ vec![(Occur::Must, Box::new(BooleanQuery::new(phrases)))]
+ }
Term::Not(subterm) => vec![(
Occur::MustNot,
Box::new(BooleanQuery::new(subterm.as_tantivy_query(
@@ -126,9 +149,10 @@ impl Term {
)
})
.unwrap();
+
vec![(
Occur::Must,
- Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, title),
+ Term::tantivy_text_query(field, entry, tokenizer_manager, field_boost, title),
)]
}
Term::Body(body) => {
@@ -141,9 +165,10 @@ impl Term {
)
})
.unwrap();
+
vec![(
Occur::Must,
- Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, body),
+ Term::tantivy_text_query(field, entry, tokenizer_manager, field_boost, body),
)]
}
Term::Url(url) => {
@@ -156,13 +181,15 @@ impl Term {
)
})
.unwrap();
+
vec![(
Occur::Must,
- Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, url),
+ Term::tantivy_text_query(field, entry, tokenizer_manager, field_boost, url),
)]
}
Term::PossibleBang(text) => {
let mut term = String::new();
+
term.push(BANG_PREFIX);
term.push_str(text);
@@ -184,7 +211,7 @@ impl Term {
.map(|(field, entry)| {
(
Occur::Should,
- Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, term),
+ Term::tantivy_text_query(field, entry, tokenizer_manager, field_boost, term),
)
})
.collect()
@@ -209,13 +236,13 @@ impl Term {
.map(|(field, entry)| {
(
Occur::Should,
- Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, term),
+ Term::tantivy_text_query(field, entry, tokenizer_manager, field_boost, term),
)
})
.collect()
}
- fn tantivy_term_query(
+ fn tantivy_text_query(
field: &tantivy::schema::Field,
entry: &tantivy::schema::FieldEntry,
tokenizer_manager: &TokenizerManager,
@@ -318,7 +345,43 @@ fn parse_term(term: &str) -> Box<Term> {
#[allow(clippy::vec_box)]
pub fn parse(query: &str) -> Vec<Box<Term>> {
- query.split_whitespace().map(parse_term).collect()
+ let mut res = Vec::new();
+
+ let mut chars = query.char_indices();
+ let mut cur_term_begin = 0;
+
+ while let Some((offset, c)) = chars.next() {
+ if cur_term_begin > offset {
+ continue;
+ }
+
+ if query[cur_term_begin..].starts_with('"') {
+ if let Some(offset) = query[cur_term_begin + 1..].find('"') {
+ let offset = offset + cur_term_begin + 1;
+ res.push(Box::new(Term::Phrase(
+ query[cur_term_begin + 1..offset].to_string(),
+ )));
+
+ cur_term_begin = offset + 1;
+ continue;
+ }
+ }
+ if c.is_whitespace() {
+ if offset - cur_term_begin == 0 {
+ cur_term_begin = offset + 1;
+ continue;
+ }
+
+ res.push(parse_term(&query[cur_term_begin..offset]));
+ cur_term_begin = offset + 1;
+ }
+ }
+
+ if cur_term_begin < query.len() {
+ res.push(parse_term(&query[cur_term_begin..query.len()]));
+ }
+
+ res
}
#[cfg(test)]
@@ -398,4 +461,53 @@ mod tests {
]
);
}
+
+ #[test]
+ fn empty() {
+ assert_eq!(parse(""), vec![]);
+ }
+
+ #[test]
+ fn phrase() {
+ assert_eq!(
+ parse("\"this is a\" inurl:test"),
+ vec![
+ // Box::new(Term::Phrase(vec![
+ // "this".to_string(),
+ // "is".to_string(),
+ // "a".to_string()
+ // ])),
+ Box::new(Term::Phrase("this is a".to_string(),)),
+ Box::new(Term::Url("test".to_string()))
+ ]
+ );
+ assert_eq!(
+ parse("\"this is a inurl:test"),
+ vec![
+ Box::new(Term::Simple("\"this".to_string())),
+ Box::new(Term::Simple("is".to_string())),
+ Box::new(Term::Simple("a".to_string())),
+ Box::new(Term::Url("test".to_string()))
+ ]
+ );
+ assert_eq!(
+ parse("this is a\" inurl:test"),
+ vec![
+ Box::new(Term::Simple("this".to_string())),
+ Box::new(Term::Simple("is".to_string())),
+ Box::new(Term::Simple("a\"".to_string())),
+ Box::new(Term::Url("test".to_string()))
+ ]
+ );
+
+ assert_eq!(
+ parse("\"this is a inurl:test\""),
+ vec![Box::new(Term::Phrase("this is a inurl:test".to_string(),)),]
+ );
+
+ assert_eq!(
+ parse("\"\""),
+ vec![Box::new(Term::Phrase("".to_string(),)),]
+ );
+ }
}