Skip to content

web search done right

match compound terms when the query is spelled as multiple words but where the word should be spelled as a compound word

Mikkel Denkera7086ed12f28
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index cfbd625e..bfc193fb 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -16,6 +16,7 @@
use crate::{
inverted_index::InvertedIndex,
+ query::parser::TermCompound,
ranking::SignalCoefficient,
schema::{Field, TextField},
search_ctx::Ctx,
@@ -36,7 +37,7 @@ pub mod union;
use parser::Term;
-use self::optic::AsMultipleTantivyQuery;
+use self::{optic::AsMultipleTantivyQuery, parser::CompoundAwareTerm};
const MAX_SIMILAR_TERMS: usize = 10;
@@ -69,11 +70,41 @@ impl Query {
*count += 1;
}
+ let mut compound_terms: Vec<_> = terms
+ .clone()
+ .into_iter()
+ .map(|term| CompoundAwareTerm {
+ term: *term,
+ adjacent_terms: Vec::new(),
+ })
+ .collect();
+
+ let term_ids: Vec<_> = compound_terms.iter().enumerate().map(|(i, _)| i).collect();
+
+ for window_size in 2..=3 {
+ for window in term_ids.windows(window_size) {
+ let mut window_terms = Vec::new();
+ for i in window {
+ if let Term::Simple(t) = &compound_terms[*i].term {
+ window_terms.push(t.clone());
+ }
+ }
+
+ let compound = TermCompound {
+ terms: window_terms,
+ };
+
+ for i in window {
+ compound_terms[*i].adjacent_terms.push(compound.clone());
+ }
+ }
+ }
+
let schema = index.schema();
let fields: Vec<tantivy::schema::Field> = schema.fields().map(|(field, _)| field).collect();
- let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms
+ let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = compound_terms
.iter()
.map(|term| term.as_tantivy_query(&fields))
.collect();
@@ -782,6 +813,28 @@ mod tests {
)
.expect("failed to insert webpage");
+ index
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Testwebsite</title>
+ </head>
+ <body>
+ This is a testwebsite {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "https://www.second.com",
+ )
+ .unwrap(),
+ )
+ .expect("failed to insert webpage");
+
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
@@ -791,8 +844,15 @@ mod tests {
};
let result = searcher.search(&query).expect("Search failed");
- assert_eq!(result.num_hits, 1);
- assert_eq!(result.webpages[0].url, "https://www.first.com/");
+ assert_eq!(result.num_hits, 2);
+
+ let query = SearchQuery {
+ query: "test website".to_string(),
+ ..Default::default()
+ };
+
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 2);
}
#[test]
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index cebec5c9..25d2674b 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -16,7 +16,7 @@
use tantivy::{
query::{BooleanQuery, Occur, PhraseQuery, TermQuery},
- tokenizer::{TextAnalyzer, Tokenizer},
+ tokenizer::Tokenizer,
};
use crate::{
@@ -25,9 +25,49 @@ use crate::{
schema::{Field, TextField, ALL_FIELDS},
};
+#[derive(Debug, Clone)]
+pub struct TermCompound {
+ pub terms: Vec<SimpleTerm>,
+}
+
+#[derive(Debug, Clone)]
+pub struct CompoundAwareTerm {
+ pub term: Term,
+ pub adjacent_terms: Vec<TermCompound>,
+}
+impl CompoundAwareTerm {
+ pub fn as_tantivy_query(
+ &self,
+ fields: &[tantivy::schema::Field],
+ ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
+ if !self.adjacent_terms.is_empty() {
+ if let Term::Simple(simple_term) = &self.term {
+ return simple_into_tantivy(simple_term, &self.adjacent_terms, fields);
+ }
+ }
+
+ self.term.as_tantivy_query(fields)
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct SimpleTerm(String);
+
+impl From<String> for SimpleTerm {
+ fn from(value: String) -> Self {
+ SimpleTerm(value)
+ }
+}
+
+impl From<SimpleTerm> for String {
+ fn from(value: SimpleTerm) -> Self {
+ value.0
+ }
+}
+
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Term {
- Simple(String),
+ Simple(SimpleTerm),
Phrase(String),
Not(Box<Term>),
Site(String),
@@ -40,7 +80,7 @@ pub enum Term {
impl ToString for Term {
fn to_string(&self) -> String {
match self {
- Term::Simple(term) => term.clone(),
+ Term::Simple(term) => term.0.clone(),
Term::Phrase(phrase) => "\"".to_string() + phrase.as_str() + "\"",
Term::Not(term) => "-".to_string() + term.to_string().as_str(),
Term::Site(site) => "site:".to_string() + site.as_str(),
@@ -53,19 +93,44 @@ impl ToString for Term {
}
fn simple_into_tantivy(
- term: &str,
+ term: &SimpleTerm,
+ adjacent_terms: &[TermCompound],
fields: &[tantivy::schema::Field],
) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
- (
- Occur::Must,
- Box::new(BooleanQuery::new(Term::into_tantivy_simple(term, fields))),
- )
+ let mut queries = Term::into_tantivy_simple(term, fields);
+
+ let fields = fields
+ .iter()
+ .filter(|field| {
+ matches!(
+ ALL_FIELDS[field.field_id() as usize],
+ Field::Text(TextField::AllBody)
+ | Field::Text(TextField::Title)
+ | Field::Text(TextField::Url)
+ )
+ })
+ .copied()
+ .collect::<Vec<_>>();
+
+ for adjacent_term in adjacent_terms {
+ let combined = adjacent_term
+ .terms
+ .iter()
+ .map(|term| term.0.as_str())
+ .collect::<String>();
+
+ for field in &fields {
+ queries.push((Occur::Should, Term::tantivy_text_query(field, &combined)))
+ }
+ }
+
+ (Occur::Must, Box::new(BooleanQuery::new(queries)))
}
impl Term {
pub fn as_simple_text(&self) -> &str {
match self {
- Term::Simple(term) => term,
+ Term::Simple(term) => &term.0,
Term::Phrase(terms) => terms,
Term::Not(term) => term.as_simple_text(),
Term::Site(term) => term,
@@ -76,12 +141,12 @@ impl Term {
}
}
- pub fn as_tantivy_query(
+ fn as_tantivy_query(
&self,
fields: &[tantivy::schema::Field],
) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
match self {
- Term::Simple(term) => simple_into_tantivy(term, fields),
+ Term::Simple(term) => simple_into_tantivy(term, &[], fields),
Term::Phrase(phrase) => {
let mut phrases = Vec::with_capacity(fields.len());
@@ -171,19 +236,19 @@ impl Term {
term.push(BANG_PREFIX);
term.push_str(text);
- simple_into_tantivy(&term, fields)
+ simple_into_tantivy(&term.into(), &[], fields)
}
}
}
fn into_tantivy_simple(
- term: &str,
+ term: &SimpleTerm,
fields: &[tantivy::schema::Field],
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
fields
.iter()
.filter(|field| ALL_FIELDS[field.field_id() as usize].is_searchable())
- .map(|field| (Occur::Should, Term::tantivy_text_query(field, term)))
+ .map(|field| (Occur::Should, Term::tantivy_text_query(field, &term.0)))
.collect()
}
@@ -222,7 +287,7 @@ impl Term {
.collect()
}
- pub fn tantivy_text_query(
+ fn tantivy_text_query(
field: &tantivy::schema::Field,
term: &str,
) -> Box<dyn tantivy::query::Query + 'static> {
@@ -254,22 +319,7 @@ impl Term {
Box::new(processed_query)
}
- pub fn get_tantivy_analyzer(
- entry: &tantivy::schema::FieldEntry,
- tokenizer_manager: &tantivy::tokenizer::TokenizerManager,
- ) -> Option<TextAnalyzer> {
- match entry.field_type() {
- tantivy::schema::FieldType::Str(options) => {
- options.get_indexing_options().and_then(|indexing_options| {
- let tokenizer_name = indexing_options.tokenizer();
- tokenizer_manager.get(tokenizer_name)
- })
- }
- _ => None,
- }
- }
-
- pub fn process_tantivy_term(
+ fn process_tantivy_term(
term: &str,
tantivy_field: tantivy::schema::Field,
) -> Vec<tantivy::Term> {
@@ -295,36 +345,36 @@ fn parse_term(term: &str) -> Box<Term> {
if !not_term.is_empty() && !not_term.starts_with('-') {
Box::new(Term::Not(parse_term(not_term)))
} else {
- Box::new(Term::Simple(term.to_string()))
+ Box::new(Term::Simple(term.to_string().into()))
}
} else if let Some(site) = term.strip_prefix("site:") {
if !site.is_empty() {
Box::new(Term::Site(site.to_string()))
} else {
- Box::new(Term::Simple(term.to_string()))
+ Box::new(Term::Simple(term.to_string().into()))
}
} else if let Some(title) = term.strip_prefix("intitle:") {
if !title.is_empty() {
Box::new(Term::Title(title.to_string()))
} else {
- Box::new(Term::Simple(term.to_string()))
+ Box::new(Term::Simple(term.to_string().into()))
}
} else if let Some(body) = term.strip_prefix("inbody:") {
if !body.is_empty() {
Box::new(Term::Body(body.to_string()))
} else {
- Box::new(Term::Simple(term.to_string()))
+ Box::new(Term::Simple(term.to_string().into()))
}
} else if let Some(url) = term.strip_prefix("inurl:") {
if !url.is_empty() {
Box::new(Term::Url(url.to_string()))
} else {
- Box::new(Term::Simple(term.to_string()))
+ Box::new(Term::Simple(term.to_string().into()))
}
} else if let Some(bang) = term.strip_prefix(BANG_PREFIX) {
Box::new(Term::PossibleBang(bang.to_string()))
} else {
- Box::new(Term::Simple(term.to_string()))
+ Box::new(Term::Simple(term.to_string().into()))
}
}
@@ -381,16 +431,16 @@ mod tests {
assert_eq!(
parse("this -that"),
vec![
- Box::new(Term::Simple("this".to_string())),
- Box::new(Term::Not(Box::new(Term::Simple("that".to_string()))))
+ Box::new(Term::Simple("this".to_string().into())),
+ Box::new(Term::Not(Box::new(Term::Simple("that".to_string().into()))))
]
);
assert_eq!(
parse("this -"),
vec![
- Box::new(Term::Simple("this".to_string())),
- Box::new(Term::Simple("-".to_string()))
+ Box::new(Term::Simple("this".to_string().into())),
+ Box::new(Term::Simple("-".to_string().into()))
]
);
}
@@ -400,8 +450,8 @@ mod tests {
assert_eq!(
parse("this --that"),
vec![
- Box::new(Term::Simple("this".to_string())),
- Box::new(Term::Simple("--that".to_string()))
+ Box::new(Term::Simple("this".to_string().into())),
+ Box::new(Term::Simple("--that".to_string().into()))
]
);
}
@@ -411,7 +461,7 @@ mod tests {
assert_eq!(
parse("this site:test.com"),
vec![
- Box::new(Term::Simple("this".to_string())),
+ Box::new(Term::Simple("this".to_string().into())),
Box::new(Term::Site("test.com".to_string()))
]
);
@@ -422,7 +472,7 @@ mod tests {
assert_eq!(
parse("this intitle:test"),
vec![
- Box::new(Term::Simple("this".to_string())),
+ Box::new(Term::Simple("this".to_string().into())),
Box::new(Term::Title("test".to_string()))
]
);
@@ -433,7 +483,7 @@ mod tests {
assert_eq!(
parse("this inbody:test"),
vec![
- Box::new(Term::Simple("this".to_string())),
+ Box::new(Term::Simple("this".to_string().into())),
Box::new(Term::Body("test".to_string()))
]
);
@@ -444,7 +494,7 @@ mod tests {
assert_eq!(
parse("this inurl:test"),
vec![
- Box::new(Term::Simple("this".to_string())),
+ Box::new(Term::Simple("this".to_string().into())),
Box::new(Term::Url("test".to_string()))
]
);
@@ -467,18 +517,18 @@ mod tests {
assert_eq!(
parse("\"this is a inurl:test"),
vec![
- Box::new(Term::Simple("\"this".to_string())),
- Box::new(Term::Simple("is".to_string())),
- Box::new(Term::Simple("a".to_string())),
+ Box::new(Term::Simple("\"this".to_string().into())),
+ Box::new(Term::Simple("is".to_string().into())),
+ Box::new(Term::Simple("a".to_string().into())),
Box::new(Term::Url("test".to_string()))
]
);
assert_eq!(
parse("this is a\" inurl:test"),
vec![
- Box::new(Term::Simple("this".to_string())),
- Box::new(Term::Simple("is".to_string())),
- Box::new(Term::Simple("a\"".to_string())),
+ Box::new(Term::Simple("this".to_string().into())),
+ Box::new(Term::Simple("is".to_string().into())),
+ Box::new(Term::Simple("a\"".to_string().into())),
Box::new(Term::Url("test".to_string()))
]
);
diff --git a/core/src/spell/mod.rs b/core/src/spell/mod.rs
index 71720e70..6c800843 100644
--- a/core/src/spell/mod.rs
+++ b/core/src/spell/mod.rs
@@ -227,7 +227,7 @@ impl Spell {
let terms: Vec<_> = query::parser::parse(&query.query)
.into_iter()
.filter_map(|term| match *term {
- Term::Simple(s) => Some(s),
+ Term::Simple(s) => Some(String::from(s)),
_ => None,
})
.map(|s| s.to_ascii_lowercase())