Skip to content

web search done right

Ftr/advanced query syntax (#22)

Mikkel Denker6814329e1204
diff --git a/src/index.rs b/src/index.rs
index 9785dd7e..8e3545ad 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -24,6 +24,7 @@ use itertools::intersperse;
use serde::{Deserialize, Serialize};
use tantivy::collector::Collector;
use tantivy::schema::Schema;
+use tantivy::tokenizer::TokenizerManager;
use uuid::Uuid;
use crate::directory::{self, DirEntry};
@@ -75,6 +76,10 @@ impl Index {
})
}
+ pub fn tokenizers(&self) -> &TokenizerManager {
+ self.inverted_index.tokenizers()
+ }
+
#[cfg(test)]
pub fn temporary() -> Result<Self> {
let path = crate::gen_temp_path();
@@ -308,7 +313,8 @@ mod tests {
let deserialized_frozen: FrozenIndex = bincode::deserialize(&bytes).unwrap();
let index: Index = deserialized_frozen.into();
- let query = Query::parse("website", index.schema()).expect("Failed to parse query");
+ let query = Query::parse("website", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
let result = index
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index b732bcd5..2b317860 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -18,6 +18,7 @@ use chrono::NaiveDateTime;
use tantivy::collector::{Collector, Count};
use tantivy::merge_policy::NoMergePolicy;
use tantivy::schema::Schema;
+use tantivy::tokenizer::TokenizerManager;
use tantivy::{DocAddress, Document, IndexReader, IndexWriter, LeasedItem};
use crate::image_store::Image;
@@ -74,6 +75,10 @@ impl InvertedIndex {
})
}
+ pub fn tokenizers(&self) -> &TokenizerManager {
+ self.tantivy_index.tokenizers()
+ }
+
#[cfg(test)]
pub fn temporary() -> Result<Self> {
let path = crate::gen_temp_path();
@@ -321,7 +326,8 @@ mod tests {
#[test]
fn simple_search() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("website", index.schema()).expect("Failed to parse query");
+ let query = Query::parse("website", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
let result = index
@@ -363,8 +369,12 @@ mod tests {
#[test]
fn document_not_matching() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("this query should not match", index.schema())
- .expect("Failed to parse query");
+ let query = Query::parse(
+ "this query should not match",
+ index.schema(),
+ index.tokenizers(),
+ )
+ .expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
index
@@ -399,7 +409,8 @@ mod tests {
#[test]
fn english_stemming() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("runner", index.schema()).expect("Failed to parse query");
+ let query = Query::parse("runner", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
index
@@ -434,7 +445,8 @@ mod tests {
#[test]
fn stemmed_query_english() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("runners", index.schema()).expect("Failed to parse query");
+ let query = Query::parse("runners", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
index
@@ -469,7 +481,8 @@ mod tests {
#[test]
fn searchable_backlinks() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("great site", index.schema()).expect("Failed to parse query");
+ let query = Query::parse("great site", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
index
@@ -534,7 +547,8 @@ mod tests {
#[test]
fn limited_top_docs() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("runner", index.schema()).expect("Failed to parse query");
+ let query = Query::parse("runner", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
for _ in 0..100 {
@@ -571,7 +585,8 @@ mod tests {
#[test]
fn host_search() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("dr", index.schema()).expect("Failed to parse query");
+ let query =
+ Query::parse("dr", index.schema(), index.tokenizers()).expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
index
@@ -654,7 +669,8 @@ mod tests {
let mut index = index1.merge(index2);
index.commit().unwrap();
- let query = Query::parse("website", index.schema()).expect("Failed to parse query");
+ let query = Query::parse("website", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
let result = index
@@ -669,7 +685,8 @@ mod tests {
#[test]
fn match_across_fields() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("example test", index.schema()).expect("Failed to parse query");
+ let query = Query::parse("example test", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
let result = index
@@ -711,7 +728,8 @@ mod tests {
#[test]
fn fetch_time_ranking() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("test", index.schema()).expect("Failed to parse query");
+ let query = Query::parse("test", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
index
diff --git a/src/lib.rs b/src/lib.rs
index 22de7fc9..d2acac35 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -25,7 +25,7 @@ use tantivy::TantivyError;
use thiserror::Error;
pub mod entrypoint;
-pub mod inverted_index;
+mod inverted_index;
pub mod mapreduce;
@@ -38,17 +38,17 @@ mod image_downloader;
mod image_store;
mod index;
mod kv;
-pub mod query;
-pub mod ranking;
+mod query;
+mod ranking;
mod schema;
mod schema_org;
-pub mod searcher;
+mod searcher;
mod snippet;
mod spell;
mod tokenizer;
mod warc;
-pub mod webgraph;
-pub mod webpage;
+mod webgraph;
+mod webpage;
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingMasterConfig {
@@ -194,12 +194,6 @@ pub enum Error {
EmptyQuery,
}
-impl<L, T, E> From<lalrpop_util::ParseError<L, T, E>> for Error {
- fn from(_: lalrpop_util::ParseError<L, T, E>) -> Self {
- Error::Parse
- }
-}
-
pub(crate) type Result<T> = std::result::Result<T, Error>;
// taken from https://docs.rs/sled/0.34.7/src/sled/config.rs.html#445
diff --git a/src/query/mod.rs b/src/query/mod.rs
index e3511ca7..82a7a670 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -14,56 +14,41 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{schema::ALL_FIELDS, tokenizer::NormalTokenizer, Result};
+use crate::Result;
use std::{
collections::{BTreeMap, HashMap},
sync::Arc,
};
use tantivy::{
- query::{BooleanQuery, BoostQuery, Occur, TermQuery},
- schema::{FieldType, IndexRecordOption, Schema},
- tokenizer::{TextAnalyzer, Tokenizer},
+ query::{AllQuery, BooleanQuery, Occur},
+ schema::Schema,
+ tokenizer::TokenizerManager,
};
-use lalrpop_util::lalrpop_mod;
-lalrpop_mod!(pub parser, "/query/parser.rs"); // synthesized by LALRPOP
-
-static PARSER: once_cell::sync::Lazy<parser::TermsParser> =
- once_cell::sync::Lazy::new(parser::TermsParser::new);
+mod parser;
+use parser::Term;
const MAX_SIMILAR_TERMS: usize = 10;
-#[derive(Debug)]
-pub enum Term<'a> {
- Simple(&'a str),
-}
-
#[derive(Clone, Debug)]
pub struct Query {
- terms: Vec<String>,
- schema: Arc<Schema>,
+ #[allow(clippy::vec_box)]
+ terms: Vec<Box<Term>>,
+ tantivy_query: Box<BooleanQuery>,
}
impl Query {
- pub fn parse(query: &str, schema: Arc<Schema>) -> Result<Query> {
- let mut raw_terms = Vec::new();
-
- let terms = PARSER.parse(query)?;
- for term in terms {
- match term {
- Term::Simple(term) => {
- let mut stream = NormalTokenizer::default().token_stream(term);
-
- while let Some(token) = stream.next() {
- raw_terms.push(token.text.to_string());
- }
- }
- }
- }
+ pub fn parse(
+ query: &str,
+ schema: Arc<Schema>,
+ tokenizer_manager: &TokenizerManager,
+ ) -> Result<Query> {
+ let parsed_terms = parser::parse(query);
let mut term_count = HashMap::new();
let mut terms = Vec::new();
- for term in raw_terms {
+
+ for term in parsed_terms {
let count = term_count.entry(term.clone()).or_insert(0);
if *count < MAX_SIMILAR_TERMS {
@@ -73,52 +58,40 @@ impl Query {
*count += 1;
}
- Ok(Query { terms, schema })
- }
+ let fields: Vec<(tantivy::schema::Field, &tantivy::schema::FieldEntry)> =
+ schema.fields().collect();
- pub fn simple_terms(&self) -> &[String] {
- &self.terms
- }
+ let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms
+ .iter()
+ .map(|term| term.as_tantivy_query(&fields, tokenizer_manager))
+ // .map(|term_query| (Occur::Must, term_query))
+ .collect();
- pub fn is_empty(&self) -> bool {
- self.terms.is_empty()
+ queries.push((Occur::Should, Box::new(AllQuery)));
+
+ let tantivy_query = dbg!(Box::new(BooleanQuery::new(queries)));
+
+ Ok(Query {
+ terms,
+ tantivy_query,
+ })
}
- fn get_tantivy_analyzer(
- entry: &tantivy::schema::FieldEntry,
- tokenizer_manager: &tantivy::tokenizer::TokenizerManager,
- ) -> Option<TextAnalyzer> {
- match entry.field_type() {
- tantivy::schema::FieldType::Str(options) => {
- options.get_indexing_options().map(|indexing_options| {
- let tokenizer_name = indexing_options.tokenizer();
- tokenizer_manager
- .get(tokenizer_name)
- .expect("Unknown tokenizer")
- })
- }
- _ => None,
- }
+ pub fn simple_terms(&self) -> Vec<String> {
+ self.terms
+ .iter()
+ .filter_map(|term| {
+ if let Term::Simple(term) = term.as_ref() {
+ Some(term.clone())
+ } else {
+ None
+ }
+ })
+ .collect()
}
- fn process_tantivy_term(
- term: &str,
- analyzer: Option<TextAnalyzer>,
- tantivy_field: tantivy::schema::Field,
- ) -> impl Iterator<Item = tantivy::Term> {
- match analyzer {
- None => vec![tantivy::Term::from_field_text(tantivy_field, term)].into_iter(),
- Some(tokenizer) => {
- let mut terms: Vec<tantivy::Term> = Vec::new();
- let mut token_stream = tokenizer.token_stream(term);
- token_stream.process(&mut |token| {
- let term = tantivy::Term::from_field_text(tantivy_field, &token.text);
- terms.push(term);
- });
-
- terms.into_iter()
- }
- }
+ pub fn is_empty(&self) -> bool {
+ self.terms.is_empty()
}
}
@@ -128,69 +101,19 @@ impl tantivy::query::Query for Query {
searcher: &tantivy::Searcher,
scoring_enabled: bool,
) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
- let schema = searcher.schema();
- let tokenizer_manager = searcher.index().tokenizers();
-
- let queries = self
- .terms
- .iter()
- .map(|term| {
- let one_term = schema
- .fields()
- .filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].is_searchable())
- .into_iter()
- .map(|(field, entry)| {
- let analyzer = Query::get_tantivy_analyzer(entry, tokenizer_manager);
- let processed_terms = Query::process_tantivy_term(term, analyzer, field);
-
- let processed_queries = processed_terms
- .map(|term| {
- (
- Occur::Should,
- Box::new(TermQuery::new(
- term,
- IndexRecordOption::WithFreqsAndPositions,
- ))
- as Box<dyn tantivy::query::Query>,
- )
- })
- .collect();
- let boost = ALL_FIELDS[field.field_id() as usize].boost().unwrap_or(1.0);
-
- (
- Occur::Should,
- Box::new(BoostQuery::new(
- Box::new(BooleanQuery::new(processed_queries)),
- boost,
- )) as Box<dyn tantivy::query::Query>,
- )
- })
- .collect();
-
- Box::new(BooleanQuery::new(one_term)) as Box<dyn tantivy::query::Query>
- })
- .map(|term_query| (Occur::Must, term_query))
- .collect();
-
- let query = Box::new(BooleanQuery::new(queries));
-
- query.weight(searcher, scoring_enabled)
+ self.tantivy_query.weight(searcher, scoring_enabled)
}
fn query_terms(&self, terms: &mut BTreeMap<tantivy::Term, bool>) {
- for term_text in &self.terms {
- for (field, entry) in self.schema.fields() {
- if let FieldType::Str(_) = entry.field_type() {
- terms.insert(tantivy::Term::from_field_text(field, term_text), true);
- }
- }
- }
+ self.tantivy_query.query_terms(terms)
}
}
#[cfg(test)]
mod tests {
- use crate::schema::create_schema;
+ use crate::{
+ inverted_index::InvertedIndex, ranking::Ranker, schema::create_schema, webpage::Webpage,
+ };
use super::*;
@@ -199,13 +122,14 @@ mod tests {
let schema = Arc::new(create_schema());
let query = Query::parse(
- "This is a simple query the the the the the the the the the the the the the",
+ "this is a simple query the the the the the the the the the the the the the",
Arc::clone(&schema),
+ &TokenizerManager::new(),
)
.expect("Failed to parse query");
assert_eq!(
- query.terms,
+ query.simple_terms(),
vec![
"this".to_string(),
"is".to_string(),
@@ -230,11 +154,15 @@ mod tests {
fn parse_trailing_leading_whitespace() {
let schema = Arc::new(create_schema());
- let query = Query::parse(" This is a simple query ", Arc::clone(&schema))
- .expect("Failed to parse query");
+ let query = Query::parse(
+ " this is a simple query ",
+ Arc::clone(&schema),
+ &TokenizerManager::new(),
+ )
+ .expect("Failed to parse query");
assert_eq!(
- query.terms,
+ query.simple_terms(),
vec![
"this".to_string(),
"is".to_string(),
@@ -249,36 +177,271 @@ mod tests {
fn parse_weird_characters() {
let schema = Arc::new(create_schema());
- let terms = Query::parse("123", Arc::clone(&schema))
+ let terms = Query::parse("123", Arc::clone(&schema), &TokenizerManager::new())
.expect("Failed to parse query")
- .terms;
+ .simple_terms();
assert_eq!(terms, vec!["123".to_string()]);
- let terms = Query::parse("123 33", Arc::clone(&schema))
+ let terms = Query::parse("123 33", Arc::clone(&schema), &TokenizerManager::new())
.expect("Failed to parse query")
- .terms;
+ .simple_terms();
assert_eq!(terms, vec!["123".to_string(), "33".to_string()]);
- let terms = Query::parse("term! term# $", Arc::clone(&schema))
- .expect("Failed to parse query")
- .terms;
+ let terms = Query::parse(
+ "term! term# $",
+ Arc::clone(&schema),
+ &TokenizerManager::new(),
+ )
+ .expect("Failed to parse query")
+ .simple_terms();
assert_eq!(
terms,
- vec![
- "term".to_string(),
- "!".to_string(),
- "term".to_string(),
- "#".to_string(),
- "$".to_string()
- ]
+ vec!["term!".to_string(), "term#".to_string(), "$".to_string()]
);
}
+ #[test]
+ fn not_query() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let query = Query::parse("test -website", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ This is a test website
+ </body>
+ </html>
+ "#,
+ "https://www.first.com",
+ vec![],
+ 1.0,
+ 0,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test test</title>
+ </head>
+ <body>
+ This test page does not contain the forbidden word
+ </body>
+ </html>
+ "#,
+ "https://www.second.com",
+ vec![],
+ 1.0,
+ 0,
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.second.com");
+ }
+
+ #[test]
+ fn site_query() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ This is a test website
+ </body>
+ </html>
+ "#,
+ "https://www.first.com",
+ vec![],
+ 1.0,
+ 0,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test test</title>
+ </head>
+ <body>
+ This test page does not contain the forbidden word
+ </body>
+ </html>
+ "#,
+ "https://www.second.com",
+ vec![],
+ 1.0,
+ 0,
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let query = Query::parse("test site:first.com", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.first.com");
+
+ let query = Query::parse(
+ "test site:www.first.com",
+ index.schema(),
+ index.tokenizers(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.first.com");
+
+ let query = Query::parse("test -site:first.com", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.second.com");
+ }
+
+ #[test]
+ fn title_query() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ This is a test website
+ </body>
+ </html>
+ "#,
+ "https://www.first.com",
+ vec![],
+ 1.0,
+ 0,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test test</title>
+ </head>
+ <body>
+ This is a test website
+ </body>
+ </html>
+ "#,
+ "https://www.second.com",
+ vec![],
+ 1.0,
+ 0,
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let query = Query::parse("intitle:website", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.first.com");
+ }
+
+ #[test]
+ fn url_query() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ This is a test website
+ </body>
+ </html>
+ "#,
+ "https://www.first.com/forum",
+ vec![],
+ 1.0,
+ 0,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test test</title>
+ </head>
+ <body>
+ This is a test website
+ </body>
+ </html>
+ "#,
+ "https://www.second.com",
+ vec![],
+ 1.0,
+ 0,
+ ))
+ .expect("failed to parse webpage");
+ index.commit().expect("failed to commit index");
+
+ let query = Query::parse("test inurl:forum", index.schema(), index.tokenizers())
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.num_docs, 1);
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.first.com/forum");
+ }
+
#[test]
fn empty_query() {
let schema = Arc::new(create_schema());
- let query = Query::parse("", Arc::clone(&schema)).expect("Failed to parse query");
+ let query = Query::parse("", Arc::clone(&schema), &TokenizerManager::new())
+ .expect("Failed to parse query");
assert!(query.is_empty())
}
diff --git a/src/query/parser.lalrpop b/src/query/parser.lalrpop
deleted file mode 100644
index 9ec70847..00000000
--- a/src/query/parser.lalrpop
+++ /dev/null
@@ -1,12 +0,0 @@
-use crate::query::Term;
-
-grammar;
-
-
-pub Terms: Vec<Term<'input>> = <Term*> => <>;
-
-Term: Term<'input> = {
- <BasicTerm> => Term::Simple(<>),
-}
-
-BasicTerm: &'input str = r"[\S]*" => <>;
\ No newline at end of file
diff --git a/src/query/parser.rs b/src/query/parser.rs
new file mode 100644
index 00000000..7948486e
--- /dev/null
+++ b/src/query/parser.rs
@@ -0,0 +1,325 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::{
+ query::{BooleanQuery, BoostQuery, Occur, TermQuery},
+ schema::IndexRecordOption,
+ tokenizer::{TextAnalyzer, TokenizerManager},
+};
+
+use crate::schema::{Field, ALL_FIELDS};
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum Term {
+ Simple(String),
+ Not(Box<Term>),
+ Site(String),
+ Title(String),
+ Body(String),
+ Url(String),
+}
+
+impl Term {
+ pub fn as_tantivy_query(
+ &self,
+ fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
+ tokenizer_manager: &TokenizerManager,
+ ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
+ match self {
+ Term::Simple(term) => (
+ Occur::Must,
+ Box::new(BooleanQuery::new(Term::into_tantivy_simple(
+ term,
+ fields,
+ tokenizer_manager,
+ ))),
+ ),
+ Term::Not(subterm) => (
+ Occur::MustNot,
+ Box::new(BooleanQuery::new(vec![
+ subterm.as_tantivy_query(fields, tokenizer_manager)
+ ])),
+ ),
+ Term::Site(site) => (
+ Occur::Must,
+ Box::new(BooleanQuery::new(Term::into_tantivy_site(
+ site,
+ fields,
+ tokenizer_manager,
+ ))),
+ ),
+ Term::Title(title) => {
+ let (field, entry) = fields
+ .iter()
+ .find(|(field, _)| {
+ matches!(ALL_FIELDS[field.field_id() as usize], Field::Title)
+ })
+ .unwrap();
+ (
+ Occur::Must,
+ Term::tantivy_term_query(field, entry, tokenizer_manager, title),
+ )
+ }
+ Term::Body(body) => {
+ let (field, entry) = fields
+ .iter()
+ .find(|(field, _)| {
+ matches!(ALL_FIELDS[field.field_id() as usize], Field::AllBody)
+ })
+ .unwrap();
+ (
+ Occur::Must,
+ Term::tantivy_term_query(field, entry, tokenizer_manager, body),
+ )
+ }
+ Term::Url(url) => {
+ let (field, entry) = fields
+ .iter()
+ .find(|(field, _)| matches!(ALL_FIELDS[field.field_id() as usize], Field::Url))
+ .unwrap();
+ (
+ Occur::Must,
+ Term::tantivy_term_query(field, entry, tokenizer_manager, url),
+ )
+ }
+ }
+ }
+
+ fn into_tantivy_simple(
+ term: &str,
+ fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
+ tokenizer_manager: &TokenizerManager,
+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
+ fields
+ .iter()
+ .filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].is_searchable())
+ .into_iter()
+ .map(|(field, entry)| {
+ (
+ Occur::Should,
+ Term::tantivy_term_query(field, entry, tokenizer_manager, term),
+ )
+ })
+ .collect()
+ }
+
+ fn into_tantivy_site(
+ term: &str,
+ fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
+ tokenizer_manager: &TokenizerManager,
+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
+ fields
+ .iter()
+ .filter(|(field, _)| {
+ matches!(
+ ALL_FIELDS[field.field_id() as usize],
+ Field::Domain | Field::Host
+ )
+ })
+ .into_iter()
+ .map(|(field, entry)| {
+ (
+ Occur::Should,
+ Term::tantivy_term_query(field, entry, tokenizer_manager, term),
+ )
+ })
+ .collect()
+ }
+
+ fn tantivy_term_query(
+ field: &tantivy::schema::Field,
+ entry: &tantivy::schema::FieldEntry,
+ tokenizer_manager: &TokenizerManager,
+ term: &str,
+ ) -> Box<dyn tantivy::query::Query + 'static> {
+ let analyzer = Term::get_tantivy_analyzer(entry, tokenizer_manager);
+ let processed_terms = Term::process_tantivy_term(term, analyzer, *field);
+
+ let processed_queries = processed_terms
+ .map(|term| {
+ (
+ Occur::Must,
+ Box::new(TermQuery::new(
+ term,
+ IndexRecordOption::WithFreqsAndPositions,
+ )) as Box<dyn tantivy::query::Query>,
+ )
+ })
+ .collect();
+ let boost = ALL_FIELDS[field.field_id() as usize].boost().unwrap_or(1.0);
+
+ Box::new(BoostQuery::new(
+ Box::new(BooleanQuery::new(processed_queries)),
+ boost,
+ ))
+ }
+
+ fn get_tantivy_analyzer(
+ entry: &tantivy::schema::FieldEntry,
+ tokenizer_manager: &tantivy::tokenizer::TokenizerManager,
+ ) -> Option<TextAnalyzer> {
+ match entry.field_type() {
+ tantivy::schema::FieldType::Str(options) => {
+ options.get_indexing_options().and_then(|indexing_options| {
+ let tokenizer_name = indexing_options.tokenizer();
+ tokenizer_manager.get(tokenizer_name)
+ })
+ }
+ _ => None,
+ }
+ }
+
+ fn process_tantivy_term(
+ term: &str,
+ analyzer: Option<TextAnalyzer>,
+ tantivy_field: tantivy::schema::Field,
+ ) -> impl Iterator<Item = tantivy::Term> {
+ match analyzer {
+ None => vec![tantivy::Term::from_field_text(tantivy_field, term)].into_iter(),
+ Some(tokenizer) => {
+ let mut terms: Vec<tantivy::Term> = Vec::new();
+ let mut token_stream = tokenizer.token_stream(term);
+ token_stream.process(&mut |token| {
+ let term = tantivy::Term::from_field_text(tantivy_field, &token.text);
+ terms.push(term);
+ });
+
+ terms.into_iter()
+ }
+ }
+ }
+}
+
+fn parse_term(term: &str) -> Box<Term> {
+ // TODO: re-write this entire function once if-let chains become stable
+ if let Some(not_term) = term.strip_prefix('-') {
+ if !not_term.is_empty() && !not_term.starts_with('-') {
+ Box::new(Term::Not(parse_term(not_term)))
+ } else {
+ Box::new(Term::Simple(term.to_string()))
+ }
+ } else if let Some(site) = term.strip_prefix("site:") {
+ if !site.is_empty() {
+ Box::new(Term::Site(site.to_string()))
+ } else {
+ Box::new(Term::Simple(term.to_string()))
+ }
+ } else if let Some(title) = term.strip_prefix("intitle:") {
+ if !title.is_empty() {
+ Box::new(Term::Title(title.to_string()))
+ } else {
+ Box::new(Term::Simple(term.to_string()))
+ }
+ } else if let Some(body) = term.strip_prefix("inbody:") {
+ if !body.is_empty() {
+ Box::new(Term::Body(body.to_string()))
+ } else {
+ Box::new(Term::Simple(term.to_string()))
+ }
+ } else if let Some(url) = term.strip_prefix("inurl:") {
+ if !url.is_empty() {
+ Box::new(Term::Url(url.to_string()))
+ } else {
+ Box::new(Term::Simple(term.to_string()))
+ }
+ } else {
+ Box::new(Term::Simple(term.to_string()))
+ }
+}
+
+#[allow(clippy::vec_box)]
+pub fn parse(query: &str) -> Vec<Box<Term>> {
+ query.split_whitespace().map(parse_term).collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn parse_not() {
+ assert_eq!(
+ parse("this -that"),
+ vec![
+ Box::new(Term::Simple("this".to_string())),
+ Box::new(Term::Not(Box::new(Term::Simple("that".to_string()))))
+ ]
+ );
+
+ assert_eq!(
+ parse("this -"),
+ vec![
+ Box::new(Term::Simple("this".to_string())),
+ Box::new(Term::Simple("-".to_string()))
+ ]
+ );
+ }
+
+ #[test]
+ fn double_not() {
+ assert_eq!(
+ parse("this --that"),
+ vec![
+ Box::new(Term::Simple("this".to_string())),
+ Box::new(Term::Simple("--that".to_string()))
+ ]
+ );
+ }
+
+ #[test]
+ fn site() {
+ assert_eq!(
+ parse("this site:test.com"),
+ vec![
+ Box::new(Term::Simple("this".to_string())),
+ Box::new(Term::Site("test.com".to_string()))
+ ]
+ );
+ }
+
+ #[test]
+ fn title() {
+ assert_eq!(
+ parse("this intitle:test"),
+ vec![
+ Box::new(Term::Simple("this".to_string())),
+ Box::new(Term::Title("test".to_string()))
+ ]
+ );
+ }
+
+ #[test]
+ fn body() {
+ assert_eq!(
+ parse("this inbody:test"),
+ vec![
+ Box::new(Term::Simple("this".to_string())),
+ Box::new(Term::Body("test".to_string()))
+ ]
+ );
+ }
+
+ #[test]
+ fn url() {
+ assert_eq!(
+ parse("this inurl:test"),
+ vec![
+ Box::new(Term::Simple("this".to_string())),
+ Box::new(Term::Url("test".to_string()))
+ ]
+ );
+ }
+}
diff --git a/src/searcher.rs b/src/searcher.rs
index c61f8d90..0daf536b 100644
--- a/src/searcher.rs
+++ b/src/searcher.rs
@@ -57,7 +57,7 @@ impl Searcher {
impl Searcher {
pub fn search(&self, query: &str) -> Result<SearchResult> {
let raw_query = query.to_string();
- let query = Query::parse(query, self.index.schema())?;
+ let query = Query::parse(query, self.index.schema(), self.index.tokenizers())?;
if query.is_empty() {
return Err(Error::EmptyQuery);
@@ -65,7 +65,7 @@ impl Searcher {
let ranker = Ranker::new(query.clone());
let webpages = self.index.search(&query, ranker.collector())?;
- let correction = self.index.spell_correction(query.simple_terms());
+ let correction = self.index.spell_correction(&query.simple_terms());
let entity = self
.entity_index
diff --git a/src/webgraph/graph_store.rs b/src/webgraph/graph_store.rs
index 7ecd931a..e9c483c6 100644
--- a/src/webgraph/graph_store.rs
+++ b/src/webgraph/graph_store.rs
@@ -201,6 +201,8 @@ impl<S: Store> GraphStore<S> {
self.assign_id(node, id);
id
}
+
+ #[allow(unused)]
pub fn outgoing_edges(&self, node: NodeID) -> Vec<Edge> {
self.adjacency
.borrow_mut()
diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rs
index 41c28779..31032212 100644
--- a/src/webgraph/mod.rs
+++ b/src/webgraph/mod.rs
@@ -274,6 +274,7 @@ impl<S: Store> Webgraph<S> {
distances
}
+ #[allow(unused)]
pub fn distances(&self, source: Node) -> HashMap<Node, usize> {
self.full_graph
.as_ref()
@@ -293,6 +294,7 @@ impl<S: Store> Webgraph<S> {
.unwrap_or_default()
}
+ #[allow(unused)]
fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {
self.full_graph
.as_ref()
@@ -307,6 +309,7 @@ impl<S: Store> Webgraph<S> {
.unwrap_or_default()
}
+ #[allow(unused)]
pub fn reversed_distances(&self, source: Node) -> HashMap<Node, usize> {
self.full_graph
.as_ref()
@@ -319,6 +322,7 @@ impl<S: Store> Webgraph<S> {
.unwrap_or_default()
}
+ #[allow(unused)]
pub fn host_distances(&self, source: Node) -> HashMap<Node, usize> {
self.host_graph
.as_ref()
@@ -352,6 +356,7 @@ impl<S: Store> Webgraph<S> {
.unwrap_or_default()
}
+ #[allow(unused)]
pub fn host_reversed_distances(&self, source: Node) -> HashMap<Node, usize> {
self.host_graph
.as_ref()
@@ -401,6 +406,7 @@ impl<S: Store> Webgraph<S> {
.collect()
}
+ #[allow(unused)]
pub fn harmonic_centrality(&self) -> HashMap<Node, f64> {
self.full_graph
.as_ref()
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 3a699570..1572d948 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -93,6 +93,7 @@ pub struct Webpage<'a> {
}
impl<'a> Webpage<'a> {
+ #[cfg(test)]
pub fn new(
html: &'a str,
url: &str,
@@ -435,7 +436,7 @@ impl<'a> Html<'a> {
return Err(Error::EmptyField("title"));
}
- doc.add_text(tantivy_field, title.unwrap())
+ doc.add_text(tantivy_field, title.unwrap());
}
Field::CleanBody | Field::StemmedCleanBody => {
let text = self.clean_text();