web search done right
Ftr/advanced query syntax (#22)
Mikkel Denker6814329e1204
diff --git a/src/index.rs b/src/index.rsindex 9785dd7e..8e3545ad 100644--- a/src/index.rs+++ b/src/index.rs@@ -24,6 +24,7 @@ use itertools::intersperse;use serde::{Deserialize, Serialize};use tantivy::collector::Collector;use tantivy::schema::Schema;+use tantivy::tokenizer::TokenizerManager;use uuid::Uuid;use crate::directory::{self, DirEntry};@@ -75,6 +76,10 @@ impl Index {})}+ pub fn tokenizers(&self) -> &TokenizerManager {+ self.inverted_index.tokenizers()+ }+#[cfg(test)]pub fn temporary() -> Result<Self> {let path = crate::gen_temp_path();@@ -308,7 +313,8 @@ mod tests {let deserialized_frozen: FrozenIndex = bincode::deserialize(&bytes).unwrap();let index: Index = deserialized_frozen.into();- let query = Query::parse("website", index.schema()).expect("Failed to parse query");+ let query = Query::parse("website", index.schema(), index.tokenizers())+ .expect("Failed to parse query");let ranker = Ranker::new(query.clone());let result = indexdiff --git a/src/inverted_index.rs b/src/inverted_index.rsindex b732bcd5..2b317860 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -18,6 +18,7 @@ use chrono::NaiveDateTime;use tantivy::collector::{Collector, Count};use tantivy::merge_policy::NoMergePolicy;use tantivy::schema::Schema;+use tantivy::tokenizer::TokenizerManager;use tantivy::{DocAddress, Document, IndexReader, IndexWriter, LeasedItem};use crate::image_store::Image;@@ -74,6 +75,10 @@ impl InvertedIndex {})}+ pub fn tokenizers(&self) -> &TokenizerManager {+ self.tantivy_index.tokenizers()+ }+#[cfg(test)]pub fn temporary() -> Result<Self> {let path = crate::gen_temp_path();@@ -321,7 +326,8 @@ mod tests {#[test]fn simple_search() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("website", index.schema()).expect("Failed to parse query");+ let query = Query::parse("website", index.schema(), index.tokenizers())+ .expect("Failed to parse query");let ranker = Ranker::new(query.clone());let result = index@@ -363,8 +369,12 @@ mod tests {#[test]fn document_not_matching() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("this query should not match", index.schema())- .expect("Failed to parse query");+ let query = Query::parse(+ "this query should not match",+ index.schema(),+ index.tokenizers(),+ )+ .expect("Failed to parse query");let ranker = Ranker::new(query.clone());index@@ -399,7 +409,8 @@ mod tests {#[test]fn english_stemming() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("runner", index.schema()).expect("Failed to parse query");+ let query = Query::parse("runner", index.schema(), index.tokenizers())+ .expect("Failed to parse query");let ranker = Ranker::new(query.clone());index@@ -434,7 +445,8 @@ mod tests {#[test]fn stemmed_query_english() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("runners", index.schema()).expect("Failed to parse query");+ let query = Query::parse("runners", index.schema(), index.tokenizers())+ .expect("Failed to parse query");let ranker = Ranker::new(query.clone());index@@ -469,7 +481,8 @@ mod tests {#[test]fn searchable_backlinks() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("great site", index.schema()).expect("Failed to parse query");+ let query = Query::parse("great site", index.schema(), index.tokenizers())+ .expect("Failed to parse query");let ranker = Ranker::new(query.clone());index@@ -534,7 +547,8 @@ mod tests {#[test]fn limited_top_docs() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("runner", index.schema()).expect("Failed to parse query");+ let query = Query::parse("runner", index.schema(), index.tokenizers())+ .expect("Failed to parse query");let ranker = Ranker::new(query.clone());for _ in 0..100 {@@ -571,7 +585,8 @@ mod tests {#[test]fn host_search() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("dr", index.schema()).expect("Failed to parse query");+ let query =+ Query::parse("dr", index.schema(), index.tokenizers()).expect("Failed to parse query");let ranker = Ranker::new(query.clone());index@@ -654,7 +669,8 @@ mod tests {let mut index = index1.merge(index2);index.commit().unwrap();- let query = Query::parse("website", index.schema()).expect("Failed to parse query");+ let query = Query::parse("website", index.schema(), index.tokenizers())+ .expect("Failed to parse query");let ranker = Ranker::new(query.clone());let result = index@@ -669,7 +685,8 @@ mod tests {#[test]fn match_across_fields() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("example test", index.schema()).expect("Failed to parse query");+ let query = Query::parse("example test", index.schema(), index.tokenizers())+ .expect("Failed to parse query");let ranker = Ranker::new(query.clone());let result = index@@ -711,7 +728,8 @@ mod tests {#[test]fn fetch_time_ranking() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("test", index.schema()).expect("Failed to parse query");+ let query = Query::parse("test", index.schema(), index.tokenizers())+ .expect("Failed to parse query");let ranker = Ranker::new(query.clone());indexdiff --git a/src/lib.rs b/src/lib.rsindex 22de7fc9..d2acac35 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -25,7 +25,7 @@ use tantivy::TantivyError;use thiserror::Error;pub mod entrypoint;-pub mod inverted_index;+mod inverted_index;pub mod mapreduce;@@ -38,17 +38,17 @@ mod image_downloader;mod image_store;mod index;mod kv;-pub mod query;-pub mod ranking;+mod query;+mod ranking;mod schema;mod schema_org;-pub mod searcher;+mod searcher;mod snippet;mod spell;mod tokenizer;mod warc;-pub mod webgraph;-pub mod webpage;+mod webgraph;+mod webpage;#[derive(Debug, Deserialize, Clone)]pub struct IndexingMasterConfig {@@ -194,12 +194,6 @@ pub enum Error {EmptyQuery,}-impl<L, T, E> From<lalrpop_util::ParseError<L, T, E>> for Error {- fn from(_: lalrpop_util::ParseError<L, T, E>) -> Self {- Error::Parse- }-}-pub(crate) type Result<T> = std::result::Result<T, Error>;// taken from https://docs.rs/sled/0.34.7/src/sled/config.rs.html#445diff --git a/src/query/mod.rs b/src/query/mod.rsindex e3511ca7..82a7a670 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -14,56 +14,41 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{schema::ALL_FIELDS, tokenizer::NormalTokenizer, Result};+use crate::Result;use std::{collections::{BTreeMap, HashMap},sync::Arc,};use tantivy::{- query::{BooleanQuery, BoostQuery, Occur, TermQuery},- schema::{FieldType, IndexRecordOption, Schema},- tokenizer::{TextAnalyzer, Tokenizer},+ query::{AllQuery, BooleanQuery, Occur},+ schema::Schema,+ tokenizer::TokenizerManager,};-use lalrpop_util::lalrpop_mod;-lalrpop_mod!(pub parser, "/query/parser.rs"); // synthesized by LALRPOP--static PARSER: once_cell::sync::Lazy<parser::TermsParser> =- once_cell::sync::Lazy::new(parser::TermsParser::new);+mod parser;+use parser::Term;const MAX_SIMILAR_TERMS: usize = 10;-#[derive(Debug)]-pub enum Term<'a> {- Simple(&'a str),-}-#[derive(Clone, Debug)]pub struct Query {- terms: Vec<String>,- schema: Arc<Schema>,+ #[allow(clippy::vec_box)]+ terms: Vec<Box<Term>>,+ tantivy_query: Box<BooleanQuery>,}impl Query {- pub fn parse(query: &str, schema: Arc<Schema>) -> Result<Query> {- let mut raw_terms = Vec::new();-- let terms = PARSER.parse(query)?;- for term in terms {- match term {- Term::Simple(term) => {- let mut stream = NormalTokenizer::default().token_stream(term);-- while let Some(token) = stream.next() {- raw_terms.push(token.text.to_string());- }- }- }- }+ pub fn parse(+ query: &str,+ schema: Arc<Schema>,+ tokenizer_manager: &TokenizerManager,+ ) -> Result<Query> {+ let parsed_terms = parser::parse(query);let mut term_count = HashMap::new();let mut terms = Vec::new();- for term in raw_terms {++ for term in parsed_terms {let count = term_count.entry(term.clone()).or_insert(0);if *count < MAX_SIMILAR_TERMS {@@ -73,52 +58,40 @@ impl Query {*count += 1;}- Ok(Query { terms, schema })- }+ let fields: Vec<(tantivy::schema::Field, &tantivy::schema::FieldEntry)> =+ schema.fields().collect();- pub fn simple_terms(&self) -> &[String] {- &self.terms- }+ let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms+ .iter()+ .map(|term| term.as_tantivy_query(&fields, tokenizer_manager))+ // .map(|term_query| (Occur::Must, term_query))+ .collect();- pub fn is_empty(&self) -> bool {- self.terms.is_empty()+ queries.push((Occur::Should, Box::new(AllQuery)));++ let tantivy_query = dbg!(Box::new(BooleanQuery::new(queries)));++ Ok(Query {+ terms,+ tantivy_query,+ })}- fn get_tantivy_analyzer(- entry: &tantivy::schema::FieldEntry,- tokenizer_manager: &tantivy::tokenizer::TokenizerManager,- ) -> Option<TextAnalyzer> {- match entry.field_type() {- tantivy::schema::FieldType::Str(options) => {- options.get_indexing_options().map(|indexing_options| {- let tokenizer_name = indexing_options.tokenizer();- tokenizer_manager- .get(tokenizer_name)- .expect("Unknown tokenizer")- })- }- _ => None,- }+ pub fn simple_terms(&self) -> Vec<String> {+ self.terms+ .iter()+ .filter_map(|term| {+ if let Term::Simple(term) = term.as_ref() {+ Some(term.clone())+ } else {+ None+ }+ })+ .collect()}- fn process_tantivy_term(- term: &str,- analyzer: Option<TextAnalyzer>,- tantivy_field: tantivy::schema::Field,- ) -> impl Iterator<Item = tantivy::Term> {- match analyzer {- None => vec![tantivy::Term::from_field_text(tantivy_field, term)].into_iter(),- Some(tokenizer) => {- let mut terms: Vec<tantivy::Term> = Vec::new();- let mut token_stream = tokenizer.token_stream(term);- token_stream.process(&mut |token| {- let term = tantivy::Term::from_field_text(tantivy_field, &token.text);- terms.push(term);- });-- terms.into_iter()- }- }+ pub fn is_empty(&self) -> bool {+ self.terms.is_empty()}}@@ -128,69 +101,19 @@ impl tantivy::query::Query for Query {searcher: &tantivy::Searcher,scoring_enabled: bool,) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {- let schema = searcher.schema();- let tokenizer_manager = searcher.index().tokenizers();-- let queries = self- .terms- .iter()- .map(|term| {- let one_term = schema- .fields()- .filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].is_searchable())- .into_iter()- .map(|(field, entry)| {- let analyzer = Query::get_tantivy_analyzer(entry, tokenizer_manager);- let processed_terms = Query::process_tantivy_term(term, analyzer, field);-- let processed_queries = processed_terms- .map(|term| {- (- Occur::Should,- Box::new(TermQuery::new(- term,- IndexRecordOption::WithFreqsAndPositions,- ))- as Box<dyn tantivy::query::Query>,- )- })- .collect();- let boost = ALL_FIELDS[field.field_id() as usize].boost().unwrap_or(1.0);-- (- Occur::Should,- Box::new(BoostQuery::new(- Box::new(BooleanQuery::new(processed_queries)),- boost,- )) as Box<dyn tantivy::query::Query>,- )- })- .collect();-- Box::new(BooleanQuery::new(one_term)) as Box<dyn tantivy::query::Query>- })- .map(|term_query| (Occur::Must, term_query))- .collect();-- let query = Box::new(BooleanQuery::new(queries));-- query.weight(searcher, scoring_enabled)+ self.tantivy_query.weight(searcher, scoring_enabled)}fn query_terms(&self, terms: &mut BTreeMap<tantivy::Term, bool>) {- for term_text in &self.terms {- for (field, entry) in self.schema.fields() {- if let FieldType::Str(_) = entry.field_type() {- terms.insert(tantivy::Term::from_field_text(field, term_text), true);- }- }- }+ self.tantivy_query.query_terms(terms)}}#[cfg(test)]mod tests {- use crate::schema::create_schema;+ use crate::{+ inverted_index::InvertedIndex, ranking::Ranker, schema::create_schema, webpage::Webpage,+ };use super::*;@@ -199,13 +122,14 @@ mod tests {let schema = Arc::new(create_schema());let query = Query::parse(- "This is a simple query the the the the the the the the the the the the the",+ "this is a simple query the the the the the the the the the the the the the",Arc::clone(&schema),+ &TokenizerManager::new(),).expect("Failed to parse query");assert_eq!(- query.terms,+ query.simple_terms(),vec!["this".to_string(),"is".to_string(),@@ -230,11 +154,15 @@ mod tests {fn parse_trailing_leading_whitespace() {let schema = Arc::new(create_schema());- let query = Query::parse(" This is a simple query ", Arc::clone(&schema))- .expect("Failed to parse query");+ let query = Query::parse(+ " this is a simple query ",+ Arc::clone(&schema),+ &TokenizerManager::new(),+ )+ .expect("Failed to parse query");assert_eq!(- query.terms,+ query.simple_terms(),vec!["this".to_string(),"is".to_string(),@@ -249,36 +177,271 @@ mod tests {fn parse_weird_characters() {let schema = Arc::new(create_schema());- let terms = Query::parse("123", Arc::clone(&schema))+ let terms = Query::parse("123", Arc::clone(&schema), &TokenizerManager::new()).expect("Failed to parse query")- .terms;+ .simple_terms();assert_eq!(terms, vec!["123".to_string()]);- let terms = Query::parse("123 33", Arc::clone(&schema))+ let terms = Query::parse("123 33", Arc::clone(&schema), &TokenizerManager::new()).expect("Failed to parse query")- .terms;+ .simple_terms();assert_eq!(terms, vec!["123".to_string(), "33".to_string()]);- let terms = Query::parse("term! term# $", Arc::clone(&schema))- .expect("Failed to parse query")- .terms;+ let terms = Query::parse(+ "term! term# $",+ Arc::clone(&schema),+ &TokenizerManager::new(),+ )+ .expect("Failed to parse query")+ .simple_terms();assert_eq!(terms,- vec![- "term".to_string(),- "!".to_string(),- "term".to_string(),- "#".to_string(),- "$".to_string()- ]+ vec!["term!".to_string(), "term#".to_string(), "$".to_string()]);}+ #[test]+ fn not_query() {+ let mut index = InvertedIndex::temporary().expect("Unable to open index");+ let query = Query::parse("test -website", index.schema(), index.tokenizers())+ .expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ This is a test website+ </body>+ </html>+ "#,+ "https://www.first.com",+ vec![],+ 1.0,+ 0,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test test</title>+ </head>+ <body>+ This test page does not contain the forbidden word+ </body>+ </html>+ "#,+ "https://www.second.com",+ vec![],+ 1.0,+ 0,+ ))+ .expect("failed to parse webpage");+ index.commit().expect("failed to commit index");++ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");+ assert_eq!(result.num_docs, 1);+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.second.com");+ }++ #[test]+ fn site_query() {+ let mut index = InvertedIndex::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ This is a test website+ </body>+ </html>+ "#,+ "https://www.first.com",+ vec![],+ 1.0,+ 0,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test test</title>+ </head>+ <body>+ This test page does not contain the forbidden word+ </body>+ </html>+ "#,+ "https://www.second.com",+ vec![],+ 1.0,+ 0,+ ))+ .expect("failed to parse webpage");+ index.commit().expect("failed to commit index");++ let query = Query::parse("test site:first.com", index.schema(), index.tokenizers())+ .expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());+ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");+ assert_eq!(result.num_docs, 1);+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.first.com");++ let query = Query::parse(+ "test site:www.first.com",+ index.schema(),+ index.tokenizers(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());+ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");+ assert_eq!(result.num_docs, 1);+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.first.com");++ let query = Query::parse("test -site:first.com", index.schema(), index.tokenizers())+ .expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());+ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");+ assert_eq!(result.num_docs, 1);+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.second.com");+ }++ #[test]+ fn title_query() {+ let mut index = InvertedIndex::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ This is a test website+ </body>+ </html>+ "#,+ "https://www.first.com",+ vec![],+ 1.0,+ 0,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test test</title>+ </head>+ <body>+ This is a test website+ </body>+ </html>+ "#,+ "https://www.second.com",+ vec![],+ 1.0,+ 0,+ ))+ .expect("failed to parse webpage");+ index.commit().expect("failed to commit index");++ let query = Query::parse("intitle:website", index.schema(), index.tokenizers())+ .expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());+ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");+ assert_eq!(result.num_docs, 1);+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.first.com");+ }++ #[test]+ fn url_query() {+ let mut index = InvertedIndex::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ This is a test website+ </body>+ </html>+ "#,+ "https://www.first.com/forum",+ vec![],+ 1.0,+ 0,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test test</title>+ </head>+ <body>+ This is a test website+ </body>+ </html>+ "#,+ "https://www.second.com",+ vec![],+ 1.0,+ 0,+ ))+ .expect("failed to parse webpage");+ index.commit().expect("failed to commit index");++ let query = Query::parse("test inurl:forum", index.schema(), index.tokenizers())+ .expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());+ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");+ assert_eq!(result.num_docs, 1);+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.first.com/forum");+ }+#[test]fn empty_query() {let schema = Arc::new(create_schema());- let query = Query::parse("", Arc::clone(&schema)).expect("Failed to parse query");+ let query = Query::parse("", Arc::clone(&schema), &TokenizerManager::new())+ .expect("Failed to parse query");assert!(query.is_empty())}diff --git a/src/query/parser.lalrpop b/src/query/parser.lalrpopdeleted file mode 100644index 9ec70847..00000000--- a/src/query/parser.lalrpop+++ /dev/null@@ -1,12 +0,0 @@-use crate::query::Term;--grammar;---pub Terms: Vec<Term<'input>> = <Term*> => <>;--Term: Term<'input> = {- <BasicTerm> => Term::Simple(<>),-}--BasicTerm: &'input str = r"[\S]*" => <>;\ No newline at end of filediff --git a/src/query/parser.rs b/src/query/parser.rsnew file mode 100644index 00000000..7948486e--- /dev/null+++ b/src/query/parser.rs@@ -0,0 +1,325 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::{+ query::{BooleanQuery, BoostQuery, Occur, TermQuery},+ schema::IndexRecordOption,+ tokenizer::{TextAnalyzer, TokenizerManager},+};++use crate::schema::{Field, ALL_FIELDS};++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub enum Term {+ Simple(String),+ Not(Box<Term>),+ Site(String),+ Title(String),+ Body(String),+ Url(String),+}++impl Term {+ pub fn as_tantivy_query(+ &self,+ fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],+ tokenizer_manager: &TokenizerManager,+ ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {+ match self {+ Term::Simple(term) => (+ Occur::Must,+ Box::new(BooleanQuery::new(Term::into_tantivy_simple(+ term,+ fields,+ tokenizer_manager,+ ))),+ ),+ Term::Not(subterm) => (+ Occur::MustNot,+ Box::new(BooleanQuery::new(vec![+ subterm.as_tantivy_query(fields, tokenizer_manager)+ ])),+ ),+ Term::Site(site) => (+ Occur::Must,+ Box::new(BooleanQuery::new(Term::into_tantivy_site(+ site,+ fields,+ tokenizer_manager,+ ))),+ ),+ Term::Title(title) => {+ let (field, entry) = fields+ .iter()+ .find(|(field, _)| {+ matches!(ALL_FIELDS[field.field_id() as usize], Field::Title)+ })+ .unwrap();+ (+ Occur::Must,+ Term::tantivy_term_query(field, entry, tokenizer_manager, title),+ )+ }+ Term::Body(body) => {+ let (field, entry) = fields+ .iter()+ .find(|(field, _)| {+ matches!(ALL_FIELDS[field.field_id() as usize], Field::AllBody)+ })+ .unwrap();+ (+ Occur::Must,+ Term::tantivy_term_query(field, entry, tokenizer_manager, body),+ )+ }+ Term::Url(url) => {+ let (field, entry) = fields+ .iter()+ .find(|(field, _)| matches!(ALL_FIELDS[field.field_id() as usize], Field::Url))+ .unwrap();+ (+ Occur::Must,+ Term::tantivy_term_query(field, entry, tokenizer_manager, url),+ )+ }+ }+ }++ fn into_tantivy_simple(+ term: &str,+ fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],+ tokenizer_manager: &TokenizerManager,+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {+ fields+ .iter()+ .filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].is_searchable())+ .into_iter()+ .map(|(field, entry)| {+ (+ Occur::Should,+ Term::tantivy_term_query(field, entry, tokenizer_manager, term),+ )+ })+ .collect()+ }++ fn into_tantivy_site(+ term: &str,+ fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],+ tokenizer_manager: &TokenizerManager,+ ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {+ fields+ .iter()+ .filter(|(field, _)| {+ matches!(+ ALL_FIELDS[field.field_id() as usize],+ Field::Domain | Field::Host+ )+ })+ .into_iter()+ .map(|(field, entry)| {+ (+ Occur::Should,+ Term::tantivy_term_query(field, entry, tokenizer_manager, term),+ )+ })+ .collect()+ }++ fn tantivy_term_query(+ field: &tantivy::schema::Field,+ entry: &tantivy::schema::FieldEntry,+ tokenizer_manager: &TokenizerManager,+ term: &str,+ ) -> Box<dyn tantivy::query::Query + 'static> {+ let analyzer = Term::get_tantivy_analyzer(entry, tokenizer_manager);+ let processed_terms = Term::process_tantivy_term(term, analyzer, *field);++ let processed_queries = processed_terms+ .map(|term| {+ (+ Occur::Must,+ Box::new(TermQuery::new(+ term,+ IndexRecordOption::WithFreqsAndPositions,+ )) as Box<dyn tantivy::query::Query>,+ )+ })+ .collect();+ let boost = ALL_FIELDS[field.field_id() as usize].boost().unwrap_or(1.0);++ Box::new(BoostQuery::new(+ Box::new(BooleanQuery::new(processed_queries)),+ boost,+ ))+ }++ fn get_tantivy_analyzer(+ entry: &tantivy::schema::FieldEntry,+ tokenizer_manager: &tantivy::tokenizer::TokenizerManager,+ ) -> Option<TextAnalyzer> {+ match entry.field_type() {+ tantivy::schema::FieldType::Str(options) => {+ options.get_indexing_options().and_then(|indexing_options| {+ let tokenizer_name = indexing_options.tokenizer();+ tokenizer_manager.get(tokenizer_name)+ })+ }+ _ => None,+ }+ }++ fn process_tantivy_term(+ term: &str,+ analyzer: Option<TextAnalyzer>,+ tantivy_field: tantivy::schema::Field,+ ) -> impl Iterator<Item = tantivy::Term> {+ match analyzer {+ None => vec![tantivy::Term::from_field_text(tantivy_field, term)].into_iter(),+ Some(tokenizer) => {+ let mut terms: Vec<tantivy::Term> = Vec::new();+ let mut token_stream = tokenizer.token_stream(term);+ token_stream.process(&mut |token| {+ let term = tantivy::Term::from_field_text(tantivy_field, &token.text);+ terms.push(term);+ });++ terms.into_iter()+ }+ }+ }+}++fn parse_term(term: &str) -> Box<Term> {+ // TODO: re-write this entire function once if-let chains become stable+ if let Some(not_term) = term.strip_prefix('-') {+ if !not_term.is_empty() && !not_term.starts_with('-') {+ Box::new(Term::Not(parse_term(not_term)))+ } else {+ Box::new(Term::Simple(term.to_string()))+ }+ } else if let Some(site) = term.strip_prefix("site:") {+ if !site.is_empty() {+ Box::new(Term::Site(site.to_string()))+ } else {+ Box::new(Term::Simple(term.to_string()))+ }+ } else if let Some(title) = term.strip_prefix("intitle:") {+ if !title.is_empty() {+ Box::new(Term::Title(title.to_string()))+ } else {+ Box::new(Term::Simple(term.to_string()))+ }+ } else if let Some(body) = term.strip_prefix("inbody:") {+ if !body.is_empty() {+ Box::new(Term::Body(body.to_string()))+ } else {+ Box::new(Term::Simple(term.to_string()))+ }+ } else if let Some(url) = term.strip_prefix("inurl:") {+ if !url.is_empty() {+ Box::new(Term::Url(url.to_string()))+ } else {+ Box::new(Term::Simple(term.to_string()))+ }+ } else {+ Box::new(Term::Simple(term.to_string()))+ }+}++#[allow(clippy::vec_box)]+pub fn parse(query: &str) -> Vec<Box<Term>> {+ query.split_whitespace().map(parse_term).collect()+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn parse_not() {+ assert_eq!(+ parse("this -that"),+ vec![+ Box::new(Term::Simple("this".to_string())),+ Box::new(Term::Not(Box::new(Term::Simple("that".to_string()))))+ ]+ );++ assert_eq!(+ parse("this -"),+ vec![+ Box::new(Term::Simple("this".to_string())),+ Box::new(Term::Simple("-".to_string()))+ ]+ );+ }++ #[test]+ fn double_not() {+ assert_eq!(+ parse("this --that"),+ vec![+ Box::new(Term::Simple("this".to_string())),+ Box::new(Term::Simple("--that".to_string()))+ ]+ );+ }++ #[test]+ fn site() {+ assert_eq!(+ parse("this site:test.com"),+ vec![+ Box::new(Term::Simple("this".to_string())),+ Box::new(Term::Site("test.com".to_string()))+ ]+ );+ }++ #[test]+ fn title() {+ assert_eq!(+ parse("this intitle:test"),+ vec![+ Box::new(Term::Simple("this".to_string())),+ Box::new(Term::Title("test".to_string()))+ ]+ );+ }++ #[test]+ fn body() {+ assert_eq!(+ parse("this inbody:test"),+ vec![+ Box::new(Term::Simple("this".to_string())),+ Box::new(Term::Body("test".to_string()))+ ]+ );+ }++ #[test]+ fn url() {+ assert_eq!(+ parse("this inurl:test"),+ vec![+ Box::new(Term::Simple("this".to_string())),+ Box::new(Term::Url("test".to_string()))+ ]+ );+ }+}diff --git a/src/searcher.rs b/src/searcher.rsindex c61f8d90..0daf536b 100644--- a/src/searcher.rs+++ b/src/searcher.rs@@ -57,7 +57,7 @@ impl Searcher {impl Searcher {pub fn search(&self, query: &str) -> Result<SearchResult> {let raw_query = query.to_string();- let query = Query::parse(query, self.index.schema())?;+ let query = Query::parse(query, self.index.schema(), self.index.tokenizers())?;if query.is_empty() {return Err(Error::EmptyQuery);@@ -65,7 +65,7 @@ impl Searcher {let ranker = Ranker::new(query.clone());let webpages = self.index.search(&query, ranker.collector())?;- let correction = self.index.spell_correction(query.simple_terms());+ let correction = self.index.spell_correction(&query.simple_terms());let entity = self.entity_indexdiff --git a/src/webgraph/graph_store.rs b/src/webgraph/graph_store.rsindex 7ecd931a..e9c483c6 100644--- a/src/webgraph/graph_store.rs+++ b/src/webgraph/graph_store.rs@@ -201,6 +201,8 @@ impl<S: Store> GraphStore<S> {self.assign_id(node, id);id}++ #[allow(unused)]pub fn outgoing_edges(&self, node: NodeID) -> Vec<Edge> {self.adjacency.borrow_mut()diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rsindex 41c28779..31032212 100644--- a/src/webgraph/mod.rs+++ b/src/webgraph/mod.rs@@ -274,6 +274,7 @@ impl<S: Store> Webgraph<S> {distances}+ #[allow(unused)]pub fn distances(&self, source: Node) -> HashMap<Node, usize> {self.full_graph.as_ref()@@ -293,6 +294,7 @@ impl<S: Store> Webgraph<S> {.unwrap_or_default()}+ #[allow(unused)]fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {self.full_graph.as_ref()@@ -307,6 +309,7 @@ impl<S: Store> Webgraph<S> {.unwrap_or_default()}+ #[allow(unused)]pub fn reversed_distances(&self, source: Node) -> HashMap<Node, usize> {self.full_graph.as_ref()@@ -319,6 +322,7 @@ impl<S: Store> Webgraph<S> {.unwrap_or_default()}+ #[allow(unused)]pub fn host_distances(&self, source: Node) -> HashMap<Node, usize> {self.host_graph.as_ref()@@ -352,6 +356,7 @@ impl<S: Store> Webgraph<S> {.unwrap_or_default()}+ #[allow(unused)]pub fn host_reversed_distances(&self, source: Node) -> HashMap<Node, usize> {self.host_graph.as_ref()@@ -401,6 +406,7 @@ impl<S: Store> Webgraph<S> {.collect()}+ #[allow(unused)]pub fn harmonic_centrality(&self) -> HashMap<Node, f64> {self.full_graph.as_ref()diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 3a699570..1572d948 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -93,6 +93,7 @@ pub struct Webpage<'a> {}impl<'a> Webpage<'a> {+ #[cfg(test)]pub fn new(html: &'a str,url: &str,@@ -435,7 +436,7 @@ impl<'a> Html<'a> {return Err(Error::EmptyField("title"));}- doc.add_text(tantivy_field, title.unwrap())+ doc.add_text(tantivy_field, title.unwrap());}Field::CleanBody | Field::StemmedCleanBody => {let text = self.clean_text();