Skip to content

web search done right

Custom tantivy query implementation. Preparations for expanded spans ranking which allows us to rank documents based on term proximity. Overall this change gives more control over the search+ranking procedure.

Mikkel Denkerd5efc66c3163
diff --git a/justfile b/justfile
index 038f2e35..3657d8d8 100644
--- a/justfile
+++ b/justfile
@@ -10,3 +10,10 @@
@frontend:
cargo watch -x 'run -- frontend data/index queries_us.csv'
+
+@local:
+ rm -rf data/index/
+ cargo build --release
+ ./target/release/cuely indexer local configs/indexer/local.toml
+ mv data/index/CC-MAIN-20220116093137-20220116123137-00049.warc.gz/* data/index/
+ ./target/release/cuely frontend data/index/ queries_us.csv
\ No newline at end of file
diff --git a/src/frontend/mod.rs b/src/frontend/mod.rs
index 0689a94c..8fbaecd8 100644
--- a/src/frontend/mod.rs
+++ b/src/frontend/mod.rs
@@ -16,7 +16,7 @@
use axum::{Extension, Router};
-use crate::{autosuggest::Autosuggest, index::Index};
+use crate::{autosuggest::Autosuggest, index::Index, searcher::Searcher};
use anyhow::Result;
use std::sync::Arc;
@@ -35,7 +35,7 @@ pub mod search;
pub struct HtmlTemplate<T>(T);
pub struct State {
- pub index: Index,
+ pub searcher: Searcher,
pub autosuggest: Autosuggest,
}
@@ -58,9 +58,10 @@ where
pub fn router(index_path: &str, queries_csv_path: &str) -> Result<Router> {
let search_index = Index::open(index_path)?;
let autosuggest = Autosuggest::load_csv(queries_csv_path)?;
+ let searcher = Searcher::from(search_index);
let state = Arc::new(State {
- index: search_index,
+ searcher,
autosuggest,
});
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index 441d4318..7d81e44d 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -24,8 +24,6 @@ use std::collections::HashMap;
use std::sync::Arc;
use super::{HtmlTemplate, State};
-use crate::query::Query;
-use crate::ranking::Ranker;
use askama::Template;
use axum::{extract, response::IntoResponse};
@@ -95,13 +93,7 @@ pub async fn route(
if let Some(query) = params.get("q") {
displayed_query = query.clone();
- let query = Query::parse(query).expect("Failed to parse query");
- let ranker = Ranker::new(query.clone());
-
- let result = state
- .index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = state.searcher.search(query).expect("Search failed");
search_result = result
.documents
diff --git a/src/index.rs b/src/index.rs
index f32943b2..d02f503b 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -16,7 +16,8 @@ use serde::{Deserialize, Serialize};
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use tantivy::collector::{Collector, Count};
use tantivy::merge_policy::NoMergePolicy;
-use tantivy::{DocAddress, Document, LeasedItem};
+use tantivy::schema::Schema;
+use tantivy::{DocAddress, Document, IndexReader, IndexWriter, LeasedItem};
use crate::directory::{self, DirEntry};
use crate::query::Query;
@@ -28,13 +29,14 @@ use crate::Result;
use crate::{schema::create_schema, tokenizer::Tokenizer};
use std::fs;
use std::path::Path;
+use std::sync::Arc;
pub struct Index {
pub path: String,
tantivy_index: tantivy::Index,
- writer: tantivy::IndexWriter,
- reader: tantivy::IndexReader,
- schema: tantivy::schema::Schema,
+ writer: IndexWriter,
+ reader: IndexReader,
+ schema: Arc<Schema>,
}
impl Index {
@@ -72,7 +74,7 @@ impl Index {
Ok(Index {
writer,
reader: tantivy_index.reader()?,
- schema,
+ schema: Arc::new(schema),
path: path.as_ref().to_str().unwrap().to_string(),
tantivy_index,
})
@@ -95,10 +97,9 @@ impl Index {
where
C: Collector<Fruit = Vec<(f64, tantivy::DocAddress)>>,
{
- let tantivy_query = query.tantivy(&self.schema, self.tantivy_index.tokenizers());
let searcher = self.reader.searcher();
- let (count, docs) = searcher.search(&tantivy_query, &(Count, collector))?;
+ let (count, docs) = searcher.search(query, &(Count, collector))?;
let mut webpages: Vec<RetrievedWebpage> = docs
.into_iter()
@@ -187,6 +188,10 @@ impl Index {
Self::open(path).expect("failed to open index")
}
+
+ pub fn schema(&self) -> Arc<Schema> {
+ Arc::clone(&self.schema)
+ }
}
#[derive(Default, Debug)]
@@ -287,7 +292,7 @@ mod tests {
#[test]
fn simple_search() {
let mut index = Index::temporary().expect("Unable to open index");
- let query = Query::parse("website").expect("Failed to parse query");
+ let query = Query::parse("website", index.schema()).expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
let result = index
@@ -328,7 +333,8 @@ mod tests {
#[test]
fn document_not_matching() {
let mut index = Index::temporary().expect("Unable to open index");
- let query = Query::parse("this query should not match").expect("Failed to parse query");
+ let query = Query::parse("this query should not match", index.schema())
+ .expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
index
@@ -362,7 +368,7 @@ mod tests {
#[test]
fn english_stemming() {
let mut index = Index::temporary().expect("Unable to open index");
- let query = Query::parse("runner").expect("Failed to parse query");
+ let query = Query::parse("runner", index.schema()).expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
index
@@ -396,7 +402,7 @@ mod tests {
#[test]
fn stemmed_query_english() {
let mut index = Index::temporary().expect("Unable to open index");
- let query = Query::parse("runners").expect("Failed to parse query");
+ let query = Query::parse("runners", index.schema()).expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
index
@@ -430,7 +436,7 @@ mod tests {
#[test]
fn searchable_backlinks() {
let mut index = Index::temporary().expect("Unable to open index");
- let query = Query::parse("great site").expect("Failed to parse query");
+ let query = Query::parse("great site", index.schema()).expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
index
@@ -493,7 +499,7 @@ mod tests {
#[test]
fn limited_top_docs() {
let mut index = Index::temporary().expect("Unable to open index");
- let query = Query::parse("runner").expect("Failed to parse query");
+ let query = Query::parse("runner", index.schema()).expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
for _ in 0..100 {
@@ -529,7 +535,7 @@ mod tests {
#[test]
fn host_search() {
let mut index = Index::temporary().expect("Unable to open index");
- let query = Query::parse("dr").expect("Failed to parse query");
+ let query = Query::parse("dr", index.schema()).expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
index
@@ -592,7 +598,7 @@ mod tests {
let deserialized_frozen: FrozenIndex = bincode::deserialize(&bytes).unwrap();
let index: Index = deserialized_frozen.into();
- let query = Query::parse("website").expect("Failed to parse query");
+ let query = Query::parse("website", index.schema()).expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
let result = index
@@ -652,7 +658,7 @@ mod tests {
let mut index = index1.merge(index2);
index.commit().unwrap();
- let query = Query::parse("website").expect("Failed to parse query");
+ let query = Query::parse("website", index.schema()).expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
let result = index
diff --git a/src/query.rs b/src/query.rs
deleted file mode 100644
index 69b16205..00000000
--- a/src/query.rs
+++ /dev/null
@@ -1,206 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{schema::ALL_FIELDS, Error, Result};
-use nom::{
- bytes::complete::{tag, take_while},
- character::complete::multispace0,
- error::ParseError,
- multi::separated_list0,
- sequence::preceded,
- IResult,
-};
-use tantivy::{
- query::{BooleanQuery, BoostQuery, Occur, TermQuery},
- schema::{Field, FieldEntry, IndexRecordOption, Schema},
- tokenizer::{TextAnalyzer, TokenizerManager},
- Term,
-};
-
-/// A combinator that takes a parser `inner` and produces a parser that also consumes leading
-/// whitespace, returning the output of `inner`.
-fn ws<'a, F: 'a, O, E: ParseError<&'a str>>(
- inner: F,
-) -> impl FnMut(&'a str) -> IResult<&'a str, O, E>
-where
- F: Fn(&'a str) -> IResult<&'a str, O, E>,
-{
- preceded(multispace0, inner)
-}
-
-fn term(term: &str) -> IResult<&str, &str> {
- ws(take_while(|c: char| !c.is_whitespace()))(term)
-}
-#[derive(Clone)]
-pub struct Query {
- pub terms: Vec<String>,
-}
-
-impl Query {
- pub fn parse(query: &str) -> Result<Query> {
- match separated_list0(tag(" "), term)(query) {
- Ok((remaining, terms)) => {
- debug_assert!(remaining.is_empty());
-
- let terms = terms
- .into_iter()
- .filter(|term| !term.is_empty())
- .map(|term| term.to_string())
- .collect();
- Ok(Query { terms })
- }
- Err(error) => Err(Error::ParsingError(error.to_string())),
- }
- }
-
- fn get_tantivy_analyzer(
- entry: &FieldEntry,
- tokenizer_manager: &TokenizerManager,
- ) -> Option<TextAnalyzer> {
- match entry.field_type() {
- tantivy::schema::FieldType::Str(options) => {
- options.get_indexing_options().map(|indexing_options| {
- let tokenizer_name = indexing_options.tokenizer();
- tokenizer_manager
- .get(tokenizer_name)
- .expect("Unknown tokenizer")
- })
- }
- _ => None,
- }
- }
-
- fn process_tantivy_term(
- term: &str,
- analyzer: Option<TextAnalyzer>,
- tantivy_field: Field,
- ) -> impl Iterator<Item = Term> {
- match analyzer {
- None => vec![Term::from_field_text(tantivy_field, term)].into_iter(),
- Some(tokenizer) => {
- let mut terms: Vec<Term> = Vec::new();
- let mut token_stream = tokenizer.token_stream(term);
- token_stream.process(&mut |token| {
- let term = Term::from_field_text(tantivy_field, &token.text);
- terms.push(term);
- });
-
- terms.into_iter()
- }
- }
- }
-
- pub fn tantivy(
- &self,
- schema: &Schema,
- tokenizer_manager: &TokenizerManager,
- ) -> Box<dyn tantivy::query::Query> {
- let queries = self
- .terms
- .iter()
- .map(|term| {
- let one_term = schema
- .fields()
- .into_iter()
- .map(|(field, entry)| {
- let analyzer = Query::get_tantivy_analyzer(entry, tokenizer_manager);
- let processed_terms = Query::process_tantivy_term(term, analyzer, field);
-
- let processed_queries = processed_terms
- .map(|term| {
- (
- Occur::Should,
- Box::new(TermQuery::new(
- term,
- IndexRecordOption::WithFreqsAndPositions,
- ))
- as Box<dyn tantivy::query::Query>,
- )
- })
- .collect();
- let boost = ALL_FIELDS[field.field_id() as usize].boost();
-
- (
- Occur::Should,
- Box::new(BoostQuery::new(
- Box::new(BooleanQuery::new(processed_queries)),
- boost.unwrap_or(1.0),
- )) as Box<dyn tantivy::query::Query>,
- )
- })
- .collect();
-
- Box::new(BooleanQuery::new(one_term)) as Box<dyn tantivy::query::Query>
- })
- .map(|term_query| (Occur::Must, term_query))
- .collect();
-
- Box::new(BooleanQuery::new(queries))
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn simple_parse() {
- let query = Query::parse("This is a simple query").expect("Failed to parse query");
-
- assert_eq!(
- query.terms,
- vec![
- "This".to_string(),
- "is".to_string(),
- "a".to_string(),
- "simple".to_string(),
- "query".to_string(),
- ]
- );
- }
-
- #[test]
- fn parse_trailing_leading_whitespace() {
- let query = Query::parse(" This is a simple query ").expect("Failed to parse query");
-
- assert_eq!(
- query.terms,
- vec![
- "This".to_string(),
- "is".to_string(),
- "a".to_string(),
- "simple".to_string(),
- "query".to_string(),
- ]
- );
- }
-
- #[test]
- fn parse_weird_characters() {
- let terms = Query::parse("123").expect("Failed to parse query").terms;
- assert_eq!(terms, vec!["123".to_string()]);
-
- let terms = Query::parse("123 33").expect("Failed to parse query").terms;
- assert_eq!(terms, vec!["123".to_string(), "33".to_string()]);
-
- let terms = Query::parse("term! term# $")
- .expect("Failed to parse query")
- .terms;
- assert_eq!(
- terms,
- vec!["term!".to_string(), "term#".to_string(), "$".to_string()]
- );
- }
-}
diff --git a/src/query/bm25.rs b/src/query/bm25.rs
new file mode 100644
index 00000000..d880b776
--- /dev/null
+++ b/src/query/bm25.rs
@@ -0,0 +1,143 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use serde::{Deserialize, Serialize};
+
+use tantivy::fieldnorm::FieldNormReader;
+use tantivy::query::Explanation;
+use tantivy::{Score, Searcher, Term};
+
+const K1: Score = 1.2;
+const B: Score = 0.75;
+
+pub(crate) fn idf(doc_freq: u64, doc_count: u64) -> Score {
+ assert!(doc_count >= doc_freq, "{} >= {}", doc_count, doc_freq);
+ let x = ((doc_count - doc_freq) as Score + 0.5) / (doc_freq as Score + 0.5);
+ (1.0 + x).ln()
+}
+
+fn cached_tf_component(fieldnorm: u32, average_fieldnorm: Score) -> Score {
+ K1 * (1.0 - B + B * fieldnorm as Score / average_fieldnorm)
+}
+
+fn compute_tf_cache(average_fieldnorm: Score) -> [Score; 256] {
+ let mut cache: [Score; 256] = [0.0; 256];
+ for (fieldnorm_id, cache_mut) in cache.iter_mut().enumerate() {
+ let fieldnorm = FieldNormReader::id_to_fieldnorm(fieldnorm_id as u8);
+ *cache_mut = cached_tf_component(fieldnorm, average_fieldnorm);
+ }
+ cache
+}
+
+#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
+pub struct Bm25Params {
+ pub idf: Score,
+ pub avg_fieldnorm: Score,
+}
+
+#[derive(Clone)]
+pub struct Bm25Weight {
+ idf_explain: Explanation,
+ weight: Score,
+ cache: [Score; 256],
+ average_fieldnorm: Score,
+}
+
+impl Bm25Weight {
+ pub fn boost_by(&self, boost: Score) -> Bm25Weight {
+ Bm25Weight {
+ idf_explain: self.idf_explain.clone(),
+ weight: self.weight * boost,
+ cache: self.cache,
+ average_fieldnorm: self.average_fieldnorm,
+ }
+ }
+
+ pub fn for_terms(searcher: &Searcher, terms: &[Term]) -> tantivy::Result<Bm25Weight> {
+ assert!(!terms.is_empty(), "Bm25 requires at least one term");
+ let field = terms[0].field();
+ for term in &terms[1..] {
+ assert_eq!(
+ term.field(),
+ field,
+ "All terms must belong to the same field."
+ );
+ }
+
+ let mut total_num_tokens = 0u64;
+ let mut total_num_docs = 0u64;
+ for segment_reader in searcher.segment_readers() {
+ let inverted_index = segment_reader.inverted_index(field)?;
+ total_num_tokens += inverted_index.total_num_tokens();
+ total_num_docs += u64::from(segment_reader.max_doc());
+ }
+ let average_fieldnorm = total_num_tokens as Score / total_num_docs as Score;
+
+ if terms.len() == 1 {
+ let term_doc_freq = searcher.doc_freq(&terms[0])?;
+ Ok(Bm25Weight::for_one_term(
+ term_doc_freq,
+ total_num_docs,
+ average_fieldnorm,
+ ))
+ } else {
+ let mut idf_sum: Score = 0.0;
+ for term in terms {
+ let term_doc_freq = searcher.doc_freq(term)?;
+ idf_sum += idf(term_doc_freq, total_num_docs);
+ }
+ let idf_explain = Explanation::new("idf", idf_sum);
+ Ok(Bm25Weight::new(idf_explain, average_fieldnorm))
+ }
+ }
+
+ pub fn for_one_term(
+ term_doc_freq: u64,
+ total_num_docs: u64,
+ avg_fieldnorm: Score,
+ ) -> Bm25Weight {
+ let idf = idf(term_doc_freq, total_num_docs);
+ let mut idf_explain =
+ Explanation::new("idf, computed as log(1 + (N - n + 0.5) / (n + 0.5))", idf);
+ idf_explain.add_const(
+ "n, number of docs containing this term",
+ term_doc_freq as Score,
+ );
+ idf_explain.add_const("N, total number of docs", total_num_docs as Score);
+ Bm25Weight::new(idf_explain, avg_fieldnorm)
+ }
+
+ pub(crate) fn new(idf_explain: Explanation, average_fieldnorm: Score) -> Bm25Weight {
+ let weight = idf_explain.value() * (1.0 + K1);
+ Bm25Weight {
+ idf_explain,
+ weight,
+ cache: compute_tf_cache(average_fieldnorm),
+ average_fieldnorm,
+ }
+ }
+
+ #[inline]
+ pub fn score(&self, fieldnorm_id: u8, term_freq: u32) -> Score {
+ self.weight * self.tf_factor(fieldnorm_id, term_freq)
+ }
+
+ #[inline]
+ pub(crate) fn tf_factor(&self, fieldnorm_id: u8, term_freq: u32) -> Score {
+ let term_freq = term_freq as Score;
+ let norm = self.cache[fieldnorm_id as usize];
+ term_freq / (term_freq + norm)
+ }
+}
diff --git a/src/query/mod.rs b/src/query/mod.rs
new file mode 100644
index 00000000..021fe684
--- /dev/null
+++ b/src/query/mod.rs
@@ -0,0 +1,217 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{query::weight::Weight, schema::ALL_FIELDS, Error, Result};
+use nom::{
+ bytes::complete::{tag, take_while},
+ character::complete::multispace0,
+ error::ParseError,
+ multi::separated_list0,
+ sequence::preceded,
+ IResult,
+};
+use std::{collections::BTreeMap, sync::Arc};
+use tantivy::{
+ schema::{FieldType, IndexRecordOption, Schema},
+ tokenizer::TextAnalyzer,
+};
+
+use self::bm25::Bm25Weight;
+
+mod bm25;
+mod scorer;
+mod union;
+mod weight;
+
+/// A combinator that takes a parser `inner` and produces a parser that also consumes leading
+/// whitespace, returning the output of `inner`.
+fn ws<'a, F: 'a, O, E: ParseError<&'a str>>(
+ inner: F,
+) -> impl FnMut(&'a str) -> IResult<&'a str, O, E>
+where
+ F: Fn(&'a str) -> IResult<&'a str, O, E>,
+{
+ preceded(multispace0, inner)
+}
+
+fn term(term: &str) -> IResult<&str, &str> {
+ ws(take_while(|c: char| !c.is_whitespace()))(term)
+}
+pub struct FieldData {
+ tantivy: tantivy::schema::Field,
+ scoring: Bm25Weight,
+ index_record_option: Option<IndexRecordOption>,
+ boost: Option<f32>,
+ analyzer: Option<TextAnalyzer>,
+}
+
+#[derive(Clone, Debug)]
+pub struct Query {
+ terms: Vec<String>,
+ schema: Arc<Schema>,
+}
+
+impl Query {
+ pub fn parse(query: &str, schema: Arc<Schema>) -> Result<Query> {
+ match separated_list0(tag(" "), term)(query) {
+ Ok((remaining, terms)) => {
+ debug_assert!(remaining.is_empty());
+
+ let terms = terms
+ .into_iter()
+ .filter(|term| !term.is_empty())
+ .map(|term| term.to_string())
+ .collect();
+ Ok(Query { terms, schema })
+ }
+ Err(error) => Err(Error::ParsingError(error.to_string())),
+ }
+ }
+}
+
+impl tantivy::query::Query for Query {
+ fn weight(
+ &self,
+ searcher: &tantivy::Searcher,
+ scoring_enabled: bool,
+ ) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
+ let terms = self.terms.clone();
+
+ let fields: Vec<_> = ALL_FIELDS
+ .iter()
+ .map(|field| {
+ let tantivy_field = searcher.schema().get_field(field.as_str()).unwrap();
+ let analyzer = searcher.index().tokenizer_for_field(tantivy_field).ok();
+
+ let tantivy_terms: Vec<_> = terms
+ .iter()
+ .flat_map(|term_text| {
+ let terms = analyzer
+ .as_ref()
+ .map(|analyzer| {
+ let mut terms = Vec::new();
+
+ let mut stream = analyzer.token_stream(term_text);
+
+ while let Some(token) = stream.next() {
+ terms.push(token.text.clone());
+ }
+
+ terms
+ })
+ .unwrap_or_else(|| vec![term_text.clone()]);
+
+ terms
+ .into_iter()
+ .map(|term| tantivy::Term::from_field_text(tantivy_field, &term))
+ })
+ .collect();
+
+ FieldData {
+ tantivy: tantivy_field,
+ index_record_option: searcher
+ .schema()
+ .get_field_entry(tantivy_field)
+ .field_type()
+ .get_index_record_option(),
+ boost: field.boost(),
+ scoring: Bm25Weight::for_terms(searcher, &tantivy_terms).unwrap(),
+ analyzer,
+ }
+ })
+ .collect();
+
+ Ok(Box::new(Weight::new(terms, fields, scoring_enabled)))
+ }
+
+ fn query_terms(&self, terms: &mut BTreeMap<tantivy::Term, bool>) {
+ for term_text in &self.terms {
+ for (field, entry) in self.schema.fields() {
+ if let FieldType::Str(_) = entry.field_type() {
+ terms.insert(tantivy::Term::from_field_text(field, term_text), true);
+ }
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::schema::create_schema;
+
+ use super::*;
+
+ #[test]
+ fn simple_parse() {
+ let schema = Arc::new(create_schema());
+
+ let query = Query::parse("This is a simple query", Arc::clone(&schema))
+ .expect("Failed to parse query");
+
+ assert_eq!(
+ query.terms,
+ vec![
+ "This".to_string(),
+ "is".to_string(),
+ "a".to_string(),
+ "simple".to_string(),
+ "query".to_string(),
+ ]
+ );
+ }
+
+ #[test]
+ fn parse_trailing_leading_whitespace() {
+ let schema = Arc::new(create_schema());
+
+ let query = Query::parse(" This is a simple query ", Arc::clone(&schema))
+ .expect("Failed to parse query");
+
+ assert_eq!(
+ query.terms,
+ vec![
+ "This".to_string(),
+ "is".to_string(),
+ "a".to_string(),
+ "simple".to_string(),
+ "query".to_string(),
+ ]
+ );
+ }
+
+ #[test]
+ fn parse_weird_characters() {
+ let schema = Arc::new(create_schema());
+
+ let terms = Query::parse("123", Arc::clone(&schema))
+ .expect("Failed to parse query")
+ .terms;
+ assert_eq!(terms, vec!["123".to_string()]);
+
+ let terms = Query::parse("123 33", Arc::clone(&schema))
+ .expect("Failed to parse query")
+ .terms;
+ assert_eq!(terms, vec!["123".to_string(), "33".to_string()]);
+
+ let terms = Query::parse("term! term# $", Arc::clone(&schema))
+ .expect("Failed to parse query")
+ .terms;
+ assert_eq!(
+ terms,
+ vec!["term!".to_string(), "term#".to_string(), "$".to_string()]
+ );
+ }
+}
diff --git a/src/query/scorer.rs b/src/query/scorer.rs
new file mode 100644
index 00000000..fe60a8c7
--- /dev/null
+++ b/src/query/scorer.rs
@@ -0,0 +1,76 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::query::bm25::Bm25Weight;
+
+use tantivy::fieldnorm::FieldNormReader;
+use tantivy::postings::{Postings, SegmentPostings};
+use tantivy::DocSet;
+use tantivy::{DocId, Score};
+
+#[derive(Clone)]
+pub struct Scorer {
+ postings: SegmentPostings,
+ fieldnorm_reader: FieldNormReader,
+ similarity_weight: Bm25Weight,
+}
+
+impl Scorer {
+ pub fn new(
+ postings: SegmentPostings,
+ fieldnorm_reader: FieldNormReader,
+ similarity_weight: Bm25Weight,
+ ) -> Scorer {
+ Scorer {
+ postings,
+ fieldnorm_reader,
+ similarity_weight,
+ }
+ }
+
+ pub fn term_freq(&self) -> u32 {
+ self.postings.term_freq()
+ }
+
+ pub fn fieldnorm_id(&self) -> u8 {
+ self.fieldnorm_reader.fieldnorm_id(self.doc())
+ }
+}
+
+impl DocSet for Scorer {
+ fn advance(&mut self) -> DocId {
+ self.postings.advance()
+ }
+
+ fn seek(&mut self, target: DocId) -> DocId {
+ self.postings.seek(target)
+ }
+
+ fn doc(&self) -> DocId {
+ self.postings.doc()
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.postings.size_hint()
+ }
+}
+
+impl tantivy::query::Scorer for Scorer {
+ fn score(&mut self) -> Score {
+ let fieldnorm_id = self.fieldnorm_id();
+ let term_freq = self.term_freq();
+ self.similarity_weight.score(fieldnorm_id, term_freq)
+ }
+}
diff --git a/src/query/union.rs b/src/query/union.rs
new file mode 100644
index 00000000..3db21791
--- /dev/null
+++ b/src/query/union.rs
@@ -0,0 +1,99 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use tantivy::query::Scorer;
+use tantivy::{DocId, DocSet, Score, TERMINATED};
+
+/// Creates a `DocSet` that iterate through the union of two or more `DocSet`s.
+pub struct Union<TScorer> {
+ docsets: Vec<TScorer>,
+ score: Score,
+ doc: DocId,
+}
+
+impl<TScorer> From<Vec<TScorer>> for Union<TScorer>
+where
+ TScorer: Scorer,
+{
+ fn from(docsets: Vec<TScorer>) -> Union<TScorer> {
+ let non_empty_docsets: Vec<TScorer> = docsets
+ .into_iter()
+ .filter(|docset| docset.doc() != TERMINATED)
+ .collect();
+ let mut union = Union {
+ docsets: non_empty_docsets,
+ doc: 0,
+ score: 0.0,
+ };
+
+ union.advance();
+ union
+ }
+}
+
+impl<TScorer> DocSet for Union<TScorer>
+where
+ TScorer: Scorer,
+{
+ fn advance(&mut self) -> DocId {
+ if let Some(min_doc) = self.docsets.iter().map(|scorer| scorer.doc()).min() {
+ self.doc = min_doc;
+
+ if self.doc == TERMINATED {
+ return self.doc;
+ }
+
+ self.score = self
+ .docsets
+ .iter_mut()
+ .filter(|scorer| scorer.doc() == self.doc)
+ .map(|scorer| scorer.score())
+ .sum();
+
+ for scorer in self
+ .docsets
+ .iter_mut()
+ .filter(|scorer| scorer.doc() == min_doc)
+ {
+ scorer.advance();
+ }
+ } else {
+ self.doc = TERMINATED;
+ }
+
+ self.doc
+ }
+
+ fn doc(&self) -> DocId {
+ self.doc
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.docsets
+ .iter()
+ .map(|docset| docset.size_hint())
+ .max()
+ .unwrap_or(0u32)
+ }
+}
+
+impl<TScorer> Scorer for Union<TScorer>
+where
+ TScorer: Scorer,
+{
+ fn score(&mut self) -> Score {
+ self.score
+ }
+}
diff --git a/src/query/weight.rs b/src/query/weight.rs
new file mode 100644
index 00000000..cd9f9008
--- /dev/null
+++ b/src/query/weight.rs
@@ -0,0 +1,109 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use super::scorer::Scorer;
+use super::union::Union;
+use super::FieldData;
+use tantivy::fieldnorm::FieldNormReader;
+use tantivy::postings::SegmentPostings;
+use tantivy::query::{intersect_scorers, Explanation};
+use tantivy::{DocId, Score, SegmentReader};
+
+pub struct Weight {
+ terms: Vec<String>,
+ fields: Vec<FieldData>,
+ scoring_enabled: bool,
+}
+
+impl tantivy::query::Weight for Weight {
+ fn scorer(
+ &self,
+ reader: &SegmentReader,
+ boost: Score,
+ ) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
+ let mut field_scorers: Vec<Box<dyn tantivy::query::Scorer>> = Vec::new();
+
+ for field_data in &self.fields {
+ let inverted_index = reader.inverted_index(field_data.tantivy)?;
+ let mut term_scorers: Vec<Box<dyn tantivy::query::Scorer>> = Vec::new();
+
+ for term_text in &self.terms {
+ let processed_terms: Vec<String> = field_data
+ .analyzer
+ .as_ref()
+ .map(|analyzer| {
+ let mut terms = Vec::new();
+
+ let mut stream = analyzer.token_stream(term_text);
+ while let Some(token) = stream.next() {
+ terms.push(token.text.clone());
+ }
+
+ terms
+ })
+ .unwrap_or_else(|| vec![term_text.clone()]);
+
+ for term in processed_terms {
+ let fieldnorm_reader_opt = if self.scoring_enabled {
+ reader.get_fieldnorms_reader(field_data.tantivy).ok()
+ } else {
+ None
+ };
+ let fieldnorm_reader = fieldnorm_reader_opt
+ .unwrap_or_else(|| FieldNormReader::constant(reader.max_doc(), 1));
+ let similarity_weight = field_data
+ .scoring
+ .boost_by(field_data.boost.unwrap_or(1.0) * boost);
+
+ let term = tantivy::Term::from_field_text(field_data.tantivy, &term);
+
+ let postings_opt: Option<SegmentPostings> = field_data
+ .index_record_option
+ .and_then(|index_record_option| {
+ inverted_index
+ .read_postings(&term, index_record_option)
+ .unwrap()
+ });
+
+ let posting = postings_opt.unwrap_or_else(SegmentPostings::empty);
+
+ term_scorers.push(Box::new(Scorer::new(
+ posting,
+ fieldnorm_reader,
+ similarity_weight,
+ )));
+ }
+ }
+
+ field_scorers.push(intersect_scorers(term_scorers));
+ }
+
+ Ok(Box::new(Union::from(field_scorers)))
+ }
+
+ fn explain(&self, _reader: &SegmentReader, _doc: DocId) -> tantivy::Result<Explanation> {
+ unimplemented!();
+ }
+}
+
+impl Weight {
+ pub fn new(terms: Vec<String>, fields: Vec<FieldData>, scoring_enabled: bool) -> Weight {
+ Weight {
+ terms,
+ fields,
+ scoring_enabled,
+ }
+ }
+}
diff --git a/src/schema.rs b/src/schema.rs
index ac716d2f..83314c46 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -66,7 +66,7 @@ impl Field {
self.default_text_options_with_tokenizer(NormalTokenizer::as_str())
}
- fn options(&self) -> IndexingOption {
+ pub fn options(&self) -> IndexingOption {
match self {
Field::Title => IndexingOption::Text(self.default_text_options().set_stored()),
Field::Body => IndexingOption::Text(self.default_text_options().set_stored()),
@@ -139,7 +139,7 @@ pub fn create_schema() -> tantivy::schema::Schema {
builder.build()
}
-enum IndexingOption {
+pub enum IndexingOption {
Text(tantivy::schema::TextOptions),
Numeric(tantivy::schema::NumericOptions),
}
diff --git a/src/searcher.rs b/src/searcher.rs
index 7421de7c..2d53ebae 100644
--- a/src/searcher.rs
+++ b/src/searcher.rs
@@ -29,7 +29,6 @@ pub struct Searcher {
index: Index,
}
-#[cfg(test)]
impl From<Index> for Searcher {
fn from(index: Index) -> Self {
Searcher { index }
@@ -38,7 +37,7 @@ impl From<Index> for Searcher {
impl Searcher {
pub fn search(&self, query: &str) -> Result<SearchResult> {
- let query = Query::parse(query)?;
+ let query = Query::parse(query, self.index.schema())?;
let ranker = Ranker::new(query.clone());
self.index.search(&query, ranker.collector())
}
diff --git a/src/snippet.rs b/src/snippet.rs
index b121e8a6..7b2ec269 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -242,10 +242,10 @@ use crate::{query::Query, schema::Field, Result};
pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Result<String> {
let generator = SnippetGenerator::create(
searcher,
- &query.tantivy(searcher.schema(), searcher.index().tokenizers()),
+ query,
searcher
.schema()
- .get_field(Field::StemmedBody.as_str())
+ .get_field(Field::Body.as_str())
.expect("Failed to get body field"),
)?;
@@ -314,41 +314,6 @@ Survey in 2016, 2017, and 2018."#;
assert_eq!(result.documents[0].snippet, "<b>Rust</b> is a systems programming <b>language</b> sponsored by Mozilla which describes it as a \"safe, concurrent, practical <b>language</b>\", supporting functional and imperative-procedural paradigms. <b>Rust</b> is...".to_string());
}
- #[test]
- fn stemmed_words_snippet_highlight() {
- let mut index = Index::temporary().expect("Unable to open index");
-
- index
- .insert(Webpage::new(
- &format!(
- r#"
- <html>
- <head>
- <title>Website for runners</title>
- </head>
- <body>
- {}
- </body>
- </html>
- "#,
- TEST_TEXT
- ),
- "https://www.example.com",
- vec![],
- 1.0,
- ))
- .expect("failed to parse webpage");
- index.commit().expect("failed to commit index");
-
- let searcher = Searcher::from(index);
-
- let result = searcher.search("describe").expect("Search failed");
-
- assert_eq!(result.num_docs, 1);
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].snippet, "Rust is a systems programming language sponsored by Mozilla which <b>describes</b> it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is...".to_string());
- }
-
#[test]
fn test_snippet() {
let terms = btreemap! {
diff --git a/src/webpage/just_text.rs b/src/webpage/just_text.rs
index c01e92b4..9ca7997c 100644
--- a/src/webpage/just_text.rs
+++ b/src/webpage/just_text.rs
@@ -26,9 +26,9 @@ use super::{lexer::Token, Preprocessor};
// reference implementation: https://github.com/miso-belica/jusText/blob/main/justext/core.py
const MAX_LINK_DENSITY_DEFAULT: f64 = 0.2;
-const LENGTH_LOW_DEFAULT: usize = 70;
+const LENGTH_LOW_DEFAULT: usize = 50;
const LENGTH_HIGH_DEFAULT: usize = 200;
-const STOPWORDS_LOW_DEFAULT: f64 = 0.30;
+const STOPWORDS_LOW_DEFAULT: f64 = 0.28;
const STOPWORDS_HIGH_DEFAULT: f64 = 0.32;
const MAX_HEADING_DISTANCE_DEFAULT: usize = 200;
@@ -117,7 +117,8 @@ lazy_static! {
"../../stopwords/Serbian.txt" => Lang::Srp,
"../../stopwords/Slovak.txt" => Lang::Slk,
"../../stopwords/Slovenian.txt" => Lang::Slv,
- "../../stopwords/Spanish.txt" => Lang::Spa
+ "../../stopwords/Spanish.txt" => Lang::Spa,
+ "../../stopwords/Japanese.txt" => Lang::Jpn
)
};
}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index ff56fd90..3a08f6d2 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -566,7 +566,7 @@ mod tests {
}
#[test]
- #[ignore = "JustText doesn't find any content on the sites. Maybe we should tune parameters?"]
+ #[ignore = "JustText doesn't find any content on the sites. How should we split into words for Japanese, Chinese etc.?"]
fn hard_parsing() {
let webpage = Html::parse(include_str!("../../testcases_parsing/yasudaya.html"), "");
assert_eq!(
diff --git a/stopwords/Japanese.txt b/stopwords/Japanese.txt
new file mode 100644
index 00000000..e324115f
--- /dev/null
+++ b/stopwords/Japanese.txt
@@ -0,0 +1,44 @@
+これ
+それ
+あれ
+この
+その
+あの
+ここ
+そこ
+あそこ
+こちら
+どこ
+だれ
+なに
+なん
+何
+私
+貴方
+貴方方
+我々
+私達
+あの人
+あのかた
+彼女
+彼
+です
+あります
+おります
+います
+は
+が
+の
+に
+を
+で
+え
+から
+まで
+より
+も
+どの
+と
+し
+それで
+しかし