web search done right
small code cleanup
Mikkel Denker39f5536c309d
diff --git a/benches/search-preindexed.rs b/benches/search-preindexed.rsindex 1a845564..3345fc4a 100644--- a/benches/search-preindexed.rs+++ b/benches/search-preindexed.rs@@ -1,10 +1,14 @@use criterion::{criterion_group, criterion_main, Criterion};use cuely::{index::Index,+ ranking::centrality_store::CentralityStore,searcher::{LocalSearcher, SearchQuery},+ webgraph::centrality::topic::TopicCentrality,};const INDEX_PATH: &str = "data/index";+const CENTRALITY_PATH: &str = "data/centrality";+const TOPIC_CENTRALITY_PATH: &str = "data/topic_centrality";macro_rules! bench {($query:tt, $searcher:ident, $c:ident) => {@@ -29,14 +33,16 @@ macro_rules! bench {pub fn criterion_benchmark(c: &mut Criterion) {let index = Index::open(INDEX_PATH).unwrap();- let searcher = LocalSearcher::new(index);+ let mut searcher = LocalSearcher::new(index);+ searcher.set_centrality_store(CentralityStore::open(CENTRALITY_PATH));+ searcher.set_topic_centrality(TopicCentrality::open(TOPIC_CENTRALITY_PATH).unwrap());- // for _ in 0..100 {- bench!("the", searcher, c);- bench!("dtu", searcher, c);- bench!("the best", searcher, c);- bench!("the circle of life", searcher, c);- // }+ for _ in 0..100 {+ bench!("the", searcher, c);+ bench!("dtu", searcher, c);+ bench!("the best", searcher, c);+ bench!("the circle of life", searcher, c);+ }}criterion_group!(benches, criterion_benchmark);diff --git a/frontend/src/pages/about.mdx b/frontend/src/pages/about.mdxindex 93e684a1..f99bdc50 100644--- a/frontend/src/pages/about.mdx+++ b/frontend/src/pages/about.mdx@@ -11,7 +11,7 @@ Cuely is an open source search engine where the user has the ability to see exac- _Complete transparency_ -- You can view our code and follow the development directly on [Github](https://github.com/cuely/cuely)- _Privacy focussed_ -- We will never track you. Period. And you can always verify this in our [code.](https://github.com/cuely/cuely)-- _Immensely customizable_ -- We aim to give you the ability to customize everything about the search. e.g. block sites, boost sites, prioritize links from specific sites etc.+- _Immensely customizable_ -- We aim to give you the ability to customize everything about the search. e.g. block sites, boost sites, prioritize links from specific sites and much more.Oh, and if we ever become evil (e.g. by changing our motto) please take our code and start a competitor. The fact that you have this ability will make sure that our values will always be aligned with our users.diff --git a/src/entrypoint/configure.rs b/src/entrypoint/configure.rsindex 401357d9..22581b57 100644--- a/src/entrypoint/configure.rs+++ b/src/entrypoint/configure.rs@@ -37,7 +37,7 @@ fn download_files() {.build().unwrap().block_on(async {- let creds = StaticProvider::new("".to_string(), "".to_string(), None, None);+ let creds = StaticProvider::new(String::new(), String::new(), None, None);let client = S3Client::new_with(HttpClient::new().unwrap(),creds,diff --git a/src/entrypoint/dmoz_parser.rs b/src/entrypoint/dmoz_parser.rsindex 632ddfee..6ce5e588 100644--- a/src/entrypoint/dmoz_parser.rs+++ b/src/entrypoint/dmoz_parser.rs@@ -58,7 +58,7 @@ impl<R: BufRead> Iterator for PageIterator<R> {Event::Start(ref e) if e.name() == b"ExternalPage" => {let url_attr = e.attributes()- .filter(|attr| attr.is_ok())+ .filter(std::result::Result::is_ok).find(|attr| attr.as_ref().unwrap().key == b"about").unwrap().unwrap();@@ -72,13 +72,13 @@ impl<R: BufRead> Iterator for PageIterator<R> {inside_topic = true;}Event::End(ref e) if e.name() == b"topic" && current_page.is_some() => {- inside_topic = false+ inside_topic = false;}Event::Start(ref e) if e.name() == b"d:Description" && current_page.is_some() => {inside_desc = true;}Event::End(ref e) if e.name() == b"d:Description" && current_page.is_some() => {- inside_desc = false+ inside_desc = false;}Event::Text(ref e) => {if inside_topic {diff --git a/src/lib.rs b/src/lib.rsindex d4e26185..c4a9f897 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -15,7 +15,6 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.// #![warn(clippy::pedantic)]-#![allow(clippy::cast_precision_loss)]#![allow(clippy::missing_errors_doc)]@@ -257,9 +256,9 @@ fn gen_temp_path() -> PathBuf {if cfg!(target_os = "linux") {// use shared memory for temporary linux files- format!("/dev/shm/pagecache.tmp.{}", salt).into()+ format!("/dev/shm/pagecache.tmp.{salt}").into()} else {- std::env::temp_dir().join(format!("pagecache.tmp.{}", salt))+ std::env::temp_dir().join(format!("pagecache.tmp.{salt}"))}}diff --git a/src/snippet.rs b/src/snippet.rsindex c7ba2c77..22da15be 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -16,7 +16,7 @@use crate::schema::TextField;use crate::search_prettifier::html_escape;-use crate::tokenizer::Stemmed;+use crate::tokenizer::{Stemmed, Tokenizer};use crate::webpage::region::Region;use std::cmp::Ordering;use std::collections::{BTreeMap, BTreeSet};@@ -201,16 +201,14 @@ impl SnippetGenerator {searcher: &Searcher,query: &dyn tantivy::query::Query,tokenizer: tantivy::tokenizer::TextAnalyzer,+ field: Field,) -> crate::Result<SnippetGenerator> {let clean_field = searcher.schema().get_field(Field::Text(TextField::CleanBody).name()).unwrap();- let stemmed_field = searcher- .schema()- .get_field(Field::Text(TextField::StemmedCleanBody).name())- .unwrap();+ let field = searcher.schema().get_field(field.name()).unwrap();let mut terms: BTreeSet<&tantivy::Term> = BTreeSet::new();query.query_terms(&mut |term, _| {@@ -232,7 +230,7 @@ impl SnippetGenerator {continue;};- let term = tantivy::Term::from_field_text(stemmed_field, &term_string);+ let term = tantivy::Term::from_field_text(field, &term_string);let doc_freq = searcher.doc_freq(&term)?;if doc_freq > 0 {let score = 1.0 / (1.0 + doc_freq as Score);@@ -272,7 +270,12 @@ pub fn generate(};let tokenizer = Stemmed::with_forced_language(lang).into();- let generator = SnippetGenerator::create(searcher, query, tokenizer)?;+ let generator = SnippetGenerator::create(+ searcher,+ query,+ tokenizer,+ Field::Text(TextField::StemmedCleanBody),+ )?;let mut snippet = generator.snippet(text);@@ -280,49 +283,29 @@ pub fn generate(if text.is_empty() {match dmoz_description {Some(desc) => {- let lang = match region.lang() {- Some(lang) => lang,- None => whatlang::detect_lang(desc).unwrap_or(Lang::Eng),- };-- let tokenizer = Stemmed::with_forced_language(lang).into();- let generator = SnippetGenerator::create(searcher, query, tokenizer)?;-- snippet = generator.snippet(desc);- if snippet.fragment.is_empty() {- snippet.fragment = desc.chars().take(DEFAULT_MAX_NUM_CHARS).collect();- }+ snippet = generate_snippet_without_stem(+ desc,+ searcher,+ query,+ Field::Text(TextField::DmozDescription),+ )?;}None => match description {Some(desc) => {- let lang = match region.lang() {- Some(lang) => lang,- None => whatlang::detect_lang(desc).unwrap_or(Lang::Eng),- };-- let tokenizer = Stemmed::with_forced_language(lang).into();- let generator = SnippetGenerator::create(searcher, query, tokenizer)?;-- snippet = generator.snippet(desc);- if snippet.fragment.is_empty() {- snippet.fragment = desc.chars().take(DEFAULT_MAX_NUM_CHARS).collect();- }+ snippet = generate_snippet_without_stem(+ desc,+ searcher,+ query,+ Field::Text(TextField::Description),+ )?;}None => {- let lang = match region.lang() {- Some(lang) => lang,- None => whatlang::detect_lang(dirty_text).unwrap_or(Lang::Eng),- };-- let tokenizer = Stemmed::with_forced_language(lang).into();- let generator = SnippetGenerator::create(searcher, query, tokenizer)?;-- snippet = generator.snippet(dirty_text);-- if snippet.fragment.is_empty() {- snippet.fragment =- dirty_text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();- }+ snippet = generate_snippet_without_stem(+ dirty_text,+ searcher,+ query,+ Field::Text(TextField::AllBody),+ )?;}},}@@ -336,6 +319,24 @@ pub fn generate(Ok(highlighted)}+fn generate_snippet_without_stem(+ text: &str,+ searcher: &Searcher,+ query: &Query,+ field: Field,+) -> Result<Snippet> {+ let tokenizer = Tokenizer::default().into();+ let generator = SnippetGenerator::create(searcher, query, tokenizer, field)?;++ let mut snippet = generator.snippet(text);++ if snippet.fragment.is_empty() {+ snippet.fragment = text.chars().take(DEFAULT_MAX_NUM_CHARS).collect();+ }++ Ok(snippet)+}+#[cfg(test)]mod tests {use super::*;