web search done right
clippy shenanigans
Mikkel Denker467ed085deb2
diff --git a/src/directory.rs b/src/directory.rsindex 13eb9024..776933cc 100644--- a/src/directory.rs+++ b/src/directory.rs@@ -41,12 +41,12 @@ fn iterate_children(path: &str) -> Result<Vec<DirEntry>> {res.push(DirEntry::Folder {name: entry.path().as_os_str().to_str().unwrap().to_string(),entries: iterate_children(entry.path().as_os_str().to_str().unwrap())?,- })+ });} else if metadata.is_file() {res.push(DirEntry::File {name: entry.path().as_os_str().to_str().unwrap().to_string(),content: fs::read(entry.path())?,- })+ });}}diff --git a/src/entity_index/entity.rs b/src/entity_index/entity.rsindex 85e4177d..81a82053 100644--- a/src/entity_index/entity.rs+++ b/src/entity_index/entity.rs@@ -57,11 +57,11 @@ impl Span {}}- pub fn add_link(&mut self, text: String, link: Link) {+ pub fn add_link(&mut self, text: &str, link: Link) {debug_assert_eq!(self.text.chars().count() + text.chars().count(), link.end);debug_assert_eq!(self.text.chars().count(), link.start);self.links.push(link);- self.text.push_str(&text);+ self.text.push_str(text);}}@@ -105,7 +105,7 @@ impl<'a> From<Vec<Node<'a>>> for Span {start: span.text.chars().count(),end: span.text.chars().count() + text.chars().count(),};- span.add_link(text, link);+ span.add_link(&text, link);}Node::Text {end: _,@@ -117,9 +117,9 @@ impl<'a> From<Vec<Node<'a>>> for Span {span.text.push_str(". ");}continue;- } else {- span.text.push_str(value);}++ span.text.push_str(value);}Node::Template {end: _,diff --git a/src/entity_index/mod.rs b/src/entity_index/mod.rsindex 6b58c63a..12738a99 100644--- a/src/entity_index/mod.rs+++ b/src/entity_index/mod.rs@@ -35,7 +35,7 @@ use crate::{image_downloader::{ImageDownloadJob, ImageDownloader},image_store::{EntityImageStore, Image, ImageStore},kv::{rocksdb_store::RocksDbStore, Kv},- tokenizer::NormalTokenizer,+ tokenizer::Normal,webpage::Url,Result,};@@ -61,7 +61,7 @@ fn schema() -> Schema {TextOptions::default().set_indexing_options(TextFieldIndexing::default()- .set_tokenizer(NormalTokenizer::as_str())+ .set_tokenizer(Normal::as_str()).set_index_option(IndexRecordOption::WithFreqsAndPositions),).set_stored(),@@ -71,7 +71,7 @@ fn schema() -> Schema {TextOptions::default().set_indexing_options(TextFieldIndexing::default()- .set_tokenizer(NormalTokenizer::as_str())+ .set_tokenizer(Normal::as_str()).set_index_option(IndexRecordOption::WithFreqsAndPositions),).set_stored(),@@ -118,7 +118,7 @@ fn entity_to_tantivy(entity: Entity, schema: &tantivy::schema::Schema) -> tantivdoc}-fn wikipedify_url(url: Url) -> Vec<Url> {+fn wikipedify_url(url: &Url) -> Vec<Url> {let mut name = url.raw().replace(' ', "_");if name.starts_with("File:") {@@ -174,12 +174,12 @@ impl EntityIndex {let stopwords: HashSet<String> = include_str!("../../stopwords/English.txt").lines().take(50)- .map(|word| word.to_ascii_lowercase())+ .map(str::to_ascii_lowercase).collect();tantivy_index.tokenizers().register(- NormalTokenizer::as_str(),- NormalTokenizer::with_stopwords(stopwords.clone().into_iter().collect()),+ Normal::as_str(),+ Normal::with_stopwords(stopwords.clone().into_iter().collect()),);let image_store = EntityImageStore::open(path.as_ref().join("images"));@@ -206,15 +206,16 @@ impl EntityIndex {}if let Some(image) = entity.image.clone() {- let image = wikipedify_url(image)+ let image = wikipedify_url(&image).into_iter()- .filter(|url| url.is_valid_uri())+ .filter(Url::is_valid_uri).collect();+self.image_downloader.schedule(ImageDownloadJob {key: entity.title.clone(),urls: image,timeout: Some(Duration::from_secs(10)),- })+ });}let doc = entity_to_tantivy(entity, &self.schema);self.writer.add_document(doc).unwrap();@@ -252,7 +253,7 @@ impl EntityIndex {.into_iter().filter(|(_, related_doc)| doc != *related_doc).map(|(_, doc_address)| {- self.retrieve_stored_entity(&searcher, &doc_address, false, false, false)+ self.retrieve_stored_entity(&searcher, doc_address, false, false, false)}).filter(|entity| entity.image.is_some()).take(4)@@ -261,14 +262,14 @@ impl EntityIndex {}}- pub fn search(&self, query: String) -> Option<StoredEntity> {+ pub fn search(&self, query: &str) -> Option<StoredEntity> {let searcher = self.reader.searcher();let title = self.schema.get_field("title").unwrap();let entity_abstract = self.schema.get_field("abstract").unwrap();let mut term_queries = Vec::new();- let mut stream = NormalTokenizer::default().token_stream(query.as_str());+ let mut stream = Normal::default().token_stream(query);while let Some(token) = stream.next() {if self.stopwords.contains(&token.text) {continue;@@ -300,14 +301,14 @@ impl EntityIndex {.unwrap().first().map(|(_score, doc_address)| {- self.retrieve_stored_entity(&searcher, doc_address, true, true, true)+ self.retrieve_stored_entity(&searcher, *doc_address, true, true, true)})}fn retrieve_stored_entity(&self,searcher: &LeasedItem<Searcher>,- doc_address: &DocAddress,+ doc_address: DocAddress,get_related: bool,decode_info: bool,get_links: bool,@@ -317,7 +318,7 @@ impl EntityIndex {let info = self.schema.get_field("info").unwrap();let links = self.schema.get_field("links").unwrap();- let doc = searcher.doc(*doc_address).unwrap();+ let doc = searcher.doc(doc_address).unwrap();let title = doc.get_first(title).and_then(|val| match val {@@ -349,7 +350,7 @@ impl EntityIndex {};let related_entities = if get_related {- self.related_entities(*doc_address)+ self.related_entities(doc_address)} else {Vec::new()};@@ -398,7 +399,7 @@ mod tests {#[test]fn wikipedia_image_url_aristotle() {assert_eq!(- wikipedify_url("Aristotle Altemps Inv8575.jpg".to_string().into())+ wikipedify_url(&"Aristotle Altemps Inv8575.jpg".to_string().into()).first().unwrap().full(),@@ -410,7 +411,7 @@ mod tests {#[test]fn wikipedia_image_url_with_file() {assert_eq!(- wikipedify_url("File:Aristotle Altemps Inv8575.jpg".to_string().into())+ wikipedify_url(&"File:Aristotle Altemps Inv8575.jpg".to_string().into()).first().unwrap().full(),@@ -437,17 +438,10 @@ mod tests {index.commit();- assert_eq!(index.search("the".to_string()), None);- assert_eq!(- index.search("ashes".to_string()).unwrap().title.as_str(),- "the ashes"- );+ assert_eq!(index.search("the"), None);+ assert_eq!(index.search("ashes").unwrap().title.as_str(), "the ashes");assert_eq!(- index- .search("the ashes".to_string())- .unwrap()- .title- .as_str(),+ index.search("the ashes").unwrap().title.as_str(),"the ashes");}diff --git a/src/entrypoint/centrality.rs b/src/entrypoint/centrality.rsindex f7a40fbb..baad8828 100644--- a/src/entrypoint/centrality.rs+++ b/src/entrypoint/centrality.rs@@ -18,9 +18,9 @@ use std::path::Path;use crate::{ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilder};-pub struct CentralityEntrypoint {}+pub struct Centrality {}-impl CentralityEntrypoint {+impl Centrality {pub fn run<P: AsRef<Path>>(webgraph_path: P, output_path: P) {let graph = WebgraphBuilder::new(webgraph_path).with_host_graph().open();diff --git a/src/entrypoint/entity.rs b/src/entrypoint/entity.rsindex 530ebc5b..6ec057e5 100644--- a/src/entrypoint/entity.rs+++ b/src/entrypoint/entity.rs@@ -263,7 +263,7 @@ impl EntityBuilder {+ text.chars().count(),target: target.to_string(),};- current_paragraph.content.add_link(text, link)+ current_paragraph.content.add_link(&text, link)}None => {let link = Link {diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rsindex eba431b0..847c99cf 100644--- a/src/entrypoint/mod.rs+++ b/src/entrypoint/mod.rs@@ -19,7 +19,7 @@ pub mod frontend;mod indexer;mod webgraph;-pub use centrality::CentralityEntrypoint;+pub use centrality::Centrality;pub use entity::EntityIndexer;pub use indexer::Indexer;-pub use webgraph::WebgraphEntrypoint;+pub use webgraph::Webgraph;diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex e8349210..e43ff525 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -14,9 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{+ directory::DirEntry,mapreduce::{Map, MapReduce, Reduce, StatelessWorker, Worker},warc::WarcFile,- webgraph::{FrozenWebgraph, Node, Webgraph, WebgraphBuilder},+ webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},webpage::Html,HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,};@@ -81,11 +82,12 @@ impl Map<StatelessWorker, FrozenWebgraph> for Job {}}-impl Reduce<FrozenWebgraph> for Webgraph {- fn reduce(mut self, other: FrozenWebgraph) -> Webgraph {+impl Reduce<FrozenWebgraph> for webgraph::Webgraph {+ fn reduce(mut self, other: FrozenWebgraph) -> webgraph::Webgraph {let other_path = match &other.root {- crate::directory::DirEntry::Folder { name, entries: _ } => name.clone(),- crate::directory::DirEntry::File { name, content: _ } => name.clone(),+ DirEntry::Folder { name, entries: _ } | DirEntry::File { name, content: _ } => {+ name.clone()+ }};let other = other.into();@@ -98,8 +100,8 @@ impl Reduce<FrozenWebgraph> for Webgraph {}}-impl Reduce<Webgraph> for Webgraph {- fn reduce(mut self, element: Webgraph) -> Self {+impl Reduce<webgraph::Webgraph> for webgraph::Webgraph {+ fn reduce(mut self, element: webgraph::Webgraph) -> Self {let other_path = element.path.clone();self.merge(element);@@ -109,9 +111,9 @@ impl Reduce<Webgraph> for Webgraph {}}-pub struct WebgraphEntrypoint {}+pub struct Webgraph {}-impl WebgraphEntrypoint {+impl Webgraph {pub fn run_master(config: &WebgraphMasterConfig) -> Result<()> {info!("Running master for webgraph construction");@@ -144,7 +146,7 @@ impl WebgraphEntrypoint {warc_paths = Box::new(warc_paths.take(limit));}- let _graph: Webgraph = warc_paths+ let _graph: webgraph::Webgraph = warc_paths.map_reduce(&workers).expect("failed to build webgraph");@@ -179,7 +181,7 @@ impl WebgraphEntrypoint {}).par_bridge().map(|job| job.map(&worker))- .map(Webgraph::from)+ .map(webgraph::Webgraph::from).reduce_with(|a, b| a.reduce(b));Ok(())diff --git a/src/frontend/autosuggest.rs b/src/frontend/autosuggest.rsindex b72f0752..3617567d 100644--- a/src/frontend/autosuggest.rs+++ b/src/frontend/autosuggest.rs@@ -41,6 +41,7 @@ struct Suggestion {raw: String,}+#[allow(clippy::unused_async)]pub async fn route(extract::Query(params): extract::Query<HashMap<String, String>>,Extension(state): Extension<Arc<State>>,diff --git a/src/frontend/entity_image.rs b/src/frontend/entity_image.rsindex d33440b5..49afd79d 100644--- a/src/frontend/entity_image.rs+++ b/src/frontend/entity_image.rs@@ -25,6 +25,7 @@ use reqwest::StatusCode;use super::State;+#[allow(clippy::unused_async)]pub async fn route(Path(entity): Path<String>,Extension(state): Extension<Arc<State>>,diff --git a/src/frontend/favicons.rs b/src/frontend/favicons.rsindex 9a4b9ce2..047bbccc 100644--- a/src/frontend/favicons.rs+++ b/src/frontend/favicons.rs@@ -27,6 +27,7 @@ use crate::webpage::Url;use super::State;+#[allow(clippy::unused_async)]pub async fn route(Path(site): Path<String>,Extension(state): Extension<Arc<State>>,diff --git a/src/frontend/index.rs b/src/frontend/index.rsindex 39736791..0245910b 100644--- a/src/frontend/index.rs+++ b/src/frontend/index.rs@@ -18,6 +18,7 @@ use super::HtmlTemplate;use askama::Template;use axum::response::IntoResponse;+#[allow(clippy::unused_async)]pub async fn route() -> impl IntoResponse {let template = IndexTemplate {};HtmlTemplate(template)diff --git a/src/frontend/primary_image.rs b/src/frontend/primary_image.rsindex 90ecc190..cb19cb6b 100644--- a/src/frontend/primary_image.rs+++ b/src/frontend/primary_image.rs@@ -25,6 +25,7 @@ use reqwest::StatusCode;use super::State;+#[allow(clippy::unused_async)]pub async fn route(Path(uuid): Path<String>,Extension(state): Extension<Arc<State>>,diff --git a/src/frontend/search.rs b/src/frontend/search.rsindex 7cb46c57..7c0e257c 100644--- a/src/frontend/search.rs+++ b/src/frontend/search.rs@@ -59,7 +59,7 @@ pub struct DisplayedWebpage {const MAX_PRETTY_URL_LEN: usize = 50;const MAX_TITLE_LEN: usize = 50;-fn prettify_url(url: Url) -> String {+fn prettify_url(url: &Url) -> String {let mut pretty_url = url.strip_query().to_string();if pretty_url.ends_with('/') {@@ -88,18 +88,18 @@ fn prettify_date(date: NaiveDateTime) -> String {if num_hours < 24 {if num_hours <= 1 {return "1 hour ago".to_string();- } else {- return format!("{num_hours} hours ago");}++ return format!("{num_hours} hours ago");}let num_days = diff.num_days();if num_days < 30 {if num_days <= 1 {return "1 day ago".to_string();- } else {- return format!("{num_days} days ago");}++ return format!("{num_days} days ago");}format!("{}", date.format("%d. %b. %Y"))@@ -111,7 +111,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {let url: Url = webpage.url.clone().into();let domain = url.domain().to_string();- let pretty_url = prettify_url(url);+ let pretty_url = prettify_url(&url);let mut title = html_escape(&webpage.title);@@ -253,6 +253,8 @@ enum RegionSelection {Unselected(Region),}+#[allow(clippy::unused_async)]+#[allow(clippy::match_wild_err_arm)]pub async fn route(extract::Query(params): extract::Query<HashMap<String, String>>,Extension(state): Extension<Arc<State>>,@@ -454,7 +456,7 @@ mod tests {),"some <a href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link".to_string()- )+ );}#[test]@@ -472,7 +474,7 @@ mod tests {7),"some <a href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string()- )+ );}#[test]@@ -497,9 +499,9 @@ mod tests {assert_eq!(thousand_sep_number(10), "10".to_string());assert_eq!(thousand_sep_number(100), "100".to_string());assert_eq!(thousand_sep_number(1000), "1.000".to_string());- assert_eq!(thousand_sep_number(10000), "10.000".to_string());- assert_eq!(thousand_sep_number(100000), "100.000".to_string());- assert_eq!(thousand_sep_number(512854), "512.854".to_string());- assert_eq!(thousand_sep_number(9512854), "9.512.854".to_string());+ assert_eq!(thousand_sep_number(10_000), "10.000".to_string());+ assert_eq!(thousand_sep_number(100_000), "100.000".to_string());+ assert_eq!(thousand_sep_number(512_854), "512.854".to_string());+ assert_eq!(thousand_sep_number(9_512_854), "9.512.854".to_string());}}diff --git a/src/kv/mod.rs b/src/kv/mod.rsindex 32b6713d..386a7028 100644--- a/src/kv/mod.rs+++ b/src/kv/mod.rs@@ -40,6 +40,6 @@ wherelet key_bytes = bincode::serialize(&key).expect("failed to serialize key");let val_bytes = bincode::serialize(&value).expect("failed to serialize value");- self.insert_raw(key_bytes, val_bytes)+ self.insert_raw(key_bytes, val_bytes);}}diff --git a/src/kv/rocksdb_store.rs b/src/kv/rocksdb_store.rsindex c6f02dc2..435a3885 100644--- a/src/kv/rocksdb_store.rs+++ b/src/kv/rocksdb_store.rs@@ -60,8 +60,8 @@ whereBox::new(IntoIter {inner: iter,- key: Default::default(),- value: Default::default(),+ key: PhantomData::default(),+ value: PhantomData::default(),})}}diff --git a/src/lib.rs b/src/lib.rsindex c33bca29..1fba47e3 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -16,6 +16,9 @@// #![warn(clippy::pedantic)]+#![allow(clippy::cast_precision_loss)]+#![allow(clippy::missing_errors_doc)]+use serde::{Deserialize, Serialize};use std::fs::File;use std::io::{self, BufRead};diff --git a/src/main.rs b/src/main.rsindex 7a6ae6d8..c9cc364c 100644--- a/src/main.rs+++ b/src/main.rs@@ -15,9 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::Result;use clap::{Parser, Subcommand};-use cuely::entrypoint::{- frontend, CentralityEntrypoint, EntityIndexer, Indexer, WebgraphEntrypoint,-};+use cuely::entrypoint::{self, frontend};use serde::de::DeserializeOwned;use std::fs;use std::path::Path;@@ -97,38 +95,38 @@ fn main() -> Result<()> {Commands::Indexer { options } => match options {IndexingOptions::Master { config_path } => {let config = load_toml_config(&config_path);- Indexer::run_master(&config)?;+ entrypoint::Indexer::run_master(&config)?;}IndexingOptions::Worker {address,centrality_store_path,} => {- Indexer::run_worker(address, centrality_store_path)?;+ entrypoint::Indexer::run_worker(address, centrality_store_path)?;}IndexingOptions::Local { config_path } => {let config = load_toml_config(&config_path);- Indexer::run_locally(&config)?;+ entrypoint::Indexer::run_locally(&config)?;}IndexingOptions::Entity {wikipedia_dump_path,output_path,- } => EntityIndexer::run(wikipedia_dump_path, output_path)?,+ } => entrypoint::EntityIndexer::run(wikipedia_dump_path, output_path)?,},Commands::Centrality {webgraph_path,output_path,- } => CentralityEntrypoint::run(webgraph_path, output_path),+ } => entrypoint::Centrality::run(webgraph_path, output_path),Commands::Webgraph { options } => match options {WebgraphOptions::Master { config_path } => {let config = load_toml_config(config_path);- WebgraphEntrypoint::run_master(&config)?;+ entrypoint::Webgraph::run_master(&config)?;}WebgraphOptions::Worker { address } => {- WebgraphEntrypoint::run_worker(address)?;+ entrypoint::Webgraph::run_worker(address)?;}WebgraphOptions::Local { config_path } => {let config = load_toml_config(config_path);- WebgraphEntrypoint::run_locally(&config)?;+ entrypoint::Webgraph::run_locally(&config)?;}},Commands::Frontend {diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rsindex 27ed2fdd..8ebea4c5 100644--- a/src/mapreduce/manager.rs+++ b/src/mapreduce/manager.rs@@ -108,7 +108,7 @@ impl<'a> WorkerGuard<'a> {}fn success(self) {- self.from_pool.insert(Arc::clone(&self.worker))+ self.from_pool.insert(Arc::clone(&self.worker));}}@@ -122,7 +122,7 @@ impl<'a> Deref for WorkerGuard<'a> {impl<'a> Drop for WorkerGuard<'a> {fn drop(&mut self) {- self.from_pool.put_back()+ self.from_pool.put_back();}}@@ -185,7 +185,7 @@ impl WorkerPool {let mut failing_workers = Vec::new();for worker in &self.all_workers {if worker.stop::<W, I, O>().is_err() {- failing_workers.push(worker)+ failing_workers.push(worker);}}@@ -272,6 +272,7 @@ impl Manager {}}+ #[allow(clippy::trait_duplication_in_bounds)]fn get_results<W, I, O1, O2>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>whereW: Worker,@@ -327,6 +328,7 @@ impl Manager {x}+ #[allow(clippy::trait_duplication_in_bounds)]pub fn run<W, I, O1, O2>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>whereW: Worker,diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rsindex c55f919c..570a62e9 100644--- a/src/mapreduce/mod.rs+++ b/src/mapreduce/mod.rs@@ -52,9 +52,11 @@ where}pub trait Reduce<T> {+ #[must_use]fn reduce(self, element: T) -> Self;}+#[allow(clippy::trait_duplication_in_bounds)]pub trait MapReduce<W, I, O1, O2>whereSelf: Sized + Iterator<Item = I> + Send,@@ -68,6 +70,8 @@ wheremanager.run(self)}}++#[allow(clippy::trait_duplication_in_bounds)]impl<W, I, O1, O2, T> MapReduce<W, I, O1, O2> for TwhereW: Worker,diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex 4535304b..36e1578e 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -132,9 +132,10 @@ fn region_score(selected_region: Option<Region>,webpage_region: Region,) -> f64 {- let boost = selected_region- .map(|region| if region == webpage_region { 50.0 } else { 0.0 })- .unwrap_or(0.0);+ let boost = selected_region.map_or(+ 0.0,+ |region| if region == webpage_region { 50.0 } else { 0.0 },+ );boost + region_count.score(&webpage_region)}diff --git a/src/schema.rs b/src/schema.rsindex 4ce019f5..6ca34087 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -18,7 +18,7 @@ use tantivy::schema::{Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,};-use crate::tokenizer::{NormalTokenizer, StemmedTokenizer};+use crate::tokenizer::{Normal, Stemmed};pub const CENTRALITY_SCALING: u64 = 1_000_000_000;@@ -79,7 +79,7 @@ impl Field {}fn default_text_options(&self) -> tantivy::schema::TextOptions {- self.default_text_options_with_tokenizer(NormalTokenizer::as_str())+ self.default_text_options_with_tokenizer(Normal::as_str())}pub fn options(&self) -> IndexingOption {@@ -91,7 +91,7 @@ impl Field {Field::Domain => IndexingOption::Text(self.default_text_options()),Field::AllBody => IndexingOption::Text(self.default_text_options()),Field::StemmedAllBody => IndexingOption::Text(- self.default_text_options_with_tokenizer(StemmedTokenizer::as_str())+ self.default_text_options_with_tokenizer(Stemmed::as_str()).set_stored(),),Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),@@ -106,11 +106,11 @@ impl Field {.set_fast(Cardinality::SingleValue).set_indexed(),),- Field::StemmedTitle => IndexingOption::Text(- self.default_text_options_with_tokenizer(StemmedTokenizer::as_str()),- ),+ Field::StemmedTitle => {+ IndexingOption::Text(self.default_text_options_with_tokenizer(Stemmed::as_str()))+ }Field::StemmedCleanBody => IndexingOption::Text(- self.default_text_options_with_tokenizer(StemmedTokenizer::as_str())+ self.default_text_options_with_tokenizer(Stemmed::as_str()).set_stored(),),Field::FetchTimeMs => IndexingOption::Numeric(diff --git a/src/schema_org.rs b/src/schema_org.rsindex a953b781..0ca00289 100644--- a/src/schema_org.rs+++ b/src/schema_org.rs@@ -61,6 +61,6 @@ mod tests {author: Some("Jane Doe".to_string()),content_url: Some("mexico-beach.jpg".to_string()),}),- )+ );}}diff --git a/src/searcher.rs b/src/searcher.rsindex 698474f0..79329109 100644--- a/src/searcher.rs+++ b/src/searcher.rs@@ -82,7 +82,7 @@ impl Searcher {let entity = self.entity_index.as_ref()- .and_then(|index| index.search(raw_query));+ .and_then(|index| index.search(&raw_query));let search_duration_ms = start.elapsed().as_millis();diff --git a/src/snippet.rs b/src/snippet.rsindex e917e480..78849a76 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::frontend::search::html_escape;-use crate::tokenizer::StemmedTokenizer;+use crate::tokenizer::Stemmed;use std::cmp::Ordering;use std::collections::BTreeMap;use std::ops::Range;@@ -24,8 +24,8 @@ use tantivy::tokenizer::{TextAnalyzer, Token};use tantivy::{Score, Searcher};use whatlang::Lang;-/// For now we use the snippet generator from tantivy with a minor modification to support our TokenStreamMerger.-/// In the future we want to implement something closer to the method described in https://cs.pomona.edu/~dkauchak/ir_project/whitepapers/Snippet-IL.pdf.+/// For now we use the snippet generator from tantivy with a minor modifications.+/// In the future we want to implement something closer to the method described in <https://cs.pomona.edu/~dkauchak/ir_project/whitepapers/Snippet-IL.pdf>./// This will require us to store each paragraph of the webpage separately to get adequate performance./// Implementing SnippetIL will also allow us to correctly add "..." to the snippet.@@ -44,7 +44,7 @@ impl FragmentCandidate {////// `score`, `num_chars` are set to 0/// and `highlighted` is set to empty vec- /// stop_offset is set to start_offset, which is taken as a param.+ /// `stop_offset` is set to `start_offset`, which is taken as a param.fn new(start_offset: usize) -> FragmentCandidate {FragmentCandidate {score: 0.0,@@ -261,8 +261,7 @@ pub fn generate(.expect("Failed to get body field");let tokenizer =- StemmedTokenizer::with_forced_language(whatlang::detect_lang(text).unwrap_or(Lang::Eng))- .into();+ Stemmed::with_forced_language(whatlang::detect_lang(text).unwrap_or(Lang::Eng)).into();let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;let mut snippet = generator.snippet(text);@@ -270,10 +269,9 @@ pub fn generate(if snippet.fragment.is_empty() {match description {Some(desc) => {- let tokenizer = StemmedTokenizer::with_forced_language(- whatlang::detect_lang(desc).unwrap_or(Lang::Eng),- )- .into();+ let tokenizer =+ Stemmed::with_forced_language(whatlang::detect_lang(desc).unwrap_or(Lang::Eng))+ .into();let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;snippet = generator.snippet(desc);@@ -283,7 +281,7 @@ pub fn generate(}None => {if text.is_empty() {- let tokenizer = StemmedTokenizer::with_forced_language(+ let tokenizer = Stemmed::with_forced_language(whatlang::detect_lang(dirty_text).unwrap_or(Lang::Eng),).into();diff --git a/src/spell/dictionary.rs b/src/spell/dictionary.rsindex 9ab1c6f2..4459ffee 100644--- a/src/spell/dictionary.rs+++ b/src/spell/dictionary.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::spell::distance::LevenshteinDistance;-use crate::tokenizer::NormalTokenizer;+use crate::tokenizer::Normal;use crate::webpage::Webpage;use fst::map::Union;use fst::{Automaton, IntoStreamer, Map, MapBuilder, Streamer};@@ -336,7 +336,7 @@ impl<const TOP_N: usize> Dictionary<TOP_N> {pub fn insert_page(&mut self, webpage: &Webpage) {let text = webpage.html.clean_text().unwrap_or_default();- let mut stream = NormalTokenizer::default().token_stream(text.as_str());+ let mut stream = Normal::default().token_stream(text.as_str());while let Some(token) = stream.next() {self.insert(&token.text);diff --git a/src/tokenizer.rs b/src/tokenizer.rsindex 092cdd23..3c0bd193 100644--- a/src/tokenizer.rs+++ b/src/tokenizer.rs@@ -29,7 +29,6 @@ impl From<Lang> for MyStemmer {Lang::Dan => MyStemmer(Stemmer::new(Language::Danish)),Lang::Ara => MyStemmer(Stemmer::new(Language::Arabic)),Lang::Nld => MyStemmer(Stemmer::new(Language::Dutch)),- Lang::Eng => MyStemmer(Stemmer::new(Language::English)),Lang::Fin => MyStemmer(Stemmer::new(Language::Finnish)),Lang::Fra => MyStemmer(Stemmer::new(Language::French)),Lang::Deu => MyStemmer(Stemmer::new(Language::German)),@@ -49,34 +48,34 @@ impl From<Lang> for MyStemmer {#[derive(Clone)]pub enum Tokenizer {- NormalTokenizer(NormalTokenizer),- StemmedTokenizer(StemmedTokenizer),+ NormalTokenizer(Normal),+ StemmedTokenizer(Stemmed),}impl Tokenizer {pub fn new_stemmed() -> Self {- Self::StemmedTokenizer(Default::default())+ Self::StemmedTokenizer(Stemmed::default())}pub fn as_str(&self) -> &'static str {match self {- Tokenizer::NormalTokenizer(_) => NormalTokenizer::as_str(),- Tokenizer::StemmedTokenizer(_) => StemmedTokenizer::as_str(),+ Tokenizer::NormalTokenizer(_) => Normal::as_str(),+ Tokenizer::StemmedTokenizer(_) => Stemmed::as_str(),}}}impl Default for Tokenizer {fn default() -> Self {- Self::NormalTokenizer(Default::default())+ Self::NormalTokenizer(Normal::default())}}#[derive(Clone, Default)]-pub struct NormalTokenizer {+pub struct Normal {stopwords: Option<Vec<String>>,}-impl NormalTokenizer {+impl Normal {pub fn as_str() -> &'static str {"tokenizer"}@@ -89,11 +88,11 @@ impl NormalTokenizer {}#[derive(Clone, Default)]-pub struct StemmedTokenizer {+pub struct Stemmed {force_language: Option<Lang>,}-impl StemmedTokenizer {+impl Stemmed {pub fn as_str() -> &'static str {"stemmed_tokenizer"}@@ -113,9 +112,9 @@ impl tantivy::tokenizer::Tokenizer for Tokenizer {}}-impl tantivy::tokenizer::Tokenizer for NormalTokenizer {+impl tantivy::tokenizer::Tokenizer for Normal {fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {- let mut analyzer = TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser);+ let mut analyzer = TextAnalyzer::from(Simple).filter(LowerCaser);if let Some(stopwords) = &self.stopwords {analyzer = analyzer.filter(StopWordFilter::remove(stopwords.clone()));@@ -125,9 +124,9 @@ impl tantivy::tokenizer::Tokenizer for NormalTokenizer {}}-impl tantivy::tokenizer::Tokenizer for StemmedTokenizer {+impl tantivy::tokenizer::Tokenizer for Stemmed {fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {- let analyzer = TextAnalyzer::from(SimpleTokenizer)+ let analyzer = TextAnalyzer::from(Simple).filter(LowerCaser).filter(StopWordFilter::remove(vec![]));@@ -154,7 +153,7 @@ enum Token {}#[derive(Clone)]-pub struct SimpleTokenizer;+pub struct Simple;pub struct SimpleTokenStream<'a> {lexer: Lexer<'a, Token>,@@ -162,7 +161,7 @@ pub struct SimpleTokenStream<'a> {next_position: usize,}-impl tantivy::tokenizer::Tokenizer for SimpleTokenizer {+impl tantivy::tokenizer::Tokenizer for Simple {fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {let lexer = Token::lexer(text);BoxTokenStream::from(SimpleTokenStream {@@ -208,7 +207,7 @@ mod tests {fn tokenize(s: &str) -> Vec<String> {let mut res = Vec::new();- let mut stream = NormalTokenizer::default().token_stream(s);+ let mut stream = Normal::default().token_stream(s);while let Some(token) = stream.next() {res.push(token.text.clone());diff --git a/src/warc.rs b/src/warc.rsindex 9e174bc6..04fa3428 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -32,27 +32,26 @@ fn rtrim(s: &mut String) {s.truncate(s.trim_end().len());}-fn decode(raw: Vec<u8>) -> String {- match String::from_utf8(raw.clone()) {- Ok(res) => res,- Err(_) => {- let encodings = [- encoding_rs::WINDOWS_1251,- encoding_rs::GBK,- encoding_rs::SHIFT_JIS,- encoding_rs::EUC_JP,- encoding_rs::EUC_KR,- ];-- for enc in encodings {- let (cow, _, had_errors) = enc.decode(&raw[..]);- if !had_errors {- return cow.to_string();- }+fn decode(raw: &[u8]) -> String {+ if let Ok(res) = String::from_utf8(raw.to_owned()) {+ res+ } else {+ let encodings = [+ encoding_rs::WINDOWS_1251,+ encoding_rs::GBK,+ encoding_rs::SHIFT_JIS,+ encoding_rs::EUC_JP,+ encoding_rs::EUC_KR,+ ];++ for enc in encodings {+ let (cow, _, had_errors) = enc.decode(raw);+ if !had_errors {+ return cow.to_string();}-- return String::from_utf8_lossy(&raw).to_string();}++ String::from_utf8_lossy(raw).to_string()}}@@ -157,7 +156,7 @@ pub(crate) struct Response {impl Response {fn from_raw(record: RawWarcRecord) -> Result<Self> {- let content = decode(record.content);+ let content = decode(&record.content[..]);let (_header, content) = content.split_once("\r\n\r\n")diff --git a/src/webpage/just_text.rs b/src/webpage/just_text.rsindex c9c2b80a..f3b44aa4 100644--- a/src/webpage/just_text.rs+++ b/src/webpage/just_text.rs@@ -182,6 +182,7 @@ impl Paragraph {text.len()}+ #[allow(clippy::cast_precision_loss)]fn link_density(&self) -> f64 {self.chars_count_in_links as f64 / self.text.len() as f64}@@ -332,11 +333,8 @@ impl JustText {res}- fn calculate_stopword_density(- &self,- paragraph: &Paragraph,- stopwords: &HashSet<String>,- ) -> f64 {+ #[allow(clippy::cast_precision_loss)]+ fn calculate_stopword_density(paragraph: &Paragraph, stopwords: &HashSet<String>) -> f64 {paragraph.text.split_whitespace()@@ -359,7 +357,7 @@ impl JustText {for paragraph in paragraphs {let classification = {- let stopword_density: f64 = self.calculate_stopword_density(¶graph, stopwords);+ let stopword_density: f64 = Self::calculate_stopword_density(¶graph, stopwords);if paragraph.link_density() > self.max_link_density|| paragraph.text.contains("\\xa9")@@ -508,16 +506,10 @@ impl JustText {paragraph.classification = match ¶graph.classification {Classification::Final(_) => paragraph.classification.clone(),Classification::Intermediate(intermediate) => match intermediate {- IntermediateClassification::Good => {+ IntermediateClassification::Good | IntermediateClassification::NearGood => {Classification::Final(FinalClassification::Good)}- IntermediateClassification::NearGood => {- Classification::Final(FinalClassification::Good)- }- IntermediateClassification::Short => {- Classification::Final(FinalClassification::Bad)- }- IntermediateClassification::Bad => {+ IntermediateClassification::Short | IntermediateClassification::Bad => {Classification::Final(FinalClassification::Bad)}},@@ -571,7 +563,7 @@ impl JustText {Classification::Intermediate(IntermediateClassification::Bad)}- fn extract_text(&self, paragraphs: Vec<ClassifiedParagraph>) -> String {+ fn extract_text(paragraphs: Vec<ClassifiedParagraph>) -> String {paragraphs.into_iter().filter(|paragraph| {@@ -595,7 +587,7 @@ impl JustText {self.contextual_classification(&mut classified);- self.extract_text(classified)+ Self::extract_text(classified).split_whitespace().collect::<Vec<_>>().join(" ")diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 0db54a36..778a8e3e 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -16,7 +16,7 @@use crate::{schema_org::SchemaOrg, Error, Result};use chrono::{DateTime, FixedOffset};use itertools::Itertools;-use kuchiki::{iter::NodeEdge, traits::*, NodeRef};+use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};use regex::Regex;use std::{borrow::Borrow, collections::HashMap};use uuid::Uuid;@@ -242,7 +242,7 @@ impl Html {source: self.url.clone(),destination: dest.to_string().into(),text: text.trim().to_string(),- })+ });}}}@@ -293,9 +293,9 @@ impl Html {let favicon = FaviconLink {link,- image_type,width,height,+ image_type,};return Some(favicon);@@ -344,10 +344,10 @@ impl Html {pub fn title(&self) -> Option<String> {if let Ok(title) = self.root.select_first("title") {let title = title.text_contents().trim().to_string();- if !title.is_empty() {- Some(title)- } else {+ if title.is_empty() {None+ } else {+ Some(title)}} else {None@@ -416,29 +416,28 @@ impl Html {Field::CleanBody | Field::StemmedCleanBody => {let text = self.clean_text();- doc.add_text(tantivy_field, text.unwrap_or_default())+ doc.add_text(tantivy_field, text.unwrap_or_default());}Field::Description => {- doc.add_text(tantivy_field, self.description().unwrap_or_default())+ doc.add_text(tantivy_field, self.description().unwrap_or_default());}Field::Url => doc.add_text(tantivy_field, self.url()),Field::Host => doc.add_text(tantivy_field, self.host()),Field::Domain => doc.add_text(tantivy_field, self.domain()),Field::DomainIfHomepage => {if self.is_homepage() {- doc.add_text(tantivy_field, self.domain())+ doc.add_text(tantivy_field, self.domain());} else {- doc.add_text(tantivy_field, "")+ doc.add_text(tantivy_field, "");}}Field::IsHomepage => {- doc.add_u64(tantivy_field, if self.is_homepage() { 1 } else { 0 })+ doc.add_u64(tantivy_field, if self.is_homepage() { 1 } else { 0 });}Field::LastUpdated => doc.add_u64(tantivy_field,self.updated_time()- .map(|time| time.timestamp().max(0) as u64)- .unwrap_or(0),+ .map_or(0, |time| time.timestamp().max(0) as u64),),Field::StemmedAllBody | Field::AllBody => {let text = self.all_text();@@ -447,7 +446,7 @@ impl Html {return Err(Error::EmptyField("all body"));}- doc.add_text(tantivy_field, text.unwrap())+ doc.add_text(tantivy_field, text.unwrap());}Field::NumTrackers => doc.add_u64(tantivy_field, self.trackers().len() as u64),Field::BacklinkText@@ -482,7 +481,7 @@ impl Html {scripts.push(Script {attributes,content,- })+ });}scripts@@ -493,7 +492,7 @@ impl Html {for schema in self.scripts().into_iter().filter(|script| {matches!(- script.attributes.get("type").map(|s| s.as_str()),+ script.attributes.get("type").map(String::as_str),Some("application/ld+json"))}) {@@ -520,7 +519,7 @@ impl Html {for node in self.root.select("link").unwrap() {if let Some(link) = node.attributes.borrow().get("href") {- links.push(link.to_string().into())+ links.push(link.to_string().into());}}@@ -584,9 +583,9 @@ impl Html {self.schema_org().into_iter().filter(|schema| matches!(schema, SchemaOrg::ImageObject(_)))- .flat_map(|schema| {+ .filter_map(|schema| {match schema {- SchemaOrg::ImageObject(image) => image.content_url.map(|url| url.into()),+ SchemaOrg::ImageObject(image) => image.content_url.map(Url::from),_ => None, // has been filtered, so only image is possible}})@@ -874,7 +873,7 @@ mod tests {height: Some(192),image_type: None})- )+ );}fn full_link_favicon(href: &str, site_url: &str, expected: &str) {@@ -973,7 +972,7 @@ mod tests {author: Some("Jane Doe".to_string()),content_url: Some("mexico-beach.jpg".to_string()),})]- )+ );}#[test]@@ -1012,7 +1011,7 @@ mod tests {let html = Html::parse(html, "example.com");- assert!(html.schema_org().is_empty())+ assert!(html.schema_org().is_empty());}#[test]@@ -1220,7 +1219,7 @@ mod tests {"thirdparty.com".to_string(),"securepubads.g.doubleclick.net".to_string()]- )+ );}#[test]@@ -1254,6 +1253,6 @@ mod tests {);assert_eq!(html.title(), Some("Test site".to_string()));- assert_eq!(html.all_text(), Some("test".to_string()))+ assert_eq!(html.all_text(), Some("test".to_string()));}}