Skip to content

web search done right

clippy shenanigans

Mikkel Denker467ed085deb2
diff --git a/src/directory.rs b/src/directory.rs
index 13eb9024..776933cc 100644
--- a/src/directory.rs
+++ b/src/directory.rs
@@ -41,12 +41,12 @@ fn iterate_children(path: &str) -> Result<Vec<DirEntry>> {
res.push(DirEntry::Folder {
name: entry.path().as_os_str().to_str().unwrap().to_string(),
entries: iterate_children(entry.path().as_os_str().to_str().unwrap())?,
- })
+ });
} else if metadata.is_file() {
res.push(DirEntry::File {
name: entry.path().as_os_str().to_str().unwrap().to_string(),
content: fs::read(entry.path())?,
- })
+ });
}
}
diff --git a/src/entity_index/entity.rs b/src/entity_index/entity.rs
index 85e4177d..81a82053 100644
--- a/src/entity_index/entity.rs
+++ b/src/entity_index/entity.rs
@@ -57,11 +57,11 @@ impl Span {
}
}
- pub fn add_link(&mut self, text: String, link: Link) {
+ pub fn add_link(&mut self, text: &str, link: Link) {
debug_assert_eq!(self.text.chars().count() + text.chars().count(), link.end);
debug_assert_eq!(self.text.chars().count(), link.start);
self.links.push(link);
- self.text.push_str(&text);
+ self.text.push_str(text);
}
}
@@ -105,7 +105,7 @@ impl<'a> From<Vec<Node<'a>>> for Span {
start: span.text.chars().count(),
end: span.text.chars().count() + text.chars().count(),
};
- span.add_link(text, link);
+ span.add_link(&text, link);
}
Node::Text {
end: _,
@@ -117,9 +117,9 @@ impl<'a> From<Vec<Node<'a>>> for Span {
span.text.push_str(". ");
}
continue;
- } else {
- span.text.push_str(value);
}
+
+ span.text.push_str(value);
}
Node::Template {
end: _,
diff --git a/src/entity_index/mod.rs b/src/entity_index/mod.rs
index 6b58c63a..12738a99 100644
--- a/src/entity_index/mod.rs
+++ b/src/entity_index/mod.rs
@@ -35,7 +35,7 @@ use crate::{
image_downloader::{ImageDownloadJob, ImageDownloader},
image_store::{EntityImageStore, Image, ImageStore},
kv::{rocksdb_store::RocksDbStore, Kv},
- tokenizer::NormalTokenizer,
+ tokenizer::Normal,
webpage::Url,
Result,
};
@@ -61,7 +61,7 @@ fn schema() -> Schema {
TextOptions::default()
.set_indexing_options(
TextFieldIndexing::default()
- .set_tokenizer(NormalTokenizer::as_str())
+ .set_tokenizer(Normal::as_str())
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
)
.set_stored(),
@@ -71,7 +71,7 @@ fn schema() -> Schema {
TextOptions::default()
.set_indexing_options(
TextFieldIndexing::default()
- .set_tokenizer(NormalTokenizer::as_str())
+ .set_tokenizer(Normal::as_str())
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
)
.set_stored(),
@@ -118,7 +118,7 @@ fn entity_to_tantivy(entity: Entity, schema: &tantivy::schema::Schema) -> tantiv
doc
}
-fn wikipedify_url(url: Url) -> Vec<Url> {
+fn wikipedify_url(url: &Url) -> Vec<Url> {
let mut name = url.raw().replace(' ', "_");
if name.starts_with("File:") {
@@ -174,12 +174,12 @@ impl EntityIndex {
let stopwords: HashSet<String> = include_str!("../../stopwords/English.txt")
.lines()
.take(50)
- .map(|word| word.to_ascii_lowercase())
+ .map(str::to_ascii_lowercase)
.collect();
tantivy_index.tokenizers().register(
- NormalTokenizer::as_str(),
- NormalTokenizer::with_stopwords(stopwords.clone().into_iter().collect()),
+ Normal::as_str(),
+ Normal::with_stopwords(stopwords.clone().into_iter().collect()),
);
let image_store = EntityImageStore::open(path.as_ref().join("images"));
@@ -206,15 +206,16 @@ impl EntityIndex {
}
if let Some(image) = entity.image.clone() {
- let image = wikipedify_url(image)
+ let image = wikipedify_url(&image)
.into_iter()
- .filter(|url| url.is_valid_uri())
+ .filter(Url::is_valid_uri)
.collect();
+
self.image_downloader.schedule(ImageDownloadJob {
key: entity.title.clone(),
urls: image,
timeout: Some(Duration::from_secs(10)),
- })
+ });
}
let doc = entity_to_tantivy(entity, &self.schema);
self.writer.add_document(doc).unwrap();
@@ -252,7 +253,7 @@ impl EntityIndex {
.into_iter()
.filter(|(_, related_doc)| doc != *related_doc)
.map(|(_, doc_address)| {
- self.retrieve_stored_entity(&searcher, &doc_address, false, false, false)
+ self.retrieve_stored_entity(&searcher, doc_address, false, false, false)
})
.filter(|entity| entity.image.is_some())
.take(4)
@@ -261,14 +262,14 @@ impl EntityIndex {
}
}
- pub fn search(&self, query: String) -> Option<StoredEntity> {
+ pub fn search(&self, query: &str) -> Option<StoredEntity> {
let searcher = self.reader.searcher();
let title = self.schema.get_field("title").unwrap();
let entity_abstract = self.schema.get_field("abstract").unwrap();
let mut term_queries = Vec::new();
- let mut stream = NormalTokenizer::default().token_stream(query.as_str());
+ let mut stream = Normal::default().token_stream(query);
while let Some(token) = stream.next() {
if self.stopwords.contains(&token.text) {
continue;
@@ -300,14 +301,14 @@ impl EntityIndex {
.unwrap()
.first()
.map(|(_score, doc_address)| {
- self.retrieve_stored_entity(&searcher, doc_address, true, true, true)
+ self.retrieve_stored_entity(&searcher, *doc_address, true, true, true)
})
}
fn retrieve_stored_entity(
&self,
searcher: &LeasedItem<Searcher>,
- doc_address: &DocAddress,
+ doc_address: DocAddress,
get_related: bool,
decode_info: bool,
get_links: bool,
@@ -317,7 +318,7 @@ impl EntityIndex {
let info = self.schema.get_field("info").unwrap();
let links = self.schema.get_field("links").unwrap();
- let doc = searcher.doc(*doc_address).unwrap();
+ let doc = searcher.doc(doc_address).unwrap();
let title = doc
.get_first(title)
.and_then(|val| match val {
@@ -349,7 +350,7 @@ impl EntityIndex {
};
let related_entities = if get_related {
- self.related_entities(*doc_address)
+ self.related_entities(doc_address)
} else {
Vec::new()
};
@@ -398,7 +399,7 @@ mod tests {
#[test]
fn wikipedia_image_url_aristotle() {
assert_eq!(
- wikipedify_url("Aristotle Altemps Inv8575.jpg".to_string().into())
+ wikipedify_url(&"Aristotle Altemps Inv8575.jpg".to_string().into())
.first()
.unwrap()
.full(),
@@ -410,7 +411,7 @@ mod tests {
#[test]
fn wikipedia_image_url_with_file() {
assert_eq!(
- wikipedify_url("File:Aristotle Altemps Inv8575.jpg".to_string().into())
+ wikipedify_url(&"File:Aristotle Altemps Inv8575.jpg".to_string().into())
.first()
.unwrap()
.full(),
@@ -437,17 +438,10 @@ mod tests {
index.commit();
- assert_eq!(index.search("the".to_string()), None);
- assert_eq!(
- index.search("ashes".to_string()).unwrap().title.as_str(),
- "the ashes"
- );
+ assert_eq!(index.search("the"), None);
+ assert_eq!(index.search("ashes").unwrap().title.as_str(), "the ashes");
assert_eq!(
- index
- .search("the ashes".to_string())
- .unwrap()
- .title
- .as_str(),
+ index.search("the ashes").unwrap().title.as_str(),
"the ashes"
);
}
diff --git a/src/entrypoint/centrality.rs b/src/entrypoint/centrality.rs
index f7a40fbb..baad8828 100644
--- a/src/entrypoint/centrality.rs
+++ b/src/entrypoint/centrality.rs
@@ -18,9 +18,9 @@ use std::path::Path;
use crate::{ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilder};
-pub struct CentralityEntrypoint {}
+pub struct Centrality {}
-impl CentralityEntrypoint {
+impl Centrality {
pub fn run<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
let graph = WebgraphBuilder::new(webgraph_path).with_host_graph().open();
diff --git a/src/entrypoint/entity.rs b/src/entrypoint/entity.rs
index 530ebc5b..6ec057e5 100644
--- a/src/entrypoint/entity.rs
+++ b/src/entrypoint/entity.rs
@@ -263,7 +263,7 @@ impl EntityBuilder {
+ text.chars().count(),
target: target.to_string(),
};
- current_paragraph.content.add_link(text, link)
+ current_paragraph.content.add_link(&text, link)
}
None => {
let link = Link {
diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rs
index eba431b0..847c99cf 100644
--- a/src/entrypoint/mod.rs
+++ b/src/entrypoint/mod.rs
@@ -19,7 +19,7 @@ pub mod frontend;
mod indexer;
mod webgraph;
-pub use centrality::CentralityEntrypoint;
+pub use centrality::Centrality;
pub use entity::EntityIndexer;
pub use indexer::Indexer;
-pub use webgraph::WebgraphEntrypoint;
+pub use webgraph::Webgraph;
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index e8349210..e43ff525 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -14,9 +14,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
+ directory::DirEntry,
mapreduce::{Map, MapReduce, Reduce, StatelessWorker, Worker},
warc::WarcFile,
- webgraph::{FrozenWebgraph, Node, Webgraph, WebgraphBuilder},
+ webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},
webpage::Html,
HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,
};
@@ -81,11 +82,12 @@ impl Map<StatelessWorker, FrozenWebgraph> for Job {
}
}
-impl Reduce<FrozenWebgraph> for Webgraph {
- fn reduce(mut self, other: FrozenWebgraph) -> Webgraph {
+impl Reduce<FrozenWebgraph> for webgraph::Webgraph {
+ fn reduce(mut self, other: FrozenWebgraph) -> webgraph::Webgraph {
let other_path = match &other.root {
- crate::directory::DirEntry::Folder { name, entries: _ } => name.clone(),
- crate::directory::DirEntry::File { name, content: _ } => name.clone(),
+ DirEntry::Folder { name, entries: _ } | DirEntry::File { name, content: _ } => {
+ name.clone()
+ }
};
let other = other.into();
@@ -98,8 +100,8 @@ impl Reduce<FrozenWebgraph> for Webgraph {
}
}
-impl Reduce<Webgraph> for Webgraph {
- fn reduce(mut self, element: Webgraph) -> Self {
+impl Reduce<webgraph::Webgraph> for webgraph::Webgraph {
+ fn reduce(mut self, element: webgraph::Webgraph) -> Self {
let other_path = element.path.clone();
self.merge(element);
@@ -109,9 +111,9 @@ impl Reduce<Webgraph> for Webgraph {
}
}
-pub struct WebgraphEntrypoint {}
+pub struct Webgraph {}
-impl WebgraphEntrypoint {
+impl Webgraph {
pub fn run_master(config: &WebgraphMasterConfig) -> Result<()> {
info!("Running master for webgraph construction");
@@ -144,7 +146,7 @@ impl WebgraphEntrypoint {
warc_paths = Box::new(warc_paths.take(limit));
}
- let _graph: Webgraph = warc_paths
+ let _graph: webgraph::Webgraph = warc_paths
.map_reduce(&workers)
.expect("failed to build webgraph");
@@ -179,7 +181,7 @@ impl WebgraphEntrypoint {
})
.par_bridge()
.map(|job| job.map(&worker))
- .map(Webgraph::from)
+ .map(webgraph::Webgraph::from)
.reduce_with(|a, b| a.reduce(b));
Ok(())
diff --git a/src/frontend/autosuggest.rs b/src/frontend/autosuggest.rs
index b72f0752..3617567d 100644
--- a/src/frontend/autosuggest.rs
+++ b/src/frontend/autosuggest.rs
@@ -41,6 +41,7 @@ struct Suggestion {
raw: String,
}
+#[allow(clippy::unused_async)]
pub async fn route(
extract::Query(params): extract::Query<HashMap<String, String>>,
Extension(state): Extension<Arc<State>>,
diff --git a/src/frontend/entity_image.rs b/src/frontend/entity_image.rs
index d33440b5..49afd79d 100644
--- a/src/frontend/entity_image.rs
+++ b/src/frontend/entity_image.rs
@@ -25,6 +25,7 @@ use reqwest::StatusCode;
use super::State;
+#[allow(clippy::unused_async)]
pub async fn route(
Path(entity): Path<String>,
Extension(state): Extension<Arc<State>>,
diff --git a/src/frontend/favicons.rs b/src/frontend/favicons.rs
index 9a4b9ce2..047bbccc 100644
--- a/src/frontend/favicons.rs
+++ b/src/frontend/favicons.rs
@@ -27,6 +27,7 @@ use crate::webpage::Url;
use super::State;
+#[allow(clippy::unused_async)]
pub async fn route(
Path(site): Path<String>,
Extension(state): Extension<Arc<State>>,
diff --git a/src/frontend/index.rs b/src/frontend/index.rs
index 39736791..0245910b 100644
--- a/src/frontend/index.rs
+++ b/src/frontend/index.rs
@@ -18,6 +18,7 @@ use super::HtmlTemplate;
use askama::Template;
use axum::response::IntoResponse;
+#[allow(clippy::unused_async)]
pub async fn route() -> impl IntoResponse {
let template = IndexTemplate {};
HtmlTemplate(template)
diff --git a/src/frontend/primary_image.rs b/src/frontend/primary_image.rs
index 90ecc190..cb19cb6b 100644
--- a/src/frontend/primary_image.rs
+++ b/src/frontend/primary_image.rs
@@ -25,6 +25,7 @@ use reqwest::StatusCode;
use super::State;
+#[allow(clippy::unused_async)]
pub async fn route(
Path(uuid): Path<String>,
Extension(state): Extension<Arc<State>>,
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index 7cb46c57..7c0e257c 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -59,7 +59,7 @@ pub struct DisplayedWebpage {
const MAX_PRETTY_URL_LEN: usize = 50;
const MAX_TITLE_LEN: usize = 50;
-fn prettify_url(url: Url) -> String {
+fn prettify_url(url: &Url) -> String {
let mut pretty_url = url.strip_query().to_string();
if pretty_url.ends_with('/') {
@@ -88,18 +88,18 @@ fn prettify_date(date: NaiveDateTime) -> String {
if num_hours < 24 {
if num_hours <= 1 {
return "1 hour ago".to_string();
- } else {
- return format!("{num_hours} hours ago");
}
+
+ return format!("{num_hours} hours ago");
}
let num_days = diff.num_days();
if num_days < 30 {
if num_days <= 1 {
return "1 day ago".to_string();
- } else {
- return format!("{num_days} days ago");
}
+
+ return format!("{num_days} days ago");
}
format!("{}", date.format("%d. %b. %Y"))
@@ -111,7 +111,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
let url: Url = webpage.url.clone().into();
let domain = url.domain().to_string();
- let pretty_url = prettify_url(url);
+ let pretty_url = prettify_url(&url);
let mut title = html_escape(&webpage.title);
@@ -253,6 +253,8 @@ enum RegionSelection {
Unselected(Region),
}
+#[allow(clippy::unused_async)]
+#[allow(clippy::match_wild_err_arm)]
pub async fn route(
extract::Query(params): extract::Query<HashMap<String, String>>,
Extension(state): Extension<Arc<State>>,
@@ -454,7 +456,7 @@ mod tests {
),
"some <a href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link"
.to_string()
- )
+ );
}
#[test]
@@ -472,7 +474,7 @@ mod tests {
7
),
"some <a href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string()
- )
+ );
}
#[test]
@@ -497,9 +499,9 @@ mod tests {
assert_eq!(thousand_sep_number(10), "10".to_string());
assert_eq!(thousand_sep_number(100), "100".to_string());
assert_eq!(thousand_sep_number(1000), "1.000".to_string());
- assert_eq!(thousand_sep_number(10000), "10.000".to_string());
- assert_eq!(thousand_sep_number(100000), "100.000".to_string());
- assert_eq!(thousand_sep_number(512854), "512.854".to_string());
- assert_eq!(thousand_sep_number(9512854), "9.512.854".to_string());
+ assert_eq!(thousand_sep_number(10_000), "10.000".to_string());
+ assert_eq!(thousand_sep_number(100_000), "100.000".to_string());
+ assert_eq!(thousand_sep_number(512_854), "512.854".to_string());
+ assert_eq!(thousand_sep_number(9_512_854), "9.512.854".to_string());
}
}
diff --git a/src/kv/mod.rs b/src/kv/mod.rs
index 32b6713d..386a7028 100644
--- a/src/kv/mod.rs
+++ b/src/kv/mod.rs
@@ -40,6 +40,6 @@ where
let key_bytes = bincode::serialize(&key).expect("failed to serialize key");
let val_bytes = bincode::serialize(&value).expect("failed to serialize value");
- self.insert_raw(key_bytes, val_bytes)
+ self.insert_raw(key_bytes, val_bytes);
}
}
diff --git a/src/kv/rocksdb_store.rs b/src/kv/rocksdb_store.rs
index c6f02dc2..435a3885 100644
--- a/src/kv/rocksdb_store.rs
+++ b/src/kv/rocksdb_store.rs
@@ -60,8 +60,8 @@ where
Box::new(IntoIter {
inner: iter,
- key: Default::default(),
- value: Default::default(),
+ key: PhantomData::default(),
+ value: PhantomData::default(),
})
}
}
diff --git a/src/lib.rs b/src/lib.rs
index c33bca29..1fba47e3 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -16,6 +16,9 @@
// #![warn(clippy::pedantic)]
+#![allow(clippy::cast_precision_loss)]
+#![allow(clippy::missing_errors_doc)]
+
use serde::{Deserialize, Serialize};
use std::fs::File;
use std::io::{self, BufRead};
diff --git a/src/main.rs b/src/main.rs
index 7a6ae6d8..c9cc364c 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -15,9 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::Result;
use clap::{Parser, Subcommand};
-use cuely::entrypoint::{
- frontend, CentralityEntrypoint, EntityIndexer, Indexer, WebgraphEntrypoint,
-};
+use cuely::entrypoint::{self, frontend};
use serde::de::DeserializeOwned;
use std::fs;
use std::path::Path;
@@ -97,38 +95,38 @@ fn main() -> Result<()> {
Commands::Indexer { options } => match options {
IndexingOptions::Master { config_path } => {
let config = load_toml_config(&config_path);
- Indexer::run_master(&config)?;
+ entrypoint::Indexer::run_master(&config)?;
}
IndexingOptions::Worker {
address,
centrality_store_path,
} => {
- Indexer::run_worker(address, centrality_store_path)?;
+ entrypoint::Indexer::run_worker(address, centrality_store_path)?;
}
IndexingOptions::Local { config_path } => {
let config = load_toml_config(&config_path);
- Indexer::run_locally(&config)?;
+ entrypoint::Indexer::run_locally(&config)?;
}
IndexingOptions::Entity {
wikipedia_dump_path,
output_path,
- } => EntityIndexer::run(wikipedia_dump_path, output_path)?,
+ } => entrypoint::EntityIndexer::run(wikipedia_dump_path, output_path)?,
},
Commands::Centrality {
webgraph_path,
output_path,
- } => CentralityEntrypoint::run(webgraph_path, output_path),
+ } => entrypoint::Centrality::run(webgraph_path, output_path),
Commands::Webgraph { options } => match options {
WebgraphOptions::Master { config_path } => {
let config = load_toml_config(config_path);
- WebgraphEntrypoint::run_master(&config)?;
+ entrypoint::Webgraph::run_master(&config)?;
}
WebgraphOptions::Worker { address } => {
- WebgraphEntrypoint::run_worker(address)?;
+ entrypoint::Webgraph::run_worker(address)?;
}
WebgraphOptions::Local { config_path } => {
let config = load_toml_config(config_path);
- WebgraphEntrypoint::run_locally(&config)?;
+ entrypoint::Webgraph::run_locally(&config)?;
}
},
Commands::Frontend {
diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rs
index 27ed2fdd..8ebea4c5 100644
--- a/src/mapreduce/manager.rs
+++ b/src/mapreduce/manager.rs
@@ -108,7 +108,7 @@ impl<'a> WorkerGuard<'a> {
}
fn success(self) {
- self.from_pool.insert(Arc::clone(&self.worker))
+ self.from_pool.insert(Arc::clone(&self.worker));
}
}
@@ -122,7 +122,7 @@ impl<'a> Deref for WorkerGuard<'a> {
impl<'a> Drop for WorkerGuard<'a> {
fn drop(&mut self) {
- self.from_pool.put_back()
+ self.from_pool.put_back();
}
}
@@ -185,7 +185,7 @@ impl WorkerPool {
let mut failing_workers = Vec::new();
for worker in &self.all_workers {
if worker.stop::<W, I, O>().is_err() {
- failing_workers.push(worker)
+ failing_workers.push(worker);
}
}
@@ -272,6 +272,7 @@ impl Manager {
}
}
+ #[allow(clippy::trait_duplication_in_bounds)]
fn get_results<W, I, O1, O2>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>
where
W: Worker,
@@ -327,6 +328,7 @@ impl Manager {
x
}
+ #[allow(clippy::trait_duplication_in_bounds)]
pub fn run<W, I, O1, O2>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>
where
W: Worker,
diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rs
index c55f919c..570a62e9 100644
--- a/src/mapreduce/mod.rs
+++ b/src/mapreduce/mod.rs
@@ -52,9 +52,11 @@ where
}
pub trait Reduce<T> {
+ #[must_use]
fn reduce(self, element: T) -> Self;
}
+#[allow(clippy::trait_duplication_in_bounds)]
pub trait MapReduce<W, I, O1, O2>
where
Self: Sized + Iterator<Item = I> + Send,
@@ -68,6 +70,8 @@ where
manager.run(self)
}
}
+
+#[allow(clippy::trait_duplication_in_bounds)]
impl<W, I, O1, O2, T> MapReduce<W, I, O1, O2> for T
where
W: Worker,
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rs
index 4535304b..36e1578e 100644
--- a/src/ranking/initial.rs
+++ b/src/ranking/initial.rs
@@ -132,9 +132,10 @@ fn region_score(
selected_region: Option<Region>,
webpage_region: Region,
) -> f64 {
- let boost = selected_region
- .map(|region| if region == webpage_region { 50.0 } else { 0.0 })
- .unwrap_or(0.0);
+ let boost = selected_region.map_or(
+ 0.0,
+ |region| if region == webpage_region { 50.0 } else { 0.0 },
+ );
boost + region_count.score(&webpage_region)
}
diff --git a/src/schema.rs b/src/schema.rs
index 4ce019f5..6ca34087 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -18,7 +18,7 @@ use tantivy::schema::{
Cardinality, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions,
};
-use crate::tokenizer::{NormalTokenizer, StemmedTokenizer};
+use crate::tokenizer::{Normal, Stemmed};
pub const CENTRALITY_SCALING: u64 = 1_000_000_000;
@@ -79,7 +79,7 @@ impl Field {
}
fn default_text_options(&self) -> tantivy::schema::TextOptions {
- self.default_text_options_with_tokenizer(NormalTokenizer::as_str())
+ self.default_text_options_with_tokenizer(Normal::as_str())
}
pub fn options(&self) -> IndexingOption {
@@ -91,7 +91,7 @@ impl Field {
Field::Domain => IndexingOption::Text(self.default_text_options()),
Field::AllBody => IndexingOption::Text(self.default_text_options()),
Field::StemmedAllBody => IndexingOption::Text(
- self.default_text_options_with_tokenizer(StemmedTokenizer::as_str())
+ self.default_text_options_with_tokenizer(Stemmed::as_str())
.set_stored(),
),
Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),
@@ -106,11 +106,11 @@ impl Field {
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
- Field::StemmedTitle => IndexingOption::Text(
- self.default_text_options_with_tokenizer(StemmedTokenizer::as_str()),
- ),
+ Field::StemmedTitle => {
+ IndexingOption::Text(self.default_text_options_with_tokenizer(Stemmed::as_str()))
+ }
Field::StemmedCleanBody => IndexingOption::Text(
- self.default_text_options_with_tokenizer(StemmedTokenizer::as_str())
+ self.default_text_options_with_tokenizer(Stemmed::as_str())
.set_stored(),
),
Field::FetchTimeMs => IndexingOption::Numeric(
diff --git a/src/schema_org.rs b/src/schema_org.rs
index a953b781..0ca00289 100644
--- a/src/schema_org.rs
+++ b/src/schema_org.rs
@@ -61,6 +61,6 @@ mod tests {
author: Some("Jane Doe".to_string()),
content_url: Some("mexico-beach.jpg".to_string()),
}),
- )
+ );
}
}
diff --git a/src/searcher.rs b/src/searcher.rs
index 698474f0..79329109 100644
--- a/src/searcher.rs
+++ b/src/searcher.rs
@@ -82,7 +82,7 @@ impl Searcher {
let entity = self
.entity_index
.as_ref()
- .and_then(|index| index.search(raw_query));
+ .and_then(|index| index.search(&raw_query));
let search_duration_ms = start.elapsed().as_millis();
diff --git a/src/snippet.rs b/src/snippet.rs
index e917e480..78849a76 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::frontend::search::html_escape;
-use crate::tokenizer::StemmedTokenizer;
+use crate::tokenizer::Stemmed;
use std::cmp::Ordering;
use std::collections::BTreeMap;
use std::ops::Range;
@@ -24,8 +24,8 @@ use tantivy::tokenizer::{TextAnalyzer, Token};
use tantivy::{Score, Searcher};
use whatlang::Lang;
-/// For now we use the snippet generator from tantivy with a minor modification to support our TokenStreamMerger.
-/// In the future we want to implement something closer to the method described in https://cs.pomona.edu/~dkauchak/ir_project/whitepapers/Snippet-IL.pdf.
+/// For now we use the snippet generator from tantivy with a minor modifications.
+/// In the future we want to implement something closer to the method described in <https://cs.pomona.edu/~dkauchak/ir_project/whitepapers/Snippet-IL.pdf>.
/// This will require us to store each paragraph of the webpage separately to get adequate performance.
/// Implementing SnippetIL will also allow us to correctly add "..." to the snippet.
@@ -44,7 +44,7 @@ impl FragmentCandidate {
///
/// `score`, `num_chars` are set to 0
/// and `highlighted` is set to empty vec
- /// stop_offset is set to start_offset, which is taken as a param.
+ /// `stop_offset` is set to `start_offset`, which is taken as a param.
fn new(start_offset: usize) -> FragmentCandidate {
FragmentCandidate {
score: 0.0,
@@ -261,8 +261,7 @@ pub fn generate(
.expect("Failed to get body field");
let tokenizer =
- StemmedTokenizer::with_forced_language(whatlang::detect_lang(text).unwrap_or(Lang::Eng))
- .into();
+ Stemmed::with_forced_language(whatlang::detect_lang(text).unwrap_or(Lang::Eng)).into();
let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;
let mut snippet = generator.snippet(text);
@@ -270,10 +269,9 @@ pub fn generate(
if snippet.fragment.is_empty() {
match description {
Some(desc) => {
- let tokenizer = StemmedTokenizer::with_forced_language(
- whatlang::detect_lang(desc).unwrap_or(Lang::Eng),
- )
- .into();
+ let tokenizer =
+ Stemmed::with_forced_language(whatlang::detect_lang(desc).unwrap_or(Lang::Eng))
+ .into();
let generator = SnippetGenerator::create(searcher, query, field, tokenizer)?;
snippet = generator.snippet(desc);
@@ -283,7 +281,7 @@ pub fn generate(
}
None => {
if text.is_empty() {
- let tokenizer = StemmedTokenizer::with_forced_language(
+ let tokenizer = Stemmed::with_forced_language(
whatlang::detect_lang(dirty_text).unwrap_or(Lang::Eng),
)
.into();
diff --git a/src/spell/dictionary.rs b/src/spell/dictionary.rs
index 9ab1c6f2..4459ffee 100644
--- a/src/spell/dictionary.rs
+++ b/src/spell/dictionary.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::spell::distance::LevenshteinDistance;
-use crate::tokenizer::NormalTokenizer;
+use crate::tokenizer::Normal;
use crate::webpage::Webpage;
use fst::map::Union;
use fst::{Automaton, IntoStreamer, Map, MapBuilder, Streamer};
@@ -336,7 +336,7 @@ impl<const TOP_N: usize> Dictionary<TOP_N> {
pub fn insert_page(&mut self, webpage: &Webpage) {
let text = webpage.html.clean_text().unwrap_or_default();
- let mut stream = NormalTokenizer::default().token_stream(text.as_str());
+ let mut stream = Normal::default().token_stream(text.as_str());
while let Some(token) = stream.next() {
self.insert(&token.text);
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 092cdd23..3c0bd193 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -29,7 +29,6 @@ impl From<Lang> for MyStemmer {
Lang::Dan => MyStemmer(Stemmer::new(Language::Danish)),
Lang::Ara => MyStemmer(Stemmer::new(Language::Arabic)),
Lang::Nld => MyStemmer(Stemmer::new(Language::Dutch)),
- Lang::Eng => MyStemmer(Stemmer::new(Language::English)),
Lang::Fin => MyStemmer(Stemmer::new(Language::Finnish)),
Lang::Fra => MyStemmer(Stemmer::new(Language::French)),
Lang::Deu => MyStemmer(Stemmer::new(Language::German)),
@@ -49,34 +48,34 @@ impl From<Lang> for MyStemmer {
#[derive(Clone)]
pub enum Tokenizer {
- NormalTokenizer(NormalTokenizer),
- StemmedTokenizer(StemmedTokenizer),
+ NormalTokenizer(Normal),
+ StemmedTokenizer(Stemmed),
}
impl Tokenizer {
pub fn new_stemmed() -> Self {
- Self::StemmedTokenizer(Default::default())
+ Self::StemmedTokenizer(Stemmed::default())
}
pub fn as_str(&self) -> &'static str {
match self {
- Tokenizer::NormalTokenizer(_) => NormalTokenizer::as_str(),
- Tokenizer::StemmedTokenizer(_) => StemmedTokenizer::as_str(),
+ Tokenizer::NormalTokenizer(_) => Normal::as_str(),
+ Tokenizer::StemmedTokenizer(_) => Stemmed::as_str(),
}
}
}
impl Default for Tokenizer {
fn default() -> Self {
- Self::NormalTokenizer(Default::default())
+ Self::NormalTokenizer(Normal::default())
}
}
#[derive(Clone, Default)]
-pub struct NormalTokenizer {
+pub struct Normal {
stopwords: Option<Vec<String>>,
}
-impl NormalTokenizer {
+impl Normal {
pub fn as_str() -> &'static str {
"tokenizer"
}
@@ -89,11 +88,11 @@ impl NormalTokenizer {
}
#[derive(Clone, Default)]
-pub struct StemmedTokenizer {
+pub struct Stemmed {
force_language: Option<Lang>,
}
-impl StemmedTokenizer {
+impl Stemmed {
pub fn as_str() -> &'static str {
"stemmed_tokenizer"
}
@@ -113,9 +112,9 @@ impl tantivy::tokenizer::Tokenizer for Tokenizer {
}
}
-impl tantivy::tokenizer::Tokenizer for NormalTokenizer {
+impl tantivy::tokenizer::Tokenizer for Normal {
fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
- let mut analyzer = TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser);
+ let mut analyzer = TextAnalyzer::from(Simple).filter(LowerCaser);
if let Some(stopwords) = &self.stopwords {
analyzer = analyzer.filter(StopWordFilter::remove(stopwords.clone()));
@@ -125,9 +124,9 @@ impl tantivy::tokenizer::Tokenizer for NormalTokenizer {
}
}
-impl tantivy::tokenizer::Tokenizer for StemmedTokenizer {
+impl tantivy::tokenizer::Tokenizer for Stemmed {
fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
- let analyzer = TextAnalyzer::from(SimpleTokenizer)
+ let analyzer = TextAnalyzer::from(Simple)
.filter(LowerCaser)
.filter(StopWordFilter::remove(vec![]));
@@ -154,7 +153,7 @@ enum Token {
}
#[derive(Clone)]
-pub struct SimpleTokenizer;
+pub struct Simple;
pub struct SimpleTokenStream<'a> {
lexer: Lexer<'a, Token>,
@@ -162,7 +161,7 @@ pub struct SimpleTokenStream<'a> {
next_position: usize,
}
-impl tantivy::tokenizer::Tokenizer for SimpleTokenizer {
+impl tantivy::tokenizer::Tokenizer for Simple {
fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
let lexer = Token::lexer(text);
BoxTokenStream::from(SimpleTokenStream {
@@ -208,7 +207,7 @@ mod tests {
fn tokenize(s: &str) -> Vec<String> {
let mut res = Vec::new();
- let mut stream = NormalTokenizer::default().token_stream(s);
+ let mut stream = Normal::default().token_stream(s);
while let Some(token) = stream.next() {
res.push(token.text.clone());
diff --git a/src/warc.rs b/src/warc.rs
index 9e174bc6..04fa3428 100644
--- a/src/warc.rs
+++ b/src/warc.rs
@@ -32,27 +32,26 @@ fn rtrim(s: &mut String) {
s.truncate(s.trim_end().len());
}
-fn decode(raw: Vec<u8>) -> String {
- match String::from_utf8(raw.clone()) {
- Ok(res) => res,
- Err(_) => {
- let encodings = [
- encoding_rs::WINDOWS_1251,
- encoding_rs::GBK,
- encoding_rs::SHIFT_JIS,
- encoding_rs::EUC_JP,
- encoding_rs::EUC_KR,
- ];
-
- for enc in encodings {
- let (cow, _, had_errors) = enc.decode(&raw[..]);
- if !had_errors {
- return cow.to_string();
- }
+fn decode(raw: &[u8]) -> String {
+ if let Ok(res) = String::from_utf8(raw.to_owned()) {
+ res
+ } else {
+ let encodings = [
+ encoding_rs::WINDOWS_1251,
+ encoding_rs::GBK,
+ encoding_rs::SHIFT_JIS,
+ encoding_rs::EUC_JP,
+ encoding_rs::EUC_KR,
+ ];
+
+ for enc in encodings {
+ let (cow, _, had_errors) = enc.decode(raw);
+ if !had_errors {
+ return cow.to_string();
}
-
- return String::from_utf8_lossy(&raw).to_string();
}
+
+ String::from_utf8_lossy(raw).to_string()
}
}
@@ -157,7 +156,7 @@ pub(crate) struct Response {
impl Response {
fn from_raw(record: RawWarcRecord) -> Result<Self> {
- let content = decode(record.content);
+ let content = decode(&record.content[..]);
let (_header, content) = content
.split_once("\r\n\r\n")
diff --git a/src/webpage/just_text.rs b/src/webpage/just_text.rs
index c9c2b80a..f3b44aa4 100644
--- a/src/webpage/just_text.rs
+++ b/src/webpage/just_text.rs
@@ -182,6 +182,7 @@ impl Paragraph {
text.len()
}
+ #[allow(clippy::cast_precision_loss)]
fn link_density(&self) -> f64 {
self.chars_count_in_links as f64 / self.text.len() as f64
}
@@ -332,11 +333,8 @@ impl JustText {
res
}
- fn calculate_stopword_density(
- &self,
- paragraph: &Paragraph,
- stopwords: &HashSet<String>,
- ) -> f64 {
+ #[allow(clippy::cast_precision_loss)]
+ fn calculate_stopword_density(paragraph: &Paragraph, stopwords: &HashSet<String>) -> f64 {
paragraph
.text
.split_whitespace()
@@ -359,7 +357,7 @@ impl JustText {
for paragraph in paragraphs {
let classification = {
- let stopword_density: f64 = self.calculate_stopword_density(&paragraph, stopwords);
+ let stopword_density: f64 = Self::calculate_stopword_density(&paragraph, stopwords);
if paragraph.link_density() > self.max_link_density
|| paragraph.text.contains("\\xa9")
@@ -508,16 +506,10 @@ impl JustText {
paragraph.classification = match &paragraph.classification {
Classification::Final(_) => paragraph.classification.clone(),
Classification::Intermediate(intermediate) => match intermediate {
- IntermediateClassification::Good => {
+ IntermediateClassification::Good | IntermediateClassification::NearGood => {
Classification::Final(FinalClassification::Good)
}
- IntermediateClassification::NearGood => {
- Classification::Final(FinalClassification::Good)
- }
- IntermediateClassification::Short => {
- Classification::Final(FinalClassification::Bad)
- }
- IntermediateClassification::Bad => {
+ IntermediateClassification::Short | IntermediateClassification::Bad => {
Classification::Final(FinalClassification::Bad)
}
},
@@ -571,7 +563,7 @@ impl JustText {
Classification::Intermediate(IntermediateClassification::Bad)
}
- fn extract_text(&self, paragraphs: Vec<ClassifiedParagraph>) -> String {
+ fn extract_text(paragraphs: Vec<ClassifiedParagraph>) -> String {
paragraphs
.into_iter()
.filter(|paragraph| {
@@ -595,7 +587,7 @@ impl JustText {
self.contextual_classification(&mut classified);
- self.extract_text(classified)
+ Self::extract_text(classified)
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 0db54a36..778a8e3e 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -16,7 +16,7 @@
use crate::{schema_org::SchemaOrg, Error, Result};
use chrono::{DateTime, FixedOffset};
use itertools::Itertools;
-use kuchiki::{iter::NodeEdge, traits::*, NodeRef};
+use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};
use regex::Regex;
use std::{borrow::Borrow, collections::HashMap};
use uuid::Uuid;
@@ -242,7 +242,7 @@ impl Html {
source: self.url.clone(),
destination: dest.to_string().into(),
text: text.trim().to_string(),
- })
+ });
}
}
}
@@ -293,9 +293,9 @@ impl Html {
let favicon = FaviconLink {
link,
- image_type,
width,
height,
+ image_type,
};
return Some(favicon);
@@ -344,10 +344,10 @@ impl Html {
pub fn title(&self) -> Option<String> {
if let Ok(title) = self.root.select_first("title") {
let title = title.text_contents().trim().to_string();
- if !title.is_empty() {
- Some(title)
- } else {
+ if title.is_empty() {
None
+ } else {
+ Some(title)
}
} else {
None
@@ -416,29 +416,28 @@ impl Html {
Field::CleanBody | Field::StemmedCleanBody => {
let text = self.clean_text();
- doc.add_text(tantivy_field, text.unwrap_or_default())
+ doc.add_text(tantivy_field, text.unwrap_or_default());
}
Field::Description => {
- doc.add_text(tantivy_field, self.description().unwrap_or_default())
+ doc.add_text(tantivy_field, self.description().unwrap_or_default());
}
Field::Url => doc.add_text(tantivy_field, self.url()),
Field::Host => doc.add_text(tantivy_field, self.host()),
Field::Domain => doc.add_text(tantivy_field, self.domain()),
Field::DomainIfHomepage => {
if self.is_homepage() {
- doc.add_text(tantivy_field, self.domain())
+ doc.add_text(tantivy_field, self.domain());
} else {
- doc.add_text(tantivy_field, "")
+ doc.add_text(tantivy_field, "");
}
}
Field::IsHomepage => {
- doc.add_u64(tantivy_field, if self.is_homepage() { 1 } else { 0 })
+ doc.add_u64(tantivy_field, if self.is_homepage() { 1 } else { 0 });
}
Field::LastUpdated => doc.add_u64(
tantivy_field,
self.updated_time()
- .map(|time| time.timestamp().max(0) as u64)
- .unwrap_or(0),
+ .map_or(0, |time| time.timestamp().max(0) as u64),
),
Field::StemmedAllBody | Field::AllBody => {
let text = self.all_text();
@@ -447,7 +446,7 @@ impl Html {
return Err(Error::EmptyField("all body"));
}
- doc.add_text(tantivy_field, text.unwrap())
+ doc.add_text(tantivy_field, text.unwrap());
}
Field::NumTrackers => doc.add_u64(tantivy_field, self.trackers().len() as u64),
Field::BacklinkText
@@ -482,7 +481,7 @@ impl Html {
scripts.push(Script {
attributes,
content,
- })
+ });
}
scripts
@@ -493,7 +492,7 @@ impl Html {
for schema in self.scripts().into_iter().filter(|script| {
matches!(
- script.attributes.get("type").map(|s| s.as_str()),
+ script.attributes.get("type").map(String::as_str),
Some("application/ld+json")
)
}) {
@@ -520,7 +519,7 @@ impl Html {
for node in self.root.select("link").unwrap() {
if let Some(link) = node.attributes.borrow().get("href") {
- links.push(link.to_string().into())
+ links.push(link.to_string().into());
}
}
@@ -584,9 +583,9 @@ impl Html {
self.schema_org()
.into_iter()
.filter(|schema| matches!(schema, SchemaOrg::ImageObject(_)))
- .flat_map(|schema| {
+ .filter_map(|schema| {
match schema {
- SchemaOrg::ImageObject(image) => image.content_url.map(|url| url.into()),
+ SchemaOrg::ImageObject(image) => image.content_url.map(Url::from),
_ => None, // has been filtered, so only image is possible
}
})
@@ -874,7 +873,7 @@ mod tests {
height: Some(192),
image_type: None
})
- )
+ );
}
fn full_link_favicon(href: &str, site_url: &str, expected: &str) {
@@ -973,7 +972,7 @@ mod tests {
author: Some("Jane Doe".to_string()),
content_url: Some("mexico-beach.jpg".to_string()),
})]
- )
+ );
}
#[test]
@@ -1012,7 +1011,7 @@ mod tests {
let html = Html::parse(html, "example.com");
- assert!(html.schema_org().is_empty())
+ assert!(html.schema_org().is_empty());
}
#[test]
@@ -1220,7 +1219,7 @@ mod tests {
"thirdparty.com".to_string(),
"securepubads.g.doubleclick.net".to_string()
]
- )
+ );
}
#[test]
@@ -1254,6 +1253,6 @@ mod tests {
);
assert_eq!(html.title(), Some("Test site".to_string()));
- assert_eq!(html.all_text(), Some("test".to_string()))
+ assert_eq!(html.all_text(), Some("test".to_string()));
}
}