web search done right
refactor long functions
Mikkel Denkerbba251d7e98a
diff --git a/clippy.toml b/clippy.tomlnew file mode 100644index 00000000..9ca075c1--- /dev/null+++ b/clippy.toml@@ -0,0 +1,2 @@+too-many-lines-threshold = 100+diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex d96012a0..de39fe06 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -15,8 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::anyhow;use encoding_rs::{Encoding, UTF_8};-use futures::{future::BoxFuture, FutureExt};-use hashbrown::{HashMap, HashSet};+use hashbrown::HashSet;use mime::Mime;use quick_xml::events::Event;use rand::seq::SliceRandom;@@ -48,6 +47,12 @@ use super::{const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB+const IGNORED_EXTENSIONS: [&str; 27] = [+ ".pdf", ".jpg", ".zip", ".png", ".css", ".js", ".json", ".jsonp", ".woff2", ".woff", ".ttf",+ ".svg", ".gif", ".jpeg", ".ico", ".mp4", ".mp3", ".avi", ".mov", ".mpeg", ".webm", ".wav",+ ".flac", ".aac", ".ogg", ".m4a", ".m4v",+];+struct ProcessedUrl {new_urls: Vec<Url>,response: UrlResponse,@@ -238,7 +243,7 @@ impl<S: DatumStream> JobExecutor<S> {if let Some(sitemap) = self.robotstxt.sitemap(retryable_url.url()).await {self.sitemap_urls- .extend(self.urls_from_sitemap(sitemap, 0, 5).await);+ .extend(self.urls_from_sitemap(sitemap, 5).await);}}@@ -276,92 +281,70 @@ impl<S: DatumStream> JobExecutor<S> {}}+ fn increase_politeness(&mut self) {+ self.politeness_factor *= 2.0;++ if self.politeness_factor > self.config.max_politeness_factor {+ self.politeness_factor = self.config.max_politeness_factor;+ }++ tracing::warn!("politeness factor increased to {}", self.politeness_factor);+ }++ fn new_urls(html: &Html) -> Vec<Url> {+ html.all_links()+ .into_iter()+ .map(|link| link.destination)+ .filter(|url| url.as_str().len() <= MAX_URL_LEN_BYTES)+ .filter(|url| {+ IGNORED_EXTENSIONS+ .iter()+ .all(|ext| !url.as_str().ends_with(ext))+ })+ .collect()+ }+async fn process_url(&mut self, url: Url) -> ProcessedUrl {let fetch = self.crawl_url(url.clone()).await;match fetch {- Ok(datum) => {- if matches!(datum.status_code, 200 | 301 | 302) {- if datum.status_code == 200 {- self.save_datum(datum.clone()).await;-- match Html::parse(&datum.body, datum.url.as_str()) {- Ok(html) => {- let new_urls = html- .all_links()- .into_iter()- .map(|link| link.destination)- .filter(|url| url.as_str().len() <= MAX_URL_LEN_BYTES)- .filter(|url| {- !url.path().ends_with(".pdf")- && !url.path().ends_with(".jpg")- && !url.path().ends_with(".zip")- && !url.path().ends_with(".png")- && !url.path().ends_with(".css")- && !url.path().ends_with(".js")- && !url.path().ends_with(".json")- && !url.path().ends_with(".jsonp")- && !url.path().ends_with(".woff2")- && !url.path().ends_with(".woff")- && !url.path().ends_with(".ttf")- && !url.path().ends_with(".svg")- && !url.path().ends_with(".gif")- && !url.path().ends_with(".jpeg")- && !url.path().ends_with(".ico")- && !url.path().ends_with(".mp4")- && !url.path().ends_with(".mp3")- && !url.path().ends_with(".avi")- && !url.path().ends_with(".mov")- && !url.path().ends_with(".mpeg")- && !url.path().ends_with(".webm")- && !url.path().ends_with(".wav")- && !url.path().ends_with(".flac")- && !url.path().ends_with(".aac")- && !url.path().ends_with(".ogg")- && !url.path().ends_with(".m4a")- && !url.path().ends_with(".m4v")- })- .collect();-- let url_res = UrlResponse::Success { url: datum.url };-- ProcessedUrl {- new_urls,- response: url_res,- }+ Ok(datum) => match datum.status_code {+ 200 => {+ self.save_datum(datum.clone()).await;++ match Html::parse(&datum.body, datum.url.as_str()) {+ Ok(html) => {+ let new_urls = Self::new_urls(&html);+ let url_res = UrlResponse::Success { url: datum.url };+ ProcessedUrl {+ new_urls,+ response: url_res,}- Err(_) => ProcessedUrl {- new_urls: Vec::new(),- response: UrlResponse::Failed {- url,- status_code: None,- },- },}- } else {- let url_res = UrlResponse::Redirected {- url,- new_url: datum.url,- };-- ProcessedUrl {+ Err(_) => ProcessedUrl {new_urls: Vec::new(),- response: url_res,- }+ response: UrlResponse::Failed {+ url,+ status_code: None,+ },+ },}- } else {- if datum.status_code == 429 {- self.politeness_factor *= 2.0;+ }+ 301 | 302 => {+ let url_res = UrlResponse::Redirected {+ url,+ new_url: datum.url,+ };- if self.politeness_factor > self.config.max_politeness_factor {- self.politeness_factor = self.config.max_politeness_factor;- }+ ProcessedUrl {+ new_urls: Vec::new(),+ response: url_res,+ }+ }- tracing::warn!(- "politeness factor increased to {} for {}",- self.politeness_factor,- &url- );+ _ => {+ if datum.status_code == 429 {+ self.increase_politeness();}tracing::debug!("failed to fetch url ({}): {}", &url, datum.status_code);@@ -373,7 +356,7 @@ impl<S: DatumStream> JobExecutor<S> {},}}- }+ },Err(err) => {tracing::debug!("failed to fetch url ({}): {}", &url, err);@@ -409,10 +392,8 @@ impl<S: DatumStream> JobExecutor<S> {.map_err(|e| e.into())}- async fn crawl_url(&self, url: Url) -> Result<CrawlDatum> {- let start = Instant::now();-- let res = if url.scheme() == "http" {+ async fn fetch_with_https_priority(&self, url: Url) -> Result<reqwest::Response> {+ if url.scheme() == "http" {let mut https = url.clone();https.set_scheme("https")@@ -427,10 +408,10 @@ impl<S: DatumStream> JobExecutor<S> {}} else {self.fetch(url.clone()).await- };-- let fetch_time = start.elapsed();+ }+ }+ async fn politeness_delay(&self, fetch_time: Duration) {let mut delay = fetch_time;if delay < Duration::from_millis(self.config.min_crawl_delay_ms) {@@ -444,37 +425,41 @@ impl<S: DatumStream> JobExecutor<S> {}tokio::time::sleep(delay).await;+ }- let res = res?;-- let headers: HashMap<_, _> = res- .headers()- .iter()- .filter_map(|(k, v)| {- let k = k.to_string();- if let Ok(v) = v.to_str() {- return Some((k, v.to_string()));- }-- None- })- .collect();-+ fn check_headers(&self, res: &reqwest::Response) -> Result<warc::PayloadType> {// check if content length is too large- if let Some(content_length) = headers.get("content-length") {- if content_length.parse::<usize>().unwrap_or(0) > MAX_CONTENT_LENGTH {+ if let Some(content_length) = res+ .headers()+ .get(reqwest::header::CONTENT_LENGTH)+ .and_then(|value| value.to_str().ok())+ .and_then(|value| value.parse::<usize>().ok())+ {+ if content_length > MAX_CONTENT_LENGTH {return Err(Error::ContentTooLarge.into());}}// check if content type is html- let payload_type = match headers.get("content-type") {- Some(ct) if ct.contains("text/html") => warc::PayloadType::Html,- Some(ct) if ct.contains("application/rss") => warc::PayloadType::Rss,- Some(ct) if ct.contains("application/atom") => warc::PayloadType::Atom,- ct => return Err(Error::InvalidContentType(format!("{ct:?}")).into()),- };+ match res+ .headers()+ .get(reqwest::header::CONTENT_TYPE)+ .and_then(|val| val.to_str().ok())+ {+ Some(ct) if ct.contains("text/html") => Ok(warc::PayloadType::Html),+ Some(ct) if ct.contains("application/rss") => Ok(warc::PayloadType::Rss),+ Some(ct) if ct.contains("application/atom") => Ok(warc::PayloadType::Atom),+ ct => Err(Error::InvalidContentType(format!("{ct:?}")).into()),+ }+ }+ fn redirect_datum(+ &self,+ res: &reqwest::Response,+ url: &Url,+ payload_type: warc::PayloadType,+ fetch_time: Duration,+ ) -> Result<Option<CrawlDatum>> {let status_code = res.status().as_u16();if status_code == 301 || status_code == 302 {@@ -489,17 +474,19 @@ impl<S: DatumStream> JobExecutor<S> {.or_else(|_| url.join(location)).map_err(|_| Error::InvalidRedirect)?;- return Ok(CrawlDatum {+ Ok(Some(CrawlDatum {url,status_code,payload_type,body: String::new(),fetch_time_ms: fetch_time.as_millis() as u64,- });+ }))+ } else {+ Ok(None)}+ }- let res_url = res.url().clone();-+ async fn encoded_body(&self, res: reqwest::Response) -> Result<String> {let content_type = res.headers().get(reqwest::header::CONTENT_TYPE)@@ -529,7 +516,28 @@ impl<S: DatumStream> JobExecutor<S> {}let (text, _, _) = encoding.decode(&bytes);- let body = text.to_string();+ Ok(text.to_string())+ }++ async fn crawl_url(&self, url: Url) -> Result<CrawlDatum> {+ let start = Instant::now();+ let res = self.fetch_with_https_priority(url.clone()).await;+ let fetch_time = start.elapsed();+ self.politeness_delay(fetch_time).await;++ // we want to delay before returning the error+ let res = res?;++ let payload_type = self.check_headers(&res)?;++ if let Some(datum) = self.redirect_datum(&res, &url, payload_type, fetch_time)? {+ return Ok(datum);+ }++ let status_code = res.status().as_u16();++ let res_url = res.url().clone();+ let body = self.encoded_body(res).await?;Ok(CrawlDatum {url: res_url,@@ -540,58 +548,51 @@ impl<S: DatumStream> JobExecutor<S> {})}- fn urls_from_sitemap(- &self,- sitemap: Url,- depth: usize,- max_depth: usize,- ) -> BoxFuture<'_, Vec<Url>> {- async move {- if depth == max_depth {- return vec![];+ async fn urls_from_sitemap(&self, sitemap: Url, max_depth: usize) -> Vec<Url> {+ let mut stack = vec![(sitemap, 0)];+ let mut urls = vec![];++ while let Some((url, depth)) = stack.pop() {+ if depth >= max_depth {+ continue;}- // fetch url- let res = self.fetch(sitemap).await;+ let res = self.fetch(url).await;+ tokio::time::sleep(Duration::from_millis(self.config.min_crawl_delay_ms)).await;if res.is_err() {- return vec![];+ continue;}+let res = res.unwrap();if res.status() != reqwest::StatusCode::OK {- return vec![];+ continue;}let body = res.text().await;if body.is_err() {- return vec![];+ continue;}let body = body.unwrap();- // parse xmllet entries = parse_sitemap(&body);- let mut urls = vec![];-for entry in entries {match entry {SitemapEntry::Url(url) => {urls.push(url);}SitemapEntry::Sitemap(url) => {- tokio::time::sleep(Duration::from_millis(self.config.min_crawl_delay_ms))- .await;- urls.append(&mut self.urls_from_sitemap(url, depth + 1, max_depth).await);+ stack.push((url, depth + 1));}}}-- urls}- .boxed()++ urls}}diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 78feed64..ce0d9f1e 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -17,7 +17,7 @@//! Main library for Stract.#![doc(html_logo_url = "https://stract.com/images/biglogo.svg")]-// #![warn(clippy::too_many_lines)]+#![warn(clippy::too_many_lines)]// pedantic stuff// #![warn(clippy::pedantic)]// #![allow(clippy::unreadable_literal)]diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rsindex b019292f..78d0ba11 100644--- a/crates/core/src/snippet.rs+++ b/crates/core/src/snippet.rs@@ -122,30 +122,19 @@ impl SnippetBuilder {TextSnippet { fragments }}-}-fn snippet_string_builder(- text: &str,- terms: &[String],- lang: whatlang::Lang,- config: SnippetConfig,- mut tokenizer: Tokenizer,-) -> SnippetBuilder {- let terms: HashSet<String> = terms- .iter()- .flat_map(|term| {- let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, term);-- let mut res = Vec::new();- while let Some(tok) = stream.next() {- res.push(tok.text.clone());- }+ fn trim_to_chars(&mut self, chars: usize) {+ self.fragment = self.fragment.chars().take(chars).collect()+ }- res.into_iter()- })- .collect();+ fn add_passage(&mut self, passage: &PassageCandidate) {+ self.fragment.push(' ');+ self.fragment.push_str(&passage.text);+ }+}- let mut passages: Vec<_> = sentence_ranges(text)+fn passages(text: &str, mut tokenizer: Tokenizer, config: &SnippetConfig) -> Vec<PassageCandidate> {+ sentence_ranges(text).into_iter().filter(|offset| offset.end - offset.start > config.min_passage_width).map(|offset| {@@ -167,39 +156,21 @@ fn snippet_string_builder(doc_terms,}})- .collect();-- if passages.is_empty() {- let mut snippet = SnippetBuilder {- fragment: text.chars().take(config.desired_num_chars).collect(),- highlights: Vec::new(),- };-- snippet.highlight(&terms, lang);-- return snippet;- }-- let mut total_d_size = 0;-- for passage in &passages {- total_d_size += passage.doc_terms.len();- }-- let avg_d_size = total_d_size / passages.len();+ .collect()+}+fn calculate_idf(terms: &HashSet<String>, passages: &[PassageCandidate]) -> HashMap<String, f64> {let mut n: HashMap<_, _> = terms.iter().map(|term| (term.to_string(), 0)).collect();- for term in &terms {- for passage in &passages {+ for term in terms.iter() {+ for passage in passages.iter() {if passage.doc_terms.contains_key(term) {*n.entry(term.to_string()).or_insert(0) += 1;}}}- let idf: HashMap<_, _> = n- .into_iter()+ n.into_iter().map(|(term, freq)| {(term,@@ -207,10 +178,23 @@ fn snippet_string_builder(.ln(),)})- .collect();+ .collect()+}++fn score_passages_with_bm25(+ passages: &mut [PassageCandidate],+ terms: &HashSet<String>,+ idf: &HashMap<String, f64>,+) {+ let mut total_d_size = 0;++ for passage in passages.iter() {+ total_d_size += passage.doc_terms.len();+ }- for passage in &mut passages {- for term in &terms {+ let avg_d_size = total_d_size / passages.len();+ for passage in passages.iter_mut() {+ for term in terms.iter() {let f = *passage.doc_terms.get(term).unwrap_or(&0) as f64;passage.score += idf[term]* ((f * (K1 + 1.0))@@ -218,6 +202,45 @@ fn snippet_string_builder(* (1.0 - B + B * (passage.doc_terms.len() as f64 / avg_d_size as f64))));}}+}++fn snippet_string_builder(+ text: &str,+ terms: &[String],+ lang: whatlang::Lang,+ config: SnippetConfig,+ mut tokenizer: Tokenizer,+) -> SnippetBuilder {+ let terms: HashSet<String> = terms+ .iter()+ .flat_map(|term| {+ let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, term);++ let mut res = Vec::new();+ while let Some(tok) = stream.next() {+ res.push(tok.text.clone());+ }++ res.into_iter()+ })+ .collect();++ let mut passages = passages(text, tokenizer, &config);++ if passages.is_empty() {+ let mut snippet = SnippetBuilder {+ fragment: text.chars().take(config.desired_num_chars).collect(),+ highlights: Vec::new(),+ };++ snippet.highlight(&terms, lang);++ return snippet;+ }++ let idf = calculate_idf(&terms, &passages);++ score_passages_with_bm25(&mut passages, &terms, &idf);let best_idx = passages.iter()@@ -233,28 +256,20 @@ fn snippet_string_builder(if snippet.fragment.len() > config.desired_num_chars + config.delta_num_chars {// TODO: find 'desired_num_chars' sized window that contains most highlights// instead of taking the prefix of the passage as a snippet- snippet.fragment = snippet- .fragment- .chars()- .take(config.desired_num_chars + config.delta_num_chars)- .collect();++ snippet.trim_to_chars(config.desired_num_chars + config.delta_num_chars);} else {let mut next_passage_idx = best_idx + 1;while snippet.fragment.len() < config.desired_num_chars - config.delta_num_chars&& next_passage_idx < passages.len(){- snippet.fragment += " ";- snippet.fragment += &passages[next_passage_idx].text;+ snippet.add_passage(&passages[next_passage_idx]);next_passage_idx += 1;}if snippet.fragment.len() > config.desired_num_chars + config.delta_num_chars {- snippet.fragment = snippet- .fragment- .chars()- .take(config.desired_num_chars + config.delta_num_chars)- .collect();+ snippet.trim_to_chars(config.desired_num_chars + config.delta_num_chars);}}snippet.highlight(&terms, lang);diff --git a/crates/core/src/warc.rs b/crates/core/src/warc.rsindex 757c656d..d7922da9 100644--- a/crates/core/src/warc.rs+++ b/crates/core/src/warc.rs@@ -234,7 +234,7 @@ impl Request {}}-#[derive(Debug, Clone, PartialEq, Eq)]+#[derive(Debug, Clone, Copy, PartialEq, Eq)]#[cfg_attr(test, derive(Arbitrary))]pub enum PayloadType {Html,diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rsindex f7d7fec2..b6c43559 100644--- a/crates/core/src/webgraph/centrality/harmonic.rs+++ b/crates/core/src/webgraph/centrality/harmonic.rs@@ -32,24 +32,169 @@ use crate::{const HYPERLOGLOG_COUNTERS: usize = 64;-fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {- let mut num_nodes = 0;-- let mut counters: BTreeMap<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>> = BTreeMap::new();+type Counter = BTreeMap<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>>;- let mut centralities: BTreeMap<NodeID, KahanSum> = BTreeMap::new();+fn initialize(+ graph: &Webgraph,+ counters: &mut Counters,+ centralities: &mut BTreeMap<NodeID, KahanSum>,+) -> u64 {+ let mut num_nodes = 0;for node in graph.nodes() {let mut counter = HyperLogLog::default();counter.add(node.as_u64());- counters.insert(node, counter);+ counters.old.insert(node, counter);centralities.insert(node, KahanSum::default());num_nodes += 1;}- let mut changed_nodes = BloomFilter::new(num_nodes as u64, 0.05);+ counters.new = counters.old.clone();++ num_nodes+}++fn update_changed_counters(+ graph: &Webgraph,+ exact_changed_nodes: &mut BTreeSet<NodeID>,+ counters: &mut Counters,+ new_changed_nodes: &mut BloomFilter,+) -> bool {+ let mut new_exact_changed_nodes = BTreeSet::default();+ let has_changes = AtomicBool::new(false);++ exact_changed_nodes.iter().for_each(|changed_node| {+ for edge in graph.raw_outgoing_edges(changed_node) {+ if let (Some(counter_to), Some(counter_from)) =+ (counters.new.get_mut(&edge.to), counters.old.get(&edge.from))+ {+ if counter_to+ .registers()+ .iter()+ .zip(counter_from.registers().iter())+ .any(|(to, from)| *from > *to)+ {+ counter_to.merge(counter_from);+ new_changed_nodes.insert(edge.to.as_u64());++ new_exact_changed_nodes.insert(edge.to);++ has_changes.store(true, Ordering::Relaxed);+ }+ }+ }+ });++ *exact_changed_nodes = new_exact_changed_nodes;++ has_changes.load(Ordering::Relaxed)+}++fn update_all_counters(+ graph: &Webgraph,+ mut exact_changed_nodes: Option<&mut BTreeSet<NodeID>>,+ counters: &mut Counters,+ changed_nodes: &BloomFilter,+ new_changed_nodes: &mut BloomFilter,+) -> bool {+ let has_changes = AtomicBool::new(false);++ if let Some(exact_changed_nodes) = &mut exact_changed_nodes {+ **exact_changed_nodes = BTreeSet::default();+ }++ graph.edges().for_each(|edge| {+ if changed_nodes.contains(edge.from.as_u64()) {+ if let (Some(counter_to), Some(counter_from)) =+ (counters.new.get_mut(&edge.to), counters.old.get(&edge.from))+ {+ if counter_to+ .registers()+ .iter()+ .zip(counter_from.registers().iter())+ .any(|(to, from)| *from > *to)+ {+ counter_to.merge(counter_from);+ new_changed_nodes.insert(edge.to.as_u64());++ if let Some(exact_changed_nodes) = &mut exact_changed_nodes {+ exact_changed_nodes.insert(edge.to);+ }++ has_changes.store(true, Ordering::Relaxed);+ }+ }+ }+ });++ has_changes.load(Ordering::Relaxed)+}++fn update_centralities(centralities: &mut BTreeMap<NodeID, KahanSum>, counters: &Counters, t: u64) {+ centralities.iter_mut().for_each(|(node, score)| {+ *score += counters+ .new+ .get(node)+ .map(|counter| counter.size())+ .unwrap_or_default()+ .checked_sub(+ counters+ .old+ .get(node)+ .map(|counter| counter.size())+ .unwrap_or_default(),+ )+ .unwrap_or_default() as f64+ / (t + 1) as f64;+ });+}++fn normalize_centralities(+ centralities: BTreeMap<NodeID, KahanSum>,+ norm_factor: f64,+) -> BTreeMap<NodeID, f64> {+ centralities+ .into_iter()+ .map(|(node_id, sum)| (node_id, f64::from(sum)))+ .filter(|(_, centrality)| *centrality > 0.0)+ .map(|(node_id, centrality)| (node_id, centrality / norm_factor))+ .map(|(node_id, centrality)| {+ if !centrality.is_finite() {+ (node_id, 0.0)+ } else {+ (node_id, centrality)+ }+ })+ .collect()+}++struct Counters {+ old: Counter,+ new: Counter,+}++impl Counters {+ fn new() -> Self {+ Self {+ old: BTreeMap::default(),+ new: BTreeMap::default(),+ }+ }++ fn step(&mut self) {+ self.old = self.new.clone();+ }+}++fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {+ let mut counters = Counters::new();+ let mut centralities: BTreeMap<NodeID, KahanSum> = BTreeMap::new();++ let num_nodes = initialize(graph, &mut counters, &mut centralities);++ let mut changed_nodes = BloomFilter::new(num_nodes, 0.05);for node in graph.nodes() {changed_nodes.insert(node.as_u64());@@ -60,92 +205,48 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {let norm_factor = (num_nodes - 1) as f64;let mut exact_counting = false;- let has_changes = AtomicBool::new(true);+ let mut has_changes = true;let mut t = 0;let mut exact_changed_nodes: BTreeSet<NodeID> = BTreeSet::default();loop {- if !has_changes.load(Ordering::Relaxed) {+ if !has_changes {break;}- let mut new_counters = counters.clone();-- has_changes.store(false, Ordering::Relaxed);- let mut new_changed_nodes = BloomFilter::new(num_nodes as u64, 0.05);+ let mut new_changed_nodes = BloomFilter::new(num_nodes, 0.05);if !exact_changed_nodes.is_empty()&& exact_changed_nodes.len() as u64 <= exact_counting_threshold{- let mut new_exact_changed_nodes = BTreeSet::default();-- exact_changed_nodes.iter().for_each(|changed_node| {- for edge in graph.raw_outgoing_edges(changed_node) {- if let (Some(counter_to), Some(counter_from)) =- (new_counters.get_mut(&edge.to), counters.get(&edge.from))- {- if counter_to- .registers()- .iter()- .zip(counter_from.registers().iter())- .any(|(to, from)| *from > *to)- {- counter_to.merge(counter_from);- new_changed_nodes.insert(edge.to.as_u64());-- new_exact_changed_nodes.insert(edge.to);-- has_changes.store(true, Ordering::Relaxed);- }- }- }- });-- exact_changed_nodes = new_exact_changed_nodes;+ has_changes = update_changed_counters(+ graph,+ &mut exact_changed_nodes,+ &mut counters,+ &mut new_changed_nodes,+ );+ } else if exact_counting {+ has_changes = update_all_counters(+ graph,+ Some(&mut exact_changed_nodes),+ &mut counters,+ &changed_nodes,+ &mut new_changed_nodes,+ );} else {- exact_changed_nodes = BTreeSet::default();- graph.edges().for_each(|edge| {- if changed_nodes.contains(edge.from.as_u64()) {- if let (Some(counter_to), Some(counter_from)) =- (new_counters.get_mut(&edge.to), counters.get(&edge.from))- {- if counter_to- .registers()- .iter()- .zip(counter_from.registers().iter())- .any(|(to, from)| *from > *to)- {- counter_to.merge(counter_from);- new_changed_nodes.insert(edge.to.as_u64());-- if exact_counting {- exact_changed_nodes.insert(edge.to);- }-- has_changes.store(true, Ordering::Relaxed);- }- }- }- })+ has_changes = update_all_counters(+ graph,+ None,+ &mut counters,+ &changed_nodes,+ &mut new_changed_nodes,+ );}- centralities.iter_mut().for_each(|(node, score)| {- *score += new_counters- .get(node)- .map(|counter| counter.size())- .unwrap_or_default()- .checked_sub(- counters- .get(node)- .map(|counter| counter.size())- .unwrap_or_default(),- )- .unwrap_or_default() as f64- / (t + 1) as f64;- });-- counters = new_counters;+ update_centralities(&mut centralities, &counters, t);++ counters.step();changed_nodes = new_changed_nodes;t += 1;@@ -154,19 +255,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {}}- let res = centralities- .into_iter()- .map(|(node_id, sum)| (node_id, f64::from(sum)))- .filter(|(_, centrality)| *centrality > 0.0)- .map(|(node_id, centrality)| (node_id, centrality / norm_factor))- .map(|(node_id, centrality)| {- if !centrality.is_finite() {- (node_id, 0.0)- } else {- (node_id, centrality)- }- })- .collect();+ let res = normalize_centralities(centralities, norm_factor);info!("Harmonic centrality calculated");diff --git a/crates/core/src/webpage/just_text.rs b/crates/core/src/webpage/just_text.rsindex e774f72f..6192f936 100644--- a/crates/core/src/webpage/just_text.rs+++ b/crates/core/src/webpage/just_text.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use hashbrown::HashSet;-use kuchiki::{iter::NodeEdge, NodeRef};+use kuchiki::{iter::NodeEdge, ElementData, NodeRef};use whatlang::Lang;use crate::stopwords;@@ -190,6 +190,51 @@ struct ClassifiedParagraph {classification: Classification,}+fn is_heading(elem: &ElementData) -> bool {+ matches!(+ elem.name.local.as_ref(),+ "h1" | "h2" | "h3" | "h4" | "h5" | "h6"+ )+}++fn is_paragraph_breaking(elem: &ElementData) -> bool {+ matches!(+ elem.name.local.as_ref(),+ "body"+ | "blockquote"+ | "caption"+ | "center"+ | "col"+ | "colgroup"+ | "dd"+ | "div"+ | "dl"+ | "dt"+ | "fieldset"+ | "form"+ | "legend"+ | "optgroup"+ | "option"+ | "p"+ | "pre"+ | "table"+ | "td"+ | "textarea"+ | "tfoot"+ | "th"+ | "thead"+ | "tr"+ | "ul"+ | "li"+ | "h1"+ | "h2"+ | "h3"+ | "h4"+ | "h5"+ | "h6"+ )+}+impl JustText {pub fn paragraphs(root: NodeRef) -> Vec<Paragraph> {let mut res = Vec::new();@@ -212,49 +257,14 @@ impl JustText {match edge {NodeEdge::Start(node) => {if let Some(element) = node.as_element() {- let name: &str = &element.name.local;-- if matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {+ if is_heading(element) {heading_count += 1;}- if matches!(- name,- "body"- | "blockquote"- | "caption"- | "center"- | "col"- | "colgroup"- | "dd"- | "div"- | "dl"- | "dt"- | "fieldset"- | "form"- | "legend"- | "optgroup"- | "option"- | "p"- | "pre"- | "table"- | "td"- | "textarea"- | "tfoot"- | "th"- | "thead"- | "tr"- | "ul"- | "li"- | "h1"- | "h2"- | "h3"- | "h4"- | "h5"- | "h6"- ) || (name == "br" && br)+ if is_paragraph_breaking(element)+ || (element.name.local.as_ref() == "br" && br){- if name == "br" {+ if element.name.local.as_ref() == "br" {// the <br><br> is a paragraph separator and should// not be included in the number of tags within the// paragraph@@ -271,10 +281,10 @@ impl JustText {paragraph = Paragraph::new();} else {- br = name == "br";+ br = element.name.local.as_ref() == "br";if br {paragraph.append_text(" ");- } else if name == "a" {+ } else if element.name.local.as_ref() == "a" {link = true;}paragraph.tags_count += 1;@@ -295,29 +305,21 @@ impl JustText {}NodeEdge::End(node) => {if let Some(element) = node.as_element() {- let name: &str = &element.name.local;- if matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {+ if is_heading(element) {heading_count -= 1;}paragraph.append_text(" ");- match name {- "body" | "blockquote" | "caption" | "center" | "col" | "colgroup"- | "dd" | "div" | "dl" | "dt" | "fieldset" | "form" | "legend"- | "optgroup" | "option" | "p" | "pre" | "table" | "td" | "textarea"- | "tfoot" | "th" | "thead" | "tr" | "ul" | "li" | "h1" | "h2"- | "h3" | "h4" | "h5" | "h6" => {- if paragraph.contains_text() {- res.push(paragraph);- }-- paragraph = Paragraph::new();+ if is_paragraph_breaking(element) {+ if paragraph.contains_text() {+ res.push(paragraph);}- _ => {}++ paragraph = Paragraph::new();}- if name == "a" {+ if element.name.local.as_ref() == "a" {link = false;}}@@ -388,8 +390,48 @@ impl JustText {res}- fn contextual_classification(&self, paragraphs: &mut [ClassifiedParagraph]) {- // good headings+ fn new_class(+ prev_neighbour: &Classification,+ next_neighbour: &Classification,+ paragraphs: &[ClassifiedParagraph],+ i: usize,+ ) -> Classification {+ if matches!(+ prev_neighbour,+ Classification::Intermediate(IntermediateClassification::Good)+ ) && matches!(+ next_neighbour,+ Classification::Intermediate(IntermediateClassification::Good)+ ) {+ Classification::Intermediate(IntermediateClassification::Good)+ } else if matches!(+ prev_neighbour,+ Classification::Intermediate(IntermediateClassification::Bad)+ ) && matches!(+ next_neighbour,+ Classification::Intermediate(IntermediateClassification::Bad)+ ) {+ Classification::Intermediate(IntermediateClassification::Bad)+ } else if (matches!(+ prev_neighbour,+ Classification::Intermediate(IntermediateClassification::Bad)+ ) && matches!(+ Self::get_prev_neighbour(paragraphs, i, false),+ Classification::Intermediate(IntermediateClassification::NearGood)+ )) || (matches!(+ next_neighbour,+ Classification::Intermediate(IntermediateClassification::Bad)+ ) && matches!(+ Self::get_next_neighbour(paragraphs, i, false),+ Classification::Intermediate(IntermediateClassification::NearGood)+ )) {+ Classification::Intermediate(IntermediateClassification::Good)+ } else {+ Classification::Intermediate(IntermediateClassification::Bad)+ }+ }++ fn update_good_headings(&self, paragraphs: &mut [ClassifiedParagraph]) {let num_paragraphs = paragraphs.len();for i in 0..num_paragraphs {let paragraph = ¶graphs[i];@@ -412,14 +454,15 @@ impl JustText {j += 1;}}+ }- // classify short+ fn classify_short(paragraphs: &mut [ClassifiedParagraph]) {let mut new_classes: Vec<_> = paragraphs.iter().map(|par| par.classification.clone()).collect();- for i in 0..num_paragraphs {+ for i in 0..paragraphs.len() {if !matches!(paragraphs[i].classification,Classification::Intermediate(IntermediateClassification::Short)@@ -429,48 +472,16 @@ impl JustText {let prev_neighbour = Self::get_prev_neighbour(paragraphs, i, true);let next_neighbour = Self::get_next_neighbour(paragraphs, i, true);-- if matches!(- prev_neighbour,- Classification::Intermediate(IntermediateClassification::Good)- ) && matches!(- next_neighbour,- Classification::Intermediate(IntermediateClassification::Good)- ) {- new_classes[i] = Classification::Intermediate(IntermediateClassification::Good);- } else if matches!(- prev_neighbour,- Classification::Intermediate(IntermediateClassification::Bad)- ) && matches!(- next_neighbour,- Classification::Intermediate(IntermediateClassification::Bad)- ) {- new_classes[i] = Classification::Intermediate(IntermediateClassification::Bad);- } else if (matches!(- prev_neighbour,- Classification::Intermediate(IntermediateClassification::Bad)- ) && matches!(- Self::get_prev_neighbour(paragraphs, i, false),- Classification::Intermediate(IntermediateClassification::NearGood)- )) || (matches!(- next_neighbour,- Classification::Intermediate(IntermediateClassification::Bad)- ) && matches!(- Self::get_next_neighbour(paragraphs, i, false),- Classification::Intermediate(IntermediateClassification::NearGood)- )) {- new_classes[i] = Classification::Intermediate(IntermediateClassification::Good);- } else {- new_classes[i] = Classification::Intermediate(IntermediateClassification::Bad);- }+ new_classes[i] = Self::new_class(&prev_neighbour, &next_neighbour, paragraphs, i);}for (paragraph, classification) in paragraphs.iter_mut().zip(new_classes) {paragraph.classification = classification;}+ }- // revise neargood- for i in 0..num_paragraphs {+ fn revise_neargood(paragraphs: &mut [ClassifiedParagraph]) {+ for i in 0..paragraphs.len() {if !matches!(paragraphs[i].classification,Classification::Intermediate(IntermediateClassification::NearGood)@@ -513,6 +524,12 @@ impl JustText {}}+ fn contextual_classification(&self, paragraphs: &mut [ClassifiedParagraph]) {+ self.update_good_headings(paragraphs);+ Self::classify_short(paragraphs);+ Self::revise_neargood(paragraphs);+ }+fn get_prev_neighbour(paragraphs: &[ClassifiedParagraph],idx: usize,diff --git a/crates/core/src/webpage/schema_org/microdata.rs b/crates/core/src/webpage/schema_org/microdata.rsindex 15cb25fa..aa8383a2 100644--- a/crates/core/src/webpage/schema_org/microdata.rs+++ b/crates/core/src/webpage/schema_org/microdata.rs@@ -16,7 +16,7 @@//! Almost spec compliant microdata parser: https://html.spec.whatwg.org/multipage/microdata.htm-use kuchiki::NodeRef;+use kuchiki::{ElementData, NodeRef};use std::collections::{HashMap, VecDeque};use thiserror::Error;@@ -151,8 +151,63 @@ fn text_contents(node: NodeRef) -> Vec<RawProperty> {res}+fn properties_for_prop(elem: &ElementData, current: NodeRef) -> Result<Vec<RawProperty>> {+ if elem.attributes.borrow().contains("itemscope") {+ Ok(vec![RawProperty::Item(parse_item(current)?)])+ } else {+ match elem.name.local.to_string().as_str() {+ "meta" => Ok(vec![RawProperty::String(+ elem.attributes+ .borrow()+ .get("content")+ .map(String::from)+ .unwrap_or_default(),+ )]),+ "audio" | "embed" | "iframe" | "img" | "source" | "track" | "video" => {+ if let Some(url) = elem.attributes.borrow().get("src") {+ Ok(vec![RawProperty::String(url.to_string())])+ } else {+ Ok(vec![RawProperty::String(String::new())])+ }+ }+ "a" | "area" | "link" => {+ if let Some(url) = elem.attributes.borrow().get("href") {+ Ok(vec![RawProperty::String(url.to_string())])+ } else {+ Ok(vec![RawProperty::String(String::new())])+ }+ }+ "object" => Ok(vec![RawProperty::String(+ elem.attributes+ .borrow()+ .get("data")+ .map(String::from)+ .unwrap_or_default(),+ )]),+ "data" | "meter" => Ok(vec![RawProperty::String(+ elem.attributes+ .borrow()+ .get("value")+ .map(String::from)+ .unwrap_or_default(),+ )]),+ "time" => {+ let time = elem+ .attributes+ .borrow()+ .get("datetime")+ .map(String::from)+ .unwrap_or_else(|| current.text_contents());++ Ok(vec![RawProperty::String(time)])+ }+ _ => Ok(text_contents(current.clone())),+ }+ }+}+/// implementation of https://html.spec.whatwg.org/multipage/microdata.html#associating-names-with-items-/// TODO: handle itemrefs+/// TODO: handle itemrefs (https://html.spec.whatwg.org/multipage/microdata.html#attr-itemref)fn parse_item(root: NodeRef) -> Result<RawItem> {if !root.as_element()@@ -190,58 +245,7 @@ fn parse_item(root: NodeRef) -> Result<RawItem> {}if let Some(itemprop) = elem.attributes.borrow().get("itemprop") {- let properties_for_prop = if elem.attributes.borrow().contains("itemscope") {- vec![RawProperty::Item(parse_item(current)?)]- } else {- match elem.name.local.to_string().as_str() {- "meta" => vec![RawProperty::String(- elem.attributes- .borrow()- .get("content")- .map(String::from)- .unwrap_or_default(),- )],- "audio" | "embed" | "iframe" | "img" | "source" | "track" | "video" => {- if let Some(url) = elem.attributes.borrow().get("src") {- vec![RawProperty::String(url.to_string())]- } else {- vec![RawProperty::String(String::new())]- }- }- "a" | "area" | "link" => {- if let Some(url) = elem.attributes.borrow().get("href") {- vec![RawProperty::String(url.to_string())]- } else {- vec![RawProperty::String(String::new())]- }- }- "object" => vec![RawProperty::String(- elem.attributes- .borrow()- .get("data")- .map(String::from)- .unwrap_or_default(),- )],- "data" | "meter" => vec![RawProperty::String(- elem.attributes- .borrow()- .get("value")- .map(String::from)- .unwrap_or_default(),- )],- "time" => {- let time = elem- .attributes- .borrow()- .get("datetime")- .map(String::from)- .unwrap_or_else(|| current.text_contents());-- vec![RawProperty::String(time)]- }- _ => text_contents(current.clone()),- }- };+ let properties_for_prop = properties_for_prop(elem, current)?;for itemprop in itemprop.split_ascii_whitespace() {propertiesdiff --git a/crates/core/src/widgets/thesaurus.rs b/crates/core/src/widgets/thesaurus.rsindex adeed6a2..88737db4 100644--- a/crates/core/src/widgets/thesaurus.rs+++ b/crates/core/src/widgets/thesaurus.rs@@ -17,7 +17,11 @@use itertools::Itertools;use regex::Regex;use serde::{Deserialize, Serialize};-use std::{fs::File, io::BufReader, path::Path};+use std::{+ fs::File,+ io::{BufRead, BufReader},+ path::Path,+};use utoipa::ToSchema;use anyhow::{anyhow, Result};@@ -25,6 +29,10 @@ use rio_api::parser::TriplesParser;use rio_turtle::TurtleParser;use std::collections::HashMap;+static VALUE_STR_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(|| {+ regex::Regex::new(r#""(.*)"@"#).expect("Failed to compile regex")+});+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)]#[serde(rename_all = "camelCase", transparent)]pub struct Lemma(String);@@ -70,6 +78,103 @@ pub struct Info {pos: PartOfSpeech,}+impl Info {+ fn build_definition(concept: &NodeQuery<'_>) -> Definition {+ let def = concept+ .single_edge(|e| e.label.contains("#definition"))+ .unwrap()+ .to+ .single_edge(|e| e.label.contains("#value"))+ .unwrap()+ .to+ .node+ .0+ .clone();++ let def = VALUE_STR_REGEX+ .captures(&def)+ .unwrap()+ .get(1)+ .unwrap()+ .as_str()+ .to_string();++ Definition(def)+ }++ fn build_examples(concept: &NodeQuery<'_>) -> Vec<Example> {+ concept+ .filtered_edges(|e| e.label.contains("#example"))+ .into_iter()+ .map(|e| {+ e.to.single_edge(|e| e.label.contains("#value"))+ .unwrap()+ .to+ .node+ .0+ .clone()+ })+ .map(|e| {+ VALUE_STR_REGEX+ .captures(&e)+ .unwrap()+ .get(1)+ .unwrap()+ .as_str()+ .to_string()+ })+ .map(Example)+ .collect()+ }++ fn build_similar(concept: &NodeQuery<'_>) -> Vec<Id> {+ concept+ .filtered_edges(|e| e.label.contains("#similar"))+ .into_iter()+ .map(|e| Id(e.to.node.0.clone()))+ .collect()+ }++ fn build_hyponyms(concept: &NodeQuery<'_>) -> Vec<Id> {+ concept+ .filtered_edges(|e| e.label.contains("#hyponym"))+ .into_iter()+ .map(|e| Id(e.to.node.0.clone()))+ .collect()+ }++ fn build_hypernyms(concept: &NodeQuery<'_>) -> Vec<Id> {+ concept+ .filtered_edges(|e| e.label.contains("#hypernym"))+ .into_iter()+ .map(|e| Id(e.to.node.0.clone()))+ .collect()+ }++ fn build_pos(concept: &NodeQuery<'_>) -> Result<PartOfSpeech> {+ let pos = concept+ .single_edge(|e| e.label.contains("#partOfSpeech"))+ .unwrap()+ .to+ .node+ .0+ .split_once('#')+ .unwrap()+ .1+ .to_string()+ .replace('>', "");++ match pos.as_str() {+ "noun" => Ok(PartOfSpeech::Noun),+ "adjective_satellite" => Ok(PartOfSpeech::AdjectiveSatellite),+ "verb" => Ok(PartOfSpeech::Verb),+ "adverb" => Ok(PartOfSpeech::Adverb),+ "adjective" => Ok(PartOfSpeech::Adjective),+ _ => Err(anyhow!("Unknown part of speech: {}", pos)),+ }+ }+}+pub struct Dictionary {map: HashMap<Id, Info>,lemmas: HashMap<NormalizedLemma, Vec<Id>>,@@ -293,9 +398,38 @@ impl Dictionary {self.spellings.insert(normalized, lemma);}- pub fn build<P: AsRef<Path>>(path: P) -> Result<Self> {- let reader = BufReader::new(File::open(path)?);+ fn entry_info(entry: NodeQuery<'_>) -> Result<Vec<Info>> {+ let mut res = Vec::new();+ for sense in entry.filtered_edges(|e| e.label.contains("#sense")) {+ let concept = sense+ .to+ .single_edge(|e| e.label.contains("#isLexicalizedSenseOf"))+ .unwrap()+ .to;+ let id = Id(concept.node.0.clone());++ let definition = Info::build_definition(&concept);+ let examples = Info::build_examples(&concept);+ let similar = Info::build_similar(&concept);+ let hyponyms = Info::build_hyponyms(&concept);+ let hypernyms = Info::build_hypernyms(&concept);+ let pos = Info::build_pos(&concept)?;++ res.push(Info {+ id,+ definition,+ examples,+ similar,+ hyponyms,+ hypernyms,+ pos,+ });+ }++ Ok(res)+ }+ fn build_graph<R: BufRead>(reader: R) -> Result<Graph> {let mut parser = TurtleParser::new(reader, None);let mut graph = Graph::new();@@ -311,6 +445,10 @@ impl Dictionary {Ok::<_, anyhow::Error>(())})?;+ Ok(graph)+ }++ fn build_lexical_entries(graph: &Graph) -> Vec<&'_ Node> {let mut lexical_entries = Vec::new();for (node, edges) in &graph.map {@@ -321,8 +459,15 @@ impl Dictionary {}}+ lexical_entries+ }++ pub fn build<P: AsRef<Path>>(path: P) -> Result<Self> {+ let reader = BufReader::new(File::open(path)?);+ let graph = Self::build_graph(reader)?;+ let lexical_entries = Self::build_lexical_entries(&graph);+let mut dict = Self::empty()?;- let value_str_regex = regex::Regex::new(r#""(.*)"@"#).unwrap();for entry in lexical_entries {let entry = NodeQuery::new(entry, &graph);@@ -338,7 +483,7 @@ impl Dictionary {.0.clone();- let written_rep = value_str_regex+ let written_rep = VALUE_STR_REGEX.captures(&written_rep).unwrap().get(1)@@ -348,106 +493,7 @@ impl Dictionary {let lemma = Lemma(written_rep);- for sense in entry.filtered_edges(|e| e.label.contains("#sense")) {- let concept = sense- .to- .single_edge(|e| e.label.contains("#isLexicalizedSenseOf"))- .unwrap()- .to;- let id = Id(concept.node.0.clone());-- let definition = concept- .single_edge(|e| e.label.contains("#definition"))- .unwrap()- .to- .single_edge(|e| e.label.contains("#value"))- .unwrap()- .to- .node- .0- .clone();-- let definition = value_str_regex- .captures(&definition)- .unwrap()- .get(1)- .unwrap()- .as_str()- .to_string();- let definition = Definition(definition);-- let examples: Vec<_> = concept- .filtered_edges(|e| e.label.contains("#example"))- .into_iter()- .map(|e| {- e.to.single_edge(|e| e.label.contains("#value"))- .unwrap()- .to- .node- .0- .clone()- })- .map(|e| {- value_str_regex- .captures(&e)- .unwrap()- .get(1)- .unwrap()- .as_str()- .to_string()- })- .map(Example)- .collect();-- let similar: Vec<_> = concept- .filtered_edges(|e| e.label.contains("#similar"))- .into_iter()- .map(|e| Id(e.to.node.0.clone()))- .collect();-- let hyponyms: Vec<_> = concept- .filtered_edges(|e| e.label.contains("#hyponym"))- .into_iter()- .map(|e| Id(e.to.node.0.clone()))- .collect();-- let hypernyms: Vec<_> = concept- .filtered_edges(|e| e.label.contains("#hypernym"))- .into_iter()- .map(|e| Id(e.to.node.0.clone()))- .collect();-- let pos = concept- .single_edge(|e| e.label.contains("#partOfSpeech"))- .unwrap()- .to- .node- .0- .split_once('#')- .unwrap()- .1- .to_string()- .replace('>', "");-- let pos = match pos.as_str() {- "noun" => PartOfSpeech::Noun,- "adjective_satellite" => PartOfSpeech::AdjectiveSatellite,- "verb" => PartOfSpeech::Verb,- "adverb" => PartOfSpeech::Adverb,- "adjective" => PartOfSpeech::Adjective,- _ => return Err(anyhow!("Unknown part of speech: {}", pos)),- };-- let info = Info {- id,- definition,- examples,- similar,- hyponyms,- hypernyms,- pos,- };-+ for info in Self::entry_info(entry)? {dict.insert(lemma.clone(), info);}}