Skip to content

web search done right

refactor long functions

Mikkel Denkerbba251d7e98a
diff --git a/clippy.toml b/clippy.toml
new file mode 100644
index 00000000..9ca075c1
--- /dev/null
+++ b/clippy.toml
@@ -0,0 +1,2 @@
+too-many-lines-threshold = 100
+
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rs
index d96012a0..de39fe06 100644
--- a/crates/core/src/crawler/worker.rs
+++ b/crates/core/src/crawler/worker.rs
@@ -15,8 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::anyhow;
use encoding_rs::{Encoding, UTF_8};
-use futures::{future::BoxFuture, FutureExt};
-use hashbrown::{HashMap, HashSet};
+use hashbrown::HashSet;
use mime::Mime;
use quick_xml::events::Event;
use rand::seq::SliceRandom;
@@ -48,6 +47,12 @@ use super::{
const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB
+const IGNORED_EXTENSIONS: [&str; 27] = [
+ ".pdf", ".jpg", ".zip", ".png", ".css", ".js", ".json", ".jsonp", ".woff2", ".woff", ".ttf",
+ ".svg", ".gif", ".jpeg", ".ico", ".mp4", ".mp3", ".avi", ".mov", ".mpeg", ".webm", ".wav",
+ ".flac", ".aac", ".ogg", ".m4a", ".m4v",
+];
+
struct ProcessedUrl {
new_urls: Vec<Url>,
response: UrlResponse,
@@ -238,7 +243,7 @@ impl<S: DatumStream> JobExecutor<S> {
if let Some(sitemap) = self.robotstxt.sitemap(retryable_url.url()).await {
self.sitemap_urls
- .extend(self.urls_from_sitemap(sitemap, 0, 5).await);
+ .extend(self.urls_from_sitemap(sitemap, 5).await);
}
}
@@ -276,92 +281,70 @@ impl<S: DatumStream> JobExecutor<S> {
}
}
+ fn increase_politeness(&mut self) {
+ self.politeness_factor *= 2.0;
+
+ if self.politeness_factor > self.config.max_politeness_factor {
+ self.politeness_factor = self.config.max_politeness_factor;
+ }
+
+ tracing::warn!("politeness factor increased to {}", self.politeness_factor);
+ }
+
+ fn new_urls(html: &Html) -> Vec<Url> {
+ html.all_links()
+ .into_iter()
+ .map(|link| link.destination)
+ .filter(|url| url.as_str().len() <= MAX_URL_LEN_BYTES)
+ .filter(|url| {
+ IGNORED_EXTENSIONS
+ .iter()
+ .all(|ext| !url.as_str().ends_with(ext))
+ })
+ .collect()
+ }
+
async fn process_url(&mut self, url: Url) -> ProcessedUrl {
let fetch = self.crawl_url(url.clone()).await;
match fetch {
- Ok(datum) => {
- if matches!(datum.status_code, 200 | 301 | 302) {
- if datum.status_code == 200 {
- self.save_datum(datum.clone()).await;
-
- match Html::parse(&datum.body, datum.url.as_str()) {
- Ok(html) => {
- let new_urls = html
- .all_links()
- .into_iter()
- .map(|link| link.destination)
- .filter(|url| url.as_str().len() <= MAX_URL_LEN_BYTES)
- .filter(|url| {
- !url.path().ends_with(".pdf")
- && !url.path().ends_with(".jpg")
- && !url.path().ends_with(".zip")
- && !url.path().ends_with(".png")
- && !url.path().ends_with(".css")
- && !url.path().ends_with(".js")
- && !url.path().ends_with(".json")
- && !url.path().ends_with(".jsonp")
- && !url.path().ends_with(".woff2")
- && !url.path().ends_with(".woff")
- && !url.path().ends_with(".ttf")
- && !url.path().ends_with(".svg")
- && !url.path().ends_with(".gif")
- && !url.path().ends_with(".jpeg")
- && !url.path().ends_with(".ico")
- && !url.path().ends_with(".mp4")
- && !url.path().ends_with(".mp3")
- && !url.path().ends_with(".avi")
- && !url.path().ends_with(".mov")
- && !url.path().ends_with(".mpeg")
- && !url.path().ends_with(".webm")
- && !url.path().ends_with(".wav")
- && !url.path().ends_with(".flac")
- && !url.path().ends_with(".aac")
- && !url.path().ends_with(".ogg")
- && !url.path().ends_with(".m4a")
- && !url.path().ends_with(".m4v")
- })
- .collect();
-
- let url_res = UrlResponse::Success { url: datum.url };
-
- ProcessedUrl {
- new_urls,
- response: url_res,
- }
+ Ok(datum) => match datum.status_code {
+ 200 => {
+ self.save_datum(datum.clone()).await;
+
+ match Html::parse(&datum.body, datum.url.as_str()) {
+ Ok(html) => {
+ let new_urls = Self::new_urls(&html);
+ let url_res = UrlResponse::Success { url: datum.url };
+ ProcessedUrl {
+ new_urls,
+ response: url_res,
}
- Err(_) => ProcessedUrl {
- new_urls: Vec::new(),
- response: UrlResponse::Failed {
- url,
- status_code: None,
- },
- },
}
- } else {
- let url_res = UrlResponse::Redirected {
- url,
- new_url: datum.url,
- };
-
- ProcessedUrl {
+ Err(_) => ProcessedUrl {
new_urls: Vec::new(),
- response: url_res,
- }
+ response: UrlResponse::Failed {
+ url,
+ status_code: None,
+ },
+ },
}
- } else {
- if datum.status_code == 429 {
- self.politeness_factor *= 2.0;
+ }
+ 301 | 302 => {
+ let url_res = UrlResponse::Redirected {
+ url,
+ new_url: datum.url,
+ };
- if self.politeness_factor > self.config.max_politeness_factor {
- self.politeness_factor = self.config.max_politeness_factor;
- }
+ ProcessedUrl {
+ new_urls: Vec::new(),
+ response: url_res,
+ }
+ }
- tracing::warn!(
- "politeness factor increased to {} for {}",
- self.politeness_factor,
- &url
- );
+ _ => {
+ if datum.status_code == 429 {
+ self.increase_politeness();
}
tracing::debug!("failed to fetch url ({}): {}", &url, datum.status_code);
@@ -373,7 +356,7 @@ impl<S: DatumStream> JobExecutor<S> {
},
}
}
- }
+ },
Err(err) => {
tracing::debug!("failed to fetch url ({}): {}", &url, err);
@@ -409,10 +392,8 @@ impl<S: DatumStream> JobExecutor<S> {
.map_err(|e| e.into())
}
- async fn crawl_url(&self, url: Url) -> Result<CrawlDatum> {
- let start = Instant::now();
-
- let res = if url.scheme() == "http" {
+ async fn fetch_with_https_priority(&self, url: Url) -> Result<reqwest::Response> {
+ if url.scheme() == "http" {
let mut https = url.clone();
https
.set_scheme("https")
@@ -427,10 +408,10 @@ impl<S: DatumStream> JobExecutor<S> {
}
} else {
self.fetch(url.clone()).await
- };
-
- let fetch_time = start.elapsed();
+ }
+ }
+ async fn politeness_delay(&self, fetch_time: Duration) {
let mut delay = fetch_time;
if delay < Duration::from_millis(self.config.min_crawl_delay_ms) {
@@ -444,37 +425,41 @@ impl<S: DatumStream> JobExecutor<S> {
}
tokio::time::sleep(delay).await;
+ }
- let res = res?;
-
- let headers: HashMap<_, _> = res
- .headers()
- .iter()
- .filter_map(|(k, v)| {
- let k = k.to_string();
- if let Ok(v) = v.to_str() {
- return Some((k, v.to_string()));
- }
-
- None
- })
- .collect();
-
+ fn check_headers(&self, res: &reqwest::Response) -> Result<warc::PayloadType> {
// check if content length is too large
- if let Some(content_length) = headers.get("content-length") {
- if content_length.parse::<usize>().unwrap_or(0) > MAX_CONTENT_LENGTH {
+ if let Some(content_length) = res
+ .headers()
+ .get(reqwest::header::CONTENT_LENGTH)
+ .and_then(|value| value.to_str().ok())
+ .and_then(|value| value.parse::<usize>().ok())
+ {
+ if content_length > MAX_CONTENT_LENGTH {
return Err(Error::ContentTooLarge.into());
}
}
// check if content type is html
- let payload_type = match headers.get("content-type") {
- Some(ct) if ct.contains("text/html") => warc::PayloadType::Html,
- Some(ct) if ct.contains("application/rss") => warc::PayloadType::Rss,
- Some(ct) if ct.contains("application/atom") => warc::PayloadType::Atom,
- ct => return Err(Error::InvalidContentType(format!("{ct:?}")).into()),
- };
+ match res
+ .headers()
+ .get(reqwest::header::CONTENT_TYPE)
+ .and_then(|val| val.to_str().ok())
+ {
+ Some(ct) if ct.contains("text/html") => Ok(warc::PayloadType::Html),
+ Some(ct) if ct.contains("application/rss") => Ok(warc::PayloadType::Rss),
+ Some(ct) if ct.contains("application/atom") => Ok(warc::PayloadType::Atom),
+ ct => Err(Error::InvalidContentType(format!("{ct:?}")).into()),
+ }
+ }
+ fn redirect_datum(
+ &self,
+ res: &reqwest::Response,
+ url: &Url,
+ payload_type: warc::PayloadType,
+ fetch_time: Duration,
+ ) -> Result<Option<CrawlDatum>> {
let status_code = res.status().as_u16();
if status_code == 301 || status_code == 302 {
@@ -489,17 +474,19 @@ impl<S: DatumStream> JobExecutor<S> {
.or_else(|_| url.join(location))
.map_err(|_| Error::InvalidRedirect)?;
- return Ok(CrawlDatum {
+ Ok(Some(CrawlDatum {
url,
status_code,
payload_type,
body: String::new(),
fetch_time_ms: fetch_time.as_millis() as u64,
- });
+ }))
+ } else {
+ Ok(None)
}
+ }
- let res_url = res.url().clone();
-
+ async fn encoded_body(&self, res: reqwest::Response) -> Result<String> {
let content_type = res
.headers()
.get(reqwest::header::CONTENT_TYPE)
@@ -529,7 +516,28 @@ impl<S: DatumStream> JobExecutor<S> {
}
let (text, _, _) = encoding.decode(&bytes);
- let body = text.to_string();
+ Ok(text.to_string())
+ }
+
+ async fn crawl_url(&self, url: Url) -> Result<CrawlDatum> {
+ let start = Instant::now();
+ let res = self.fetch_with_https_priority(url.clone()).await;
+ let fetch_time = start.elapsed();
+ self.politeness_delay(fetch_time).await;
+
+ // we want to delay before returning the error
+ let res = res?;
+
+ let payload_type = self.check_headers(&res)?;
+
+ if let Some(datum) = self.redirect_datum(&res, &url, payload_type, fetch_time)? {
+ return Ok(datum);
+ }
+
+ let status_code = res.status().as_u16();
+
+ let res_url = res.url().clone();
+ let body = self.encoded_body(res).await?;
Ok(CrawlDatum {
url: res_url,
@@ -540,58 +548,51 @@ impl<S: DatumStream> JobExecutor<S> {
})
}
- fn urls_from_sitemap(
- &self,
- sitemap: Url,
- depth: usize,
- max_depth: usize,
- ) -> BoxFuture<'_, Vec<Url>> {
- async move {
- if depth == max_depth {
- return vec![];
+ async fn urls_from_sitemap(&self, sitemap: Url, max_depth: usize) -> Vec<Url> {
+ let mut stack = vec![(sitemap, 0)];
+ let mut urls = vec![];
+
+ while let Some((url, depth)) = stack.pop() {
+ if depth >= max_depth {
+ continue;
}
- // fetch url
- let res = self.fetch(sitemap).await;
+ let res = self.fetch(url).await;
+ tokio::time::sleep(Duration::from_millis(self.config.min_crawl_delay_ms)).await;
if res.is_err() {
- return vec![];
+ continue;
}
+
let res = res.unwrap();
if res.status() != reqwest::StatusCode::OK {
- return vec![];
+ continue;
}
let body = res.text().await;
if body.is_err() {
- return vec![];
+ continue;
}
let body = body.unwrap();
- // parse xml
let entries = parse_sitemap(&body);
- let mut urls = vec![];
-
for entry in entries {
match entry {
SitemapEntry::Url(url) => {
urls.push(url);
}
SitemapEntry::Sitemap(url) => {
- tokio::time::sleep(Duration::from_millis(self.config.min_crawl_delay_ms))
- .await;
- urls.append(&mut self.urls_from_sitemap(url, depth + 1, max_depth).await);
+ stack.push((url, depth + 1));
}
}
}
-
- urls
}
- .boxed()
+
+ urls
}
}
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 78feed64..ce0d9f1e 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -17,7 +17,7 @@
//! Main library for Stract.
#![doc(html_logo_url = "https://stract.com/images/biglogo.svg")]
-// #![warn(clippy::too_many_lines)]
+#![warn(clippy::too_many_lines)]
// pedantic stuff
// #![warn(clippy::pedantic)]
// #![allow(clippy::unreadable_literal)]
diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rs
index b019292f..78d0ba11 100644
--- a/crates/core/src/snippet.rs
+++ b/crates/core/src/snippet.rs
@@ -122,30 +122,19 @@ impl SnippetBuilder {
TextSnippet { fragments }
}
-}
-fn snippet_string_builder(
- text: &str,
- terms: &[String],
- lang: whatlang::Lang,
- config: SnippetConfig,
- mut tokenizer: Tokenizer,
-) -> SnippetBuilder {
- let terms: HashSet<String> = terms
- .iter()
- .flat_map(|term| {
- let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, term);
-
- let mut res = Vec::new();
- while let Some(tok) = stream.next() {
- res.push(tok.text.clone());
- }
+ fn trim_to_chars(&mut self, chars: usize) {
+ self.fragment = self.fragment.chars().take(chars).collect()
+ }
- res.into_iter()
- })
- .collect();
+ fn add_passage(&mut self, passage: &PassageCandidate) {
+ self.fragment.push(' ');
+ self.fragment.push_str(&passage.text);
+ }
+}
- let mut passages: Vec<_> = sentence_ranges(text)
+fn passages(text: &str, mut tokenizer: Tokenizer, config: &SnippetConfig) -> Vec<PassageCandidate> {
+ sentence_ranges(text)
.into_iter()
.filter(|offset| offset.end - offset.start > config.min_passage_width)
.map(|offset| {
@@ -167,39 +156,21 @@ fn snippet_string_builder(
doc_terms,
}
})
- .collect();
-
- if passages.is_empty() {
- let mut snippet = SnippetBuilder {
- fragment: text.chars().take(config.desired_num_chars).collect(),
- highlights: Vec::new(),
- };
-
- snippet.highlight(&terms, lang);
-
- return snippet;
- }
-
- let mut total_d_size = 0;
-
- for passage in &passages {
- total_d_size += passage.doc_terms.len();
- }
-
- let avg_d_size = total_d_size / passages.len();
+ .collect()
+}
+fn calculate_idf(terms: &HashSet<String>, passages: &[PassageCandidate]) -> HashMap<String, f64> {
let mut n: HashMap<_, _> = terms.iter().map(|term| (term.to_string(), 0)).collect();
- for term in &terms {
- for passage in &passages {
+ for term in terms.iter() {
+ for passage in passages.iter() {
if passage.doc_terms.contains_key(term) {
*n.entry(term.to_string()).or_insert(0) += 1;
}
}
}
- let idf: HashMap<_, _> = n
- .into_iter()
+ n.into_iter()
.map(|(term, freq)| {
(
term,
@@ -207,10 +178,23 @@ fn snippet_string_builder(
.ln(),
)
})
- .collect();
+ .collect()
+}
+
+fn score_passages_with_bm25(
+ passages: &mut [PassageCandidate],
+ terms: &HashSet<String>,
+ idf: &HashMap<String, f64>,
+) {
+ let mut total_d_size = 0;
+
+ for passage in passages.iter() {
+ total_d_size += passage.doc_terms.len();
+ }
- for passage in &mut passages {
- for term in &terms {
+ let avg_d_size = total_d_size / passages.len();
+ for passage in passages.iter_mut() {
+ for term in terms.iter() {
let f = *passage.doc_terms.get(term).unwrap_or(&0) as f64;
passage.score += idf[term]
* ((f * (K1 + 1.0))
@@ -218,6 +202,45 @@ fn snippet_string_builder(
* (1.0 - B + B * (passage.doc_terms.len() as f64 / avg_d_size as f64))));
}
}
+}
+
+fn snippet_string_builder(
+ text: &str,
+ terms: &[String],
+ lang: whatlang::Lang,
+ config: SnippetConfig,
+ mut tokenizer: Tokenizer,
+) -> SnippetBuilder {
+ let terms: HashSet<String> = terms
+ .iter()
+ .flat_map(|term| {
+ let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, term);
+
+ let mut res = Vec::new();
+ while let Some(tok) = stream.next() {
+ res.push(tok.text.clone());
+ }
+
+ res.into_iter()
+ })
+ .collect();
+
+ let mut passages = passages(text, tokenizer, &config);
+
+ if passages.is_empty() {
+ let mut snippet = SnippetBuilder {
+ fragment: text.chars().take(config.desired_num_chars).collect(),
+ highlights: Vec::new(),
+ };
+
+ snippet.highlight(&terms, lang);
+
+ return snippet;
+ }
+
+ let idf = calculate_idf(&terms, &passages);
+
+ score_passages_with_bm25(&mut passages, &terms, &idf);
let best_idx = passages
.iter()
@@ -233,28 +256,20 @@ fn snippet_string_builder(
if snippet.fragment.len() > config.desired_num_chars + config.delta_num_chars {
// TODO: find 'desired_num_chars' sized window that contains most highlights
// instead of taking the prefix of the passage as a snippet
- snippet.fragment = snippet
- .fragment
- .chars()
- .take(config.desired_num_chars + config.delta_num_chars)
- .collect();
+
+ snippet.trim_to_chars(config.desired_num_chars + config.delta_num_chars);
} else {
let mut next_passage_idx = best_idx + 1;
while snippet.fragment.len() < config.desired_num_chars - config.delta_num_chars
&& next_passage_idx < passages.len()
{
- snippet.fragment += " ";
- snippet.fragment += &passages[next_passage_idx].text;
+ snippet.add_passage(&passages[next_passage_idx]);
next_passage_idx += 1;
}
if snippet.fragment.len() > config.desired_num_chars + config.delta_num_chars {
- snippet.fragment = snippet
- .fragment
- .chars()
- .take(config.desired_num_chars + config.delta_num_chars)
- .collect();
+ snippet.trim_to_chars(config.desired_num_chars + config.delta_num_chars);
}
}
snippet.highlight(&terms, lang);
diff --git a/crates/core/src/warc.rs b/crates/core/src/warc.rs
index 757c656d..d7922da9 100644
--- a/crates/core/src/warc.rs
+++ b/crates/core/src/warc.rs
@@ -234,7 +234,7 @@ impl Request {
}
}
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[cfg_attr(test, derive(Arbitrary))]
pub enum PayloadType {
Html,
diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rs
index f7d7fec2..b6c43559 100644
--- a/crates/core/src/webgraph/centrality/harmonic.rs
+++ b/crates/core/src/webgraph/centrality/harmonic.rs
@@ -32,24 +32,169 @@ use crate::{
const HYPERLOGLOG_COUNTERS: usize = 64;
-fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
- let mut num_nodes = 0;
-
- let mut counters: BTreeMap<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>> = BTreeMap::new();
+type Counter = BTreeMap<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>>;
- let mut centralities: BTreeMap<NodeID, KahanSum> = BTreeMap::new();
+fn initialize(
+ graph: &Webgraph,
+ counters: &mut Counters,
+ centralities: &mut BTreeMap<NodeID, KahanSum>,
+) -> u64 {
+ let mut num_nodes = 0;
for node in graph.nodes() {
let mut counter = HyperLogLog::default();
counter.add(node.as_u64());
- counters.insert(node, counter);
+ counters.old.insert(node, counter);
centralities.insert(node, KahanSum::default());
num_nodes += 1;
}
- let mut changed_nodes = BloomFilter::new(num_nodes as u64, 0.05);
+ counters.new = counters.old.clone();
+
+ num_nodes
+}
+
+fn update_changed_counters(
+ graph: &Webgraph,
+ exact_changed_nodes: &mut BTreeSet<NodeID>,
+ counters: &mut Counters,
+ new_changed_nodes: &mut BloomFilter,
+) -> bool {
+ let mut new_exact_changed_nodes = BTreeSet::default();
+ let has_changes = AtomicBool::new(false);
+
+ exact_changed_nodes.iter().for_each(|changed_node| {
+ for edge in graph.raw_outgoing_edges(changed_node) {
+ if let (Some(counter_to), Some(counter_from)) =
+ (counters.new.get_mut(&edge.to), counters.old.get(&edge.from))
+ {
+ if counter_to
+ .registers()
+ .iter()
+ .zip(counter_from.registers().iter())
+ .any(|(to, from)| *from > *to)
+ {
+ counter_to.merge(counter_from);
+ new_changed_nodes.insert(edge.to.as_u64());
+
+ new_exact_changed_nodes.insert(edge.to);
+
+ has_changes.store(true, Ordering::Relaxed);
+ }
+ }
+ }
+ });
+
+ *exact_changed_nodes = new_exact_changed_nodes;
+
+ has_changes.load(Ordering::Relaxed)
+}
+
+fn update_all_counters(
+ graph: &Webgraph,
+ mut exact_changed_nodes: Option<&mut BTreeSet<NodeID>>,
+ counters: &mut Counters,
+ changed_nodes: &BloomFilter,
+ new_changed_nodes: &mut BloomFilter,
+) -> bool {
+ let has_changes = AtomicBool::new(false);
+
+ if let Some(exact_changed_nodes) = &mut exact_changed_nodes {
+ **exact_changed_nodes = BTreeSet::default();
+ }
+
+ graph.edges().for_each(|edge| {
+ if changed_nodes.contains(edge.from.as_u64()) {
+ if let (Some(counter_to), Some(counter_from)) =
+ (counters.new.get_mut(&edge.to), counters.old.get(&edge.from))
+ {
+ if counter_to
+ .registers()
+ .iter()
+ .zip(counter_from.registers().iter())
+ .any(|(to, from)| *from > *to)
+ {
+ counter_to.merge(counter_from);
+ new_changed_nodes.insert(edge.to.as_u64());
+
+ if let Some(exact_changed_nodes) = &mut exact_changed_nodes {
+ exact_changed_nodes.insert(edge.to);
+ }
+
+ has_changes.store(true, Ordering::Relaxed);
+ }
+ }
+ }
+ });
+
+ has_changes.load(Ordering::Relaxed)
+}
+
+fn update_centralities(centralities: &mut BTreeMap<NodeID, KahanSum>, counters: &Counters, t: u64) {
+ centralities.iter_mut().for_each(|(node, score)| {
+ *score += counters
+ .new
+ .get(node)
+ .map(|counter| counter.size())
+ .unwrap_or_default()
+ .checked_sub(
+ counters
+ .old
+ .get(node)
+ .map(|counter| counter.size())
+ .unwrap_or_default(),
+ )
+ .unwrap_or_default() as f64
+ / (t + 1) as f64;
+ });
+}
+
+fn normalize_centralities(
+ centralities: BTreeMap<NodeID, KahanSum>,
+ norm_factor: f64,
+) -> BTreeMap<NodeID, f64> {
+ centralities
+ .into_iter()
+ .map(|(node_id, sum)| (node_id, f64::from(sum)))
+ .filter(|(_, centrality)| *centrality > 0.0)
+ .map(|(node_id, centrality)| (node_id, centrality / norm_factor))
+ .map(|(node_id, centrality)| {
+ if !centrality.is_finite() {
+ (node_id, 0.0)
+ } else {
+ (node_id, centrality)
+ }
+ })
+ .collect()
+}
+
+struct Counters {
+ old: Counter,
+ new: Counter,
+}
+
+impl Counters {
+ fn new() -> Self {
+ Self {
+ old: BTreeMap::default(),
+ new: BTreeMap::default(),
+ }
+ }
+
+ fn step(&mut self) {
+ self.old = self.new.clone();
+ }
+}
+
+fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
+ let mut counters = Counters::new();
+ let mut centralities: BTreeMap<NodeID, KahanSum> = BTreeMap::new();
+
+ let num_nodes = initialize(graph, &mut counters, &mut centralities);
+
+ let mut changed_nodes = BloomFilter::new(num_nodes, 0.05);
for node in graph.nodes() {
changed_nodes.insert(node.as_u64());
@@ -60,92 +205,48 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
let norm_factor = (num_nodes - 1) as f64;
let mut exact_counting = false;
- let has_changes = AtomicBool::new(true);
+ let mut has_changes = true;
let mut t = 0;
let mut exact_changed_nodes: BTreeSet<NodeID> = BTreeSet::default();
loop {
- if !has_changes.load(Ordering::Relaxed) {
+ if !has_changes {
break;
}
- let mut new_counters = counters.clone();
-
- has_changes.store(false, Ordering::Relaxed);
- let mut new_changed_nodes = BloomFilter::new(num_nodes as u64, 0.05);
+ let mut new_changed_nodes = BloomFilter::new(num_nodes, 0.05);
if !exact_changed_nodes.is_empty()
&& exact_changed_nodes.len() as u64 <= exact_counting_threshold
{
- let mut new_exact_changed_nodes = BTreeSet::default();
-
- exact_changed_nodes.iter().for_each(|changed_node| {
- for edge in graph.raw_outgoing_edges(changed_node) {
- if let (Some(counter_to), Some(counter_from)) =
- (new_counters.get_mut(&edge.to), counters.get(&edge.from))
- {
- if counter_to
- .registers()
- .iter()
- .zip(counter_from.registers().iter())
- .any(|(to, from)| *from > *to)
- {
- counter_to.merge(counter_from);
- new_changed_nodes.insert(edge.to.as_u64());
-
- new_exact_changed_nodes.insert(edge.to);
-
- has_changes.store(true, Ordering::Relaxed);
- }
- }
- }
- });
-
- exact_changed_nodes = new_exact_changed_nodes;
+ has_changes = update_changed_counters(
+ graph,
+ &mut exact_changed_nodes,
+ &mut counters,
+ &mut new_changed_nodes,
+ );
+ } else if exact_counting {
+ has_changes = update_all_counters(
+ graph,
+ Some(&mut exact_changed_nodes),
+ &mut counters,
+ &changed_nodes,
+ &mut new_changed_nodes,
+ );
} else {
- exact_changed_nodes = BTreeSet::default();
- graph.edges().for_each(|edge| {
- if changed_nodes.contains(edge.from.as_u64()) {
- if let (Some(counter_to), Some(counter_from)) =
- (new_counters.get_mut(&edge.to), counters.get(&edge.from))
- {
- if counter_to
- .registers()
- .iter()
- .zip(counter_from.registers().iter())
- .any(|(to, from)| *from > *to)
- {
- counter_to.merge(counter_from);
- new_changed_nodes.insert(edge.to.as_u64());
-
- if exact_counting {
- exact_changed_nodes.insert(edge.to);
- }
-
- has_changes.store(true, Ordering::Relaxed);
- }
- }
- }
- })
+ has_changes = update_all_counters(
+ graph,
+ None,
+ &mut counters,
+ &changed_nodes,
+ &mut new_changed_nodes,
+ );
}
- centralities.iter_mut().for_each(|(node, score)| {
- *score += new_counters
- .get(node)
- .map(|counter| counter.size())
- .unwrap_or_default()
- .checked_sub(
- counters
- .get(node)
- .map(|counter| counter.size())
- .unwrap_or_default(),
- )
- .unwrap_or_default() as f64
- / (t + 1) as f64;
- });
-
- counters = new_counters;
+ update_centralities(&mut centralities, &counters, t);
+
+ counters.step();
changed_nodes = new_changed_nodes;
t += 1;
@@ -154,19 +255,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
}
}
- let res = centralities
- .into_iter()
- .map(|(node_id, sum)| (node_id, f64::from(sum)))
- .filter(|(_, centrality)| *centrality > 0.0)
- .map(|(node_id, centrality)| (node_id, centrality / norm_factor))
- .map(|(node_id, centrality)| {
- if !centrality.is_finite() {
- (node_id, 0.0)
- } else {
- (node_id, centrality)
- }
- })
- .collect();
+ let res = normalize_centralities(centralities, norm_factor);
info!("Harmonic centrality calculated");
diff --git a/crates/core/src/webpage/just_text.rs b/crates/core/src/webpage/just_text.rs
index e774f72f..6192f936 100644
--- a/crates/core/src/webpage/just_text.rs
+++ b/crates/core/src/webpage/just_text.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use hashbrown::HashSet;
-use kuchiki::{iter::NodeEdge, NodeRef};
+use kuchiki::{iter::NodeEdge, ElementData, NodeRef};
use whatlang::Lang;
use crate::stopwords;
@@ -190,6 +190,51 @@ struct ClassifiedParagraph {
classification: Classification,
}
+fn is_heading(elem: &ElementData) -> bool {
+ matches!(
+ elem.name.local.as_ref(),
+ "h1" | "h2" | "h3" | "h4" | "h5" | "h6"
+ )
+}
+
+fn is_paragraph_breaking(elem: &ElementData) -> bool {
+ matches!(
+ elem.name.local.as_ref(),
+ "body"
+ | "blockquote"
+ | "caption"
+ | "center"
+ | "col"
+ | "colgroup"
+ | "dd"
+ | "div"
+ | "dl"
+ | "dt"
+ | "fieldset"
+ | "form"
+ | "legend"
+ | "optgroup"
+ | "option"
+ | "p"
+ | "pre"
+ | "table"
+ | "td"
+ | "textarea"
+ | "tfoot"
+ | "th"
+ | "thead"
+ | "tr"
+ | "ul"
+ | "li"
+ | "h1"
+ | "h2"
+ | "h3"
+ | "h4"
+ | "h5"
+ | "h6"
+ )
+}
+
impl JustText {
pub fn paragraphs(root: NodeRef) -> Vec<Paragraph> {
let mut res = Vec::new();
@@ -212,49 +257,14 @@ impl JustText {
match edge {
NodeEdge::Start(node) => {
if let Some(element) = node.as_element() {
- let name: &str = &element.name.local;
-
- if matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {
+ if is_heading(element) {
heading_count += 1;
}
- if matches!(
- name,
- "body"
- | "blockquote"
- | "caption"
- | "center"
- | "col"
- | "colgroup"
- | "dd"
- | "div"
- | "dl"
- | "dt"
- | "fieldset"
- | "form"
- | "legend"
- | "optgroup"
- | "option"
- | "p"
- | "pre"
- | "table"
- | "td"
- | "textarea"
- | "tfoot"
- | "th"
- | "thead"
- | "tr"
- | "ul"
- | "li"
- | "h1"
- | "h2"
- | "h3"
- | "h4"
- | "h5"
- | "h6"
- ) || (name == "br" && br)
+ if is_paragraph_breaking(element)
+ || (element.name.local.as_ref() == "br" && br)
{
- if name == "br" {
+ if element.name.local.as_ref() == "br" {
// the <br><br> is a paragraph separator and should
// not be included in the number of tags within the
// paragraph
@@ -271,10 +281,10 @@ impl JustText {
paragraph = Paragraph::new();
} else {
- br = name == "br";
+ br = element.name.local.as_ref() == "br";
if br {
paragraph.append_text(" ");
- } else if name == "a" {
+ } else if element.name.local.as_ref() == "a" {
link = true;
}
paragraph.tags_count += 1;
@@ -295,29 +305,21 @@ impl JustText {
}
NodeEdge::End(node) => {
if let Some(element) = node.as_element() {
- let name: &str = &element.name.local;
- if matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {
+ if is_heading(element) {
heading_count -= 1;
}
paragraph.append_text(" ");
- match name {
- "body" | "blockquote" | "caption" | "center" | "col" | "colgroup"
- | "dd" | "div" | "dl" | "dt" | "fieldset" | "form" | "legend"
- | "optgroup" | "option" | "p" | "pre" | "table" | "td" | "textarea"
- | "tfoot" | "th" | "thead" | "tr" | "ul" | "li" | "h1" | "h2"
- | "h3" | "h4" | "h5" | "h6" => {
- if paragraph.contains_text() {
- res.push(paragraph);
- }
-
- paragraph = Paragraph::new();
+ if is_paragraph_breaking(element) {
+ if paragraph.contains_text() {
+ res.push(paragraph);
}
- _ => {}
+
+ paragraph = Paragraph::new();
}
- if name == "a" {
+ if element.name.local.as_ref() == "a" {
link = false;
}
}
@@ -388,8 +390,48 @@ impl JustText {
res
}
- fn contextual_classification(&self, paragraphs: &mut [ClassifiedParagraph]) {
- // good headings
+ fn new_class(
+ prev_neighbour: &Classification,
+ next_neighbour: &Classification,
+ paragraphs: &[ClassifiedParagraph],
+ i: usize,
+ ) -> Classification {
+ if matches!(
+ prev_neighbour,
+ Classification::Intermediate(IntermediateClassification::Good)
+ ) && matches!(
+ next_neighbour,
+ Classification::Intermediate(IntermediateClassification::Good)
+ ) {
+ Classification::Intermediate(IntermediateClassification::Good)
+ } else if matches!(
+ prev_neighbour,
+ Classification::Intermediate(IntermediateClassification::Bad)
+ ) && matches!(
+ next_neighbour,
+ Classification::Intermediate(IntermediateClassification::Bad)
+ ) {
+ Classification::Intermediate(IntermediateClassification::Bad)
+ } else if (matches!(
+ prev_neighbour,
+ Classification::Intermediate(IntermediateClassification::Bad)
+ ) && matches!(
+ Self::get_prev_neighbour(paragraphs, i, false),
+ Classification::Intermediate(IntermediateClassification::NearGood)
+ )) || (matches!(
+ next_neighbour,
+ Classification::Intermediate(IntermediateClassification::Bad)
+ ) && matches!(
+ Self::get_next_neighbour(paragraphs, i, false),
+ Classification::Intermediate(IntermediateClassification::NearGood)
+ )) {
+ Classification::Intermediate(IntermediateClassification::Good)
+ } else {
+ Classification::Intermediate(IntermediateClassification::Bad)
+ }
+ }
+
+ fn update_good_headings(&self, paragraphs: &mut [ClassifiedParagraph]) {
let num_paragraphs = paragraphs.len();
for i in 0..num_paragraphs {
let paragraph = &paragraphs[i];
@@ -412,14 +454,15 @@ impl JustText {
j += 1;
}
}
+ }
- // classify short
+ fn classify_short(paragraphs: &mut [ClassifiedParagraph]) {
let mut new_classes: Vec<_> = paragraphs
.iter()
.map(|par| par.classification.clone())
.collect();
- for i in 0..num_paragraphs {
+ for i in 0..paragraphs.len() {
if !matches!(
paragraphs[i].classification,
Classification::Intermediate(IntermediateClassification::Short)
@@ -429,48 +472,16 @@ impl JustText {
let prev_neighbour = Self::get_prev_neighbour(paragraphs, i, true);
let next_neighbour = Self::get_next_neighbour(paragraphs, i, true);
-
- if matches!(
- prev_neighbour,
- Classification::Intermediate(IntermediateClassification::Good)
- ) && matches!(
- next_neighbour,
- Classification::Intermediate(IntermediateClassification::Good)
- ) {
- new_classes[i] = Classification::Intermediate(IntermediateClassification::Good);
- } else if matches!(
- prev_neighbour,
- Classification::Intermediate(IntermediateClassification::Bad)
- ) && matches!(
- next_neighbour,
- Classification::Intermediate(IntermediateClassification::Bad)
- ) {
- new_classes[i] = Classification::Intermediate(IntermediateClassification::Bad);
- } else if (matches!(
- prev_neighbour,
- Classification::Intermediate(IntermediateClassification::Bad)
- ) && matches!(
- Self::get_prev_neighbour(paragraphs, i, false),
- Classification::Intermediate(IntermediateClassification::NearGood)
- )) || (matches!(
- next_neighbour,
- Classification::Intermediate(IntermediateClassification::Bad)
- ) && matches!(
- Self::get_next_neighbour(paragraphs, i, false),
- Classification::Intermediate(IntermediateClassification::NearGood)
- )) {
- new_classes[i] = Classification::Intermediate(IntermediateClassification::Good);
- } else {
- new_classes[i] = Classification::Intermediate(IntermediateClassification::Bad);
- }
+ new_classes[i] = Self::new_class(&prev_neighbour, &next_neighbour, paragraphs, i);
}
for (paragraph, classification) in paragraphs.iter_mut().zip(new_classes) {
paragraph.classification = classification;
}
+ }
- // revise neargood
- for i in 0..num_paragraphs {
+ fn revise_neargood(paragraphs: &mut [ClassifiedParagraph]) {
+ for i in 0..paragraphs.len() {
if !matches!(
paragraphs[i].classification,
Classification::Intermediate(IntermediateClassification::NearGood)
@@ -513,6 +524,12 @@ impl JustText {
}
}
+ fn contextual_classification(&self, paragraphs: &mut [ClassifiedParagraph]) {
+ self.update_good_headings(paragraphs);
+ Self::classify_short(paragraphs);
+ Self::revise_neargood(paragraphs);
+ }
+
fn get_prev_neighbour(
paragraphs: &[ClassifiedParagraph],
idx: usize,
diff --git a/crates/core/src/webpage/schema_org/microdata.rs b/crates/core/src/webpage/schema_org/microdata.rs
index 15cb25fa..aa8383a2 100644
--- a/crates/core/src/webpage/schema_org/microdata.rs
+++ b/crates/core/src/webpage/schema_org/microdata.rs
@@ -16,7 +16,7 @@
//! Almost spec compliant microdata parser: https://html.spec.whatwg.org/multipage/microdata.htm
-use kuchiki::NodeRef;
+use kuchiki::{ElementData, NodeRef};
use std::collections::{HashMap, VecDeque};
use thiserror::Error;
@@ -151,8 +151,63 @@ fn text_contents(node: NodeRef) -> Vec<RawProperty> {
res
}
+fn properties_for_prop(elem: &ElementData, current: NodeRef) -> Result<Vec<RawProperty>> {
+ if elem.attributes.borrow().contains("itemscope") {
+ Ok(vec![RawProperty::Item(parse_item(current)?)])
+ } else {
+ match elem.name.local.to_string().as_str() {
+ "meta" => Ok(vec![RawProperty::String(
+ elem.attributes
+ .borrow()
+ .get("content")
+ .map(String::from)
+ .unwrap_or_default(),
+ )]),
+ "audio" | "embed" | "iframe" | "img" | "source" | "track" | "video" => {
+ if let Some(url) = elem.attributes.borrow().get("src") {
+ Ok(vec![RawProperty::String(url.to_string())])
+ } else {
+ Ok(vec![RawProperty::String(String::new())])
+ }
+ }
+ "a" | "area" | "link" => {
+ if let Some(url) = elem.attributes.borrow().get("href") {
+ Ok(vec![RawProperty::String(url.to_string())])
+ } else {
+ Ok(vec![RawProperty::String(String::new())])
+ }
+ }
+ "object" => Ok(vec![RawProperty::String(
+ elem.attributes
+ .borrow()
+ .get("data")
+ .map(String::from)
+ .unwrap_or_default(),
+ )]),
+ "data" | "meter" => Ok(vec![RawProperty::String(
+ elem.attributes
+ .borrow()
+ .get("value")
+ .map(String::from)
+ .unwrap_or_default(),
+ )]),
+ "time" => {
+ let time = elem
+ .attributes
+ .borrow()
+ .get("datetime")
+ .map(String::from)
+ .unwrap_or_else(|| current.text_contents());
+
+ Ok(vec![RawProperty::String(time)])
+ }
+ _ => Ok(text_contents(current.clone())),
+ }
+ }
+}
+
/// implementation of https://html.spec.whatwg.org/multipage/microdata.html#associating-names-with-items
-/// TODO: handle itemrefs
+/// TODO: handle itemrefs (https://html.spec.whatwg.org/multipage/microdata.html#attr-itemref)
fn parse_item(root: NodeRef) -> Result<RawItem> {
if !root
.as_element()
@@ -190,58 +245,7 @@ fn parse_item(root: NodeRef) -> Result<RawItem> {
}
if let Some(itemprop) = elem.attributes.borrow().get("itemprop") {
- let properties_for_prop = if elem.attributes.borrow().contains("itemscope") {
- vec![RawProperty::Item(parse_item(current)?)]
- } else {
- match elem.name.local.to_string().as_str() {
- "meta" => vec![RawProperty::String(
- elem.attributes
- .borrow()
- .get("content")
- .map(String::from)
- .unwrap_or_default(),
- )],
- "audio" | "embed" | "iframe" | "img" | "source" | "track" | "video" => {
- if let Some(url) = elem.attributes.borrow().get("src") {
- vec![RawProperty::String(url.to_string())]
- } else {
- vec![RawProperty::String(String::new())]
- }
- }
- "a" | "area" | "link" => {
- if let Some(url) = elem.attributes.borrow().get("href") {
- vec![RawProperty::String(url.to_string())]
- } else {
- vec![RawProperty::String(String::new())]
- }
- }
- "object" => vec![RawProperty::String(
- elem.attributes
- .borrow()
- .get("data")
- .map(String::from)
- .unwrap_or_default(),
- )],
- "data" | "meter" => vec![RawProperty::String(
- elem.attributes
- .borrow()
- .get("value")
- .map(String::from)
- .unwrap_or_default(),
- )],
- "time" => {
- let time = elem
- .attributes
- .borrow()
- .get("datetime")
- .map(String::from)
- .unwrap_or_else(|| current.text_contents());
-
- vec![RawProperty::String(time)]
- }
- _ => text_contents(current.clone()),
- }
- };
+ let properties_for_prop = properties_for_prop(elem, current)?;
for itemprop in itemprop.split_ascii_whitespace() {
properties
diff --git a/crates/core/src/widgets/thesaurus.rs b/crates/core/src/widgets/thesaurus.rs
index adeed6a2..88737db4 100644
--- a/crates/core/src/widgets/thesaurus.rs
+++ b/crates/core/src/widgets/thesaurus.rs
@@ -17,7 +17,11 @@
use itertools::Itertools;
use regex::Regex;
use serde::{Deserialize, Serialize};
-use std::{fs::File, io::BufReader, path::Path};
+use std::{
+ fs::File,
+ io::{BufRead, BufReader},
+ path::Path,
+};
use utoipa::ToSchema;
use anyhow::{anyhow, Result};
@@ -25,6 +29,10 @@ use rio_api::parser::TriplesParser;
use rio_turtle::TurtleParser;
use std::collections::HashMap;
+static VALUE_STR_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(|| {
+ regex::Regex::new(r#""(.*)"@"#).expect("Failed to compile regex")
+});
+
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "camelCase", transparent)]
pub struct Lemma(String);
@@ -70,6 +78,103 @@ pub struct Info {
pos: PartOfSpeech,
}
+impl Info {
+ fn build_definition(concept: &NodeQuery<'_>) -> Definition {
+ let def = concept
+ .single_edge(|e| e.label.contains("#definition"))
+ .unwrap()
+ .to
+ .single_edge(|e| e.label.contains("#value"))
+ .unwrap()
+ .to
+ .node
+ .0
+ .clone();
+
+ let def = VALUE_STR_REGEX
+ .captures(&def)
+ .unwrap()
+ .get(1)
+ .unwrap()
+ .as_str()
+ .to_string();
+
+ Definition(def)
+ }
+
+ fn build_examples(concept: &NodeQuery<'_>) -> Vec<Example> {
+ concept
+ .filtered_edges(|e| e.label.contains("#example"))
+ .into_iter()
+ .map(|e| {
+ e.to.single_edge(|e| e.label.contains("#value"))
+ .unwrap()
+ .to
+ .node
+ .0
+ .clone()
+ })
+ .map(|e| {
+ VALUE_STR_REGEX
+ .captures(&e)
+ .unwrap()
+ .get(1)
+ .unwrap()
+ .as_str()
+ .to_string()
+ })
+ .map(Example)
+ .collect()
+ }
+
+ fn build_similar(concept: &NodeQuery<'_>) -> Vec<Id> {
+ concept
+ .filtered_edges(|e| e.label.contains("#similar"))
+ .into_iter()
+ .map(|e| Id(e.to.node.0.clone()))
+ .collect()
+ }
+
+ fn build_hyponyms(concept: &NodeQuery<'_>) -> Vec<Id> {
+ concept
+ .filtered_edges(|e| e.label.contains("#hyponym"))
+ .into_iter()
+ .map(|e| Id(e.to.node.0.clone()))
+ .collect()
+ }
+
+ fn build_hypernyms(concept: &NodeQuery<'_>) -> Vec<Id> {
+ concept
+ .filtered_edges(|e| e.label.contains("#hypernym"))
+ .into_iter()
+ .map(|e| Id(e.to.node.0.clone()))
+ .collect()
+ }
+
+ fn build_pos(concept: &NodeQuery<'_>) -> Result<PartOfSpeech> {
+ let pos = concept
+ .single_edge(|e| e.label.contains("#partOfSpeech"))
+ .unwrap()
+ .to
+ .node
+ .0
+ .split_once('#')
+ .unwrap()
+ .1
+ .to_string()
+ .replace('>', "");
+
+ match pos.as_str() {
+ "noun" => Ok(PartOfSpeech::Noun),
+ "adjective_satellite" => Ok(PartOfSpeech::AdjectiveSatellite),
+ "verb" => Ok(PartOfSpeech::Verb),
+ "adverb" => Ok(PartOfSpeech::Adverb),
+ "adjective" => Ok(PartOfSpeech::Adjective),
+ _ => Err(anyhow!("Unknown part of speech: {}", pos)),
+ }
+ }
+}
+
pub struct Dictionary {
map: HashMap<Id, Info>,
lemmas: HashMap<NormalizedLemma, Vec<Id>>,
@@ -293,9 +398,38 @@ impl Dictionary {
self.spellings.insert(normalized, lemma);
}
- pub fn build<P: AsRef<Path>>(path: P) -> Result<Self> {
- let reader = BufReader::new(File::open(path)?);
+ fn entry_info(entry: NodeQuery<'_>) -> Result<Vec<Info>> {
+ let mut res = Vec::new();
+ for sense in entry.filtered_edges(|e| e.label.contains("#sense")) {
+ let concept = sense
+ .to
+ .single_edge(|e| e.label.contains("#isLexicalizedSenseOf"))
+ .unwrap()
+ .to;
+ let id = Id(concept.node.0.clone());
+
+ let definition = Info::build_definition(&concept);
+ let examples = Info::build_examples(&concept);
+ let similar = Info::build_similar(&concept);
+ let hyponyms = Info::build_hyponyms(&concept);
+ let hypernyms = Info::build_hypernyms(&concept);
+ let pos = Info::build_pos(&concept)?;
+
+ res.push(Info {
+ id,
+ definition,
+ examples,
+ similar,
+ hyponyms,
+ hypernyms,
+ pos,
+ });
+ }
+
+ Ok(res)
+ }
+ fn build_graph<R: BufRead>(reader: R) -> Result<Graph> {
let mut parser = TurtleParser::new(reader, None);
let mut graph = Graph::new();
@@ -311,6 +445,10 @@ impl Dictionary {
Ok::<_, anyhow::Error>(())
})?;
+ Ok(graph)
+ }
+
+ fn build_lexical_entries(graph: &Graph) -> Vec<&'_ Node> {
let mut lexical_entries = Vec::new();
for (node, edges) in &graph.map {
@@ -321,8 +459,15 @@ impl Dictionary {
}
}
+ lexical_entries
+ }
+
+ pub fn build<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let reader = BufReader::new(File::open(path)?);
+ let graph = Self::build_graph(reader)?;
+ let lexical_entries = Self::build_lexical_entries(&graph);
+
let mut dict = Self::empty()?;
- let value_str_regex = regex::Regex::new(r#""(.*)"@"#).unwrap();
for entry in lexical_entries {
let entry = NodeQuery::new(entry, &graph);
@@ -338,7 +483,7 @@ impl Dictionary {
.0
.clone();
- let written_rep = value_str_regex
+ let written_rep = VALUE_STR_REGEX
.captures(&written_rep)
.unwrap()
.get(1)
@@ -348,106 +493,7 @@ impl Dictionary {
let lemma = Lemma(written_rep);
- for sense in entry.filtered_edges(|e| e.label.contains("#sense")) {
- let concept = sense
- .to
- .single_edge(|e| e.label.contains("#isLexicalizedSenseOf"))
- .unwrap()
- .to;
- let id = Id(concept.node.0.clone());
-
- let definition = concept
- .single_edge(|e| e.label.contains("#definition"))
- .unwrap()
- .to
- .single_edge(|e| e.label.contains("#value"))
- .unwrap()
- .to
- .node
- .0
- .clone();
-
- let definition = value_str_regex
- .captures(&definition)
- .unwrap()
- .get(1)
- .unwrap()
- .as_str()
- .to_string();
- let definition = Definition(definition);
-
- let examples: Vec<_> = concept
- .filtered_edges(|e| e.label.contains("#example"))
- .into_iter()
- .map(|e| {
- e.to.single_edge(|e| e.label.contains("#value"))
- .unwrap()
- .to
- .node
- .0
- .clone()
- })
- .map(|e| {
- value_str_regex
- .captures(&e)
- .unwrap()
- .get(1)
- .unwrap()
- .as_str()
- .to_string()
- })
- .map(Example)
- .collect();
-
- let similar: Vec<_> = concept
- .filtered_edges(|e| e.label.contains("#similar"))
- .into_iter()
- .map(|e| Id(e.to.node.0.clone()))
- .collect();
-
- let hyponyms: Vec<_> = concept
- .filtered_edges(|e| e.label.contains("#hyponym"))
- .into_iter()
- .map(|e| Id(e.to.node.0.clone()))
- .collect();
-
- let hypernyms: Vec<_> = concept
- .filtered_edges(|e| e.label.contains("#hypernym"))
- .into_iter()
- .map(|e| Id(e.to.node.0.clone()))
- .collect();
-
- let pos = concept
- .single_edge(|e| e.label.contains("#partOfSpeech"))
- .unwrap()
- .to
- .node
- .0
- .split_once('#')
- .unwrap()
- .1
- .to_string()
- .replace('>', "");
-
- let pos = match pos.as_str() {
- "noun" => PartOfSpeech::Noun,
- "adjective_satellite" => PartOfSpeech::AdjectiveSatellite,
- "verb" => PartOfSpeech::Verb,
- "adverb" => PartOfSpeech::Adverb,
- "adjective" => PartOfSpeech::Adjective,
- _ => return Err(anyhow!("Unknown part of speech: {}", pos)),
- };
-
- let info = Info {
- id,
- definition,
- examples,
- similar,
- hyponyms,
- hypernyms,
- pos,
- };
-
+ for info in Self::entry_info(entry)? {
dict.insert(lemma.clone(), info);
}
}