web search done right
speedup webgraph by not parsing texts
Mikkel Denkerefa1c229fb38
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 5aefe252..4c15faa9 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -68,7 +68,8 @@ fn process_job(job: &Job) -> webgraph::Webgraph {for warc_path in &warc_files {if let Ok(file) = WarcFile::open(warc_path) {for record in file.records().flatten() {- let webpage = Html::parse(&record.response.body, &record.request.url);+ let webpage =+ Html::parse_including_text(&record.response.body, &record.request.url, false);for link in webpage.links().into_iter()diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex d4f83f68..68ae3ec2 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -201,10 +201,22 @@ pub struct Html {impl Html {pub fn parse(html: &str, url: &str) -> Self {+ Self::parse_including_text(html, url, true)+ }++ pub fn parse_including_text(html: &str, url: &str, include_text: bool) -> Self {let root = kuchiki::parse_html().one(html);- let all_text = Html::calculate_all_text(root.clone());- let clean_text = Html::calculate_clean_text(root.clone());+ let all_text = if include_text {+ Html::calculate_all_text(root.clone())+ } else {+ None+ };+ let clean_text = if include_text {+ Html::calculate_clean_text(root.clone())+ } else {+ None+ };Self {root,