Skip to content

web search done right

speedup webgraph by not parsing texts

Mikkel Denkerefa1c229fb38
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index 5aefe252..4c15faa9 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -68,7 +68,8 @@ fn process_job(job: &Job) -> webgraph::Webgraph {
for warc_path in &warc_files {
if let Ok(file) = WarcFile::open(warc_path) {
for record in file.records().flatten() {
- let webpage = Html::parse(&record.response.body, &record.request.url);
+ let webpage =
+ Html::parse_including_text(&record.response.body, &record.request.url, false);
for link in webpage
.links()
.into_iter()
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index d4f83f68..68ae3ec2 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -201,10 +201,22 @@ pub struct Html {
impl Html {
pub fn parse(html: &str, url: &str) -> Self {
+ Self::parse_including_text(html, url, true)
+ }
+
+ pub fn parse_including_text(html: &str, url: &str, include_text: bool) -> Self {
let root = kuchiki::parse_html().one(html);
- let all_text = Html::calculate_all_text(root.clone());
- let clean_text = Html::calculate_clean_text(root.clone());
+ let all_text = if include_text {
+ Html::calculate_all_text(root.clone())
+ } else {
+ None
+ };
+ let clean_text = if include_text {
+ Html::calculate_clean_text(root.clone())
+ } else {
+ None
+ };
Self {
root,