web search done right
various webgraph insert + merge optimizations
Mikkel Denker5b98e498fcf4
diff --git a/justfile b/justfileindex b770803e..dd580fd0 100644--- a/justfile+++ b/justfile@@ -1,4 +1,5 @@@flamegraph:+ sudo rm -rf webgraphcargo flamegraph --root -- configs/webgraph/local/profile.toml@worker:diff --git a/src/bin/cuely.rs b/src/bin/cuely.rsindex ff84f3a6..931e15ea 100644--- a/src/bin/cuely.rs+++ b/src/bin/cuely.rs@@ -28,7 +28,7 @@ struct Args {#[tokio::main]async fn main() {let subscriber = FmtSubscriber::builder()- .with_max_level(Level::DEBUG)+ .with_max_level(Level::INFO).finish();tracing::subscriber::set_global_default(subscriber).unwrap();diff --git a/src/directory.rs b/src/directory.rsindex d98a66d3..cf5075e2 100644--- a/src/directory.rs+++ b/src/directory.rs@@ -31,11 +31,10 @@ pub enum DirEntry {}fn iterate_children(path: &str) -> Result<Vec<DirEntry>> {- dbg!(&path);let mut res = Vec::new();for f in fs::read_dir(path)? {- let entry = dbg!(f?);+ let entry = f?;let metadata = entry.metadata()?;if metadata.is_dir() {diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex b3086dc0..75c85561 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -21,6 +21,7 @@ use crate::{HttpConfig, LocalConfig, Result, WarcSource, WebgraphConfig, WebgraphLocalConfig,WebgraphMasterConfig, WebgraphWorkerConfig,};+use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};use serde::{Deserialize, Serialize};use std::{net::SocketAddr, path::Path};use tracing::{debug, info};@@ -118,12 +119,22 @@ impl WebgraphBuilder {WarcSource::Local(config) => JobConfig::Local(config),};+ let pb = ProgressBar::new(warc_paths.len() as u64);+ pb.set_style(+ ProgressStyle::default_bar()+ .template(+ "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})",+ )+ .progress_chars("#>-"),+ );+warc_paths.into_iter().map(|warc_path| Job {config: job_config.clone(),warc_path,})+ .progress_with(pb).map_reduce(&workers).await.expect("failed to build webgraph");diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rsindex 83749155..e686d6fc 100644--- a/src/mapreduce/manager.rs+++ b/src/mapreduce/manager.rs@@ -258,6 +258,10 @@ impl Manager {jobs.into_iter().map(|job| self.map::<I, O>(job)).collect::<FuturesUnordered<_>>()+ .map(|d| {+ println!("TEST");+ d+ }).fold(None,|acc, elem| async move { Some(Manager::reduce(acc, elem)) },diff --git a/src/webpage.rs b/src/webpage.rsindex da7f07f3..4cc3a0b8 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -172,7 +172,13 @@ impl<'a> Html<'a> {tag.attributes().get("href").flatten().map(|bytes| {let destination = String::from_utf8(bytes.as_bytes().to_vec()).unwrap();- let text = tag.inner_text(parser).to_string();+ let children = tag.children();+ let text: String = Itertools::intersperse(+ Html::children_text(children, parser).into_iter(),+ ". ".to_string(),+ )+ .collect();+Link {source: self.url.clone(),destination,@@ -250,24 +256,28 @@ impl<'a> Html<'a> {}fn children_text(children: Children, parser: &tl::Parser) -> Vec<String> {+ let mut stack = vec![children];+let mut res = Vec::new();let allowed_tags = ["a", "div", "span", "p", "h1", "h2", "h3", "h4", "li", "ul", "ol", "nav", "pre","body", "html",];- for id in children.top().iter() {- let node = id.get(parser).unwrap();- match node {- tl::Node::Tag(t) => {- let tag = node.as_tag().unwrap();- let name = tag.name().as_utf8_str();- if allowed_tags.iter().any(|allowed| name == *allowed) {- res.extend(Html::children_text(t.children(), parser))+ while let Some(children) = stack.pop() {+ for id in children.top().iter() {+ let node = id.get(parser).unwrap();+ match node {+ tl::Node::Tag(t) => {+ let tag = node.as_tag().unwrap();+ let name = tag.name().as_utf8_str();+ if allowed_tags.iter().any(|allowed| name == *allowed) {+ stack.push(t.children())+ }}+ tl::Node::Raw(raw) => res.push(raw.as_utf8_str().trim().to_string()),+ tl::Node::Comment(_) => {}}- tl::Node::Raw(raw) => res.push(raw.as_utf8_str().trim().to_string()),- tl::Node::Comment(_) => {}}}