Skip to content

web search done right

various webgraph insert + merge optimizations

Mikkel Denker5b98e498fcf4
diff --git a/justfile b/justfile
index b770803e..dd580fd0 100644
--- a/justfile
+++ b/justfile
@@ -1,4 +1,5 @@
@flamegraph:
+ sudo rm -rf webgraph
cargo flamegraph --root -- configs/webgraph/local/profile.toml
@worker:
diff --git a/src/bin/cuely.rs b/src/bin/cuely.rs
index ff84f3a6..931e15ea 100644
--- a/src/bin/cuely.rs
+++ b/src/bin/cuely.rs
@@ -28,7 +28,7 @@ struct Args {
#[tokio::main]
async fn main() {
let subscriber = FmtSubscriber::builder()
- .with_max_level(Level::DEBUG)
+ .with_max_level(Level::INFO)
.finish();
tracing::subscriber::set_global_default(subscriber).unwrap();
diff --git a/src/directory.rs b/src/directory.rs
index d98a66d3..cf5075e2 100644
--- a/src/directory.rs
+++ b/src/directory.rs
@@ -31,11 +31,10 @@ pub enum DirEntry {
}
fn iterate_children(path: &str) -> Result<Vec<DirEntry>> {
- dbg!(&path);
let mut res = Vec::new();
for f in fs::read_dir(path)? {
- let entry = dbg!(f?);
+ let entry = f?;
let metadata = entry.metadata()?;
if metadata.is_dir() {
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index b3086dc0..75c85561 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -21,6 +21,7 @@ use crate::{
HttpConfig, LocalConfig, Result, WarcSource, WebgraphConfig, WebgraphLocalConfig,
WebgraphMasterConfig, WebgraphWorkerConfig,
};
+use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};
use serde::{Deserialize, Serialize};
use std::{net::SocketAddr, path::Path};
use tracing::{debug, info};
@@ -118,12 +119,22 @@ impl WebgraphBuilder {
WarcSource::Local(config) => JobConfig::Local(config),
};
+ let pb = ProgressBar::new(warc_paths.len() as u64);
+ pb.set_style(
+ ProgressStyle::default_bar()
+ .template(
+ "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})",
+ )
+ .progress_chars("#>-"),
+ );
+
warc_paths
.into_iter()
.map(|warc_path| Job {
config: job_config.clone(),
warc_path,
})
+ .progress_with(pb)
.map_reduce(&workers)
.await
.expect("failed to build webgraph");
diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rs
index 83749155..e686d6fc 100644
--- a/src/mapreduce/manager.rs
+++ b/src/mapreduce/manager.rs
@@ -258,6 +258,10 @@ impl Manager {
jobs.into_iter()
.map(|job| self.map::<I, O>(job))
.collect::<FuturesUnordered<_>>()
+ .map(|d| {
+ println!("TEST");
+ d
+ })
.fold(
None,
|acc, elem| async move { Some(Manager::reduce(acc, elem)) },
diff --git a/src/webpage.rs b/src/webpage.rs
index da7f07f3..4cc3a0b8 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -172,7 +172,13 @@ impl<'a> Html<'a> {
tag.attributes().get("href").flatten().map(|bytes| {
let destination = String::from_utf8(bytes.as_bytes().to_vec()).unwrap();
- let text = tag.inner_text(parser).to_string();
+ let children = tag.children();
+ let text: String = Itertools::intersperse(
+ Html::children_text(children, parser).into_iter(),
+ ". ".to_string(),
+ )
+ .collect();
+
Link {
source: self.url.clone(),
destination,
@@ -250,24 +256,28 @@ impl<'a> Html<'a> {
}
fn children_text(children: Children, parser: &tl::Parser) -> Vec<String> {
+ let mut stack = vec![children];
+
let mut res = Vec::new();
let allowed_tags = [
"a", "div", "span", "p", "h1", "h2", "h3", "h4", "li", "ul", "ol", "nav", "pre",
"body", "html",
];
- for id in children.top().iter() {
- let node = id.get(parser).unwrap();
- match node {
- tl::Node::Tag(t) => {
- let tag = node.as_tag().unwrap();
- let name = tag.name().as_utf8_str();
- if allowed_tags.iter().any(|allowed| name == *allowed) {
- res.extend(Html::children_text(t.children(), parser))
+ while let Some(children) = stack.pop() {
+ for id in children.top().iter() {
+ let node = id.get(parser).unwrap();
+ match node {
+ tl::Node::Tag(t) => {
+ let tag = node.as_tag().unwrap();
+ let name = tag.name().as_utf8_str();
+ if allowed_tags.iter().any(|allowed| name == *allowed) {
+ stack.push(t.children())
+ }
}
+ tl::Node::Raw(raw) => res.push(raw.as_utf8_str().trim().to_string()),
+ tl::Node::Comment(_) => {}
}
- tl::Node::Raw(raw) => res.push(raw.as_utf8_str().trim().to_string()),
- tl::Node::Comment(_) => {}
}
}