Skip to content

web search done right

configurable webgraph level

Mikkel Denker6aa3766d790b
diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.toml
index 1074ec96..6a3efa49 100644
--- a/configs/webgraph/local.toml
+++ b/configs/webgraph/local.toml
@@ -1,3 +1,4 @@
+level = "host"
limit_warc_files = 5
graph_base_path = "data/webgraph"
batch_size = 1
diff --git a/configs/webgraph/master.toml b/configs/webgraph/master.toml
index 69e3e024..8cd58012 100644
--- a/configs/webgraph/master.toml
+++ b/configs/webgraph/master.toml
@@ -1,3 +1,4 @@
+level = "host"
limit_warc_files = 10000
workers = ["0.0.0.0:1337", "0.0.0.0:1338", "0.0.0.0:1339", "0.0.0.0:1340", "0.0.0.0:1341", "0.0.0.0:1342", "0.0.0.0:1343"]
graph_base_path = "/mnt/webgraph"
diff --git a/configs/webgraph/profile.toml b/configs/webgraph/profile.toml
index 2070795d..b56c05d8 100644
--- a/configs/webgraph/profile.toml
+++ b/configs/webgraph/profile.toml
@@ -1,3 +1,4 @@
+level = "host"
graph_base_path = "data/webgraph"
[warc_source]
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 0f06ed52..1a1dcb13 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -80,6 +80,7 @@ fn create_webgraph() -> Result<()> {
}),
warc_paths: vec![warc_path.to_str().unwrap().to_string()],
graph_base_path: out_path_tmp.to_str().unwrap().to_string(),
+ level: crate::WebgraphLevel::Host,
};
let graph = webgraph::process_job(&job);
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index 22028373..f6d2cd2a 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -19,7 +19,7 @@ use crate::{
warc::WarcFile,
webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},
webpage::Html,
- HttpConfig, LocalConfig, Result, S3Config, WarcSource, WebgraphLocalConfig,
+ HttpConfig, LocalConfig, Result, S3Config, WarcSource, WebgraphLevel, WebgraphLocalConfig,
WebgraphMasterConfig,
};
use itertools::Itertools;
@@ -63,6 +63,7 @@ impl From<JobConfig> for WarcSource {
#[derive(Debug, Serialize, Deserialize)]
pub struct Job {
+ pub level: WebgraphLevel,
pub config: JobConfig,
pub warc_paths: Vec<String>,
pub graph_base_path: String,
@@ -101,11 +102,18 @@ pub fn process_job(job: &Job) -> webgraph::Webgraph {
.filter(|link| link.matches_url_regex())
{
trace!("inserting link {:?}", link);
- graph.insert(
- Node::from(link.source),
- Node::from(link.destination),
- link.text,
- );
+ let mut source = Node::from(link.source);
+ source.remove_protocol();
+
+ let mut destination = Node::from(link.destination);
+ destination.remove_protocol();
+
+ if let WebgraphLevel::Host = job.level {
+ source = source.into_host();
+ destination = destination.into_host();
+ }
+
+ graph.insert(source, destination, link.text);
}
}
}
@@ -203,6 +211,7 @@ impl Webgraph {
.into_iter()
.map(|warc_paths| Job {
config: job_config.clone(),
+ level: config.level.clone(),
warc_paths: warc_paths.into_iter().collect(),
graph_base_path: config
.graph_base_path
@@ -260,6 +269,7 @@ impl Webgraph {
.into_iter()
.map(|warc_paths| Job {
config: job_config.clone(),
+ level: config.level.clone(),
warc_paths: warc_paths.collect_vec(),
graph_base_path: config
.graph_base_path
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 94a02291..5c62e364 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -110,8 +110,15 @@ pub struct IndexingLocalConfig {
minimum_clean_words: Option<usize>,
}
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum WebgraphLevel {
+ Url,
+ Host,
+}
+
#[derive(Debug, Deserialize, Clone)]
pub struct WebgraphMasterConfig {
+ level: WebgraphLevel,
limit_warc_files: Option<usize>,
warc_source: WarcSource,
workers: Vec<String>,
@@ -121,6 +128,7 @@ pub struct WebgraphMasterConfig {
#[derive(Debug, Deserialize, Clone)]
pub struct WebgraphLocalConfig {
+ level: WebgraphLevel,
limit_warc_files: Option<usize>,
warc_source: WarcSource,
graph_base_path: Option<String>,
diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rs
index ceb1b90d..9bf34e3a 100644
--- a/core/src/webgraph/centrality/harmonic.rs
+++ b/core/src/webgraph/centrality/harmonic.rs
@@ -248,10 +248,26 @@ mod tests {
fn www_subdomain_ignored() {
let mut graph = WebgraphBuilder::new_memory().open();
- graph.insert(Node::from("B.com"), Node::from("A.com"), String::new());
- graph.insert(Node::from("B.com"), Node::from("www.A.com"), String::new());
- graph.insert(Node::from("C.com"), Node::from("A.com"), String::new());
- graph.insert(Node::from("C.com"), Node::from("www.A.com"), String::new());
+ graph.insert(
+ Node::from("B.com").into_host(),
+ Node::from("A.com").into_host(),
+ String::new(),
+ );
+ graph.insert(
+ Node::from("B.com").into_host(),
+ Node::from("www.A.com").into_host(),
+ String::new(),
+ );
+ graph.insert(
+ Node::from("C.com").into_host(),
+ Node::from("A.com").into_host(),
+ String::new(),
+ );
+ graph.insert(
+ Node::from("C.com").into_host(),
+ Node::from("www.A.com").into_host(),
+ String::new(),
+ );
graph.commit();
let centrality = HarmonicCentrality::calculate(&graph);
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index a2754950..4674442c 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -69,7 +69,7 @@ impl Node {
Node::from(url.full())
}
- fn remove_protocol(&mut self) {
+ pub fn remove_protocol(&mut self) {
let url = Url::from(self.name.clone());
self.name = url.without_protocol().to_string();
}
@@ -372,13 +372,8 @@ impl Webgraph {
}
}
- pub fn insert(&mut self, mut from: Node, mut to: Node, label: String) {
- from.remove_protocol();
- to.remove_protocol();
-
- let (from, to) = (from.into_host(), to.into_host());
+ pub fn insert(&mut self, from: Node, to: Node, label: String) {
let (from_id, to_id) = (self.id_or_assign(&from), self.id_or_assign(&to));
-
self.live_segment.insert(from_id, to_id, label);
}
@@ -792,22 +787,6 @@ mod test {
n.remove_protocol();
assert_eq!(&n.name, "www.example.com?test");
-
- let mut graph = WebgraphBuilder::new_memory().open();
-
- graph.insert(
- Node::from("http://A"),
- Node::from("https://B"),
- String::new(),
- );
-
- graph.commit();
-
- let distances = graph.distances(Node::from("A"));
- assert_eq!(distances.get(&Node::from("B")), Some(&1));
-
- let distances = graph.distances(Node::from("http://A"));
- assert!(distances.is_empty());
}
#[test]