web search done right
configurable webgraph level
Mikkel Denker6aa3766d790b
diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.tomlindex 1074ec96..6a3efa49 100644--- a/configs/webgraph/local.toml+++ b/configs/webgraph/local.toml@@ -1,3 +1,4 @@+level = "host"limit_warc_files = 5graph_base_path = "data/webgraph"batch_size = 1diff --git a/configs/webgraph/master.toml b/configs/webgraph/master.tomlindex 69e3e024..8cd58012 100644--- a/configs/webgraph/master.toml+++ b/configs/webgraph/master.toml@@ -1,3 +1,4 @@+level = "host"limit_warc_files = 10000workers = ["0.0.0.0:1337", "0.0.0.0:1338", "0.0.0.0:1339", "0.0.0.0:1340", "0.0.0.0:1341", "0.0.0.0:1342", "0.0.0.0:1343"]graph_base_path = "/mnt/webgraph"diff --git a/configs/webgraph/profile.toml b/configs/webgraph/profile.tomlindex 2070795d..b56c05d8 100644--- a/configs/webgraph/profile.toml+++ b/configs/webgraph/profile.toml@@ -1,3 +1,4 @@+level = "host"graph_base_path = "data/webgraph"[warc_source]diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 0f06ed52..1a1dcb13 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -80,6 +80,7 @@ fn create_webgraph() -> Result<()> {}),warc_paths: vec![warc_path.to_str().unwrap().to_string()],graph_base_path: out_path_tmp.to_str().unwrap().to_string(),+ level: crate::WebgraphLevel::Host,};let graph = webgraph::process_job(&job);diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 22028373..f6d2cd2a 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -19,7 +19,7 @@ use crate::{warc::WarcFile,webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},webpage::Html,- HttpConfig, LocalConfig, Result, S3Config, WarcSource, WebgraphLocalConfig,+ HttpConfig, LocalConfig, Result, S3Config, WarcSource, WebgraphLevel, WebgraphLocalConfig,WebgraphMasterConfig,};use itertools::Itertools;@@ -63,6 +63,7 @@ impl From<JobConfig> for WarcSource {#[derive(Debug, Serialize, Deserialize)]pub struct Job {+ pub level: WebgraphLevel,pub config: JobConfig,pub warc_paths: Vec<String>,pub graph_base_path: String,@@ -101,11 +102,18 @@ pub fn process_job(job: &Job) -> webgraph::Webgraph {.filter(|link| link.matches_url_regex()){trace!("inserting link {:?}", link);- graph.insert(- Node::from(link.source),- Node::from(link.destination),- link.text,- );+ let mut source = Node::from(link.source);+ source.remove_protocol();++ let mut destination = Node::from(link.destination);+ destination.remove_protocol();++ if let WebgraphLevel::Host = job.level {+ source = source.into_host();+ destination = destination.into_host();+ }++ graph.insert(source, destination, link.text);}}}@@ -203,6 +211,7 @@ impl Webgraph {.into_iter().map(|warc_paths| Job {config: job_config.clone(),+ level: config.level.clone(),warc_paths: warc_paths.into_iter().collect(),graph_base_path: config.graph_base_path@@ -260,6 +269,7 @@ impl Webgraph {.into_iter().map(|warc_paths| Job {config: job_config.clone(),+ level: config.level.clone(),warc_paths: warc_paths.collect_vec(),graph_base_path: config.graph_base_pathdiff --git a/core/src/lib.rs b/core/src/lib.rsindex 94a02291..5c62e364 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -110,8 +110,15 @@ pub struct IndexingLocalConfig {minimum_clean_words: Option<usize>,}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub enum WebgraphLevel {+ Url,+ Host,+}+#[derive(Debug, Deserialize, Clone)]pub struct WebgraphMasterConfig {+ level: WebgraphLevel,limit_warc_files: Option<usize>,warc_source: WarcSource,workers: Vec<String>,@@ -121,6 +128,7 @@ pub struct WebgraphMasterConfig {#[derive(Debug, Deserialize, Clone)]pub struct WebgraphLocalConfig {+ level: WebgraphLevel,limit_warc_files: Option<usize>,warc_source: WarcSource,graph_base_path: Option<String>,diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex ceb1b90d..9bf34e3a 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -248,10 +248,26 @@ mod tests {fn www_subdomain_ignored() {let mut graph = WebgraphBuilder::new_memory().open();- graph.insert(Node::from("B.com"), Node::from("A.com"), String::new());- graph.insert(Node::from("B.com"), Node::from("www.A.com"), String::new());- graph.insert(Node::from("C.com"), Node::from("A.com"), String::new());- graph.insert(Node::from("C.com"), Node::from("www.A.com"), String::new());+ graph.insert(+ Node::from("B.com").into_host(),+ Node::from("A.com").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("B.com").into_host(),+ Node::from("www.A.com").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("C.com").into_host(),+ Node::from("A.com").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("C.com").into_host(),+ Node::from("www.A.com").into_host(),+ String::new(),+ );graph.commit();let centrality = HarmonicCentrality::calculate(&graph);diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex a2754950..4674442c 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -69,7 +69,7 @@ impl Node {Node::from(url.full())}- fn remove_protocol(&mut self) {+ pub fn remove_protocol(&mut self) {let url = Url::from(self.name.clone());self.name = url.without_protocol().to_string();}@@ -372,13 +372,8 @@ impl Webgraph {}}- pub fn insert(&mut self, mut from: Node, mut to: Node, label: String) {- from.remove_protocol();- to.remove_protocol();-- let (from, to) = (from.into_host(), to.into_host());+ pub fn insert(&mut self, from: Node, to: Node, label: String) {let (from_id, to_id) = (self.id_or_assign(&from), self.id_or_assign(&to));-self.live_segment.insert(from_id, to_id, label);}@@ -792,22 +787,6 @@ mod test {n.remove_protocol();assert_eq!(&n.name, "www.example.com?test");-- let mut graph = WebgraphBuilder::new_memory().open();-- graph.insert(- Node::from("http://A"),- Node::from("https://B"),- String::new(),- );-- graph.commit();-- let distances = graph.distances(Node::from("A"));- assert_eq!(distances.get(&Node::from("B")), Some(&1));-- let distances = graph.distances(Node::from("http://A"));- assert!(distances.is_empty());}#[test]