web search done right
build page and host level webgraph at the same time
Mikkel Denker9ae87df04b1e
diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.tomlindex 6a3efa49..781ef943 100644--- a/configs/webgraph/local.toml+++ b/configs/webgraph/local.toml@@ -1,6 +1,6 @@-level = "host"limit_warc_files = 5-graph_base_path = "data/webgraph"+host_graph_base_path = "data/webgraph"+page_graph_base_path = "data/webgraph_page"batch_size = 1[warc_source]diff --git a/configs/webgraph/server.toml b/configs/webgraph/server.tomlindex dd220804..2f537362 100644--- a/configs/webgraph/server.toml+++ b/configs/webgraph/server.toml@@ -1,5 +1,5 @@host = "0.0.0.0:3003"cluster_id = "dev_webgraph"gossip_addr = "0.0.0.0:3007"-graph_path = "data/webgraph"-inbound_similarity_path = "data/centrality/inbound_similarity"\ No newline at end of file+graph_path = "data/webgraph_host"+inbound_similarity_path = "data/centrality/inbound_similarity"diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 3fac570f..b67f111c 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -39,20 +39,13 @@ pub struct IndexingLocalConfig {pub minimum_clean_words: Option<usize>,}-#[derive(Debug, Serialize, Deserialize, Clone)]-#[serde(rename_all = "lowercase")]-pub enum WebgraphLevel {- Page,- Host,-}-#[derive(Debug, Deserialize, Clone)]pub struct WebgraphConstructConfig {- pub level: WebgraphLevel,+ pub host_graph_base_path: String,+ pub page_graph_base_path: String,+ pub warc_source: WarcSource,pub redirect_db_path: Option<String>,pub limit_warc_files: Option<usize>,- pub warc_source: WarcSource,- pub graph_base_path: Option<String>,pub batch_size: Option<usize>,}diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex c177cf0e..1ecee632 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -61,10 +61,15 @@ fn download_files() {fn create_webgraph() -> Result<()> {debug!("Creating webgraph");- let out_path = Path::new(DATA_PATH).join("webgraph");+ let out_path_host = Path::new(DATA_PATH).join("webgraph_host");+ let out_path_page = Path::new(DATA_PATH).join("webgraph_page");- if out_path.exists() {- std::fs::remove_dir_all(&out_path)?;+ if out_path_host.exists() {+ std::fs::remove_dir_all(&out_path_host)?;+ }++ if out_path_page.exists() {+ std::fs::remove_dir_all(&out_path_page)?;}let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");@@ -75,13 +80,12 @@ fn create_webgraph() -> Result<()> {names: vec![warc_path.to_str().unwrap().to_string()],}),warc_paths: vec![warc_path.to_str().unwrap().to_string()],- graph_base_path: out_path.to_str().unwrap().to_string(),- level: crate::config::WebgraphLevel::Host,};let mut worker = webgraph::WebgraphWorker {redirect: None,- graph: webgraph::open_graph(&job.graph_base_path),+ host_graph: webgraph::open_graph(&out_path_host),+ page_graph: webgraph::open_graph(&out_path_page),};worker.process_job(&job);@@ -91,7 +95,7 @@ fn create_webgraph() -> Result<()> {fn calculate_centrality() {debug!("Calculating centrality");- let webgraph_path = Path::new(DATA_PATH).join("webgraph");+ let webgraph_path = Path::new(DATA_PATH).join("webgraph_host");let out_path = Path::new(DATA_PATH).join("centrality");Centrality::build_harmonic(&webgraph_path, &out_path);@@ -123,7 +127,7 @@ fn create_inverted_index() -> Result<()> {minimum_clean_words: None,};- let webgraph_path = Path::new(DATA_PATH).join("webgraph");+ let webgraph_path = Path::new(DATA_PATH).join("webgraph_page");let centrality_path = Path::new(DATA_PATH).join("centrality");let worker = indexer::IndexingWorker::new(diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 302ace83..1456e05d 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -15,7 +15,6 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{config::WarcSource,- config::WebgraphLevel,config::{self, WebgraphConstructConfig},crawler::crawl_db::RedirectDb,entrypoint::download_all_warc_files,@@ -64,10 +63,8 @@ impl From<JobConfig> for config::WarcSource {#[derive(Debug, Serialize, Deserialize, Clone)]pub struct Job {- pub level: WebgraphLevel,pub config: JobConfig,pub warc_paths: Vec<String>,- pub graph_base_path: String,}pub fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {@@ -75,7 +72,8 @@ pub fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {}pub struct WebgraphWorker {- pub graph: webgraph::Webgraph,+ pub host_graph: webgraph::Webgraph,+ pub page_graph: webgraph::Webgraph,pub redirect: Option<Arc<RedirectDb>>,}@@ -130,18 +128,21 @@ impl WebgraphWorker {let mut destination = Node::from(destination);- if let WebgraphLevel::Host = job.level {- source = source.into_host();- destination = destination.into_host();- }+ self.page_graph+ .insert(source.clone(), destination.clone(), link.text.clone());++ source = source.into_host();+ destination = destination.into_host();- self.graph.insert(source, destination, link.text);+ self.host_graph.insert(source, destination, link.text);}}- self.graph.commit();+ self.host_graph.commit();+ self.page_graph.commit();}- self.graph.merge_segments(1);+ self.host_graph.merge_segments(1);+ self.page_graph.merge_segments(1);info!("{} done", name);}@@ -169,31 +170,29 @@ impl Webgraph {.into_iter().map(|warc_paths| Job {config: job_config.clone(),- level: config.level.clone(),warc_paths: warc_paths.collect_vec(),- graph_base_path: config- .graph_base_path- .clone()- .unwrap_or_else(|| "data/webgraph".to_string()),}).collect_vec();let num_workers = num_cpus::get();let mut handlers = Vec::new();- let parent_path = config- .graph_base_path- .clone()- .unwrap_or_else(|| "data/webgraph".to_string());+ let host_path = &config.host_graph_base_path;+ let page_path = &config.page_graph_base_path;for i in 0..num_workers {- let path = parent_path.clone();- let path = Path::new(&path);- let path = path.join(format!("worker_{i}"));+ let host_path = host_path.clone();+ let host_path = Path::new(&host_path);+ let host_path = host_path.join(format!("worker_{i}"));++ let page_path = page_path.clone();+ let page_path = Path::new(&page_path);+ let page_path = page_path.join(format!("worker_{i}"));let mut worker = WebgraphWorker {redirect: redirect.clone(),- graph: open_graph(path),+ host_graph: open_graph(host_path),+ page_graph: open_graph(page_path),};let jobs = jobs.clone();@@ -202,7 +201,7 @@ impl Webgraph {worker.process_job(job);}- worker.graph+ (worker.host_graph, worker.page_graph)}));}@@ -211,11 +210,17 @@ impl Webgraph {graphs.push(handler.join().unwrap());}- let mut graph = graphs.pop().unwrap();- for other_graph in graphs {- let other_path = other_graph.path.clone();- graph.merge(other_graph);- fs::remove_dir_all(other_path)?;+ let (mut host_graph, mut page_graph) = graphs.pop().unwrap();++ for (other_host, other_page) in graphs {+ let other_host_path = other_host.path.clone();+ let other_page_path = other_page.path.clone();++ host_graph.merge(other_host);+ page_graph.merge(other_page);++ fs::remove_dir_all(other_host_path)?;+ fs::remove_dir_all(other_page_path)?;}Ok(())