Skip to content

web search done right

build page and host level webgraph at the same time

Mikkel Denker9ae87df04b1e
diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.toml
index 6a3efa49..781ef943 100644
--- a/configs/webgraph/local.toml
+++ b/configs/webgraph/local.toml
@@ -1,6 +1,6 @@
-level = "host"
limit_warc_files = 5
-graph_base_path = "data/webgraph"
+host_graph_base_path = "data/webgraph"
+page_graph_base_path = "data/webgraph_page"
batch_size = 1
[warc_source]
diff --git a/configs/webgraph/server.toml b/configs/webgraph/server.toml
index dd220804..2f537362 100644
--- a/configs/webgraph/server.toml
+++ b/configs/webgraph/server.toml
@@ -1,5 +1,5 @@
host = "0.0.0.0:3003"
cluster_id = "dev_webgraph"
gossip_addr = "0.0.0.0:3007"
-graph_path = "data/webgraph"
-inbound_similarity_path = "data/centrality/inbound_similarity"
\ No newline at end of file
+graph_path = "data/webgraph_host"
+inbound_similarity_path = "data/centrality/inbound_similarity"
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 3fac570f..b67f111c 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -39,20 +39,13 @@ pub struct IndexingLocalConfig {
pub minimum_clean_words: Option<usize>,
}
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(rename_all = "lowercase")]
-pub enum WebgraphLevel {
- Page,
- Host,
-}
-
#[derive(Debug, Deserialize, Clone)]
pub struct WebgraphConstructConfig {
- pub level: WebgraphLevel,
+ pub host_graph_base_path: String,
+ pub page_graph_base_path: String,
+ pub warc_source: WarcSource,
pub redirect_db_path: Option<String>,
pub limit_warc_files: Option<usize>,
- pub warc_source: WarcSource,
- pub graph_base_path: Option<String>,
pub batch_size: Option<usize>,
}
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index c177cf0e..1ecee632 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -61,10 +61,15 @@ fn download_files() {
fn create_webgraph() -> Result<()> {
debug!("Creating webgraph");
- let out_path = Path::new(DATA_PATH).join("webgraph");
+ let out_path_host = Path::new(DATA_PATH).join("webgraph_host");
+ let out_path_page = Path::new(DATA_PATH).join("webgraph_page");
- if out_path.exists() {
- std::fs::remove_dir_all(&out_path)?;
+ if out_path_host.exists() {
+ std::fs::remove_dir_all(&out_path_host)?;
+ }
+
+ if out_path_page.exists() {
+ std::fs::remove_dir_all(&out_path_page)?;
}
let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");
@@ -75,13 +80,12 @@ fn create_webgraph() -> Result<()> {
names: vec![warc_path.to_str().unwrap().to_string()],
}),
warc_paths: vec![warc_path.to_str().unwrap().to_string()],
- graph_base_path: out_path.to_str().unwrap().to_string(),
- level: crate::config::WebgraphLevel::Host,
};
let mut worker = webgraph::WebgraphWorker {
redirect: None,
- graph: webgraph::open_graph(&job.graph_base_path),
+ host_graph: webgraph::open_graph(&out_path_host),
+ page_graph: webgraph::open_graph(&out_path_page),
};
worker.process_job(&job);
@@ -91,7 +95,7 @@ fn create_webgraph() -> Result<()> {
fn calculate_centrality() {
debug!("Calculating centrality");
- let webgraph_path = Path::new(DATA_PATH).join("webgraph");
+ let webgraph_path = Path::new(DATA_PATH).join("webgraph_host");
let out_path = Path::new(DATA_PATH).join("centrality");
Centrality::build_harmonic(&webgraph_path, &out_path);
@@ -123,7 +127,7 @@ fn create_inverted_index() -> Result<()> {
minimum_clean_words: None,
};
- let webgraph_path = Path::new(DATA_PATH).join("webgraph");
+ let webgraph_path = Path::new(DATA_PATH).join("webgraph_page");
let centrality_path = Path::new(DATA_PATH).join("centrality");
let worker = indexer::IndexingWorker::new(
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index 302ace83..1456e05d 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -15,7 +15,6 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
config::WarcSource,
- config::WebgraphLevel,
config::{self, WebgraphConstructConfig},
crawler::crawl_db::RedirectDb,
entrypoint::download_all_warc_files,
@@ -64,10 +63,8 @@ impl From<JobConfig> for config::WarcSource {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct Job {
- pub level: WebgraphLevel,
pub config: JobConfig,
pub warc_paths: Vec<String>,
- pub graph_base_path: String,
}
pub fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {
@@ -75,7 +72,8 @@ pub fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {
}
pub struct WebgraphWorker {
- pub graph: webgraph::Webgraph,
+ pub host_graph: webgraph::Webgraph,
+ pub page_graph: webgraph::Webgraph,
pub redirect: Option<Arc<RedirectDb>>,
}
@@ -130,18 +128,21 @@ impl WebgraphWorker {
let mut destination = Node::from(destination);
- if let WebgraphLevel::Host = job.level {
- source = source.into_host();
- destination = destination.into_host();
- }
+ self.page_graph
+ .insert(source.clone(), destination.clone(), link.text.clone());
+
+ source = source.into_host();
+ destination = destination.into_host();
- self.graph.insert(source, destination, link.text);
+ self.host_graph.insert(source, destination, link.text);
}
}
- self.graph.commit();
+ self.host_graph.commit();
+ self.page_graph.commit();
}
- self.graph.merge_segments(1);
+ self.host_graph.merge_segments(1);
+ self.page_graph.merge_segments(1);
info!("{} done", name);
}
@@ -169,31 +170,29 @@ impl Webgraph {
.into_iter()
.map(|warc_paths| Job {
config: job_config.clone(),
- level: config.level.clone(),
warc_paths: warc_paths.collect_vec(),
- graph_base_path: config
- .graph_base_path
- .clone()
- .unwrap_or_else(|| "data/webgraph".to_string()),
})
.collect_vec();
let num_workers = num_cpus::get();
let mut handlers = Vec::new();
- let parent_path = config
- .graph_base_path
- .clone()
- .unwrap_or_else(|| "data/webgraph".to_string());
+ let host_path = &config.host_graph_base_path;
+ let page_path = &config.page_graph_base_path;
for i in 0..num_workers {
- let path = parent_path.clone();
- let path = Path::new(&path);
- let path = path.join(format!("worker_{i}"));
+ let host_path = host_path.clone();
+ let host_path = Path::new(&host_path);
+ let host_path = host_path.join(format!("worker_{i}"));
+
+ let page_path = page_path.clone();
+ let page_path = Path::new(&page_path);
+ let page_path = page_path.join(format!("worker_{i}"));
let mut worker = WebgraphWorker {
redirect: redirect.clone(),
- graph: open_graph(path),
+ host_graph: open_graph(host_path),
+ page_graph: open_graph(page_path),
};
let jobs = jobs.clone();
@@ -202,7 +201,7 @@ impl Webgraph {
worker.process_job(job);
}
- worker.graph
+ (worker.host_graph, worker.page_graph)
}));
}
@@ -211,11 +210,17 @@ impl Webgraph {
graphs.push(handler.join().unwrap());
}
- let mut graph = graphs.pop().unwrap();
- for other_graph in graphs {
- let other_path = other_graph.path.clone();
- graph.merge(other_graph);
- fs::remove_dir_all(other_path)?;
+ let (mut host_graph, mut page_graph) = graphs.pop().unwrap();
+
+ for (other_host, other_page) in graphs {
+ let other_host_path = other_host.path.clone();
+ let other_page_path = other_page.path.clone();
+
+ host_graph.merge(other_host);
+ page_graph.merge(other_page);
+
+ fs::remove_dir_all(other_host_path)?;
+ fs::remove_dir_all(other_page_path)?;
}
Ok(())