web search done right
ability to skip warc files in webgraph construction
Mikkel Denkerc52653abd032
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex ef96315d..659b8fae 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -46,6 +46,7 @@ pub struct WebgraphConstructConfig {pub page_graph_base_path: String,pub warc_source: WarcSource,pub limit_warc_files: Option<usize>,+ pub skip_warc_files: Option<usize>,pub batch_size: Option<usize>,}diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 1544d1ca..cd1d325a 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -160,6 +160,7 @@ impl Webgraph {let jobs: Vec<_> = warc_paths.into_iter()+ .skip(config.skip_warc_files.unwrap_or(0)).take(config.limit_warc_files.unwrap_or(usize::MAX)).chunks(config.batch_size.unwrap_or(1)).into_iter()diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 1061fc1e..dd1729a5 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -459,8 +459,9 @@ impl EdgeStore {if let Some(last_node) = last_node {if (reversed && edge.to.id != last_node) || (!reversed && edge.from.id != last_node){- batch- .sort_by_key(|e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id });+ batch.sort_unstable_by_key(+ |e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id },+ );batch.dedup_by_key(|e| if reversed { e.from.id } else { e.to.id });s.put(&batch);batch.clear();@@ -472,7 +473,9 @@ impl EdgeStore {}if !batch.is_empty() {- batch.sort_by_key(|e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id });+ batch.sort_unstable_by_key(+ |e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id },+ );batch.dedup_by_key(|e| if reversed { e.from.id } else { e.to.id });s.put(&batch);}