web search done right
Remove old webgraph files when merged from CLI and merge into 1/2 num_segments during commit to make use of parallelism
Mikkel Denker7f40cac3c2a5
diff --git a/core/src/main.rs b/core/src/main.rsindex 51d6d485..4ca73b97 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -201,8 +201,9 @@ fn main() -> Result<()> {let mut webgraph = WebgraphBuilder::new(paths.remove(0)).open();for other_path in paths {- let other = WebgraphBuilder::new(other_path).open();+ let other = WebgraphBuilder::new(&other_path).open();webgraph.merge(other);+ std::fs::remove_dir_all(other_path).unwrap();}if let Some(num_segments) = num_segments {@@ -216,12 +217,7 @@ fn main() -> Result<()> {tokio::runtime::Builder::new_multi_thread().enable_all().build()?- .block_on(frontend::run(- &config.queries_csv_path,- &config.crossencoder_model_path,- &config.host,- config.search_servers,- ))?+ .block_on(frontend::run(config))?}Commands::SearchServer { config_path } => {let config: SearchServerConfig = load_toml_config(config_path);diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex b0a648d6..1eab9a9f 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -36,7 +36,7 @@ mod executor;use self::executor::Executor;use self::segment::{CachedStore, SegmentNodeID};-const MAX_NUM_SEGMENTS: usize = 200;+const MAX_NUM_SEGMENTS: usize = 400;#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]pub struct NodeID(pub u64);@@ -461,7 +461,9 @@ impl<S: Store> Webgraph<S> {self.node2id.flush();self.id2node.flush();- self.merge_segments(MAX_NUM_SEGMENTS);+ if self.segments.len() > MAX_NUM_SEGMENTS {+ self.merge_segments((MAX_NUM_SEGMENTS / 2).max(1));+ }}pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {