Skip to content

web search done right

Remove old webgraph files when merged from CLI and merge into 1/2 num_segments during commit to make use of parallelism

Mikkel Denker7f40cac3c2a5
diff --git a/core/src/main.rs b/core/src/main.rs
index 51d6d485..4ca73b97 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -201,8 +201,9 @@ fn main() -> Result<()> {
let mut webgraph = WebgraphBuilder::new(paths.remove(0)).open();
for other_path in paths {
- let other = WebgraphBuilder::new(other_path).open();
+ let other = WebgraphBuilder::new(&other_path).open();
webgraph.merge(other);
+ std::fs::remove_dir_all(other_path).unwrap();
}
if let Some(num_segments) = num_segments {
@@ -216,12 +217,7 @@ fn main() -> Result<()> {
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()?
- .block_on(frontend::run(
- &config.queries_csv_path,
- &config.crossencoder_model_path,
- &config.host,
- config.search_servers,
- ))?
+ .block_on(frontend::run(config))?
}
Commands::SearchServer { config_path } => {
let config: SearchServerConfig = load_toml_config(config_path);
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index b0a648d6..1eab9a9f 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -36,7 +36,7 @@ mod executor;
use self::executor::Executor;
use self::segment::{CachedStore, SegmentNodeID};
-const MAX_NUM_SEGMENTS: usize = 200;
+const MAX_NUM_SEGMENTS: usize = 400;
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct NodeID(pub u64);
@@ -461,7 +461,9 @@ impl<S: Store> Webgraph<S> {
self.node2id.flush();
self.id2node.flush();
- self.merge_segments(MAX_NUM_SEGMENTS);
+ if self.segments.len() > MAX_NUM_SEGMENTS {
+ self.merge_segments((MAX_NUM_SEGMENTS / 2).max(1));
+ }
}
pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {