web search done right
forgot to remove old index after merge
Mikkel Denker29ee8edfebf1
diff --git a/configs/frontend.toml b/configs/frontend.tomlindex 47f15ee4..223e24e3 100644--- a/configs/frontend.toml+++ b/configs/frontend.toml@@ -2,7 +2,6 @@ cluster_id = "dev_frontend"queries_csv_path = "data/queries_us.csv"host = "0.0.0.0:3000"prometheus_host = "0.0.0.0:3001"-search_servers = [["0.0.0.0:3002"]]gossip_addr = "0.0.0.0:3003"gossip_seed_nodes = ["0.0.0.0:3004"]crossencoder_model_path = "data/cross_encoder"diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex f1c15e37..4ee0162a 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -438,7 +438,10 @@ impl Indexer {let mut threads = Vec::new();for _ in 0..(num_cores + 1) {- let indexes = it.by_ref().take(num_indexes / num_cores).collect_vec();+ let indexes = it+ .by_ref()+ .take(((num_indexes as f64) / (num_cores as f64)).ceil() as usize)+ .collect_vec();if indexes.is_empty() {break;@@ -468,8 +471,11 @@ impl Indexer {let mut it = indexes.into_iter();let mut index = it.next().unwrap();+for other in it {+ let other_path = other.path.clone();index = index.merge(other);+ std::fs::remove_dir_all(other_path).unwrap();}Ok(())diff --git a/core/src/lib.rs b/core/src/lib.rsindex 1b66f0db..2f21f1a8 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -171,7 +171,6 @@ pub struct FrontendConfig {pub qa_model_path: Option<String>,pub bangs_path: String,pub summarizer_path: String,- pub search_servers: Vec<Vec<String>>,pub query_store_db_host: Option<String>,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,