Skip to content

web search done right

forgot to remove old index after merge

Mikkel Denker29ee8edfebf1
diff --git a/configs/frontend.toml b/configs/frontend.toml
index 47f15ee4..223e24e3 100644
--- a/configs/frontend.toml
+++ b/configs/frontend.toml
@@ -2,7 +2,6 @@ cluster_id = "dev_frontend"
queries_csv_path = "data/queries_us.csv"
host = "0.0.0.0:3000"
prometheus_host = "0.0.0.0:3001"
-search_servers = [["0.0.0.0:3002"]]
gossip_addr = "0.0.0.0:3003"
gossip_seed_nodes = ["0.0.0.0:3004"]
crossencoder_model_path = "data/cross_encoder"
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index f1c15e37..4ee0162a 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -438,7 +438,10 @@ impl Indexer {
let mut threads = Vec::new();
for _ in 0..(num_cores + 1) {
- let indexes = it.by_ref().take(num_indexes / num_cores).collect_vec();
+ let indexes = it
+ .by_ref()
+ .take(((num_indexes as f64) / (num_cores as f64)).ceil() as usize)
+ .collect_vec();
if indexes.is_empty() {
break;
@@ -468,8 +471,11 @@ impl Indexer {
let mut it = indexes.into_iter();
let mut index = it.next().unwrap();
+
for other in it {
+ let other_path = other.path.clone();
index = index.merge(other);
+ std::fs::remove_dir_all(other_path).unwrap();
}
Ok(())
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 1b66f0db..2f21f1a8 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -171,7 +171,6 @@ pub struct FrontendConfig {
pub qa_model_path: Option<String>,
pub bangs_path: String,
pub summarizer_path: String,
- pub search_servers: Vec<Vec<String>>,
pub query_store_db_host: Option<String>,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,