Skip to content

web search done right

move last worker folder into correct output location after indexing

Mikkel Denkerde5c946b580b
diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rs
index 07e80b43..076e2bd3 100644
--- a/crates/core/examples/search_preindexed.rs
+++ b/crates/core/examples/search_preindexed.rs
@@ -22,13 +22,13 @@ pub async fn main() {
};
let config = ApiConfig {
- queries_csv_path: "data/queries_us.csv".to_string(),
+ queries_csv_path: Some("data/queries_us.csv".to_string()),
host: "0.0.0.0:8000".parse().unwrap(),
prometheus_host: "0.0.0.0:8001".parse().unwrap(),
crossencoder_model_path: None,
lambda_model_path: None,
dual_encoder_model_path: None,
- bangs_path: "data/bangs.json".to_string(),
+ bangs_path: Some("data/bangs.json".to_string()),
query_store_db_host: None,
cluster_id: "api".to_string(),
gossip_seed_nodes: None,
@@ -47,10 +47,11 @@ pub async fn main() {
max_similar_hosts: defaults::Api::max_similar_hosts(),
};
- let mut queries = stract::autosuggest::Autosuggest::load_csv(&config.queries_csv_path)
- .unwrap()
- .all()
- .unwrap();
+ let mut queries =
+ stract::autosuggest::Autosuggest::load_csv(config.queries_csv_path.as_ref().unwrap())
+ .unwrap()
+ .all()
+ .unwrap();
queries.shuffle(&mut rand::thread_rng());
@@ -61,7 +62,7 @@ pub async fn main() {
max_considered_words: Some(10_000),
..Default::default()
});
- let bangs = Bangs::from_path(&config.bangs_path);
+ let bangs = Bangs::from_path(config.bangs_path.as_ref().unwrap());
let searcher = stract::searcher::LocalSearchClient::from(searcher);
diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rs
index 64f7c8a6..7daf6dc9 100644
--- a/crates/core/src/api/mod.rs
+++ b/crates/core/src/api/mod.rs
@@ -148,7 +148,10 @@ fn build_router(state: Arc<State>) -> Router {
}
pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {
- let autosuggest = Autosuggest::load_csv(&config.queries_csv_path)?;
+ let autosuggest = match &config.queries_csv_path {
+ Some(queries_csv_path) => Autosuggest::load_csv(queries_csv_path)?,
+ None => Autosuggest::empty(),
+ };
let lambda_model = match &config.lambda_model_path {
Some(path) => Some(LambdaMART::open(path)?),
@@ -166,7 +169,10 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {
query_store_queue
});
- let bangs = Bangs::from_path(&config.bangs_path);
+ let bangs = match &config.bangs_path {
+ Some(bangs_path) => Bangs::from_path(bangs_path),
+ None => Bangs::empty(),
+ };
let cluster = Arc::new(
Cluster::join(
diff --git a/crates/core/src/autosuggest.rs b/crates/core/src/autosuggest.rs
index b04520dd..72902535 100644
--- a/crates/core/src/autosuggest.rs
+++ b/crates/core/src/autosuggest.rs
@@ -47,6 +47,12 @@ impl Autosuggest {
Ok(Self { queries })
}
+ pub fn empty() -> Self {
+ Self {
+ queries: fst::Set::default(),
+ }
+ }
+
pub fn suggestions(&self, query: &str) -> Result<Vec<String>> {
let query = query.to_ascii_lowercase();
let q = Str::new(query.as_str()).starts_with();
diff --git a/crates/core/src/bangs.rs b/crates/core/src/bangs.rs
index fb366299..bd97a8df 100644
--- a/crates/core/src/bangs.rs
+++ b/crates/core/src/bangs.rs
@@ -122,7 +122,6 @@ pub struct Bangs {
}
impl Bangs {
- #[cfg(test)]
pub fn empty() -> Self {
Self {
bangs: HashMap::new(),
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 39c005d1..7e388ab5 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -225,13 +225,13 @@ pub struct ApiSpellCheck {
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct ApiConfig {
- pub queries_csv_path: String,
+ pub queries_csv_path: Option<String>,
pub host: SocketAddr,
pub prometheus_host: SocketAddr,
pub crossencoder_model_path: Option<String>,
pub lambda_model_path: Option<String>,
pub dual_encoder_model_path: Option<String>,
- pub bangs_path: String,
+ pub bangs_path: Option<String>,
pub query_store_db_host: Option<String>,
pub cluster_id: String,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
@@ -245,6 +245,7 @@ pub struct ApiConfig {
#[serde(default)]
pub thresholds: ApiThresholds,
+ #[serde(default)]
pub widgets: WidgetsConfig,
#[serde(default)]
diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rs
index bcb7857b..af03e0a4 100644
--- a/crates/core/src/entrypoint/configure.rs
+++ b/crates/core/src/entrypoint/configure.rs
@@ -155,11 +155,7 @@ fn calculate_centrality() {
fn create_inverted_index() -> Result<()> {
debug!("Creating inverted index");
let out_path = Path::new(DATA_PATH).join("index");
- let out_path_tmp = Path::new(DATA_PATH).join("index_tmp");
- if out_path_tmp.exists() {
- std::fs::remove_dir_all(&out_path_tmp)?;
- }
if out_path.exists() {
std::fs::remove_dir_all(&out_path)?;
}
@@ -172,7 +168,7 @@ fn create_inverted_index() -> Result<()> {
names: vec![warc_path.to_str().unwrap().to_string()],
}),
warc_path: warc_path.to_str().unwrap().to_string(),
- base_path: out_path_tmp.to_str().unwrap().to_string(),
+ base_path: out_path.to_str().unwrap().to_string(),
settings: JobSettings {
host_centrality_threshold: None,
minimum_clean_words: None,
@@ -199,7 +195,7 @@ fn create_inverted_index() -> Result<()> {
.unwrap()
.to_string(),
),
- output_path: out_path_tmp.to_str().unwrap().to_string(),
+ output_path: out_path.to_str().unwrap().to_string(),
limit_warc_files: None,
skip_warc_files: None,
warc_source: job.source_config.clone(),
@@ -215,8 +211,7 @@ fn create_inverted_index() -> Result<()> {
});
let index = job.process(&worker);
- std::fs::rename(index.path, out_path)?;
- std::fs::remove_dir_all(&out_path_tmp)?;
+ crate::mv(index.path(), &out_path)?;
Ok(())
}
diff --git a/crates/core/src/entrypoint/indexer/mod.rs b/crates/core/src/entrypoint/indexer/mod.rs
index 068c8f7c..d1e3ef88 100644
--- a/crates/core/src/entrypoint/indexer/mod.rs
+++ b/crates/core/src/entrypoint/indexer/mod.rs
@@ -62,14 +62,16 @@ pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {
autocommit_after_num_inserts: config.autocommit_after_num_inserts,
},
})
- .map(|job| IndexPointer(job.process(&worker).path))
+ .map(|job| IndexPointer(job.process(&worker).path().as_os_str().to_str().unwrap().to_string()))
.collect();
- merge(indexes)?;
+ let index = merge(indexes)?;
+ crate::mv(index.path(), &config.output_path)?;
+
Ok(())
}
-pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {
+pub fn merge(indexes: Vec<IndexPointer>) -> Result<Index> {
let num_indexes = indexes.len();
let mut it = indexes.into_iter();
let num_cores = usize::from(std::thread::available_parallelism()?);
@@ -114,12 +116,12 @@ pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {
let mut index = it.next().unwrap();
for other in it {
- let other_path = other.path.clone();
+ let other_path = other.path();
index = index.merge(other);
std::fs::remove_dir_all(other_path).unwrap();
}
index.inverted_index.merge_into_max_segments(1).unwrap();
- Ok(())
+ Ok(index)
}
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index e9f75cad..7713f405 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -276,6 +276,9 @@ impl Webgraph {
host_graph.optimize_read();
page_graph.optimize_read();
+ crate::mv(host_graph.path(), &config.host_graph_base_path)?;
+ crate::mv(page_graph.path(), &config.page_graph_base_path)?;
+
Ok(())
}
}
diff --git a/crates/core/src/index.rs b/crates/core/src/index.rs
index fc54185b..a4f77253 100644
--- a/crates/core/src/index.rs
+++ b/crates/core/src/index.rs
@@ -16,7 +16,7 @@
use std::collections::HashSet;
use std::fs;
-use std::path::Path;
+use std::path::{Path, PathBuf};
use std::sync::Mutex;
use std::time::SystemTime;
@@ -37,7 +37,7 @@ const REGION_COUNT_FILE_NAME: &str = "region_count.json";
pub struct Index {
pub inverted_index: InvertedIndex,
pub region_count: Mutex<RegionCount>,
- pub path: String,
+ path: String,
}
impl Index {
@@ -58,6 +58,10 @@ impl Index {
})
}
+ pub fn path(&self) -> PathBuf {
+ PathBuf::from(&self.path)
+ }
+
pub fn set_auto_merge_policy(&mut self) {
self.inverted_index.set_auto_merge_policy();
}
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 10e474a8..44dc1d28 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -401,3 +401,28 @@ where
top_k.truncate(k);
top_k
}
+
+/// Recursively move a file or directory to a new location.
+/// Intended to be similar to the `mv` command in Unix.
+pub fn mv<P1: AsRef<std::path::Path>, P2: AsRef<std::path::Path>>(
+ from: P1,
+ to: P2,
+) -> std::io::Result<()> {
+ let from = from.as_ref();
+ let to = to.as_ref();
+
+ if from.is_dir() {
+ std::fs::create_dir_all(to)?;
+ for entry in std::fs::read_dir(from)? {
+ let entry = entry?;
+ let new_from = entry.path();
+ let new_to = to.join(entry.file_name());
+ mv(new_from, new_to)?;
+ }
+ std::fs::remove_dir(from)?;
+ } else {
+ std::fs::rename(from, to)?;
+ }
+
+ Ok(())
+}
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 120f5a0e..47fcde1d 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::fs::File;
use std::io::{BufReader, BufWriter, Read, Write};
-use std::path::Path;
+use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::{fs, io};
@@ -122,6 +122,10 @@ impl Webgraph {
WebgraphBuilder::new(path)
}
+ pub fn path(&self) -> PathBuf {
+ PathBuf::from(&self.path)
+ }
+
fn meta<P: AsRef<Path>>(path: P) -> Meta {
let meta_path = path.as_ref().join("metadata.json");
Meta::open(meta_path)