web search done right
move last worker folder into correct output location after indexing
Mikkel Denkerde5c946b580b
diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rsindex 07e80b43..076e2bd3 100644--- a/crates/core/examples/search_preindexed.rs+++ b/crates/core/examples/search_preindexed.rs@@ -22,13 +22,13 @@ pub async fn main() {};let config = ApiConfig {- queries_csv_path: "data/queries_us.csv".to_string(),+ queries_csv_path: Some("data/queries_us.csv".to_string()),host: "0.0.0.0:8000".parse().unwrap(),prometheus_host: "0.0.0.0:8001".parse().unwrap(),crossencoder_model_path: None,lambda_model_path: None,dual_encoder_model_path: None,- bangs_path: "data/bangs.json".to_string(),+ bangs_path: Some("data/bangs.json".to_string()),query_store_db_host: None,cluster_id: "api".to_string(),gossip_seed_nodes: None,@@ -47,10 +47,11 @@ pub async fn main() {max_similar_hosts: defaults::Api::max_similar_hosts(),};- let mut queries = stract::autosuggest::Autosuggest::load_csv(&config.queries_csv_path)- .unwrap()- .all()- .unwrap();+ let mut queries =+ stract::autosuggest::Autosuggest::load_csv(config.queries_csv_path.as_ref().unwrap())+ .unwrap()+ .all()+ .unwrap();queries.shuffle(&mut rand::thread_rng());@@ -61,7 +62,7 @@ pub async fn main() {max_considered_words: Some(10_000),..Default::default()});- let bangs = Bangs::from_path(&config.bangs_path);+ let bangs = Bangs::from_path(config.bangs_path.as_ref().unwrap());let searcher = stract::searcher::LocalSearchClient::from(searcher);diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rsindex 64f7c8a6..7daf6dc9 100644--- a/crates/core/src/api/mod.rs+++ b/crates/core/src/api/mod.rs@@ -148,7 +148,10 @@ fn build_router(state: Arc<State>) -> Router {}pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {- let autosuggest = Autosuggest::load_csv(&config.queries_csv_path)?;+ let autosuggest = match &config.queries_csv_path {+ Some(queries_csv_path) => Autosuggest::load_csv(queries_csv_path)?,+ None => Autosuggest::empty(),+ };let lambda_model = match &config.lambda_model_path {Some(path) => Some(LambdaMART::open(path)?),@@ -166,7 +169,10 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {query_store_queue});- let bangs = Bangs::from_path(&config.bangs_path);+ let bangs = match &config.bangs_path {+ Some(bangs_path) => Bangs::from_path(bangs_path),+ None => Bangs::empty(),+ };let cluster = Arc::new(Cluster::join(diff --git a/crates/core/src/autosuggest.rs b/crates/core/src/autosuggest.rsindex b04520dd..72902535 100644--- a/crates/core/src/autosuggest.rs+++ b/crates/core/src/autosuggest.rs@@ -47,6 +47,12 @@ impl Autosuggest {Ok(Self { queries })}+ pub fn empty() -> Self {+ Self {+ queries: fst::Set::default(),+ }+ }+pub fn suggestions(&self, query: &str) -> Result<Vec<String>> {let query = query.to_ascii_lowercase();let q = Str::new(query.as_str()).starts_with();diff --git a/crates/core/src/bangs.rs b/crates/core/src/bangs.rsindex fb366299..bd97a8df 100644--- a/crates/core/src/bangs.rs+++ b/crates/core/src/bangs.rs@@ -122,7 +122,6 @@ pub struct Bangs {}impl Bangs {- #[cfg(test)]pub fn empty() -> Self {Self {bangs: HashMap::new(),diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 39c005d1..7e388ab5 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -225,13 +225,13 @@ pub struct ApiSpellCheck {#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct ApiConfig {- pub queries_csv_path: String,+ pub queries_csv_path: Option<String>,pub host: SocketAddr,pub prometheus_host: SocketAddr,pub crossencoder_model_path: Option<String>,pub lambda_model_path: Option<String>,pub dual_encoder_model_path: Option<String>,- pub bangs_path: String,+ pub bangs_path: Option<String>,pub query_store_db_host: Option<String>,pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,@@ -245,6 +245,7 @@ pub struct ApiConfig {#[serde(default)]pub thresholds: ApiThresholds,+ #[serde(default)]pub widgets: WidgetsConfig,#[serde(default)]diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rsindex bcb7857b..af03e0a4 100644--- a/crates/core/src/entrypoint/configure.rs+++ b/crates/core/src/entrypoint/configure.rs@@ -155,11 +155,7 @@ fn calculate_centrality() {fn create_inverted_index() -> Result<()> {debug!("Creating inverted index");let out_path = Path::new(DATA_PATH).join("index");- let out_path_tmp = Path::new(DATA_PATH).join("index_tmp");- if out_path_tmp.exists() {- std::fs::remove_dir_all(&out_path_tmp)?;- }if out_path.exists() {std::fs::remove_dir_all(&out_path)?;}@@ -172,7 +168,7 @@ fn create_inverted_index() -> Result<()> {names: vec![warc_path.to_str().unwrap().to_string()],}),warc_path: warc_path.to_str().unwrap().to_string(),- base_path: out_path_tmp.to_str().unwrap().to_string(),+ base_path: out_path.to_str().unwrap().to_string(),settings: JobSettings {host_centrality_threshold: None,minimum_clean_words: None,@@ -199,7 +195,7 @@ fn create_inverted_index() -> Result<()> {.unwrap().to_string(),),- output_path: out_path_tmp.to_str().unwrap().to_string(),+ output_path: out_path.to_str().unwrap().to_string(),limit_warc_files: None,skip_warc_files: None,warc_source: job.source_config.clone(),@@ -215,8 +211,7 @@ fn create_inverted_index() -> Result<()> {});let index = job.process(&worker);- std::fs::rename(index.path, out_path)?;- std::fs::remove_dir_all(&out_path_tmp)?;+ crate::mv(index.path(), &out_path)?;Ok(())}diff --git a/crates/core/src/entrypoint/indexer/mod.rs b/crates/core/src/entrypoint/indexer/mod.rsindex 068c8f7c..d1e3ef88 100644--- a/crates/core/src/entrypoint/indexer/mod.rs+++ b/crates/core/src/entrypoint/indexer/mod.rs@@ -62,14 +62,16 @@ pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {autocommit_after_num_inserts: config.autocommit_after_num_inserts,},})- .map(|job| IndexPointer(job.process(&worker).path))+ .map(|job| IndexPointer(job.process(&worker).path().as_os_str().to_str().unwrap().to_string())).collect();- merge(indexes)?;+ let index = merge(indexes)?;+ crate::mv(index.path(), &config.output_path)?;+Ok(())}-pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {+pub fn merge(indexes: Vec<IndexPointer>) -> Result<Index> {let num_indexes = indexes.len();let mut it = indexes.into_iter();let num_cores = usize::from(std::thread::available_parallelism()?);@@ -114,12 +116,12 @@ pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {let mut index = it.next().unwrap();for other in it {- let other_path = other.path.clone();+ let other_path = other.path();index = index.merge(other);std::fs::remove_dir_all(other_path).unwrap();}index.inverted_index.merge_into_max_segments(1).unwrap();- Ok(())+ Ok(index)}diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex e9f75cad..7713f405 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -276,6 +276,9 @@ impl Webgraph {host_graph.optimize_read();page_graph.optimize_read();+ crate::mv(host_graph.path(), &config.host_graph_base_path)?;+ crate::mv(page_graph.path(), &config.page_graph_base_path)?;+Ok(())}}diff --git a/crates/core/src/index.rs b/crates/core/src/index.rsindex fc54185b..a4f77253 100644--- a/crates/core/src/index.rs+++ b/crates/core/src/index.rs@@ -16,7 +16,7 @@use std::collections::HashSet;use std::fs;-use std::path::Path;+use std::path::{Path, PathBuf};use std::sync::Mutex;use std::time::SystemTime;@@ -37,7 +37,7 @@ const REGION_COUNT_FILE_NAME: &str = "region_count.json";pub struct Index {pub inverted_index: InvertedIndex,pub region_count: Mutex<RegionCount>,- pub path: String,+ path: String,}impl Index {@@ -58,6 +58,10 @@ impl Index {})}+ pub fn path(&self) -> PathBuf {+ PathBuf::from(&self.path)+ }+pub fn set_auto_merge_policy(&mut self) {self.inverted_index.set_auto_merge_policy();}diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 10e474a8..44dc1d28 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -401,3 +401,28 @@ wheretop_k.truncate(k);top_k}++/// Recursively move a file or directory to a new location.+/// Intended to be similar to the `mv` command in Unix.+pub fn mv<P1: AsRef<std::path::Path>, P2: AsRef<std::path::Path>>(+ from: P1,+ to: P2,+) -> std::io::Result<()> {+ let from = from.as_ref();+ let to = to.as_ref();++ if from.is_dir() {+ std::fs::create_dir_all(to)?;+ for entry in std::fs::read_dir(from)? {+ let entry = entry?;+ let new_from = entry.path();+ let new_to = to.join(entry.file_name());+ mv(new_from, new_to)?;+ }+ std::fs::remove_dir(from)?;+ } else {+ std::fs::rename(from, to)?;+ }++ Ok(())+}diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 120f5a0e..47fcde1d 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use std::fs::File;use std::io::{BufReader, BufWriter, Read, Write};-use std::path::Path;+use std::path::{Path, PathBuf};use std::sync::Arc;use std::{fs, io};@@ -122,6 +122,10 @@ impl Webgraph {WebgraphBuilder::new(path)}+ pub fn path(&self) -> PathBuf {+ PathBuf::from(&self.path)+ }+fn meta<P: AsRef<Path>>(path: P) -> Meta {let meta_path = path.as_ref().join("metadata.json");Meta::open(meta_path)