web search done right
merge indexes from cli
Mikkel Denkercfabb5a453c9
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 673b05ed..3b742674 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -205,7 +205,13 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {}#[derive(Debug, Serialize, Deserialize)]-struct IndexPointer(String);+pub struct IndexPointer(String);++impl From<String> for IndexPointer {+ fn from(path: String) -> Self {+ IndexPointer(path)+ }+}impl Worker for IndexingWorker {}@@ -237,20 +243,6 @@ impl Reduce<FrozenIndex> for Index {}}-impl Reduce<IndexPointer> for IndexPointer {- fn reduce(self, element: IndexPointer) -> Self {- let index = Index::open(self.0).unwrap();- let other_path = element.0;- let other = Index::open(&other_path).unwrap();-- let res = index.merge(other);-- std::fs::remove_dir_all(other_path).unwrap();-- IndexPointer(res.path)- }-}-impl Reduce<Index> for Index {fn reduce(self, element: Index) -> Self {let other = element;@@ -359,7 +351,7 @@ impl Indexer {config.webgraph_path.clone(),);- let index = warc_paths+ let indexes: Vec<_> = warc_paths.into_iter().take(config.limit_warc_files.unwrap_or(usize::MAX)).chunks(config.batch_size.unwrap_or(1))@@ -375,33 +367,26 @@ impl Indexer {.unwrap_or_else(|| "data/index".to_string()),}).collect_vec()- // .into_iter()- // .map(|job| job.map(&worker))- // .fold(None, |acc: Option<Index>, elem: FrozenIndex| match acc {- // Some(acc) => Some(acc.reduce(elem)),- // None => Some(elem.into()),- // });.into_par_iter().panic_fuse().map(|job| -> IndexPointer { job.map(&worker) })- .map(Some)- .reduce(- || None,- |a, b| match (a, b) {- (Some(a), Some(b)) => Some(a.reduce(b)),- (Some(graph), None) | (None, Some(graph)) => Some(graph),- (None, None) => None,- },- );-- if let Some(pointer) = index {- let mut index = Index::open(pointer.0).unwrap();-- index- .inverted_index- .merge_into_segments(config.final_num_segments.unwrap_or(20))?;+ .collect();++ Self::merge(indexes, config.final_num_segments.unwrap_or(20))?;+ Ok(())+ }++ pub fn merge(indexes: Vec<IndexPointer>, num_segments: u32) -> Result<()> {+ let mut it = indexes.into_iter();+ let mut index = Index::open(it.next().unwrap().0)?;++ for other in it {+ let other = Index::open(other.0)?;+ index = index.merge(other);}+ index.inverted_index.merge_into_segments(num_segments)?;+Ok(())}}diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rsindex c642ccb0..9ab62d1f 100644--- a/src/entrypoint/mod.rs+++ b/src/entrypoint/mod.rs@@ -16,7 +16,7 @@mod centrality;mod entity;pub mod frontend;-mod indexer;+pub mod indexer;pub mod search_server;mod webgraph;diff --git a/src/main.rs b/src/main.rsindex 03865e9a..aa0dfaab 100644--- a/src/main.rs+++ b/src/main.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::Result;use clap::{Parser, Subcommand};+use cuely::entrypoint::indexer::IndexPointer;use cuely::entrypoint::{self, frontend, search_server};use cuely::{FrontendConfig, SearchServerConfig};use serde::de::DeserializeOwned;@@ -77,6 +78,10 @@ enum IndexingOptions {wikipedia_dump_path: String,output_path: String,},+ Merge {+ num_segments: u32,+ indexes: Vec<String>,+ },}fn load_toml_config<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> T {@@ -113,6 +118,13 @@ fn main() -> Result<()> {wikipedia_dump_path,output_path,} => entrypoint::EntityIndexer::run(wikipedia_dump_path, output_path)?,+ IndexingOptions::Merge {+ num_segments,+ indexes,+ } => entrypoint::Indexer::merge(+ indexes.into_iter().map(IndexPointer::from).collect(),+ num_segments,+ )?,},Commands::Centrality {webgraph_path,