Skip to content

web search done right

merge indexes from cli

Mikkel Denkercfabb5a453c9
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 673b05ed..3b742674 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -205,7 +205,13 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
}
#[derive(Debug, Serialize, Deserialize)]
-struct IndexPointer(String);
+pub struct IndexPointer(String);
+
+impl From<String> for IndexPointer {
+ fn from(path: String) -> Self {
+ IndexPointer(path)
+ }
+}
impl Worker for IndexingWorker {}
@@ -237,20 +243,6 @@ impl Reduce<FrozenIndex> for Index {
}
}
-impl Reduce<IndexPointer> for IndexPointer {
- fn reduce(self, element: IndexPointer) -> Self {
- let index = Index::open(self.0).unwrap();
- let other_path = element.0;
- let other = Index::open(&other_path).unwrap();
-
- let res = index.merge(other);
-
- std::fs::remove_dir_all(other_path).unwrap();
-
- IndexPointer(res.path)
- }
-}
-
impl Reduce<Index> for Index {
fn reduce(self, element: Index) -> Self {
let other = element;
@@ -359,7 +351,7 @@ impl Indexer {
config.webgraph_path.clone(),
);
- let index = warc_paths
+ let indexes: Vec<_> = warc_paths
.into_iter()
.take(config.limit_warc_files.unwrap_or(usize::MAX))
.chunks(config.batch_size.unwrap_or(1))
@@ -375,33 +367,26 @@ impl Indexer {
.unwrap_or_else(|| "data/index".to_string()),
})
.collect_vec()
- // .into_iter()
- // .map(|job| job.map(&worker))
- // .fold(None, |acc: Option<Index>, elem: FrozenIndex| match acc {
- // Some(acc) => Some(acc.reduce(elem)),
- // None => Some(elem.into()),
- // });
.into_par_iter()
.panic_fuse()
.map(|job| -> IndexPointer { job.map(&worker) })
- .map(Some)
- .reduce(
- || None,
- |a, b| match (a, b) {
- (Some(a), Some(b)) => Some(a.reduce(b)),
- (Some(graph), None) | (None, Some(graph)) => Some(graph),
- (None, None) => None,
- },
- );
-
- if let Some(pointer) = index {
- let mut index = Index::open(pointer.0).unwrap();
-
- index
- .inverted_index
- .merge_into_segments(config.final_num_segments.unwrap_or(20))?;
+ .collect();
+
+ Self::merge(indexes, config.final_num_segments.unwrap_or(20))?;
+ Ok(())
+ }
+
+ pub fn merge(indexes: Vec<IndexPointer>, num_segments: u32) -> Result<()> {
+ let mut it = indexes.into_iter();
+ let mut index = Index::open(it.next().unwrap().0)?;
+
+ for other in it {
+ let other = Index::open(other.0)?;
+ index = index.merge(other);
}
+ index.inverted_index.merge_into_segments(num_segments)?;
+
Ok(())
}
}
diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rs
index c642ccb0..9ab62d1f 100644
--- a/src/entrypoint/mod.rs
+++ b/src/entrypoint/mod.rs
@@ -16,7 +16,7 @@
mod centrality;
mod entity;
pub mod frontend;
-mod indexer;
+pub mod indexer;
pub mod search_server;
mod webgraph;
diff --git a/src/main.rs b/src/main.rs
index 03865e9a..aa0dfaab 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::Result;
use clap::{Parser, Subcommand};
+use cuely::entrypoint::indexer::IndexPointer;
use cuely::entrypoint::{self, frontend, search_server};
use cuely::{FrontendConfig, SearchServerConfig};
use serde::de::DeserializeOwned;
@@ -77,6 +78,10 @@ enum IndexingOptions {
wikipedia_dump_path: String,
output_path: String,
},
+ Merge {
+ num_segments: u32,
+ indexes: Vec<String>,
+ },
}
fn load_toml_config<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> T {
@@ -113,6 +118,13 @@ fn main() -> Result<()> {
wikipedia_dump_path,
output_path,
} => entrypoint::EntityIndexer::run(wikipedia_dump_path, output_path)?,
+ IndexingOptions::Merge {
+ num_segments,
+ indexes,
+ } => entrypoint::Indexer::merge(
+ indexes.into_iter().map(IndexPointer::from).collect(),
+ num_segments,
+ )?,
},
Commands::Centrality {
webgraph_path,