Skip to content

web search done right

reduce indexer memory by closing indexes

Mikkel Denker2f40f7fc9a24
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index a9bfff1a..fd12a988 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -1,8 +1,7 @@
warc_paths_file = "warc.paths"
centrality_store_path = "./data/centrality"
-limit_warc_files = 25
+limit_warc_files = 3
download_images = false
-batch_size = 3
[warc_source]
# type = "Local"
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index cc095839..c0070087 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -73,97 +73,110 @@ impl IndexingWorker {
}
}
-impl Worker for IndexingWorker {}
-
-impl Map<IndexingWorker, FrozenIndex> for Job {
- fn map(self, worker: &IndexingWorker) -> FrozenIndex {
- let name = self.warc_paths.first().unwrap().split('/').last().unwrap();
-
- info!("processing {}", name);
+fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
+ let name = job.warc_paths.first().unwrap().split('/').last().unwrap();
+
+ info!("processing {}", name);
+
+ let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();
+
+ let source = match job.source_config.clone() {
+ JobConfig::Http(config) => WarcSource::HTTP(config),
+ JobConfig::Local(config) => WarcSource::Local(config),
+ };
+
+ let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.base_path);
+
+ for file in warc_files {
+ if let Ok(file) = WarcFile::open(&file) {
+ for record in
+ file.records()
+ .flatten()
+ .filter(|record| match &record.response.payload_type {
+ Some(payload_type) => !matches!(payload_type.as_str(), "application/pdf"),
+ None => true,
+ })
+ {
+ let html = Html::parse(&record.response.body, &record.request.url);
+ let backlinks: Vec<Link> = worker
+ .webgraph
+ .as_ref()
+ .map(|webgraph| {
+ webgraph
+ .ingoing_edges(Node::from(html.url()))
+ .into_iter()
+ .map(|edge| Link {
+ source: edge.from.name.into(),
+ destination: edge.to.name.into(),
+ text: edge.label,
+ })
+ .collect()
+ })
+ .unwrap_or_else(Vec::new);
+
+ let host_centrality = worker
+ .host_centrality_store
+ .get(html.url().host_without_specific_subdomains())
+ .unwrap_or_default();
+
+ let page_centrality = worker
+ .page_centrality_store
+ .get(html.url().raw())
+ .unwrap_or_default();
+
+ let fetch_time_ms = record.metadata.fetch_time_ms as u64;
+
+ trace!("inserting webpage: {:?}", html.url());
+
+ trace!("title = {:?}", html.title());
+ trace!("text = {:?}", html.clean_text());
+
+ let webpage = Webpage {
+ html,
+ backlinks,
+ page_centrality,
+ host_centrality,
+ fetch_time_ms,
+ primary_image: None,
+ };
+ if let Err(err) = index.insert(webpage) {
+ debug!("{:?}", err);
+ }
+ }
+ if job.download_images {
+ info!("downloading images");
+ index.download_pending_images();
+ }
+ }
- let mut index = Index::open(Path::new(&self.base_path).join(name)).unwrap();
+ std::fs::remove_file(file).ok();
+ }
+ index.commit().unwrap();
- let source = match self.source_config {
- JobConfig::Http(config) => WarcSource::HTTP(config),
- JobConfig::Local(config) => WarcSource::Local(config),
- };
+ info!("{} done", name);
- let warc_files = download_all_warc_files(&self.warc_paths, &source, &self.base_path);
-
- for file in warc_files {
- if let Ok(file) = WarcFile::open(&file) {
- for record in
- file.records()
- .flatten()
- .filter(|record| match &record.response.payload_type {
- Some(payload_type) => {
- !matches!(payload_type.as_str(), "application/pdf")
- }
- None => true,
- })
- {
- let html = Html::parse(&record.response.body, &record.request.url);
- let backlinks: Vec<Link> = worker
- .webgraph
- .as_ref()
- .map(|webgraph| {
- webgraph
- .ingoing_edges(Node::from(html.url()))
- .into_iter()
- .map(|edge| Link {
- source: edge.from.name.into(),
- destination: edge.to.name.into(),
- text: edge.label,
- })
- .collect()
- })
- .unwrap_or_else(Vec::new);
-
- let host_centrality = worker
- .host_centrality_store
- .get(html.url().host_without_specific_subdomains())
- .unwrap_or_default();
-
- let page_centrality = worker
- .page_centrality_store
- .get(html.url().raw())
- .unwrap_or_default();
-
- let fetch_time_ms = record.metadata.fetch_time_ms as u64;
-
- trace!("inserting webpage: {:?}", html.url());
-
- trace!("title = {:?}", html.title());
- trace!("text = {:?}", html.clean_text());
-
- let webpage = Webpage {
- html,
- backlinks,
- page_centrality,
- host_centrality,
- fetch_time_ms,
- primary_image: None,
- };
- if let Err(err) = index.insert(webpage) {
- debug!("{:?}", err);
- }
- }
- if self.download_images {
- info!("downloading images");
- index.download_pending_images();
- }
- }
+ index
+}
- std::fs::remove_file(file).ok();
- }
- index.commit().unwrap();
+#[derive(Debug, Serialize, Deserialize)]
+struct IndexPointer(String);
- info!("{} done", name);
+impl Worker for IndexingWorker {}
+impl Map<IndexingWorker, FrozenIndex> for Job {
+ fn map(self, worker: &IndexingWorker) -> FrozenIndex {
+ let index = process_job(&self, worker);
index.into()
}
}
+impl Map<IndexingWorker, IndexPointer> for Job {
+ fn map(self, worker: &IndexingWorker) -> IndexPointer {
+ let index = process_job(&self, worker);
+ IndexPointer(index.path)
+ }
+}
+
impl Reduce<FrozenIndex> for Index {
fn reduce(self, element: FrozenIndex) -> Self {
let other: Index = element.into();
@@ -178,6 +191,20 @@ impl Reduce<FrozenIndex> for Index {
}
}
+impl Reduce<IndexPointer> for IndexPointer {
+ fn reduce(self, element: IndexPointer) -> Self {
+ let index = Index::open(self.0).unwrap();
+ let other_path = element.0;
+ let other = Index::open(&other_path).unwrap();
+
+ let res = index.merge(other);
+
+ std::fs::remove_dir_all(other_path).unwrap();
+
+ IndexPointer(res.path)
+ }
+}
+
impl Reduce<Index> for Index {
fn reduce(self, element: Index) -> Self {
let other = element;
@@ -230,9 +257,13 @@ impl Indexer {
warc_paths = Box::new(warc_paths.take(limit));
}
- let mut index: Index = warc_paths
- .map_reduce(&workers)
- .expect("failed to build index");
+ let mut index = <Box<dyn Iterator<Item = Job> + std::marker::Send> as MapReduce<
+ IndexingWorker,
+ Job,
+ FrozenIndex,
+ Index,
+ >>::map_reduce(warc_paths, &workers)
+ .expect("failed to build index");
index
.inverted_index
@@ -267,7 +298,7 @@ impl Indexer {
config.webgraph_path.clone(),
);
- let mut index = warc_paths
+ let index = warc_paths
.into_iter()
.take(config.limit_warc_files.unwrap_or(usize::MAX))
.chunks(config.batch_size.unwrap_or(1))
@@ -290,8 +321,7 @@ impl Indexer {
// });
.into_par_iter()
.panic_fuse()
- .map(|job| job.map(&worker))
- .map(Index::from)
+ .map(|job| -> IndexPointer { job.map(&worker) })
.map(Some)
.reduce(
|| None,
@@ -302,7 +332,9 @@ impl Indexer {
},
);
- if let Some(index) = index.as_mut() {
+ if let Some(pointer) = index {
+ let mut index = Index::open(pointer.0).unwrap();
+
index
.inverted_index
.merge_into_segments(config.final_num_segments.unwrap_or(20))?;
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index 4c15faa9..928aa02b 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -29,7 +29,7 @@ use std::{net::SocketAddr, path::Path};
use tracing::{info, trace};
#[derive(Debug, Serialize, Deserialize)]
-struct GraphPath(String);
+struct GraphPointer(String);
#[derive(Debug, Serialize, Deserialize, Clone)]
enum JobConfig {
@@ -103,10 +103,10 @@ impl Map<StatelessWorker, FrozenWebgraph> for Job {
}
}
-impl Map<StatelessWorker, GraphPath> for Job {
- fn map(self, _worker: &StatelessWorker) -> GraphPath {
+impl Map<StatelessWorker, GraphPointer> for Job {
+ fn map(self, _worker: &StatelessWorker) -> GraphPointer {
let graph = process_job(&self);
- GraphPath(graph.path)
+ GraphPointer(graph.path)
}
}
@@ -139,8 +139,8 @@ impl Reduce<webgraph::Webgraph> for webgraph::Webgraph {
}
}
-impl Reduce<GraphPath> for GraphPath {
- fn reduce(self, other: GraphPath) -> Self {
+impl Reduce<GraphPointer> for GraphPointer {
+ fn reduce(self, other: GraphPointer) -> Self {
let other_path = other.0.clone();
let self_path = self.0.clone();
@@ -153,7 +153,7 @@ impl Reduce<GraphPath> for GraphPath {
std::fs::remove_dir_all(other_path).unwrap();
- GraphPath(self_path)
+ GraphPointer(self_path)
}
}
@@ -242,7 +242,7 @@ impl Webgraph {
})
.collect_vec()
.into_par_iter()
- .map(|job| -> GraphPath { job.map(&worker) })
+ .map(|job| -> GraphPointer { job.map(&worker) })
.map(Some)
.reduce(
|| None,