web search done right
reduce indexer memory by closing indexes
Mikkel Denker2f40f7fc9a24
diff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex a9bfff1a..fd12a988 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -1,8 +1,7 @@warc_paths_file = "warc.paths"centrality_store_path = "./data/centrality"-limit_warc_files = 25+limit_warc_files = 3download_images = false-batch_size = 3[warc_source]# type = "Local"diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex cc095839..c0070087 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -73,97 +73,110 @@ impl IndexingWorker {}}-impl Worker for IndexingWorker {}--impl Map<IndexingWorker, FrozenIndex> for Job {- fn map(self, worker: &IndexingWorker) -> FrozenIndex {- let name = self.warc_paths.first().unwrap().split('/').last().unwrap();-- info!("processing {}", name);+fn process_job(job: &Job, worker: &IndexingWorker) -> Index {+ let name = job.warc_paths.first().unwrap().split('/').last().unwrap();++ info!("processing {}", name);++ let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();++ let source = match job.source_config.clone() {+ JobConfig::Http(config) => WarcSource::HTTP(config),+ JobConfig::Local(config) => WarcSource::Local(config),+ };++ let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.base_path);++ for file in warc_files {+ if let Ok(file) = WarcFile::open(&file) {+ for record in+ file.records()+ .flatten()+ .filter(|record| match &record.response.payload_type {+ Some(payload_type) => !matches!(payload_type.as_str(), "application/pdf"),+ None => true,+ })+ {+ let html = Html::parse(&record.response.body, &record.request.url);+ let backlinks: Vec<Link> = worker+ .webgraph+ .as_ref()+ .map(|webgraph| {+ webgraph+ .ingoing_edges(Node::from(html.url()))+ .into_iter()+ .map(|edge| Link {+ source: edge.from.name.into(),+ destination: edge.to.name.into(),+ text: edge.label,+ })+ .collect()+ })+ .unwrap_or_else(Vec::new);++ let host_centrality = worker+ .host_centrality_store+ .get(html.url().host_without_specific_subdomains())+ .unwrap_or_default();++ let page_centrality = worker+ .page_centrality_store+ .get(html.url().raw())+ .unwrap_or_default();++ let fetch_time_ms = record.metadata.fetch_time_ms as u64;++ trace!("inserting webpage: {:?}", html.url());++ trace!("title = {:?}", html.title());+ trace!("text = {:?}", html.clean_text());++ let webpage = Webpage {+ html,+ backlinks,+ page_centrality,+ host_centrality,+ fetch_time_ms,+ primary_image: None,+ };+ if let Err(err) = index.insert(webpage) {+ debug!("{:?}", err);+ }+ }+ if job.download_images {+ info!("downloading images");+ index.download_pending_images();+ }+ }- let mut index = Index::open(Path::new(&self.base_path).join(name)).unwrap();+ std::fs::remove_file(file).ok();+ }+ index.commit().unwrap();- let source = match self.source_config {- JobConfig::Http(config) => WarcSource::HTTP(config),- JobConfig::Local(config) => WarcSource::Local(config),- };+ info!("{} done", name);- let warc_files = download_all_warc_files(&self.warc_paths, &source, &self.base_path);-- for file in warc_files {- if let Ok(file) = WarcFile::open(&file) {- for record in- file.records()- .flatten()- .filter(|record| match &record.response.payload_type {- Some(payload_type) => {- !matches!(payload_type.as_str(), "application/pdf")- }- None => true,- })- {- let html = Html::parse(&record.response.body, &record.request.url);- let backlinks: Vec<Link> = worker- .webgraph- .as_ref()- .map(|webgraph| {- webgraph- .ingoing_edges(Node::from(html.url()))- .into_iter()- .map(|edge| Link {- source: edge.from.name.into(),- destination: edge.to.name.into(),- text: edge.label,- })- .collect()- })- .unwrap_or_else(Vec::new);-- let host_centrality = worker- .host_centrality_store- .get(html.url().host_without_specific_subdomains())- .unwrap_or_default();-- let page_centrality = worker- .page_centrality_store- .get(html.url().raw())- .unwrap_or_default();-- let fetch_time_ms = record.metadata.fetch_time_ms as u64;-- trace!("inserting webpage: {:?}", html.url());-- trace!("title = {:?}", html.title());- trace!("text = {:?}", html.clean_text());-- let webpage = Webpage {- html,- backlinks,- page_centrality,- host_centrality,- fetch_time_ms,- primary_image: None,- };- if let Err(err) = index.insert(webpage) {- debug!("{:?}", err);- }- }- if self.download_images {- info!("downloading images");- index.download_pending_images();- }- }+ index+}- std::fs::remove_file(file).ok();- }- index.commit().unwrap();+#[derive(Debug, Serialize, Deserialize)]+struct IndexPointer(String);- info!("{} done", name);+impl Worker for IndexingWorker {}+impl Map<IndexingWorker, FrozenIndex> for Job {+ fn map(self, worker: &IndexingWorker) -> FrozenIndex {+ let index = process_job(&self, worker);index.into()}}+impl Map<IndexingWorker, IndexPointer> for Job {+ fn map(self, worker: &IndexingWorker) -> IndexPointer {+ let index = process_job(&self, worker);+ IndexPointer(index.path)+ }+}+impl Reduce<FrozenIndex> for Index {fn reduce(self, element: FrozenIndex) -> Self {let other: Index = element.into();@@ -178,6 +191,20 @@ impl Reduce<FrozenIndex> for Index {}}+impl Reduce<IndexPointer> for IndexPointer {+ fn reduce(self, element: IndexPointer) -> Self {+ let index = Index::open(self.0).unwrap();+ let other_path = element.0;+ let other = Index::open(&other_path).unwrap();++ let res = index.merge(other);++ std::fs::remove_dir_all(other_path).unwrap();++ IndexPointer(res.path)+ }+}+impl Reduce<Index> for Index {fn reduce(self, element: Index) -> Self {let other = element;@@ -230,9 +257,13 @@ impl Indexer {warc_paths = Box::new(warc_paths.take(limit));}- let mut index: Index = warc_paths- .map_reduce(&workers)- .expect("failed to build index");+ let mut index = <Box<dyn Iterator<Item = Job> + std::marker::Send> as MapReduce<+ IndexingWorker,+ Job,+ FrozenIndex,+ Index,+ >>::map_reduce(warc_paths, &workers)+ .expect("failed to build index");index.inverted_index@@ -267,7 +298,7 @@ impl Indexer {config.webgraph_path.clone(),);- let mut index = warc_paths+ let index = warc_paths.into_iter().take(config.limit_warc_files.unwrap_or(usize::MAX)).chunks(config.batch_size.unwrap_or(1))@@ -290,8 +321,7 @@ impl Indexer {// });.into_par_iter().panic_fuse()- .map(|job| job.map(&worker))- .map(Index::from)+ .map(|job| -> IndexPointer { job.map(&worker) }).map(Some).reduce(|| None,@@ -302,7 +332,9 @@ impl Indexer {},);- if let Some(index) = index.as_mut() {+ if let Some(pointer) = index {+ let mut index = Index::open(pointer.0).unwrap();+index.inverted_index.merge_into_segments(config.final_num_segments.unwrap_or(20))?;diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 4c15faa9..928aa02b 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -29,7 +29,7 @@ use std::{net::SocketAddr, path::Path};use tracing::{info, trace};#[derive(Debug, Serialize, Deserialize)]-struct GraphPath(String);+struct GraphPointer(String);#[derive(Debug, Serialize, Deserialize, Clone)]enum JobConfig {@@ -103,10 +103,10 @@ impl Map<StatelessWorker, FrozenWebgraph> for Job {}}-impl Map<StatelessWorker, GraphPath> for Job {- fn map(self, _worker: &StatelessWorker) -> GraphPath {+impl Map<StatelessWorker, GraphPointer> for Job {+ fn map(self, _worker: &StatelessWorker) -> GraphPointer {let graph = process_job(&self);- GraphPath(graph.path)+ GraphPointer(graph.path)}}@@ -139,8 +139,8 @@ impl Reduce<webgraph::Webgraph> for webgraph::Webgraph {}}-impl Reduce<GraphPath> for GraphPath {- fn reduce(self, other: GraphPath) -> Self {+impl Reduce<GraphPointer> for GraphPointer {+ fn reduce(self, other: GraphPointer) -> Self {let other_path = other.0.clone();let self_path = self.0.clone();@@ -153,7 +153,7 @@ impl Reduce<GraphPath> for GraphPath {std::fs::remove_dir_all(other_path).unwrap();- GraphPath(self_path)+ GraphPointer(self_path)}}@@ -242,7 +242,7 @@ impl Webgraph {}).collect_vec().into_par_iter()- .map(|job| -> GraphPath { job.map(&worker) })+ .map(|job| -> GraphPointer { job.map(&worker) }).map(Some).reduce(|| None,