web search done right
Optional download images during indexing
Mikkel Denker49d71ee0dbbc
diff --git a/README.md b/README.mdindex 3085cde3..ac90fca1 100644--- a/README.md+++ b/README.md@@ -16,7 +16,7 @@ Cuely is an open source web search engine hosted at [cuely.io](https://cuely.io/* Keyword search that respects your search query.* Advanced query syntax (`site:`, `intitle:` etc.).* Rank webpages based on their [harmonic centrality](https://en.wikipedia.org/wiki/Centrality#Harmonic_centrality)-* DDG-like [!bang syntax](https://duckduckgo.com/bang)+* DDG-style [!bang syntax](https://duckduckgo.com/bang)* Entity sidebar* De-rank websites with third-party trackers* Prioritize fresh contentdiff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 58fc05b1..8446cfee 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -42,7 +42,8 @@ enum JobConfig {#[derive(Debug, Serialize, Deserialize)]struct Job {- config: JobConfig,+ source_config: JobConfig,+ download_images: bool,warc_paths: Vec<String>,base_path: String,}@@ -77,7 +78,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {let mut index = Index::open(Path::new(&self.base_path).join(name)).unwrap();- let source = match self.config {+ let source = match self.source_config {JobConfig::Http(config) => WarcSource::HTTP(config),JobConfig::Local(config) => WarcSource::Local(config),};@@ -135,8 +136,10 @@ impl Map<IndexingWorker, FrozenIndex> for Job {}}index.commit().unwrap();- info!("downloading images");- index.download_pending_images();+ if self.download_images {+ info!("downloading images");+ index.download_pending_images();+ }}std::fs::remove_file(file).ok();@@ -198,8 +201,9 @@ impl Indexer {.chunks(config.batch_size.unwrap_or(1)).into_iter().map(|warc_paths| Job {- config: job_config.clone(),+ source_config: job_config.clone(),warc_paths: warc_paths.collect_vec(),+ download_images: config.download_images.unwrap_or(true),base_path: config.index_base_path.clone()@@ -252,8 +256,9 @@ impl Indexer {.chunks(config.batch_size.unwrap_or(1)).into_iter().map(|warc_paths| Job {- config: job_config.clone(),+ source_config: job_config.clone(),warc_paths: warc_paths.collect_vec(),+ download_images: config.download_images.unwrap_or(true),base_path: config.output_path.clone()diff --git a/src/lib.rs b/src/lib.rsindex 590f8731..1fa5191b 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -60,6 +60,7 @@ pub struct IndexingMasterConfig {warc_source: WarcSource,workers: Vec<String>,batch_size: Option<usize>,+ download_images: Option<bool>,index_base_path: Option<String>,}@@ -70,6 +71,7 @@ pub struct IndexingLocalConfig {batch_size: Option<usize>,webgraph_path: Option<String>,output_path: Option<String>,+ download_images: Option<bool>,centrality_store_path: String,}