Skip to content

web search done right

Optional download images during indexing

Mikkel Denker49d71ee0dbbc
diff --git a/README.md b/README.md
index 3085cde3..ac90fca1 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ Cuely is an open source web search engine hosted at [cuely.io](https://cuely.io/
* Keyword search that respects your search query.
* Advanced query syntax (`site:`, `intitle:` etc.).
* Rank webpages based on their [harmonic centrality](https://en.wikipedia.org/wiki/Centrality#Harmonic_centrality)
-* DDG-like [!bang syntax](https://duckduckgo.com/bang)
+* DDG-style [!bang syntax](https://duckduckgo.com/bang)
* Entity sidebar
* De-rank websites with third-party trackers
* Prioritize fresh content
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 58fc05b1..8446cfee 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -42,7 +42,8 @@ enum JobConfig {
#[derive(Debug, Serialize, Deserialize)]
struct Job {
- config: JobConfig,
+ source_config: JobConfig,
+ download_images: bool,
warc_paths: Vec<String>,
base_path: String,
}
@@ -77,7 +78,7 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
let mut index = Index::open(Path::new(&self.base_path).join(name)).unwrap();
- let source = match self.config {
+ let source = match self.source_config {
JobConfig::Http(config) => WarcSource::HTTP(config),
JobConfig::Local(config) => WarcSource::Local(config),
};
@@ -135,8 +136,10 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
}
}
index.commit().unwrap();
- info!("downloading images");
- index.download_pending_images();
+ if self.download_images {
+ info!("downloading images");
+ index.download_pending_images();
+ }
}
std::fs::remove_file(file).ok();
@@ -198,8 +201,9 @@ impl Indexer {
.chunks(config.batch_size.unwrap_or(1))
.into_iter()
.map(|warc_paths| Job {
- config: job_config.clone(),
+ source_config: job_config.clone(),
warc_paths: warc_paths.collect_vec(),
+ download_images: config.download_images.unwrap_or(true),
base_path: config
.index_base_path
.clone()
@@ -252,8 +256,9 @@ impl Indexer {
.chunks(config.batch_size.unwrap_or(1))
.into_iter()
.map(|warc_paths| Job {
- config: job_config.clone(),
+ source_config: job_config.clone(),
warc_paths: warc_paths.collect_vec(),
+ download_images: config.download_images.unwrap_or(true),
base_path: config
.output_path
.clone()
diff --git a/src/lib.rs b/src/lib.rs
index 590f8731..1fa5191b 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -60,6 +60,7 @@ pub struct IndexingMasterConfig {
warc_source: WarcSource,
workers: Vec<String>,
batch_size: Option<usize>,
+ download_images: Option<bool>,
index_base_path: Option<String>,
}
@@ -70,6 +71,7 @@ pub struct IndexingLocalConfig {
batch_size: Option<usize>,
webgraph_path: Option<String>,
output_path: Option<String>,
+ download_images: Option<bool>,
centrality_store_path: String,
}