Skip to content

web search done right

internet archive warc files does not seem to store the payload type. let's just assume it's html (records that can't be parsed are skipped anyway)

Mikkel Denkeraed64be27e2c
diff --git a/configs/indexer/create.toml b/configs/indexer/create.toml
index 8cc9656d..6ccc4021 100644
--- a/configs/indexer/create.toml
+++ b/configs/indexer/create.toml
@@ -1,5 +1,6 @@
host_centrality_store_path = "./data/centrality"
limit_warc_files = 20
+output_path = "./data/index"
# minimum_clean_words = 40
[warc_source]
diff --git a/configs/webgraph/create.toml b/configs/webgraph/create.toml
index 781ef943..ee3b4726 100644
--- a/configs/webgraph/create.toml
+++ b/configs/webgraph/create.toml
@@ -4,6 +4,6 @@ page_graph_base_path = "data/webgraph_page"
batch_size = 1
[warc_source]
-type = "HTTP"
-base_url = "https://data.commoncrawl.org/"
-warc_paths_file = "data/warc.paths"
+folder = "./data"
+names = ["sample.warc.gz"]
+type = "Local"
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 822b11a4..bcf5caf9 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -26,12 +26,12 @@ use std::net::SocketAddr;
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingLocalConfig {
+ pub output_path: String,
pub limit_warc_files: Option<usize>,
pub skip_warc_files: Option<usize>,
pub warc_source: WarcSource,
pub batch_size: Option<usize>,
pub page_webgraph_path: Option<String>,
- pub output_path: Option<String>,
pub host_centrality_threshold: Option<f64>,
pub topics_path: Option<String>,
pub host_centrality_store_path: String,
diff --git a/crates/core/src/entrypoint/indexer.rs b/crates/core/src/entrypoint/indexer.rs
index 05b4d811..d63b5f9a 100644
--- a/crates/core/src/entrypoint/indexer.rs
+++ b/crates/core/src/entrypoint/indexer.rs
@@ -277,7 +277,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
.flatten()
.filter(|record| match &record.response.payload_type {
Some(payload_type) => matches!(payload_type, PayloadType::Html),
- None => false,
+ None => true,
})
{
if let Ok(webpage) = worker.prepare_webpage(
@@ -387,7 +387,7 @@ impl Indexer {
.map(|warc_paths| Job {
source_config: job_config.clone(),
warc_paths,
- base_path: config.output_path.clone().expect("output path not set"),
+ base_path: config.output_path.clone(),
settings: JobSettings {
host_centrality_threshold: config.host_centrality_threshold,
minimum_clean_words: config.minimum_clean_words,
diff --git a/crates/core/src/warc.rs b/crates/core/src/warc.rs
index 9f627d9a..44336fa8 100644
--- a/crates/core/src/warc.rs
+++ b/crates/core/src/warc.rs
@@ -727,21 +727,23 @@ mod tests {
#[test]
fn internet_archive_parse() {
- if !Path::new("../../../data/internet_archive.warc.gz").exists() {
+ if !Path::new("../../data/internet_archive.warc.gz").exists() {
return;
}
- let file = File::open("../../../data/internet_archive.warc.gz").unwrap();
- let mut reader = BufReader::new(MultiGzDecoder::new(BufReader::new(file)));
+ let mut records = 0;
- let mut bytes = Vec::new();
- reader.read_to_end(&mut bytes).unwrap();
-
- for record in WarcFile::new(bytes).records() {
+ for record in WarcFile::open("../../data/internet_archive.warc.gz")
+ .unwrap()
+ .records()
+ {
+ records += 1;
if let Err(err) = record {
panic!("Error: {:?}", err);
}
}
+
+ assert!(records > 0);
}
#[test]