Skip to content

web search done right

store atom and rss feeds during crawl

Mikkel Denkere786bc0bdb61
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index 6e898121..aae8416b 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -20,7 +20,7 @@ use hashbrown::HashMap;
use url::Url;
-use crate::{config::CrawlerConfig, webpage::url_ext::UrlExt};
+use crate::{config::CrawlerConfig, warc, webpage::url_ext::UrlExt};
use self::{warc_writer::WarcWriter, worker::WorkerThread};
@@ -206,7 +206,7 @@ impl From<Job> for WorkerJob {
pub struct CrawlDatum {
url: Url,
status_code: u16,
- headers: HashMap<String, String>,
+ payload_type: warc::PayloadType,
body: String,
fetch_time_ms: u64,
}
diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rs
index 4d49fb72..a2b06e54 100644
--- a/core/src/crawler/warc_writer.rs
+++ b/core/src/crawler/warc_writer.rs
@@ -88,25 +88,13 @@ async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3:
rayon::scope(move |s| {
s.spawn(move |_| {
- let is_pdf = datum
- .headers
- .get("content-type")
- .map(|ct| ct.contains("application/pdf"))
- .unwrap_or(false);
-
- let payload = if is_pdf {
- Some("application/pdf".to_string())
- } else {
- None
- };
-
let warc_record = warc::WarcRecord {
request: warc::Request {
url: datum.url.to_string(),
},
response: warc::Response {
body: datum.body,
- payload_type: payload,
+ payload_type: Some(datum.payload_type),
},
metadata: warc::Metadata {
fetch_time_ms: datum.fetch_time_ms as usize,
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 3f796aa2..0e67ad9c 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -35,6 +35,7 @@ use crate::{
crawler::MAX_URL_LEN_BYTES,
distributed::{retry_strategy::ExponentialBackoff, sonic},
entrypoint::crawler::router::{NewJob, RouterService},
+ warc,
webpage::Html,
};
@@ -472,11 +473,12 @@ impl JobExecutor {
}
// check if content type is html
- if let Some(content_type) = headers.get("content-type") {
- if !content_type.contains("text/html") {
- return Err(Error::InvalidContentType(content_type.to_string()).into());
- }
- }
+ let payload_type = match headers.get("content-type") {
+ Some(ct) if ct.contains("text/html") => warc::PayloadType::Html,
+ Some(ct) if ct.contains("application/rss") => warc::PayloadType::Rss,
+ Some(ct) if ct.contains("application/atom") => warc::PayloadType::Atom,
+ ct => return Err(Error::InvalidContentType(format!("{ct:?}")).into()),
+ };
let status_code = res.status().as_u16();
@@ -495,7 +497,7 @@ impl JobExecutor {
return Ok(CrawlDatum {
url,
status_code,
- headers,
+ payload_type,
body: String::new(),
fetch_time_ms: fetch_time.as_millis() as u64,
});
@@ -537,8 +539,8 @@ impl JobExecutor {
Ok(CrawlDatum {
url: res_url,
status_code,
- headers,
body,
+ payload_type,
fetch_time_ms: fetch_time.as_millis() as u64,
})
}
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 0104e1e4..feb7b224 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -30,6 +30,7 @@ use crate::kv::rocksdb_store::RocksDbStore;
use crate::kv::Kv;
use crate::mapreduce::{Map, Reduce, Worker};
use crate::ranking::SignalAggregator;
+use crate::warc::PayloadType;
use crate::webgraph::{Node, NodeID, Webgraph, WebgraphBuilder};
use crate::webpage::{safety_classifier, Html, Webpage};
use crate::{human_website_annotations, Result};
@@ -125,8 +126,8 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
file.records()
.flatten()
.filter(|record| match &record.response.payload_type {
- Some(payload_type) => !matches!(payload_type.as_str(), "application/pdf"),
- None => true,
+ Some(payload_type) => matches!(payload_type, PayloadType::Html),
+ None => false,
})
{
let mut html =
diff --git a/core/src/warc.rs b/core/src/warc.rs
index d8bcea83..06af44eb 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -20,6 +20,7 @@ use std::collections::BTreeMap;
use std::fs::File;
use std::io::{BufRead, BufReader, Cursor, Read, Seek, Write};
use std::path::Path;
+use std::str::FromStr;
use std::thread::sleep;
use std::time::Duration;
@@ -234,11 +235,45 @@ impl Request {
}
}
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[cfg_attr(test, derive(Arbitrary))]
+pub enum PayloadType {
+ Html,
+ Pdf,
+ Rss,
+ Atom,
+}
+
+impl FromStr for PayloadType {
+ type Err = Error;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s {
+ "application/html" => Ok(Self::Html),
+ "application/pdf" => Ok(Self::Pdf),
+ "application/rss" => Ok(Self::Rss),
+ "application/atom" => Ok(Self::Atom),
+ _ => Err(Error::WarcParse("Unknown payload type")),
+ }
+ }
+}
+
+impl ToString for PayloadType {
+ fn to_string(&self) -> String {
+ match self {
+ Self::Html => "application/html".to_string(),
+ Self::Pdf => "application/pdf".to_string(),
+ Self::Rss => "application/rss".to_string(),
+ Self::Atom => "application/atom".to_string(),
+ }
+ }
+}
+
#[derive(Debug)]
#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]
pub struct Response {
pub body: String,
- pub payload_type: Option<String>,
+ pub payload_type: Option<PayloadType>,
}
impl Response {
@@ -251,7 +286,10 @@ impl Response {
Ok(Self {
body: content.to_string(),
- payload_type: record.header.get("WARC-IDENTIFIED-PAYLOAD-TYPE").cloned(),
+ payload_type: record
+ .header
+ .get("WARC-IDENTIFIED-PAYLOAD-TYPE")
+ .and_then(|p| PayloadType::from_str(p).ok()),
})
}
}
@@ -528,7 +566,11 @@ impl WarcWriter {
if let Some(payload_type) = &record.response.payload_type {
self.writer.write_all(
- format!("WARC-Identified-Payload-Type: {}\r\n", payload_type).as_bytes(),
+ format!(
+ "WARC-Identified-Payload-Type: {}\r\n",
+ payload_type.to_string()
+ )
+ .as_bytes(),
)?;
}
@@ -645,7 +687,7 @@ mod tests {
},
response: Response {
body: "body of a".to_string(),
- payload_type: Some("text/html".to_string()),
+ payload_type: Some(PayloadType::Html),
},
metadata: Metadata {
fetch_time_ms: 1337,
@@ -695,7 +737,7 @@ mod tests {
},
response: Response {
body: utf8.to_string(),
- payload_type: Some("text/html".to_string()),
+ payload_type: Some(PayloadType::Html),
},
metadata: Metadata { fetch_time_ms: 0 },
};
@@ -727,7 +769,7 @@ mod tests {
},
response: Response {
body: body.to_string(),
- payload_type: Some("text/html".to_string()),
+ payload_type: Some(PayloadType::Html),
},
metadata: Metadata { fetch_time_ms: 0 },
};