web search done right
store atom and rss feeds during crawl
Mikkel Denkere786bc0bdb61
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex 6e898121..aae8416b 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -20,7 +20,7 @@ use hashbrown::HashMap;use url::Url;-use crate::{config::CrawlerConfig, webpage::url_ext::UrlExt};+use crate::{config::CrawlerConfig, warc, webpage::url_ext::UrlExt};use self::{warc_writer::WarcWriter, worker::WorkerThread};@@ -206,7 +206,7 @@ impl From<Job> for WorkerJob {pub struct CrawlDatum {url: Url,status_code: u16,- headers: HashMap<String, String>,+ payload_type: warc::PayloadType,body: String,fetch_time_ms: u64,}diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rsindex 4d49fb72..a2b06e54 100644--- a/core/src/crawler/warc_writer.rs+++ b/core/src/crawler/warc_writer.rs@@ -88,25 +88,13 @@ async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3:rayon::scope(move |s| {s.spawn(move |_| {- let is_pdf = datum- .headers- .get("content-type")- .map(|ct| ct.contains("application/pdf"))- .unwrap_or(false);-- let payload = if is_pdf {- Some("application/pdf".to_string())- } else {- None- };-let warc_record = warc::WarcRecord {request: warc::Request {url: datum.url.to_string(),},response: warc::Response {body: datum.body,- payload_type: payload,+ payload_type: Some(datum.payload_type),},metadata: warc::Metadata {fetch_time_ms: datum.fetch_time_ms as usize,diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 3f796aa2..0e67ad9c 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -35,6 +35,7 @@ use crate::{crawler::MAX_URL_LEN_BYTES,distributed::{retry_strategy::ExponentialBackoff, sonic},entrypoint::crawler::router::{NewJob, RouterService},+ warc,webpage::Html,};@@ -472,11 +473,12 @@ impl JobExecutor {}// check if content type is html- if let Some(content_type) = headers.get("content-type") {- if !content_type.contains("text/html") {- return Err(Error::InvalidContentType(content_type.to_string()).into());- }- }+ let payload_type = match headers.get("content-type") {+ Some(ct) if ct.contains("text/html") => warc::PayloadType::Html,+ Some(ct) if ct.contains("application/rss") => warc::PayloadType::Rss,+ Some(ct) if ct.contains("application/atom") => warc::PayloadType::Atom,+ ct => return Err(Error::InvalidContentType(format!("{ct:?}")).into()),+ };let status_code = res.status().as_u16();@@ -495,7 +497,7 @@ impl JobExecutor {return Ok(CrawlDatum {url,status_code,- headers,+ payload_type,body: String::new(),fetch_time_ms: fetch_time.as_millis() as u64,});@@ -537,8 +539,8 @@ impl JobExecutor {Ok(CrawlDatum {url: res_url,status_code,- headers,body,+ payload_type,fetch_time_ms: fetch_time.as_millis() as u64,})}diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 0104e1e4..feb7b224 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -30,6 +30,7 @@ use crate::kv::rocksdb_store::RocksDbStore;use crate::kv::Kv;use crate::mapreduce::{Map, Reduce, Worker};use crate::ranking::SignalAggregator;+use crate::warc::PayloadType;use crate::webgraph::{Node, NodeID, Webgraph, WebgraphBuilder};use crate::webpage::{safety_classifier, Html, Webpage};use crate::{human_website_annotations, Result};@@ -125,8 +126,8 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {file.records().flatten().filter(|record| match &record.response.payload_type {- Some(payload_type) => !matches!(payload_type.as_str(), "application/pdf"),- None => true,+ Some(payload_type) => matches!(payload_type, PayloadType::Html),+ None => false,}){let mut html =diff --git a/core/src/warc.rs b/core/src/warc.rsindex d8bcea83..06af44eb 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -20,6 +20,7 @@ use std::collections::BTreeMap;use std::fs::File;use std::io::{BufRead, BufReader, Cursor, Read, Seek, Write};use std::path::Path;+use std::str::FromStr;use std::thread::sleep;use std::time::Duration;@@ -234,11 +235,45 @@ impl Request {}}+#[derive(Debug, Clone, PartialEq, Eq)]+#[cfg_attr(test, derive(Arbitrary))]+pub enum PayloadType {+ Html,+ Pdf,+ Rss,+ Atom,+}++impl FromStr for PayloadType {+ type Err = Error;++ fn from_str(s: &str) -> Result<Self, Self::Err> {+ match s {+ "application/html" => Ok(Self::Html),+ "application/pdf" => Ok(Self::Pdf),+ "application/rss" => Ok(Self::Rss),+ "application/atom" => Ok(Self::Atom),+ _ => Err(Error::WarcParse("Unknown payload type")),+ }+ }+}++impl ToString for PayloadType {+ fn to_string(&self) -> String {+ match self {+ Self::Html => "application/html".to_string(),+ Self::Pdf => "application/pdf".to_string(),+ Self::Rss => "application/rss".to_string(),+ Self::Atom => "application/atom".to_string(),+ }+ }+}+#[derive(Debug)]#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]pub struct Response {pub body: String,- pub payload_type: Option<String>,+ pub payload_type: Option<PayloadType>,}impl Response {@@ -251,7 +286,10 @@ impl Response {Ok(Self {body: content.to_string(),- payload_type: record.header.get("WARC-IDENTIFIED-PAYLOAD-TYPE").cloned(),+ payload_type: record+ .header+ .get("WARC-IDENTIFIED-PAYLOAD-TYPE")+ .and_then(|p| PayloadType::from_str(p).ok()),})}}@@ -528,7 +566,11 @@ impl WarcWriter {if let Some(payload_type) = &record.response.payload_type {self.writer.write_all(- format!("WARC-Identified-Payload-Type: {}\r\n", payload_type).as_bytes(),+ format!(+ "WARC-Identified-Payload-Type: {}\r\n",+ payload_type.to_string()+ )+ .as_bytes(),)?;}@@ -645,7 +687,7 @@ mod tests {},response: Response {body: "body of a".to_string(),- payload_type: Some("text/html".to_string()),+ payload_type: Some(PayloadType::Html),},metadata: Metadata {fetch_time_ms: 1337,@@ -695,7 +737,7 @@ mod tests {},response: Response {body: utf8.to_string(),- payload_type: Some("text/html".to_string()),+ payload_type: Some(PayloadType::Html),},metadata: Metadata { fetch_time_ms: 0 },};@@ -727,7 +769,7 @@ mod tests {},response: Response {body: body.to_string(),- payload_type: Some("text/html".to_string()),+ payload_type: Some(PayloadType::Html),},metadata: Metadata { fetch_time_ms: 0 },};