Skip to content

web search done right

8.5 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

//! # Crawler
//!
//! The crawler is responsible for fetching webpages and storing them in WARC files
//! for later processing.
//!
//! Before starting a crawl, a plan needs to be created. This plan is then used by
//! the crawler coordinator to assign sites to crawl to different workers.
//! A site is only assigned to one worker at a time for politeness.

use std::{collections::VecDeque, future::Future, net::SocketAddr, sync::Arc};

type HashMap<K, V> = std::collections::HashMap<K, V, ahash::RandomState>;

use anyhow::anyhow;
use futures::StreamExt;
use robot_client::RobotClient;
use url::Url;

use crate::{config::CrawlerConfig, warc, webpage::url_ext::UrlExt};

use self::{warc_writer::WarcWriter, worker::WorkerThread};
pub use worker::JobExecutor;

pub mod coordinator;
pub mod robots_txt;
pub mod router;
pub use router::Router;
mod file_queue;
pub mod planner;
pub mod robot_client;
mod wander_prioritiser;
mod warc_writer;
mod worker;

pub use coordinator::CrawlCoordinator;

pub const MAX_URL_LEN_BYTES: usize = 8192;

pub const MAX_OUTGOING_URLS_PER_PAGE: usize = 512;
pub const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB

#[derive(thiserror::Error, Debug)]
pub enum Error {
    #[error("invalid content type: {0}")]
    InvalidContentType(String),

    #[error("fetch failed: {status_code}")]
    FetchFailed {
        status_code: reqwest::StatusCode,
        headers: reqwest::header::HeaderMap,
    },

    #[error("content too large")]
    ContentTooLarge,

    #[error("couldn't read response body")]
    ResponseBodyReadFailed,

    #[error("invalid politeness factor")]
    InvalidPolitenessFactor,

    #[error("invalid redirect")]
    InvalidRedirect,

    #[error("couldn't parse html")]
    InvalidHtml,

    #[error("request path is disallowed by robots.txt")]
    DisallowedPath,

    #[error("an error occurred: {0}")]
    Anyhow(#[from] anyhow::Error),
}

type Result<T, E = Error> = std::result::Result<T, E>;

#[derive(
    Clone,
    Debug,
    PartialEq,
    Eq,
    Hash,
    PartialOrd,
    Ord,
    serde::Serialize,
    serde::Deserialize,
    bincode::Encode,
    bincode::Decode,
)]
struct Site(String);

#[derive(
    Clone,
    Debug,
    PartialEq,
    Eq,
    PartialOrd,
    Ord,
    Hash,
    serde::Serialize,
    serde::Deserialize,
    bincode::Encode,
    bincode::Decode,
)]
pub struct Domain(String);

impl From<&Url> for Domain {
    fn from(url: &Url) -> Self {
        Self(url.icann_domain().unwrap_or_default().to_string())
    }
}

impl From<Url> for Domain {
    fn from(url: Url) -> Self {
        Self::from(&url)
    }
}

impl From<String> for Domain {
    fn from(s: String) -> Self {
        Self(s)
    }
}

impl Domain {
    pub fn as_str(&self) -> &str {
        &self.0
    }
}

#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
pub struct WeightedUrl {
    #[bincode(with_serde)]
    pub url: Url,
    pub weight: f64,
}

impl PartialEq for WeightedUrl {
    fn eq(&self, other: &Self) -> bool {
        self.url == other.url
    }
}

impl Eq for WeightedUrl {}

impl std::hash::Hash for WeightedUrl {
    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
        self.url.hash(state);
    }
}

/// All urls in a job must be from the same domain and only one job per domain.
/// at a time. This ensures that we stay polite when crawling.
#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
pub struct Job {
    pub domain: Domain,
    pub urls: VecDeque<WeightedUrl>,
    pub wandering_urls: u64,
}

#[derive(
    Debug,
    Clone,
    PartialEq,
    Eq,
    Hash,
    PartialOrd,
    Ord,
    serde::Serialize,
    serde::Deserialize,
    bincode::Encode,
    bincode::Decode,
)]
pub struct UrlString(String);

impl From<&Url> for UrlString {
    fn from(url: &Url) -> Self {
        Self(url.as_str().to_string())
    }
}

impl From<Url> for UrlString {
    fn from(url: Url) -> Self {
        Self(url.as_str().to_string())
    }
}

impl TryFrom<&UrlString> for Url {
    type Error = anyhow::Error;
    fn try_from(url: &UrlString) -> Result<Self, Self::Error> {
        Ok(Url::parse(&url.0)?)
    }
}

#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct UrlToInsert {
    pub url: UrlString,
    pub weight: f64,
}

#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct DiscoveredUrls {
    pub urls: HashMap<Domain, Vec<UrlToInsert>>,
}

#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct DomainCrawled {
    pub domain: Domain,
    pub budget_used: f64,
}

pub struct RetrieableUrl {
    weighted_url: WeightedUrl,
    retries: u8,
}

impl RetrieableUrl {
    pub fn url(&self) -> &Url {
        &self.weighted_url.url
    }
}

impl From<WeightedUrl> for RetrieableUrl {
    fn from(weighted_url: WeightedUrl) -> Self {
        Self {
            weighted_url,
            retries: 0,
        }
    }
}

pub struct WorkerJob {
    pub domain: Domain,
    pub urls: VecDeque<RetrieableUrl>,
    pub wandering_urls: u64,
}

impl From<Job> for WorkerJob {
    fn from(value: Job) -> Self {
        Self {
            domain: value.domain,
            urls: value.urls.into_iter().map(RetrieableUrl::from).collect(),
            wandering_urls: value.wandering_urls,
        }
    }
}

#[derive(Debug, Clone)]
pub struct CrawlDatum {
    pub url: Url,
    pub payload_type: warc::PayloadType,
    pub body: String,
    pub fetch_time_ms: u64,
    pub date: chrono::DateTime<chrono::Utc>,
}

pub struct Crawler {
    writer: Arc<WarcWriter>,
    handles: Vec<tokio::task::JoinHandle<()>>,
}

impl Crawler {
    pub async fn new(config: CrawlerConfig) -> Result<Self> {
        let writer = Arc::new(WarcWriter::new(config.s3.clone()));
        let mut handles = Vec::new();
        let mut router_hosts = Vec::new();
        let client = RobotClient::new(&config)?;

        for host in &config.router_hosts {
            router_hosts.push(
                host.parse::<SocketAddr>()
                    .map_err(|e| Error::from(anyhow!(e)))?,
            );
        }

        for _ in 0..config.num_worker_threads {
            let worker = WorkerThread::new(
                Arc::clone(&writer),
                client.clone(),
                config.clone(),
                router_hosts.clone(),
            )?;

            handles.push(tokio::spawn(async move {
                worker.run().await;
            }));
        }

        Ok(Self { writer, handles })
    }

    pub async fn run(self) {
        for handle in self.handles {
            handle.await.ok();
        }

        self.writer.finish().await.unwrap();
    }
}

pub trait DatumSink: Send + Sync {
    fn write(&self, crawl_datum: CrawlDatum) -> impl Future<Output = Result<()>> + Send;
    fn finish(&self) -> impl Future<Output = Result<()>> + Send;
}

pub async fn encoded_body(res: reqwest::Response) -> Result<String> {
    let content_type = res
        .headers()
        .get(reqwest::header::CONTENT_TYPE)
        .and_then(|value| value.to_str().ok())
        .and_then(|value| value.parse::<mime::Mime>().ok());
    let encoding_name = content_type
        .as_ref()
        .and_then(|mime| mime.get_param("charset").map(|charset| charset.as_str()))
        .unwrap_or("utf-8");
    let encoding =
        encoding_rs::Encoding::for_label(encoding_name.as_bytes()).unwrap_or(encoding_rs::UTF_8);

    let mut bytes = Vec::new();

    let mut stream = res.bytes_stream();
    while let Some(b) = stream.next().await {
        if b.is_err() {
            return Err(Error::ResponseBodyReadFailed);
        }

        let b = b.unwrap();

        bytes.extend_from_slice(&b);

        if bytes.len() > MAX_CONTENT_LENGTH {
            return Err(Error::ContentTooLarge);
        }
    }

    let (text, _, _) = encoding.decode(&bytes);
    Ok(text.to_string())
}