Skip to content

web search done right

3.9 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use super::{Error, Result};
use crate::config::CrawlerConfig;
use crate::crawler::robots_txt::RobotsTxtManager;
use anyhow::anyhow;
use std::time::Duration;
use url::Url;

pub(super) fn reqwest_client(config: &CrawlerConfig) -> Result<reqwest::Client> {
    let timeout = Duration::from_secs(config.timeout_seconds);

    let mut headers = reqwest::header::HeaderMap::default();
    headers.insert(
        reqwest::header::ACCEPT,
        reqwest::header::HeaderValue::from_static("text/html"),
    );
    headers.insert(
        reqwest::header::ACCEPT_LANGUAGE,
        reqwest::header::HeaderValue::from_static("en-US,en;q=0.9,*;q=0.8"),
    );

    reqwest::Client::builder()
        .timeout(timeout)
        .connect_timeout(timeout)
        .http2_keep_alive_interval(None)
        .default_headers(headers)
        .redirect(reqwest::redirect::Policy::limited(0))
        .user_agent(&config.user_agent.full)
        .build()
        .map_err(|e| Error::from(anyhow!(e)))
}

/// Reqwest client that respects robots.txt for each request.
#[derive(Clone)]
pub struct RobotClient {
    robots_txt_manager: RobotsTxtManager,
    client: reqwest::Client,
}

impl RobotClient {
    pub fn new(config: &CrawlerConfig) -> Result<Self> {
        Ok(Self {
            client: reqwest_client(config)?,
            robots_txt_manager: RobotsTxtManager::new(config),
        })
    }

    pub fn robots_txt_manager(&self) -> &RobotsTxtManager {
        &self.robots_txt_manager
    }

    pub async fn get(&self, url: Url) -> Result<RequestBuilder> {
        if !self.robots_txt_manager.is_allowed(&url).await {
            return Err(Error::DisallowedPath);
        }

        Ok(RequestBuilder::new(self.client.get(url)))
    }
}

#[allow(dead_code)] // not used in tests, but required for public API
pub struct RequestBuilder(reqwest::RequestBuilder);

impl From<reqwest::RequestBuilder> for RequestBuilder {
    fn from(builder: reqwest::RequestBuilder) -> Self {
        Self::new(builder)
    }
}

impl RequestBuilder {
    pub fn new(builder: reqwest::RequestBuilder) -> Self {
        Self(builder)
    }

    #[cfg(not(test))]
    pub async fn send(self) -> Result<reqwest::Response> {
        self.0.send().await.map_err(|e| Error::from(anyhow!(e)))
    }

    #[cfg(test)]
    pub async fn send(&self) -> Result<reqwest::Response> {
        Err(Error::from(anyhow!("Not allowed in tests")))
    }

    pub fn timeout(self, timeout: Duration) -> Self {
        Self(self.0.timeout(timeout))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_errs_disallowed_path() {
        let config = CrawlerConfig::for_tests();
        let client = RobotClient::new(&config).unwrap();

        let robots_txt =
            robotstxt::Robots::parse("TestBot", "User-agent: *\nDisallow: /test\nAllow: /example")
                .unwrap();

        client
            .robots_txt_manager()
            .insert("example.com".to_string(), robots_txt);

        let url = Url::parse("http://example.com/test").unwrap();
        assert!(client.get(url).await.is_err());

        let url = Url::parse("http://example.com/example").unwrap();
        assert!(client.get(url).await.is_ok());
    }
}