Skip to content

web search done right

9.2 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use std::{
    collections::{HashMap, HashSet},
    ops::AddAssign,
    sync::{Arc, Mutex},
};

use bloom::BytesBloomFilter;
use url::Url;

use crate::{
    config::{self, SiteStatsConfig},
    entrypoint::download_all_warc_files,
    feed::Feed,
    webgraph::Node,
    webpage::{url_ext::UrlExt, Html},
    Result,
};

const TOP_FEEDS_PER_SITE: Option<usize> = Some(10);
const MIN_FEED_COUNT: u64 = 1;

#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Default)]
pub struct SiteStats {
    pages: u64,
    blogposts: u64,
    news_articles: u64,
    feeds: HashMap<Feed, u64>,
}

impl AddAssign<SiteStats> for SiteStats {
    fn add_assign(&mut self, rhs: SiteStats) {
        self.pages += rhs.pages;
        self.blogposts += rhs.blogposts;
        self.news_articles += rhs.news_articles;

        for (feed, count) in rhs.feeds {
            *self.feeds.entry(feed).or_default() += count;
        }
    }
}

#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
pub struct FinalStats {
    pub pages: u64,
    pub blogposts: u64,
    pub news_articles: u64,
    pub feeds: Vec<FeedCount>,
}

#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
pub struct FeedCount {
    feed: Feed,
    count: u64,
}

impl From<FeedCount> for Feed {
    fn from(feed_count: FeedCount) -> Self {
        feed_count.feed
    }
}

impl From<(Feed, u64)> for FeedCount {
    fn from((feed, count): (Feed, u64)) -> Self {
        Self { feed, count }
    }
}

impl From<SiteStats> for FinalStats {
    fn from(stats: SiteStats) -> Self {
        Self {
            pages: stats.pages,
            blogposts: stats.blogposts,
            news_articles: stats.news_articles,
            feeds: stats
                .feeds
                .into_iter()
                .map(|(feed, count)| FeedCount::from((feed, count)))
                .collect(),
        }
    }
}

#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
pub struct FinalSiteStats {
    site: Site,
    #[serde(flatten)]
    stats: FinalStats,
}

impl FinalSiteStats {
    pub fn site(&self) -> &Site {
        &self.site
    }

    pub fn stats(&self) -> &FinalStats {
        &self.stats
    }
}

#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Hash, PartialEq, Eq)]
#[serde(transparent)]
pub struct Site(String);

impl Site {
    fn from_url(url: &str) -> Result<Self> {
        let url = Url::robust_parse(url)?;
        let root_domain = url
            .root_domain()
            .ok_or(anyhow::anyhow!("Failed to get root domain from url"))?;
        Ok(Self(root_domain.to_string()))
    }

    pub fn as_str(&self) -> &str {
        &self.0
    }

    pub fn url(&self) -> Result<Url> {
        Url::robust_parse(&self.0).map_err(|_| anyhow::anyhow!("Failed to parse url"))
    }
}

#[derive(Clone, Debug, Hash, PartialEq, Eq)]
struct SiteId([u8; 16]);

impl SiteId {
    fn from_url(url: &str) -> Result<Self> {
        let url = Url::robust_parse(url)?;
        let node_id = Node::from(&url).into_host().id();
        Ok(Self(node_id.as_u128().to_be_bytes()))
    }
}

impl AsRef<[u8]> for SiteId {
    fn as_ref(&self) -> &[u8] {
        self.0.as_ref()
    }
}

struct SiteFilter {
    bloom: BytesBloomFilter<SiteId>,
    full: HashSet<SiteId>,
}

impl SiteFilter {
    fn new(sites: Vec<SiteId>) -> Self {
        let mut bloom = BytesBloomFilter::new(sites.len() as u64, 0.01);
        let mut full = HashSet::new();

        for site in sites {
            bloom.insert(&site);
            full.insert(site);
        }

        Self { bloom, full }
    }
}

impl SiteFilter {
    fn should_process(&self, site: &SiteId) -> bool {
        self.bloom.contains(site) && self.full.contains(site)
    }
}

pub struct StatsWorker {
    stats: Mutex<HashMap<Site, SiteStats>>,
    site_filter: SiteFilter,
}

impl StatsWorker {
    fn new(site_ids: Vec<SiteId>) -> Self {
        Self {
            stats: Mutex::new(HashMap::new()),
            site_filter: SiteFilter::new(site_ids),
        }
    }

    fn process(&self, job: Job) -> Result<()> {
        let name = job.warc_path.split('/').last().unwrap();

        tracing::info!("processing {}", name);

        let source = job.source_config.clone();

        let paths = vec![job.warc_path.clone()];
        let warc_files = download_all_warc_files(&paths, &source);
        tokio::pin!(warc_files);

        for file in warc_files.by_ref() {
            for record in file.records().flatten() {
                let site_id = SiteId::from_url(&record.request.url);

                if site_id.is_err() {
                    tracing::error!("error parsing url: {}", site_id.err().unwrap());
                    continue;
                }

                let site_id = site_id.unwrap();

                if !self.site_filter.should_process(&site_id) {
                    continue;
                }

                let webpage = match Html::parse(&record.response.body, &record.request.url) {
                    Ok(webpage) => webpage,
                    Err(err) => {
                        tracing::error!("error parsing webpage: {}", err);
                        continue;
                    }
                };

                let mut stats = SiteStats {
                    feeds: HashMap::new(),
                    pages: 1,
                    blogposts: 0,
                    news_articles: 0,
                };

                if let Ok(feeds) = webpage.feeds() {
                    let root_domain = webpage.url().root_domain();

                    for feed in feeds {
                        if feed.url.root_domain() == root_domain {
                            *stats.feeds.entry(feed).or_default() += 1;
                        }
                    }
                }

                for schema in webpage.schema_org() {
                    if schema.types_contains("NewsArticle") {
                        stats.news_articles = 1;
                    }

                    if schema.types_contains("BlogPosting") {
                        stats.blogposts = 1;
                    }
                }
                if let Ok(site) = Site::from_url(&record.request.url) {
                    *self.stats.lock().unwrap().entry(site).or_default() += stats;
                }
            }
        }

        Ok(())
    }
}

#[derive(Debug, Clone)]
pub struct Job {
    pub source_config: config::WarcSource,
    pub warc_path: String,
}

pub fn run(config: SiteStatsConfig) -> Result<()> {
    let host_centrality = speedy_kv::Db::open_or_create(&config.host_centrality_path)?;

    let site_ids: Vec<_> = crate::webgraph::centrality::top_nodes(
        &host_centrality,
        crate::webgraph::centrality::TopNodes::Top(config.top_sites),
    )
    .into_iter()
    .map(|(node, _)| SiteId(node.as_u128().to_be_bytes()))
    .collect();

    let jobs: Vec<_> = config
        .warc_source
        .paths()?
        .into_iter()
        .skip(config.skip_warc_files.unwrap_or(0))
        .take(config.limit_warc_files.unwrap_or(usize::MAX))
        .map(|warc_path| Job {
            source_config: config.warc_source.clone(),
            warc_path,
        })
        .collect();

    let num_workers = usize::from(std::thread::available_parallelism()?).min(jobs.len());
    let mut handlers = Vec::new();
    let worker = Arc::new(StatsWorker::new(site_ids.clone()));

    for i in 0..num_workers {
        let jobs = jobs.clone();
        let worker = Arc::clone(&worker);

        handlers.push(std::thread::spawn(move || {
            for job in jobs.into_iter().skip(i).step_by(num_workers) {
                worker.process(job).unwrap();
            }
        }));
    }

    for handler in handlers {
        handler.join().ok();
    }

    let mut final_stats: Vec<_> = worker
        .stats
        .lock()
        .unwrap()
        .clone()
        .into_iter()
        .map(|(site, stats)| FinalSiteStats {
            site,
            stats: stats.into(),
        })
        .collect();

    final_stats.iter_mut().for_each(|site_stats| {
        site_stats
            .stats
            .feeds
            .retain(|feed_count| feed_count.count > MIN_FEED_COUNT);

        site_stats.stats.feeds.sort_by(|a, b| b.count.cmp(&a.count));

        if let Some(top_feeds) = TOP_FEEDS_PER_SITE {
            site_stats.stats.feeds.truncate(top_feeds);
        }
    });

    final_stats.sort_by(|a, b| b.stats.pages.cmp(&a.stats.pages));

    let writer = std::fs::File::create(&config.output_path)?;
    let writer = std::io::BufWriter::new(writer);

    serde_json::to_writer_pretty(writer, &final_stats)?;

    Ok(())
}