web search done right
Crawler: Use pre-computed crawl plan. The crawl coordinator now uses a crawl plan computed by the crawl planner to determine which urls and sites to schedule. This allows us to use heavier computations like harmonic centrality to determine the prioritation during crawl. This should be much more resistant to linkfarms but comes with the disadvantage that we will always use the previously crawled data to determine the current scheduling (which may be somewhat out-of-date).
Mikkel Denkerf54ac8901319
diff --git a/configs/crawler/coordinator.toml b/configs/crawler/coordinator.tomlindex c5c38269..f5af4afa 100644--- a/configs/crawler/coordinator.toml+++ b/configs/crawler/coordinator.toml@@ -1,2 +1,2 @@-crawldb_folder = "data/crawldb"host = "0.0.0.0:8080"+job_queue = "data/crawlplan/job_queue/0.queue"diff --git a/configs/crawler/crawler.toml b/configs/crawler/crawler.tomlindex c65bc35e..226afc0d 100644--- a/configs/crawler/crawler.toml+++ b/configs/crawler/crawler.toml@@ -1,3 +1,4 @@+dry_run = truenum_jobs_per_fetch = 10num_worker_threads = 1politeness_factor = 1.0diff --git a/configs/crawler/router.toml b/configs/crawler/router.tomlindex dcc3fa08..fca9e23a 100644--- a/configs/crawler/router.toml+++ b/configs/crawler/router.toml@@ -1,3 +1,2 @@coordinator_addrs = ["0.0.0.0:8080"]host = "0.0.0.0:8181"-seed_urls = ["https://en.wikipedia.org"]diff --git a/core/examples/crawldb_insert.rs b/core/examples/crawldb_insert.rsdeleted file mode 100644index 2639cf3c..00000000--- a/core/examples/crawldb_insert.rs+++ /dev/null@@ -1,52 +0,0 @@-use anyhow::Result;-use hashbrown::HashMap;-use stract::crawler::{crawl_db::CrawlDb, Domain, UrlString, UrlToInsert};--fn random_domains() -> Vec<String> {- let mut domains = Vec::new();-- for i in 0..1024 {- domains.push(format!("domain-{}", i));- }-- domains-}--fn random_urls(domain: &str) -> Vec<String> {- let mut urls = Vec::new();-- for i in 0..256 {- urls.push(format!("https://{domain}/{i}"));- }-- urls-}--fn main() -> Result<()> {- let mut db = CrawlDb::open("data/crawldb")?;-- loop {- let domains = random_domains();-- let mut urls = HashMap::new();-- for domain in &domains {- let rand_urls = random_urls(domain);-- urls.insert(- Domain::from(domain.clone()),- rand_urls- .into_iter()- .map(|url| UrlToInsert {- url: UrlString(url),- weight: 1.0,- })- .collect(),- );- }-- let start = std::time::Instant::now();- db.insert_urls(urls)?;- println!("inserted in {:?}", start.elapsed());- }-}diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsindex cd80b07c..bf574a22 100644--- a/core/src/config/defaults.rs+++ b/core/src/config/defaults.rs@@ -96,6 +96,10 @@ impl Crawler {pub fn max_redirects() -> usize {5}++ pub fn dry_run() -> bool {+ false+ }}pub struct WebgraphServer;diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 8b25b03e..ddfd1724 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -260,7 +260,7 @@ pub struct SearchServerConfig {#[derive(Debug, Serialize, Deserialize, Clone)]pub struct CrawlCoordinatorConfig {- pub crawldb_folder: String,+ pub job_queue: String,pub host: SocketAddr,}@@ -274,7 +274,6 @@ pub struct UserAgent {pub struct CrawlerConfig {pub num_worker_threads: usize,pub user_agent: UserAgent,- pub num_jobs_per_fetch: usize,#[serde(default = "defaults::Crawler::robots_txt_cache_sec")]pub robots_txt_cache_sec: u64,@@ -297,6 +296,9 @@ pub struct CrawlerConfig {#[serde(default = "defaults::Crawler::max_redirects")]pub max_redirects: usize,+ #[serde(default = "defaults::Crawler::dry_run")]+ pub dry_run: bool,+pub timeout_seconds: u64,pub s3: S3Config,pub router_hosts: Vec<String>,@@ -306,7 +308,6 @@ pub struct CrawlerConfig {pub struct CrawlRouterConfig {pub host: SocketAddr,pub coordinator_addrs: Vec<SocketAddr>,- pub seed_urls: Vec<String>,}#[derive(Debug, Serialize, Deserialize, Clone)]@@ -364,3 +365,18 @@ pub struct WebgraphServerConfig {pub struct WidgetsConfig {pub thesaurus_paths: Vec<String>,}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct CrawlPlannerConfig {+ pub page_harmonic_path: String,+ pub host_harmonic_path: String,+ pub page_graph_path: String,+ pub host_graph_path: String,+ pub output_path: String,++ pub num_job_queues: usize,++ pub crawl_budget: usize,+ pub top_host_fraction: f64,+ pub wander_fraction: f64,+}diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex 4fa52ec9..cc4f20e3 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -14,57 +14,21 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use hashbrown::HashMap;-use std::sync::Mutex;--use super::{crawl_db::CrawlDb, Domain, DomainCrawled, Job, Result, UrlToInsert};-use std::{path::Path, time::Instant};--const DEFAULT_JOB_URLS: usize = 20;+use super::{file_queue::FileQueue, Job, Result};+use std::{path::Path, sync::Mutex};pub struct CrawlCoordinator {- db: Mutex<CrawlDb>,+ jobs: Mutex<FileQueue<Job>>,}impl CrawlCoordinator {- pub fn new<P: AsRef<Path>>(crawldb_folder: P) -> Result<Self> {- let db = CrawlDb::open(crawldb_folder)?;-- Ok(Self { db: Mutex::new(db) })- }-- pub fn insert_urls(&self, urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {- let mut db = self.db.lock().unwrap_or_else(|e| e.into_inner());- let start = Instant::now();-- db.insert_urls(urls)?;-- tracing::info!("inserted responses in {:?}", start.elapsed());-- Ok(())+ pub fn new<P: AsRef<Path>>(jobs_queue: P) -> Result<Self> {+ Ok(Self {+ jobs: Mutex::new(FileQueue::new(jobs_queue)?),+ })}- pub fn mark_jobs_complete(&self, domains: &[DomainCrawled]) -> Result<()> {- let mut db = self.db.lock().unwrap_or_else(|e| e.into_inner());- let start = Instant::now();-- db.mark_jobs_complete(domains)?;-- tracing::info!("marked jobs complete in {:?}", start.elapsed());-- Ok(())- }-- pub fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {- let mut db = self.db.lock().unwrap_or_else(|e| e.into_inner());- let start = Instant::now();-- let domains = db.sample_domains(num_jobs)?;- tracing::debug!("sampled domains: {:?}", domains);- let jobs = db.prepare_jobs(&domains, DEFAULT_JOB_URLS)?;- tracing::debug!("sampled jobs: {:?}", jobs);- tracing::info!("sampled {} jobs in {:?}", jobs.len(), start.elapsed());-- Ok(jobs)+ pub fn sample_job(&self) -> Result<Option<Job>> {+ self.jobs.lock().unwrap_or_else(|e| e.into_inner()).pop()}}diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsdeleted file mode 100644index 68a29dcc..00000000--- a/core/src/crawler/crawl_db.rs+++ /dev/null@@ -1,753 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use hashbrown::HashMap;-use rand::Rng;-use std::path::PathBuf;-use std::{collections::BinaryHeap, path::Path};-use url::Url;--use super::{Domain, DomainCrawled, Job, Result, UrlString, UrlToInsert, MAX_URL_LEN_BYTES};--const MAX_URL_DB_SIZE_BYTES: u64 = 20 * 1024 * 1024 * 1024; // 20GB--#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]-pub enum UrlStatus {- Pending,- Crawling,- Failed { status_code: Option<u16> },- Done,-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]-pub enum DomainStatus {- Pending,- CrawlInProgress,-}--struct SampledItem<T> {- item: T,- priority: f64,-}--impl<T> PartialEq for SampledItem<T> {- fn eq(&self, other: &Self) -> bool {- self.priority == other.priority- }-}--impl<T> Eq for SampledItem<T> {}--impl<T> PartialOrd for SampledItem<T> {- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {- Some(self.cmp(other))- }-}--impl<T> Ord for SampledItem<T> {- fn cmp(&self, other: &Self) -> std::cmp::Ordering {- self.priority.total_cmp(&other.priority)- }-}--fn weighted_sample<T>(items: impl Iterator<Item = (T, f64)>, num_items: usize) -> Vec<(T, f64)> {- let mut sampled_items: BinaryHeap<SampledItem<T>> = BinaryHeap::with_capacity(num_items);-- let mut rng = rand::thread_rng();-- for (item, weight) in items {- // see https://www.kaggle.com/code/kotamori/random-sample-with-weights-on-sql/notebook for details on math- let priority = -(rng.gen::<f64>().abs() + f64::EPSILON).ln() / (weight + 1.0);-- if sampled_items.len() < num_items {- sampled_items.push(SampledItem { item, priority });- } else if let Some(mut max) = sampled_items.peek_mut() {- if priority < max.priority {- max.item = item;- max.priority = priority;- }- }- }-- sampled_items- .into_iter()- .map(|s| (s.item, s.priority))- .collect()-}--#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]-struct UrlState {- weight: f64,- status: UrlStatus,-}--impl Default for UrlState {- fn default() -> Self {- Self {- weight: 0.0,- status: UrlStatus::Pending,- }- }-}--#[derive(Clone, serde::Serialize, serde::Deserialize)]-struct DomainState {- weight: f64,- status: DomainStatus,-}--impl Default for DomainState {- fn default() -> Self {- Self {- weight: 1.0,- status: DomainStatus::Pending,- }- }-}-struct CachedValue<T> {- value: T,- last_updated: std::time::Instant,-}--impl<T> From<T> for CachedValue<T> {- fn from(value: T) -> Self {- Self {- value,- last_updated: std::time::Instant::now(),- }- }-}--struct UrlStateDbShard {- db: rocksdb::DB,- /// from rocksdb docs: "Cache must outlive DB instance which uses it."- _cache: rocksdb::Cache,- approx_size_bytes: CachedValue<u64>,-}--impl UrlStateDbShard {- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let mut options = rocksdb::Options::default();-- options.create_if_missing(true);-- options.set_max_background_jobs(8);- options.increase_parallelism(8);- options.set_max_subcompactions(8);- options.set_write_buffer_size(512 * 1024 * 1024);- options.set_allow_mmap_reads(true);- options.set_allow_mmap_writes(true);- options.set_target_file_size_base(512 * 1024 * 1024); // 512 MB- options.set_target_file_size_multiplier(10);-- options.set_level_zero_slowdown_writes_trigger(-1);- options.set_level_zero_stop_writes_trigger(-1);-- // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)- options.set_level_compaction_dynamic_level_bytes(true);- options.set_bytes_per_sync(1048576);- let mut block_options = rocksdb::BlockBasedOptions::default();- let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024); // 1GB- block_options.set_block_cache(&cache);- block_options.set_ribbon_filter(10.0);- block_options.set_format_version(5);- block_options.set_block_size(16 * 1024);- block_options.set_cache_index_and_filter_blocks(true);- block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);-- options.set_block_based_table_factory(&block_options);-- options.set_compression_type(rocksdb::DBCompressionType::None);-- let db = rocksdb::DB::open(&options, path.as_ref().join("urls"))?;- let approx_size_bytes = db- .property_int_value(rocksdb::properties::TOTAL_SST_FILES_SIZE)?- .unwrap_or_default()- .into();-- Ok(Self {- db,- approx_size_bytes,- _cache: cache,- })- }-- pub fn put_batch(&mut self, batch: &[(Domain, Vec<(UrlString, UrlState)>)]) -> Result<()> {- let mut rocksdb_batch = rocksdb::WriteBatch::default();-- for (domain, urls) in batch {- let domain_bytes = bincode::serialize(domain)?;-- for (url, state) in urls {- let url_bytes = bincode::serialize(url)?;- if url_bytes.len() > MAX_URL_LEN_BYTES {- continue;- }-- let key_bytes = [domain_bytes.as_slice(), url_bytes.as_slice()].concat();-- let state_bytes = bincode::serialize(state)?;-- rocksdb_batch.put(key_bytes, state_bytes);- }- }-- let mut write_options = rocksdb::WriteOptions::default();- write_options.disable_wal(true);- write_options.set_no_slowdown(true);-- self.db.write_opt(rocksdb_batch, &write_options)?;-- Ok(())- }-- pub fn get_all_urls(&self, domain: &Domain) -> Result<Vec<(UrlString, UrlState)>> {- let domain_bytes = bincode::serialize(domain)?;-- let start = [domain_bytes.as_slice(), &[0].repeat(MAX_URL_LEN_BYTES)].concat();-- let iter = self.db.iterator(rocksdb::IteratorMode::From(- &start,- rocksdb::Direction::Forward,- ));-- Ok(iter- .take_while(|r| {- if let Ok((key, _)) = r.as_ref() {- if domain_bytes.len() >= key.len() {- return false;- }-- key[..domain_bytes.len()] == domain_bytes- } else {- false- }- })- .filter_map(|r| {- let (key, value) = r.ok()?;-- if domain_bytes.len() >= key.len() {- return None;- }-- let url = bincode::deserialize(&key[domain_bytes.len()..]).ok()?;-- let state = bincode::deserialize(&value).ok()?;-- Some((url, state))- })- .collect())- }-- pub fn approximate_size_bytes(&mut self) -> Result<u64> {- if self.approx_size_bytes.last_updated.elapsed().as_secs() > 10 {- self.approx_size_bytes = self- .db- .property_int_value(rocksdb::properties::TOTAL_SST_FILES_SIZE)?- .unwrap_or_default()- .into();- }-- Ok(self.approx_size_bytes.value)- }-- fn multi_get(&self, urls: &[(Domain, UrlString)]) -> Result<HashMap<UrlString, UrlState>> {- let mut res = HashMap::new();-- let mut keys = Vec::with_capacity(urls.len());-- for (domain, url) in urls {- let domain_bytes = bincode::serialize(domain)?;- let url_bytes = bincode::serialize(url)?;-- let key_bytes = [domain_bytes.as_slice(), url_bytes.as_slice()].concat();-- keys.push(key_bytes);- }-- for (val, (_, url)) in self- .db- .multi_get(keys.into_iter())- .into_iter()- .zip(urls.iter())- {- if let Some(val) = val? {- let state: UrlState = bincode::deserialize(&val)?;- res.insert(url.clone(), state);- }- }-- Ok(res)- }-}--struct UrlStateDb {- shards: Vec<UrlStateDbShard>,- path: PathBuf,-}--impl UrlStateDb {- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- if path.as_ref().exists() {- let mut shard_names = Vec::new();- for entry in std::fs::read_dir(path.as_ref())? {- let entry = entry?;- let path = entry.path();-- if path.is_dir() {- shard_names.push(path.to_str().unwrap().to_string());- }- }-- shard_names.sort();-- let mut shards = Vec::new();-- for shard_name in shard_names {- shards.push(UrlStateDbShard::open(shard_name)?);- }-- Ok(Self {- shards,- path: path.as_ref().to_path_buf(),- })- } else {- let shard_id =- chrono::Utc::now().to_rfc3339() + "_" + uuid::Uuid::new_v4().to_string().as_str();- let shard_path = path.as_ref().join(shard_id);-- std::fs::create_dir_all(&shard_path)?;-- let shard = UrlStateDbShard::open(&shard_path)?;-- Ok(Self {- shards: vec![shard],- path: path.as_ref().to_path_buf(),- })- }- }-- fn new_shard(&mut self) -> Result<()> {- let shard_id =- chrono::Utc::now().to_rfc3339() + "_" + uuid::Uuid::new_v4().to_string().as_str();- let shard_path = self.path.as_path().join(shard_id);-- std::fs::create_dir_all(&shard_path)?;-- let shard = UrlStateDbShard::open(&shard_path)?;-- self.shards.push(shard);-- Ok(())- }-- pub fn put_batch(&mut self, batch: &[(Domain, Vec<(UrlString, UrlState)>)]) -> Result<()> {- if self.shards.is_empty() {- self.new_shard()?;- }- let last_shard = self.shards.last_mut().unwrap();-- if last_shard.approximate_size_bytes()? > MAX_URL_DB_SIZE_BYTES {- self.new_shard()?;- }-- self.shards.last_mut().unwrap().put_batch(batch)?;-- Ok(())- }-- pub fn get_all_urls(&self, domain: &Domain) -> Result<HashMap<UrlString, UrlState>> {- let mut res = HashMap::new();-- for shard in &self.shards {- for (url, state) in shard.get_all_urls(domain)? {- res.insert(url, state);- }- }-- Ok(res)- }-- fn multi_get(&self, urls: &[(Domain, UrlString)]) -> Result<HashMap<UrlString, UrlState>> {- let mut res = HashMap::new();-- for shard in &self.shards {- res.extend(shard.multi_get(urls)?.into_iter());- }-- Ok(res)- }-}--struct DomainStateDb {- db: rocksdb::DB,-}--impl DomainStateDb {- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let mut options = rocksdb::Options::default();-- options.create_if_missing(true);-- let mut block_options = rocksdb::BlockBasedOptions::default();- block_options.set_ribbon_filter(10.0);- block_options.set_format_version(5);-- options.set_block_based_table_factory(&block_options);- options.set_optimize_filters_for_hits(true);- options.set_max_background_jobs(8);- options.increase_parallelism(8);- options.set_write_buffer_size(512 * 1024 * 1024);- options.set_allow_mmap_reads(true);- options.set_allow_mmap_writes(true);- options.set_max_subcompactions(8);-- options.set_level_zero_slowdown_writes_trigger(-1);- options.set_level_zero_stop_writes_trigger(-1);-- let db = rocksdb::DB::open(&options, path.as_ref())?;-- Ok(Self { db })- }-- fn multi_get(&self, domains: &[Domain]) -> Result<HashMap<Domain, DomainState>> {- let mut res = HashMap::new();- let mut domain_bytes = Vec::with_capacity(domains.len());-- for domain in domains {- domain_bytes.push(bincode::serialize(domain)?);- }-- for (val, domain) in self- .db- .multi_get(domain_bytes.into_iter())- .into_iter()- .zip(domains.iter())- {- if let Some(val) = val? {- let domain_state: DomainState = bincode::deserialize(&val)?;- res.insert(domain.clone(), domain_state);- }- }-- Ok(res)- }-- fn put_batch(&self, batch: &[(Domain, DomainState)]) -> Result<()> {- let mut rocksdb_batch = rocksdb::WriteBatch::default();-- for (domain, state) in batch {- let domain_bytes = bincode::serialize(domain)?;- let state_bytes = bincode::serialize(state)?;-- rocksdb_batch.put(domain_bytes, state_bytes);- }-- let mut write_options = rocksdb::WriteOptions::default();- write_options.disable_wal(true);- write_options.set_no_slowdown(true);-- self.db.write_opt(rocksdb_batch, &write_options)?;-- Ok(())- }-- fn iter(&self) -> impl Iterator<Item = (Domain, DomainState)> + '_ {- let iter = self.db.iterator(rocksdb::IteratorMode::Start);-- iter.filter_map(|r| {- let (key, value) = r.ok()?;- let domain = bincode::deserialize(&key).ok()?;- let state = bincode::deserialize(&value).ok()?;-- Some((domain, state))- })- }-}--pub struct CrawlDb {- domain_state: DomainStateDb,- urls: UrlStateDb,-}--impl CrawlDb {- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- Ok(Self {- domain_state: DomainStateDb::open(path.as_ref().join("domains"))?,- urls: UrlStateDb::open(path.as_ref().join("urls"))?,- })- }-- pub fn insert_urls(&mut self, domain_urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {- let mut url_batches = Vec::new();- let mut domain_batches = Vec::new();-- let domains = domain_urls.keys().cloned().collect::<Vec<_>>();- let domain_states = self.domain_state.multi_get(&domains)?;- let mut url_states = self.urls.multi_get(- domain_urls- .iter()- .flat_map(|(domain, urls)| urls.iter().map(|url| (domain.clone(), url.url.clone())))- .collect::<Vec<_>>()- .as_slice(),- )?;-- for (domain, urls) in domain_urls.into_iter() {- let mut domain_state = domain_states.get(&domain).cloned().unwrap_or_default();-- let mut updated_url_states = Vec::new();-- for url in urls {- if url.url.as_str().len() > MAX_URL_LEN_BYTES {- continue;- }-- match url_states.get_mut(&url.url) {- Some(state) => {- state.weight += url.weight;- domain_state.weight += url.weight;- if url.weight > 0.0 {- updated_url_states.push((url.url.clone(), state.clone()));- }- }- None => {- let mut state = UrlState::default();- state.weight += url.weight;- domain_state.weight += url.weight;-- updated_url_states.push((url.url.clone(), state));- }- };- }-- url_batches.push((domain.clone(), updated_url_states));- domain_batches.push((domain, domain_state));- }-- self.urls.put_batch(&url_batches)?;- self.domain_state.put_batch(&domain_batches)?;-- Ok(())- }-- fn set_domain_status(&mut self, domains: &[Domain], status: DomainStatus) -> Result<()> {- let domain_states = self.domain_state.multi_get(domains)?;-- let mut batches = Vec::new();-- for domain in domains {- let mut domain_state = domain_states.get(domain).cloned().unwrap_or_default();- domain_state.status = status;-- batches.push((domain.clone(), domain_state));- }-- self.domain_state.put_batch(&batches)?;-- Ok(())- }-- pub fn mark_jobs_complete(&mut self, domains: &[DomainCrawled]) -> Result<()> {- let domain_states = self.domain_state.multi_get(- domains- .iter()- .map(|d| d.domain.clone())- .collect::<Vec<_>>()- .as_slice(),- )?;-- let mut batches = Vec::new();-- for domain in domains {- let mut domain_state = domain_states- .get(&domain.domain)- .cloned()- .unwrap_or_default();- domain_state.status = DomainStatus::Pending;- domain_state.weight -= domain.budget_used;-- batches.push((domain.domain.clone(), domain_state));- }-- self.domain_state.put_batch(&batches)?;-- Ok(())- }-- pub fn sample_domains(&mut self, num_jobs: usize) -> Result<Vec<Domain>> {- let sampled = weighted_sample(- self.domain_state.iter().filter_map(|(domain, state)| {- if state.status == DomainStatus::Pending {- Some((domain, state.weight))- } else {- None- }- }),- num_jobs,- );-- let domains = sampled.into_iter().map(|(d, _)| d).collect::<Vec<_>>();-- self.set_domain_status(&domains, DomainStatus::CrawlInProgress)?;-- Ok(domains)- }-- pub fn prepare_jobs(&mut self, domains: &[Domain], urls_per_job: usize) -> Result<Vec<Job>> {- let mut jobs = Vec::with_capacity(domains.len());- let mut url_batches = Vec::new();-- let domain_states = self.domain_state.multi_get(domains)?;-- for domain in domains {- let urls = self.urls.get_all_urls(domain)?;-- let available_urls: Vec<_> = urls- .iter()- .filter_map(|(url, state)| {- if state.status == UrlStatus::Pending {- Some((url.clone(), state.weight))- } else {- None- }- })- .collect();-- let sampled: Vec<_> = weighted_sample(- available_urls.iter().map(|(url, weight)| (url, *weight)),- urls_per_job,- );-- let mut new_url_states = Vec::new();-- for (url, _) in &sampled {- let mut state = urls.get(*url).cloned().unwrap_or_default();- state.status = UrlStatus::Crawling;-- new_url_states.push(((*url).clone(), state));- }-- url_batches.push((domain.clone(), new_url_states));-- let domain_state = domain_states.get(domain).cloned().unwrap_or_default();-- let job = Job {- domain: domain.clone(),- fetch_sitemap: false, // todo: fetch for new sites- urls: sampled- .iter()- .filter_map(|(url, _)| Url::try_from(*url).ok())- .collect(),- weight_budget: domain_state.weight,- };-- jobs.push(job);- }-- self.urls.put_batch(&url_batches)?;-- Ok(jobs)- }-}--#[cfg(test)]-mod tests {- use crate::gen_temp_path;-- use super::*;-- fn insert_seed_urls(urls: &[Url], db: &mut CrawlDb) {- let mut domain_urls = HashMap::new();-- for url in urls {- let domain = Domain::from(url);- let url_to_insert = UrlToInsert {- url: UrlString::from(url),- weight: 0.0,- };-- domain_urls- .entry(domain)- .or_insert_with(Vec::new)- .push(url_to_insert);- }-- db.insert_urls(domain_urls).unwrap();- }-- #[test]- fn sampling() {- let items: Vec<(usize, f64)> = vec![(0, 1.0), (1, 2.0), (2, 3.0), (3, 4.0)];- let sampled = weighted_sample(items.iter().map(|(i, w)| (i, *w)), 10);- assert_eq!(sampled.len(), items.len());-- let items: Vec<(usize, f64)> = vec![(0, 1.0), (1, 2.0), (2, 3.0), (3, 4.0)];- let sampled = weighted_sample(items.iter().map(|(i, w)| (i, *w)), 1);- assert_eq!(sampled.len(), 1);-- let items: Vec<(usize, f64)> = vec![(0, 1.0), (1, 2.0), (2, 3.0), (3, 4.0)];- let sampled = weighted_sample(items.iter().map(|(i, w)| (i, *w)), 0);- assert_eq!(sampled.len(), 0);-- let items: Vec<(usize, f64)> = vec![(0, 1000000000.0), (1, 2.0)];- let sampled = weighted_sample(items.iter().map(|(i, w)| (i, *w)), 1);- assert_eq!(sampled.len(), 1);- assert_eq!(*sampled[0].0, 0);- }-- #[test]- fn simple_politeness() {- let mut db = CrawlDb::open(gen_temp_path()).unwrap();-- let urls = vec![Url::parse("https://example.com").unwrap()];- insert_seed_urls(&urls, &mut db);-- let domain = Domain::from(&Url::parse("https://example.com").unwrap());-- let sample = db.sample_domains(128).unwrap();-- assert_eq!(sample.len(), 1);- assert_eq!(&sample[0], &domain);- assert_eq!(- db.domain_state.multi_get(&[domain.clone()]).unwrap()[&domain].status,- DomainStatus::CrawlInProgress- );-- let new_sample = db.sample_domains(128).unwrap();- assert_eq!(new_sample.len(), 0);- }-- #[test]- fn get_all_urls() {- let mut db = CrawlDb::open(gen_temp_path()).unwrap();-- let urls = vec![- Url::parse("https://a.com").unwrap(),- Url::parse("https://b.com").unwrap(),- ];- insert_seed_urls(&urls, &mut db);-- let domain = Domain::from(&Url::parse("https://a.com").unwrap());-- let urls: Vec<_> = db.urls.get_all_urls(&domain).unwrap().into_iter().collect();-- assert_eq!(urls.len(), 1);- assert_eq!(- urls[0].0,- UrlString::from(&Url::parse("https://a.com").unwrap())- );- }-}diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex 93cb640f..6e898121 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -14,13 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{- collections::{HashMap, VecDeque},- sync::Arc,- time::Duration,-};+use std::{collections::VecDeque, sync::Arc, time::Duration};++use hashbrown::HashMap;-use tokio::sync::Mutex;use url::Url;use crate::{config::CrawlerConfig, webpage::url_ext::UrlExt};@@ -28,10 +25,12 @@ use crate::{config::CrawlerConfig, webpage::url_ext::UrlExt};use self::{warc_writer::WarcWriter, worker::WorkerThread};pub mod coordinator;-pub mod crawl_db;mod robots_txt;pub mod router;pub use router::Router;+mod file_queue;+pub mod planner;+mod site_graph;mod warc_writer;mod worker;@@ -101,6 +100,12 @@ impl From<&Url> for Domain {}}+impl From<Url> for Domain {+ fn from(url: Url) -> Self {+ Self::from(&url)+ }+}+impl From<String> for Domain {fn from(s: String) -> Self {Self(s)@@ -118,9 +123,8 @@ impl Domain {#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]pub struct Job {pub domain: Domain,- pub fetch_sitemap: bool,pub urls: VecDeque<Url>,- pub weight_budget: f64,+ pub wandering_urls: u64,}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]@@ -130,23 +134,10 @@ pub enum UrlResponse {Redirected { url: Url, new_url: Url },}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]-pub struct JobResponse {- pub domain: Domain,- pub url_responses: Vec<UrlResponse>,- pub discovered_urls: Vec<Url>,- pub weight_budget: f64,-}-#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,)]-pub struct UrlString(pub String);-impl UrlString {- fn as_str(&self) -> &str {- &self.0- }-}+pub struct UrlString(String);impl From<&Url> for UrlString {fn from(url: &Url) -> Self {@@ -197,27 +188,20 @@ impl From<Url> for RetrieableUrl {struct WorkerJob {pub domain: Domain,- pub fetch_sitemap: bool,pub urls: VecDeque<RetrieableUrl>,- pub weight_budget: f64,+ pub wanrdering_urls: u64,}impl From<Job> for WorkerJob {fn from(value: Job) -> Self {Self {domain: value.domain,- fetch_sitemap: value.fetch_sitemap,urls: value.urls.into_iter().map(RetrieableUrl::from).collect(),- weight_budget: value.weight_budget,+ wanrdering_urls: value.wandering_urls,}}}-pub enum Command {- Job(Job),- Shutdown,-}-#[derive(Debug, Clone)]pub struct CrawlDatum {url: Url,@@ -234,8 +218,6 @@ pub struct Crawler {impl Crawler {pub async fn new(config: CrawlerConfig) -> Result<Self> {- let pending_commands = Arc::new(Mutex::new(VecDeque::new()));- let results = Arc::new(Mutex::new(Vec::new()));let writer = Arc::new(WarcWriter::new(config.s3.clone()));let timeout = Duration::from_secs(config.timeout_seconds);let mut handles = Vec::new();@@ -247,9 +229,7 @@ impl Crawler {for _ in 0..config.num_worker_threads {let worker = WorkerThread::new(- Arc::clone(&pending_commands),Arc::clone(&writer),- Arc::clone(&results),config.clone(),timeout,router_hosts.clone(),@@ -271,9 +251,3 @@ impl Crawler {self.writer.finish().await.unwrap();}}--#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]-pub enum Response {- NewJobs { jobs: Vec<Job> },- Done,-}diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rsindex 4ee20937..fe6863f6 100644--- a/core/src/crawler/robots_txt.rs+++ b/core/src/crawler/robots_txt.rs@@ -46,11 +46,6 @@ impl RobotsTxtManager {}}- pub fn clear(&mut self) {- self.cache.clear();- self.last_prune = std::time::Instant::now();- }-pub async fn is_allowed(&mut self, url: &Url, user_agent: &str) -> bool {match self.get_mut(url).await {Lookup::Found(robots_txt) => robots_txtdiff --git a/core/src/crawler/router.rs b/core/src/crawler/router.rsindex 6337a3b7..fa4fcec2 100644--- a/core/src/crawler/router.rs+++ b/core/src/crawler/router.rs@@ -1,19 +1,14 @@use anyhow::Result;-use hashbrown::HashMap;-use rand::seq::SliceRandom;+use rand::Rng;use std::{net::SocketAddr, time::Duration};-use url::Url;+use tokio::sync::Mutex;use crate::{- crawler::{DomainCrawled, UrlToInsert},distributed::{retry_strategy::ExponentialBackoff, sonic},- entrypoint::crawler::coordinator::{CoordinatorService, GetJobs, InsertUrls, MarkJobsComplete},+ entrypoint::crawler::coordinator::{CoordinatorService, GetJob},};-use super::{- Domain, Job, JobResponse, UrlString, MAX_DOMAIN_DISCOVERY_FACTOR,- MAX_URLS_FOR_DOMAIN_PER_INSERT, MAX_URL_LEN_BYTES,-};+use super::Job;struct RemoteCoordinator {addr: SocketAddr,@@ -31,192 +26,59 @@ impl RemoteCoordinator {.await?)}- async fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {+ async fn sample_job(&self) -> Result<Option<Job>> {let conn = self.conn().await?;let response = conn- .send_with_timeout(&GetJobs { num_jobs }, Duration::from_secs(90))+ .send_with_timeout(&GetJob {}, Duration::from_secs(90)).await?;Ok(response)}-- async fn insert_urls(&self, urls: HashMap<Domain, Vec<UrlToInsert>>) -> Result<()> {- let conn = self.conn().await?;-- conn.send_with_timeout(&InsertUrls { urls }, Duration::from_secs(90))- .await?;-- Ok(())- }-- async fn mark_jobs_complete(&self, domains: Vec<DomainCrawled>) -> Result<()> {- let conn = self.conn().await?;-- conn.send_with_timeout(&MarkJobsComplete { domains }, Duration::from_secs(90))- .await?;-- Ok(())- }}-pub struct Router {+struct InnerRouter {coordinators: Vec<RemoteCoordinator>,}-impl Router {- pub async fn new(coordinator_addrs: Vec<SocketAddr>, seed_urls: Vec<String>) -> Result<Self> {- let s = Self {+impl InnerRouter {+ async fn new(coordinator_addrs: Vec<SocketAddr>) -> Result<Self> {+ Ok(Self {coordinators: coordinator_addrs.into_iter().map(|addr| RemoteCoordinator { addr }).collect(),- };-- let mut coordinator_urls: HashMap<usize, HashMap<Domain, Vec<UrlToInsert>>> =- HashMap::new();-- for url in seed_urls {- let url = Url::parse(&url)?;- let domain = Domain::from(&url);-- let coordinator_index = s.coordinator_index(&domain);-- coordinator_urls- .entry(coordinator_index)- .or_default()- .entry(domain)- .or_default()- .push(UrlToInsert {- url: UrlString::from(url),- weight: 0.0,- });- }-- let mut futures = Vec::new();- for (coordinator_index, urls) in coordinator_urls {- let coordinator = &s.coordinators[coordinator_index];-- futures.push(coordinator.insert_urls(urls));- }-- futures::future::join_all(futures).await;-- Ok(s)- }-- fn coordinator_index(&self, domain: &Domain) -> usize {- let hash = md5::compute(domain.0.as_bytes());- let hash = u128::from_be_bytes(hash.0) as usize;- hash % self.coordinators.len()- }-- pub async fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {- let random_coordinator = self.coordinators.choose(&mut rand::thread_rng()).unwrap();-- random_coordinator.sample_jobs(num_jobs).await+ })}- pub async fn add_responses(&self, responses: &[JobResponse]) -> Result<()> {- let mut domain_urls: HashMap<Domain, HashMap<UrlString, f64>> = HashMap::new();- let mut domain_budgets: Vec<DomainCrawled> = Vec::new();-- for res in responses {- let urls: Vec<(Domain, Url)> = res- .discovered_urls- .iter()- .map(|url| {- let domain = Domain::from(url);- (domain, url.clone())- })- .collect();-- let diff_domains = urls- .iter()- .filter(|(domain, _)| res.domain != *domain)- .count() as f64;-- let mut used_budget = 0.0;+ async fn sample_job(&mut self) -> Result<Option<Job>> {+ while !self.coordinators.is_empty() {+ let idx = rand::thread_rng().gen_range(0..self.coordinators.len());+ let res = self.coordinators[idx].sample_job().await?;- for (domain, url) in urls {- if url.as_str().len() > MAX_URL_LEN_BYTES {- continue;- }-- let different_domain = res.domain != domain;-- let weight = if different_domain {- (res.weight_budget / diff_domains).min(1.0)- } else {- 0.0- };-- used_budget += weight;-- let urls = domain_urls.entry(domain).or_default();-- if urls.len() >= MAX_URLS_FOR_DOMAIN_PER_INSERT {- continue;- }-- urls.entry(UrlString::from(url.clone()))- .and_modify(|w| *w = (*w + weight).min(1.0))- .or_insert(weight);+ if res.is_some() {+ return Ok(res);}- domain_budgets.push(DomainCrawled {- domain: res.domain.clone(),- budget_used: used_budget,- });+ self.coordinators.remove(idx);}- let mut coordinator_urls: HashMap<usize, HashMap<Domain, Vec<UrlToInsert>>> =- HashMap::new();-- for (domain, urls) in domain_urls {- let urls = urls- .into_iter()- .map(|(url, weight)| UrlToInsert { url, weight })- .collect();- let coordinator_index = self.coordinator_index(&domain);-- let domains = coordinator_urls.entry(coordinator_index).or_default();-- if domains.len() < MAX_DOMAIN_DISCOVERY_FACTOR * responses.len() {- domains.insert(domain, urls);- }- }-- let mut futures = Vec::new();-- for (coordinator_index, urls) in coordinator_urls {- let coordinator = &self.coordinators[coordinator_index];-- futures.push(coordinator.insert_urls(urls));- }-- futures::future::join_all(futures).await;-- let mut futures = Vec::new();- let mut coordinator_budgets: HashMap<usize, Vec<DomainCrawled>> = HashMap::new();-- for domain in domain_budgets {- let coordinator_index = self.coordinator_index(&domain.domain);-- coordinator_budgets- .entry(coordinator_index)- .or_default()- .push(domain);- }-- for (coordinator_index, budgets) in coordinator_budgets {- let coordinator = &self.coordinators[coordinator_index];+ Ok(None)+ }+}- futures.push(coordinator.mark_jobs_complete(budgets));- }+pub struct Router {+ inner: Mutex<InnerRouter>,+}- futures::future::join_all(futures).await;+impl Router {+ pub async fn new(coordinator_addrs: Vec<SocketAddr>) -> Result<Self> {+ Ok(Self {+ inner: Mutex::new(InnerRouter::new(coordinator_addrs).await?),+ })+ }- Ok(())+ pub async fn sample_job(&self) -> Result<Option<Job>> {+ self.inner.lock().await.sample_job().await}}diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 2d58dd48..3f796aa2 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -1,4 +1,3 @@-use encoding_rs::{Encoding, UTF_8};// Stract is an open source web search engine.// Copyright (C) 2023 Stract ApS//@@ -14,55 +13,54 @@ use encoding_rs::{Encoding, UTF_8};//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use encoding_rs::{Encoding, UTF_8};use futures::{future::BoxFuture, FutureExt};+use hashbrown::{HashMap, HashSet};use mime::Mime;use quick_xml::events::Event;use rand::seq::SliceRandom;use tokio_stream::StreamExt;use std::{- collections::{HashMap, HashSet, VecDeque},+ collections::VecDeque,net::SocketAddr,sync::Arc,time::{Duration, Instant},};-use tokio::sync::Mutex;use url::Url;use crate::{config::CrawlerConfig,- crawler::{JobResponse, Response, MAX_OUTGOING_URLS_PER_PAGE, MAX_URL_LEN_BYTES},+ crawler::MAX_URL_LEN_BYTES,distributed::{retry_strategy::ExponentialBackoff, sonic},- entrypoint::crawler::router::{NewJobs, RouterService},+ entrypoint::crawler::router::{NewJob, RouterService},webpage::Html,};use super::{- robots_txt::RobotsTxtManager, Command, CrawlDatum, Error, Result, Site, UrlResponse,- WarcWriter, WorkerJob,+ robots_txt::RobotsTxtManager, site_graph::SiteGraph, CrawlDatum, Domain, Error, Result,+ RetrieableUrl, Site, UrlResponse, WarcWriter, WorkerJob,};const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB+struct ProcessedUrl {+ new_urls: Vec<Url>,+ response: UrlResponse,+ fetch_time: Duration,+}+pub struct WorkerThread {- current_job: Option<WorkerJob>,- pending_commands: Arc<Mutex<VecDeque<Command>>>,writer: Arc<WarcWriter>,- results: Arc<Mutex<Vec<JobResponse>>>,client: reqwest::Client,- config: CrawlerConfig,- politeness_factor: f32,+ config: Arc<CrawlerConfig>,router_hosts: Vec<SocketAddr>,- num_jobs_per_fetch: usize,- robotstxt: RobotsTxtManager,}impl WorkerThread {pub fn new(- pending_commands: Arc<Mutex<VecDeque<Command>>>,writer: Arc<WarcWriter>,- results: Arc<Mutex<Vec<JobResponse>>>,config: CrawlerConfig,timeout: Duration,router_hosts: Vec<SocketAddr>,@@ -86,22 +84,11 @@ impl WorkerThread {.user_agent(&config.user_agent.full).build()?;- let robotstxt = RobotsTxtManager::new(- client.clone(),- Duration::from_secs(config.robots_txt_cache_sec),- );-Ok(Self {writer,client,- results,- current_job: None,- pending_commands,- num_jobs_per_fetch: config.num_jobs_per_fetch,- politeness_factor: config.politeness_factor,- config,+ config: Arc::new(config),router_hosts,- robotstxt,})}@@ -118,82 +105,121 @@ impl WorkerThread {.await?)}- pub async fn run(mut self) {+ pub async fn run(self) {loop {- let mut guard = self.pending_commands.lock().await;- let command = guard.pop_front();-- if let Some(command) = command {- drop(guard); // let other workers get jobs-- match command {- Command::Job(job) => {- self.current_job = Some(job.into());- self.process_job().await;- }- Command::Shutdown => {- self.pending_commands- .lock()- .await- .push_back(Command::Shutdown);-- break;- }+ let conn = self.router_conn().await.unwrap();+ let res = conn+ .send_with_timeout(&NewJob {}, Duration::from_secs(90))+ .await;++ match res {+ Ok(Some(job)) => {+ let executor = JobExecutor::new(+ job.into(),+ self.client.clone(),+ self.config.clone(),+ self.writer.clone(),+ );+ executor.run().await;}- } else {- let conn = self.router_conn().await.unwrap();- let results = self.results.lock().await.drain(..).collect::<Vec<_>>();-- let res = conn- .send_with_timeout(- &NewJobs {- responses: results,- num_jobs: self.num_jobs_per_fetch,- },- Duration::from_secs(90),- )- .await;-- match res {- Ok(Response::NewJobs { jobs }) => {- if jobs.is_empty() {- drop(guard);- tokio::time::sleep(Duration::from_secs(30)).await;- continue;- }-- guard.extend(jobs.into_iter().map(Command::Job));- }- Ok(Response::Done) => {- guard.push_back(Command::Shutdown);-- break;- }- _ => {- drop(guard);- tokio::time::sleep(Duration::from_secs(30)).await;- continue;- }+ Ok(None) => {+ return;+ }+ _ => {+ tokio::time::sleep(Duration::from_secs(60)).await;+ continue;}- drop(guard)}}}+}++struct JobExecutor {+ writer: Arc<WarcWriter>,+ client: reqwest::Client,+ politeness_factor: f32,+ robotstxt: RobotsTxtManager,+ crawled_urls: HashSet<Url>,+ crawled_sitemaps: HashSet<Site>,+ sitemap_urls: HashSet<Url>,+ config: Arc<CrawlerConfig>,+ site_graph: SiteGraph,+ job: WorkerJob,+}++impl JobExecutor {+ fn new(+ job: WorkerJob,+ client: reqwest::Client,+ config: Arc<CrawlerConfig>,+ writer: Arc<WarcWriter>,+ ) -> Self {+ Self {+ writer,+ politeness_factor: config.politeness_factor,+ robotstxt: RobotsTxtManager::new(+ client.clone(),+ Duration::from_secs(config.robots_txt_cache_sec),+ ),+ client,+ crawled_urls: HashSet::new(),+ crawled_sitemaps: HashSet::new(),+ sitemap_urls: HashSet::new(),+ config,+ site_graph: SiteGraph::new(),+ job,+ }+ }+ async fn run(mut self) {+ tracing::info!("Processing job: {:?}", self.job.domain);++ self.scheduled_urls().await;+ self.wander().await;+ }++ async fn scheduled_urls(&mut self) {+ let urls = self.job.urls.drain(..).collect();+ self.process_urls(urls, true).await;+ }- async fn process_job(&mut self) {- self.politeness_factor = self.config.politeness_factor;+ async fn wander(&mut self) {+ let mut urls: Vec<(Url, f64)> = self+ .site_graph+ .compute_centralities()+ .into_iter()+ .map(|(node_ref, score)| {+ (+ Url::from(self.site_graph.get_node(node_ref).unwrap().clone()),+ score,+ )+ })+ .chain(self.sitemap_urls.drain().map(|url| (url.clone(), 0.0)))+ .filter(|(url, _)| !self.crawled_urls.contains(url))+ .filter(|(url, _)| self.job.domain == Domain::from(url))+ .filter(|(_, score)| score.is_finite())+ .collect();- let mut job = self.current_job.take().unwrap();- tracing::info!("Processing job: {:?}", job.domain);+ urls.sort_by(|(a, _), (b, _)| a.cmp(b));+ urls.dedup();- self.robotstxt.clear();+ urls.sort_by(|(_, a), (_, b)| b.total_cmp(a));- let mut url_responses: Vec<UrlResponse> = Vec::new();- let mut discovered_urls: HashSet<Url> = HashSet::new();+ let urls = urls+ .into_iter()+ .take(self.job.wanrdering_urls as usize)+ .map(|(url, _)| url)+ .map(RetrieableUrl::from)+ .collect();- let mut crawled_sitemaps: HashSet<Site> = HashSet::new();+ self.process_urls(urls, false).await;+ }++ async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>, fetch_sitemap: bool) {+ while let Some(retryable_url) = urls.pop_front() {+ if self.crawled_urls.contains(&retryable_url.url) {+ continue;+ }- while let Some(retryable_url) = job.urls.pop_front() {if retryable_url.retries > self.config.max_url_slowdown_retry {continue;}@@ -204,25 +230,22 @@ impl WorkerThread {continue;}- if !self- .robotstxt- .is_allowed(&retryable_url.url, &self.config.user_agent.token)- .await+ if !self.config.dry_run+ && !self+ .robotstxt+ .is_allowed(&retryable_url.url, &self.config.user_agent.token)+ .await{continue;}let site = Site(retryable_url.url.host_str().unwrap_or_default().to_string());- if job.fetch_sitemap && !crawled_sitemaps.contains(&site) {- crawled_sitemaps.insert(site.clone());+ if !self.config.dry_run && fetch_sitemap && !self.crawled_sitemaps.contains(&site) {+ self.crawled_sitemaps.insert(site.clone());if let Some(sitemap) = self.robotstxt.sitemap(&retryable_url.url).await {- let sitemap_urls = self.urls_from_sitemap(sitemap, 0, 5).await;- discovered_urls.extend(- sitemap_urls- .into_iter()- .filter(|url| url.as_str().len() < MAX_URL_LEN_BYTES),- );+ self.sitemap_urls+ .extend(self.urls_from_sitemap(sitemap, 0, 5).await);}}@@ -240,39 +263,39 @@ impl WorkerThread {tokio::time::sleep(delay).await;- if let UrlResponse::Failed {- url: _,- status_code,- } = res.response- {- if matches!(status_code, Some(429)) {- let mut retryable_url = retryable_url;- retryable_url.retries += 1;- job.urls.push_back(retryable_url);- continue;+ match res.response {+ UrlResponse::Success { url } => {+ self.crawled_urls.insert(url.clone());++ let from_node = self.site_graph.add_node(url.clone().into());++ for new_url in res.new_urls {+ if new_url.host_str().is_none() {+ continue;+ }++ if new_url.host_str() != retryable_url.url.host_str() {+ continue;+ }++ let to_node = self.site_graph.add_node(new_url.clone().into());+ self.site_graph.add_edge(from_node, to_node);+ }}+ UrlResponse::Failed {+ url: _,+ status_code,+ } => {+ if matches!(status_code, Some(429)) {+ let mut retryable_url = retryable_url;+ retryable_url.retries += 1;+ urls.push_back(retryable_url);+ continue;+ }+ }+ UrlResponse::Redirected { url: _, new_url: _ } => {}}-- discovered_urls.extend(- res.new_urls- .into_iter()- .filter(|url| url.as_str().len() < MAX_URL_LEN_BYTES)- .take(MAX_OUTGOING_URLS_PER_PAGE),- );- url_responses.push(res.response);}-- let job_response = JobResponse {- domain: job.domain,- url_responses,- discovered_urls: discovered_urls- .into_iter()- .filter(|url| url.as_str().len() < MAX_URL_LEN_BYTES)- .collect(),- weight_budget: job.weight_budget,- };-- self.results.lock().await.push(job_response);}async fn process_url(&mut self, url: Url) -> ProcessedUrl {@@ -290,6 +313,7 @@ impl WorkerThread {.all_links().into_iter().map(|link| link.destination)+ .filter(|url| url.as_str().len() <= MAX_URL_LEN_BYTES).filter(|url| {!url.path().ends_with(".pdf")&& !url.path().ends_with(".jpg")@@ -400,8 +424,13 @@ impl WorkerThread {}async fn fetch(&self, url: Url) -> Result<reqwest::Response> {- let backoff = ExponentialBackoff::from_millis(1000)- .with_limit(Duration::from_secs(20))+ if self.config.dry_run {+ tracing::debug!("dry run: {}", url);+ return Err(Error::FetchFailed(reqwest::StatusCode::IM_A_TEAPOT).into());+ }++ let backoff = ExponentialBackoff::from_millis(self.config.min_crawl_delay_ms)+ .with_limit(Duration::from_millis(self.config.max_crawl_delay_ms)).take(3);let mut res = Err(Error::FetchFailed(reqwest::StatusCode::IM_A_TEAPOT).into());@@ -546,57 +575,20 @@ impl WorkerThread {let body = body.unwrap();// parse xml- let mut reader = quick_xml::Reader::from_str(&body);+ let entries = parse_sitemap(&body);let mut urls = vec![];- let mut in_sitemap = false;- let mut in_url = false;- let mut in_loc = false;-- loop {- match reader.read_event() {- Ok(Event::Start(ref e)) => {- if e.name().as_ref() == b"sitemap" {- in_sitemap = true;- } else if e.name().as_ref() == b"url" {- in_url = true;- } else if e.name().as_ref() == b"loc" {- in_loc = true;- }- }- Ok(Event::End(ref e)) => {- if e.name().as_ref() == b"sitemap" {- in_sitemap = false;- } else if e.name().as_ref() == b"url" {- in_url = false;- } else if e.name().as_ref() == b"loc" {- in_loc = false;- }- }- Ok(Event::Text(e)) => {- if in_sitemap && in_loc {- if let Ok(url) = Url::parse(&e.unescape().unwrap()) {- urls.append(- &mut self.urls_from_sitemap(url, depth + 1, max_depth).await,- );- tokio::time::sleep(Duration::from_millis(- self.config.min_crawl_delay_ms,- ))- .await;- }- } else if in_url && in_loc {- if let Ok(url) = Url::parse(&e.unescape().unwrap()) {- urls.push(url);- }- }+ for entry in entries {+ match entry {+ SitemapEntry::Url(url) => {+ urls.push(url);}- Ok(Event::Eof) => break,- Err(e) => {- tracing::debug!("failed to parse sitemap: {}", e);- break;+ SitemapEntry::Sitemap(url) => {+ tokio::time::sleep(Duration::from_millis(self.config.min_crawl_delay_ms))+ .await;+ urls.append(&mut self.urls_from_sitemap(url, depth + 1, max_depth).await);}- _ => (),}}@@ -606,8 +598,154 @@ impl WorkerThread {}}-struct ProcessedUrl {- new_urls: Vec<Url>,- response: UrlResponse,- fetch_time: Duration,+#[derive(Debug, Clone, PartialEq, Eq)]+enum SitemapEntry {+ Url(Url),+ Sitemap(Url),+}++fn parse_sitemap(s: &str) -> Vec<SitemapEntry> {+ let mut reader = quick_xml::Reader::from_str(s);++ let mut res = vec![];++ let mut in_sitemap = false;+ let mut in_url = false;+ let mut in_loc = false;++ loop {+ match reader.read_event() {+ Ok(Event::Start(ref e)) => {+ if e.name().as_ref() == b"sitemap" {+ in_sitemap = true;+ } else if e.name().as_ref() == b"url" {+ in_url = true;+ } else if e.name().as_ref() == b"loc" {+ in_loc = true;+ }+ }+ Ok(Event::End(ref e)) => {+ if e.name().as_ref() == b"sitemap" {+ in_sitemap = false;+ } else if e.name().as_ref() == b"url" {+ in_url = false;+ } else if e.name().as_ref() == b"loc" {+ in_loc = false;+ }+ }+ Ok(Event::Text(e)) => {+ if in_sitemap && in_loc {+ if let Ok(url) = Url::parse(&e.unescape().unwrap()) {+ res.push(SitemapEntry::Sitemap(url));+ }+ } else if in_url && in_loc {+ if let Ok(url) = Url::parse(&e.unescape().unwrap()) {+ res.push(SitemapEntry::Url(url));+ }+ }+ }+ Ok(Event::Eof) => break,+ Err(e) => {+ tracing::debug!("failed to parse sitemap: {}", e);+ break;+ }+ _ => (),+ }+ }++ res+}++#[cfg(test)]+mod tests {+ #[test]+ fn parse_sitemap() {+ let dr = r#"<sitemapindex>+ <sitemap>+ <loc>https://www.dr.dk/drtv/sitemap.xml</loc>+ </sitemap>+ <sitemap>+ <loc>https://www.dr.dk/sitemap.tvguide.xml</loc>+ </sitemap>+ <sitemap>+ <loc>+ https://www.dr.dk/sitemap.kommunalvalg.resultater.xml+ </loc>+ </sitemap>+ <sitemap>+ <loc>https://www.dr.dk/sitemap.folketingsvalg2022.xml</loc>+ </sitemap>+ </sitemapindex>"#;++ let entries = super::parse_sitemap(dr);+ assert_eq!(+ entries,+ vec![+ super::SitemapEntry::Sitemap("https://www.dr.dk/drtv/sitemap.xml".parse().unwrap()),+ super::SitemapEntry::Sitemap(+ "https://www.dr.dk/sitemap.tvguide.xml".parse().unwrap()+ ),+ super::SitemapEntry::Sitemap(+ "https://www.dr.dk/sitemap.kommunalvalg.resultater.xml"+ .parse()+ .unwrap()+ ),+ super::SitemapEntry::Sitemap(+ "https://www.dr.dk/sitemap.folketingsvalg2022.xml"+ .parse()+ .unwrap()+ ),+ ]+ );++ let dr = r#"<urlset>+ <url>+ <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>+ <loc>https://www.dr.dk/drtv/serie/sleepover_6382</loc>+ </url>+ <url>+ <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>+ <loc>https://www.dr.dk/drtv/saeson/sleepover_9673</loc>+ </url>+ <url>+ <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>+ <loc>+ https://www.dr.dk/drtv/episode/sleepover_-zoologisk-museum_52239+ </loc>+ </url>+ <url>+ <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>+ <loc>+ https://www.dr.dk/drtv/episode/sleepover_-koebenhavns-raadhus_52252+ </loc>+ </url>+ </urlset>"#;++ let entries = super::parse_sitemap(dr);+ assert_eq!(+ entries,+ vec![+ super::SitemapEntry::Url(+ "https://www.dr.dk/drtv/serie/sleepover_6382"+ .parse()+ .unwrap()+ ),+ super::SitemapEntry::Url(+ "https://www.dr.dk/drtv/saeson/sleepover_9673"+ .parse()+ .unwrap()+ ),+ super::SitemapEntry::Url(+ "https://www.dr.dk/drtv/episode/sleepover_-zoologisk-museum_52239"+ .parse()+ .unwrap()+ ),+ super::SitemapEntry::Url(+ "https://www.dr.dk/drtv/episode/sleepover_-koebenhavns-raadhus_52252"+ .parse()+ .unwrap()+ ),+ ]+ );+ }}diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rsindex e314f2ec..fe3f96e8 100644--- a/core/src/entrypoint/crawler.rs+++ b/core/src/entrypoint/crawler.rs@@ -16,14 +16,16 @@use std::{net::SocketAddr, sync::Arc};-use hashbrown::HashMap;use serde::{Deserialize, Serialize};use crate::{config,- crawler::{self, CrawlCoordinator, Crawler, Domain, DomainCrawled, JobResponse, UrlToInsert},+ crawler::{self, planner::make_crawl_plan, CrawlCoordinator, Crawler},distributed::sonic::{self, service::Message},- sonic_service, Result,+ kv::rocksdb_store::RocksDbStore,+ sonic_service,+ webgraph::WebgraphBuilder,+ Result,};pub async fn worker(config: config::CrawlerConfig) -> Result<()> {@@ -35,7 +37,7 @@ pub async fn worker(config: config::CrawlerConfig) -> Result<()> {}pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {- let coordinator = Arc::new(CrawlCoordinator::new(config.crawldb_folder)?);+ let coordinator = Arc::new(CrawlCoordinator::new(config.job_queue)?);let addr: SocketAddr = config.host;let server = coordinator::CoordinatorService { coordinator }@@ -51,7 +53,7 @@ pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {}pub async fn router(config: config::CrawlRouterConfig) -> Result<()> {- let router = crawler::Router::new(config.coordinator_addrs.clone(), config.seed_urls).await?;+ let router = crawler::Router::new(config.coordinator_addrs.clone()).await?;let addr: SocketAddr = config.host;@@ -64,28 +66,44 @@ pub async fn router(config: config::CrawlRouterConfig) -> Result<()> {}}+pub fn planner(config: config::CrawlPlannerConfig) -> Result<()> {+ let page_centrality = RocksDbStore::open(&config.page_harmonic_path);+ let host_centrality = RocksDbStore::open(&config.host_harmonic_path);+ let page_graph = WebgraphBuilder::new(&config.page_graph_path).open();+ let host_graph = WebgraphBuilder::new(&config.host_graph_path).open();+ let output_path = config.output_path.clone();++ make_crawl_plan(+ host_centrality,+ page_centrality,+ host_graph,+ page_graph,+ config,+ output_path,+ )?;++ Ok(())+}+pub mod router {+ use crate::crawler::Job;+use super::*;pub struct RouterService {pub router: crawler::Router,}- sonic_service!(RouterService, [NewJobs]);+ sonic_service!(RouterService, [NewJob]);#[derive(Debug, Clone, Serialize, Deserialize)]- pub struct NewJobs {- pub responses: Vec<JobResponse>,- pub num_jobs: usize,- }+ pub struct NewJob {}#[async_trait::async_trait]- impl Message<RouterService> for NewJobs {- type Response = crate::crawler::Response;+ impl Message<RouterService> for NewJob {+ type Response = Option<Job>;async fn handle(self, server: &RouterService) -> sonic::Result<Self::Response> {- server.router.add_responses(&self.responses).await?;- let jobs = server.router.sample_jobs(self.num_jobs).await?;- Ok(crate::crawler::Response::NewJobs { jobs })+ Ok(server.router.sample_job().await?)}}}@@ -99,56 +117,18 @@ pub mod coordinator {pub coordinator: Arc<CrawlCoordinator>,}- sonic_service!(CoordinatorService, [InsertUrls, GetJobs, MarkJobsComplete]);+ sonic_service!(CoordinatorService, [GetJob]);#[derive(Debug, Clone, Serialize, Deserialize)]- pub struct InsertUrls {- pub urls: HashMap<Domain, Vec<UrlToInsert>>,- }-- #[derive(Debug, Clone, Serialize, Deserialize)]- pub struct GetJobs {- pub num_jobs: usize,- }-- #[derive(Debug, Clone, Serialize, Deserialize)]- pub struct MarkJobsComplete {- pub domains: Vec<DomainCrawled>,- }+ pub struct GetJob {}#[async_trait::async_trait]- impl Message<CoordinatorService> for InsertUrls {- type Response = ();+ impl Message<CoordinatorService> for GetJob {+ type Response = Option<Job>;async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {- if let Err(e) = server.coordinator.insert_urls(self.urls) {- tracing::error!("failed to insert urls: {}", e);- }-- Ok(())- }- }-- #[async_trait::async_trait]- impl Message<CoordinatorService> for GetJobs {- type Response = Vec<Job>;-- async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {- let jobs = server.coordinator.sample_jobs(self.num_jobs)?;- Ok(jobs)- }- }-- #[async_trait::async_trait]- impl Message<CoordinatorService> for MarkJobsComplete {- type Response = ();-- async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {- if let Err(e) = server.coordinator.mark_jobs_complete(&self.domains) {- tracing::error!("failed to mark jobs complete: {}", e);- }-- Ok(())+ let job = server.coordinator.sample_job()?;+ Ok(job)}}}diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 897a5e39..0104e1e4 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -212,11 +212,11 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {has_backlinks = true;}- if !page_centrality.is_normal() {+ if !page_centrality.is_finite() {page_centrality = 0.0;}- if !host_centrality.is_normal() {+ if !host_centrality.is_finite() {host_centrality = 0.0;}diff --git a/core/src/main.rs b/core/src/main.rsindex ad06b3ed..9876fcdd 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -122,6 +122,9 @@ enum Crawler {/// Deploy the crawl router. The crawl router is responsible for routing job responses and requests/// from the workers to the correct crawl coordinators.Router { config_path: String },++ /// Create a crawl plan.+ Plan { config_path: String },}/// Commands to train or run inference on the classifier that predicts if a webpage is NSFW or SFW.@@ -326,6 +329,11 @@ fn main() -> Result<()> {.build()?.block_on(entrypoint::crawler::router(config))?}+ Crawler::Plan { config_path } => {+ let config: config::CrawlPlannerConfig = load_toml_config(config_path);++ entrypoint::crawler::planner(config)?;+ }},Commands::SafetyClassifier { options } => match options {SafetyClassifierOptions::Train {diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex 14eab905..3b1d55f6 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -159,6 +159,13 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {.map(|(node_id, sum)| (node_id, f64::from(sum))).filter(|(_, centrality)| *centrality > 0.0).map(|(node_id, centrality)| (node_id, centrality / norm_factor))+ .map(|(node_id, centrality)| {+ if !centrality.is_finite() {+ (node_id, 0.0)+ } else {+ (node_id, centrality)+ }+ }).collect();info!("Harmonic centrality calculated");