web search done right
Live index crawler (#226)
Mikkel Denker5cd174e415f4
diff --git a/crates/core/examples/distributed_harmonic.rs b/crates/core/examples/distributed_harmonic.rsindex a2960914..28eb6d9a 100644--- a/crates/core/examples/distributed_harmonic.rs+++ b/crates/core/examples/distributed_harmonic.rs@@ -19,7 +19,6 @@ fn start_dht_thread(id: u64, host: SocketAddr, gossip: SocketAddr) {shard: ShardId::new(id),seed_node: None,gossip: Some(stract::config::GossipConfig {- cluster_id: "test".to_string(),seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),addr: gossip,}),@@ -38,7 +37,6 @@ fn start_worker_thread(graph_path: String, shard: ShardId, host: SocketAddr, gosstd::thread::spawn(move || {let config = stract::config::HarmonicWorkerConfig {gossip: stract::config::GossipConfig {- cluster_id: "test".to_string(),seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),addr: gossip,},@@ -158,7 +156,6 @@ fn main() -> anyhow::Result<()> {let config = stract::config::HarmonicCoordinatorConfig {gossip: stract::config::GossipConfig {- cluster_id: "test".to_string(),seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),addr: "0.0.0.0:3007".parse().unwrap(),},diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rsindex 2c3081f1..c338cbe6 100644--- a/crates/core/examples/search_preindexed.rs+++ b/crates/core/examples/search_preindexed.rs@@ -30,7 +30,6 @@ pub async fn main() {dual_encoder_model_path: None,bangs_path: Some("data/bangs.json".to_string()),query_store_db_host: None,- cluster_id: "api".to_string(),gossip_seed_nodes: None,gossip_addr: "0.0.0.0:8002".parse().unwrap(),collector: collector_conf.clone(),diff --git a/crates/core/public_suffix_list.dat b/crates/core/public_suffix_list.datindex 021578e9..88dda820 100644--- a/crates/core/public_suffix_list.dat+++ b/crates/core/public_suffix_list.dat@@ -13765,4 +13765,8 @@ enterprisecloud.nu// Added by Stractwordpress.com+// tumblr+// Added by Stract+tumblr.com+// ===END PRIVATE DOMAINS===diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rsindex f7d059ed..b01c04a3 100644--- a/crates/core/src/api/mod.rs+++ b/crates/core/src/api/mod.rs@@ -33,7 +33,7 @@ use crate::{ranking::models::lambdamart::LambdaMART,searcher::{api::ApiSearcher, live::LiveSearcher, DistributedSearcher, SearchClient},similar_hosts::SimilarHostsFinder,- webgraph::remote::RemoteWebgraph,+ webgraph::remote::{Host, Page, RemoteWebgraph},};use crate::ranking::models::cross_encoder::CrossEncoderModel;@@ -72,14 +72,14 @@ pub struct Counters {pub struct State {pub config: ApiConfig,- pub searcher: Arc<ApiSearcher<DistributedSearcher, LiveSearcher, Arc<RemoteWebgraph>>>,- pub page_webgraph: Arc<RemoteWebgraph>,- pub host_webgraph: Arc<RemoteWebgraph>,+ pub searcher: Arc<ApiSearcher<DistributedSearcher, LiveSearcher, Arc<RemoteWebgraph<Host>>>>,+ pub page_webgraph: Arc<RemoteWebgraph<Page>>,+ pub host_webgraph: Arc<RemoteWebgraph<Host>>,pub autosuggest: Autosuggest,pub counters: Counters,pub improvement_queue: Option<Arc<Mutex<LeakyQueue<ImprovementEvent>>>>,pub _cluster: Arc<Cluster>,- pub similar_hosts: SimilarHostsFinder,+ pub similar_hosts: SimilarHostsFinder<Host>,}pub async fn favicon() -> impl IntoResponse {@@ -172,10 +172,8 @@ pub async fn router(None => Bangs::empty(),};- let host_webgraph =- RemoteWebgraph::new(cluster.clone(), crate::config::WebgraphGranularity::Host).await;- let page_webgraph =- RemoteWebgraph::new(cluster.clone(), crate::config::WebgraphGranularity::Page).await;+ let host_webgraph = RemoteWebgraph::<Host>::new(cluster.clone()).await;+ let page_webgraph = RemoteWebgraph::<Page>::new(cluster.clone()).await;let dist_searcher = DistributedSearcher::new(Arc::clone(&cluster)).await;let live_searcher = LiveSearcher::new(Arc::clone(&cluster));diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rsindex b6c03a4b..4cd6b521 100644--- a/crates/core/src/api/webgraph.rs+++ b/crates/core/src/api/webgraph.rs@@ -156,6 +156,10 @@ pub mod host {}pub mod page {+ use url::Url;++ use crate::webpage::url_ext::UrlExt;+use super::*;#[derive(serde::Deserialize, IntoParams)]@@ -175,7 +179,8 @@ pub mod page {extract::State(state): extract::State<Arc<State>>,extract::Query(params): extract::Query<PageLinksParams>,) -> std::result::Result<impl IntoResponse, StatusCode> {- let node = Node::from(params.page);+ let page = Url::robust_parse(¶ms.page).map_err(|_| StatusCode::BAD_REQUEST)?;+ let node = Node::from(page);let links = ingoing_links(state, node, WebgraphGranularity::Page).await.map_err(|_| {@@ -197,7 +202,8 @@ pub mod page {extract::State(state): extract::State<Arc<State>>,extract::Query(params): extract::Query<PageLinksParams>,) -> std::result::Result<impl IntoResponse, StatusCode> {- let node = Node::from(params.page);+ let url = Url::robust_parse(¶ms.page).map_err(|_| StatusCode::BAD_REQUEST)?;+ let node = Node::from(url);let links = outgoing_links(state, node, WebgraphGranularity::Page).await.map_err(|_| {@@ -214,12 +220,20 @@ async fn ingoing_links(node: Node,level: WebgraphGranularity,) -> anyhow::Result<Vec<FullEdge>> {- let graph = match level {- WebgraphGranularity::Host => &state.host_webgraph,- WebgraphGranularity::Page => &state.page_webgraph,- };-- graph.ingoing_edges(node, EdgeLimit::Limit(1024)).await+ match level {+ WebgraphGranularity::Host => {+ state+ .host_webgraph+ .ingoing_edges(node, EdgeLimit::Limit(1024))+ .await+ }+ WebgraphGranularity::Page => {+ state+ .page_webgraph+ .ingoing_edges(node, EdgeLimit::Limit(1024))+ .await+ }+ }}async fn outgoing_links(@@ -227,12 +241,20 @@ async fn outgoing_links(node: Node,level: WebgraphGranularity,) -> anyhow::Result<Vec<FullEdge>> {- let graph = match level {- WebgraphGranularity::Host => &state.host_webgraph,- WebgraphGranularity::Page => &state.page_webgraph,- };-- graph.outgoing_edges(node, EdgeLimit::Limit(1024)).await+ match level {+ WebgraphGranularity::Host => {+ state+ .host_webgraph+ .outgoing_edges(node, EdgeLimit::Limit(1024))+ .await+ }+ WebgraphGranularity::Page => {+ state+ .page_webgraph+ .outgoing_edges(node, EdgeLimit::Limit(1024))+ .await+ }+ }}#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema)]diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex ab52f037..c6ef0910 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -14,6 +14,8 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::time::Duration;+pub struct Collector;impl Collector {@@ -211,3 +213,31 @@ impl Webgraph {true}}++pub struct LiveIndex;++impl LiveIndex {+ pub fn feeds_crawl_interval() -> Duration {+ Duration::from_secs(30 * 60) // 30 minutes+ }++ pub fn sitemap_crawl_interval() -> Duration {+ Duration::from_secs(24 * 60 * 60) // daily+ }++ pub fn frontpage_crawl_interval() -> Duration {+ Duration::from_secs(12 * 60 * 60) // 12 hours+ }++ pub fn blogs_budget() -> u64 {+ 300_000+ }++ pub fn news_budget() -> u64 {+ 500_000+ }++ pub fn remaining_budget() -> u64 {+ 200_000+ }+}diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 44532423..731288d5 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -23,6 +23,38 @@ use crate::distributed::member::ShardId;use std::fs::File;use std::io::{self, BufRead};use std::net::SocketAddr;+use std::path::PathBuf;+use std::time::Duration;++pub fn parse_duration<'de, D: serde::de::Deserializer<'de>>(+ deserializer: D,+) -> Result<Duration, D::Error> {+ let err = <D::Error as serde::de::Error>::custom;+ let s: String = serde::de::Deserialize::deserialize(deserializer)?;+ let num_part = s.trim_end_matches(|c: char| !c.is_numeric());+ let suffix = &s[num_part.len()..];+ let num: u64 = num_part+ .parse()+ .map_err(|_| err("invalid number".to_string()))?;++ let ret = match suffix.trim() {+ "s" => Duration::from_secs(num),+ "sec" => Duration::from_secs(num),+ "secs" => Duration::from_secs(num),+ "ms" => Duration::from_millis(num),+ "millis" => Duration::from_millis(num),+ "milliseconds" => Duration::from_millis(num),+ "m" => Duration::from_secs(num * 60),+ "mins" => Duration::from_secs(num * 60),+ "minutes" => Duration::from_secs(num * 60),+ "h" => Duration::from_secs(num * 60 * 60),+ "hours" => Duration::from_secs(num * 60 * 60),+ "d" => Duration::from_secs(num * 24 * 60 * 60),+ "days" => Duration::from_secs(num * 24 * 60 * 60),+ other => return Err(err(format!("invalid suffix {other}"))),+ };+ Ok(ret)+}#[derive(Debug, serde::Deserialize, Clone)]pub struct IndexerConfig {@@ -230,7 +262,6 @@ pub struct ApiConfig {pub dual_encoder_model_path: Option<String>,pub bangs_path: Option<String>,pub query_store_db_host: Option<String>,- pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,@@ -299,7 +330,6 @@ impl Default for SnippetConfig {#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct SearchServerConfig {- pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,pub shard: ShardId,@@ -317,7 +347,6 @@ pub struct SearchServerConfig {#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct EntitySearchServerConfig {- pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,pub index_path: String,@@ -367,6 +396,47 @@ pub struct CrawlerConfig {pub router_hosts: Vec<String>,}+#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]+pub struct DailyLiveIndexCrawlerBudget {+ #[serde(default = "defaults::LiveIndex::blogs_budget")]+ pub blogs: u64,+ #[serde(default = "defaults::LiveIndex::news_budget")]+ pub news: u64,+ #[serde(default = "defaults::LiveIndex::remaining_budget")]+ pub remaining: u64,+}++#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]+pub struct CheckIntervals {+ #[serde(+ deserialize_with = "parse_duration",+ default = "defaults::LiveIndex::feeds_crawl_interval"+ )]+ pub feeds: Duration,+ #[serde(+ deserialize_with = "parse_duration",+ default = "defaults::LiveIndex::sitemap_crawl_interval"+ )]+ pub sitemap: Duration,+ #[serde(+ deserialize_with = "parse_duration",+ default = "defaults::LiveIndex::frontpage_crawl_interval"+ )]+ pub frontpage: Duration,+}++#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]+pub struct LiveCrawlerConfig {+ pub crawled_db_path: PathBuf,+ pub gossip: GossipConfig,+ pub site_stats_path: PathBuf,+ pub host_centrality_path: PathBuf,+ pub user_agent: UserAgent,+ pub num_worker_threads: usize,+ pub check_intervals: CheckIntervals,+ pub daily_budget: DailyLiveIndexCrawlerBudget,+}+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct CrawlRouterConfig {pub host: SocketAddr,@@ -415,7 +485,6 @@ pub struct WebgraphServerConfig {pub graph_path: String,pub granularity: WebgraphGranularity,- pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,}@@ -472,34 +541,11 @@ pub struct CrawlPlannerConfig {#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct LiveIndexConfig {- // crawler- pub user_agent: UserAgent,- #[serde(default = "defaults::Crawler::robots_txt_cache_sec")]- pub robots_txt_cache_sec: u64,- #[serde(default = "defaults::Crawler::min_politeness_factor")]- pub min_politeness_factor: u32,- #[serde(default = "defaults::Crawler::start_politeness_factor")]- pub start_politeness_factor: u32,- #[serde(default = "defaults::Crawler::min_crawl_delay_ms")]- pub min_crawl_delay_ms: u64,- #[serde(default = "defaults::Crawler::max_crawl_delay_ms")]- pub max_crawl_delay_ms: u64,- #[serde(default = "defaults::Crawler::max_politeness_factor")]- pub max_politeness_factor: u32,- #[serde(default = "defaults::Crawler::max_url_slowdown_retry")]- pub max_url_slowdown_retry: u8,- #[serde(default = "defaults::Crawler::timeout_seconds")]- pub timeout_seconds: u64,-- // indexerpub host_centrality_store_path: String,pub page_centrality_store_path: Option<String>,pub safety_classifier_path: Option<String>,pub host_centrality_threshold: Option<f64>,pub minimum_clean_words: Option<usize>,-- // search- pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,pub shard_id: ShardId,@@ -513,22 +559,6 @@ pub struct LiveIndexConfig {pub snippet: SnippetConfig,}-#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]-pub struct LiveIndexSchedulerConfig {- pub schedule_path: String,- pub feed_index_path: String,- pub host_centrality_store_path: String,- pub host_graph_path: String,- pub num_splits: u64,-}-#[derive(Debug, serde::Deserialize, Clone)]-pub struct FeedIndexingConfig {- pub output_path: String,- pub warc_source: WarcSource,- pub limit_warc_files: Option<usize>,- pub skip_warc_files: Option<usize>,-}-#[derive(Debug, serde::Deserialize, Clone)]pub struct WebSpellConfig {pub output_path: String,@@ -577,7 +607,6 @@ impl Default for CorrectionConfig {#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]pub struct GossipConfig {- pub cluster_id: String,pub seed_nodes: Option<Vec<SocketAddr>>,pub addr: SocketAddr,}diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rsindex 2aee88f3..531f8409 100644--- a/crates/core/src/crawler/mod.rs+++ b/crates/core/src/crawler/mod.rs@@ -58,6 +58,9 @@ pub enum Error {#[error("content too large")]ContentTooLarge,+ #[error("couldn't read response body")]+ ResponseBodyReadFailed,+#[error("invalid politeness factor")]InvalidPolitenessFactor,@@ -338,7 +341,7 @@ pub async fn encoded_body(res: reqwest::Response) -> Result<String> {let mut stream = res.bytes_stream();while let Some(b) = stream.next().await {if b.is_err() {- return Err(Error::ContentTooLarge);+ return Err(Error::ResponseBodyReadFailed);}let b = b.unwrap();diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rsindex f1ab3029..4a10bbec 100644--- a/crates/core/src/crawler/planner.rs+++ b/crates/core/src/crawler/planner.rs@@ -29,11 +29,10 @@ use std::{};use url::Url;-use crate::config::WebgraphGranularity;use crate::crawler::WeightedUrl;use crate::distributed::cluster::Cluster;use crate::external_sort::ExternalSorter;-use crate::webgraph::remote::RemoteWebgraph;+use crate::webgraph::remote::{Host, Page, RemoteWebgraph};use crate::webgraph::Node;use crate::webpage::url_ext::UrlExt;use crate::SortableFloat;@@ -148,8 +147,8 @@ struct Budget {pub struct CrawlPlanner {host_centrality: speedy_kv::Db<NodeID, f64>,page_centrality: speedy_kv::Db<NodeID, f64>,- page_graph: RemoteWebgraph,- host_graph: RemoteWebgraph,+ page_graph: RemoteWebgraph<Page>,+ host_graph: RemoteWebgraph<Host>,config: CrawlPlannerConfig,excluded_domains: HashSet<Domain>,@@ -166,8 +165,8 @@ impl CrawlPlanner {) -> Result<Self> {Self::check_config(&config)?;- let page_graph = RemoteWebgraph::new(cluster.clone(), WebgraphGranularity::Page).await;- let host_graph = RemoteWebgraph::new(cluster, WebgraphGranularity::Host).await;+ let page_graph = RemoteWebgraph::new(cluster.clone()).await;+ let host_graph = RemoteWebgraph::new(cluster.clone()).await;let domain_boosts = config.domain_boostsdiff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rsindex d62b5b03..88033c11 100644--- a/crates/core/src/crawler/robots_txt.rs+++ b/crates/core/src/crawler/robots_txt.rs@@ -24,6 +24,7 @@ use super::{encoded_body, Result, Site};const RETRY_ROBOTSTXT_UNREACHABLE: bool = false;+#[derive(Debug)]enum Lookup<T> {Found(T),/// 404@@ -113,12 +114,12 @@ impl RobotsTxtManager {async fn fetch_robots_txt_without_retry(&self, site: &Site) -> Lookup<RobotsTxt> {match self- .fetch_robots_txt_from_url(&format!("http://{}/robots.txt", site.0))+ .fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0)).await{- Lookup::Unavailable => {+ Lookup::Unavailable | Lookup::Unreachable => {match self- .fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0))+ .fetch_robots_txt_from_url(&format!("http://{}/robots.txt", site.0)).await{Lookup::Found(robots_txt) => Lookup::Found(robots_txt),@@ -136,7 +137,7 @@ impl RobotsTxtManager {Lookup::Unavailable => Lookup::Unavailable,}}- res => res,+ Lookup::Found(robots_txt) => Lookup::Found(robots_txt),}}@@ -202,6 +203,7 @@ impl RobotsTxtManager {}}+#[derive(Debug)]struct RobotsTxt {download_time: std::time::Instant,robots: robotstxt::Robots,diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex 9ee134e6..f245377d 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -15,7 +15,6 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::anyhow;use hashbrown::HashSet;-use quick_xml::events::Event;use rand::seq::SliceRandom;use std::{@@ -30,8 +29,10 @@ use url::Url;use crate::{config::CrawlerConfig,crawler::MAX_URL_LEN_BYTES,+ dated_url::DatedUrl,distributed::{retry_strategy::ExponentialBackoff, sonic},entrypoint::crawler::router::{NewJob, RouterService},+ sitemap::{parse_sitemap, SitemapEntry},warc,webpage::{url_ext::UrlExt, Html},};@@ -195,7 +196,10 @@ impl<S: DatumStream> JobExecutor<S> {}self.scheduled_urls().await;- self.crawl_sitemaps().await;++ if self.wandered_urls < self.job.wandering_urls {+ self.crawl_sitemaps().await;+ }while self.wandered_urls < self.job.wandering_urls&& self.wander_prioritiser.known_urls() > 0@@ -288,16 +292,22 @@ impl<S: DatumStream> JobExecutor<S> {let sitemaps = self.robotstxt.sitemaps(url).await;for sitemap in sitemaps {- self.sitemap_urls- .extend(self.urls_from_sitemap(sitemap, 5).await);+ self.sitemap_urls.extend(+ self.urls_from_sitemap(sitemap, 5)+ .await+ .into_iter()+ .map(|dated_url| dated_url.url),+ );}}}}- async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>) {+ pub async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>) {+ tracing::debug!("processing {} urls", urls.len());while let Some(retryable_url) = urls.pop_front() {if let UrlVisit::Skip = self.verify_url(&retryable_url).await {+ tracing::debug!("skipping url: {}", retryable_url.url());continue;}@@ -420,7 +430,7 @@ impl<S: DatumStream> JobExecutor<S> {}async fn process_url(&mut self, url: Url) -> Result<ProcessedUrl> {- let datum = self.crawl_url(url.clone()).await?;+ let datum = self.polite_crawl_url(url.clone()).await?;self.save_datum(datum.clone()).await;match Html::parse(&datum.body, datum.url.as_str()) {@@ -527,7 +537,6 @@ impl<S: DatumStream> JobExecutor<S> {res: &reqwest::Response,url: &Url,payload_type: warc::PayloadType,- fetch_time: Duration,) -> Result<Option<CrawlDatum>> {let status_code = res.status().as_u16();@@ -547,38 +556,27 @@ impl<S: DatumStream> JobExecutor<S> {url,payload_type,body: String::new(),- fetch_time_ms: fetch_time.as_millis() as u64,+ fetch_time_ms: 0,}))} else {Ok(None)}}- async fn crawl_url(&mut self, url: Url) -> Result<CrawlDatum> {- let mut url = url;- url.normalize_in_place();-- if self.crawled_urls.contains(&url) {- return Err(Error::from(anyhow!("url already crawled: {}", url)));- }-- let start = Instant::now();- let res = self.fetch_with_https_priority(url.clone()).await;- let fetch_time = start.elapsed();- self.politeness_delay(fetch_time).await;-- // we want to delay before returning the error- let res = res?;+ async fn unpolite_crawl_url(&mut self, url: Url) -> Result<CrawlDatum> {+ tracing::debug!("crawling url: {}", url);- self.crawled_urls.insert(url.clone());- let payload_type = self.check_headers(&res)?;-- if let Some(datum) = self.redirect_datum(&res, &url, payload_type, fetch_time)? {- return Ok(datum);- }+ let res = self.fetch_with_https_priority(url.clone()).await?;+ let payload_type = self.check_headers(&res);let status_code = res.status();+ let mut res_url = res.url().clone();+ if let Ok(payload_type) = payload_type {+ if let Ok(Some(datum)) = self.redirect_datum(&res, &url, payload_type) {+ return Ok(datum);+ }+ }if status_code != reqwest::StatusCode::OK {return Err(Error::FetchFailed {status_code,@@ -586,22 +584,40 @@ impl<S: DatumStream> JobExecutor<S> {});}- let mut res_url = res.url().clone();+ let body = encoded_body(res).await;++ self.crawled_urls.insert(url.clone());+res_url.normalize_in_place();self.crawled_urls.insert(res_url.clone());- let body = encoded_body(res).await?;-Ok(CrawlDatum {url: res_url,- body,- payload_type,- fetch_time_ms: fetch_time.as_millis() as u64,+ body: body?,+ payload_type: payload_type?,+ fetch_time_ms: 0,})}- async fn urls_from_sitemap(&self, sitemap: Url, max_depth: usize) -> Vec<Url> {+ async fn polite_crawl_url(&mut self, url: Url) -> Result<CrawlDatum> {+ let mut url = url;+ url.normalize_in_place();++ if self.crawled_urls.contains(&url) {+ return Err(Error::from(anyhow!("url already crawled: {}", url)));+ }++ let start = Instant::now();+ let res = self.unpolite_crawl_url(url).await;+ let fetch_time = start.elapsed();+ self.politeness_delay(fetch_time).await;+ let mut datum = res?;+ datum.fetch_time_ms = fetch_time.as_millis() as u64;+ Ok(datum)+ }++ async fn urls_from_sitemap(&self, sitemap: Url, max_depth: usize) -> Vec<DatedUrl> {let mut stack = vec![(sitemap, 0)];let mut urls = vec![];@@ -648,155 +664,3 @@ impl<S: DatumStream> JobExecutor<S> {urls}}--#[derive(Debug, Clone, PartialEq, Eq)]-enum SitemapEntry {- Url(Url),- Sitemap(Url),-}--fn parse_sitemap(s: &str) -> Vec<SitemapEntry> {- let mut reader = quick_xml::Reader::from_str(s);-- let mut res = vec![];-- let mut in_sitemap = false;- let mut in_url = false;- let mut in_loc = false;-- loop {- match reader.read_event() {- Ok(Event::Start(ref e)) => {- if e.name().as_ref() == b"sitemap" {- in_sitemap = true;- } else if e.name().as_ref() == b"url" {- in_url = true;- } else if e.name().as_ref() == b"loc" {- in_loc = true;- }- }- Ok(Event::End(ref e)) => {- if e.name().as_ref() == b"sitemap" {- in_sitemap = false;- } else if e.name().as_ref() == b"url" {- in_url = false;- } else if e.name().as_ref() == b"loc" {- in_loc = false;- }- }- Ok(Event::Text(e)) => {- if in_sitemap && in_loc {- if let Ok(url) = Url::parse(&e.unescape().unwrap()) {- res.push(SitemapEntry::Sitemap(url));- }- } else if in_url && in_loc {- if let Ok(url) = Url::parse(&e.unescape().unwrap()) {- res.push(SitemapEntry::Url(url));- }- }- }- Ok(Event::Eof) => break,- Err(e) => {- tracing::debug!("failed to parse sitemap: {}", e);- break;- }- _ => (),- }- }-- res-}--#[cfg(test)]-mod tests {- #[test]- fn parse_sitemap() {- let dr = r#"<sitemapindex>- <sitemap>- <loc>https://www.dr.dk/drtv/sitemap.xml</loc>- </sitemap>- <sitemap>- <loc>https://www.dr.dk/sitemap.tvguide.xml</loc>- </sitemap>- <sitemap>- <loc>- https://www.dr.dk/sitemap.kommunalvalg.resultater.xml- </loc>- </sitemap>- <sitemap>- <loc>https://www.dr.dk/sitemap.folketingsvalg2022.xml</loc>- </sitemap>- </sitemapindex>"#;-- let entries = super::parse_sitemap(dr);- assert_eq!(- entries,- vec![- super::SitemapEntry::Sitemap("https://www.dr.dk/drtv/sitemap.xml".parse().unwrap()),- super::SitemapEntry::Sitemap(- "https://www.dr.dk/sitemap.tvguide.xml".parse().unwrap()- ),- super::SitemapEntry::Sitemap(- "https://www.dr.dk/sitemap.kommunalvalg.resultater.xml"- .parse()- .unwrap()- ),- super::SitemapEntry::Sitemap(- "https://www.dr.dk/sitemap.folketingsvalg2022.xml"- .parse()- .unwrap()- ),- ]- );-- let dr = r#"<urlset>- <url>- <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>- <loc>https://www.dr.dk/drtv/serie/sleepover_6382</loc>- </url>- <url>- <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>- <loc>https://www.dr.dk/drtv/saeson/sleepover_9673</loc>- </url>- <url>- <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>- <loc>- https://www.dr.dk/drtv/episode/sleepover_-zoologisk-museum_52239- </loc>- </url>- <url>- <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>- <loc>- https://www.dr.dk/drtv/episode/sleepover_-koebenhavns-raadhus_52252- </loc>- </url>- </urlset>"#;-- let entries = super::parse_sitemap(dr);- assert_eq!(- entries,- vec![- super::SitemapEntry::Url(- "https://www.dr.dk/drtv/serie/sleepover_6382"- .parse()- .unwrap()- ),- super::SitemapEntry::Url(- "https://www.dr.dk/drtv/saeson/sleepover_9673"- .parse()- .unwrap()- ),- super::SitemapEntry::Url(- "https://www.dr.dk/drtv/episode/sleepover_-zoologisk-museum_52239"- .parse()- .unwrap()- ),- super::SitemapEntry::Url(- "https://www.dr.dk/drtv/episode/sleepover_-koebenhavns-raadhus_52252"- .parse()- .unwrap()- ),- ]- );- }-}diff --git a/crates/core/src/dated_url.rs b/crates/core/src/dated_url.rsnew file mode 100644index 00000000..09b68136--- /dev/null+++ b/crates/core/src/dated_url.rs@@ -0,0 +1,36 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use chrono::{DateTime, Utc};+use url::Url;++#[derive(+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Hash,+)]+pub struct DatedUrl {+ #[bincode(with_serde)]+ pub url: Url,+ #[bincode(with_serde)]+ pub last_modified: Option<DateTime<Utc>>,+}diff --git a/crates/core/src/distributed/cluster.rs b/crates/core/src/distributed/cluster.rsindex 1cf1f1cb..f31dc39d 100644--- a/crates/core/src/distributed/cluster.rs+++ b/crates/core/src/distributed/cluster.rs@@ -92,7 +92,6 @@ impl Cluster {}pub async fn join_as_spectator(- cluster_id: String,gossip_addr: SocketAddr,seed_addrs: Vec<SocketAddr>,) -> Result<Self> {@@ -104,7 +103,7 @@ impl Cluster {let uuid = uuid::Uuid::new_v4().to_string();let node_id = NodeId {- id: format!("{}_{}", cluster_id, uuid),+ id: format!("{}_{}", CLUSTER_ID, uuid),gossip_public_address: gossip_addr,};let config = ChitchatConfig {diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rsindex 2aac7228..512bdedc 100644--- a/crates/core/src/distributed/member.rs+++ b/crates/core/src/distributed/member.rs@@ -179,3 +179,10 @@ pub struct Member {pub id: String,pub service: Service,}++impl Member {+ pub fn new(service: Service) -> Self {+ let id = uuid::Uuid::new_v4().to_string();+ Self { id, service }+ }+}diff --git a/crates/core/src/distributed/sonic/replication.rs b/crates/core/src/distributed/sonic/replication.rsindex 7727a6a0..80e344e1 100644--- a/crates/core/src/distributed/sonic/replication.rs+++ b/crates/core/src/distributed/sonic/replication.rs@@ -341,6 +341,10 @@ wherepub fn new(id: Id, replicas: ReplicatedClient<S>) -> Self {Self { replicas, id }}++ pub fn id(&self) -> &Id {+ &self.id+ }}pub struct ShardedClient<S: sonic::service::Service, Id: ShardIdentifier> {@@ -360,6 +364,10 @@ whereself.shards.is_empty()}+ pub fn shards(&self) -> &[Shard<S, Id>] {+ &self.shards+ }+async fn send_single<Req, Sel>(&self,req: Req,@@ -397,6 +405,10 @@ wherefutures.push(self.send_single(req.clone(), shard, replica_selector, timeout));}+ if futures.is_empty() {+ return Err(anyhow::anyhow!("no shards available").into());+ }+let mut results = Vec::new();for r in join_all(futures).await {match r {@@ -478,7 +490,6 @@ pub trait ReusableClientManager {type ShardId: ShardIdentifier;fn new_client(- &self,cluster: &Cluster,) -> impl std::future::Future<Output = ShardedClient<Self::Service, Self::ShardId>>;}@@ -490,29 +501,27 @@ wherecluster: Arc<Cluster>,client: Arc<sonic::replication::ShardedClient<M::Service, M::ShardId>>,last_client_update: std::time::Instant,- manager: M,}impl<M> ReusableShardedClient<M>whereM: ReusableClientManager,{- pub async fn new(cluster: Arc<Cluster>, manager: M) -> Self {- let client = Arc::new(manager.new_client(&cluster).await);+ pub async fn new(cluster: Arc<Cluster>) -> Self {+ let client = Arc::new(M::new_client(&cluster).await);let last_client_update = std::time::Instant::now();Self {cluster,client,last_client_update,- manager,}}pub async fn conn(&mut self) -> Arc<sonic::replication::ShardedClient<M::Service, M::ShardId>> {if self.client.is_empty() || self.last_client_update.elapsed() > M::CLIENT_REFRESH_INTERVAL{- self.client = Arc::new(self.manager.new_client(&self.cluster).await);+ self.client = Arc::new(M::new_client(&self.cluster).await);self.last_client_update = std::time::Instant::now();}diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rsindex a5d41d38..50b9b614 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs@@ -162,10 +162,7 @@ struct ClusterInfo {async fn setup_gossip(config: ApproxHarmonicCoordinatorConfig) -> Result<ClusterInfo> {let handle = Cluster::join(- Member {- id: config.gossip.cluster_id,- service: Service::ApproxHarmonicCoordinator { host: config.host },- },+ Member::new(Service::ApproxHarmonicCoordinator { host: config.host }),config.gossip.addr,config.gossip.seed_nodes.unwrap_or_default(),)diff --git a/crates/core/src/entrypoint/ampc/dht.rs b/crates/core/src/entrypoint/ampc/dht.rsindex 74afb2c7..9b359508 100644--- a/crates/core/src/entrypoint/ampc/dht.rs+++ b/crates/core/src/entrypoint/ampc/dht.rs@@ -113,13 +113,10 @@ pub async fn run<C: Into<Config>>(config: C) -> Result<()> {let _cluster_handle = match config.gossip {Some(gossip) => Some(Cluster::join(- Member {- id: gossip.cluster_id,- service: Service::Dht {- host: config.host,- shard: config.shard,- },- },+ Member::new(Service::Dht {+ host: config.host,+ shard: config.shard,+ }),gossip.addr,gossip.seed_nodes.unwrap_or_default(),)diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rsindex 84822151..cf28c839 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs@@ -144,10 +144,7 @@ struct ClusterInfo {async fn setup_gossip(config: HarmonicCoordinatorConfig) -> Result<ClusterInfo> {let handle = Cluster::join(- Member {- id: config.gossip.cluster_id,- service: Service::HarmonicCoordinator { host: config.host },- },+ Member::new(Service::HarmonicCoordinator { host: config.host }),config.gossip.addr,config.gossip.seed_nodes.unwrap_or_default(),)diff --git a/crates/core/src/entrypoint/api.rs b/crates/core/src/entrypoint/api.rsindex 5f4025af..3efe16a3 100644--- a/crates/core/src/entrypoint/api.rs+++ b/crates/core/src/entrypoint/api.rs@@ -90,10 +90,7 @@ fn counters(registry: &mut crate::metrics::PrometheusRegistry) -> Result<Counterasync fn cluster(config: &config::ApiConfig) -> Result<Cluster> {Cluster::join(- Member {- id: config.cluster_id.clone(),- service: Service::Api { host: config.host },- },+ Member::new(Service::Api { host: config.host }),config.gossip_addr,config.gossip_seed_nodes.clone().unwrap_or_default(),)diff --git a/crates/core/src/entrypoint/crawler.rs b/crates/core/src/entrypoint/crawler.rsindex c6aa3136..62951fea 100644--- a/crates/core/src/entrypoint/crawler.rs+++ b/crates/core/src/entrypoint/crawler.rs@@ -68,7 +68,6 @@ pub async fn planner(config: config::CrawlPlannerConfig) -> Result<()> {let gossip = config.gossip.clone();let cluster = Arc::new(crate::distributed::cluster::Cluster::join_as_spectator(- gossip.cluster_id,gossip.addr,gossip.seed_nodes.unwrap_or_default(),)diff --git a/crates/core/src/entrypoint/entity_search_server.rs b/crates/core/src/entrypoint/entity_search_server.rsindex fabb815a..68def5f9 100644--- a/crates/core/src/entrypoint/entity_search_server.rs+++ b/crates/core/src/entrypoint/entity_search_server.rs@@ -40,10 +40,7 @@ impl SearchService {let index = EntityIndex::open(config.index_path)?;let cluster_handle = Cluster::join(- Member {- id: config.cluster_id,- service: Service::EntitySearcher { host: config.host },- },+ Member::new(Service::EntitySearcher { host: config.host }),config.gossip_addr,config.gossip_seed_nodes.unwrap_or_default(),)diff --git a/crates/core/src/entrypoint/feed_indexer.rs b/crates/core/src/entrypoint/feed_indexer.rsdeleted file mode 100644index 01f38444..00000000--- a/crates/core/src/entrypoint/feed_indexer.rs+++ /dev/null@@ -1,176 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::{- path::{Path, PathBuf},- thread,-};--use anyhow::Result;-use itertools::Itertools;-use rayon::prelude::*;--use tracing::info;-use url::Url;--use crate::{- config::{FeedIndexingConfig, WarcSource},- feed::{index::FeedIndex, Feed, FeedKind},- warc::{PayloadType, WarcFile},-};--#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]-pub struct Job {- pub source_config: WarcSource,- pub warc_path: String,- pub base_path: String,-}--#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]-pub struct IndexPointer(PathBuf);--impl From<String> for IndexPointer {- fn from(path: String) -> Self {- IndexPointer(Path::new(&path).to_path_buf())- }-}--pub struct IndexingWorker;-impl IndexingWorker {- fn process_job(&self, job: &Job) -> FeedIndex {- let name = job.warc_path.split('/').last().unwrap().to_string();- info!("processing {}", name);-- let mut index = FeedIndex::open(Path::new(&job.base_path).join(&name)).unwrap();-- if let Ok(file) = WarcFile::download(&job.source_config, &job.warc_path) {- for record in- file.records()- .flatten()- .filter(|record| match &record.response.payload_type {- Some(payload_type) => {- matches!(payload_type, PayloadType::Rss | PayloadType::Atom)- }- None => false,- })- {- let kind = match &record.response.payload_type {- Some(payload_type) => match payload_type {- PayloadType::Rss => FeedKind::Rss,- PayloadType::Atom => FeedKind::Atom,- _ => unreachable!(),- },- None => unreachable!(),- };-- let url = Url::parse(&record.request.url);-- if url.is_err() {- continue;- }-- let feed = Feed {- url: url.unwrap(),- kind,- };-- index.insert(&feed).unwrap();- }-- index.commit().unwrap();- index.merge_into_max_segments(1).unwrap();- }-- info!("{} done", name);-- index- }-}--pub fn build(config: FeedIndexingConfig) -> Result<()> {- let warc_paths = config.warc_source.paths()?;-- let job_config: WarcSource = config.warc_source.clone();-- let worker = IndexingWorker;-- let indexes = warc_paths- .into_par_iter()- .skip(config.skip_warc_files.unwrap_or(0))- .take(config.limit_warc_files.unwrap_or(usize::MAX))- .map(|warc_path| Job {- source_config: job_config.clone(),- warc_path,- base_path: config.output_path.clone(),- })- .map(|job| IndexPointer(worker.process_job(&job).path))- .collect();-- merge(indexes)?;- Ok(())-}--pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {- let num_indexes = indexes.len();- let mut it = indexes.into_iter();- let num_cores = usize::from(std::thread::available_parallelism()?);-- let mut threads = Vec::new();-- for _ in 0..(num_cores + 1) {- let indexes = it- .by_ref()- .take(((num_indexes as f64) / (num_cores as f64)).ceil() as usize)- .collect_vec();-- if indexes.is_empty() {- break;- }-- threads.push(thread::spawn(move || {- let mut it = indexes.into_iter();- let mut index = FeedIndex::open(it.next().unwrap().0).unwrap();-- for other in it {- let other_path = other.0;- let other = FeedIndex::open(&other_path).unwrap();- index = index.merge(other);-- std::fs::remove_dir_all(other_path).unwrap();- }-- index.merge_into_max_segments(1).unwrap();-- index- }));- }-- let mut indexes = Vec::new();- for thread in threads {- indexes.push(thread.join().unwrap());- }-- let mut it = indexes.into_iter();- let mut index = it.next().unwrap();-- for other in it {- let other_path = other.path.clone();- index = index.merge(other);- std::fs::remove_dir_all(other_path).unwrap();- }-- Ok(())-}diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsindex 0ba35cb8..979b9218 100644--- a/crates/core/src/entrypoint/indexer/worker.rs+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -24,10 +24,10 @@ use tracing::debug;pub use super::indexable_webpage::IndexableWebpage;pub use super::job::{Job, JobSettings};use crate::backlink_grouper::BacklinkGrouper;-use crate::config::{GossipConfig, IndexerConfig, IndexerDualEncoderConfig, WebgraphGranularity};+use crate::config::{GossipConfig, IndexerConfig, IndexerDualEncoderConfig};use crate::distributed::cluster::Cluster;use crate::models::dual_encoder::DualEncoder as DualEncoderModel;-use crate::webgraph::remote::RemoteWebgraph;+use crate::webgraph::remote::{Page, RemoteWebgraph};use crate::Result;use crate::index::Index;@@ -81,7 +81,7 @@ struct DualEncoder {}pub(super) enum Webgraph {- Remote(RemoteWebgraph),+ Remote(RemoteWebgraph<Page>),Local(webgraph::Webgraph),}@@ -152,10 +152,10 @@ impl Webgraph {),IndexerGraphConfig::Remote { gossip } => {let cluster = crate::start_gossip_cluster_thread(gossip.clone(), None);- Self::Remote(RemoteWebgraph::new(cluster, WebgraphGranularity::Page).await)+ Self::Remote(RemoteWebgraph::new(cluster).await)}IndexerGraphConfig::Existing { cluster } => {- Self::Remote(RemoteWebgraph::new(cluster.clone(), WebgraphGranularity::Page).await)+ Self::Remote(RemoteWebgraph::new(cluster.clone()).await)}}}diff --git a/crates/core/src/entrypoint/live_index/crawler.rs b/crates/core/src/entrypoint/live_index/crawler.rsnew file mode 100644index 00000000..d03cc016--- /dev/null+++ b/crates/core/src/entrypoint/live_index/crawler.rs@@ -0,0 +1,27 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::config;+use crate::live_index;+use crate::Result;++pub async fn run(config: config::LiveCrawlerConfig) -> Result<()> {+ let crawler = live_index::Crawler::new(config).await?;++ crawler.run().await?;++ Ok(())+}diff --git a/crates/core/src/entrypoint/live_index/mod.rs b/crates/core/src/entrypoint/live_index/mod.rsindex d782a04f..b9b0ad01 100644--- a/crates/core/src/entrypoint/live_index/mod.rs+++ b/crates/core/src/entrypoint/live_index/mod.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -14,423 +14,13 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+pub mod crawler;+pub mod search_server;+#[cfg(test)]mod tests;-use std::{- net::SocketAddr,- path::{Path, PathBuf},- sync::Arc,- time::Duration,-};--use crate::{- ampc::dht::ShardId,- config::LiveIndexConfig,- distributed::{- cluster::Cluster,- member::{LiveIndexState, Member, Service},- remote_cp,- sonic::{self, service::sonic_service},- },- inverted_index,- live_index::{IndexManager, LiveIndex},- searcher::{InitialWebsiteResult, LocalSearcher},-};-use anyhow::{Context, Result};-use futures::stream::FuturesUnordered;-use itertools::Itertools;-use simple_wal::Wal;-use tokio::sync::Mutex;-use tokio_stream::StreamExt;-use tracing::info;--use super::{- indexer::{self, IndexableWebpage},- search_server::{RetrieveWebsites, Search},-};--const INDEXING_TIMEOUT: Duration = Duration::from_secs(60);-const INDEXING_RETRIES: usize = 3;--sonic_service!(- LiveIndexService,- [- RetrieveWebsites,- Search,- IndexWebpages,- GetIndexPath,- RemoteDownload- ]-);--fn start_manager(index: Arc<LiveIndex>) {- let manager = IndexManager::new(index);- std::thread::spawn(|| manager.run());-}--struct FileDownloadStepper {- conn: Mutex<sonic::service::Connection<LiveIndexService>>,-}--impl remote_cp::Stepper for FileDownloadStepper {- async fn step(&self, req: remote_cp::Request) -> remote_cp::Response {- self.conn- .lock()- .await- .send(RemoteDownload { req })- .await- .unwrap()- }-}--async fn other_replicas(cluster: &Cluster, shard: &ShardId, id: &str) -> Vec<SocketAddr> {- tokio::time::sleep(std::time::Duration::from_secs(10)).await;- cluster- .members()- .await- .into_iter()- .filter_map(|member| {- if member.id != id {- if let Service::LiveIndex {- host,- shard: member_shard,- state,- } = member.service- {- if &member_shard == shard && matches!(state, LiveIndexState::Ready) {- return Some(host);- }- }- }-- None- })- .collect()-}--async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal) -> Result<()> {- let self_node = cluster- .self_node()- .expect("cluster should not be joined as spectator");-- let Service::LiveIndex {- host,- shard,- state: _,- } = self_node.service.clone()- else {- panic!("self node should be a live index")- };-- let mut others = other_replicas(&cluster, &shard, &self_node.id).await;-- if let Some(other) = others.pop() {- let mut conn: sonic::service::Connection<LiveIndexService> =- sonic::service::Connection::create(other).await?;- index.delete_all_pages();- let local_path = index.path();- let remote_path = conn.send(GetIndexPath).await?;-- remote_cp::Request::download(- remote_path,- local_path,- &FileDownloadStepper {- conn: Mutex::new(conn),- },- )- .await;- }-- let mut wal = temp_wal- .lock()- .await- .take()- .expect("temp_wal should always exist before setup has been run");-- for pages in wal.iter().unwrap().chunks(512).into_iter() {- let pages: Vec<_> = pages.into_iter().collect();- index.insert(&pages);- }-- wal.clear().unwrap();-- start_manager(index);-- cluster- .set_service(Service::LiveIndex {- host,- shard,- state: LiveIndexState::Ready,- })- .await?;-- Ok(())-}--type TempWal = Arc<Mutex<Option<Wal<IndexableWebpage>>>>;--pub struct LiveIndexService {- local_searcher: LocalSearcher<Arc<LiveIndex>>,- temp_wal: TempWal,- index: Arc<LiveIndex>,- cluster_handle: Arc<Cluster>,-}--impl LiveIndexService {- async fn new(config: LiveIndexConfig) -> Result<Self> {- let cluster_handle = Arc::new(- Cluster::join(- Member {- id: config.cluster_id.clone(),- service: Service::LiveIndex {- host: config.host,- shard: config.shard_id,- state: crate::distributed::member::LiveIndexState::InSetup,- },- },- config.gossip_addr,- config.gossip_seed_nodes.clone().unwrap_or_default(),- )- .await?,- );- let index_path = Path::new(&config.index_path);-- let index = Arc::new(- LiveIndex::new(- index_path.join("index"),- indexer::worker::Config {- host_centrality_store_path: config.host_centrality_store_path.clone(),- page_centrality_store_path: config.page_centrality_store_path.clone(),- page_webgraph: Some(indexer::worker::IndexerGraphConfig::Existing {- cluster: cluster_handle.clone(),- }),- safety_classifier_path: config.safety_classifier_path.clone(),- dual_encoder: None,- },- )- .await?,- );- let local_searcher = LocalSearcher::new(index.clone());-- let temp_wal = Arc::new(Mutex::new(Some(Wal::open(index_path.join("wal.temp"))?)));-- Ok(Self {- local_searcher,- cluster_handle,- index,- temp_wal,- })- }-- fn background_setup(&self) {- let index = self.index.clone();- let temp_wal = self.temp_wal.clone();- let cluster = self.cluster_handle.clone();-- tokio::task::spawn(async move { setup(index, cluster, temp_wal).await.unwrap() });- }-- async fn index_webpages_in_replicas(- &self,- webpages: &[IndexableWebpage],- consistency_fraction: f64,- ) -> Result<(), IndexingError> {- let self_member = self- .cluster_handle- .self_node()- .expect("node should be participating part of cluster");-- let self_id = self_member.id.clone();- let Service::LiveIndex {- host: _,- shard: self_shard,- state: _,- } = self_member.service- else {- panic!("self_member should always be a live index")- };-- let live_indexes: Vec<_> = self- .cluster_handle- .members()- .await- .into_iter()- .filter_map(|member| {- if let Service::LiveIndex { host, shard, state } = member.service {- if member.id != self_id && shard == self_shard {- Some(RemoteIndex { host, state })- } else {- None- }- } else {- None- }- })- .collect();-- let ready = live_indexes- .iter()- .filter(|index| matches!(index.state, LiveIndexState::Ready))- .count();-- let mut futures = FuturesUnordered::new();-- for index in live_indexes {- let pages = webpages.to_vec();-- futures.push(tokio::spawn(async move {- match index.index_webpages_without_consistency(&pages).await {- Ok(_) => Ok(index),- Err(e) => Err(e),- }- }));- }-- let mut missing_responses =- (((ready as f64) * consistency_fraction).ceil() as u64).min(ready as u64);-- while let Some(Ok(index)) = futures.next().await {- if let Ok(index) = index {- if matches!(index.state, LiveIndexState::Ready) {- missing_responses = missing_responses.saturating_sub(1);- }-- if missing_responses == 0 {- break;- }- }- }-- if missing_responses > 0 {- return Err(IndexingError::InsufficientReplication);- }-- Ok(())- }-}--impl sonic::service::Message<LiveIndexService> for RetrieveWebsites {- type Response = Option<Vec<inverted_index::RetrievedWebpage>>;- async fn handle(self, server: &LiveIndexService) -> Self::Response {- server- .local_searcher- .retrieve_websites(&self.websites, &self.query)- .ok()- }-}--impl sonic::service::Message<LiveIndexService> for Search {- type Response = Option<InitialWebsiteResult>;- async fn handle(self, server: &LiveIndexService) -> Self::Response {- server.local_searcher.search_initial(&self.query, true).ok()- }-}--#[derive(Clone, Debug)]-struct RemoteIndex {- host: SocketAddr,- state: LiveIndexState,-}--impl RemoteIndex {- async fn index_webpages_without_consistency(- &self,- webpages: &[IndexableWebpage],- ) -> Result<()> {- let req = IndexWebpages {- pages: webpages.to_vec(),- consistency_fraction: None,- };-- for _ in 0..INDEXING_RETRIES {- let mut conn: sonic::service::Connection<LiveIndexService> =- sonic::service::Connection::create(self.host)- .await- .with_context(|| format!("failed to connect to {}", self.host))?;-- if conn- .send_with_timeout(req.clone(), INDEXING_TIMEOUT)- .await- .is_ok()- {- return Ok(());- }- }-- Err(anyhow::anyhow!("failed to replicate webpages"))- }-}--#[derive(thiserror::Error, Debug, bincode::Encode, bincode::Decode)]-pub enum IndexingError {- #[error("failed to replicate to the necessary quorom")]- InsufficientReplication,-}--#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]-pub struct IndexWebpages {- pub pages: Vec<IndexableWebpage>,- pub consistency_fraction: Option<f64>,-}--impl sonic::service::Message<LiveIndexService> for IndexWebpages {- type Response = Result<(), IndexingError>;-- async fn handle(self, server: &LiveIndexService) -> Self::Response {- if let Some(wal) = server.temp_wal.lock().await.as_mut() {- wal.batch_write(self.pages.iter()).unwrap();- } else {- server.index.insert(&self.pages);-- if let Some(consistency_fraction) = self.consistency_fraction {- server- .index_webpages_in_replicas(&self.pages, consistency_fraction)- .await?;- }- }-- Ok(())- }-}--#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]-pub struct GetIndexPath;--impl sonic::service::Message<LiveIndexService> for GetIndexPath {- type Response = PathBuf;-- async fn handle(self, server: &LiveIndexService) -> Self::Response {- server.index.path()- }-}--#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]-pub struct RemoteDownload {- req: remote_cp::Request,-}-impl sonic::service::Message<LiveIndexService> for RemoteDownload {- type Response = remote_cp::Response;-- async fn handle(self, _: &LiveIndexService) -> Self::Response {- remote_cp::Response::handle(self.req).unwrap()- }-}--pub async fn serve(config: LiveIndexConfig) -> Result<()> {- let addr = config.host;-- let service = LiveIndexService::new(config).await?;-- service.background_setup();-- let server = service.bind(&addr).await.unwrap();-- info!("live index is ready to accept requests on {}", addr);-- loop {- if let Err(e) = server.accept().await {- tracing::error!("{:?}", e);- }- }-}+pub use self::search_server::GetIndexPath;+pub use self::search_server::IndexWebpages;+pub use self::search_server::LiveIndexService;+pub use self::search_server::RemoteDownload;diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rsnew file mode 100644index 00000000..3474bc53--- /dev/null+++ b/crates/core/src/entrypoint/live_index/search_server.rs@@ -0,0 +1,450 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{+ net::SocketAddr,+ path::{Path, PathBuf},+ sync::Arc,+ time::Duration,+};++use crate::{+ ampc::dht::ShardId,+ config::LiveIndexConfig,+ distributed::{+ cluster::Cluster,+ member::{LiveIndexState, Member, Service},+ remote_cp,+ sonic::{self, service::sonic_service},+ },+ inverted_index,+ live_index::{IndexManager, LiveIndex},+ searcher::{InitialWebsiteResult, LocalSearcher},+};+use anyhow::{Context, Result};+use futures::stream::FuturesUnordered;+use itertools::Itertools;+use simple_wal::Wal;+use tokio::sync::Mutex;+use tokio_stream::StreamExt;+use tracing::info;++use crate::entrypoint::{+ indexer::{self, IndexableWebpage},+ search_server::{GetSiteUrls, RetrieveWebsites, Search, SiteUrls},+};++const INDEXING_TIMEOUT: Duration = Duration::from_secs(60);+const INDEXING_RETRIES: usize = 3;++sonic_service!(+ LiveIndexService,+ [+ RetrieveWebsites,+ Search,+ GetSiteUrls,+ IndexWebpages,+ GetIndexPath,+ RemoteDownload+ ]+);++fn start_manager(index: Arc<LiveIndex>) {+ let manager = IndexManager::new(index);+ std::thread::spawn(|| manager.run());+}++struct FileDownloadStepper {+ conn: Mutex<sonic::service::Connection<LiveIndexService>>,+}++impl remote_cp::Stepper for FileDownloadStepper {+ async fn step(&self, req: remote_cp::Request) -> remote_cp::Response {+ self.conn+ .lock()+ .await+ .send(RemoteDownload { req })+ .await+ .unwrap()+ }+}++async fn other_replicas(cluster: &Cluster, shard: &ShardId, id: &str) -> Vec<SocketAddr> {+ tokio::time::sleep(std::time::Duration::from_secs(10)).await;+ cluster+ .members()+ .await+ .into_iter()+ .filter_map(|member| {+ if member.id != id {+ if let Service::LiveIndex {+ host,+ shard: member_shard,+ state,+ } = member.service+ {+ if &member_shard == shard && matches!(state, LiveIndexState::Ready) {+ return Some(host);+ }+ }+ }++ None+ })+ .collect()+}++async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal) -> Result<()> {+ let self_node = cluster+ .self_node()+ .expect("cluster should not be joined as spectator");++ let Service::LiveIndex {+ host,+ shard,+ state: _,+ } = self_node.service.clone()+ else {+ panic!("self node should be a live index")+ };++ let mut others = other_replicas(&cluster, &shard, &self_node.id).await;++ if let Some(other) = others.pop() {+ let mut conn: sonic::service::Connection<LiveIndexService> =+ sonic::service::Connection::create(other).await?;+ index.delete_all_pages();+ let local_path = index.path();+ let remote_path = conn.send(GetIndexPath).await?;++ remote_cp::Request::download(+ remote_path,+ local_path,+ &FileDownloadStepper {+ conn: Mutex::new(conn),+ },+ )+ .await;+ }++ let mut wal = temp_wal+ .lock()+ .await+ .take()+ .expect("temp_wal should always exist before setup has been run");++ for pages in wal.iter().unwrap().chunks(512).into_iter() {+ let pages: Vec<_> = pages.into_iter().collect();+ index.insert(&pages);+ }++ wal.clear().unwrap();++ start_manager(index);++ cluster+ .set_service(Service::LiveIndex {+ host,+ shard,+ state: LiveIndexState::Ready,+ })+ .await?;++ Ok(())+}++type TempWal = Arc<Mutex<Option<Wal<IndexableWebpage>>>>;++pub struct LiveIndexService {+ local_searcher: LocalSearcher<Arc<LiveIndex>>,+ temp_wal: TempWal,+ index: Arc<LiveIndex>,+ cluster_handle: Arc<Cluster>,+}++impl LiveIndexService {+ pub async fn new(config: LiveIndexConfig) -> Result<Self> {+ let cluster_handle = Arc::new(+ Cluster::join(+ Member::new(Service::LiveIndex {+ host: config.host,+ shard: config.shard_id,+ state: crate::distributed::member::LiveIndexState::InSetup,+ }),+ config.gossip_addr,+ config.gossip_seed_nodes.clone().unwrap_or_default(),+ )+ .await?,+ );+ let index_path = Path::new(&config.index_path);++ let index = Arc::new(+ LiveIndex::new(+ index_path.join("index"),+ indexer::worker::Config {+ host_centrality_store_path: config.host_centrality_store_path.clone(),+ page_centrality_store_path: config.page_centrality_store_path.clone(),+ page_webgraph: Some(indexer::worker::IndexerGraphConfig::Existing {+ cluster: cluster_handle.clone(),+ }),+ safety_classifier_path: config.safety_classifier_path.clone(),+ dual_encoder: None,+ },+ )+ .await?,+ );+ let local_searcher = LocalSearcher::new(index.clone());++ let temp_wal = Arc::new(Mutex::new(Some(Wal::open(index_path.join("wal.temp"))?)));++ Ok(Self {+ local_searcher,+ cluster_handle,+ index,+ temp_wal,+ })+ }++ pub fn cluster_handle(&self) -> Arc<Cluster> {+ self.cluster_handle.clone()+ }++ pub fn index(&self) -> Arc<LiveIndex> {+ self.index.clone()+ }++ pub fn background_setup(&self) {+ let index = self.index.clone();+ let temp_wal = self.temp_wal.clone();+ let cluster = self.cluster_handle.clone();++ tokio::task::spawn(async move { setup(index, cluster, temp_wal).await.unwrap() });+ }++ async fn index_webpages_in_replicas(+ &self,+ webpages: &[IndexableWebpage],+ consistency_fraction: f64,+ ) -> Result<(), IndexingError> {+ let self_member = self+ .cluster_handle+ .self_node()+ .expect("node should be participating part of cluster");++ let self_id = self_member.id.clone();+ let Service::LiveIndex {+ host: _,+ shard: self_shard,+ state: _,+ } = self_member.service+ else {+ panic!("self_member should always be a live index")+ };++ let live_indexes: Vec<_> = self+ .cluster_handle+ .members()+ .await+ .into_iter()+ .filter_map(|member| {+ if let Service::LiveIndex { host, shard, state } = member.service {+ if member.id != self_id && shard == self_shard {+ Some(RemoteIndex { host, state })+ } else {+ None+ }+ } else {+ None+ }+ })+ .collect();++ let ready = live_indexes+ .iter()+ .filter(|index| matches!(index.state, LiveIndexState::Ready))+ .count();++ let mut futures = FuturesUnordered::new();++ for index in live_indexes {+ let pages = webpages.to_vec();++ futures.push(tokio::spawn(async move {+ match index.index_webpages_without_consistency(&pages).await {+ Ok(_) => Ok(index),+ Err(e) => Err(e),+ }+ }));+ }++ let mut missing_responses =+ (((ready as f64) * consistency_fraction).ceil() as u64).min(ready as u64);++ while let Some(Ok(index)) = futures.next().await {+ if let Ok(index) = index {+ if matches!(index.state, LiveIndexState::Ready) {+ missing_responses = missing_responses.saturating_sub(1);+ }++ if missing_responses == 0 {+ break;+ }+ }+ }++ if missing_responses > 0 {+ return Err(IndexingError::InsufficientReplication);+ }++ Ok(())+ }+}++impl sonic::service::Message<LiveIndexService> for RetrieveWebsites {+ type Response = Option<Vec<inverted_index::RetrievedWebpage>>;+ async fn handle(self, server: &LiveIndexService) -> Self::Response {+ server+ .local_searcher+ .retrieve_websites(&self.websites, &self.query)+ .ok()+ }+}++impl sonic::service::Message<LiveIndexService> for Search {+ type Response = Option<InitialWebsiteResult>;+ async fn handle(self, server: &LiveIndexService) -> Self::Response {+ server.local_searcher.search_initial(&self.query, true).ok()+ }+}++impl sonic::service::Message<LiveIndexService> for GetSiteUrls {+ type Response = SiteUrls;+ async fn handle(self, server: &LiveIndexService) -> Self::Response {+ let urls = server+ .local_searcher+ .get_site_urls(&self.site, self.offset, self.limit);++ SiteUrls { urls }+ }+}++#[derive(Clone, Debug)]+struct RemoteIndex {+ host: SocketAddr,+ state: LiveIndexState,+}++impl RemoteIndex {+ async fn index_webpages_without_consistency(+ &self,+ webpages: &[IndexableWebpage],+ ) -> Result<()> {+ let req = IndexWebpages {+ pages: webpages.to_vec(),+ consistency_fraction: None,+ };++ for _ in 0..INDEXING_RETRIES {+ let mut conn: sonic::service::Connection<LiveIndexService> =+ sonic::service::Connection::create(self.host)+ .await+ .with_context(|| format!("failed to connect to {}", self.host))?;++ if conn+ .send_with_timeout(req.clone(), INDEXING_TIMEOUT)+ .await+ .is_ok()+ {+ return Ok(());+ }+ }++ Err(anyhow::anyhow!("failed to replicate webpages"))+ }+}++#[derive(thiserror::Error, Debug, bincode::Encode, bincode::Decode)]+pub enum IndexingError {+ #[error("failed to replicate to the necessary quorom")]+ InsufficientReplication,+}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct IndexWebpages {+ pub pages: Vec<IndexableWebpage>,+ pub consistency_fraction: Option<f64>,+}++impl sonic::service::Message<LiveIndexService> for IndexWebpages {+ type Response = Result<(), IndexingError>;++ async fn handle(self, server: &LiveIndexService) -> Self::Response {+ if let Some(wal) = server.temp_wal.lock().await.as_mut() {+ wal.batch_write(self.pages.iter()).unwrap();+ } else {+ server.index.insert(&self.pages);++ if let Some(consistency_fraction) = self.consistency_fraction {+ server+ .index_webpages_in_replicas(&self.pages, consistency_fraction)+ .await?;+ }+ }++ Ok(())+ }+}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct GetIndexPath;++impl sonic::service::Message<LiveIndexService> for GetIndexPath {+ type Response = PathBuf;++ async fn handle(self, server: &LiveIndexService) -> Self::Response {+ server.index.path()+ }+}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct RemoteDownload {+ req: remote_cp::Request,+}+impl sonic::service::Message<LiveIndexService> for RemoteDownload {+ type Response = remote_cp::Response;++ async fn handle(self, _: &LiveIndexService) -> Self::Response {+ remote_cp::Response::handle(self.req).unwrap()+ }+}++pub async fn serve(config: LiveIndexConfig) -> Result<()> {+ let addr = config.host;++ let service = LiveIndexService::new(config).await?;++ service.background_setup();++ let server = service.bind(&addr).await.unwrap();++ info!("live index is ready to accept requests on {}", addr);++ loop {+ if let Err(e) = server.accept().await {+ tracing::error!("{:?}", e);+ }+ }+}diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rsindex 048f32f0..3d376769 100644--- a/crates/core/src/entrypoint/live_index/tests.rs+++ b/crates/core/src/entrypoint/live_index/tests.rs@@ -57,18 +57,6 @@ impl RemoteIndex {};let config = LiveIndexConfig {- user_agent: crate::config::UserAgent {- full: "TestBot".to_string(),- token: "TestBot".to_string(),- },- robots_txt_cache_sec: 60 * 60,- min_politeness_factor: 1,- start_politeness_factor: 3,- min_crawl_delay_ms: 5_000,- max_crawl_delay_ms: 60_000,- max_politeness_factor: 2048,- max_url_slowdown_retry: 5,- timeout_seconds: 30,host_centrality_store_path: path.as_path().join("host_centrality")@@ -79,7 +67,6 @@ impl RemoteIndex {safety_classifier_path: None,host_centrality_threshold: None,minimum_clean_words: None,- cluster_id: "test-cluster".to_string(),gossip_seed_nodes: gossip_seed,gossip_addr,shard_id: shard,@@ -92,8 +79,8 @@ impl RemoteIndex {};let service = LiveIndexService::new(config).await?;- let cluster = service.cluster_handle.clone();- let index = service.index.clone();+ let cluster = service.cluster_handle();+ let index = service.index();service.background_setup();@@ -186,19 +173,12 @@ impl RemoteIndex {}}-const CLUSTER_ID: &str = "test-cluster";-#[tokio::test]async fn test_shard_without_replica() -> Result<()> {let shard1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;let shard2 = RemoteIndex::start(ShardId::new(2), vec![shard1.gossip_addr]).await?;- let cluster = Cluster::join_as_spectator(- CLUSTER_ID.to_string(),- free_socket_addr(),- vec![shard1.gossip_addr],- )- .await?;+ let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![shard1.gossip_addr]).await?;shard1.await_ready(&cluster).await;shard2.await_ready(&cluster).await;@@ -252,12 +232,7 @@ async fn test_replica_no_fails() -> Result<()> {let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;- let cluster = Cluster::join_as_spectator(- CLUSTER_ID.to_string(),- free_socket_addr(),- vec![rep1.gossip_addr],- )- .await?;+ let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![rep1.gossip_addr]).await?;rep1.await_ready(&cluster).await;rep2.await_ready(&cluster).await;@@ -306,12 +281,7 @@ async fn test_replica_no_fails() -> Result<()> {async fn test_replica_setup_after_inserts() -> Result<()> {let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;- let cluster = Cluster::join_as_spectator(- CLUSTER_ID.to_string(),- free_socket_addr(),- vec![rep1.gossip_addr],- )- .await?;+ let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![rep1.gossip_addr]).await?;rep1.await_ready(&cluster).await;@@ -364,12 +334,7 @@ async fn test_replica_recovery() -> Result<()> {let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;- let cluster = Cluster::join_as_spectator(- CLUSTER_ID.to_string(),- free_socket_addr(),- vec![rep1.gossip_addr],- )- .await?;+ let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![rep1.gossip_addr]).await?;rep1.await_ready(&cluster).await;rep2.await_ready(&cluster).await;diff --git a/crates/core/src/entrypoint/mod.rs b/crates/core/src/entrypoint/mod.rsindex 63e3e075..fb2ab188 100644--- a/crates/core/src/entrypoint/mod.rs+++ b/crates/core/src/entrypoint/mod.rs@@ -24,7 +24,6 @@ pub mod configure;pub mod crawler;mod entity;pub mod entity_search_server;-pub mod feed_indexer;pub mod indexer;pub mod safety_classifier;pub mod search_server;diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex f4a2ebbf..f7e92786 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -45,6 +45,7 @@ sonic_service!(GetHomepageDescriptions,TopKeyPhrases,Size,+ GetSiteUrls,]);@@ -73,13 +74,10 @@ impl SearchService {local_searcher.set_snippet_config(config.snippet);let cluster_handle = Cluster::join(- Member {- id: config.cluster_id,- service: Service::Searcher {- host: config.host,- shard: config.shard,- },- },+ Member::new(Service::Searcher {+ host: config.host,+ shard: config.shard,+ }),config.gossip_addr,config.gossip_seed_nodes.unwrap_or_default(),)@@ -188,3 +186,27 @@ impl sonic::service::Message<SearchService> for Size {}}}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct GetSiteUrls {+ pub site: String,+ pub offset: usize,+ pub limit: usize,+}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct SiteUrls {+ #[bincode(with_serde)]+ pub urls: Vec<Url>,+}++impl sonic::service::Message<SearchService> for GetSiteUrls {+ type Response = SiteUrls;+ async fn handle(self, server: &SearchService) -> Self::Response {+ let urls = server+ .local_searcher+ .get_site_urls(&self.site, self.offset, self.limit);++ SiteUrls { urls }+ }+}diff --git a/crates/core/src/entrypoint/site_stats.rs b/crates/core/src/entrypoint/site_stats.rsindex c06d373b..730600a6 100644--- a/crates/core/src/entrypoint/site_stats.rs+++ b/crates/core/src/entrypoint/site_stats.rs@@ -21,41 +21,128 @@ use std::{};use bloom::BytesBloomFilter;+use url::Url;use crate::{config::{self, SiteStatsConfig},entrypoint::download_all_warc_files,+ feed::Feed,webgraph::Node,webpage::{url_ext::UrlExt, Html},Result,};+const TOP_FEEDS_PER_SITE: Option<usize> = Some(10);+const MIN_FEED_COUNT: u64 = 1;+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Default)]pub struct SiteStats {pages: u64,blogposts: u64,news_articles: u64,+ feeds: HashMap<Feed, u64>,}impl AddAssign<SiteStats> for SiteStats {fn add_assign(&mut self, rhs: SiteStats) {self.pages += rhs.pages;self.blogposts += rhs.blogposts;- self.news_articles += rhs.news_articles+ self.news_articles += rhs.news_articles;++ for (feed, count) in rhs.feeds {+ *self.feeds.entry(feed).or_default() += count;+ }+ }+}++#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]+pub struct FinalStats {+ pub pages: u64,+ pub blogposts: u64,+ pub news_articles: u64,+ pub feeds: Vec<FeedCount>,+}++#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]+pub struct FeedCount {+ feed: Feed,+ count: u64,+}++impl From<FeedCount> for Feed {+ fn from(feed_count: FeedCount) -> Self {+ feed_count.feed+ }+}++impl From<(Feed, u64)> for FeedCount {+ fn from((feed, count): (Feed, u64)) -> Self {+ Self { feed, count }+ }+}++impl From<SiteStats> for FinalStats {+ fn from(stats: SiteStats) -> Self {+ Self {+ pages: stats.pages,+ blogposts: stats.blogposts,+ news_articles: stats.news_articles,+ feeds: stats+ .feeds+ .into_iter()+ .map(|(feed, count)| FeedCount::from((feed, count)))+ .collect(),+ }+ }+}++#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]+pub struct FinalSiteStats {+ site: Site,+ #[serde(flatten)]+ stats: FinalStats,+}++impl FinalSiteStats {+ pub fn site(&self) -> &Site {+ &self.site+ }++ pub fn stats(&self) -> &FinalStats {+ &self.stats}}#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Hash, PartialEq, Eq)]#[serde(transparent)]-struct Site(String);+pub struct Site(String);++impl Site {+ fn from_url(url: &str) -> Result<Self> {+ let url = Url::robust_parse(url)?;+ let root_domain = url+ .root_domain()+ .ok_or(anyhow::anyhow!("Failed to get root domain from url"))?;+ Ok(Self(root_domain.to_string()))+ }++ pub fn as_str(&self) -> &str {+ &self.0+ }++ pub fn url(&self) -> Result<Url> {+ Url::robust_parse(&self.0).map_err(|_| anyhow::anyhow!("Failed to parse url"))+ }+}#[derive(Clone, Debug, Hash, PartialEq, Eq)]struct SiteId([u8; 8]);impl SiteId {- fn from_url(url: String) -> Self {- let node_id = Node::from(url).into_host().id();- Self(node_id.as_u64().to_be_bytes())+ fn from_url(url: &str) -> Result<Self> {+ let url = Url::robust_parse(url)?;+ let node_id = Node::from(&url).into_host().id();+ Ok(Self(node_id.as_u64().to_be_bytes()))}}@@ -65,13 +152,6 @@ impl AsRef<[u8]> for SiteId {}}-#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]-struct FinalSiteStats {- site: Site,- #[serde(flatten)]- stats: SiteStats,-}-struct SiteFilter {bloom: BytesBloomFilter<SiteId>,full: HashSet<SiteId>,@@ -123,10 +203,16 @@ impl StatsWorker {for file in warc_files.by_ref() {for record in file.records().flatten() {- if !self- .site_filter- .should_process(&SiteId::from_url(record.request.url.clone()))- {+ let site_id = SiteId::from_url(&record.request.url);++ if site_id.is_err() {+ tracing::error!("error parsing url: {}", site_id.err().unwrap());+ continue;+ }++ let site_id = site_id.unwrap();++ if !self.site_filter.should_process(&site_id) {continue;}@@ -139,11 +225,22 @@ impl StatsWorker {};let mut stats = SiteStats {+ feeds: HashMap::new(),pages: 1,blogposts: 0,news_articles: 0,};+ if let Ok(feeds) = webpage.feeds() {+ let root_domain = webpage.url().root_domain();++ for feed in feeds {+ if feed.url.root_domain() == root_domain {+ *stats.feeds.entry(feed).or_default() += 1;+ }+ }+ }+for schema in webpage.schema_org() {if schema.types_contains("NewsArticle") {stats.news_articles = 1;@@ -153,11 +250,7 @@ impl StatsWorker {stats.blogposts = 1;}}- if let Some(site) = webpage- .url()- .root_domain()- .map(|site| Site(site.to_string()))- {+ if let Ok(site) = Site::from_url(&record.request.url) {*self.stats.lock().unwrap().entry(site).or_default() += stats;}}@@ -212,7 +305,7 @@ pub fn run(config: SiteStatsConfig) -> Result<()> {}for handler in handlers {- handler.join().unwrap();+ handler.join().ok();}let mut final_stats: Vec<_> = worker@@ -221,9 +314,25 @@ pub fn run(config: SiteStatsConfig) -> Result<()> {.unwrap().clone().into_iter()- .map(|(site, stats)| FinalSiteStats { site, stats })+ .map(|(site, stats)| FinalSiteStats {+ site,+ stats: stats.into(),+ }).collect();+ final_stats.iter_mut().for_each(|site_stats| {+ site_stats+ .stats+ .feeds+ .retain(|feed_count| feed_count.count > MIN_FEED_COUNT);++ site_stats.stats.feeds.sort_by(|a, b| b.count.cmp(&a.count));++ if let Some(top_feeds) = TOP_FEEDS_PER_SITE {+ site_stats.stats.feeds.truncate(top_feeds);+ }+ });+final_stats.sort_by(|a, b| b.stats.pages.cmp(&a.stats.pages));let writer = std::fs::File::create(&config.output_path)?;diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rsindex 42ef0156..fb374436 100644--- a/crates/core/src/entrypoint/webgraph_server.rs+++ b/crates/core/src/entrypoint/webgraph_server.rs@@ -190,14 +190,11 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {// dropping the handle leaves the clusterlet _cluster = Arc::new(Cluster::join(- Member {- id: config.cluster_id,- service: Service::Webgraph {- host: addr,- granularity: config.granularity,- shard: config.shard,- },- },+ Member::new(Service::Webgraph {+ host: addr,+ granularity: config.granularity,+ shard: config.shard,+ }),config.gossip_addr,config.gossip_seed_nodes.unwrap_or_default(),)diff --git a/crates/core/src/feed/index.rs b/crates/core/src/feed/index.rsdeleted file mode 100644index 6b0ca1e8..00000000--- a/crates/core/src/feed/index.rs+++ /dev/null@@ -1,292 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::{- fs,- path::{Path, PathBuf},-};--use crate::{- inverted_index::merge_tantivy_segments,- tokenizer::fields::{FieldTokenizer, UrlTokenizer},-};-use anyhow::Result;-use hashbrown::HashSet;-use lending_iter::LendingIterator;-use tantivy::{- merge_policy::NoMergePolicy,- query::{PhraseQuery, TermQuery},- schema::{IndexRecordOption, TextFieldIndexing, TextOptions, Value},- tokenizer::Tokenizer as TantivyTokenizer,-};-use url::Url;--use super::{Feed, FeedKind};--pub struct FeedIndex {- tantivy_index: tantivy::Index,- writer: tantivy::IndexWriter,- reader: tantivy::IndexReader,- schema: tantivy::schema::Schema,- pub path: PathBuf,-}--impl FeedIndex {- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let url_tokenizer = FieldTokenizer::Url(UrlTokenizer);- let kind_tokenizer = FieldTokenizer::default();-- let mut builder = tantivy::schema::Schema::builder();-- builder.add_text_field(- "url",- TextOptions::default()- .set_indexing_options(- TextFieldIndexing::default()- .set_tokenizer(url_tokenizer.as_str())- .set_index_option(IndexRecordOption::WithFreqsAndPositions),- )- .set_stored(),- );- builder.add_text_field(- "kind",- TextOptions::default()- .set_indexing_options(- TextFieldIndexing::default()- .set_tokenizer(kind_tokenizer.as_str())- .set_index_option(IndexRecordOption::Basic),- )- .set_stored(),- );-- let schema = builder.build();-- let tv_index = if !path.as_ref().exists() {- fs::create_dir_all(path.as_ref())?;- tantivy::Index::create_in_dir(path.as_ref(), schema.clone())?- } else {- tantivy::Index::open_in_dir(path.as_ref())?- };-- tv_index- .tokenizers()- .register(url_tokenizer.as_str(), url_tokenizer);-- tv_index- .tokenizers()- .register(kind_tokenizer.as_str(), kind_tokenizer);-- let writer = tv_index.writer(50_000_000)?;-- let merge_policy = NoMergePolicy;- writer.set_merge_policy(Box::new(merge_policy));-- let reader = tv_index.reader()?;-- Ok(Self {- tantivy_index: tv_index,- writer,- reader,- schema,- path: path.as_ref().to_path_buf(),- })- }-- pub fn insert(&mut self, feed: &Feed) -> Result<()> {- let mut doc = tantivy::TantivyDocument::default();-- doc.add_text(self.schema.get_field("url")?, feed.url.as_str());- doc.add_text(- self.schema.get_field("kind")?,- &serde_json::to_string(&feed.kind)?,- );-- self.writer.add_document(doc)?;-- Ok(())- }-- pub fn commit(&mut self) -> Result<()> {- self.writer.commit()?;- self.reader.reload()?;-- Ok(())- }-- fn terms(&self, query: &str) -> Vec<tantivy::Term> {- let mut tokenizer = UrlTokenizer;- let mut res = Vec::new();- let tv_field = self.schema.get_field("url").unwrap();-- let mut stream = tokenizer.token_stream(query);- let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);-- while let Some(token) = it.next() {- res.push(tantivy::Term::from_field_text(tv_field, &token.text));- }-- res- }-- pub fn search(&self, query: &str) -> Result<Vec<Feed>> {- let searcher = self.reader.searcher();-- let terms = self.terms(query);- if terms.is_empty() {- return Ok(Vec::new());- }-- let query = if terms.len() == 1 {- let term = terms.first().unwrap().clone();- Box::new(TermQuery::new(- term,- tantivy::schema::IndexRecordOption::Basic,- )) as Box<dyn tantivy::query::Query>- } else {- Box::new(PhraseQuery::new(terms)) as Box<dyn tantivy::query::Query>- };-- let docs = searcher.search(&query, &tantivy::collector::DocSetCollector)?;-- let mut res = Vec::new();-- for address in docs {- let doc: tantivy::TantivyDocument = searcher.doc(address)?;-- let url = doc- .get_first(self.schema.get_field("url")?)- .unwrap()- .as_str()- .unwrap();-- let url = Url::parse(url)?;-- let kind = doc- .get_first(self.schema.get_field("kind")?)- .unwrap()- .as_str()- .unwrap();-- let kind: FeedKind = serde_json::from_str(kind)?;-- res.push(Feed { url, kind });- }-- Ok(res)- }-- pub fn merge_into_max_segments(&mut self, max_num_segments: u64) -> Result<()> {- let segments: Vec<_> = self- .tantivy_index- .load_metas()?- .segments- .into_iter()- .collect();-- merge_tantivy_segments(&mut self.writer, segments, &self.path, max_num_segments)?;-- Ok(())- }-- pub fn merge(mut self, mut other: Self) -> Self {- let path = self.path.clone();-- {- other.commit().expect("failed to commit index");- self.commit().expect("failed to commit index");-- let other_meta = other- .tantivy_index- .load_metas()- .expect("failed to load tantivy metadata for index");-- let mut meta = self- .tantivy_index- .load_metas()- .expect("failed to load tantivy metadata for index");-- let other_path = other.path.clone();- let other_path = other_path.as_path();- other.writer.wait_merging_threads().unwrap();-- let path = self.path.clone();- let self_path = path.as_path();- self.writer.wait_merging_threads().unwrap();-- let ids: HashSet<_> = meta.segments.iter().map(|segment| segment.id()).collect();-- for segment in other_meta.segments {- if ids.contains(&segment.id()) {- continue;- }-- // TODO: handle case where current index has segment with same name- for file in segment.list_files() {- let p = other_path.join(&file);- if p.exists() {- fs::rename(p, self_path.join(&file)).unwrap();- }- }- meta.segments.push(segment);- }-- meta.segments- .sort_by_key(|a| std::cmp::Reverse(a.max_doc()));-- fs::remove_dir_all(other_path).ok();-- let self_path = Path::new(&path);-- std::fs::write(- self_path.join("meta.json"),- serde_json::to_string_pretty(&meta).unwrap(),- )- .unwrap();- }-- Self::open(path).expect("failed to open index")- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn feed_index() {- let mut index = FeedIndex::open(crate::gen_temp_path()).unwrap();-- let a = Feed {- url: Url::parse("https://a.com/feed.xml").unwrap(),- kind: FeedKind::Atom,- };- index.insert(&a).unwrap();-- let b = Feed {- url: Url::parse("https://b.com/another/feed").unwrap(),- kind: FeedKind::Rss,- };- index.insert(&b).unwrap();-- index.commit().unwrap();-- let res = index.search("a.com").unwrap();- assert_eq!(res, vec![a]);-- let res = index.search("b.com").unwrap();- assert_eq!(res, vec![b]);- }-}diff --git a/crates/core/src/feed/mod.rs b/crates/core/src/feed/mod.rsindex 8f9ffd6d..9f6c4f40 100644--- a/crates/core/src/feed/mod.rs+++ b/crates/core/src/feed/mod.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -14,14 +14,16 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::str::FromStr;+use url::Url;-pub mod index;mod parser;-pub mod scheduler;pub use parser::parse;+use crate::dated_url::DatedUrl;+#[derive(Debug,Clone,@@ -39,6 +41,20 @@ pub enum FeedKind {Rss,}+impl FromStr for FeedKind {+ type Err = anyhow::Error;++ fn from_str(s: &str) -> Result<Self, Self::Err> {+ match s {+ "application/atom" => Ok(Self::Atom),+ "application/atom+xml" => Ok(Self::Atom),+ "application/rss" => Ok(Self::Rss),+ "application/rss+xml" => Ok(Self::Rss),+ s => anyhow::bail!("Unknown feed kind: {s}"),+ }+ }+}+#[derive(Debug,Clone,@@ -57,5 +73,5 @@ pub struct Feed {}pub struct ParsedFeed {- pub links: Vec<Url>,+ pub links: Vec<DatedUrl>,}diff --git a/crates/core/src/feed/parser.rs b/crates/core/src/feed/parser.rsindex cfb56920..94b349f3 100644--- a/crates/core/src/feed/parser.rs+++ b/crates/core/src/feed/parser.rs@@ -15,9 +15,12 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::{anyhow, Result};+use chrono::{DateTime, Utc};use quick_xml::events::Event;use url::Url;+use crate::dated_url::DatedUrl;+use super::{FeedKind, ParsedFeed};fn parse_rss(feed: &str) -> Result<ParsedFeed> {@@ -27,6 +30,8 @@ fn parse_rss(feed: &str) -> Result<ParsedFeed> {let mut links = Vec::new();let mut inside_link = false;let mut inside_item = false;+ let mut current_link: Option<Url> = None;+ let mut current_date: Option<DateTime<Utc>> = None;loop {match reader.read_event_into(&mut buf) {@@ -38,14 +43,33 @@ fn parse_rss(feed: &str) -> Result<ParsedFeed> {}Ok(Event::Start(ref e)) if e.name().as_ref() == b"item" => {inside_item = true;+ current_link = None;+ current_date = None;}Ok(Event::End(ref e)) if e.name().as_ref() == b"item" => {inside_item = false;+ if let Some(link) = current_link.take() {+ links.push(DatedUrl {+ url: link,+ last_modified: current_date,+ });+ }}Ok(Event::Text(e)) if inside_item && inside_link => {let link = e.unescape()?;if let Ok(link) = Url::parse(&link) {- links.push(link);+ current_link = Some(link);+ }+ }+ Ok(Event::Start(ref e))+ if e.name().as_ref() == b"pubDate" || e.name().as_ref() == b"lastBuildDate" =>+ {+ if inside_item {+ if let Ok(Event::Text(e)) = reader.read_event_into(&mut buf) {+ if let Ok(date) = DateTime::parse_from_rfc2822(&e.unescape()?) {+ current_date = Some(date.with_timezone(&Utc));+ }+ }}}Err(e) => {@@ -67,14 +91,24 @@ fn parse_atom(feed: &str) -> Result<ParsedFeed> {let mut links = Vec::new();let mut inside_entry = false;+ let mut current_link: Option<Url> = None;+ let mut current_date: Option<DateTime<Utc>> = None;loop {match reader.read_event_into(&mut buf) {Ok(Event::Start(ref e)) if e.name().as_ref() == b"entry" => {inside_entry = true;+ current_link = None;+ current_date = None;}Ok(Event::End(ref e)) if e.name().as_ref() == b"entry" => {inside_entry = false;+ if let Some(link) = current_link.take() {+ links.push(DatedUrl {+ url: link,+ last_modified: current_date,+ });+ }}Ok(Event::Start(ref e)) | Ok(Event::Empty(ref e)) => {if inside_entry && e.name().as_ref() == b"link" {@@ -88,7 +122,15 @@ fn parse_atom(feed: &str) -> Result<ParsedFeed> {.map_err(|e| anyhow!(e)).and_then(|v| Url::parse(&v).map_err(|e| anyhow!(e))){- links.push(href);+ current_link = Some(href);+ }+ }+ } else if inside_entry+ && (e.name().as_ref() == b"updated" || e.name().as_ref() == b"published")+ {+ if let Ok(Event::Text(e)) = reader.read_event_into(&mut buf) {+ if let Ok(date) = DateTime::parse_from_rfc3339(&e.unescape()?) {+ current_date = Some(date.with_timezone(&Utc));}}}@@ -142,7 +184,14 @@ mod tests {assert_eq!(parsed.links,- vec![Url::parse("https://example.com/a").unwrap()]+ vec![DatedUrl {+ url: Url::parse("https://example.com/a").unwrap(),+ last_modified: Some(+ DateTime::parse_from_rfc2822("Mon, 30 Oct 2023 08:55:00 GMT")+ .unwrap()+ .with_timezone(&Utc)+ ),+ }]);}@@ -163,7 +212,14 @@ mod tests {assert_eq!(parsed.links,- vec![Url::parse("https://example.com/a").unwrap()]+ vec![DatedUrl {+ url: Url::parse("https://example.com/a").unwrap(),+ last_modified: Some(+ DateTime::parse_from_rfc3339("2003-12-13T18:30:02Z")+ .unwrap()+ .with_timezone(&Utc)+ ),+ }]);}}diff --git a/crates/core/src/feed/scheduler.rs b/crates/core/src/feed/scheduler.rsdeleted file mode 100644index 24ad9a91..00000000--- a/crates/core/src/feed/scheduler.rs+++ /dev/null@@ -1,177 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::path::Path;--use anyhow::Result;-use hashbrown::{HashMap, HashSet};-use url::Url;--use crate::{- webgraph::{- centrality::{top_nodes, TopNodes},- NodeID, Webgraph,- },- webpage::url_ext::UrlExt,-};--use super::{index::FeedIndex, Feed};--#[derive(- Debug,- Clone,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,- PartialEq,- Eq,- Hash,-)]-pub struct Domain(String);--#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]-pub struct DomainFeeds {- pub domain: Domain,- pub feeds: Vec<Feed>,-}--impl From<&Url> for Domain {- fn from(url: &Url) -> Self {- Domain(url.icann_domain().unwrap_or_default().to_string())- }-}--#[derive(- Debug,- Clone,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,- PartialEq,- Eq,- Hash,-)]-pub struct SplitId(#[bincode(with_serde)] uuid::Uuid);--impl std::fmt::Display for SplitId {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- write!(f, "SplitId({})", self.0)- }-}--impl SplitId {- pub fn id(&self) -> uuid::Uuid {- self.0- }-}--#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]-pub struct Split {- pub id: SplitId,- pub feeds: Vec<DomainFeeds>,-}--impl Split {- pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {- let file = std::fs::OpenOptions::new()- .write(true)- .create(true)- .truncate(true)- .open(path.as_ref())?;- let writer = std::io::BufWriter::new(file);-- serde_json::to_writer_pretty(writer, &self)- .map_err(|e| anyhow::anyhow!("Failed to serialize split: {}", e))?;-- Ok(())- }-- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let file = std::fs::OpenOptions::new().read(true).open(path.as_ref())?;- let reader = std::io::BufReader::new(file);-- serde_json::from_reader(reader)- .map_err(|e| anyhow::anyhow!("Failed to deserialize split: {}", e))- }-}--pub struct Schedule {- splits: Vec<Split>,-}-impl Schedule {- pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {- if !path.as_ref().exists() {- std::fs::create_dir_all(&path)?;- }-- for split in self.splits {- let name = split.id.0.to_string() + ".json";- split.save(path.as_ref().join(name))?;- }-- Ok(())- }-}--pub fn schedule(- index: &FeedIndex,- host_centrality: &speedy_kv::Db<NodeID, f64>,- host_graph: &Webgraph,- num_splits: u64,-) -> Schedule {- let top_hosts = top_nodes(host_centrality, TopNodes::Top(1_000_000));-- let mut all_feeds = HashMap::new();-- for (host, _) in top_hosts {- let host = host_graph.id2node(&host).unwrap();- let url = Url::parse(&format!("http://{}", host.as_str()));-- if url.is_err() {- continue;- }-- let url = url.unwrap();- let domain = Domain::from(&url);-- all_feeds- .entry(domain)- .or_insert(HashSet::new())- .extend(index.search(host.as_str()).unwrap().into_iter());- }-- let mut splits = Vec::new();-- for _ in 0..num_splits {- splits.push(Split {- id: SplitId(uuid::Uuid::new_v4()),- feeds: Vec::new(),- });- }-- for (i, (domain, feeds)) in all_feeds.into_iter().enumerate() {- let split = &mut splits[i % num_splits as usize];-- split.feeds.push(DomainFeeds {- domain,- feeds: feeds.into_iter().collect(),- });- }-- Schedule { splits }-}diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rsindex 0efdce1e..8aea95ba 100644--- a/crates/core/src/inverted_index/search.rs+++ b/crates/core/src/inverted_index/search.rs@@ -39,6 +39,7 @@ use crate::snippet;use crate::snippet::TextSnippet;use crate::webgraph::NodeID;+use crate::schema::text_field::TextField;use crate::webpage::url_ext::UrlExt;use crate::Result;@@ -186,8 +187,7 @@ impl InvertedIndex {let tv_searcher = self.reader.searcher();let mut webpages: Vec<RetrievedWebpage> = websites.iter()- .map(|website| self.retrieve_doc(website.address, &tv_searcher))- .filter_map(|res| res.ok())+ .filter_map(|website| self.retrieve_doc(website.address, &tv_searcher).ok()).collect();for (url, page) in webpages.iter_mut().filter_map(|page| {@@ -310,4 +310,30 @@ impl InvertedIndex {pub(crate) fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {KeyPhrase::compute_top(&self.reader, top_n)}++ pub(crate) fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {+ let ctx = self.local_search_ctx();+ let tv_searcher = ctx.tv_searcher;++ let field = tv_searcher+ .schema()+ .get_field(text_field::SiteNoTokenizer.name())+ .unwrap();++ let term = tantivy::Term::from_field_text(field, site);++ let query = tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic);++ match tv_searcher.search(+ &query,+ &tantivy::collector::TopDocs::with_limit(limit).and_offset(offset),+ ) {+ Ok(res) => res+ .into_iter()+ .filter_map(|(_, doc)| self.retrieve_doc(doc.into(), &tv_searcher).ok())+ .filter_map(|page| Url::parse(&page.url).ok())+ .collect(),+ Err(_) => vec![],+ }+ }}diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 23b43519..28979087 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -52,6 +52,7 @@ pub mod canon_index;mod collector;pub mod config;pub mod crawler;+mod dated_url;pub mod distributed;pub mod entity_index;mod enum_map;@@ -82,6 +83,7 @@ mod search_prettifier;pub mod searcher;mod simhash;pub mod similar_hosts;+mod sitemap;mod snippet;mod stopwords;pub mod summarizer;@@ -145,22 +147,17 @@ pub fn start_gossip_cluster_thread(config: GossipConfig, service: Option<Servicert.block_on(async {let cluster = match service {Some(service) => Cluster::join(- Member {- id: config.cluster_id,- service,- },- config.addr,- config.seed_nodes.unwrap_or_default(),- )- .await- .unwrap(),- None => Cluster::join_as_spectator(- config.cluster_id,+ Member::new(service),config.addr,config.seed_nodes.unwrap_or_default(),).await.unwrap(),+ None => {+ Cluster::join_as_spectator(config.addr, config.seed_nodes.unwrap_or_default())+ .await+ .unwrap()+ }};let cluster = Arc::new(cluster);diff --git a/crates/core/src/live_index/crawler/budgets.rs b/crates/core/src/live_index/crawler/budgets.rsnew file mode 100644index 00000000..f4eb3114--- /dev/null+++ b/crates/core/src/live_index/crawler/budgets.rs@@ -0,0 +1,147 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/++use url::Url;++use crate::config::DailyLiveIndexCrawlerBudget;+use crate::entrypoint::site_stats::{FinalSiteStats, Site};+use crate::webgraph;+use crate::webpage::url_ext::UrlExt;+use crate::Result;+use std::collections::HashMap;+use std::path::Path;+use std::time::Duration;++use super::SiteStats;++const MILLIS_PER_DAY: u64 = 24 * 60 * 60 * 1000;++fn assign_budgets(+ sites: &mut HashMap<Site, f64>,+ daily_budget: u64,+ total_centrality: f64,+ host_centrality: &speedy_kv::Db<webgraph::NodeID, f64>,+) {+ for (site, budget) in sites.iter_mut() {+ if let Some(centrality) = Url::robust_parse(site.as_str())+ .ok()+ .map(webgraph::Node::from)+ .map(|node| node.id())+ .and_then(|id| host_centrality.get(&id).ok().flatten())+ {+ *budget = (centrality * daily_budget as f64) / total_centrality;+ }+ }+}++fn calculate_total<'a>(+ sites: impl Iterator<Item = &'a FinalSiteStats>,+ host_centrality: &speedy_kv::Db<webgraph::NodeID, f64>,+) -> f64 {+ sites+ .filter_map(|site| Url::robust_parse(site.site().as_str()).ok())+ .map(webgraph::Node::from)+ .filter_map(|node| host_centrality.get(&node.id()).ok().flatten())+ .sum()+}++pub struct SiteBudgets {+ blogs: HashMap<Site, f64>,+ news: HashMap<Site, f64>,+ remaining: HashMap<Site, f64>,+}++impl SiteBudgets {+ pub fn new<P: AsRef<Path>>(+ host_centrality: P,+ stats: &SiteStats,+ daily_budget: DailyLiveIndexCrawlerBudget,+ ) -> Result<Self> {+ tracing::debug!("Assigning budgets");+ let host_centrality: speedy_kv::Db<webgraph::NodeID, f64> =+ speedy_kv::Db::open_or_create(host_centrality)?;++ let mut blogs = HashMap::new();+ let mut news = HashMap::new();+ let mut remaining = HashMap::new();++ for site in stats.blogs() {+ blogs.insert(site.site().clone(), 0.0);+ }++ for site in stats.news() {+ news.insert(site.site().clone(), 0.0);+ }++ for site in stats.all() {+ if blogs.contains_key(site.site()) || news.contains_key(site.site()) {+ continue;+ }++ remaining.insert(site.site().clone(), 0.0);+ }++ let blogs_total = calculate_total(stats.blogs(), &host_centrality);+ let news_total = calculate_total(stats.news(), &host_centrality);+ let remaining_total = calculate_total(+ stats+ .all()+ .filter(|site| !blogs.contains_key(site.site()) && !news.contains_key(site.site())),+ &host_centrality,+ );++ assign_budgets(+ &mut blogs,+ daily_budget.blogs,+ blogs_total,+ &host_centrality,+ );+ assign_budgets(&mut news, daily_budget.news, news_total, &host_centrality);+ assign_budgets(+ &mut remaining,+ daily_budget.remaining,+ remaining_total,+ &host_centrality,+ );++ tracing::debug!("Budgets assigned");+ tracing::debug!(+ "Number of sites with budgets: blogs: {}, news: {}, remaining: {}",+ blogs.len(),+ news.len(),+ remaining.len(),+ );+ Ok(Self {+ blogs,+ news,+ remaining,+ })+ }++ pub fn drip_rate(&self, site: &Site) -> Option<Duration> {+ let budget = *self+ .blogs+ .get(site)+ .or_else(|| self.news.get(site))+ .or_else(|| self.remaining.get(site))?;++ if budget == 0.0 {+ Some(Duration::from_millis(MILLIS_PER_DAY)) // once per day+ } else {+ Some(Duration::from_millis(MILLIS_PER_DAY / budget as u64))+ }+ }+}diff --git a/crates/core/src/live_index/crawler/checker/feeds.rs b/crates/core/src/live_index/crawler/checker/feeds.rsnew file mode 100644index 00000000..1e394de5--- /dev/null+++ b/crates/core/src/live_index/crawler/checker/feeds.rs@@ -0,0 +1,66 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::time::Duration;++use crate::feed::{parse, Feed};+use crate::Result;++use super::{CheckIntervals, Checker, CrawlableUrl};++const CRAWL_DELAY: Duration = Duration::from_secs(5);++pub struct Feeds {+ feeds: Vec<Feed>,+ last_check: std::time::Instant,+ client: reqwest::Client,+}++impl Feeds {+ pub fn new(feeds: Vec<Feed>, client: reqwest::Client) -> Self {+ Self {+ feeds,+ last_check: std::time::Instant::now(),+ client,+ }+ }+}++impl Checker for Feeds {+ async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {+ let mut urls = Vec::new();++ for feed in &self.feeds {+ let resp = self.client.get(feed.url.clone()).send().await?;+ let text = resp.text().await?;+ let parsed_feed = parse(&text, feed.kind)?;++ for link in parsed_feed.links {+ urls.push(CrawlableUrl::from(link));+ }++ tokio::time::sleep(CRAWL_DELAY).await;+ }++ self.last_check = std::time::Instant::now();++ Ok(urls)+ }++ fn should_check(&self, interval: &CheckIntervals) -> bool {+ self.last_check.elapsed() > interval.feeds+ }+}diff --git a/crates/core/src/live_index/crawler/checker/frontpage.rs b/crates/core/src/live_index/crawler/checker/frontpage.rsnew file mode 100644index 00000000..bf4d92f9--- /dev/null+++ b/crates/core/src/live_index/crawler/checker/frontpage.rs@@ -0,0 +1,65 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use url::Url;++use crate::config::CheckIntervals;+use crate::webpage::Html;+use crate::Result;+use crate::{entrypoint::site_stats, webpage::url_ext::UrlExt};++use super::{Checker, CrawlableUrl};++pub struct Frontpage {+ url: Url,+ last_check: std::time::Instant,+ client: reqwest::Client,+}++impl Frontpage {+ pub fn new(site: &site_stats::Site, client: reqwest::Client) -> Result<Self> {+ let url = Url::robust_parse(&format!("https://{}/", site.as_str()))?;++ Ok(Self {+ url,+ last_check: std::time::Instant::now(),+ client,+ })+ }+}++impl Checker for Frontpage {+ async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {+ let res = self.client.get(self.url.clone()).send().await?;+ let body = res.text().await?;++ let page = Html::parse(&body, self.url.as_str())?;++ let urls = page+ .anchor_links()+ .into_iter()+ .map(|link| CrawlableUrl::from(link.destination))+ .collect::<Vec<_>>();++ self.last_check = std::time::Instant::now();++ Ok(urls)+ }++ fn should_check(&self, interval: &CheckIntervals) -> bool {+ self.last_check.elapsed() > interval.frontpage+ }+}diff --git a/crates/core/src/live_index/crawler/checker/mod.rs b/crates/core/src/live_index/crawler/checker/mod.rsnew file mode 100644index 00000000..ec349c80--- /dev/null+++ b/crates/core/src/live_index/crawler/checker/mod.rs@@ -0,0 +1,60 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/++use chrono::{DateTime, Utc};+use url::Url;++mod feeds;+mod frontpage;+mod sitemap;++pub use self::feeds::Feeds;+pub use frontpage::Frontpage;+pub use sitemap::Sitemap;++use crate::config::CheckIntervals;+use crate::dated_url::DatedUrl;++use crate::Result;++#[derive(Debug)]+pub struct CrawlableUrl {+ pub url: Url,+ pub last_modified: Option<DateTime<Utc>>,+}++impl From<DatedUrl> for CrawlableUrl {+ fn from(url: DatedUrl) -> Self {+ Self {+ url: url.url,+ last_modified: url.last_modified,+ }+ }+}++impl From<Url> for CrawlableUrl {+ fn from(url: Url) -> Self {+ Self {+ url,+ last_modified: None,+ }+ }+}++pub trait Checker {+ async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>>;+ fn should_check(&self, interval: &CheckIntervals) -> bool;+}diff --git a/crates/core/src/live_index/crawler/checker/sitemap.rs b/crates/core/src/live_index/crawler/checker/sitemap.rsnew file mode 100644index 00000000..eeb598a2--- /dev/null+++ b/crates/core/src/live_index/crawler/checker/sitemap.rs@@ -0,0 +1,132 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::time::Duration;++use url::Url;++use crate::dated_url::DatedUrl;+use crate::sitemap::{parse_sitemap, SitemapEntry};+use crate::Result;+use crate::{entrypoint::site_stats, webpage::url_ext::UrlExt};++use super::{CheckIntervals, Checker, CrawlableUrl};++const MAX_SITEMAP_DEPTH: usize = 10;+const SITEMAP_DELAY: Duration = Duration::from_secs(10);++pub struct Sitemap {+ robots_txt: Url,+ last_check: std::time::Instant,+ client: reqwest::Client,+}++impl Sitemap {+ pub fn new(site: &site_stats::Site, client: reqwest::Client) -> Result<Self> {+ let robots_txt = Url::robust_parse(&format!("{}/robots.txt", site.as_str()))?;++ Ok(Self {+ robots_txt,+ last_check: std::time::Instant::now(),+ client,+ })+ }++ async fn sitemap_urls(&self) -> Result<Vec<Url>> {+ let res = self.client.get(self.robots_txt.clone()).send().await?;+ let body = res.text().await?;++ // wildcard useragent is okay as we only use it to check for sitemap directive+ let robots = robotstxt::Robots::parse("*", &body)?;++ Ok(robots+ .sitemaps()+ .iter()+ .filter_map(|s| Url::parse(s).ok())+ .collect())+ }++ async fn urls_from_sitemap(&self, sitemap: Url) -> Vec<DatedUrl> {+ let mut stack = vec![(sitemap, 0)];+ let mut urls = vec![];++ while let Some((url, depth)) = stack.pop() {+ if depth >= MAX_SITEMAP_DEPTH {+ continue;+ }++ let res = self.client.get(url).send().await;+ tokio::time::sleep(SITEMAP_DELAY).await;++ if res.is_err() {+ continue;+ }++ let res = res.unwrap();++ if res.status() != reqwest::StatusCode::OK {+ continue;+ }++ let body = res.text().await;++ if body.is_err() {+ continue;+ }++ let body = body.unwrap();++ let entries = parse_sitemap(&body);++ for entry in entries {+ match entry {+ SitemapEntry::Url(url) => {+ urls.push(url);+ }+ SitemapEntry::Sitemap(url) => {+ stack.push((url, depth + 1));+ }+ }+ }+ }++ urls+ }+}++impl Checker for Sitemap {+ async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {+ let sitemap_urls = self.sitemap_urls().await?;+ let mut urls = vec![];++ for sitemap_url in sitemap_urls {+ urls.extend(+ self.urls_from_sitemap(sitemap_url)+ .await+ .into_iter()+ .map(CrawlableUrl::from),+ );+ }++ self.last_check = std::time::Instant::now();++ Ok(urls)+ }++ fn should_check(&self, interval: &CheckIntervals) -> bool {+ self.last_check.elapsed() > interval.sitemap+ }+}diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rsnew file mode 100644index 00000000..3c80a516--- /dev/null+++ b/crates/core/src/live_index/crawler/crawlable_site.rs@@ -0,0 +1,340 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/++use std::{+ sync::{+ atomic::{AtomicBool, Ordering},+ Arc,+ },+ time::{Duration, Instant},+};++use itertools::Itertools;+use tokio::sync::Mutex;+use url::Url;++use crate::{+ config::{CheckIntervals, CrawlerConfig},+ crawler,+ entrypoint::{+ indexer::IndexableWebpage,+ site_stats::{self, FinalSiteStats},+ },+ live_index::crawler::checker::{Feeds, Frontpage, Sitemap},+ webpage::url_ext::UrlExt,+ Result,+};++use super::{+ checker::{Checker, CrawlableUrl},+ crawled_db::ShardedCrawledDb,+ Client,+};++pub const MAX_PENDING_BUDGET: u64 = 128;++struct InnerCrawlableSite {+ site: site_stats::Site,+ feeds: Feeds,+ sitemap: Sitemap,+ frontpage: Frontpage,+ last_drip: Instant,+ drip_rate: Duration,+ budget: u64,+}++impl InnerCrawlableSite {+ pub fn new(site: FinalSiteStats, client: &Client, drip_rate: Duration) -> Result<Self> {+ Ok(Self {+ site: site.site().clone(),+ feeds: Feeds::new(+ site.stats()+ .feeds+ .clone()+ .into_iter()+ .map(|feed| feed.into())+ .collect(),+ client.reqwest().clone(),+ ),+ sitemap: Sitemap::new(site.site(), client.reqwest().clone())?,+ frontpage: Frontpage::new(site.site(), client.reqwest().clone())?,+ last_drip: Instant::now(),+ drip_rate,+ budget: 0,+ })+ }++ pub fn drip(&mut self) {+ let pages_to_drip =+ (self.last_drip.elapsed().as_secs_f32() / self.drip_rate.as_secs_f32()) as u64;++ if pages_to_drip > 0 {+ self.last_drip = Instant::now();+ self.budget += pages_to_drip;++ if self.budget > MAX_PENDING_BUDGET {+ self.budget = MAX_PENDING_BUDGET;+ }+ }+ }+ pub fn should_crawl(&self, interval: &CheckIntervals) -> bool {+ (self.frontpage.should_check(interval)+ || self.sitemap.should_check(interval)+ || self.feeds.should_check(interval))+ && self.budget > 0+ }+}++pub struct CrawlableSite {+ inner: Arc<Mutex<InnerCrawlableSite>>,+ currently_crawling: AtomicBool,+ site: site_stats::Site,+}++impl CrawlableSite {+ pub fn new(site: FinalSiteStats, client: &Client, drip_rate: Duration) -> Result<Self> {+ Ok(Self {+ site: site.site().clone(),+ inner: Arc::new(Mutex::new(InnerCrawlableSite::new(+ site, client, drip_rate,+ )?)),+ currently_crawling: AtomicBool::new(false),+ })+ }++ pub fn currently_crawling(&self) -> bool {+ self.currently_crawling.load(Ordering::Relaxed)+ }++ pub fn site(&self) -> &site_stats::Site {+ &self.site+ }++ pub async fn should_crawl(&self, interval: &CheckIntervals) -> bool {+ !self.currently_crawling() && self.inner.lock().await.should_crawl(interval)+ }++ pub async fn drip(&self) {+ if self.currently_crawling() {+ return;+ }++ self.inner.lock().await.drip();+ }+}++impl crawler::DatumStream for tokio::sync::Mutex<Vec<crawler::CrawlDatum>> {+ async fn write(&self, crawl_datum: crawler::CrawlDatum) -> Result<(), crawler::Error> {+ self.lock().await.push(crawl_datum);+ Ok(())+ }++ async fn finish(&self) -> Result<(), crawler::Error> {+ Ok(())+ }+}++pub struct CrawlableSiteGuard {+ site: Arc<CrawlableSite>,+ crawled_db: Arc<ShardedCrawledDb>,+ config: Arc<CrawlerConfig>,+}++impl CrawlableSiteGuard {+ pub async fn new(+ site: Arc<CrawlableSite>,+ crawled_db: Arc<ShardedCrawledDb>,+ config: Arc<CrawlerConfig>,+ ) -> Self {+ {+ let currently_crawling = site.currently_crawling.swap(true, Ordering::Relaxed);+ if currently_crawling {+ panic!("site is already being crawled");+ }+ }++ Self {+ site,+ crawled_db,+ config,+ }+ }++ pub fn url(&self) -> Result<Url> {+ self.site.site().url()+ }+}++impl Drop for CrawlableSiteGuard {+ fn drop(&mut self) {+ self.site.currently_crawling.store(false, Ordering::Relaxed);++ tracing::debug!(+ "Dropping crawlable site guard for site {}",+ self.site.site().as_str()+ );+ }+}++impl CrawlableSiteGuard {+ pub async fn crawl(self, client: &Client, interval: &CheckIntervals) -> Result<()> {+ let mut site = self.site.inner.lock().await;+ let mut urls = Vec::new();++ if site.feeds.should_check(interval) {+ urls.extend(site.feeds.get_urls().await.unwrap_or_default());+ }++ if site.sitemap.should_check(interval) {+ urls.extend(site.sitemap.get_urls().await.unwrap_or_default());+ }++ if site.frontpage.should_check(interval) {+ urls.extend(site.frontpage.get_urls().await.unwrap_or_default());+ }++ let url = site.site.url()?;+ let icann_domain = url.icann_domain();+ urls = urls+ .into_iter()+ .filter(|u| u.url.icann_domain() == icann_domain)+ .unique_by(|u| u.url.clone())+ .collect();++ urls.retain(|url| !self.crawled_db.has_crawled(&url.url).unwrap_or(false));++ order_urls(&mut urls);++ let budget = site.budget.min(urls.len() as u64);+ site.budget = site.budget.saturating_sub(budget);++ urls.truncate(budget as usize);++ if urls.is_empty() {+ tracing::debug!("No new urls to crawl for site {}", site.site.as_str());+ return Ok(());+ }++ tracing::debug!(+ "Crawling {} urls for site {}",+ urls.len(),+ site.site.as_str()+ );++ for crawlable_url in &urls {+ self.crawled_db.insert(&crawlable_url.url)?;+ }++ let crawl_data = Arc::new(tokio::sync::Mutex::new(Vec::new()));++ let executor = crawler::JobExecutor::new(+ crawler::WorkerJob {+ domain: crawler::Domain::from(site.site.url()?),+ urls: urls+ .into_iter()+ .take(budget as usize)+ .map(|url| {+ let weighted_url = crawler::WeightedUrl {+ url: url.url,+ weight: 1.0,+ };++ crawler::RetrieableUrl::from(weighted_url)+ })+ .collect(),+ wandering_urls: 0,+ },+ client.reqwest().clone(),+ self.config.clone(),+ Arc::clone(&crawl_data),+ );++ executor.run().await;++ let crawl_data = crawl_data.lock().await.clone();++ tracing::debug!(+ "Indexing {} urls for site {}",+ crawl_data.len(),+ site.site.as_str()+ );+ client+ .index(crawl_data.into_iter().map(IndexableWebpage::from).collect())+ .await?;++ tracing::debug!("Finished crawling site {}", site.site.as_str());++ Ok(())+ }+}++fn order_urls(urls: &mut [CrawlableUrl]) {+ urls.sort_by(|a, b| match (a.last_modified, b.last_modified) {+ (Some(a), Some(b)) => a.cmp(&b).reverse(),+ (Some(_), None) => std::cmp::Ordering::Greater,+ (None, Some(_)) => std::cmp::Ordering::Less,+ (None, None) => std::cmp::Ordering::Equal,+ });+}++#[cfg(test)]+mod tests {+ use chrono::DateTime;+ use url::Url;++ use super::*;++ #[test]+ fn test_latest_urls_crawled_first() {+ let mut urls = vec![+ CrawlableUrl {+ url: Url::parse("https://example.com/page1").unwrap(),+ last_modified: Some(+ DateTime::parse_from_rfc2822("Mon, 01 Jan 2024 00:00:00 GMT")+ .unwrap()+ .into(),+ ),+ },+ CrawlableUrl {+ url: Url::parse("https://example.com/page2").unwrap(),+ last_modified: Some(+ DateTime::parse_from_rfc2822("Tue, 02 Jan 2024 00:00:01 GMT")+ .unwrap()+ .into(),+ ),+ },+ CrawlableUrl {+ url: Url::parse("https://example.com/page3").unwrap(),+ last_modified: None,+ },+ ];++ order_urls(&mut urls);++ assert_eq!(+ urls[0].url,+ Url::parse("https://example.com/page3").unwrap()+ );+ assert_eq!(+ urls[1].url,+ Url::parse("https://example.com/page2").unwrap()+ );+ assert_eq!(+ urls[2].url,+ Url::parse("https://example.com/page1").unwrap()+ );+ }+}diff --git a/crates/core/src/live_index/crawler/downloaded_db.rs b/crates/core/src/live_index/crawler/crawled_db.rssimilarity index 67%rename from crates/core/src/live_index/crawler/downloaded_db.rsrename to crates/core/src/live_index/crawler/crawled_db.rsindex 30bfc9e7..9e90eb06 100644--- a/crates/core/src/live_index/crawler/downloaded_db.rs+++ b/crates/core/src/live_index/crawler/crawled_db.rs@@ -14,11 +14,16 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/+use dashmap::DashMap;use redb::ReadableTable;use url::Url;-use crate::Result;-use std::{path::Path, time::Duration};+use crate::{webpage::url_ext::UrlExt, Result};+use std::{+ path::{Path, PathBuf},+ sync::Arc,+ time::Duration,+};use crate::live_index::TTL;@@ -172,7 +177,7 @@ impl InnerDb {redb::TableDefinition::new("times")}- fn has_downloaded(&self, url: &Url) -> Result<bool> {+ fn has_crawled(&self, url: &Url) -> Result<bool> {let url = TruncatedUrl::new(url);Ok(self@@ -232,7 +237,7 @@ impl InnerDb {self.truncate(TTL)}- pub fn insert(&mut self, url: &Url) -> Result<()> {+ fn insert(&mut self, url: &Url) -> Result<()> {{let key = TruncatedUrl::new(url);let txn = self.db.begin_write()?;@@ -258,11 +263,11 @@ impl InnerDb {}}-pub struct DownloadedDb {+struct CrawledDb {inner: std::sync::Mutex<InnerDb>,}-impl DownloadedDb {+impl CrawledDb {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {let db = InnerDb::open(path)?;@@ -271,8 +276,8 @@ impl DownloadedDb {})}- pub fn has_downloaded(&self, url: &Url) -> Result<bool> {- self.inner.lock().unwrap().has_downloaded(url)+ pub fn has_crawled(&self, url: &Url) -> Result<bool> {+ self.inner.lock().unwrap().has_crawled(url)}pub fn insert(&self, url: &Url) -> Result<()> {@@ -280,35 +285,120 @@ impl DownloadedDb {}}+#[derive(Debug, Clone, Hash, Eq, PartialEq)]+struct ShardedHost(String);++impl ShardedHost {+ fn from_url(url: &Url) -> Option<Self> {+ let host = url.normalized_host()?;+ let hash = md5::compute(host);+ let hash = format!("{:x}", hash).chars().take(2).collect();++ Some(Self(hash))+ }+}++pub struct ShardedCrawledDb {+ inner: Arc<DashMap<ShardedHost, CrawledDb>>,+ folder: PathBuf,+}++impl ShardedCrawledDb {+ pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {+ if !folder.as_ref().exists() {+ std::fs::create_dir_all(folder.as_ref())?;+ }++ let inner = Arc::new(+ std::fs::read_dir(&folder)?+ .filter_map(|entry| entry.ok())+ .filter(|entry| entry.file_type().map(|ft| ft.is_dir()).unwrap_or(false))+ .filter_map(|entry| {+ let path = entry.path();+ let shard = ShardedHost(path.file_name()?.to_str()?.to_string());+ let db = CrawledDb::open(path).ok()?;+ Some((shard, db))+ })+ .collect::<DashMap<ShardedHost, CrawledDb>>(),+ );++ Ok(Self {+ folder: folder.as_ref().to_path_buf(),+ inner,+ })+ }++ pub fn has_crawled(&self, url: &Url) -> Result<bool> {+ let shard =+ ShardedHost::from_url(url).ok_or(anyhow::anyhow!("Failed to get shard from url"))?;+ match self.inner.get(&shard) {+ Some(db) => db.has_crawled(url),+ None => Ok(false),+ }+ }++ pub fn insert(&self, url: &Url) -> Result<()> {+ let shard =+ ShardedHost::from_url(url).ok_or(anyhow::anyhow!("Failed to get shard from url"))?;++ let db = self.inner.entry(shard.clone()).or_insert_with(|| {+ let path = self.folder.join(shard.0);+ CrawledDb::open(path).unwrap()+ });++ db.insert(url)+ }+}+#[cfg(test)]mod tests {use super::*;#[test]fn test_downloaded_db() {- let db = DownloadedDb::open(crate::gen_temp_path()).unwrap();+ let db = CrawledDb::open(crate::gen_temp_path()).unwrap();let url = Url::parse("https://example.com").unwrap();- assert!(!db.has_downloaded(&url).unwrap());+ assert!(!db.has_crawled(&url).unwrap());db.insert(&url).unwrap();- assert!(db.has_downloaded(&url).unwrap());+ assert!(db.has_crawled(&url).unwrap());}#[test]fn test_truncate_ttl() {- let db = DownloadedDb::open(crate::gen_temp_path()).unwrap();+ let db = CrawledDb::open(crate::gen_temp_path()).unwrap();let url = Url::parse("https://example.com").unwrap();db.insert(&url).unwrap();- assert!(db.has_downloaded(&url).unwrap());+ assert!(db.has_crawled(&url).unwrap());let ttl = Duration::from_secs(1);std::thread::sleep(ttl + Duration::from_secs(1));db.inner.lock().unwrap().truncate(ttl).unwrap();- assert!(!db.has_downloaded(&url).unwrap());+ assert!(!db.has_crawled(&url).unwrap());+ }++ #[test]+ fn test_sharded_downloaded_db() {+ let db = ShardedCrawledDb::open(crate::gen_temp_path()).unwrap();++ let url = Url::parse("https://example.com").unwrap();+ assert!(!db.has_crawled(&url).unwrap());++ db.insert(&url).unwrap();+ assert!(db.has_crawled(&url).unwrap());++ let url = Url::parse("https://example.com/foo").unwrap();+ assert!(!db.has_crawled(&url).unwrap());++ db.insert(&url).unwrap();+ assert!(db.has_crawled(&url).unwrap());++ let url = Url::parse("https://another_example.com/bar").unwrap();+ assert!(!db.has_crawled(&url).unwrap());}}diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rsindex 84637499..f1209505 100644--- a/crates/core/src/live_index/crawler/mod.rs+++ b/crates/core/src/live_index/crawler/mod.rs@@ -12,135 +12,326 @@// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/+// along with this program. If not, see <https://www.gnu.org/licenses/>-mod downloaded_db;--use crate::Result;+mod budgets;+mod checker;+mod crawlable_site;+mod crawled_db;+use std::fs::File;+use std::path::PathBuf;use std::sync::Arc;+use std::time::Duration;-use chrono::Utc;-use url::Url;-+use crate::ampc::dht::ShardId;+use crate::config::{CheckIntervals, CrawlerConfig, LiveCrawlerConfig};+use crate::distributed::cluster::Cluster;+use crate::distributed::sonic::replication::{+ AllShardsSelector, RandomReplicaSelector, RandomShardSelector, ShardedClient,+};+use crate::entrypoint::search_server;+use crate::entrypoint::site_stats::FinalSiteStats;+use crate::{crawler, Result};use crate::{- config::CrawlerConfig,- crawler::reqwest_client,- feed::{- self,- scheduler::{Domain, DomainFeeds, Split},- },+ distributed::sonic::replication::ReusableShardedClient,+ entrypoint::{indexer::IndexableWebpage, live_index},};+use crawlable_site::{CrawlableSite, CrawlableSiteGuard};+use crawled_db::ShardedCrawledDb;+use tokio::sync::Mutex;+use tokio::sync::Semaphore;+use url::Url;++const SITE_URL_BATCH_SIZE: usize = 100;+const DEFAULT_CONSISTENCY_FRACTION: f64 = 0.5;+const BLOG_FRACTION_THRESHOLD: f64 = 0.5;+const NEWS_FRACTION_THRESHOLD: f64 = 0.5;+const MIN_CRAWL_DELAY: Duration = Duration::from_secs(30);+const MAX_CRAWL_DELAY: Duration = Duration::from_secs(300);+const TICK_INTERVAL: Duration = Duration::from_secs(5);++struct Client {+ live_index: Mutex<ReusableShardedClient<live_index::LiveIndexService>>,+ search: Mutex<ReusableShardedClient<search_server::SearchService>>,+ reqwest: reqwest::Client,+}++impl Client {+ pub async fn new(cluster: Arc<Cluster>, crawler_config: &CrawlerConfig) -> Result<Self> {+ let live_index = Mutex::new(ReusableShardedClient::new(cluster.clone()).await);+ let search = Mutex::new(ReusableShardedClient::new(cluster.clone()).await);++ Ok(Self {+ live_index,+ search,+ reqwest: crawler::reqwest_client(crawler_config)?,+ })+ }++ pub fn reqwest(&self) -> &reqwest::Client {+ &self.reqwest+ }++ async fn live_conn(&self) -> Arc<ShardedClient<live_index::LiveIndexService, ShardId>> {+ self.live_index.lock().await.conn().await+ }++ async fn search_conn(&self) -> Arc<ShardedClient<search_server::SearchService, ShardId>> {+ self.search.lock().await.conn().await+ }++ pub async fn index(&self, pages: Vec<IndexableWebpage>) -> Result<()> {+ let conn = self.live_conn().await;-use downloaded_db::DownloadedDb;+ let req = live_index::IndexWebpages {+ pages,+ consistency_fraction: Some(DEFAULT_CONSISTENCY_FRACTION),+ };++ while let Err(e) = conn+ .send(req.clone(), &RandomShardSelector, &RandomReplicaSelector)+ .await+ {+ tracing::error!("Failed to index pages: {e}");+ tokio::time::sleep(Duration::from_millis(1_000)).await;+ }++ Ok(())+ }++ async fn get_site_urls_from_backbone(&self, site: &str) -> Result<Vec<Url>> {+ let mut res = Vec::new();+ let conn = self.search_conn().await;+ let mut offset = 0;++ while let Ok(resp) = conn+ .send(+ search_server::GetSiteUrls {+ site: site.to_string(),+ offset,+ limit: SITE_URL_BATCH_SIZE,+ },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await+ {+ let urls: Vec<_> = resp+ .into_iter()+ .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))+ .collect();++ if urls.is_empty() {+ break;+ }++ res.extend(urls);+ offset += SITE_URL_BATCH_SIZE;+ }++ Ok(res)+ }++ async fn get_site_urls_from_live(&self, site: &str) -> Result<Vec<Url>> {+ let mut res = Vec::new();+ let conn = self.live_conn().await;+ let mut offset = 0;++ while let Ok(resp) = conn+ .send(+ search_server::GetSiteUrls {+ site: site.to_string(),+ offset,+ limit: SITE_URL_BATCH_SIZE,+ },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await+ {+ let urls: Vec<_> = resp+ .into_iter()+ .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))+ .collect();++ if urls.is_empty() {+ break;+ }++ res.extend(urls);+ offset += SITE_URL_BATCH_SIZE;+ }++ Ok(res)+ }-use super::{Feeds, FEED_CHECK_INTERVAL};+ pub async fn get_site_urls(&self, site: &str) -> Vec<Url> {+ let mut res = Vec::new();-pub enum CrawlResults {- None,- HasInserts,+ if let Ok(urls) = self.get_site_urls_from_backbone(site).await {+ res.extend(urls);+ }++ if let Ok(urls) = self.get_site_urls_from_live(site).await {+ res.extend(urls);+ }++ res+ }}-pub struct Crawler {- feeds: Vec<Feeds>,- downloaded_db: DownloadedDb,- _config: Arc<CrawlerConfig>,- client: reqwest::Client,+impl From<LiveCrawlerConfig> for CrawlerConfig {+ fn from(config: LiveCrawlerConfig) -> Self {+ Self {+ num_worker_threads: 1,+ user_agent: config.user_agent,+ robots_txt_cache_sec: crate::config::defaults::Crawler::robots_txt_cache_sec(),+ min_politeness_factor: 0,+ start_politeness_factor: 1,+ min_crawl_delay_ms: MIN_CRAWL_DELAY.as_millis() as u64,+ max_crawl_delay_ms: MAX_CRAWL_DELAY.as_millis() as u64,+ max_politeness_factor: crate::config::defaults::Crawler::max_politeness_factor(),+ max_url_slowdown_retry: crate::config::defaults::Crawler::max_url_slowdown_retry(),+ timeout_seconds: 60,+ s3: crate::config::S3Config {+ bucket: String::new(),+ folder: String::new(),+ access_key: String::new(),+ secret_key: String::new(),+ endpoint: String::new(),+ },+ router_hosts: vec![],+ }+ }}-impl Crawler {- pub fn new(- split: Split,- downloaded_db: DownloadedDb,- config: Arc<CrawlerConfig>,- ) -> Result<Self> {- let client = reqwest_client(&config)?;+struct SiteStats {+ sites: Vec<FinalSiteStats>,+}- Ok(Self {- feeds: split.into(),- downloaded_db,- _config: config,- client,+impl SiteStats {+ pub fn open(path: PathBuf) -> Result<Self> {+ let file = File::open(path)?;+ let sites = serde_json::from_reader(file)?;+ Ok(Self { sites })+ }++ pub fn blogs(&self) -> impl Iterator<Item = &FinalSiteStats> {+ self.sites.iter().filter(|site| {+ site.stats().blogposts as f64 / site.stats().pages as f64 > BLOG_FRACTION_THRESHOLD+ })+ }++ pub fn news(&self) -> impl Iterator<Item = &FinalSiteStats> {+ self.sites.iter().filter(|site| {+ site.stats().news_articles as f64 / site.stats().pages as f64 > NEWS_FRACTION_THRESHOLD})}- async fn process_urls(&self, _urls: Vec<Url>) -> Result<bool> {- todo!()-- // if urls.is_empty() {- // return Ok(false);- // }-- // let domain = urls.first().unwrap().into();- // let job = WorkerJob {- // domain,- // urls: urls- // .clone()- // .into_iter()- // .map(|url| RetrieableUrl::from(WeightedUrl { url, weight: 1.0 }))- // .collect(),- // wandering_urls: 0,- // };-- // let executor = JobExecutor::new(- // job,- // self.client.clone(),- // self.config.clone(),- // self.indexer.clone(),- // );- // executor.run().await;-- // for url in &urls {- // self.downloaded_db.insert(url)?;- // }-- // Ok(true)+ pub fn all(&self) -> impl Iterator<Item = &FinalSiteStats> {+ self.sites.iter()}+}- async fn process_feed(&self, domain_feeds: &DomainFeeds) -> Result<bool> {- let mut urls = Vec::new();-- for feed in domain_feeds.feeds.iter() {- let content = self- .client- .get(feed.url.as_str())- .send()- .await?- .text()- .await?;-- let feed = feed::parse(&content, feed.kind)?;-- for url in feed.links {- if !self.downloaded_db.has_downloaded(&url)?- && Domain::from(&url) == domain_feeds.domain- {- urls.push(url);+pub struct Crawler {+ client: Arc<Client>,+ db: Arc<ShardedCrawledDb>,+ sites: Vec<Arc<CrawlableSite>>,+ num_worker_threads: usize,+ check_intervals: CheckIntervals,+ crawler_config: Arc<CrawlerConfig>,+}++impl Crawler {+ pub async fn new(config: LiveCrawlerConfig) -> Result<Self> {+ let crawler_config = Arc::new(CrawlerConfig::from(config.clone()));++ let cluster = Arc::new(+ Cluster::join_as_spectator(+ config.gossip.addr,+ config.gossip.seed_nodes.unwrap_or_default(),+ )+ .await?,+ );++ let client = Arc::new(Client::new(cluster, &crawler_config).await?);+ let db = Arc::new(ShardedCrawledDb::open(config.crawled_db_path)?);++ let site_stats = SiteStats::open(config.site_stats_path)?;+ let sites: Vec<_> = site_stats.all().cloned().collect();++ let budgets = budgets::SiteBudgets::new(+ &config.host_centrality_path,+ &site_stats,+ config.daily_budget.clone(),+ )?;++ let mut crawlable_sites = Vec::new();+ tracing::debug!("Initializing crawler db with previously crawled urls");+ for site in sites {+ for url in client.get_site_urls(site.site().as_str()).await {+ if !db.has_crawled(&url)? {+ db.insert(&url)?;}}++ if let Some(drip_rate) = budgets.drip_rate(site.site()) {+ crawlable_sites.push(Arc::new(CrawlableSite::new(site, &client, drip_rate)?));+ }}- self.process_urls(urls).await+ Ok(Self {+ client,+ db,+ sites: crawlable_sites,+ num_worker_threads: config.num_worker_threads,+ check_intervals: config.check_intervals,+ crawler_config,+ })}- pub async fn check_feeds(&mut self) -> CrawlResults {- let mut feeds = self.feeds.clone();+ pub async fn run(mut self) -> Result<()> {+ let mut interval = tokio::time::interval(TICK_INTERVAL);+ let semaphore = Arc::new(Semaphore::new(self.num_worker_threads));- let mut futures = Vec::new();- for feeds in feeds.iter_mut() {- if feeds.last_checked + FEED_CHECK_INTERVAL < Utc::now() {- futures.push(self.process_feed(&feeds.feed));- feeds.last_checked = Utc::now();- }- }+ loop {+ interval.tick().await;+ tracing::debug!("Tick");- let res = futures::future::join_all(futures).await;+ for site in &mut self.sites {+ if site.currently_crawling() {+ tracing::debug!("Site {} is currently crawling", site.site().as_str());+ continue;+ }- self.feeds = feeds;+ site.drip().await;- if res.iter().filter_map(|r| r.as_ref().ok()).any(|r| *r) {- CrawlResults::HasInserts- } else {- CrawlResults::None+ if site.should_crawl(&self.check_intervals).await {+ tracing::debug!("Site {} should crawl", site.site().as_str());+ let client = self.client.clone();+ let intervals = self.check_intervals.clone();+ let guard = CrawlableSiteGuard::new(+ site.clone(),+ self.db.clone(),+ self.crawler_config.clone(),+ )+ .await;+ let semaphore = semaphore.clone();++ tokio::task::spawn(async move {+ let _permit = semaphore.acquire().await.unwrap();+ let url = guard.url();++ if let Err(e) = guard.crawl(&client, &intervals).await {+ if let Ok(url) = url {+ tracing::error!("Failed to crawl site {}: {:?}", url, e);+ } else {+ tracing::error!("Failed to crawl site: {:?}", e);+ }+ }+ });+ }+ }}}}diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rsindex 620df532..a0507a40 100644--- a/crates/core/src/live_index/index.rs+++ b/crates/core/src/live_index/index.rs@@ -264,6 +264,7 @@ impl LiveIndex {}pub fn commit(&self) {+ tracing::debug!("committing index");futures::executor::block_on(self.inner.write()@@ -273,6 +274,7 @@ impl LiveIndex {}pub fn prune_segments(&self) {+ tracing::debug!("pruning segments");self.inner.write().unwrap_or_else(|e| e.into_inner())@@ -287,6 +289,7 @@ impl LiveIndex {}pub fn compact_segments_by_date(&self) {+ tracing::debug!("compacting segments by date");self.inner.write().unwrap_or_else(|e| e.into_inner())@@ -294,6 +297,7 @@ impl LiveIndex {}pub fn insert(&self, pages: &[IndexableWebpage]) {+ tracing::debug!("inserting pages into index");self.inner.write().unwrap_or_else(|e| e.into_inner())diff --git a/crates/core/src/live_index/mod.rs b/crates/core/src/live_index/mod.rsindex bee8d976..afc5b732 100644--- a/crates/core/src/live_index/mod.rs+++ b/crates/core/src/live_index/mod.rs@@ -15,64 +15,18 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use std::time::Duration;-use crate::config::{CrawlerConfig, LiveIndexConfig};-pub use self::index::LiveIndex;pub use self::index_manager::IndexManager;-// mod crawler;+pub mod crawler;pub mod index;mod index_manager;+pub use self::crawler::Crawler;+const TTL: Duration = Duration::from_secs(60 * 60 * 24 * 60); // 60 daysconst PRUNE_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hourconst COMPACT_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour- // const FEED_CHECK_INTERVAL: Duration = Duration::from_secs(60 * 10); // 10 minutesconst AUTO_COMMIT_INTERVAL: Duration = Duration::from_secs(60 * 5); // 5 minutesconst EVENT_LOOP_INTERVAL: Duration = Duration::from_secs(5);const BATCH_SIZE: usize = 512;--// #[derive(Debug, Clone)]-// struct Feeds {-// last_checked: DateTime<Utc>,-// feed: DomainFeeds,-// }-//-// impl From<Split> for Vec<Feeds> {-// fn from(split: Split) -> Self {-// split-// .feeds-// .into_iter()-// .map(|feed| Feeds {-// last_checked: Utc::now(),-// feed,-// })-// .collect()-// }-// }--impl From<&LiveIndexConfig> for CrawlerConfig {- fn from(live: &LiveIndexConfig) -> Self {- Self {- num_worker_threads: 1, // no impact- user_agent: live.user_agent.clone(),- robots_txt_cache_sec: live.robots_txt_cache_sec,- start_politeness_factor: live.start_politeness_factor,- min_politeness_factor: live.min_politeness_factor,- max_politeness_factor: live.max_politeness_factor,- min_crawl_delay_ms: live.min_crawl_delay_ms,- max_crawl_delay_ms: live.max_crawl_delay_ms,- max_url_slowdown_retry: live.max_url_slowdown_retry,- timeout_seconds: live.timeout_seconds,- // no impact- s3: crate::config::S3Config {- bucket: String::new(),- folder: String::new(),- access_key: String::new(),- secret_key: String::new(),- endpoint: String::new(),- },- router_hosts: Vec::new(),- }- }-}diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex 7170ab2d..480753d8 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -177,6 +177,9 @@ enum AdminIndexOptions {enum LiveIndex {/// Serve the live index.Serve { config_path: String },++ /// Start the live index crawler.+ Crawler { config_path: String },}#[derive(Subcommand)]@@ -262,11 +265,6 @@ enum IndexingOptions {output_path: String,},- /// Create the feed index. Used to find feeds to put into the live index.- Feed {- config_path: String,- },-// Create an index of canonical urls.Canonical {config_path: String,@@ -308,10 +306,6 @@ fn main() -> Result<()> {wikipedia_dump_path,output_path,} => entrypoint::EntityIndexer::run(wikipedia_dump_path, output_path)?,- IndexingOptions::Feed { config_path } => {- let config = load_toml_config(config_path);- entrypoint::feed_indexer::build(config)?;- }IndexingOptions::MergeSearch { paths } => {let pointers = paths.into_iter()@@ -454,7 +448,15 @@ fn main() -> Result<()> {tokio::runtime::Builder::new_multi_thread().enable_all().build()?- .block_on(entrypoint::live_index::serve(config))?;+ .block_on(entrypoint::live_index::search_server::serve(config))?;+ }+ LiveIndex::Crawler { config_path } => {+ let config = load_toml_config(config_path);++ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?+ .block_on(entrypoint::live_index::crawler::run(config))?;}},Commands::WebSpell { config_path } => {diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex 5fcaa60f..77050f0e 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -41,7 +41,7 @@ use crate::ranking::{};use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};use crate::web_spell::SpellChecker;-use crate::webgraph::remote::RemoteWebgraph;+use crate::webgraph::remote::{RemoteWebgraph, WebgraphGranularity};use crate::webgraph::EdgeLimit;use crate::widgets::{Widget, Widgets};use crate::{@@ -177,7 +177,7 @@ pub trait Graph {) -> impl Future<Output = Vec<Vec<webgraph::Edge<()>>>>;}-impl Graph for RemoteWebgraph {+impl<G: WebgraphGranularity> Graph for RemoteWebgraph<G> {async fn batch_raw_ingoing(&self,nodes: &[webgraph::NodeID],@@ -570,13 +570,23 @@ wherelet liked: Vec<_> = host_rankings.liked.iter()- .map(|n| webgraph::Node::from(n.clone()).into_host().id())+ .filter_map(|n| {+ Url::parse(n)+ .or_else(|_| Url::parse(&format!("http://{}", n)))+ .ok()+ })+ .map(|n| webgraph::Node::from(n).into_host().id()).collect();let disliked: Vec<_> = host_rankings.disliked.iter()- .map(|n| webgraph::Node::from(n.clone()).into_host().id())+ .filter_map(|n| {+ Url::parse(n)+ .or_else(|_| Url::parse(&format!("http://{}", n)))+ .ok()+ })+ .map(|n| webgraph::Node::from(n).into_host().id()).collect();inbound_similarity::Scorer::new(webgraph, &liked, &disliked, false).await}diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rsindex 67d27342..dd927676 100644--- a/crates/core/src/searcher/distributed.rs+++ b/crates/core/src/searcher/distributed.rs@@ -17,7 +17,7 @@use crate::{distributed::{cluster::Cluster,- member::{Service, ShardId},+ member::{LiveIndexState, Service, ShardId},sonic::replication::{AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient,ReusableClientManager, ReusableShardedClient, Shard, ShardIdentifier, ShardedClient,@@ -27,6 +27,7 @@ use crate::{entity_index::EntityMatch,entrypoint::{entity_search_server,+ live_index::LiveIndexService,search_server::{self, SearchService},},image_store::Image,@@ -110,16 +111,14 @@ pub struct InitialSearchResultShard {pub shard: ShardId,}-struct SearchClientManager;--impl ReusableClientManager for SearchClientManager {+impl ReusableClientManager for SearchService {const CLIENT_REFRESH_INTERVAL: std::time::Duration = CLIENT_REFRESH_INTERVAL;type Service = SearchService;type ShardId = ShardId;- async fn new_client(&self, cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {+ async fn new_client(cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {let mut shards = HashMap::new();for member in cluster.members().await {if let Service::Searcher { host, shard } = member.service {@@ -140,15 +139,13 @@ impl ReusableClientManager for SearchClientManager {}}-struct EntitySearchClientManager;--impl ReusableClientManager for EntitySearchClientManager {+impl ReusableClientManager for entity_search_server::SearchService {const CLIENT_REFRESH_INTERVAL: std::time::Duration = CLIENT_REFRESH_INTERVAL;type Service = entity_search_server::SearchService;type ShardId = ();- async fn new_client(&self, cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {+ async fn new_client(cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {let mut replicas = Vec::new();for member in cluster.members().await {if let Service::EntitySearcher { host } = member.service {@@ -166,20 +163,46 @@ impl ReusableClientManager for EntitySearchClientManager {}}+impl ReusableClientManager for LiveIndexService {+ const CLIENT_REFRESH_INTERVAL: std::time::Duration = CLIENT_REFRESH_INTERVAL;++ type Service = LiveIndexService;++ type ShardId = ShardId;++ async fn new_client(cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {+ let mut shards = HashMap::new();+ for member in cluster.members().await {+ if let Service::LiveIndex { host, shard, state } = member.service {+ if state == LiveIndexState::Ready {+ shards.entry(shard).or_insert_with(Vec::new).push(host);+ }+ }+ }++ let mut shard_clients = Vec::new();++ for (id, replicas) in shards {+ let replicated =+ ReplicatedClient::new(replicas.into_iter().map(RemoteClient::new).collect());+ let shard = Shard::new(id, replicated);+ shard_clients.push(shard);+ }++ ShardedClient::new(shard_clients)+ }+}+pub struct DistributedSearcher {- client: Mutex<ReusableShardedClient<SearchClientManager>>,- entiy_client: Mutex<ReusableShardedClient<EntitySearchClientManager>>,+ client: Mutex<ReusableShardedClient<SearchService>>,+ entiy_client: Mutex<ReusableShardedClient<entity_search_server::SearchService>>,}impl DistributedSearcher {pub async fn new(cluster: Arc<Cluster>) -> Self {Self {- client: Mutex::new(- ReusableShardedClient::new(cluster.clone(), SearchClientManager).await,- ),- entiy_client: Mutex::new(- ReusableShardedClient::new(cluster, EntitySearchClientManager).await,- ),+ client: Mutex::new(ReusableShardedClient::new(cluster.clone()).await),+ entiy_client: Mutex::new(ReusableShardedClient::new(cluster).await),}}diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rsindex a8dbfd1c..e4baf94f 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local.rs@@ -343,6 +343,13 @@ wherepub fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {self.index.guard().inverted_index().top_key_phrases(top_n)}++ pub fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {+ self.index+ .guard()+ .inverted_index()+ .get_site_urls(site, offset, limit)+ }}#[cfg(test)]diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rsindex 4e24dc7b..f60322bc 100644--- a/crates/core/src/similar_hosts.rs+++ b/crates/core/src/similar_hosts.rs@@ -22,7 +22,10 @@ use url::Url;use crate::{ranking::{bitvec_similarity, inbound_similarity},- webgraph::{remote::RemoteWebgraph, EdgeLimit, Node, NodeID},+ webgraph::{+ remote::{RemoteWebgraph, WebgraphGranularity},+ EdgeLimit, Node, NodeID,+ },webpage::url_ext::UrlExt,SortableFloat,};@@ -33,13 +36,13 @@ pub struct ScoredNode {pub score: f64,}-pub struct SimilarHostsFinder {- webgraph: Arc<RemoteWebgraph>,+pub struct SimilarHostsFinder<G: WebgraphGranularity> {+ webgraph: Arc<RemoteWebgraph<G>>,max_similar_hosts: usize,}-impl SimilarHostsFinder {- pub fn new(webgraph: Arc<RemoteWebgraph>, max_similar_hosts: usize) -> Self {+impl<G: WebgraphGranularity> SimilarHostsFinder<G> {+ pub fn new(webgraph: Arc<RemoteWebgraph<G>>, max_similar_hosts: usize) -> Self {Self {webgraph,max_similar_hosts,@@ -57,7 +60,8 @@ impl SimilarHostsFinder {let nodes: Vec<_> = nodes.iter()- .map(|url| Node::from(url.to_string()).into_host())+ .filter_map(|url| Url::robust_parse(url).ok())+ .map(|url| Node::from(url).into_host()).collect();let domains = nodesdiff --git a/crates/core/src/sitemap.rs b/crates/core/src/sitemap.rsnew file mode 100644index 00000000..97ac4af2--- /dev/null+++ b/crates/core/src/sitemap.rs@@ -0,0 +1,213 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use chrono::{DateTime, Utc};+use quick_xml::events::Event;+use url::Url;++use crate::dated_url::DatedUrl;++#[derive(Debug, Clone, PartialEq, Eq)]+pub enum SitemapEntry {+ Url(DatedUrl),+ Sitemap(Url),+}++pub fn parse_sitemap(s: &str) -> Vec<SitemapEntry> {+ let mut reader = quick_xml::Reader::from_str(s);++ let mut res = vec![];++ let mut in_sitemap = false;+ let mut in_url = false;+ let mut in_loc = false;+ let mut in_lastmod = false;++ let mut current_url: Option<Url> = None;+ let mut current_lastmod: Option<DateTime<Utc>> = None;++ loop {+ match reader.read_event() {+ Ok(Event::Start(ref e)) => {+ if e.name().as_ref() == b"sitemap" {+ in_sitemap = true;+ } else if e.name().as_ref() == b"url" {+ in_url = true;+ } else if e.name().as_ref() == b"loc" {+ in_loc = true;+ } else if e.name().as_ref() == b"lastmod" {+ in_lastmod = true;+ }+ }+ Ok(Event::End(ref e)) => {+ if e.name().as_ref() == b"sitemap" {+ in_sitemap = false;+ } else if e.name().as_ref() == b"url" {+ in_url = false;+ if let Some(url) = current_url.take() {+ res.push(SitemapEntry::Url(DatedUrl {+ url,+ last_modified: current_lastmod.take(),+ }));+ }+ } else if e.name().as_ref() == b"loc" {+ in_loc = false;+ } else if e.name().as_ref() == b"lastmod" {+ in_lastmod = false;+ }+ }+ Ok(Event::Text(e)) => {+ if in_sitemap && in_loc {+ if let Ok(url) = Url::parse(&e.unescape().unwrap()) {+ res.push(SitemapEntry::Sitemap(url));+ }+ } else if in_url && in_loc {+ if let Ok(url) = Url::parse(&e.unescape().unwrap()) {+ current_url = Some(url);+ }+ } else if in_url && in_lastmod {+ if let Ok(date) = DateTime::parse_from_rfc3339(&e.unescape().unwrap()) {+ current_lastmod = Some(date.with_timezone(&Utc));+ }+ }+ }+ Ok(Event::Eof) => break,+ Err(e) => {+ tracing::debug!("failed to parse sitemap: {}", e);+ break;+ }+ _ => (),+ }+ }++ res+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn parse_sitemap() {+ let dr = r#"<sitemapindex>+ <sitemap>+ <loc>https://www.dr.dk/drtv/sitemap.xml</loc>+ </sitemap>+ <sitemap>+ <loc>https://www.dr.dk/sitemap.tvguide.xml</loc>+ </sitemap>+ <sitemap>+ <loc>+ https://www.dr.dk/sitemap.kommunalvalg.resultater.xml+ </loc>+ </sitemap>+ <sitemap>+ <loc>https://www.dr.dk/sitemap.folketingsvalg2022.xml</loc>+ </sitemap>+ </sitemapindex>"#;++ let entries = super::parse_sitemap(dr);+ assert_eq!(+ entries,+ vec![+ super::SitemapEntry::Sitemap("https://www.dr.dk/drtv/sitemap.xml".parse().unwrap()),+ super::SitemapEntry::Sitemap(+ "https://www.dr.dk/sitemap.tvguide.xml".parse().unwrap()+ ),+ super::SitemapEntry::Sitemap(+ "https://www.dr.dk/sitemap.kommunalvalg.resultater.xml"+ .parse()+ .unwrap()+ ),+ super::SitemapEntry::Sitemap(+ "https://www.dr.dk/sitemap.folketingsvalg2022.xml"+ .parse()+ .unwrap()+ ),+ ]+ );++ let dr = r#"<urlset>+ <url>+ <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>+ <loc>https://www.dr.dk/drtv/serie/sleepover_6382</loc>+ </url>+ <url>+ <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>+ <loc>https://www.dr.dk/drtv/saeson/sleepover_9673</loc>+ </url>+ <url>+ <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>+ <loc>+ https://www.dr.dk/drtv/episode/sleepover_-zoologisk-museum_52239+ </loc>+ </url>+ <url>+ <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>+ <loc>+ https://www.dr.dk/drtv/episode/sleepover_-koebenhavns-raadhus_52252+ </loc>+ </url>+ </urlset>"#;++ let entries = super::parse_sitemap(dr);+ assert_eq!(+ entries,+ vec![+ super::SitemapEntry::Url(DatedUrl {+ url: "https://www.dr.dk/drtv/serie/sleepover_6382"+ .parse()+ .unwrap(),+ last_modified: Some(+ "2023-10-18T05:40:04.7435930+00:00"+ .parse::<DateTime<Utc>>()+ .unwrap()+ ),+ }),+ super::SitemapEntry::Url(DatedUrl {+ url: "https://www.dr.dk/drtv/saeson/sleepover_9673"+ .parse()+ .unwrap(),+ last_modified: Some(+ "2023-10-18T05:40:04.7435930+00:00"+ .parse::<DateTime<Utc>>()+ .unwrap()+ ),+ }),+ super::SitemapEntry::Url(DatedUrl {+ url: "https://www.dr.dk/drtv/episode/sleepover_-zoologisk-museum_52239"+ .parse()+ .unwrap(),+ last_modified: Some(+ "2023-10-18T05:40:04.7435930+00:00"+ .parse::<DateTime<Utc>>()+ .unwrap()+ ),+ }),+ super::SitemapEntry::Url(DatedUrl {+ url: "https://www.dr.dk/drtv/episode/sleepover_-koebenhavns-raadhus_52252"+ .parse()+ .unwrap(),+ last_modified: Some(+ "2023-10-18T05:40:04.7435930+00:00"+ .parse::<DateTime<Utc>>()+ .unwrap()+ ),+ }),+ ]+ );+ }+}diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rsindex d435a27b..6eced6bd 100644--- a/crates/core/src/webgraph/centrality/harmonic.rs+++ b/crates/core/src/webgraph/centrality/harmonic.rs@@ -36,6 +36,7 @@ const HYPERLOGLOG_COUNTERS: usize = 64;pub static SKIPPED_REL: std::sync::LazyLock<RelFlags> = std::sync::LazyLock::new(|| {RelFlags::TAG| RelFlags::NOFOLLOW+ | RelFlags::SPONSORED| RelFlags::IS_IN_FOOTER| RelFlags::IS_IN_NAVIGATION| RelFlags::PRIVACY_POLICYdiff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rsindex 8e964bd3..d37ebee9 100644--- a/crates/core/src/webgraph/node.rs+++ b/crates/core/src/webgraph/node.rs@@ -153,6 +153,7 @@ impl Node {}}+#[cfg(test)]impl From<String> for Node {fn from(name: String) -> Self {let url = if name.contains("://") {@@ -172,6 +173,7 @@ impl From<&Url> for Node {}}+#[cfg(test)]impl From<&str> for Node {fn from(name: &str) -> Self {name.to_string().into()diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rsindex 9cbb407e..411529d0 100644--- a/crates/core/src/webgraph/remote.rs+++ b/crates/core/src/webgraph/remote.rs@@ -21,7 +21,7 @@ use tokio::sync::Mutex;use url::Url;use crate::{- config::WebgraphGranularity,+ config,distributed::{cluster::Cluster,member::{Service, ShardId},@@ -41,18 +41,40 @@ use crate::{use super::{Edge, EdgeLimit, FullEdge, Node, NodeID};-struct WebgraphClientManager {- granularity: WebgraphGranularity,+struct WebgraphClientManager<G: WebgraphGranularity>(std::marker::PhantomData<G>);++pub trait WebgraphGranularity: Clone {+ fn granularity() -> config::WebgraphGranularity;}-impl sonic::replication::ReusableClientManager for WebgraphClientManager {+#[derive(Clone)]+pub struct Page;++impl WebgraphGranularity for Page {+ fn granularity() -> config::WebgraphGranularity {+ config::WebgraphGranularity::Page+ }+}++#[derive(Clone)]+pub struct Host;++impl WebgraphGranularity for Host {+ fn granularity() -> config::WebgraphGranularity {+ config::WebgraphGranularity::Host+ }+}++impl<G> sonic::replication::ReusableClientManager for WebgraphClientManager<G>+where+ G: WebgraphGranularity,+{const CLIENT_REFRESH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);type Service = WebGraphService;type ShardId = ShardId;async fn new_client(- &self,cluster: &Cluster,) -> sonic::replication::ShardedClient<Self::Service, Self::ShardId> {let shards = cluster@@ -66,7 +88,7 @@ impl sonic::replication::ReusableClientManager for WebgraphClientManager {granularity,} = member.service{- if granularity == self.granularity {+ if granularity == G::granularity() {Some((shard, RemoteClient::<WebGraphService>::new(host)))} else {None@@ -90,16 +112,15 @@ impl sonic::replication::ReusableClientManager for WebgraphClientManager {}#[derive(Clone)]-pub struct RemoteWebgraph {- client: Arc<Mutex<sonic::replication::ReusableShardedClient<WebgraphClientManager>>>,+pub struct RemoteWebgraph<G: WebgraphGranularity> {+ client: Arc<Mutex<sonic::replication::ReusableShardedClient<WebgraphClientManager<G>>>>,}-impl RemoteWebgraph {- pub async fn new(cluster: Arc<Cluster>, granularity: WebgraphGranularity) -> Self {- let manager = WebgraphClientManager { granularity };-+impl<G: WebgraphGranularity> RemoteWebgraph<G> {+ pub async fn new(cluster: Arc<Cluster>) -> Self {#[cfg(feature = "dev")]{+ let granularity = G::granularity();tracing::info!("waiting for {granularity} webgraph to come online...");cluster.await_member(|member| {@@ -119,7 +140,7 @@ impl RemoteWebgraph {Self {client: Arc::new(Mutex::new(- sonic::replication::ReusableShardedClient::new(cluster, manager).await,+ sonic::replication::ReusableShardedClient::new(cluster).await,)),}}diff --git a/crates/core/src/webpage/html/feeds.rs b/crates/core/src/webpage/html/feeds.rsnew file mode 100644index 00000000..cac65215--- /dev/null+++ b/crates/core/src/webpage/html/feeds.rs@@ -0,0 +1,45 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::str::FromStr;++use url::Url;++use crate::feed::{Feed, FeedKind};+use crate::Result;++use super::Html;++impl Html {+ pub fn feeds(&self) -> Result<impl Iterator<Item = Feed>> {+ Ok(self.root.select("link")?.filter_map(|node| {+ let attributes = node.attributes.borrow();+ if let (Some(feed_kind), Some(Ok(feed_url))) = (+ attributes.get("type"),+ attributes.get("href").map(Url::parse),+ ) {+ if let Ok(feed_kind) = FeedKind::from_str(feed_kind) {+ return Some(Feed {+ url: feed_url,+ kind: feed_kind,+ });+ }+ }++ None+ }))+ }+}diff --git a/crates/core/src/webpage/html/links.rs b/crates/core/src/webpage/html/links.rsindex d889e30d..ffcced11 100644--- a/crates/core/src/webpage/html/links.rs+++ b/crates/core/src/webpage/html/links.rs@@ -322,12 +322,6 @@ impl Html {}while let Some((text, attributes)) = open_links.pop() {- if let Some(rel) = attributes.borrow().get("rel") {- if rel.contains("nofollow") || rel.contains("sponsored") || rel.contains("ugc") {- continue;- }- }-if let Some(dest) = attributes.borrow().get("href") {if dest.starts_with("mailto:") || dest.starts_with("tel:") {continue;diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rsindex dfde819c..c0eda2c1 100644--- a/crates/core/src/webpage/html/mod.rs+++ b/crates/core/src/webpage/html/mod.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -31,6 +31,7 @@ use super::url_ext::UrlExt;pub use fn_cache::FnCache;+mod feeds;mod fn_cache;mod into_tantivy;pub mod links;diff --git a/crates/core/src/webpage/url_ext.rs b/crates/core/src/webpage/url_ext.rsindex 84c3df9e..0f4a74ce 100644--- a/crates/core/src/webpage/url_ext.rs+++ b/crates/core/src/webpage/url_ext.rs@@ -33,6 +33,9 @@ pub trait UrlExt {fn parse_with_base_url(base_url: &url::Url, url: &str) -> Result<url::Url> {url::Url::parse(url).or_else(|_| base_url.join(url).map_err(|e| e.into()))}+ fn robust_parse(url: &str) -> Result<url::Url, url::ParseError> {+ url::Url::parse(url).or_else(|_| url::Url::parse(&format!("https://{}", url)))+ }fn icann_domain(&self) -> Option<&str>;fn root_domain(&self) -> Option<&str>;fn normalized_host(&self) -> Option<&str>;