Skip to content

web search done right

Live index crawler (#226)

Mikkel Denker5cd174e415f4
diff --git a/crates/core/examples/distributed_harmonic.rs b/crates/core/examples/distributed_harmonic.rs
index a2960914..28eb6d9a 100644
--- a/crates/core/examples/distributed_harmonic.rs
+++ b/crates/core/examples/distributed_harmonic.rs
@@ -19,7 +19,6 @@ fn start_dht_thread(id: u64, host: SocketAddr, gossip: SocketAddr) {
shard: ShardId::new(id),
seed_node: None,
gossip: Some(stract::config::GossipConfig {
- cluster_id: "test".to_string(),
seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),
addr: gossip,
}),
@@ -38,7 +37,6 @@ fn start_worker_thread(graph_path: String, shard: ShardId, host: SocketAddr, gos
std::thread::spawn(move || {
let config = stract::config::HarmonicWorkerConfig {
gossip: stract::config::GossipConfig {
- cluster_id: "test".to_string(),
seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),
addr: gossip,
},
@@ -158,7 +156,6 @@ fn main() -> anyhow::Result<()> {
let config = stract::config::HarmonicCoordinatorConfig {
gossip: stract::config::GossipConfig {
- cluster_id: "test".to_string(),
seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),
addr: "0.0.0.0:3007".parse().unwrap(),
},
diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rs
index 2c3081f1..c338cbe6 100644
--- a/crates/core/examples/search_preindexed.rs
+++ b/crates/core/examples/search_preindexed.rs
@@ -30,7 +30,6 @@ pub async fn main() {
dual_encoder_model_path: None,
bangs_path: Some("data/bangs.json".to_string()),
query_store_db_host: None,
- cluster_id: "api".to_string(),
gossip_seed_nodes: None,
gossip_addr: "0.0.0.0:8002".parse().unwrap(),
collector: collector_conf.clone(),
diff --git a/crates/core/public_suffix_list.dat b/crates/core/public_suffix_list.dat
index 021578e9..88dda820 100644
--- a/crates/core/public_suffix_list.dat
+++ b/crates/core/public_suffix_list.dat
@@ -13765,4 +13765,8 @@ enterprisecloud.nu
// Added by Stract
wordpress.com
+// tumblr
+// Added by Stract
+tumblr.com
+
// ===END PRIVATE DOMAINS===
diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rs
index f7d059ed..b01c04a3 100644
--- a/crates/core/src/api/mod.rs
+++ b/crates/core/src/api/mod.rs
@@ -33,7 +33,7 @@ use crate::{
ranking::models::lambdamart::LambdaMART,
searcher::{api::ApiSearcher, live::LiveSearcher, DistributedSearcher, SearchClient},
similar_hosts::SimilarHostsFinder,
- webgraph::remote::RemoteWebgraph,
+ webgraph::remote::{Host, Page, RemoteWebgraph},
};
use crate::ranking::models::cross_encoder::CrossEncoderModel;
@@ -72,14 +72,14 @@ pub struct Counters {
pub struct State {
pub config: ApiConfig,
- pub searcher: Arc<ApiSearcher<DistributedSearcher, LiveSearcher, Arc<RemoteWebgraph>>>,
- pub page_webgraph: Arc<RemoteWebgraph>,
- pub host_webgraph: Arc<RemoteWebgraph>,
+ pub searcher: Arc<ApiSearcher<DistributedSearcher, LiveSearcher, Arc<RemoteWebgraph<Host>>>>,
+ pub page_webgraph: Arc<RemoteWebgraph<Page>>,
+ pub host_webgraph: Arc<RemoteWebgraph<Host>>,
pub autosuggest: Autosuggest,
pub counters: Counters,
pub improvement_queue: Option<Arc<Mutex<LeakyQueue<ImprovementEvent>>>>,
pub _cluster: Arc<Cluster>,
- pub similar_hosts: SimilarHostsFinder,
+ pub similar_hosts: SimilarHostsFinder<Host>,
}
pub async fn favicon() -> impl IntoResponse {
@@ -172,10 +172,8 @@ pub async fn router(
None => Bangs::empty(),
};
- let host_webgraph =
- RemoteWebgraph::new(cluster.clone(), crate::config::WebgraphGranularity::Host).await;
- let page_webgraph =
- RemoteWebgraph::new(cluster.clone(), crate::config::WebgraphGranularity::Page).await;
+ let host_webgraph = RemoteWebgraph::<Host>::new(cluster.clone()).await;
+ let page_webgraph = RemoteWebgraph::<Page>::new(cluster.clone()).await;
let dist_searcher = DistributedSearcher::new(Arc::clone(&cluster)).await;
let live_searcher = LiveSearcher::new(Arc::clone(&cluster));
diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rs
index b6c03a4b..4cd6b521 100644
--- a/crates/core/src/api/webgraph.rs
+++ b/crates/core/src/api/webgraph.rs
@@ -156,6 +156,10 @@ pub mod host {
}
pub mod page {
+ use url::Url;
+
+ use crate::webpage::url_ext::UrlExt;
+
use super::*;
#[derive(serde::Deserialize, IntoParams)]
@@ -175,7 +179,8 @@ pub mod page {
extract::State(state): extract::State<Arc<State>>,
extract::Query(params): extract::Query<PageLinksParams>,
) -> std::result::Result<impl IntoResponse, StatusCode> {
- let node = Node::from(params.page);
+ let page = Url::robust_parse(&params.page).map_err(|_| StatusCode::BAD_REQUEST)?;
+ let node = Node::from(page);
let links = ingoing_links(state, node, WebgraphGranularity::Page)
.await
.map_err(|_| {
@@ -197,7 +202,8 @@ pub mod page {
extract::State(state): extract::State<Arc<State>>,
extract::Query(params): extract::Query<PageLinksParams>,
) -> std::result::Result<impl IntoResponse, StatusCode> {
- let node = Node::from(params.page);
+ let url = Url::robust_parse(&params.page).map_err(|_| StatusCode::BAD_REQUEST)?;
+ let node = Node::from(url);
let links = outgoing_links(state, node, WebgraphGranularity::Page)
.await
.map_err(|_| {
@@ -214,12 +220,20 @@ async fn ingoing_links(
node: Node,
level: WebgraphGranularity,
) -> anyhow::Result<Vec<FullEdge>> {
- let graph = match level {
- WebgraphGranularity::Host => &state.host_webgraph,
- WebgraphGranularity::Page => &state.page_webgraph,
- };
-
- graph.ingoing_edges(node, EdgeLimit::Limit(1024)).await
+ match level {
+ WebgraphGranularity::Host => {
+ state
+ .host_webgraph
+ .ingoing_edges(node, EdgeLimit::Limit(1024))
+ .await
+ }
+ WebgraphGranularity::Page => {
+ state
+ .page_webgraph
+ .ingoing_edges(node, EdgeLimit::Limit(1024))
+ .await
+ }
+ }
}
async fn outgoing_links(
@@ -227,12 +241,20 @@ async fn outgoing_links(
node: Node,
level: WebgraphGranularity,
) -> anyhow::Result<Vec<FullEdge>> {
- let graph = match level {
- WebgraphGranularity::Host => &state.host_webgraph,
- WebgraphGranularity::Page => &state.page_webgraph,
- };
-
- graph.outgoing_edges(node, EdgeLimit::Limit(1024)).await
+ match level {
+ WebgraphGranularity::Host => {
+ state
+ .host_webgraph
+ .outgoing_edges(node, EdgeLimit::Limit(1024))
+ .await
+ }
+ WebgraphGranularity::Page => {
+ state
+ .page_webgraph
+ .outgoing_edges(node, EdgeLimit::Limit(1024))
+ .await
+ }
+ }
}
#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema)]
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rs
index ab52f037..c6ef0910 100644
--- a/crates/core/src/config/defaults.rs
+++ b/crates/core/src/config/defaults.rs
@@ -14,6 +14,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::time::Duration;
+
pub struct Collector;
impl Collector {
@@ -211,3 +213,31 @@ impl Webgraph {
true
}
}
+
+pub struct LiveIndex;
+
+impl LiveIndex {
+ pub fn feeds_crawl_interval() -> Duration {
+ Duration::from_secs(30 * 60) // 30 minutes
+ }
+
+ pub fn sitemap_crawl_interval() -> Duration {
+ Duration::from_secs(24 * 60 * 60) // daily
+ }
+
+ pub fn frontpage_crawl_interval() -> Duration {
+ Duration::from_secs(12 * 60 * 60) // 12 hours
+ }
+
+ pub fn blogs_budget() -> u64 {
+ 300_000
+ }
+
+ pub fn news_budget() -> u64 {
+ 500_000
+ }
+
+ pub fn remaining_budget() -> u64 {
+ 200_000
+ }
+}
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 44532423..731288d5 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -23,6 +23,38 @@ use crate::distributed::member::ShardId;
use std::fs::File;
use std::io::{self, BufRead};
use std::net::SocketAddr;
+use std::path::PathBuf;
+use std::time::Duration;
+
+pub fn parse_duration<'de, D: serde::de::Deserializer<'de>>(
+ deserializer: D,
+) -> Result<Duration, D::Error> {
+ let err = <D::Error as serde::de::Error>::custom;
+ let s: String = serde::de::Deserialize::deserialize(deserializer)?;
+ let num_part = s.trim_end_matches(|c: char| !c.is_numeric());
+ let suffix = &s[num_part.len()..];
+ let num: u64 = num_part
+ .parse()
+ .map_err(|_| err("invalid number".to_string()))?;
+
+ let ret = match suffix.trim() {
+ "s" => Duration::from_secs(num),
+ "sec" => Duration::from_secs(num),
+ "secs" => Duration::from_secs(num),
+ "ms" => Duration::from_millis(num),
+ "millis" => Duration::from_millis(num),
+ "milliseconds" => Duration::from_millis(num),
+ "m" => Duration::from_secs(num * 60),
+ "mins" => Duration::from_secs(num * 60),
+ "minutes" => Duration::from_secs(num * 60),
+ "h" => Duration::from_secs(num * 60 * 60),
+ "hours" => Duration::from_secs(num * 60 * 60),
+ "d" => Duration::from_secs(num * 24 * 60 * 60),
+ "days" => Duration::from_secs(num * 24 * 60 * 60),
+ other => return Err(err(format!("invalid suffix {other}"))),
+ };
+ Ok(ret)
+}
#[derive(Debug, serde::Deserialize, Clone)]
pub struct IndexerConfig {
@@ -230,7 +262,6 @@ pub struct ApiConfig {
pub dual_encoder_model_path: Option<String>,
pub bangs_path: Option<String>,
pub query_store_db_host: Option<String>,
- pub cluster_id: String,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
@@ -299,7 +330,6 @@ impl Default for SnippetConfig {
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct SearchServerConfig {
- pub cluster_id: String,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
pub shard: ShardId,
@@ -317,7 +347,6 @@ pub struct SearchServerConfig {
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct EntitySearchServerConfig {
- pub cluster_id: String,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
pub index_path: String,
@@ -367,6 +396,47 @@ pub struct CrawlerConfig {
pub router_hosts: Vec<String>,
}
+#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
+pub struct DailyLiveIndexCrawlerBudget {
+ #[serde(default = "defaults::LiveIndex::blogs_budget")]
+ pub blogs: u64,
+ #[serde(default = "defaults::LiveIndex::news_budget")]
+ pub news: u64,
+ #[serde(default = "defaults::LiveIndex::remaining_budget")]
+ pub remaining: u64,
+}
+
+#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
+pub struct CheckIntervals {
+ #[serde(
+ deserialize_with = "parse_duration",
+ default = "defaults::LiveIndex::feeds_crawl_interval"
+ )]
+ pub feeds: Duration,
+ #[serde(
+ deserialize_with = "parse_duration",
+ default = "defaults::LiveIndex::sitemap_crawl_interval"
+ )]
+ pub sitemap: Duration,
+ #[serde(
+ deserialize_with = "parse_duration",
+ default = "defaults::LiveIndex::frontpage_crawl_interval"
+ )]
+ pub frontpage: Duration,
+}
+
+#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
+pub struct LiveCrawlerConfig {
+ pub crawled_db_path: PathBuf,
+ pub gossip: GossipConfig,
+ pub site_stats_path: PathBuf,
+ pub host_centrality_path: PathBuf,
+ pub user_agent: UserAgent,
+ pub num_worker_threads: usize,
+ pub check_intervals: CheckIntervals,
+ pub daily_budget: DailyLiveIndexCrawlerBudget,
+}
+
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct CrawlRouterConfig {
pub host: SocketAddr,
@@ -415,7 +485,6 @@ pub struct WebgraphServerConfig {
pub graph_path: String,
pub granularity: WebgraphGranularity,
- pub cluster_id: String,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
}
@@ -472,34 +541,11 @@ pub struct CrawlPlannerConfig {
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct LiveIndexConfig {
- // crawler
- pub user_agent: UserAgent,
- #[serde(default = "defaults::Crawler::robots_txt_cache_sec")]
- pub robots_txt_cache_sec: u64,
- #[serde(default = "defaults::Crawler::min_politeness_factor")]
- pub min_politeness_factor: u32,
- #[serde(default = "defaults::Crawler::start_politeness_factor")]
- pub start_politeness_factor: u32,
- #[serde(default = "defaults::Crawler::min_crawl_delay_ms")]
- pub min_crawl_delay_ms: u64,
- #[serde(default = "defaults::Crawler::max_crawl_delay_ms")]
- pub max_crawl_delay_ms: u64,
- #[serde(default = "defaults::Crawler::max_politeness_factor")]
- pub max_politeness_factor: u32,
- #[serde(default = "defaults::Crawler::max_url_slowdown_retry")]
- pub max_url_slowdown_retry: u8,
- #[serde(default = "defaults::Crawler::timeout_seconds")]
- pub timeout_seconds: u64,
-
- // indexer
pub host_centrality_store_path: String,
pub page_centrality_store_path: Option<String>,
pub safety_classifier_path: Option<String>,
pub host_centrality_threshold: Option<f64>,
pub minimum_clean_words: Option<usize>,
-
- // search
- pub cluster_id: String,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
pub shard_id: ShardId,
@@ -513,22 +559,6 @@ pub struct LiveIndexConfig {
pub snippet: SnippetConfig,
}
-#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
-pub struct LiveIndexSchedulerConfig {
- pub schedule_path: String,
- pub feed_index_path: String,
- pub host_centrality_store_path: String,
- pub host_graph_path: String,
- pub num_splits: u64,
-}
-#[derive(Debug, serde::Deserialize, Clone)]
-pub struct FeedIndexingConfig {
- pub output_path: String,
- pub warc_source: WarcSource,
- pub limit_warc_files: Option<usize>,
- pub skip_warc_files: Option<usize>,
-}
-
#[derive(Debug, serde::Deserialize, Clone)]
pub struct WebSpellConfig {
pub output_path: String,
@@ -577,7 +607,6 @@ impl Default for CorrectionConfig {
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub struct GossipConfig {
- pub cluster_id: String,
pub seed_nodes: Option<Vec<SocketAddr>>,
pub addr: SocketAddr,
}
diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rs
index 2aee88f3..531f8409 100644
--- a/crates/core/src/crawler/mod.rs
+++ b/crates/core/src/crawler/mod.rs
@@ -58,6 +58,9 @@ pub enum Error {
#[error("content too large")]
ContentTooLarge,
+ #[error("couldn't read response body")]
+ ResponseBodyReadFailed,
+
#[error("invalid politeness factor")]
InvalidPolitenessFactor,
@@ -338,7 +341,7 @@ pub async fn encoded_body(res: reqwest::Response) -> Result<String> {
let mut stream = res.bytes_stream();
while let Some(b) = stream.next().await {
if b.is_err() {
- return Err(Error::ContentTooLarge);
+ return Err(Error::ResponseBodyReadFailed);
}
let b = b.unwrap();
diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rs
index f1ab3029..4a10bbec 100644
--- a/crates/core/src/crawler/planner.rs
+++ b/crates/core/src/crawler/planner.rs
@@ -29,11 +29,10 @@ use std::{
};
use url::Url;
-use crate::config::WebgraphGranularity;
use crate::crawler::WeightedUrl;
use crate::distributed::cluster::Cluster;
use crate::external_sort::ExternalSorter;
-use crate::webgraph::remote::RemoteWebgraph;
+use crate::webgraph::remote::{Host, Page, RemoteWebgraph};
use crate::webgraph::Node;
use crate::webpage::url_ext::UrlExt;
use crate::SortableFloat;
@@ -148,8 +147,8 @@ struct Budget {
pub struct CrawlPlanner {
host_centrality: speedy_kv::Db<NodeID, f64>,
page_centrality: speedy_kv::Db<NodeID, f64>,
- page_graph: RemoteWebgraph,
- host_graph: RemoteWebgraph,
+ page_graph: RemoteWebgraph<Page>,
+ host_graph: RemoteWebgraph<Host>,
config: CrawlPlannerConfig,
excluded_domains: HashSet<Domain>,
@@ -166,8 +165,8 @@ impl CrawlPlanner {
) -> Result<Self> {
Self::check_config(&config)?;
- let page_graph = RemoteWebgraph::new(cluster.clone(), WebgraphGranularity::Page).await;
- let host_graph = RemoteWebgraph::new(cluster, WebgraphGranularity::Host).await;
+ let page_graph = RemoteWebgraph::new(cluster.clone()).await;
+ let host_graph = RemoteWebgraph::new(cluster.clone()).await;
let domain_boosts = config
.domain_boosts
diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rs
index d62b5b03..88033c11 100644
--- a/crates/core/src/crawler/robots_txt.rs
+++ b/crates/core/src/crawler/robots_txt.rs
@@ -24,6 +24,7 @@ use super::{encoded_body, Result, Site};
const RETRY_ROBOTSTXT_UNREACHABLE: bool = false;
+#[derive(Debug)]
enum Lookup<T> {
Found(T),
/// 404
@@ -113,12 +114,12 @@ impl RobotsTxtManager {
async fn fetch_robots_txt_without_retry(&self, site: &Site) -> Lookup<RobotsTxt> {
match self
- .fetch_robots_txt_from_url(&format!("http://{}/robots.txt", site.0))
+ .fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0))
.await
{
- Lookup::Unavailable => {
+ Lookup::Unavailable | Lookup::Unreachable => {
match self
- .fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0))
+ .fetch_robots_txt_from_url(&format!("http://{}/robots.txt", site.0))
.await
{
Lookup::Found(robots_txt) => Lookup::Found(robots_txt),
@@ -136,7 +137,7 @@ impl RobotsTxtManager {
Lookup::Unavailable => Lookup::Unavailable,
}
}
- res => res,
+ Lookup::Found(robots_txt) => Lookup::Found(robots_txt),
}
}
@@ -202,6 +203,7 @@ impl RobotsTxtManager {
}
}
+#[derive(Debug)]
struct RobotsTxt {
download_time: std::time::Instant,
robots: robotstxt::Robots,
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rs
index 9ee134e6..f245377d 100644
--- a/crates/core/src/crawler/worker.rs
+++ b/crates/core/src/crawler/worker.rs
@@ -15,7 +15,6 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::anyhow;
use hashbrown::HashSet;
-use quick_xml::events::Event;
use rand::seq::SliceRandom;
use std::{
@@ -30,8 +29,10 @@ use url::Url;
use crate::{
config::CrawlerConfig,
crawler::MAX_URL_LEN_BYTES,
+ dated_url::DatedUrl,
distributed::{retry_strategy::ExponentialBackoff, sonic},
entrypoint::crawler::router::{NewJob, RouterService},
+ sitemap::{parse_sitemap, SitemapEntry},
warc,
webpage::{url_ext::UrlExt, Html},
};
@@ -195,7 +196,10 @@ impl<S: DatumStream> JobExecutor<S> {
}
self.scheduled_urls().await;
- self.crawl_sitemaps().await;
+
+ if self.wandered_urls < self.job.wandering_urls {
+ self.crawl_sitemaps().await;
+ }
while self.wandered_urls < self.job.wandering_urls
&& self.wander_prioritiser.known_urls() > 0
@@ -288,16 +292,22 @@ impl<S: DatumStream> JobExecutor<S> {
let sitemaps = self.robotstxt.sitemaps(url).await;
for sitemap in sitemaps {
- self.sitemap_urls
- .extend(self.urls_from_sitemap(sitemap, 5).await);
+ self.sitemap_urls.extend(
+ self.urls_from_sitemap(sitemap, 5)
+ .await
+ .into_iter()
+ .map(|dated_url| dated_url.url),
+ );
}
}
}
}
- async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>) {
+ pub async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>) {
+ tracing::debug!("processing {} urls", urls.len());
while let Some(retryable_url) = urls.pop_front() {
if let UrlVisit::Skip = self.verify_url(&retryable_url).await {
+ tracing::debug!("skipping url: {}", retryable_url.url());
continue;
}
@@ -420,7 +430,7 @@ impl<S: DatumStream> JobExecutor<S> {
}
async fn process_url(&mut self, url: Url) -> Result<ProcessedUrl> {
- let datum = self.crawl_url(url.clone()).await?;
+ let datum = self.polite_crawl_url(url.clone()).await?;
self.save_datum(datum.clone()).await;
match Html::parse(&datum.body, datum.url.as_str()) {
@@ -527,7 +537,6 @@ impl<S: DatumStream> JobExecutor<S> {
res: &reqwest::Response,
url: &Url,
payload_type: warc::PayloadType,
- fetch_time: Duration,
) -> Result<Option<CrawlDatum>> {
let status_code = res.status().as_u16();
@@ -547,38 +556,27 @@ impl<S: DatumStream> JobExecutor<S> {
url,
payload_type,
body: String::new(),
- fetch_time_ms: fetch_time.as_millis() as u64,
+ fetch_time_ms: 0,
}))
} else {
Ok(None)
}
}
- async fn crawl_url(&mut self, url: Url) -> Result<CrawlDatum> {
- let mut url = url;
- url.normalize_in_place();
-
- if self.crawled_urls.contains(&url) {
- return Err(Error::from(anyhow!("url already crawled: {}", url)));
- }
-
- let start = Instant::now();
- let res = self.fetch_with_https_priority(url.clone()).await;
- let fetch_time = start.elapsed();
- self.politeness_delay(fetch_time).await;
-
- // we want to delay before returning the error
- let res = res?;
+ async fn unpolite_crawl_url(&mut self, url: Url) -> Result<CrawlDatum> {
+ tracing::debug!("crawling url: {}", url);
- self.crawled_urls.insert(url.clone());
- let payload_type = self.check_headers(&res)?;
-
- if let Some(datum) = self.redirect_datum(&res, &url, payload_type, fetch_time)? {
- return Ok(datum);
- }
+ let res = self.fetch_with_https_priority(url.clone()).await?;
+ let payload_type = self.check_headers(&res);
let status_code = res.status();
+ let mut res_url = res.url().clone();
+ if let Ok(payload_type) = payload_type {
+ if let Ok(Some(datum)) = self.redirect_datum(&res, &url, payload_type) {
+ return Ok(datum);
+ }
+ }
if status_code != reqwest::StatusCode::OK {
return Err(Error::FetchFailed {
status_code,
@@ -586,22 +584,40 @@ impl<S: DatumStream> JobExecutor<S> {
});
}
- let mut res_url = res.url().clone();
+ let body = encoded_body(res).await;
+
+ self.crawled_urls.insert(url.clone());
+
res_url.normalize_in_place();
self.crawled_urls.insert(res_url.clone());
- let body = encoded_body(res).await?;
-
Ok(CrawlDatum {
url: res_url,
- body,
- payload_type,
- fetch_time_ms: fetch_time.as_millis() as u64,
+ body: body?,
+ payload_type: payload_type?,
+ fetch_time_ms: 0,
})
}
- async fn urls_from_sitemap(&self, sitemap: Url, max_depth: usize) -> Vec<Url> {
+ async fn polite_crawl_url(&mut self, url: Url) -> Result<CrawlDatum> {
+ let mut url = url;
+ url.normalize_in_place();
+
+ if self.crawled_urls.contains(&url) {
+ return Err(Error::from(anyhow!("url already crawled: {}", url)));
+ }
+
+ let start = Instant::now();
+ let res = self.unpolite_crawl_url(url).await;
+ let fetch_time = start.elapsed();
+ self.politeness_delay(fetch_time).await;
+ let mut datum = res?;
+ datum.fetch_time_ms = fetch_time.as_millis() as u64;
+ Ok(datum)
+ }
+
+ async fn urls_from_sitemap(&self, sitemap: Url, max_depth: usize) -> Vec<DatedUrl> {
let mut stack = vec![(sitemap, 0)];
let mut urls = vec![];
@@ -648,155 +664,3 @@ impl<S: DatumStream> JobExecutor<S> {
urls
}
}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-enum SitemapEntry {
- Url(Url),
- Sitemap(Url),
-}
-
-fn parse_sitemap(s: &str) -> Vec<SitemapEntry> {
- let mut reader = quick_xml::Reader::from_str(s);
-
- let mut res = vec![];
-
- let mut in_sitemap = false;
- let mut in_url = false;
- let mut in_loc = false;
-
- loop {
- match reader.read_event() {
- Ok(Event::Start(ref e)) => {
- if e.name().as_ref() == b"sitemap" {
- in_sitemap = true;
- } else if e.name().as_ref() == b"url" {
- in_url = true;
- } else if e.name().as_ref() == b"loc" {
- in_loc = true;
- }
- }
- Ok(Event::End(ref e)) => {
- if e.name().as_ref() == b"sitemap" {
- in_sitemap = false;
- } else if e.name().as_ref() == b"url" {
- in_url = false;
- } else if e.name().as_ref() == b"loc" {
- in_loc = false;
- }
- }
- Ok(Event::Text(e)) => {
- if in_sitemap && in_loc {
- if let Ok(url) = Url::parse(&e.unescape().unwrap()) {
- res.push(SitemapEntry::Sitemap(url));
- }
- } else if in_url && in_loc {
- if let Ok(url) = Url::parse(&e.unescape().unwrap()) {
- res.push(SitemapEntry::Url(url));
- }
- }
- }
- Ok(Event::Eof) => break,
- Err(e) => {
- tracing::debug!("failed to parse sitemap: {}", e);
- break;
- }
- _ => (),
- }
- }
-
- res
-}
-
-#[cfg(test)]
-mod tests {
- #[test]
- fn parse_sitemap() {
- let dr = r#"<sitemapindex>
- <sitemap>
- <loc>https://www.dr.dk/drtv/sitemap.xml</loc>
- </sitemap>
- <sitemap>
- <loc>https://www.dr.dk/sitemap.tvguide.xml</loc>
- </sitemap>
- <sitemap>
- <loc>
- https://www.dr.dk/sitemap.kommunalvalg.resultater.xml
- </loc>
- </sitemap>
- <sitemap>
- <loc>https://www.dr.dk/sitemap.folketingsvalg2022.xml</loc>
- </sitemap>
- </sitemapindex>"#;
-
- let entries = super::parse_sitemap(dr);
- assert_eq!(
- entries,
- vec![
- super::SitemapEntry::Sitemap("https://www.dr.dk/drtv/sitemap.xml".parse().unwrap()),
- super::SitemapEntry::Sitemap(
- "https://www.dr.dk/sitemap.tvguide.xml".parse().unwrap()
- ),
- super::SitemapEntry::Sitemap(
- "https://www.dr.dk/sitemap.kommunalvalg.resultater.xml"
- .parse()
- .unwrap()
- ),
- super::SitemapEntry::Sitemap(
- "https://www.dr.dk/sitemap.folketingsvalg2022.xml"
- .parse()
- .unwrap()
- ),
- ]
- );
-
- let dr = r#"<urlset>
- <url>
- <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>
- <loc>https://www.dr.dk/drtv/serie/sleepover_6382</loc>
- </url>
- <url>
- <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>
- <loc>https://www.dr.dk/drtv/saeson/sleepover_9673</loc>
- </url>
- <url>
- <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>
- <loc>
- https://www.dr.dk/drtv/episode/sleepover_-zoologisk-museum_52239
- </loc>
- </url>
- <url>
- <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>
- <loc>
- https://www.dr.dk/drtv/episode/sleepover_-koebenhavns-raadhus_52252
- </loc>
- </url>
- </urlset>"#;
-
- let entries = super::parse_sitemap(dr);
- assert_eq!(
- entries,
- vec![
- super::SitemapEntry::Url(
- "https://www.dr.dk/drtv/serie/sleepover_6382"
- .parse()
- .unwrap()
- ),
- super::SitemapEntry::Url(
- "https://www.dr.dk/drtv/saeson/sleepover_9673"
- .parse()
- .unwrap()
- ),
- super::SitemapEntry::Url(
- "https://www.dr.dk/drtv/episode/sleepover_-zoologisk-museum_52239"
- .parse()
- .unwrap()
- ),
- super::SitemapEntry::Url(
- "https://www.dr.dk/drtv/episode/sleepover_-koebenhavns-raadhus_52252"
- .parse()
- .unwrap()
- ),
- ]
- );
- }
-}
diff --git a/crates/core/src/dated_url.rs b/crates/core/src/dated_url.rs
new file mode 100644
index 00000000..09b68136
--- /dev/null
+++ b/crates/core/src/dated_url.rs
@@ -0,0 +1,36 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use chrono::{DateTime, Utc};
+use url::Url;
+
+#[derive(
+ Debug,
+ Clone,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ PartialEq,
+ Eq,
+ Hash,
+)]
+pub struct DatedUrl {
+ #[bincode(with_serde)]
+ pub url: Url,
+ #[bincode(with_serde)]
+ pub last_modified: Option<DateTime<Utc>>,
+}
diff --git a/crates/core/src/distributed/cluster.rs b/crates/core/src/distributed/cluster.rs
index 1cf1f1cb..f31dc39d 100644
--- a/crates/core/src/distributed/cluster.rs
+++ b/crates/core/src/distributed/cluster.rs
@@ -92,7 +92,6 @@ impl Cluster {
}
pub async fn join_as_spectator(
- cluster_id: String,
gossip_addr: SocketAddr,
seed_addrs: Vec<SocketAddr>,
) -> Result<Self> {
@@ -104,7 +103,7 @@ impl Cluster {
let uuid = uuid::Uuid::new_v4().to_string();
let node_id = NodeId {
- id: format!("{}_{}", cluster_id, uuid),
+ id: format!("{}_{}", CLUSTER_ID, uuid),
gossip_public_address: gossip_addr,
};
let config = ChitchatConfig {
diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rs
index 2aac7228..512bdedc 100644
--- a/crates/core/src/distributed/member.rs
+++ b/crates/core/src/distributed/member.rs
@@ -179,3 +179,10 @@ pub struct Member {
pub id: String,
pub service: Service,
}
+
+impl Member {
+ pub fn new(service: Service) -> Self {
+ let id = uuid::Uuid::new_v4().to_string();
+ Self { id, service }
+ }
+}
diff --git a/crates/core/src/distributed/sonic/replication.rs b/crates/core/src/distributed/sonic/replication.rs
index 7727a6a0..80e344e1 100644
--- a/crates/core/src/distributed/sonic/replication.rs
+++ b/crates/core/src/distributed/sonic/replication.rs
@@ -341,6 +341,10 @@ where
pub fn new(id: Id, replicas: ReplicatedClient<S>) -> Self {
Self { replicas, id }
}
+
+ pub fn id(&self) -> &Id {
+ &self.id
+ }
}
pub struct ShardedClient<S: sonic::service::Service, Id: ShardIdentifier> {
@@ -360,6 +364,10 @@ where
self.shards.is_empty()
}
+ pub fn shards(&self) -> &[Shard<S, Id>] {
+ &self.shards
+ }
+
async fn send_single<Req, Sel>(
&self,
req: Req,
@@ -397,6 +405,10 @@ where
futures.push(self.send_single(req.clone(), shard, replica_selector, timeout));
}
+ if futures.is_empty() {
+ return Err(anyhow::anyhow!("no shards available").into());
+ }
+
let mut results = Vec::new();
for r in join_all(futures).await {
match r {
@@ -478,7 +490,6 @@ pub trait ReusableClientManager {
type ShardId: ShardIdentifier;
fn new_client(
- &self,
cluster: &Cluster,
) -> impl std::future::Future<Output = ShardedClient<Self::Service, Self::ShardId>>;
}
@@ -490,29 +501,27 @@ where
cluster: Arc<Cluster>,
client: Arc<sonic::replication::ShardedClient<M::Service, M::ShardId>>,
last_client_update: std::time::Instant,
- manager: M,
}
impl<M> ReusableShardedClient<M>
where
M: ReusableClientManager,
{
- pub async fn new(cluster: Arc<Cluster>, manager: M) -> Self {
- let client = Arc::new(manager.new_client(&cluster).await);
+ pub async fn new(cluster: Arc<Cluster>) -> Self {
+ let client = Arc::new(M::new_client(&cluster).await);
let last_client_update = std::time::Instant::now();
Self {
cluster,
client,
last_client_update,
- manager,
}
}
pub async fn conn(&mut self) -> Arc<sonic::replication::ShardedClient<M::Service, M::ShardId>> {
if self.client.is_empty() || self.last_client_update.elapsed() > M::CLIENT_REFRESH_INTERVAL
{
- self.client = Arc::new(self.manager.new_client(&self.cluster).await);
+ self.client = Arc::new(M::new_client(&self.cluster).await);
self.last_client_update = std::time::Instant::now();
}
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
index a5d41d38..50b9b614 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
@@ -162,10 +162,7 @@ struct ClusterInfo {
async fn setup_gossip(config: ApproxHarmonicCoordinatorConfig) -> Result<ClusterInfo> {
let handle = Cluster::join(
- Member {
- id: config.gossip.cluster_id,
- service: Service::ApproxHarmonicCoordinator { host: config.host },
- },
+ Member::new(Service::ApproxHarmonicCoordinator { host: config.host }),
config.gossip.addr,
config.gossip.seed_nodes.unwrap_or_default(),
)
diff --git a/crates/core/src/entrypoint/ampc/dht.rs b/crates/core/src/entrypoint/ampc/dht.rs
index 74afb2c7..9b359508 100644
--- a/crates/core/src/entrypoint/ampc/dht.rs
+++ b/crates/core/src/entrypoint/ampc/dht.rs
@@ -113,13 +113,10 @@ pub async fn run<C: Into<Config>>(config: C) -> Result<()> {
let _cluster_handle = match config.gossip {
Some(gossip) => Some(
Cluster::join(
- Member {
- id: gossip.cluster_id,
- service: Service::Dht {
- host: config.host,
- shard: config.shard,
- },
- },
+ Member::new(Service::Dht {
+ host: config.host,
+ shard: config.shard,
+ }),
gossip.addr,
gossip.seed_nodes.unwrap_or_default(),
)
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
index 84822151..cf28c839 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
@@ -144,10 +144,7 @@ struct ClusterInfo {
async fn setup_gossip(config: HarmonicCoordinatorConfig) -> Result<ClusterInfo> {
let handle = Cluster::join(
- Member {
- id: config.gossip.cluster_id,
- service: Service::HarmonicCoordinator { host: config.host },
- },
+ Member::new(Service::HarmonicCoordinator { host: config.host }),
config.gossip.addr,
config.gossip.seed_nodes.unwrap_or_default(),
)
diff --git a/crates/core/src/entrypoint/api.rs b/crates/core/src/entrypoint/api.rs
index 5f4025af..3efe16a3 100644
--- a/crates/core/src/entrypoint/api.rs
+++ b/crates/core/src/entrypoint/api.rs
@@ -90,10 +90,7 @@ fn counters(registry: &mut crate::metrics::PrometheusRegistry) -> Result<Counter
async fn cluster(config: &config::ApiConfig) -> Result<Cluster> {
Cluster::join(
- Member {
- id: config.cluster_id.clone(),
- service: Service::Api { host: config.host },
- },
+ Member::new(Service::Api { host: config.host }),
config.gossip_addr,
config.gossip_seed_nodes.clone().unwrap_or_default(),
)
diff --git a/crates/core/src/entrypoint/crawler.rs b/crates/core/src/entrypoint/crawler.rs
index c6aa3136..62951fea 100644
--- a/crates/core/src/entrypoint/crawler.rs
+++ b/crates/core/src/entrypoint/crawler.rs
@@ -68,7 +68,6 @@ pub async fn planner(config: config::CrawlPlannerConfig) -> Result<()> {
let gossip = config.gossip.clone();
let cluster = Arc::new(
crate::distributed::cluster::Cluster::join_as_spectator(
- gossip.cluster_id,
gossip.addr,
gossip.seed_nodes.unwrap_or_default(),
)
diff --git a/crates/core/src/entrypoint/entity_search_server.rs b/crates/core/src/entrypoint/entity_search_server.rs
index fabb815a..68def5f9 100644
--- a/crates/core/src/entrypoint/entity_search_server.rs
+++ b/crates/core/src/entrypoint/entity_search_server.rs
@@ -40,10 +40,7 @@ impl SearchService {
let index = EntityIndex::open(config.index_path)?;
let cluster_handle = Cluster::join(
- Member {
- id: config.cluster_id,
- service: Service::EntitySearcher { host: config.host },
- },
+ Member::new(Service::EntitySearcher { host: config.host }),
config.gossip_addr,
config.gossip_seed_nodes.unwrap_or_default(),
)
diff --git a/crates/core/src/entrypoint/feed_indexer.rs b/crates/core/src/entrypoint/feed_indexer.rs
deleted file mode 100644
index 01f38444..00000000
--- a/crates/core/src/entrypoint/feed_indexer.rs
+++ /dev/null
@@ -1,176 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::{
- path::{Path, PathBuf},
- thread,
-};
-
-use anyhow::Result;
-use itertools::Itertools;
-use rayon::prelude::*;
-
-use tracing::info;
-use url::Url;
-
-use crate::{
- config::{FeedIndexingConfig, WarcSource},
- feed::{index::FeedIndex, Feed, FeedKind},
- warc::{PayloadType, WarcFile},
-};
-
-#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
-pub struct Job {
- pub source_config: WarcSource,
- pub warc_path: String,
- pub base_path: String,
-}
-
-#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
-pub struct IndexPointer(PathBuf);
-
-impl From<String> for IndexPointer {
- fn from(path: String) -> Self {
- IndexPointer(Path::new(&path).to_path_buf())
- }
-}
-
-pub struct IndexingWorker;
-impl IndexingWorker {
- fn process_job(&self, job: &Job) -> FeedIndex {
- let name = job.warc_path.split('/').last().unwrap().to_string();
- info!("processing {}", name);
-
- let mut index = FeedIndex::open(Path::new(&job.base_path).join(&name)).unwrap();
-
- if let Ok(file) = WarcFile::download(&job.source_config, &job.warc_path) {
- for record in
- file.records()
- .flatten()
- .filter(|record| match &record.response.payload_type {
- Some(payload_type) => {
- matches!(payload_type, PayloadType::Rss | PayloadType::Atom)
- }
- None => false,
- })
- {
- let kind = match &record.response.payload_type {
- Some(payload_type) => match payload_type {
- PayloadType::Rss => FeedKind::Rss,
- PayloadType::Atom => FeedKind::Atom,
- _ => unreachable!(),
- },
- None => unreachable!(),
- };
-
- let url = Url::parse(&record.request.url);
-
- if url.is_err() {
- continue;
- }
-
- let feed = Feed {
- url: url.unwrap(),
- kind,
- };
-
- index.insert(&feed).unwrap();
- }
-
- index.commit().unwrap();
- index.merge_into_max_segments(1).unwrap();
- }
-
- info!("{} done", name);
-
- index
- }
-}
-
-pub fn build(config: FeedIndexingConfig) -> Result<()> {
- let warc_paths = config.warc_source.paths()?;
-
- let job_config: WarcSource = config.warc_source.clone();
-
- let worker = IndexingWorker;
-
- let indexes = warc_paths
- .into_par_iter()
- .skip(config.skip_warc_files.unwrap_or(0))
- .take(config.limit_warc_files.unwrap_or(usize::MAX))
- .map(|warc_path| Job {
- source_config: job_config.clone(),
- warc_path,
- base_path: config.output_path.clone(),
- })
- .map(|job| IndexPointer(worker.process_job(&job).path))
- .collect();
-
- merge(indexes)?;
- Ok(())
-}
-
-pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {
- let num_indexes = indexes.len();
- let mut it = indexes.into_iter();
- let num_cores = usize::from(std::thread::available_parallelism()?);
-
- let mut threads = Vec::new();
-
- for _ in 0..(num_cores + 1) {
- let indexes = it
- .by_ref()
- .take(((num_indexes as f64) / (num_cores as f64)).ceil() as usize)
- .collect_vec();
-
- if indexes.is_empty() {
- break;
- }
-
- threads.push(thread::spawn(move || {
- let mut it = indexes.into_iter();
- let mut index = FeedIndex::open(it.next().unwrap().0).unwrap();
-
- for other in it {
- let other_path = other.0;
- let other = FeedIndex::open(&other_path).unwrap();
- index = index.merge(other);
-
- std::fs::remove_dir_all(other_path).unwrap();
- }
-
- index.merge_into_max_segments(1).unwrap();
-
- index
- }));
- }
-
- let mut indexes = Vec::new();
- for thread in threads {
- indexes.push(thread.join().unwrap());
- }
-
- let mut it = indexes.into_iter();
- let mut index = it.next().unwrap();
-
- for other in it {
- let other_path = other.path.clone();
- index = index.merge(other);
- std::fs::remove_dir_all(other_path).unwrap();
- }
-
- Ok(())
-}
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rs
index 0ba35cb8..979b9218 100644
--- a/crates/core/src/entrypoint/indexer/worker.rs
+++ b/crates/core/src/entrypoint/indexer/worker.rs
@@ -24,10 +24,10 @@ use tracing::debug;
pub use super::indexable_webpage::IndexableWebpage;
pub use super::job::{Job, JobSettings};
use crate::backlink_grouper::BacklinkGrouper;
-use crate::config::{GossipConfig, IndexerConfig, IndexerDualEncoderConfig, WebgraphGranularity};
+use crate::config::{GossipConfig, IndexerConfig, IndexerDualEncoderConfig};
use crate::distributed::cluster::Cluster;
use crate::models::dual_encoder::DualEncoder as DualEncoderModel;
-use crate::webgraph::remote::RemoteWebgraph;
+use crate::webgraph::remote::{Page, RemoteWebgraph};
use crate::Result;
use crate::index::Index;
@@ -81,7 +81,7 @@ struct DualEncoder {
}
pub(super) enum Webgraph {
- Remote(RemoteWebgraph),
+ Remote(RemoteWebgraph<Page>),
Local(webgraph::Webgraph),
}
@@ -152,10 +152,10 @@ impl Webgraph {
),
IndexerGraphConfig::Remote { gossip } => {
let cluster = crate::start_gossip_cluster_thread(gossip.clone(), None);
- Self::Remote(RemoteWebgraph::new(cluster, WebgraphGranularity::Page).await)
+ Self::Remote(RemoteWebgraph::new(cluster).await)
}
IndexerGraphConfig::Existing { cluster } => {
- Self::Remote(RemoteWebgraph::new(cluster.clone(), WebgraphGranularity::Page).await)
+ Self::Remote(RemoteWebgraph::new(cluster.clone()).await)
}
}
}
diff --git a/crates/core/src/entrypoint/live_index/crawler.rs b/crates/core/src/entrypoint/live_index/crawler.rs
new file mode 100644
index 00000000..d03cc016
--- /dev/null
+++ b/crates/core/src/entrypoint/live_index/crawler.rs
@@ -0,0 +1,27 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::config;
+use crate::live_index;
+use crate::Result;
+
+pub async fn run(config: config::LiveCrawlerConfig) -> Result<()> {
+ let crawler = live_index::Crawler::new(config).await?;
+
+ crawler.run().await?;
+
+ Ok(())
+}
diff --git a/crates/core/src/entrypoint/live_index/mod.rs b/crates/core/src/entrypoint/live_index/mod.rs
index d782a04f..b9b0ad01 100644
--- a/crates/core/src/entrypoint/live_index/mod.rs
+++ b/crates/core/src/entrypoint/live_index/mod.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -14,423 +14,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+pub mod crawler;
+pub mod search_server;
+
#[cfg(test)]
mod tests;
-use std::{
- net::SocketAddr,
- path::{Path, PathBuf},
- sync::Arc,
- time::Duration,
-};
-
-use crate::{
- ampc::dht::ShardId,
- config::LiveIndexConfig,
- distributed::{
- cluster::Cluster,
- member::{LiveIndexState, Member, Service},
- remote_cp,
- sonic::{self, service::sonic_service},
- },
- inverted_index,
- live_index::{IndexManager, LiveIndex},
- searcher::{InitialWebsiteResult, LocalSearcher},
-};
-use anyhow::{Context, Result};
-use futures::stream::FuturesUnordered;
-use itertools::Itertools;
-use simple_wal::Wal;
-use tokio::sync::Mutex;
-use tokio_stream::StreamExt;
-use tracing::info;
-
-use super::{
- indexer::{self, IndexableWebpage},
- search_server::{RetrieveWebsites, Search},
-};
-
-const INDEXING_TIMEOUT: Duration = Duration::from_secs(60);
-const INDEXING_RETRIES: usize = 3;
-
-sonic_service!(
- LiveIndexService,
- [
- RetrieveWebsites,
- Search,
- IndexWebpages,
- GetIndexPath,
- RemoteDownload
- ]
-);
-
-fn start_manager(index: Arc<LiveIndex>) {
- let manager = IndexManager::new(index);
- std::thread::spawn(|| manager.run());
-}
-
-struct FileDownloadStepper {
- conn: Mutex<sonic::service::Connection<LiveIndexService>>,
-}
-
-impl remote_cp::Stepper for FileDownloadStepper {
- async fn step(&self, req: remote_cp::Request) -> remote_cp::Response {
- self.conn
- .lock()
- .await
- .send(RemoteDownload { req })
- .await
- .unwrap()
- }
-}
-
-async fn other_replicas(cluster: &Cluster, shard: &ShardId, id: &str) -> Vec<SocketAddr> {
- tokio::time::sleep(std::time::Duration::from_secs(10)).await;
- cluster
- .members()
- .await
- .into_iter()
- .filter_map(|member| {
- if member.id != id {
- if let Service::LiveIndex {
- host,
- shard: member_shard,
- state,
- } = member.service
- {
- if &member_shard == shard && matches!(state, LiveIndexState::Ready) {
- return Some(host);
- }
- }
- }
-
- None
- })
- .collect()
-}
-
-async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal) -> Result<()> {
- let self_node = cluster
- .self_node()
- .expect("cluster should not be joined as spectator");
-
- let Service::LiveIndex {
- host,
- shard,
- state: _,
- } = self_node.service.clone()
- else {
- panic!("self node should be a live index")
- };
-
- let mut others = other_replicas(&cluster, &shard, &self_node.id).await;
-
- if let Some(other) = others.pop() {
- let mut conn: sonic::service::Connection<LiveIndexService> =
- sonic::service::Connection::create(other).await?;
- index.delete_all_pages();
- let local_path = index.path();
- let remote_path = conn.send(GetIndexPath).await?;
-
- remote_cp::Request::download(
- remote_path,
- local_path,
- &FileDownloadStepper {
- conn: Mutex::new(conn),
- },
- )
- .await;
- }
-
- let mut wal = temp_wal
- .lock()
- .await
- .take()
- .expect("temp_wal should always exist before setup has been run");
-
- for pages in wal.iter().unwrap().chunks(512).into_iter() {
- let pages: Vec<_> = pages.into_iter().collect();
- index.insert(&pages);
- }
-
- wal.clear().unwrap();
-
- start_manager(index);
-
- cluster
- .set_service(Service::LiveIndex {
- host,
- shard,
- state: LiveIndexState::Ready,
- })
- .await?;
-
- Ok(())
-}
-
-type TempWal = Arc<Mutex<Option<Wal<IndexableWebpage>>>>;
-
-pub struct LiveIndexService {
- local_searcher: LocalSearcher<Arc<LiveIndex>>,
- temp_wal: TempWal,
- index: Arc<LiveIndex>,
- cluster_handle: Arc<Cluster>,
-}
-
-impl LiveIndexService {
- async fn new(config: LiveIndexConfig) -> Result<Self> {
- let cluster_handle = Arc::new(
- Cluster::join(
- Member {
- id: config.cluster_id.clone(),
- service: Service::LiveIndex {
- host: config.host,
- shard: config.shard_id,
- state: crate::distributed::member::LiveIndexState::InSetup,
- },
- },
- config.gossip_addr,
- config.gossip_seed_nodes.clone().unwrap_or_default(),
- )
- .await?,
- );
- let index_path = Path::new(&config.index_path);
-
- let index = Arc::new(
- LiveIndex::new(
- index_path.join("index"),
- indexer::worker::Config {
- host_centrality_store_path: config.host_centrality_store_path.clone(),
- page_centrality_store_path: config.page_centrality_store_path.clone(),
- page_webgraph: Some(indexer::worker::IndexerGraphConfig::Existing {
- cluster: cluster_handle.clone(),
- }),
- safety_classifier_path: config.safety_classifier_path.clone(),
- dual_encoder: None,
- },
- )
- .await?,
- );
- let local_searcher = LocalSearcher::new(index.clone());
-
- let temp_wal = Arc::new(Mutex::new(Some(Wal::open(index_path.join("wal.temp"))?)));
-
- Ok(Self {
- local_searcher,
- cluster_handle,
- index,
- temp_wal,
- })
- }
-
- fn background_setup(&self) {
- let index = self.index.clone();
- let temp_wal = self.temp_wal.clone();
- let cluster = self.cluster_handle.clone();
-
- tokio::task::spawn(async move { setup(index, cluster, temp_wal).await.unwrap() });
- }
-
- async fn index_webpages_in_replicas(
- &self,
- webpages: &[IndexableWebpage],
- consistency_fraction: f64,
- ) -> Result<(), IndexingError> {
- let self_member = self
- .cluster_handle
- .self_node()
- .expect("node should be participating part of cluster");
-
- let self_id = self_member.id.clone();
- let Service::LiveIndex {
- host: _,
- shard: self_shard,
- state: _,
- } = self_member.service
- else {
- panic!("self_member should always be a live index")
- };
-
- let live_indexes: Vec<_> = self
- .cluster_handle
- .members()
- .await
- .into_iter()
- .filter_map(|member| {
- if let Service::LiveIndex { host, shard, state } = member.service {
- if member.id != self_id && shard == self_shard {
- Some(RemoteIndex { host, state })
- } else {
- None
- }
- } else {
- None
- }
- })
- .collect();
-
- let ready = live_indexes
- .iter()
- .filter(|index| matches!(index.state, LiveIndexState::Ready))
- .count();
-
- let mut futures = FuturesUnordered::new();
-
- for index in live_indexes {
- let pages = webpages.to_vec();
-
- futures.push(tokio::spawn(async move {
- match index.index_webpages_without_consistency(&pages).await {
- Ok(_) => Ok(index),
- Err(e) => Err(e),
- }
- }));
- }
-
- let mut missing_responses =
- (((ready as f64) * consistency_fraction).ceil() as u64).min(ready as u64);
-
- while let Some(Ok(index)) = futures.next().await {
- if let Ok(index) = index {
- if matches!(index.state, LiveIndexState::Ready) {
- missing_responses = missing_responses.saturating_sub(1);
- }
-
- if missing_responses == 0 {
- break;
- }
- }
- }
-
- if missing_responses > 0 {
- return Err(IndexingError::InsufficientReplication);
- }
-
- Ok(())
- }
-}
-
-impl sonic::service::Message<LiveIndexService> for RetrieveWebsites {
- type Response = Option<Vec<inverted_index::RetrievedWebpage>>;
- async fn handle(self, server: &LiveIndexService) -> Self::Response {
- server
- .local_searcher
- .retrieve_websites(&self.websites, &self.query)
- .ok()
- }
-}
-
-impl sonic::service::Message<LiveIndexService> for Search {
- type Response = Option<InitialWebsiteResult>;
- async fn handle(self, server: &LiveIndexService) -> Self::Response {
- server.local_searcher.search_initial(&self.query, true).ok()
- }
-}
-
-#[derive(Clone, Debug)]
-struct RemoteIndex {
- host: SocketAddr,
- state: LiveIndexState,
-}
-
-impl RemoteIndex {
- async fn index_webpages_without_consistency(
- &self,
- webpages: &[IndexableWebpage],
- ) -> Result<()> {
- let req = IndexWebpages {
- pages: webpages.to_vec(),
- consistency_fraction: None,
- };
-
- for _ in 0..INDEXING_RETRIES {
- let mut conn: sonic::service::Connection<LiveIndexService> =
- sonic::service::Connection::create(self.host)
- .await
- .with_context(|| format!("failed to connect to {}", self.host))?;
-
- if conn
- .send_with_timeout(req.clone(), INDEXING_TIMEOUT)
- .await
- .is_ok()
- {
- return Ok(());
- }
- }
-
- Err(anyhow::anyhow!("failed to replicate webpages"))
- }
-}
-
-#[derive(thiserror::Error, Debug, bincode::Encode, bincode::Decode)]
-pub enum IndexingError {
- #[error("failed to replicate to the necessary quorom")]
- InsufficientReplication,
-}
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct IndexWebpages {
- pub pages: Vec<IndexableWebpage>,
- pub consistency_fraction: Option<f64>,
-}
-
-impl sonic::service::Message<LiveIndexService> for IndexWebpages {
- type Response = Result<(), IndexingError>;
-
- async fn handle(self, server: &LiveIndexService) -> Self::Response {
- if let Some(wal) = server.temp_wal.lock().await.as_mut() {
- wal.batch_write(self.pages.iter()).unwrap();
- } else {
- server.index.insert(&self.pages);
-
- if let Some(consistency_fraction) = self.consistency_fraction {
- server
- .index_webpages_in_replicas(&self.pages, consistency_fraction)
- .await?;
- }
- }
-
- Ok(())
- }
-}
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct GetIndexPath;
-
-impl sonic::service::Message<LiveIndexService> for GetIndexPath {
- type Response = PathBuf;
-
- async fn handle(self, server: &LiveIndexService) -> Self::Response {
- server.index.path()
- }
-}
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct RemoteDownload {
- req: remote_cp::Request,
-}
-impl sonic::service::Message<LiveIndexService> for RemoteDownload {
- type Response = remote_cp::Response;
-
- async fn handle(self, _: &LiveIndexService) -> Self::Response {
- remote_cp::Response::handle(self.req).unwrap()
- }
-}
-
-pub async fn serve(config: LiveIndexConfig) -> Result<()> {
- let addr = config.host;
-
- let service = LiveIndexService::new(config).await?;
-
- service.background_setup();
-
- let server = service.bind(&addr).await.unwrap();
-
- info!("live index is ready to accept requests on {}", addr);
-
- loop {
- if let Err(e) = server.accept().await {
- tracing::error!("{:?}", e);
- }
- }
-}
+pub use self::search_server::GetIndexPath;
+pub use self::search_server::IndexWebpages;
+pub use self::search_server::LiveIndexService;
+pub use self::search_server::RemoteDownload;
diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rs
new file mode 100644
index 00000000..3474bc53
--- /dev/null
+++ b/crates/core/src/entrypoint/live_index/search_server.rs
@@ -0,0 +1,450 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{
+ net::SocketAddr,
+ path::{Path, PathBuf},
+ sync::Arc,
+ time::Duration,
+};
+
+use crate::{
+ ampc::dht::ShardId,
+ config::LiveIndexConfig,
+ distributed::{
+ cluster::Cluster,
+ member::{LiveIndexState, Member, Service},
+ remote_cp,
+ sonic::{self, service::sonic_service},
+ },
+ inverted_index,
+ live_index::{IndexManager, LiveIndex},
+ searcher::{InitialWebsiteResult, LocalSearcher},
+};
+use anyhow::{Context, Result};
+use futures::stream::FuturesUnordered;
+use itertools::Itertools;
+use simple_wal::Wal;
+use tokio::sync::Mutex;
+use tokio_stream::StreamExt;
+use tracing::info;
+
+use crate::entrypoint::{
+ indexer::{self, IndexableWebpage},
+ search_server::{GetSiteUrls, RetrieveWebsites, Search, SiteUrls},
+};
+
+const INDEXING_TIMEOUT: Duration = Duration::from_secs(60);
+const INDEXING_RETRIES: usize = 3;
+
+sonic_service!(
+ LiveIndexService,
+ [
+ RetrieveWebsites,
+ Search,
+ GetSiteUrls,
+ IndexWebpages,
+ GetIndexPath,
+ RemoteDownload
+ ]
+);
+
+fn start_manager(index: Arc<LiveIndex>) {
+ let manager = IndexManager::new(index);
+ std::thread::spawn(|| manager.run());
+}
+
+struct FileDownloadStepper {
+ conn: Mutex<sonic::service::Connection<LiveIndexService>>,
+}
+
+impl remote_cp::Stepper for FileDownloadStepper {
+ async fn step(&self, req: remote_cp::Request) -> remote_cp::Response {
+ self.conn
+ .lock()
+ .await
+ .send(RemoteDownload { req })
+ .await
+ .unwrap()
+ }
+}
+
+async fn other_replicas(cluster: &Cluster, shard: &ShardId, id: &str) -> Vec<SocketAddr> {
+ tokio::time::sleep(std::time::Duration::from_secs(10)).await;
+ cluster
+ .members()
+ .await
+ .into_iter()
+ .filter_map(|member| {
+ if member.id != id {
+ if let Service::LiveIndex {
+ host,
+ shard: member_shard,
+ state,
+ } = member.service
+ {
+ if &member_shard == shard && matches!(state, LiveIndexState::Ready) {
+ return Some(host);
+ }
+ }
+ }
+
+ None
+ })
+ .collect()
+}
+
+async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal) -> Result<()> {
+ let self_node = cluster
+ .self_node()
+ .expect("cluster should not be joined as spectator");
+
+ let Service::LiveIndex {
+ host,
+ shard,
+ state: _,
+ } = self_node.service.clone()
+ else {
+ panic!("self node should be a live index")
+ };
+
+ let mut others = other_replicas(&cluster, &shard, &self_node.id).await;
+
+ if let Some(other) = others.pop() {
+ let mut conn: sonic::service::Connection<LiveIndexService> =
+ sonic::service::Connection::create(other).await?;
+ index.delete_all_pages();
+ let local_path = index.path();
+ let remote_path = conn.send(GetIndexPath).await?;
+
+ remote_cp::Request::download(
+ remote_path,
+ local_path,
+ &FileDownloadStepper {
+ conn: Mutex::new(conn),
+ },
+ )
+ .await;
+ }
+
+ let mut wal = temp_wal
+ .lock()
+ .await
+ .take()
+ .expect("temp_wal should always exist before setup has been run");
+
+ for pages in wal.iter().unwrap().chunks(512).into_iter() {
+ let pages: Vec<_> = pages.into_iter().collect();
+ index.insert(&pages);
+ }
+
+ wal.clear().unwrap();
+
+ start_manager(index);
+
+ cluster
+ .set_service(Service::LiveIndex {
+ host,
+ shard,
+ state: LiveIndexState::Ready,
+ })
+ .await?;
+
+ Ok(())
+}
+
+type TempWal = Arc<Mutex<Option<Wal<IndexableWebpage>>>>;
+
+pub struct LiveIndexService {
+ local_searcher: LocalSearcher<Arc<LiveIndex>>,
+ temp_wal: TempWal,
+ index: Arc<LiveIndex>,
+ cluster_handle: Arc<Cluster>,
+}
+
+impl LiveIndexService {
+ pub async fn new(config: LiveIndexConfig) -> Result<Self> {
+ let cluster_handle = Arc::new(
+ Cluster::join(
+ Member::new(Service::LiveIndex {
+ host: config.host,
+ shard: config.shard_id,
+ state: crate::distributed::member::LiveIndexState::InSetup,
+ }),
+ config.gossip_addr,
+ config.gossip_seed_nodes.clone().unwrap_or_default(),
+ )
+ .await?,
+ );
+ let index_path = Path::new(&config.index_path);
+
+ let index = Arc::new(
+ LiveIndex::new(
+ index_path.join("index"),
+ indexer::worker::Config {
+ host_centrality_store_path: config.host_centrality_store_path.clone(),
+ page_centrality_store_path: config.page_centrality_store_path.clone(),
+ page_webgraph: Some(indexer::worker::IndexerGraphConfig::Existing {
+ cluster: cluster_handle.clone(),
+ }),
+ safety_classifier_path: config.safety_classifier_path.clone(),
+ dual_encoder: None,
+ },
+ )
+ .await?,
+ );
+ let local_searcher = LocalSearcher::new(index.clone());
+
+ let temp_wal = Arc::new(Mutex::new(Some(Wal::open(index_path.join("wal.temp"))?)));
+
+ Ok(Self {
+ local_searcher,
+ cluster_handle,
+ index,
+ temp_wal,
+ })
+ }
+
+ pub fn cluster_handle(&self) -> Arc<Cluster> {
+ self.cluster_handle.clone()
+ }
+
+ pub fn index(&self) -> Arc<LiveIndex> {
+ self.index.clone()
+ }
+
+ pub fn background_setup(&self) {
+ let index = self.index.clone();
+ let temp_wal = self.temp_wal.clone();
+ let cluster = self.cluster_handle.clone();
+
+ tokio::task::spawn(async move { setup(index, cluster, temp_wal).await.unwrap() });
+ }
+
+ async fn index_webpages_in_replicas(
+ &self,
+ webpages: &[IndexableWebpage],
+ consistency_fraction: f64,
+ ) -> Result<(), IndexingError> {
+ let self_member = self
+ .cluster_handle
+ .self_node()
+ .expect("node should be participating part of cluster");
+
+ let self_id = self_member.id.clone();
+ let Service::LiveIndex {
+ host: _,
+ shard: self_shard,
+ state: _,
+ } = self_member.service
+ else {
+ panic!("self_member should always be a live index")
+ };
+
+ let live_indexes: Vec<_> = self
+ .cluster_handle
+ .members()
+ .await
+ .into_iter()
+ .filter_map(|member| {
+ if let Service::LiveIndex { host, shard, state } = member.service {
+ if member.id != self_id && shard == self_shard {
+ Some(RemoteIndex { host, state })
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ let ready = live_indexes
+ .iter()
+ .filter(|index| matches!(index.state, LiveIndexState::Ready))
+ .count();
+
+ let mut futures = FuturesUnordered::new();
+
+ for index in live_indexes {
+ let pages = webpages.to_vec();
+
+ futures.push(tokio::spawn(async move {
+ match index.index_webpages_without_consistency(&pages).await {
+ Ok(_) => Ok(index),
+ Err(e) => Err(e),
+ }
+ }));
+ }
+
+ let mut missing_responses =
+ (((ready as f64) * consistency_fraction).ceil() as u64).min(ready as u64);
+
+ while let Some(Ok(index)) = futures.next().await {
+ if let Ok(index) = index {
+ if matches!(index.state, LiveIndexState::Ready) {
+ missing_responses = missing_responses.saturating_sub(1);
+ }
+
+ if missing_responses == 0 {
+ break;
+ }
+ }
+ }
+
+ if missing_responses > 0 {
+ return Err(IndexingError::InsufficientReplication);
+ }
+
+ Ok(())
+ }
+}
+
+impl sonic::service::Message<LiveIndexService> for RetrieveWebsites {
+ type Response = Option<Vec<inverted_index::RetrievedWebpage>>;
+ async fn handle(self, server: &LiveIndexService) -> Self::Response {
+ server
+ .local_searcher
+ .retrieve_websites(&self.websites, &self.query)
+ .ok()
+ }
+}
+
+impl sonic::service::Message<LiveIndexService> for Search {
+ type Response = Option<InitialWebsiteResult>;
+ async fn handle(self, server: &LiveIndexService) -> Self::Response {
+ server.local_searcher.search_initial(&self.query, true).ok()
+ }
+}
+
+impl sonic::service::Message<LiveIndexService> for GetSiteUrls {
+ type Response = SiteUrls;
+ async fn handle(self, server: &LiveIndexService) -> Self::Response {
+ let urls = server
+ .local_searcher
+ .get_site_urls(&self.site, self.offset, self.limit);
+
+ SiteUrls { urls }
+ }
+}
+
+#[derive(Clone, Debug)]
+struct RemoteIndex {
+ host: SocketAddr,
+ state: LiveIndexState,
+}
+
+impl RemoteIndex {
+ async fn index_webpages_without_consistency(
+ &self,
+ webpages: &[IndexableWebpage],
+ ) -> Result<()> {
+ let req = IndexWebpages {
+ pages: webpages.to_vec(),
+ consistency_fraction: None,
+ };
+
+ for _ in 0..INDEXING_RETRIES {
+ let mut conn: sonic::service::Connection<LiveIndexService> =
+ sonic::service::Connection::create(self.host)
+ .await
+ .with_context(|| format!("failed to connect to {}", self.host))?;
+
+ if conn
+ .send_with_timeout(req.clone(), INDEXING_TIMEOUT)
+ .await
+ .is_ok()
+ {
+ return Ok(());
+ }
+ }
+
+ Err(anyhow::anyhow!("failed to replicate webpages"))
+ }
+}
+
+#[derive(thiserror::Error, Debug, bincode::Encode, bincode::Decode)]
+pub enum IndexingError {
+ #[error("failed to replicate to the necessary quorom")]
+ InsufficientReplication,
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct IndexWebpages {
+ pub pages: Vec<IndexableWebpage>,
+ pub consistency_fraction: Option<f64>,
+}
+
+impl sonic::service::Message<LiveIndexService> for IndexWebpages {
+ type Response = Result<(), IndexingError>;
+
+ async fn handle(self, server: &LiveIndexService) -> Self::Response {
+ if let Some(wal) = server.temp_wal.lock().await.as_mut() {
+ wal.batch_write(self.pages.iter()).unwrap();
+ } else {
+ server.index.insert(&self.pages);
+
+ if let Some(consistency_fraction) = self.consistency_fraction {
+ server
+ .index_webpages_in_replicas(&self.pages, consistency_fraction)
+ .await?;
+ }
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct GetIndexPath;
+
+impl sonic::service::Message<LiveIndexService> for GetIndexPath {
+ type Response = PathBuf;
+
+ async fn handle(self, server: &LiveIndexService) -> Self::Response {
+ server.index.path()
+ }
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct RemoteDownload {
+ req: remote_cp::Request,
+}
+impl sonic::service::Message<LiveIndexService> for RemoteDownload {
+ type Response = remote_cp::Response;
+
+ async fn handle(self, _: &LiveIndexService) -> Self::Response {
+ remote_cp::Response::handle(self.req).unwrap()
+ }
+}
+
+pub async fn serve(config: LiveIndexConfig) -> Result<()> {
+ let addr = config.host;
+
+ let service = LiveIndexService::new(config).await?;
+
+ service.background_setup();
+
+ let server = service.bind(&addr).await.unwrap();
+
+ info!("live index is ready to accept requests on {}", addr);
+
+ loop {
+ if let Err(e) = server.accept().await {
+ tracing::error!("{:?}", e);
+ }
+ }
+}
diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rs
index 048f32f0..3d376769 100644
--- a/crates/core/src/entrypoint/live_index/tests.rs
+++ b/crates/core/src/entrypoint/live_index/tests.rs
@@ -57,18 +57,6 @@ impl RemoteIndex {
};
let config = LiveIndexConfig {
- user_agent: crate::config::UserAgent {
- full: "TestBot".to_string(),
- token: "TestBot".to_string(),
- },
- robots_txt_cache_sec: 60 * 60,
- min_politeness_factor: 1,
- start_politeness_factor: 3,
- min_crawl_delay_ms: 5_000,
- max_crawl_delay_ms: 60_000,
- max_politeness_factor: 2048,
- max_url_slowdown_retry: 5,
- timeout_seconds: 30,
host_centrality_store_path: path
.as_path()
.join("host_centrality")
@@ -79,7 +67,6 @@ impl RemoteIndex {
safety_classifier_path: None,
host_centrality_threshold: None,
minimum_clean_words: None,
- cluster_id: "test-cluster".to_string(),
gossip_seed_nodes: gossip_seed,
gossip_addr,
shard_id: shard,
@@ -92,8 +79,8 @@ impl RemoteIndex {
};
let service = LiveIndexService::new(config).await?;
- let cluster = service.cluster_handle.clone();
- let index = service.index.clone();
+ let cluster = service.cluster_handle();
+ let index = service.index();
service.background_setup();
@@ -186,19 +173,12 @@ impl RemoteIndex {
}
}
-const CLUSTER_ID: &str = "test-cluster";
-
#[tokio::test]
async fn test_shard_without_replica() -> Result<()> {
let shard1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;
let shard2 = RemoteIndex::start(ShardId::new(2), vec![shard1.gossip_addr]).await?;
- let cluster = Cluster::join_as_spectator(
- CLUSTER_ID.to_string(),
- free_socket_addr(),
- vec![shard1.gossip_addr],
- )
- .await?;
+ let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![shard1.gossip_addr]).await?;
shard1.await_ready(&cluster).await;
shard2.await_ready(&cluster).await;
@@ -252,12 +232,7 @@ async fn test_replica_no_fails() -> Result<()> {
let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;
let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;
- let cluster = Cluster::join_as_spectator(
- CLUSTER_ID.to_string(),
- free_socket_addr(),
- vec![rep1.gossip_addr],
- )
- .await?;
+ let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![rep1.gossip_addr]).await?;
rep1.await_ready(&cluster).await;
rep2.await_ready(&cluster).await;
@@ -306,12 +281,7 @@ async fn test_replica_no_fails() -> Result<()> {
async fn test_replica_setup_after_inserts() -> Result<()> {
let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;
- let cluster = Cluster::join_as_spectator(
- CLUSTER_ID.to_string(),
- free_socket_addr(),
- vec![rep1.gossip_addr],
- )
- .await?;
+ let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![rep1.gossip_addr]).await?;
rep1.await_ready(&cluster).await;
@@ -364,12 +334,7 @@ async fn test_replica_recovery() -> Result<()> {
let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;
let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;
- let cluster = Cluster::join_as_spectator(
- CLUSTER_ID.to_string(),
- free_socket_addr(),
- vec![rep1.gossip_addr],
- )
- .await?;
+ let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![rep1.gossip_addr]).await?;
rep1.await_ready(&cluster).await;
rep2.await_ready(&cluster).await;
diff --git a/crates/core/src/entrypoint/mod.rs b/crates/core/src/entrypoint/mod.rs
index 63e3e075..fb2ab188 100644
--- a/crates/core/src/entrypoint/mod.rs
+++ b/crates/core/src/entrypoint/mod.rs
@@ -24,7 +24,6 @@ pub mod configure;
pub mod crawler;
mod entity;
pub mod entity_search_server;
-pub mod feed_indexer;
pub mod indexer;
pub mod safety_classifier;
pub mod search_server;
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rs
index f4a2ebbf..f7e92786 100644
--- a/crates/core/src/entrypoint/search_server.rs
+++ b/crates/core/src/entrypoint/search_server.rs
@@ -45,6 +45,7 @@ sonic_service!(
GetHomepageDescriptions,
TopKeyPhrases,
Size,
+ GetSiteUrls,
]
);
@@ -73,13 +74,10 @@ impl SearchService {
local_searcher.set_snippet_config(config.snippet);
let cluster_handle = Cluster::join(
- Member {
- id: config.cluster_id,
- service: Service::Searcher {
- host: config.host,
- shard: config.shard,
- },
- },
+ Member::new(Service::Searcher {
+ host: config.host,
+ shard: config.shard,
+ }),
config.gossip_addr,
config.gossip_seed_nodes.unwrap_or_default(),
)
@@ -188,3 +186,27 @@ impl sonic::service::Message<SearchService> for Size {
}
}
}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct GetSiteUrls {
+ pub site: String,
+ pub offset: usize,
+ pub limit: usize,
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct SiteUrls {
+ #[bincode(with_serde)]
+ pub urls: Vec<Url>,
+}
+
+impl sonic::service::Message<SearchService> for GetSiteUrls {
+ type Response = SiteUrls;
+ async fn handle(self, server: &SearchService) -> Self::Response {
+ let urls = server
+ .local_searcher
+ .get_site_urls(&self.site, self.offset, self.limit);
+
+ SiteUrls { urls }
+ }
+}
diff --git a/crates/core/src/entrypoint/site_stats.rs b/crates/core/src/entrypoint/site_stats.rs
index c06d373b..730600a6 100644
--- a/crates/core/src/entrypoint/site_stats.rs
+++ b/crates/core/src/entrypoint/site_stats.rs
@@ -21,41 +21,128 @@ use std::{
};
use bloom::BytesBloomFilter;
+use url::Url;
use crate::{
config::{self, SiteStatsConfig},
entrypoint::download_all_warc_files,
+ feed::Feed,
webgraph::Node,
webpage::{url_ext::UrlExt, Html},
Result,
};
+const TOP_FEEDS_PER_SITE: Option<usize> = Some(10);
+const MIN_FEED_COUNT: u64 = 1;
+
#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Default)]
pub struct SiteStats {
pages: u64,
blogposts: u64,
news_articles: u64,
+ feeds: HashMap<Feed, u64>,
}
impl AddAssign<SiteStats> for SiteStats {
fn add_assign(&mut self, rhs: SiteStats) {
self.pages += rhs.pages;
self.blogposts += rhs.blogposts;
- self.news_articles += rhs.news_articles
+ self.news_articles += rhs.news_articles;
+
+ for (feed, count) in rhs.feeds {
+ *self.feeds.entry(feed).or_default() += count;
+ }
+ }
+}
+
+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
+pub struct FinalStats {
+ pub pages: u64,
+ pub blogposts: u64,
+ pub news_articles: u64,
+ pub feeds: Vec<FeedCount>,
+}
+
+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
+pub struct FeedCount {
+ feed: Feed,
+ count: u64,
+}
+
+impl From<FeedCount> for Feed {
+ fn from(feed_count: FeedCount) -> Self {
+ feed_count.feed
+ }
+}
+
+impl From<(Feed, u64)> for FeedCount {
+ fn from((feed, count): (Feed, u64)) -> Self {
+ Self { feed, count }
+ }
+}
+
+impl From<SiteStats> for FinalStats {
+ fn from(stats: SiteStats) -> Self {
+ Self {
+ pages: stats.pages,
+ blogposts: stats.blogposts,
+ news_articles: stats.news_articles,
+ feeds: stats
+ .feeds
+ .into_iter()
+ .map(|(feed, count)| FeedCount::from((feed, count)))
+ .collect(),
+ }
+ }
+}
+
+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
+pub struct FinalSiteStats {
+ site: Site,
+ #[serde(flatten)]
+ stats: FinalStats,
+}
+
+impl FinalSiteStats {
+ pub fn site(&self) -> &Site {
+ &self.site
+ }
+
+ pub fn stats(&self) -> &FinalStats {
+ &self.stats
}
}
#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Hash, PartialEq, Eq)]
#[serde(transparent)]
-struct Site(String);
+pub struct Site(String);
+
+impl Site {
+ fn from_url(url: &str) -> Result<Self> {
+ let url = Url::robust_parse(url)?;
+ let root_domain = url
+ .root_domain()
+ .ok_or(anyhow::anyhow!("Failed to get root domain from url"))?;
+ Ok(Self(root_domain.to_string()))
+ }
+
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+
+ pub fn url(&self) -> Result<Url> {
+ Url::robust_parse(&self.0).map_err(|_| anyhow::anyhow!("Failed to parse url"))
+ }
+}
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
struct SiteId([u8; 8]);
impl SiteId {
- fn from_url(url: String) -> Self {
- let node_id = Node::from(url).into_host().id();
- Self(node_id.as_u64().to_be_bytes())
+ fn from_url(url: &str) -> Result<Self> {
+ let url = Url::robust_parse(url)?;
+ let node_id = Node::from(&url).into_host().id();
+ Ok(Self(node_id.as_u64().to_be_bytes()))
}
}
@@ -65,13 +152,6 @@ impl AsRef<[u8]> for SiteId {
}
}
-#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
-struct FinalSiteStats {
- site: Site,
- #[serde(flatten)]
- stats: SiteStats,
-}
-
struct SiteFilter {
bloom: BytesBloomFilter<SiteId>,
full: HashSet<SiteId>,
@@ -123,10 +203,16 @@ impl StatsWorker {
for file in warc_files.by_ref() {
for record in file.records().flatten() {
- if !self
- .site_filter
- .should_process(&SiteId::from_url(record.request.url.clone()))
- {
+ let site_id = SiteId::from_url(&record.request.url);
+
+ if site_id.is_err() {
+ tracing::error!("error parsing url: {}", site_id.err().unwrap());
+ continue;
+ }
+
+ let site_id = site_id.unwrap();
+
+ if !self.site_filter.should_process(&site_id) {
continue;
}
@@ -139,11 +225,22 @@ impl StatsWorker {
};
let mut stats = SiteStats {
+ feeds: HashMap::new(),
pages: 1,
blogposts: 0,
news_articles: 0,
};
+ if let Ok(feeds) = webpage.feeds() {
+ let root_domain = webpage.url().root_domain();
+
+ for feed in feeds {
+ if feed.url.root_domain() == root_domain {
+ *stats.feeds.entry(feed).or_default() += 1;
+ }
+ }
+ }
+
for schema in webpage.schema_org() {
if schema.types_contains("NewsArticle") {
stats.news_articles = 1;
@@ -153,11 +250,7 @@ impl StatsWorker {
stats.blogposts = 1;
}
}
- if let Some(site) = webpage
- .url()
- .root_domain()
- .map(|site| Site(site.to_string()))
- {
+ if let Ok(site) = Site::from_url(&record.request.url) {
*self.stats.lock().unwrap().entry(site).or_default() += stats;
}
}
@@ -212,7 +305,7 @@ pub fn run(config: SiteStatsConfig) -> Result<()> {
}
for handler in handlers {
- handler.join().unwrap();
+ handler.join().ok();
}
let mut final_stats: Vec<_> = worker
@@ -221,9 +314,25 @@ pub fn run(config: SiteStatsConfig) -> Result<()> {
.unwrap()
.clone()
.into_iter()
- .map(|(site, stats)| FinalSiteStats { site, stats })
+ .map(|(site, stats)| FinalSiteStats {
+ site,
+ stats: stats.into(),
+ })
.collect();
+ final_stats.iter_mut().for_each(|site_stats| {
+ site_stats
+ .stats
+ .feeds
+ .retain(|feed_count| feed_count.count > MIN_FEED_COUNT);
+
+ site_stats.stats.feeds.sort_by(|a, b| b.count.cmp(&a.count));
+
+ if let Some(top_feeds) = TOP_FEEDS_PER_SITE {
+ site_stats.stats.feeds.truncate(top_feeds);
+ }
+ });
+
final_stats.sort_by(|a, b| b.stats.pages.cmp(&a.stats.pages));
let writer = std::fs::File::create(&config.output_path)?;
diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rs
index 42ef0156..fb374436 100644
--- a/crates/core/src/entrypoint/webgraph_server.rs
+++ b/crates/core/src/entrypoint/webgraph_server.rs
@@ -190,14 +190,11 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
// dropping the handle leaves the cluster
let _cluster = Arc::new(
Cluster::join(
- Member {
- id: config.cluster_id,
- service: Service::Webgraph {
- host: addr,
- granularity: config.granularity,
- shard: config.shard,
- },
- },
+ Member::new(Service::Webgraph {
+ host: addr,
+ granularity: config.granularity,
+ shard: config.shard,
+ }),
config.gossip_addr,
config.gossip_seed_nodes.unwrap_or_default(),
)
diff --git a/crates/core/src/feed/index.rs b/crates/core/src/feed/index.rs
deleted file mode 100644
index 6b0ca1e8..00000000
--- a/crates/core/src/feed/index.rs
+++ /dev/null
@@ -1,292 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::{
- fs,
- path::{Path, PathBuf},
-};
-
-use crate::{
- inverted_index::merge_tantivy_segments,
- tokenizer::fields::{FieldTokenizer, UrlTokenizer},
-};
-use anyhow::Result;
-use hashbrown::HashSet;
-use lending_iter::LendingIterator;
-use tantivy::{
- merge_policy::NoMergePolicy,
- query::{PhraseQuery, TermQuery},
- schema::{IndexRecordOption, TextFieldIndexing, TextOptions, Value},
- tokenizer::Tokenizer as TantivyTokenizer,
-};
-use url::Url;
-
-use super::{Feed, FeedKind};
-
-pub struct FeedIndex {
- tantivy_index: tantivy::Index,
- writer: tantivy::IndexWriter,
- reader: tantivy::IndexReader,
- schema: tantivy::schema::Schema,
- pub path: PathBuf,
-}
-
-impl FeedIndex {
- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let url_tokenizer = FieldTokenizer::Url(UrlTokenizer);
- let kind_tokenizer = FieldTokenizer::default();
-
- let mut builder = tantivy::schema::Schema::builder();
-
- builder.add_text_field(
- "url",
- TextOptions::default()
- .set_indexing_options(
- TextFieldIndexing::default()
- .set_tokenizer(url_tokenizer.as_str())
- .set_index_option(IndexRecordOption::WithFreqsAndPositions),
- )
- .set_stored(),
- );
- builder.add_text_field(
- "kind",
- TextOptions::default()
- .set_indexing_options(
- TextFieldIndexing::default()
- .set_tokenizer(kind_tokenizer.as_str())
- .set_index_option(IndexRecordOption::Basic),
- )
- .set_stored(),
- );
-
- let schema = builder.build();
-
- let tv_index = if !path.as_ref().exists() {
- fs::create_dir_all(path.as_ref())?;
- tantivy::Index::create_in_dir(path.as_ref(), schema.clone())?
- } else {
- tantivy::Index::open_in_dir(path.as_ref())?
- };
-
- tv_index
- .tokenizers()
- .register(url_tokenizer.as_str(), url_tokenizer);
-
- tv_index
- .tokenizers()
- .register(kind_tokenizer.as_str(), kind_tokenizer);
-
- let writer = tv_index.writer(50_000_000)?;
-
- let merge_policy = NoMergePolicy;
- writer.set_merge_policy(Box::new(merge_policy));
-
- let reader = tv_index.reader()?;
-
- Ok(Self {
- tantivy_index: tv_index,
- writer,
- reader,
- schema,
- path: path.as_ref().to_path_buf(),
- })
- }
-
- pub fn insert(&mut self, feed: &Feed) -> Result<()> {
- let mut doc = tantivy::TantivyDocument::default();
-
- doc.add_text(self.schema.get_field("url")?, feed.url.as_str());
- doc.add_text(
- self.schema.get_field("kind")?,
- &serde_json::to_string(&feed.kind)?,
- );
-
- self.writer.add_document(doc)?;
-
- Ok(())
- }
-
- pub fn commit(&mut self) -> Result<()> {
- self.writer.commit()?;
- self.reader.reload()?;
-
- Ok(())
- }
-
- fn terms(&self, query: &str) -> Vec<tantivy::Term> {
- let mut tokenizer = UrlTokenizer;
- let mut res = Vec::new();
- let tv_field = self.schema.get_field("url").unwrap();
-
- let mut stream = tokenizer.token_stream(query);
- let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
-
- while let Some(token) = it.next() {
- res.push(tantivy::Term::from_field_text(tv_field, &token.text));
- }
-
- res
- }
-
- pub fn search(&self, query: &str) -> Result<Vec<Feed>> {
- let searcher = self.reader.searcher();
-
- let terms = self.terms(query);
- if terms.is_empty() {
- return Ok(Vec::new());
- }
-
- let query = if terms.len() == 1 {
- let term = terms.first().unwrap().clone();
- Box::new(TermQuery::new(
- term,
- tantivy::schema::IndexRecordOption::Basic,
- )) as Box<dyn tantivy::query::Query>
- } else {
- Box::new(PhraseQuery::new(terms)) as Box<dyn tantivy::query::Query>
- };
-
- let docs = searcher.search(&query, &tantivy::collector::DocSetCollector)?;
-
- let mut res = Vec::new();
-
- for address in docs {
- let doc: tantivy::TantivyDocument = searcher.doc(address)?;
-
- let url = doc
- .get_first(self.schema.get_field("url")?)
- .unwrap()
- .as_str()
- .unwrap();
-
- let url = Url::parse(url)?;
-
- let kind = doc
- .get_first(self.schema.get_field("kind")?)
- .unwrap()
- .as_str()
- .unwrap();
-
- let kind: FeedKind = serde_json::from_str(kind)?;
-
- res.push(Feed { url, kind });
- }
-
- Ok(res)
- }
-
- pub fn merge_into_max_segments(&mut self, max_num_segments: u64) -> Result<()> {
- let segments: Vec<_> = self
- .tantivy_index
- .load_metas()?
- .segments
- .into_iter()
- .collect();
-
- merge_tantivy_segments(&mut self.writer, segments, &self.path, max_num_segments)?;
-
- Ok(())
- }
-
- pub fn merge(mut self, mut other: Self) -> Self {
- let path = self.path.clone();
-
- {
- other.commit().expect("failed to commit index");
- self.commit().expect("failed to commit index");
-
- let other_meta = other
- .tantivy_index
- .load_metas()
- .expect("failed to load tantivy metadata for index");
-
- let mut meta = self
- .tantivy_index
- .load_metas()
- .expect("failed to load tantivy metadata for index");
-
- let other_path = other.path.clone();
- let other_path = other_path.as_path();
- other.writer.wait_merging_threads().unwrap();
-
- let path = self.path.clone();
- let self_path = path.as_path();
- self.writer.wait_merging_threads().unwrap();
-
- let ids: HashSet<_> = meta.segments.iter().map(|segment| segment.id()).collect();
-
- for segment in other_meta.segments {
- if ids.contains(&segment.id()) {
- continue;
- }
-
- // TODO: handle case where current index has segment with same name
- for file in segment.list_files() {
- let p = other_path.join(&file);
- if p.exists() {
- fs::rename(p, self_path.join(&file)).unwrap();
- }
- }
- meta.segments.push(segment);
- }
-
- meta.segments
- .sort_by_key(|a| std::cmp::Reverse(a.max_doc()));
-
- fs::remove_dir_all(other_path).ok();
-
- let self_path = Path::new(&path);
-
- std::fs::write(
- self_path.join("meta.json"),
- serde_json::to_string_pretty(&meta).unwrap(),
- )
- .unwrap();
- }
-
- Self::open(path).expect("failed to open index")
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn feed_index() {
- let mut index = FeedIndex::open(crate::gen_temp_path()).unwrap();
-
- let a = Feed {
- url: Url::parse("https://a.com/feed.xml").unwrap(),
- kind: FeedKind::Atom,
- };
- index.insert(&a).unwrap();
-
- let b = Feed {
- url: Url::parse("https://b.com/another/feed").unwrap(),
- kind: FeedKind::Rss,
- };
- index.insert(&b).unwrap();
-
- index.commit().unwrap();
-
- let res = index.search("a.com").unwrap();
- assert_eq!(res, vec![a]);
-
- let res = index.search("b.com").unwrap();
- assert_eq!(res, vec![b]);
- }
-}
diff --git a/crates/core/src/feed/mod.rs b/crates/core/src/feed/mod.rs
index 8f9ffd6d..9f6c4f40 100644
--- a/crates/core/src/feed/mod.rs
+++ b/crates/core/src/feed/mod.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -14,14 +14,16 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::str::FromStr;
+
use url::Url;
-pub mod index;
mod parser;
-pub mod scheduler;
pub use parser::parse;
+use crate::dated_url::DatedUrl;
+
#[derive(
Debug,
Clone,
@@ -39,6 +41,20 @@ pub enum FeedKind {
Rss,
}
+impl FromStr for FeedKind {
+ type Err = anyhow::Error;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s {
+ "application/atom" => Ok(Self::Atom),
+ "application/atom+xml" => Ok(Self::Atom),
+ "application/rss" => Ok(Self::Rss),
+ "application/rss+xml" => Ok(Self::Rss),
+ s => anyhow::bail!("Unknown feed kind: {s}"),
+ }
+ }
+}
+
#[derive(
Debug,
Clone,
@@ -57,5 +73,5 @@ pub struct Feed {
}
pub struct ParsedFeed {
- pub links: Vec<Url>,
+ pub links: Vec<DatedUrl>,
}
diff --git a/crates/core/src/feed/parser.rs b/crates/core/src/feed/parser.rs
index cfb56920..94b349f3 100644
--- a/crates/core/src/feed/parser.rs
+++ b/crates/core/src/feed/parser.rs
@@ -15,9 +15,12 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::{anyhow, Result};
+use chrono::{DateTime, Utc};
use quick_xml::events::Event;
use url::Url;
+use crate::dated_url::DatedUrl;
+
use super::{FeedKind, ParsedFeed};
fn parse_rss(feed: &str) -> Result<ParsedFeed> {
@@ -27,6 +30,8 @@ fn parse_rss(feed: &str) -> Result<ParsedFeed> {
let mut links = Vec::new();
let mut inside_link = false;
let mut inside_item = false;
+ let mut current_link: Option<Url> = None;
+ let mut current_date: Option<DateTime<Utc>> = None;
loop {
match reader.read_event_into(&mut buf) {
@@ -38,14 +43,33 @@ fn parse_rss(feed: &str) -> Result<ParsedFeed> {
}
Ok(Event::Start(ref e)) if e.name().as_ref() == b"item" => {
inside_item = true;
+ current_link = None;
+ current_date = None;
}
Ok(Event::End(ref e)) if e.name().as_ref() == b"item" => {
inside_item = false;
+ if let Some(link) = current_link.take() {
+ links.push(DatedUrl {
+ url: link,
+ last_modified: current_date,
+ });
+ }
}
Ok(Event::Text(e)) if inside_item && inside_link => {
let link = e.unescape()?;
if let Ok(link) = Url::parse(&link) {
- links.push(link);
+ current_link = Some(link);
+ }
+ }
+ Ok(Event::Start(ref e))
+ if e.name().as_ref() == b"pubDate" || e.name().as_ref() == b"lastBuildDate" =>
+ {
+ if inside_item {
+ if let Ok(Event::Text(e)) = reader.read_event_into(&mut buf) {
+ if let Ok(date) = DateTime::parse_from_rfc2822(&e.unescape()?) {
+ current_date = Some(date.with_timezone(&Utc));
+ }
+ }
}
}
Err(e) => {
@@ -67,14 +91,24 @@ fn parse_atom(feed: &str) -> Result<ParsedFeed> {
let mut links = Vec::new();
let mut inside_entry = false;
+ let mut current_link: Option<Url> = None;
+ let mut current_date: Option<DateTime<Utc>> = None;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) if e.name().as_ref() == b"entry" => {
inside_entry = true;
+ current_link = None;
+ current_date = None;
}
Ok(Event::End(ref e)) if e.name().as_ref() == b"entry" => {
inside_entry = false;
+ if let Some(link) = current_link.take() {
+ links.push(DatedUrl {
+ url: link,
+ last_modified: current_date,
+ });
+ }
}
Ok(Event::Start(ref e)) | Ok(Event::Empty(ref e)) => {
if inside_entry && e.name().as_ref() == b"link" {
@@ -88,7 +122,15 @@ fn parse_atom(feed: &str) -> Result<ParsedFeed> {
.map_err(|e| anyhow!(e))
.and_then(|v| Url::parse(&v).map_err(|e| anyhow!(e)))
{
- links.push(href);
+ current_link = Some(href);
+ }
+ }
+ } else if inside_entry
+ && (e.name().as_ref() == b"updated" || e.name().as_ref() == b"published")
+ {
+ if let Ok(Event::Text(e)) = reader.read_event_into(&mut buf) {
+ if let Ok(date) = DateTime::parse_from_rfc3339(&e.unescape()?) {
+ current_date = Some(date.with_timezone(&Utc));
}
}
}
@@ -142,7 +184,14 @@ mod tests {
assert_eq!(
parsed.links,
- vec![Url::parse("https://example.com/a").unwrap()]
+ vec![DatedUrl {
+ url: Url::parse("https://example.com/a").unwrap(),
+ last_modified: Some(
+ DateTime::parse_from_rfc2822("Mon, 30 Oct 2023 08:55:00 GMT")
+ .unwrap()
+ .with_timezone(&Utc)
+ ),
+ }]
);
}
@@ -163,7 +212,14 @@ mod tests {
assert_eq!(
parsed.links,
- vec![Url::parse("https://example.com/a").unwrap()]
+ vec![DatedUrl {
+ url: Url::parse("https://example.com/a").unwrap(),
+ last_modified: Some(
+ DateTime::parse_from_rfc3339("2003-12-13T18:30:02Z")
+ .unwrap()
+ .with_timezone(&Utc)
+ ),
+ }]
);
}
}
diff --git a/crates/core/src/feed/scheduler.rs b/crates/core/src/feed/scheduler.rs
deleted file mode 100644
index 24ad9a91..00000000
--- a/crates/core/src/feed/scheduler.rs
+++ /dev/null
@@ -1,177 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::path::Path;
-
-use anyhow::Result;
-use hashbrown::{HashMap, HashSet};
-use url::Url;
-
-use crate::{
- webgraph::{
- centrality::{top_nodes, TopNodes},
- NodeID, Webgraph,
- },
- webpage::url_ext::UrlExt,
-};
-
-use super::{index::FeedIndex, Feed};
-
-#[derive(
- Debug,
- Clone,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
- PartialEq,
- Eq,
- Hash,
-)]
-pub struct Domain(String);
-
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
-pub struct DomainFeeds {
- pub domain: Domain,
- pub feeds: Vec<Feed>,
-}
-
-impl From<&Url> for Domain {
- fn from(url: &Url) -> Self {
- Domain(url.icann_domain().unwrap_or_default().to_string())
- }
-}
-
-#[derive(
- Debug,
- Clone,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
- PartialEq,
- Eq,
- Hash,
-)]
-pub struct SplitId(#[bincode(with_serde)] uuid::Uuid);
-
-impl std::fmt::Display for SplitId {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- write!(f, "SplitId({})", self.0)
- }
-}
-
-impl SplitId {
- pub fn id(&self) -> uuid::Uuid {
- self.0
- }
-}
-
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
-pub struct Split {
- pub id: SplitId,
- pub feeds: Vec<DomainFeeds>,
-}
-
-impl Split {
- pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {
- let file = std::fs::OpenOptions::new()
- .write(true)
- .create(true)
- .truncate(true)
- .open(path.as_ref())?;
- let writer = std::io::BufWriter::new(file);
-
- serde_json::to_writer_pretty(writer, &self)
- .map_err(|e| anyhow::anyhow!("Failed to serialize split: {}", e))?;
-
- Ok(())
- }
-
- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let file = std::fs::OpenOptions::new().read(true).open(path.as_ref())?;
- let reader = std::io::BufReader::new(file);
-
- serde_json::from_reader(reader)
- .map_err(|e| anyhow::anyhow!("Failed to deserialize split: {}", e))
- }
-}
-
-pub struct Schedule {
- splits: Vec<Split>,
-}
-impl Schedule {
- pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {
- if !path.as_ref().exists() {
- std::fs::create_dir_all(&path)?;
- }
-
- for split in self.splits {
- let name = split.id.0.to_string() + ".json";
- split.save(path.as_ref().join(name))?;
- }
-
- Ok(())
- }
-}
-
-pub fn schedule(
- index: &FeedIndex,
- host_centrality: &speedy_kv::Db<NodeID, f64>,
- host_graph: &Webgraph,
- num_splits: u64,
-) -> Schedule {
- let top_hosts = top_nodes(host_centrality, TopNodes::Top(1_000_000));
-
- let mut all_feeds = HashMap::new();
-
- for (host, _) in top_hosts {
- let host = host_graph.id2node(&host).unwrap();
- let url = Url::parse(&format!("http://{}", host.as_str()));
-
- if url.is_err() {
- continue;
- }
-
- let url = url.unwrap();
- let domain = Domain::from(&url);
-
- all_feeds
- .entry(domain)
- .or_insert(HashSet::new())
- .extend(index.search(host.as_str()).unwrap().into_iter());
- }
-
- let mut splits = Vec::new();
-
- for _ in 0..num_splits {
- splits.push(Split {
- id: SplitId(uuid::Uuid::new_v4()),
- feeds: Vec::new(),
- });
- }
-
- for (i, (domain, feeds)) in all_feeds.into_iter().enumerate() {
- let split = &mut splits[i % num_splits as usize];
-
- split.feeds.push(DomainFeeds {
- domain,
- feeds: feeds.into_iter().collect(),
- });
- }
-
- Schedule { splits }
-}
diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rs
index 0efdce1e..8aea95ba 100644
--- a/crates/core/src/inverted_index/search.rs
+++ b/crates/core/src/inverted_index/search.rs
@@ -39,6 +39,7 @@ use crate::snippet;
use crate::snippet::TextSnippet;
use crate::webgraph::NodeID;
+use crate::schema::text_field::TextField;
use crate::webpage::url_ext::UrlExt;
use crate::Result;
@@ -186,8 +187,7 @@ impl InvertedIndex {
let tv_searcher = self.reader.searcher();
let mut webpages: Vec<RetrievedWebpage> = websites
.iter()
- .map(|website| self.retrieve_doc(website.address, &tv_searcher))
- .filter_map(|res| res.ok())
+ .filter_map(|website| self.retrieve_doc(website.address, &tv_searcher).ok())
.collect();
for (url, page) in webpages.iter_mut().filter_map(|page| {
@@ -310,4 +310,30 @@ impl InvertedIndex {
pub(crate) fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {
KeyPhrase::compute_top(&self.reader, top_n)
}
+
+ pub(crate) fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {
+ let ctx = self.local_search_ctx();
+ let tv_searcher = ctx.tv_searcher;
+
+ let field = tv_searcher
+ .schema()
+ .get_field(text_field::SiteNoTokenizer.name())
+ .unwrap();
+
+ let term = tantivy::Term::from_field_text(field, site);
+
+ let query = tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic);
+
+ match tv_searcher.search(
+ &query,
+ &tantivy::collector::TopDocs::with_limit(limit).and_offset(offset),
+ ) {
+ Ok(res) => res
+ .into_iter()
+ .filter_map(|(_, doc)| self.retrieve_doc(doc.into(), &tv_searcher).ok())
+ .filter_map(|page| Url::parse(&page.url).ok())
+ .collect(),
+ Err(_) => vec![],
+ }
+ }
}
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 23b43519..28979087 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -52,6 +52,7 @@ pub mod canon_index;
mod collector;
pub mod config;
pub mod crawler;
+mod dated_url;
pub mod distributed;
pub mod entity_index;
mod enum_map;
@@ -82,6 +83,7 @@ mod search_prettifier;
pub mod searcher;
mod simhash;
pub mod similar_hosts;
+mod sitemap;
mod snippet;
mod stopwords;
pub mod summarizer;
@@ -145,22 +147,17 @@ pub fn start_gossip_cluster_thread(config: GossipConfig, service: Option<Service
rt.block_on(async {
let cluster = match service {
Some(service) => Cluster::join(
- Member {
- id: config.cluster_id,
- service,
- },
- config.addr,
- config.seed_nodes.unwrap_or_default(),
- )
- .await
- .unwrap(),
- None => Cluster::join_as_spectator(
- config.cluster_id,
+ Member::new(service),
config.addr,
config.seed_nodes.unwrap_or_default(),
)
.await
.unwrap(),
+ None => {
+ Cluster::join_as_spectator(config.addr, config.seed_nodes.unwrap_or_default())
+ .await
+ .unwrap()
+ }
};
let cluster = Arc::new(cluster);
diff --git a/crates/core/src/live_index/crawler/budgets.rs b/crates/core/src/live_index/crawler/budgets.rs
new file mode 100644
index 00000000..f4eb3114
--- /dev/null
+++ b/crates/core/src/live_index/crawler/budgets.rs
@@ -0,0 +1,147 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/
+
+use url::Url;
+
+use crate::config::DailyLiveIndexCrawlerBudget;
+use crate::entrypoint::site_stats::{FinalSiteStats, Site};
+use crate::webgraph;
+use crate::webpage::url_ext::UrlExt;
+use crate::Result;
+use std::collections::HashMap;
+use std::path::Path;
+use std::time::Duration;
+
+use super::SiteStats;
+
+const MILLIS_PER_DAY: u64 = 24 * 60 * 60 * 1000;
+
+fn assign_budgets(
+ sites: &mut HashMap<Site, f64>,
+ daily_budget: u64,
+ total_centrality: f64,
+ host_centrality: &speedy_kv::Db<webgraph::NodeID, f64>,
+) {
+ for (site, budget) in sites.iter_mut() {
+ if let Some(centrality) = Url::robust_parse(site.as_str())
+ .ok()
+ .map(webgraph::Node::from)
+ .map(|node| node.id())
+ .and_then(|id| host_centrality.get(&id).ok().flatten())
+ {
+ *budget = (centrality * daily_budget as f64) / total_centrality;
+ }
+ }
+}
+
+fn calculate_total<'a>(
+ sites: impl Iterator<Item = &'a FinalSiteStats>,
+ host_centrality: &speedy_kv::Db<webgraph::NodeID, f64>,
+) -> f64 {
+ sites
+ .filter_map(|site| Url::robust_parse(site.site().as_str()).ok())
+ .map(webgraph::Node::from)
+ .filter_map(|node| host_centrality.get(&node.id()).ok().flatten())
+ .sum()
+}
+
+pub struct SiteBudgets {
+ blogs: HashMap<Site, f64>,
+ news: HashMap<Site, f64>,
+ remaining: HashMap<Site, f64>,
+}
+
+impl SiteBudgets {
+ pub fn new<P: AsRef<Path>>(
+ host_centrality: P,
+ stats: &SiteStats,
+ daily_budget: DailyLiveIndexCrawlerBudget,
+ ) -> Result<Self> {
+ tracing::debug!("Assigning budgets");
+ let host_centrality: speedy_kv::Db<webgraph::NodeID, f64> =
+ speedy_kv::Db::open_or_create(host_centrality)?;
+
+ let mut blogs = HashMap::new();
+ let mut news = HashMap::new();
+ let mut remaining = HashMap::new();
+
+ for site in stats.blogs() {
+ blogs.insert(site.site().clone(), 0.0);
+ }
+
+ for site in stats.news() {
+ news.insert(site.site().clone(), 0.0);
+ }
+
+ for site in stats.all() {
+ if blogs.contains_key(site.site()) || news.contains_key(site.site()) {
+ continue;
+ }
+
+ remaining.insert(site.site().clone(), 0.0);
+ }
+
+ let blogs_total = calculate_total(stats.blogs(), &host_centrality);
+ let news_total = calculate_total(stats.news(), &host_centrality);
+ let remaining_total = calculate_total(
+ stats
+ .all()
+ .filter(|site| !blogs.contains_key(site.site()) && !news.contains_key(site.site())),
+ &host_centrality,
+ );
+
+ assign_budgets(
+ &mut blogs,
+ daily_budget.blogs,
+ blogs_total,
+ &host_centrality,
+ );
+ assign_budgets(&mut news, daily_budget.news, news_total, &host_centrality);
+ assign_budgets(
+ &mut remaining,
+ daily_budget.remaining,
+ remaining_total,
+ &host_centrality,
+ );
+
+ tracing::debug!("Budgets assigned");
+ tracing::debug!(
+ "Number of sites with budgets: blogs: {}, news: {}, remaining: {}",
+ blogs.len(),
+ news.len(),
+ remaining.len(),
+ );
+ Ok(Self {
+ blogs,
+ news,
+ remaining,
+ })
+ }
+
+ pub fn drip_rate(&self, site: &Site) -> Option<Duration> {
+ let budget = *self
+ .blogs
+ .get(site)
+ .or_else(|| self.news.get(site))
+ .or_else(|| self.remaining.get(site))?;
+
+ if budget == 0.0 {
+ Some(Duration::from_millis(MILLIS_PER_DAY)) // once per day
+ } else {
+ Some(Duration::from_millis(MILLIS_PER_DAY / budget as u64))
+ }
+ }
+}
diff --git a/crates/core/src/live_index/crawler/checker/feeds.rs b/crates/core/src/live_index/crawler/checker/feeds.rs
new file mode 100644
index 00000000..1e394de5
--- /dev/null
+++ b/crates/core/src/live_index/crawler/checker/feeds.rs
@@ -0,0 +1,66 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::time::Duration;
+
+use crate::feed::{parse, Feed};
+use crate::Result;
+
+use super::{CheckIntervals, Checker, CrawlableUrl};
+
+const CRAWL_DELAY: Duration = Duration::from_secs(5);
+
+pub struct Feeds {
+ feeds: Vec<Feed>,
+ last_check: std::time::Instant,
+ client: reqwest::Client,
+}
+
+impl Feeds {
+ pub fn new(feeds: Vec<Feed>, client: reqwest::Client) -> Self {
+ Self {
+ feeds,
+ last_check: std::time::Instant::now(),
+ client,
+ }
+ }
+}
+
+impl Checker for Feeds {
+ async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {
+ let mut urls = Vec::new();
+
+ for feed in &self.feeds {
+ let resp = self.client.get(feed.url.clone()).send().await?;
+ let text = resp.text().await?;
+ let parsed_feed = parse(&text, feed.kind)?;
+
+ for link in parsed_feed.links {
+ urls.push(CrawlableUrl::from(link));
+ }
+
+ tokio::time::sleep(CRAWL_DELAY).await;
+ }
+
+ self.last_check = std::time::Instant::now();
+
+ Ok(urls)
+ }
+
+ fn should_check(&self, interval: &CheckIntervals) -> bool {
+ self.last_check.elapsed() > interval.feeds
+ }
+}
diff --git a/crates/core/src/live_index/crawler/checker/frontpage.rs b/crates/core/src/live_index/crawler/checker/frontpage.rs
new file mode 100644
index 00000000..bf4d92f9
--- /dev/null
+++ b/crates/core/src/live_index/crawler/checker/frontpage.rs
@@ -0,0 +1,65 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use url::Url;
+
+use crate::config::CheckIntervals;
+use crate::webpage::Html;
+use crate::Result;
+use crate::{entrypoint::site_stats, webpage::url_ext::UrlExt};
+
+use super::{Checker, CrawlableUrl};
+
+pub struct Frontpage {
+ url: Url,
+ last_check: std::time::Instant,
+ client: reqwest::Client,
+}
+
+impl Frontpage {
+ pub fn new(site: &site_stats::Site, client: reqwest::Client) -> Result<Self> {
+ let url = Url::robust_parse(&format!("https://{}/", site.as_str()))?;
+
+ Ok(Self {
+ url,
+ last_check: std::time::Instant::now(),
+ client,
+ })
+ }
+}
+
+impl Checker for Frontpage {
+ async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {
+ let res = self.client.get(self.url.clone()).send().await?;
+ let body = res.text().await?;
+
+ let page = Html::parse(&body, self.url.as_str())?;
+
+ let urls = page
+ .anchor_links()
+ .into_iter()
+ .map(|link| CrawlableUrl::from(link.destination))
+ .collect::<Vec<_>>();
+
+ self.last_check = std::time::Instant::now();
+
+ Ok(urls)
+ }
+
+ fn should_check(&self, interval: &CheckIntervals) -> bool {
+ self.last_check.elapsed() > interval.frontpage
+ }
+}
diff --git a/crates/core/src/live_index/crawler/checker/mod.rs b/crates/core/src/live_index/crawler/checker/mod.rs
new file mode 100644
index 00000000..ec349c80
--- /dev/null
+++ b/crates/core/src/live_index/crawler/checker/mod.rs
@@ -0,0 +1,60 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/
+
+use chrono::{DateTime, Utc};
+use url::Url;
+
+mod feeds;
+mod frontpage;
+mod sitemap;
+
+pub use self::feeds::Feeds;
+pub use frontpage::Frontpage;
+pub use sitemap::Sitemap;
+
+use crate::config::CheckIntervals;
+use crate::dated_url::DatedUrl;
+
+use crate::Result;
+
+#[derive(Debug)]
+pub struct CrawlableUrl {
+ pub url: Url,
+ pub last_modified: Option<DateTime<Utc>>,
+}
+
+impl From<DatedUrl> for CrawlableUrl {
+ fn from(url: DatedUrl) -> Self {
+ Self {
+ url: url.url,
+ last_modified: url.last_modified,
+ }
+ }
+}
+
+impl From<Url> for CrawlableUrl {
+ fn from(url: Url) -> Self {
+ Self {
+ url,
+ last_modified: None,
+ }
+ }
+}
+
+pub trait Checker {
+ async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>>;
+ fn should_check(&self, interval: &CheckIntervals) -> bool;
+}
diff --git a/crates/core/src/live_index/crawler/checker/sitemap.rs b/crates/core/src/live_index/crawler/checker/sitemap.rs
new file mode 100644
index 00000000..eeb598a2
--- /dev/null
+++ b/crates/core/src/live_index/crawler/checker/sitemap.rs
@@ -0,0 +1,132 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::time::Duration;
+
+use url::Url;
+
+use crate::dated_url::DatedUrl;
+use crate::sitemap::{parse_sitemap, SitemapEntry};
+use crate::Result;
+use crate::{entrypoint::site_stats, webpage::url_ext::UrlExt};
+
+use super::{CheckIntervals, Checker, CrawlableUrl};
+
+const MAX_SITEMAP_DEPTH: usize = 10;
+const SITEMAP_DELAY: Duration = Duration::from_secs(10);
+
+pub struct Sitemap {
+ robots_txt: Url,
+ last_check: std::time::Instant,
+ client: reqwest::Client,
+}
+
+impl Sitemap {
+ pub fn new(site: &site_stats::Site, client: reqwest::Client) -> Result<Self> {
+ let robots_txt = Url::robust_parse(&format!("{}/robots.txt", site.as_str()))?;
+
+ Ok(Self {
+ robots_txt,
+ last_check: std::time::Instant::now(),
+ client,
+ })
+ }
+
+ async fn sitemap_urls(&self) -> Result<Vec<Url>> {
+ let res = self.client.get(self.robots_txt.clone()).send().await?;
+ let body = res.text().await?;
+
+ // wildcard useragent is okay as we only use it to check for sitemap directive
+ let robots = robotstxt::Robots::parse("*", &body)?;
+
+ Ok(robots
+ .sitemaps()
+ .iter()
+ .filter_map(|s| Url::parse(s).ok())
+ .collect())
+ }
+
+ async fn urls_from_sitemap(&self, sitemap: Url) -> Vec<DatedUrl> {
+ let mut stack = vec![(sitemap, 0)];
+ let mut urls = vec![];
+
+ while let Some((url, depth)) = stack.pop() {
+ if depth >= MAX_SITEMAP_DEPTH {
+ continue;
+ }
+
+ let res = self.client.get(url).send().await;
+ tokio::time::sleep(SITEMAP_DELAY).await;
+
+ if res.is_err() {
+ continue;
+ }
+
+ let res = res.unwrap();
+
+ if res.status() != reqwest::StatusCode::OK {
+ continue;
+ }
+
+ let body = res.text().await;
+
+ if body.is_err() {
+ continue;
+ }
+
+ let body = body.unwrap();
+
+ let entries = parse_sitemap(&body);
+
+ for entry in entries {
+ match entry {
+ SitemapEntry::Url(url) => {
+ urls.push(url);
+ }
+ SitemapEntry::Sitemap(url) => {
+ stack.push((url, depth + 1));
+ }
+ }
+ }
+ }
+
+ urls
+ }
+}
+
+impl Checker for Sitemap {
+ async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {
+ let sitemap_urls = self.sitemap_urls().await?;
+ let mut urls = vec![];
+
+ for sitemap_url in sitemap_urls {
+ urls.extend(
+ self.urls_from_sitemap(sitemap_url)
+ .await
+ .into_iter()
+ .map(CrawlableUrl::from),
+ );
+ }
+
+ self.last_check = std::time::Instant::now();
+
+ Ok(urls)
+ }
+
+ fn should_check(&self, interval: &CheckIntervals) -> bool {
+ self.last_check.elapsed() > interval.sitemap
+ }
+}
diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rs
new file mode 100644
index 00000000..3c80a516
--- /dev/null
+++ b/crates/core/src/live_index/crawler/crawlable_site.rs
@@ -0,0 +1,340 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/
+
+use std::{
+ sync::{
+ atomic::{AtomicBool, Ordering},
+ Arc,
+ },
+ time::{Duration, Instant},
+};
+
+use itertools::Itertools;
+use tokio::sync::Mutex;
+use url::Url;
+
+use crate::{
+ config::{CheckIntervals, CrawlerConfig},
+ crawler,
+ entrypoint::{
+ indexer::IndexableWebpage,
+ site_stats::{self, FinalSiteStats},
+ },
+ live_index::crawler::checker::{Feeds, Frontpage, Sitemap},
+ webpage::url_ext::UrlExt,
+ Result,
+};
+
+use super::{
+ checker::{Checker, CrawlableUrl},
+ crawled_db::ShardedCrawledDb,
+ Client,
+};
+
+pub const MAX_PENDING_BUDGET: u64 = 128;
+
+struct InnerCrawlableSite {
+ site: site_stats::Site,
+ feeds: Feeds,
+ sitemap: Sitemap,
+ frontpage: Frontpage,
+ last_drip: Instant,
+ drip_rate: Duration,
+ budget: u64,
+}
+
+impl InnerCrawlableSite {
+ pub fn new(site: FinalSiteStats, client: &Client, drip_rate: Duration) -> Result<Self> {
+ Ok(Self {
+ site: site.site().clone(),
+ feeds: Feeds::new(
+ site.stats()
+ .feeds
+ .clone()
+ .into_iter()
+ .map(|feed| feed.into())
+ .collect(),
+ client.reqwest().clone(),
+ ),
+ sitemap: Sitemap::new(site.site(), client.reqwest().clone())?,
+ frontpage: Frontpage::new(site.site(), client.reqwest().clone())?,
+ last_drip: Instant::now(),
+ drip_rate,
+ budget: 0,
+ })
+ }
+
+ pub fn drip(&mut self) {
+ let pages_to_drip =
+ (self.last_drip.elapsed().as_secs_f32() / self.drip_rate.as_secs_f32()) as u64;
+
+ if pages_to_drip > 0 {
+ self.last_drip = Instant::now();
+ self.budget += pages_to_drip;
+
+ if self.budget > MAX_PENDING_BUDGET {
+ self.budget = MAX_PENDING_BUDGET;
+ }
+ }
+ }
+ pub fn should_crawl(&self, interval: &CheckIntervals) -> bool {
+ (self.frontpage.should_check(interval)
+ || self.sitemap.should_check(interval)
+ || self.feeds.should_check(interval))
+ && self.budget > 0
+ }
+}
+
+pub struct CrawlableSite {
+ inner: Arc<Mutex<InnerCrawlableSite>>,
+ currently_crawling: AtomicBool,
+ site: site_stats::Site,
+}
+
+impl CrawlableSite {
+ pub fn new(site: FinalSiteStats, client: &Client, drip_rate: Duration) -> Result<Self> {
+ Ok(Self {
+ site: site.site().clone(),
+ inner: Arc::new(Mutex::new(InnerCrawlableSite::new(
+ site, client, drip_rate,
+ )?)),
+ currently_crawling: AtomicBool::new(false),
+ })
+ }
+
+ pub fn currently_crawling(&self) -> bool {
+ self.currently_crawling.load(Ordering::Relaxed)
+ }
+
+ pub fn site(&self) -> &site_stats::Site {
+ &self.site
+ }
+
+ pub async fn should_crawl(&self, interval: &CheckIntervals) -> bool {
+ !self.currently_crawling() && self.inner.lock().await.should_crawl(interval)
+ }
+
+ pub async fn drip(&self) {
+ if self.currently_crawling() {
+ return;
+ }
+
+ self.inner.lock().await.drip();
+ }
+}
+
+impl crawler::DatumStream for tokio::sync::Mutex<Vec<crawler::CrawlDatum>> {
+ async fn write(&self, crawl_datum: crawler::CrawlDatum) -> Result<(), crawler::Error> {
+ self.lock().await.push(crawl_datum);
+ Ok(())
+ }
+
+ async fn finish(&self) -> Result<(), crawler::Error> {
+ Ok(())
+ }
+}
+
+pub struct CrawlableSiteGuard {
+ site: Arc<CrawlableSite>,
+ crawled_db: Arc<ShardedCrawledDb>,
+ config: Arc<CrawlerConfig>,
+}
+
+impl CrawlableSiteGuard {
+ pub async fn new(
+ site: Arc<CrawlableSite>,
+ crawled_db: Arc<ShardedCrawledDb>,
+ config: Arc<CrawlerConfig>,
+ ) -> Self {
+ {
+ let currently_crawling = site.currently_crawling.swap(true, Ordering::Relaxed);
+ if currently_crawling {
+ panic!("site is already being crawled");
+ }
+ }
+
+ Self {
+ site,
+ crawled_db,
+ config,
+ }
+ }
+
+ pub fn url(&self) -> Result<Url> {
+ self.site.site().url()
+ }
+}
+
+impl Drop for CrawlableSiteGuard {
+ fn drop(&mut self) {
+ self.site.currently_crawling.store(false, Ordering::Relaxed);
+
+ tracing::debug!(
+ "Dropping crawlable site guard for site {}",
+ self.site.site().as_str()
+ );
+ }
+}
+
+impl CrawlableSiteGuard {
+ pub async fn crawl(self, client: &Client, interval: &CheckIntervals) -> Result<()> {
+ let mut site = self.site.inner.lock().await;
+ let mut urls = Vec::new();
+
+ if site.feeds.should_check(interval) {
+ urls.extend(site.feeds.get_urls().await.unwrap_or_default());
+ }
+
+ if site.sitemap.should_check(interval) {
+ urls.extend(site.sitemap.get_urls().await.unwrap_or_default());
+ }
+
+ if site.frontpage.should_check(interval) {
+ urls.extend(site.frontpage.get_urls().await.unwrap_or_default());
+ }
+
+ let url = site.site.url()?;
+ let icann_domain = url.icann_domain();
+ urls = urls
+ .into_iter()
+ .filter(|u| u.url.icann_domain() == icann_domain)
+ .unique_by(|u| u.url.clone())
+ .collect();
+
+ urls.retain(|url| !self.crawled_db.has_crawled(&url.url).unwrap_or(false));
+
+ order_urls(&mut urls);
+
+ let budget = site.budget.min(urls.len() as u64);
+ site.budget = site.budget.saturating_sub(budget);
+
+ urls.truncate(budget as usize);
+
+ if urls.is_empty() {
+ tracing::debug!("No new urls to crawl for site {}", site.site.as_str());
+ return Ok(());
+ }
+
+ tracing::debug!(
+ "Crawling {} urls for site {}",
+ urls.len(),
+ site.site.as_str()
+ );
+
+ for crawlable_url in &urls {
+ self.crawled_db.insert(&crawlable_url.url)?;
+ }
+
+ let crawl_data = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+
+ let executor = crawler::JobExecutor::new(
+ crawler::WorkerJob {
+ domain: crawler::Domain::from(site.site.url()?),
+ urls: urls
+ .into_iter()
+ .take(budget as usize)
+ .map(|url| {
+ let weighted_url = crawler::WeightedUrl {
+ url: url.url,
+ weight: 1.0,
+ };
+
+ crawler::RetrieableUrl::from(weighted_url)
+ })
+ .collect(),
+ wandering_urls: 0,
+ },
+ client.reqwest().clone(),
+ self.config.clone(),
+ Arc::clone(&crawl_data),
+ );
+
+ executor.run().await;
+
+ let crawl_data = crawl_data.lock().await.clone();
+
+ tracing::debug!(
+ "Indexing {} urls for site {}",
+ crawl_data.len(),
+ site.site.as_str()
+ );
+ client
+ .index(crawl_data.into_iter().map(IndexableWebpage::from).collect())
+ .await?;
+
+ tracing::debug!("Finished crawling site {}", site.site.as_str());
+
+ Ok(())
+ }
+}
+
+fn order_urls(urls: &mut [CrawlableUrl]) {
+ urls.sort_by(|a, b| match (a.last_modified, b.last_modified) {
+ (Some(a), Some(b)) => a.cmp(&b).reverse(),
+ (Some(_), None) => std::cmp::Ordering::Greater,
+ (None, Some(_)) => std::cmp::Ordering::Less,
+ (None, None) => std::cmp::Ordering::Equal,
+ });
+}
+
+#[cfg(test)]
+mod tests {
+ use chrono::DateTime;
+ use url::Url;
+
+ use super::*;
+
+ #[test]
+ fn test_latest_urls_crawled_first() {
+ let mut urls = vec![
+ CrawlableUrl {
+ url: Url::parse("https://example.com/page1").unwrap(),
+ last_modified: Some(
+ DateTime::parse_from_rfc2822("Mon, 01 Jan 2024 00:00:00 GMT")
+ .unwrap()
+ .into(),
+ ),
+ },
+ CrawlableUrl {
+ url: Url::parse("https://example.com/page2").unwrap(),
+ last_modified: Some(
+ DateTime::parse_from_rfc2822("Tue, 02 Jan 2024 00:00:01 GMT")
+ .unwrap()
+ .into(),
+ ),
+ },
+ CrawlableUrl {
+ url: Url::parse("https://example.com/page3").unwrap(),
+ last_modified: None,
+ },
+ ];
+
+ order_urls(&mut urls);
+
+ assert_eq!(
+ urls[0].url,
+ Url::parse("https://example.com/page3").unwrap()
+ );
+ assert_eq!(
+ urls[1].url,
+ Url::parse("https://example.com/page2").unwrap()
+ );
+ assert_eq!(
+ urls[2].url,
+ Url::parse("https://example.com/page1").unwrap()
+ );
+ }
+}
diff --git a/crates/core/src/live_index/crawler/downloaded_db.rs b/crates/core/src/live_index/crawler/crawled_db.rs
similarity index 67%
rename from crates/core/src/live_index/crawler/downloaded_db.rs
rename to crates/core/src/live_index/crawler/crawled_db.rs
index 30bfc9e7..9e90eb06 100644
--- a/crates/core/src/live_index/crawler/downloaded_db.rs
+++ b/crates/core/src/live_index/crawler/crawled_db.rs
@@ -14,11 +14,16 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/
+use dashmap::DashMap;
use redb::ReadableTable;
use url::Url;
-use crate::Result;
-use std::{path::Path, time::Duration};
+use crate::{webpage::url_ext::UrlExt, Result};
+use std::{
+ path::{Path, PathBuf},
+ sync::Arc,
+ time::Duration,
+};
use crate::live_index::TTL;
@@ -172,7 +177,7 @@ impl InnerDb {
redb::TableDefinition::new("times")
}
- fn has_downloaded(&self, url: &Url) -> Result<bool> {
+ fn has_crawled(&self, url: &Url) -> Result<bool> {
let url = TruncatedUrl::new(url);
Ok(self
@@ -232,7 +237,7 @@ impl InnerDb {
self.truncate(TTL)
}
- pub fn insert(&mut self, url: &Url) -> Result<()> {
+ fn insert(&mut self, url: &Url) -> Result<()> {
{
let key = TruncatedUrl::new(url);
let txn = self.db.begin_write()?;
@@ -258,11 +263,11 @@ impl InnerDb {
}
}
-pub struct DownloadedDb {
+struct CrawledDb {
inner: std::sync::Mutex<InnerDb>,
}
-impl DownloadedDb {
+impl CrawledDb {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
let db = InnerDb::open(path)?;
@@ -271,8 +276,8 @@ impl DownloadedDb {
})
}
- pub fn has_downloaded(&self, url: &Url) -> Result<bool> {
- self.inner.lock().unwrap().has_downloaded(url)
+ pub fn has_crawled(&self, url: &Url) -> Result<bool> {
+ self.inner.lock().unwrap().has_crawled(url)
}
pub fn insert(&self, url: &Url) -> Result<()> {
@@ -280,35 +285,120 @@ impl DownloadedDb {
}
}
+#[derive(Debug, Clone, Hash, Eq, PartialEq)]
+struct ShardedHost(String);
+
+impl ShardedHost {
+ fn from_url(url: &Url) -> Option<Self> {
+ let host = url.normalized_host()?;
+ let hash = md5::compute(host);
+ let hash = format!("{:x}", hash).chars().take(2).collect();
+
+ Some(Self(hash))
+ }
+}
+
+pub struct ShardedCrawledDb {
+ inner: Arc<DashMap<ShardedHost, CrawledDb>>,
+ folder: PathBuf,
+}
+
+impl ShardedCrawledDb {
+ pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {
+ if !folder.as_ref().exists() {
+ std::fs::create_dir_all(folder.as_ref())?;
+ }
+
+ let inner = Arc::new(
+ std::fs::read_dir(&folder)?
+ .filter_map(|entry| entry.ok())
+ .filter(|entry| entry.file_type().map(|ft| ft.is_dir()).unwrap_or(false))
+ .filter_map(|entry| {
+ let path = entry.path();
+ let shard = ShardedHost(path.file_name()?.to_str()?.to_string());
+ let db = CrawledDb::open(path).ok()?;
+ Some((shard, db))
+ })
+ .collect::<DashMap<ShardedHost, CrawledDb>>(),
+ );
+
+ Ok(Self {
+ folder: folder.as_ref().to_path_buf(),
+ inner,
+ })
+ }
+
+ pub fn has_crawled(&self, url: &Url) -> Result<bool> {
+ let shard =
+ ShardedHost::from_url(url).ok_or(anyhow::anyhow!("Failed to get shard from url"))?;
+ match self.inner.get(&shard) {
+ Some(db) => db.has_crawled(url),
+ None => Ok(false),
+ }
+ }
+
+ pub fn insert(&self, url: &Url) -> Result<()> {
+ let shard =
+ ShardedHost::from_url(url).ok_or(anyhow::anyhow!("Failed to get shard from url"))?;
+
+ let db = self.inner.entry(shard.clone()).or_insert_with(|| {
+ let path = self.folder.join(shard.0);
+ CrawledDb::open(path).unwrap()
+ });
+
+ db.insert(url)
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_downloaded_db() {
- let db = DownloadedDb::open(crate::gen_temp_path()).unwrap();
+ let db = CrawledDb::open(crate::gen_temp_path()).unwrap();
let url = Url::parse("https://example.com").unwrap();
- assert!(!db.has_downloaded(&url).unwrap());
+ assert!(!db.has_crawled(&url).unwrap());
db.insert(&url).unwrap();
- assert!(db.has_downloaded(&url).unwrap());
+ assert!(db.has_crawled(&url).unwrap());
}
#[test]
fn test_truncate_ttl() {
- let db = DownloadedDb::open(crate::gen_temp_path()).unwrap();
+ let db = CrawledDb::open(crate::gen_temp_path()).unwrap();
let url = Url::parse("https://example.com").unwrap();
db.insert(&url).unwrap();
- assert!(db.has_downloaded(&url).unwrap());
+ assert!(db.has_crawled(&url).unwrap());
let ttl = Duration::from_secs(1);
std::thread::sleep(ttl + Duration::from_secs(1));
db.inner.lock().unwrap().truncate(ttl).unwrap();
- assert!(!db.has_downloaded(&url).unwrap());
+ assert!(!db.has_crawled(&url).unwrap());
+ }
+
+ #[test]
+ fn test_sharded_downloaded_db() {
+ let db = ShardedCrawledDb::open(crate::gen_temp_path()).unwrap();
+
+ let url = Url::parse("https://example.com").unwrap();
+ assert!(!db.has_crawled(&url).unwrap());
+
+ db.insert(&url).unwrap();
+ assert!(db.has_crawled(&url).unwrap());
+
+ let url = Url::parse("https://example.com/foo").unwrap();
+ assert!(!db.has_crawled(&url).unwrap());
+
+ db.insert(&url).unwrap();
+ assert!(db.has_crawled(&url).unwrap());
+
+ let url = Url::parse("https://another_example.com/bar").unwrap();
+ assert!(!db.has_crawled(&url).unwrap());
}
}
diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rs
index 84637499..f1209505 100644
--- a/crates/core/src/live_index/crawler/mod.rs
+++ b/crates/core/src/live_index/crawler/mod.rs
@@ -12,135 +12,326 @@
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/
+// along with this program. If not, see <https://www.gnu.org/licenses/>
-mod downloaded_db;
-
-use crate::Result;
+mod budgets;
+mod checker;
+mod crawlable_site;
+mod crawled_db;
+use std::fs::File;
+use std::path::PathBuf;
use std::sync::Arc;
+use std::time::Duration;
-use chrono::Utc;
-use url::Url;
-
+use crate::ampc::dht::ShardId;
+use crate::config::{CheckIntervals, CrawlerConfig, LiveCrawlerConfig};
+use crate::distributed::cluster::Cluster;
+use crate::distributed::sonic::replication::{
+ AllShardsSelector, RandomReplicaSelector, RandomShardSelector, ShardedClient,
+};
+use crate::entrypoint::search_server;
+use crate::entrypoint::site_stats::FinalSiteStats;
+use crate::{crawler, Result};
use crate::{
- config::CrawlerConfig,
- crawler::reqwest_client,
- feed::{
- self,
- scheduler::{Domain, DomainFeeds, Split},
- },
+ distributed::sonic::replication::ReusableShardedClient,
+ entrypoint::{indexer::IndexableWebpage, live_index},
};
+use crawlable_site::{CrawlableSite, CrawlableSiteGuard};
+use crawled_db::ShardedCrawledDb;
+use tokio::sync::Mutex;
+use tokio::sync::Semaphore;
+use url::Url;
+
+const SITE_URL_BATCH_SIZE: usize = 100;
+const DEFAULT_CONSISTENCY_FRACTION: f64 = 0.5;
+const BLOG_FRACTION_THRESHOLD: f64 = 0.5;
+const NEWS_FRACTION_THRESHOLD: f64 = 0.5;
+const MIN_CRAWL_DELAY: Duration = Duration::from_secs(30);
+const MAX_CRAWL_DELAY: Duration = Duration::from_secs(300);
+const TICK_INTERVAL: Duration = Duration::from_secs(5);
+
+struct Client {
+ live_index: Mutex<ReusableShardedClient<live_index::LiveIndexService>>,
+ search: Mutex<ReusableShardedClient<search_server::SearchService>>,
+ reqwest: reqwest::Client,
+}
+
+impl Client {
+ pub async fn new(cluster: Arc<Cluster>, crawler_config: &CrawlerConfig) -> Result<Self> {
+ let live_index = Mutex::new(ReusableShardedClient::new(cluster.clone()).await);
+ let search = Mutex::new(ReusableShardedClient::new(cluster.clone()).await);
+
+ Ok(Self {
+ live_index,
+ search,
+ reqwest: crawler::reqwest_client(crawler_config)?,
+ })
+ }
+
+ pub fn reqwest(&self) -> &reqwest::Client {
+ &self.reqwest
+ }
+
+ async fn live_conn(&self) -> Arc<ShardedClient<live_index::LiveIndexService, ShardId>> {
+ self.live_index.lock().await.conn().await
+ }
+
+ async fn search_conn(&self) -> Arc<ShardedClient<search_server::SearchService, ShardId>> {
+ self.search.lock().await.conn().await
+ }
+
+ pub async fn index(&self, pages: Vec<IndexableWebpage>) -> Result<()> {
+ let conn = self.live_conn().await;
-use downloaded_db::DownloadedDb;
+ let req = live_index::IndexWebpages {
+ pages,
+ consistency_fraction: Some(DEFAULT_CONSISTENCY_FRACTION),
+ };
+
+ while let Err(e) = conn
+ .send(req.clone(), &RandomShardSelector, &RandomReplicaSelector)
+ .await
+ {
+ tracing::error!("Failed to index pages: {e}");
+ tokio::time::sleep(Duration::from_millis(1_000)).await;
+ }
+
+ Ok(())
+ }
+
+ async fn get_site_urls_from_backbone(&self, site: &str) -> Result<Vec<Url>> {
+ let mut res = Vec::new();
+ let conn = self.search_conn().await;
+ let mut offset = 0;
+
+ while let Ok(resp) = conn
+ .send(
+ search_server::GetSiteUrls {
+ site: site.to_string(),
+ offset,
+ limit: SITE_URL_BATCH_SIZE,
+ },
+ &AllShardsSelector,
+ &RandomReplicaSelector,
+ )
+ .await
+ {
+ let urls: Vec<_> = resp
+ .into_iter()
+ .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))
+ .collect();
+
+ if urls.is_empty() {
+ break;
+ }
+
+ res.extend(urls);
+ offset += SITE_URL_BATCH_SIZE;
+ }
+
+ Ok(res)
+ }
+
+ async fn get_site_urls_from_live(&self, site: &str) -> Result<Vec<Url>> {
+ let mut res = Vec::new();
+ let conn = self.live_conn().await;
+ let mut offset = 0;
+
+ while let Ok(resp) = conn
+ .send(
+ search_server::GetSiteUrls {
+ site: site.to_string(),
+ offset,
+ limit: SITE_URL_BATCH_SIZE,
+ },
+ &AllShardsSelector,
+ &RandomReplicaSelector,
+ )
+ .await
+ {
+ let urls: Vec<_> = resp
+ .into_iter()
+ .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))
+ .collect();
+
+ if urls.is_empty() {
+ break;
+ }
+
+ res.extend(urls);
+ offset += SITE_URL_BATCH_SIZE;
+ }
+
+ Ok(res)
+ }
-use super::{Feeds, FEED_CHECK_INTERVAL};
+ pub async fn get_site_urls(&self, site: &str) -> Vec<Url> {
+ let mut res = Vec::new();
-pub enum CrawlResults {
- None,
- HasInserts,
+ if let Ok(urls) = self.get_site_urls_from_backbone(site).await {
+ res.extend(urls);
+ }
+
+ if let Ok(urls) = self.get_site_urls_from_live(site).await {
+ res.extend(urls);
+ }
+
+ res
+ }
}
-pub struct Crawler {
- feeds: Vec<Feeds>,
- downloaded_db: DownloadedDb,
- _config: Arc<CrawlerConfig>,
- client: reqwest::Client,
+impl From<LiveCrawlerConfig> for CrawlerConfig {
+ fn from(config: LiveCrawlerConfig) -> Self {
+ Self {
+ num_worker_threads: 1,
+ user_agent: config.user_agent,
+ robots_txt_cache_sec: crate::config::defaults::Crawler::robots_txt_cache_sec(),
+ min_politeness_factor: 0,
+ start_politeness_factor: 1,
+ min_crawl_delay_ms: MIN_CRAWL_DELAY.as_millis() as u64,
+ max_crawl_delay_ms: MAX_CRAWL_DELAY.as_millis() as u64,
+ max_politeness_factor: crate::config::defaults::Crawler::max_politeness_factor(),
+ max_url_slowdown_retry: crate::config::defaults::Crawler::max_url_slowdown_retry(),
+ timeout_seconds: 60,
+ s3: crate::config::S3Config {
+ bucket: String::new(),
+ folder: String::new(),
+ access_key: String::new(),
+ secret_key: String::new(),
+ endpoint: String::new(),
+ },
+ router_hosts: vec![],
+ }
+ }
}
-impl Crawler {
- pub fn new(
- split: Split,
- downloaded_db: DownloadedDb,
- config: Arc<CrawlerConfig>,
- ) -> Result<Self> {
- let client = reqwest_client(&config)?;
+struct SiteStats {
+ sites: Vec<FinalSiteStats>,
+}
- Ok(Self {
- feeds: split.into(),
- downloaded_db,
- _config: config,
- client,
+impl SiteStats {
+ pub fn open(path: PathBuf) -> Result<Self> {
+ let file = File::open(path)?;
+ let sites = serde_json::from_reader(file)?;
+ Ok(Self { sites })
+ }
+
+ pub fn blogs(&self) -> impl Iterator<Item = &FinalSiteStats> {
+ self.sites.iter().filter(|site| {
+ site.stats().blogposts as f64 / site.stats().pages as f64 > BLOG_FRACTION_THRESHOLD
+ })
+ }
+
+ pub fn news(&self) -> impl Iterator<Item = &FinalSiteStats> {
+ self.sites.iter().filter(|site| {
+ site.stats().news_articles as f64 / site.stats().pages as f64 > NEWS_FRACTION_THRESHOLD
})
}
- async fn process_urls(&self, _urls: Vec<Url>) -> Result<bool> {
- todo!()
-
- // if urls.is_empty() {
- // return Ok(false);
- // }
-
- // let domain = urls.first().unwrap().into();
- // let job = WorkerJob {
- // domain,
- // urls: urls
- // .clone()
- // .into_iter()
- // .map(|url| RetrieableUrl::from(WeightedUrl { url, weight: 1.0 }))
- // .collect(),
- // wandering_urls: 0,
- // };
-
- // let executor = JobExecutor::new(
- // job,
- // self.client.clone(),
- // self.config.clone(),
- // self.indexer.clone(),
- // );
- // executor.run().await;
-
- // for url in &urls {
- // self.downloaded_db.insert(url)?;
- // }
-
- // Ok(true)
+ pub fn all(&self) -> impl Iterator<Item = &FinalSiteStats> {
+ self.sites.iter()
}
+}
- async fn process_feed(&self, domain_feeds: &DomainFeeds) -> Result<bool> {
- let mut urls = Vec::new();
-
- for feed in domain_feeds.feeds.iter() {
- let content = self
- .client
- .get(feed.url.as_str())
- .send()
- .await?
- .text()
- .await?;
-
- let feed = feed::parse(&content, feed.kind)?;
-
- for url in feed.links {
- if !self.downloaded_db.has_downloaded(&url)?
- && Domain::from(&url) == domain_feeds.domain
- {
- urls.push(url);
+pub struct Crawler {
+ client: Arc<Client>,
+ db: Arc<ShardedCrawledDb>,
+ sites: Vec<Arc<CrawlableSite>>,
+ num_worker_threads: usize,
+ check_intervals: CheckIntervals,
+ crawler_config: Arc<CrawlerConfig>,
+}
+
+impl Crawler {
+ pub async fn new(config: LiveCrawlerConfig) -> Result<Self> {
+ let crawler_config = Arc::new(CrawlerConfig::from(config.clone()));
+
+ let cluster = Arc::new(
+ Cluster::join_as_spectator(
+ config.gossip.addr,
+ config.gossip.seed_nodes.unwrap_or_default(),
+ )
+ .await?,
+ );
+
+ let client = Arc::new(Client::new(cluster, &crawler_config).await?);
+ let db = Arc::new(ShardedCrawledDb::open(config.crawled_db_path)?);
+
+ let site_stats = SiteStats::open(config.site_stats_path)?;
+ let sites: Vec<_> = site_stats.all().cloned().collect();
+
+ let budgets = budgets::SiteBudgets::new(
+ &config.host_centrality_path,
+ &site_stats,
+ config.daily_budget.clone(),
+ )?;
+
+ let mut crawlable_sites = Vec::new();
+ tracing::debug!("Initializing crawler db with previously crawled urls");
+ for site in sites {
+ for url in client.get_site_urls(site.site().as_str()).await {
+ if !db.has_crawled(&url)? {
+ db.insert(&url)?;
}
}
+
+ if let Some(drip_rate) = budgets.drip_rate(site.site()) {
+ crawlable_sites.push(Arc::new(CrawlableSite::new(site, &client, drip_rate)?));
+ }
}
- self.process_urls(urls).await
+ Ok(Self {
+ client,
+ db,
+ sites: crawlable_sites,
+ num_worker_threads: config.num_worker_threads,
+ check_intervals: config.check_intervals,
+ crawler_config,
+ })
}
- pub async fn check_feeds(&mut self) -> CrawlResults {
- let mut feeds = self.feeds.clone();
+ pub async fn run(mut self) -> Result<()> {
+ let mut interval = tokio::time::interval(TICK_INTERVAL);
+ let semaphore = Arc::new(Semaphore::new(self.num_worker_threads));
- let mut futures = Vec::new();
- for feeds in feeds.iter_mut() {
- if feeds.last_checked + FEED_CHECK_INTERVAL < Utc::now() {
- futures.push(self.process_feed(&feeds.feed));
- feeds.last_checked = Utc::now();
- }
- }
+ loop {
+ interval.tick().await;
+ tracing::debug!("Tick");
- let res = futures::future::join_all(futures).await;
+ for site in &mut self.sites {
+ if site.currently_crawling() {
+ tracing::debug!("Site {} is currently crawling", site.site().as_str());
+ continue;
+ }
- self.feeds = feeds;
+ site.drip().await;
- if res.iter().filter_map(|r| r.as_ref().ok()).any(|r| *r) {
- CrawlResults::HasInserts
- } else {
- CrawlResults::None
+ if site.should_crawl(&self.check_intervals).await {
+ tracing::debug!("Site {} should crawl", site.site().as_str());
+ let client = self.client.clone();
+ let intervals = self.check_intervals.clone();
+ let guard = CrawlableSiteGuard::new(
+ site.clone(),
+ self.db.clone(),
+ self.crawler_config.clone(),
+ )
+ .await;
+ let semaphore = semaphore.clone();
+
+ tokio::task::spawn(async move {
+ let _permit = semaphore.acquire().await.unwrap();
+ let url = guard.url();
+
+ if let Err(e) = guard.crawl(&client, &intervals).await {
+ if let Ok(url) = url {
+ tracing::error!("Failed to crawl site {}: {:?}", url, e);
+ } else {
+ tracing::error!("Failed to crawl site: {:?}", e);
+ }
+ }
+ });
+ }
+ }
}
}
}
diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rs
index 620df532..a0507a40 100644
--- a/crates/core/src/live_index/index.rs
+++ b/crates/core/src/live_index/index.rs
@@ -264,6 +264,7 @@ impl LiveIndex {
}
pub fn commit(&self) {
+ tracing::debug!("committing index");
futures::executor::block_on(
self.inner
.write()
@@ -273,6 +274,7 @@ impl LiveIndex {
}
pub fn prune_segments(&self) {
+ tracing::debug!("pruning segments");
self.inner
.write()
.unwrap_or_else(|e| e.into_inner())
@@ -287,6 +289,7 @@ impl LiveIndex {
}
pub fn compact_segments_by_date(&self) {
+ tracing::debug!("compacting segments by date");
self.inner
.write()
.unwrap_or_else(|e| e.into_inner())
@@ -294,6 +297,7 @@ impl LiveIndex {
}
pub fn insert(&self, pages: &[IndexableWebpage]) {
+ tracing::debug!("inserting pages into index");
self.inner
.write()
.unwrap_or_else(|e| e.into_inner())
diff --git a/crates/core/src/live_index/mod.rs b/crates/core/src/live_index/mod.rs
index bee8d976..afc5b732 100644
--- a/crates/core/src/live_index/mod.rs
+++ b/crates/core/src/live_index/mod.rs
@@ -15,64 +15,18 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::time::Duration;
-use crate::config::{CrawlerConfig, LiveIndexConfig};
-
pub use self::index::LiveIndex;
pub use self::index_manager::IndexManager;
-// mod crawler;
+pub mod crawler;
pub mod index;
mod index_manager;
+pub use self::crawler::Crawler;
+
const TTL: Duration = Duration::from_secs(60 * 60 * 24 * 60); // 60 days
const PRUNE_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour
const COMPACT_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour
- // const FEED_CHECK_INTERVAL: Duration = Duration::from_secs(60 * 10); // 10 minutes
const AUTO_COMMIT_INTERVAL: Duration = Duration::from_secs(60 * 5); // 5 minutes
const EVENT_LOOP_INTERVAL: Duration = Duration::from_secs(5);
const BATCH_SIZE: usize = 512;
-
-// #[derive(Debug, Clone)]
-// struct Feeds {
-// last_checked: DateTime<Utc>,
-// feed: DomainFeeds,
-// }
-//
-// impl From<Split> for Vec<Feeds> {
-// fn from(split: Split) -> Self {
-// split
-// .feeds
-// .into_iter()
-// .map(|feed| Feeds {
-// last_checked: Utc::now(),
-// feed,
-// })
-// .collect()
-// }
-// }
-
-impl From<&LiveIndexConfig> for CrawlerConfig {
- fn from(live: &LiveIndexConfig) -> Self {
- Self {
- num_worker_threads: 1, // no impact
- user_agent: live.user_agent.clone(),
- robots_txt_cache_sec: live.robots_txt_cache_sec,
- start_politeness_factor: live.start_politeness_factor,
- min_politeness_factor: live.min_politeness_factor,
- max_politeness_factor: live.max_politeness_factor,
- min_crawl_delay_ms: live.min_crawl_delay_ms,
- max_crawl_delay_ms: live.max_crawl_delay_ms,
- max_url_slowdown_retry: live.max_url_slowdown_retry,
- timeout_seconds: live.timeout_seconds,
- // no impact
- s3: crate::config::S3Config {
- bucket: String::new(),
- folder: String::new(),
- access_key: String::new(),
- secret_key: String::new(),
- endpoint: String::new(),
- },
- router_hosts: Vec::new(),
- }
- }
-}
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index 7170ab2d..480753d8 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -177,6 +177,9 @@ enum AdminIndexOptions {
enum LiveIndex {
/// Serve the live index.
Serve { config_path: String },
+
+ /// Start the live index crawler.
+ Crawler { config_path: String },
}
#[derive(Subcommand)]
@@ -262,11 +265,6 @@ enum IndexingOptions {
output_path: String,
},
- /// Create the feed index. Used to find feeds to put into the live index.
- Feed {
- config_path: String,
- },
-
// Create an index of canonical urls.
Canonical {
config_path: String,
@@ -308,10 +306,6 @@ fn main() -> Result<()> {
wikipedia_dump_path,
output_path,
} => entrypoint::EntityIndexer::run(wikipedia_dump_path, output_path)?,
- IndexingOptions::Feed { config_path } => {
- let config = load_toml_config(config_path);
- entrypoint::feed_indexer::build(config)?;
- }
IndexingOptions::MergeSearch { paths } => {
let pointers = paths
.into_iter()
@@ -454,7 +448,15 @@ fn main() -> Result<()> {
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()?
- .block_on(entrypoint::live_index::serve(config))?;
+ .block_on(entrypoint::live_index::search_server::serve(config))?;
+ }
+ LiveIndex::Crawler { config_path } => {
+ let config = load_toml_config(config_path);
+
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()?
+ .block_on(entrypoint::live_index::crawler::run(config))?;
}
},
Commands::WebSpell { config_path } => {
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index 5fcaa60f..77050f0e 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -41,7 +41,7 @@ use crate::ranking::{
};
use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};
use crate::web_spell::SpellChecker;
-use crate::webgraph::remote::RemoteWebgraph;
+use crate::webgraph::remote::{RemoteWebgraph, WebgraphGranularity};
use crate::webgraph::EdgeLimit;
use crate::widgets::{Widget, Widgets};
use crate::{
@@ -177,7 +177,7 @@ pub trait Graph {
) -> impl Future<Output = Vec<Vec<webgraph::Edge<()>>>>;
}
-impl Graph for RemoteWebgraph {
+impl<G: WebgraphGranularity> Graph for RemoteWebgraph<G> {
async fn batch_raw_ingoing(
&self,
nodes: &[webgraph::NodeID],
@@ -570,13 +570,23 @@ where
let liked: Vec<_> = host_rankings
.liked
.iter()
- .map(|n| webgraph::Node::from(n.clone()).into_host().id())
+ .filter_map(|n| {
+ Url::parse(n)
+ .or_else(|_| Url::parse(&format!("http://{}", n)))
+ .ok()
+ })
+ .map(|n| webgraph::Node::from(n).into_host().id())
.collect();
let disliked: Vec<_> = host_rankings
.disliked
.iter()
- .map(|n| webgraph::Node::from(n.clone()).into_host().id())
+ .filter_map(|n| {
+ Url::parse(n)
+ .or_else(|_| Url::parse(&format!("http://{}", n)))
+ .ok()
+ })
+ .map(|n| webgraph::Node::from(n).into_host().id())
.collect();
inbound_similarity::Scorer::new(webgraph, &liked, &disliked, false).await
}
diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rs
index 67d27342..dd927676 100644
--- a/crates/core/src/searcher/distributed.rs
+++ b/crates/core/src/searcher/distributed.rs
@@ -17,7 +17,7 @@
use crate::{
distributed::{
cluster::Cluster,
- member::{Service, ShardId},
+ member::{LiveIndexState, Service, ShardId},
sonic::replication::{
AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient,
ReusableClientManager, ReusableShardedClient, Shard, ShardIdentifier, ShardedClient,
@@ -27,6 +27,7 @@ use crate::{
entity_index::EntityMatch,
entrypoint::{
entity_search_server,
+ live_index::LiveIndexService,
search_server::{self, SearchService},
},
image_store::Image,
@@ -110,16 +111,14 @@ pub struct InitialSearchResultShard {
pub shard: ShardId,
}
-struct SearchClientManager;
-
-impl ReusableClientManager for SearchClientManager {
+impl ReusableClientManager for SearchService {
const CLIENT_REFRESH_INTERVAL: std::time::Duration = CLIENT_REFRESH_INTERVAL;
type Service = SearchService;
type ShardId = ShardId;
- async fn new_client(&self, cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {
+ async fn new_client(cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {
let mut shards = HashMap::new();
for member in cluster.members().await {
if let Service::Searcher { host, shard } = member.service {
@@ -140,15 +139,13 @@ impl ReusableClientManager for SearchClientManager {
}
}
-struct EntitySearchClientManager;
-
-impl ReusableClientManager for EntitySearchClientManager {
+impl ReusableClientManager for entity_search_server::SearchService {
const CLIENT_REFRESH_INTERVAL: std::time::Duration = CLIENT_REFRESH_INTERVAL;
type Service = entity_search_server::SearchService;
type ShardId = ();
- async fn new_client(&self, cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {
+ async fn new_client(cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {
let mut replicas = Vec::new();
for member in cluster.members().await {
if let Service::EntitySearcher { host } = member.service {
@@ -166,20 +163,46 @@ impl ReusableClientManager for EntitySearchClientManager {
}
}
+impl ReusableClientManager for LiveIndexService {
+ const CLIENT_REFRESH_INTERVAL: std::time::Duration = CLIENT_REFRESH_INTERVAL;
+
+ type Service = LiveIndexService;
+
+ type ShardId = ShardId;
+
+ async fn new_client(cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {
+ let mut shards = HashMap::new();
+ for member in cluster.members().await {
+ if let Service::LiveIndex { host, shard, state } = member.service {
+ if state == LiveIndexState::Ready {
+ shards.entry(shard).or_insert_with(Vec::new).push(host);
+ }
+ }
+ }
+
+ let mut shard_clients = Vec::new();
+
+ for (id, replicas) in shards {
+ let replicated =
+ ReplicatedClient::new(replicas.into_iter().map(RemoteClient::new).collect());
+ let shard = Shard::new(id, replicated);
+ shard_clients.push(shard);
+ }
+
+ ShardedClient::new(shard_clients)
+ }
+}
+
pub struct DistributedSearcher {
- client: Mutex<ReusableShardedClient<SearchClientManager>>,
- entiy_client: Mutex<ReusableShardedClient<EntitySearchClientManager>>,
+ client: Mutex<ReusableShardedClient<SearchService>>,
+ entiy_client: Mutex<ReusableShardedClient<entity_search_server::SearchService>>,
}
impl DistributedSearcher {
pub async fn new(cluster: Arc<Cluster>) -> Self {
Self {
- client: Mutex::new(
- ReusableShardedClient::new(cluster.clone(), SearchClientManager).await,
- ),
- entiy_client: Mutex::new(
- ReusableShardedClient::new(cluster, EntitySearchClientManager).await,
- ),
+ client: Mutex::new(ReusableShardedClient::new(cluster.clone()).await),
+ entiy_client: Mutex::new(ReusableShardedClient::new(cluster).await),
}
}
diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rs
index a8dbfd1c..e4baf94f 100644
--- a/crates/core/src/searcher/local.rs
+++ b/crates/core/src/searcher/local.rs
@@ -343,6 +343,13 @@ where
pub fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {
self.index.guard().inverted_index().top_key_phrases(top_n)
}
+
+ pub fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {
+ self.index
+ .guard()
+ .inverted_index()
+ .get_site_urls(site, offset, limit)
+ }
}
#[cfg(test)]
diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rs
index 4e24dc7b..f60322bc 100644
--- a/crates/core/src/similar_hosts.rs
+++ b/crates/core/src/similar_hosts.rs
@@ -22,7 +22,10 @@ use url::Url;
use crate::{
ranking::{bitvec_similarity, inbound_similarity},
- webgraph::{remote::RemoteWebgraph, EdgeLimit, Node, NodeID},
+ webgraph::{
+ remote::{RemoteWebgraph, WebgraphGranularity},
+ EdgeLimit, Node, NodeID,
+ },
webpage::url_ext::UrlExt,
SortableFloat,
};
@@ -33,13 +36,13 @@ pub struct ScoredNode {
pub score: f64,
}
-pub struct SimilarHostsFinder {
- webgraph: Arc<RemoteWebgraph>,
+pub struct SimilarHostsFinder<G: WebgraphGranularity> {
+ webgraph: Arc<RemoteWebgraph<G>>,
max_similar_hosts: usize,
}
-impl SimilarHostsFinder {
- pub fn new(webgraph: Arc<RemoteWebgraph>, max_similar_hosts: usize) -> Self {
+impl<G: WebgraphGranularity> SimilarHostsFinder<G> {
+ pub fn new(webgraph: Arc<RemoteWebgraph<G>>, max_similar_hosts: usize) -> Self {
Self {
webgraph,
max_similar_hosts,
@@ -57,7 +60,8 @@ impl SimilarHostsFinder {
let nodes: Vec<_> = nodes
.iter()
- .map(|url| Node::from(url.to_string()).into_host())
+ .filter_map(|url| Url::robust_parse(url).ok())
+ .map(|url| Node::from(url).into_host())
.collect();
let domains = nodes
diff --git a/crates/core/src/sitemap.rs b/crates/core/src/sitemap.rs
new file mode 100644
index 00000000..97ac4af2
--- /dev/null
+++ b/crates/core/src/sitemap.rs
@@ -0,0 +1,213 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use chrono::{DateTime, Utc};
+use quick_xml::events::Event;
+use url::Url;
+
+use crate::dated_url::DatedUrl;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum SitemapEntry {
+ Url(DatedUrl),
+ Sitemap(Url),
+}
+
+pub fn parse_sitemap(s: &str) -> Vec<SitemapEntry> {
+ let mut reader = quick_xml::Reader::from_str(s);
+
+ let mut res = vec![];
+
+ let mut in_sitemap = false;
+ let mut in_url = false;
+ let mut in_loc = false;
+ let mut in_lastmod = false;
+
+ let mut current_url: Option<Url> = None;
+ let mut current_lastmod: Option<DateTime<Utc>> = None;
+
+ loop {
+ match reader.read_event() {
+ Ok(Event::Start(ref e)) => {
+ if e.name().as_ref() == b"sitemap" {
+ in_sitemap = true;
+ } else if e.name().as_ref() == b"url" {
+ in_url = true;
+ } else if e.name().as_ref() == b"loc" {
+ in_loc = true;
+ } else if e.name().as_ref() == b"lastmod" {
+ in_lastmod = true;
+ }
+ }
+ Ok(Event::End(ref e)) => {
+ if e.name().as_ref() == b"sitemap" {
+ in_sitemap = false;
+ } else if e.name().as_ref() == b"url" {
+ in_url = false;
+ if let Some(url) = current_url.take() {
+ res.push(SitemapEntry::Url(DatedUrl {
+ url,
+ last_modified: current_lastmod.take(),
+ }));
+ }
+ } else if e.name().as_ref() == b"loc" {
+ in_loc = false;
+ } else if e.name().as_ref() == b"lastmod" {
+ in_lastmod = false;
+ }
+ }
+ Ok(Event::Text(e)) => {
+ if in_sitemap && in_loc {
+ if let Ok(url) = Url::parse(&e.unescape().unwrap()) {
+ res.push(SitemapEntry::Sitemap(url));
+ }
+ } else if in_url && in_loc {
+ if let Ok(url) = Url::parse(&e.unescape().unwrap()) {
+ current_url = Some(url);
+ }
+ } else if in_url && in_lastmod {
+ if let Ok(date) = DateTime::parse_from_rfc3339(&e.unescape().unwrap()) {
+ current_lastmod = Some(date.with_timezone(&Utc));
+ }
+ }
+ }
+ Ok(Event::Eof) => break,
+ Err(e) => {
+ tracing::debug!("failed to parse sitemap: {}", e);
+ break;
+ }
+ _ => (),
+ }
+ }
+
+ res
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn parse_sitemap() {
+ let dr = r#"<sitemapindex>
+ <sitemap>
+ <loc>https://www.dr.dk/drtv/sitemap.xml</loc>
+ </sitemap>
+ <sitemap>
+ <loc>https://www.dr.dk/sitemap.tvguide.xml</loc>
+ </sitemap>
+ <sitemap>
+ <loc>
+ https://www.dr.dk/sitemap.kommunalvalg.resultater.xml
+ </loc>
+ </sitemap>
+ <sitemap>
+ <loc>https://www.dr.dk/sitemap.folketingsvalg2022.xml</loc>
+ </sitemap>
+ </sitemapindex>"#;
+
+ let entries = super::parse_sitemap(dr);
+ assert_eq!(
+ entries,
+ vec![
+ super::SitemapEntry::Sitemap("https://www.dr.dk/drtv/sitemap.xml".parse().unwrap()),
+ super::SitemapEntry::Sitemap(
+ "https://www.dr.dk/sitemap.tvguide.xml".parse().unwrap()
+ ),
+ super::SitemapEntry::Sitemap(
+ "https://www.dr.dk/sitemap.kommunalvalg.resultater.xml"
+ .parse()
+ .unwrap()
+ ),
+ super::SitemapEntry::Sitemap(
+ "https://www.dr.dk/sitemap.folketingsvalg2022.xml"
+ .parse()
+ .unwrap()
+ ),
+ ]
+ );
+
+ let dr = r#"<urlset>
+ <url>
+ <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>
+ <loc>https://www.dr.dk/drtv/serie/sleepover_6382</loc>
+ </url>
+ <url>
+ <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>
+ <loc>https://www.dr.dk/drtv/saeson/sleepover_9673</loc>
+ </url>
+ <url>
+ <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>
+ <loc>
+ https://www.dr.dk/drtv/episode/sleepover_-zoologisk-museum_52239
+ </loc>
+ </url>
+ <url>
+ <lastmod>2023-10-18T05:40:04.7435930+00:00</lastmod>
+ <loc>
+ https://www.dr.dk/drtv/episode/sleepover_-koebenhavns-raadhus_52252
+ </loc>
+ </url>
+ </urlset>"#;
+
+ let entries = super::parse_sitemap(dr);
+ assert_eq!(
+ entries,
+ vec![
+ super::SitemapEntry::Url(DatedUrl {
+ url: "https://www.dr.dk/drtv/serie/sleepover_6382"
+ .parse()
+ .unwrap(),
+ last_modified: Some(
+ "2023-10-18T05:40:04.7435930+00:00"
+ .parse::<DateTime<Utc>>()
+ .unwrap()
+ ),
+ }),
+ super::SitemapEntry::Url(DatedUrl {
+ url: "https://www.dr.dk/drtv/saeson/sleepover_9673"
+ .parse()
+ .unwrap(),
+ last_modified: Some(
+ "2023-10-18T05:40:04.7435930+00:00"
+ .parse::<DateTime<Utc>>()
+ .unwrap()
+ ),
+ }),
+ super::SitemapEntry::Url(DatedUrl {
+ url: "https://www.dr.dk/drtv/episode/sleepover_-zoologisk-museum_52239"
+ .parse()
+ .unwrap(),
+ last_modified: Some(
+ "2023-10-18T05:40:04.7435930+00:00"
+ .parse::<DateTime<Utc>>()
+ .unwrap()
+ ),
+ }),
+ super::SitemapEntry::Url(DatedUrl {
+ url: "https://www.dr.dk/drtv/episode/sleepover_-koebenhavns-raadhus_52252"
+ .parse()
+ .unwrap(),
+ last_modified: Some(
+ "2023-10-18T05:40:04.7435930+00:00"
+ .parse::<DateTime<Utc>>()
+ .unwrap()
+ ),
+ }),
+ ]
+ );
+ }
+}
diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rs
index d435a27b..6eced6bd 100644
--- a/crates/core/src/webgraph/centrality/harmonic.rs
+++ b/crates/core/src/webgraph/centrality/harmonic.rs
@@ -36,6 +36,7 @@ const HYPERLOGLOG_COUNTERS: usize = 64;
pub static SKIPPED_REL: std::sync::LazyLock<RelFlags> = std::sync::LazyLock::new(|| {
RelFlags::TAG
| RelFlags::NOFOLLOW
+ | RelFlags::SPONSORED
| RelFlags::IS_IN_FOOTER
| RelFlags::IS_IN_NAVIGATION
| RelFlags::PRIVACY_POLICY
diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rs
index 8e964bd3..d37ebee9 100644
--- a/crates/core/src/webgraph/node.rs
+++ b/crates/core/src/webgraph/node.rs
@@ -153,6 +153,7 @@ impl Node {
}
}
+#[cfg(test)]
impl From<String> for Node {
fn from(name: String) -> Self {
let url = if name.contains("://") {
@@ -172,6 +173,7 @@ impl From<&Url> for Node {
}
}
+#[cfg(test)]
impl From<&str> for Node {
fn from(name: &str) -> Self {
name.to_string().into()
diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rs
index 9cbb407e..411529d0 100644
--- a/crates/core/src/webgraph/remote.rs
+++ b/crates/core/src/webgraph/remote.rs
@@ -21,7 +21,7 @@ use tokio::sync::Mutex;
use url::Url;
use crate::{
- config::WebgraphGranularity,
+ config,
distributed::{
cluster::Cluster,
member::{Service, ShardId},
@@ -41,18 +41,40 @@ use crate::{
use super::{Edge, EdgeLimit, FullEdge, Node, NodeID};
-struct WebgraphClientManager {
- granularity: WebgraphGranularity,
+struct WebgraphClientManager<G: WebgraphGranularity>(std::marker::PhantomData<G>);
+
+pub trait WebgraphGranularity: Clone {
+ fn granularity() -> config::WebgraphGranularity;
}
-impl sonic::replication::ReusableClientManager for WebgraphClientManager {
+#[derive(Clone)]
+pub struct Page;
+
+impl WebgraphGranularity for Page {
+ fn granularity() -> config::WebgraphGranularity {
+ config::WebgraphGranularity::Page
+ }
+}
+
+#[derive(Clone)]
+pub struct Host;
+
+impl WebgraphGranularity for Host {
+ fn granularity() -> config::WebgraphGranularity {
+ config::WebgraphGranularity::Host
+ }
+}
+
+impl<G> sonic::replication::ReusableClientManager for WebgraphClientManager<G>
+where
+ G: WebgraphGranularity,
+{
const CLIENT_REFRESH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);
type Service = WebGraphService;
type ShardId = ShardId;
async fn new_client(
- &self,
cluster: &Cluster,
) -> sonic::replication::ShardedClient<Self::Service, Self::ShardId> {
let shards = cluster
@@ -66,7 +88,7 @@ impl sonic::replication::ReusableClientManager for WebgraphClientManager {
granularity,
} = member.service
{
- if granularity == self.granularity {
+ if granularity == G::granularity() {
Some((shard, RemoteClient::<WebGraphService>::new(host)))
} else {
None
@@ -90,16 +112,15 @@ impl sonic::replication::ReusableClientManager for WebgraphClientManager {
}
#[derive(Clone)]
-pub struct RemoteWebgraph {
- client: Arc<Mutex<sonic::replication::ReusableShardedClient<WebgraphClientManager>>>,
+pub struct RemoteWebgraph<G: WebgraphGranularity> {
+ client: Arc<Mutex<sonic::replication::ReusableShardedClient<WebgraphClientManager<G>>>>,
}
-impl RemoteWebgraph {
- pub async fn new(cluster: Arc<Cluster>, granularity: WebgraphGranularity) -> Self {
- let manager = WebgraphClientManager { granularity };
-
+impl<G: WebgraphGranularity> RemoteWebgraph<G> {
+ pub async fn new(cluster: Arc<Cluster>) -> Self {
#[cfg(feature = "dev")]
{
+ let granularity = G::granularity();
tracing::info!("waiting for {granularity} webgraph to come online...");
cluster
.await_member(|member| {
@@ -119,7 +140,7 @@ impl RemoteWebgraph {
Self {
client: Arc::new(Mutex::new(
- sonic::replication::ReusableShardedClient::new(cluster, manager).await,
+ sonic::replication::ReusableShardedClient::new(cluster).await,
)),
}
}
diff --git a/crates/core/src/webpage/html/feeds.rs b/crates/core/src/webpage/html/feeds.rs
new file mode 100644
index 00000000..cac65215
--- /dev/null
+++ b/crates/core/src/webpage/html/feeds.rs
@@ -0,0 +1,45 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::str::FromStr;
+
+use url::Url;
+
+use crate::feed::{Feed, FeedKind};
+use crate::Result;
+
+use super::Html;
+
+impl Html {
+ pub fn feeds(&self) -> Result<impl Iterator<Item = Feed>> {
+ Ok(self.root.select("link")?.filter_map(|node| {
+ let attributes = node.attributes.borrow();
+ if let (Some(feed_kind), Some(Ok(feed_url))) = (
+ attributes.get("type"),
+ attributes.get("href").map(Url::parse),
+ ) {
+ if let Ok(feed_kind) = FeedKind::from_str(feed_kind) {
+ return Some(Feed {
+ url: feed_url,
+ kind: feed_kind,
+ });
+ }
+ }
+
+ None
+ }))
+ }
+}
diff --git a/crates/core/src/webpage/html/links.rs b/crates/core/src/webpage/html/links.rs
index d889e30d..ffcced11 100644
--- a/crates/core/src/webpage/html/links.rs
+++ b/crates/core/src/webpage/html/links.rs
@@ -322,12 +322,6 @@ impl Html {
}
while let Some((text, attributes)) = open_links.pop() {
- if let Some(rel) = attributes.borrow().get("rel") {
- if rel.contains("nofollow") || rel.contains("sponsored") || rel.contains("ugc") {
- continue;
- }
- }
-
if let Some(dest) = attributes.borrow().get("href") {
if dest.starts_with("mailto:") || dest.starts_with("tel:") {
continue;
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index dfde819c..c0eda2c1 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -31,6 +31,7 @@ use super::url_ext::UrlExt;
pub use fn_cache::FnCache;
+mod feeds;
mod fn_cache;
mod into_tantivy;
pub mod links;
diff --git a/crates/core/src/webpage/url_ext.rs b/crates/core/src/webpage/url_ext.rs
index 84c3df9e..0f4a74ce 100644
--- a/crates/core/src/webpage/url_ext.rs
+++ b/crates/core/src/webpage/url_ext.rs
@@ -33,6 +33,9 @@ pub trait UrlExt {
fn parse_with_base_url(base_url: &url::Url, url: &str) -> Result<url::Url> {
url::Url::parse(url).or_else(|_| base_url.join(url).map_err(|e| e.into()))
}
+ fn robust_parse(url: &str) -> Result<url::Url, url::ParseError> {
+ url::Url::parse(url).or_else(|_| url::Url::parse(&format!("https://{}", url)))
+ }
fn icann_domain(&self) -> Option<&str>;
fn root_domain(&self) -> Option<&str>;
fn normalized_host(&self) -> Option<&str>;