Skip to content

web search done right

refactor GetSiteUrls into a response stream for reusability

Mikkel Denker9fc02932d980
diff --git a/crates/core/src/distributed/mod.rs b/crates/core/src/distributed/mod.rs
index 74b51726..1bb6b27d 100644
--- a/crates/core/src/distributed/mod.rs
+++ b/crates/core/src/distributed/mod.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
pub mod cluster;
pub mod member;
+pub mod remote_cp;
pub mod retry_strategy;
pub mod sonic;
-pub mod remote_cp;
+pub mod streaming_response;
diff --git a/crates/core/src/distributed/streaming_response.rs b/crates/core/src/distributed/streaming_response.rs
new file mode 100644
index 00000000..4a27b858
--- /dev/null
+++ b/crates/core/src/distributed/streaming_response.rs
@@ -0,0 +1,161 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use futures::{pin_mut, Stream};
+
+use crate::Result;
+use std::{
+ future::Future,
+ pin::Pin,
+ task::{Context, Poll},
+};
+
+pub trait StreamingResponse: Unpin + Sized {
+ type Item: Unpin;
+
+ fn next_batch(&mut self) -> impl Future<Output = Result<Vec<Self::Item>>>;
+
+ fn stream(self) -> impl Stream<Item = Self::Item> {
+ StreamingResponseStream::new(self)
+ }
+}
+
+pub struct StreamingResponseStream<T>
+where
+ T: StreamingResponse,
+{
+ inner: T,
+ batch: Option<Vec<T::Item>>,
+}
+
+impl<T> StreamingResponseStream<T>
+where
+ T: StreamingResponse,
+{
+ fn new(inner: T) -> Self {
+ Self { inner, batch: None }
+ }
+}
+
+impl<T> Stream for StreamingResponseStream<T>
+where
+ T: StreamingResponse,
+{
+ type Item = T::Item;
+
+ fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+ let this = self.get_mut();
+
+ if this.batch.is_none() {
+ let fut = this.inner.next_batch();
+ pin_mut!(fut);
+
+ match fut.poll(cx) {
+ Poll::Pending => return Poll::Pending,
+ Poll::Ready(batch) => match batch {
+ Ok(batch) => {
+ if batch.is_empty() {
+ return Poll::Ready(None);
+ }
+ this.batch = Some(batch);
+ }
+ Err(_) => return Poll::Ready(None),
+ },
+ }
+ }
+
+ match this.batch.as_mut() {
+ Some(batch) => {
+ if batch.is_empty() {
+ let fut = this.inner.next_batch();
+ pin_mut!(fut);
+
+ match fut.poll(cx) {
+ Poll::Pending => return Poll::Pending,
+ Poll::Ready(next_batch) => match next_batch {
+ Ok(next_batch) => {
+ if next_batch.is_empty() {
+ return Poll::Ready(None);
+ }
+
+ batch.extend(next_batch);
+ }
+ Err(_) => return Poll::Ready(None),
+ },
+ }
+ }
+
+ Poll::Ready(batch.pop())
+ }
+ None => Poll::Ready(None),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use futures::StreamExt;
+
+ use super::*;
+
+ struct TestStreamingResponse {
+ items: Vec<String>,
+ index: usize,
+ }
+
+ impl TestStreamingResponse {
+ fn new(items: Vec<String>) -> Self {
+ Self { items, index: 0 }
+ }
+ }
+
+ impl StreamingResponse for TestStreamingResponse {
+ type Item = String;
+
+ async fn next_batch(&mut self) -> Result<Vec<Self::Item>> {
+ if self.index >= self.items.len() {
+ return Ok(Vec::new());
+ }
+
+ let mut res = Vec::new();
+
+ res.push(self.items[self.index].clone());
+
+ self.index += 1;
+
+ Ok(res)
+ }
+ }
+
+ #[tokio::test]
+ async fn test_streaming_response_stream() {
+ let mut stream =
+ TestStreamingResponse::new(vec!["a".to_string(), "b".to_string(), "c".to_string()])
+ .stream();
+
+ assert_eq!(stream.next().await, Some("a".to_string()));
+ assert_eq!(stream.next().await, Some("b".to_string()));
+ assert_eq!(stream.next().await, Some("c".to_string()));
+ assert_eq!(stream.next().await, None);
+ }
+
+ #[tokio::test]
+ async fn test_empty_stream() {
+ let mut stream = TestStreamingResponse::new(Vec::new()).stream();
+
+ assert_eq!(stream.next().await, None);
+ }
+}
diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rs
index c0e598a8..ab07c2ad 100644
--- a/crates/core/src/entrypoint/live_index/search_server.rs
+++ b/crates/core/src/entrypoint/live_index/search_server.rs
@@ -343,9 +343,11 @@ impl sonic::service::Message<LiveIndexService> for Search {
impl sonic::service::Message<LiveIndexService> for GetSiteUrls {
type Response = SiteUrls;
async fn handle(self, server: &LiveIndexService) -> Self::Response {
- let urls = server
- .local_searcher
- .get_site_urls(&self.site, self.offset, self.limit);
+ let urls = server.local_searcher.get_site_urls(
+ &self.site,
+ self.offset as usize,
+ self.limit as usize,
+ );
SiteUrls { urls }
}
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rs
index f7e92786..9316f633 100644
--- a/crates/core/src/entrypoint/search_server.rs
+++ b/crates/core/src/entrypoint/search_server.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -190,8 +190,8 @@ impl sonic::service::Message<SearchService> for Size {
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct GetSiteUrls {
pub site: String,
- pub offset: usize,
- pub limit: usize,
+ pub offset: u64,
+ pub limit: u64,
}
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
@@ -203,9 +203,11 @@ pub struct SiteUrls {
impl sonic::service::Message<SearchService> for GetSiteUrls {
type Response = SiteUrls;
async fn handle(self, server: &SearchService) -> Self::Response {
- let urls = server
- .local_searcher
- .get_site_urls(&self.site, self.offset, self.limit);
+ let urls = server.local_searcher.get_site_urls(
+ &self.site,
+ self.offset as usize,
+ self.limit as usize,
+ );
SiteUrls { urls }
}
diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rs
index f1209505..c003f053 100644
--- a/crates/core/src/live_index/crawler/mod.rs
+++ b/crates/core/src/live_index/crawler/mod.rs
@@ -18,6 +18,7 @@ mod budgets;
mod checker;
mod crawlable_site;
mod crawled_db;
+mod site_url_stream;
use std::fs::File;
use std::path::PathBuf;
use std::sync::Arc;
@@ -27,8 +28,9 @@ use crate::ampc::dht::ShardId;
use crate::config::{CheckIntervals, CrawlerConfig, LiveCrawlerConfig};
use crate::distributed::cluster::Cluster;
use crate::distributed::sonic::replication::{
- AllShardsSelector, RandomReplicaSelector, RandomShardSelector, ShardedClient,
+ RandomReplicaSelector, RandomShardSelector, ShardedClient,
};
+use crate::distributed::streaming_response::StreamingResponse;
use crate::entrypoint::search_server;
use crate::entrypoint::site_stats::FinalSiteStats;
use crate::{crawler, Result};
@@ -38,11 +40,12 @@ use crate::{
};
use crawlable_site::{CrawlableSite, CrawlableSiteGuard};
use crawled_db::ShardedCrawledDb;
+use futures::StreamExt;
+use site_url_stream::SiteUrlStream;
use tokio::sync::Mutex;
use tokio::sync::Semaphore;
use url::Url;
-const SITE_URL_BATCH_SIZE: usize = 100;
const DEFAULT_CONSISTENCY_FRACTION: f64 = 0.5;
const BLOG_FRACTION_THRESHOLD: f64 = 0.5;
const NEWS_FRACTION_THRESHOLD: f64 = 0.5;
@@ -99,82 +102,36 @@ impl Client {
Ok(())
}
- async fn get_site_urls_from_backbone(&self, site: &str) -> Result<Vec<Url>> {
+ async fn get_site_urls_from_backbone(&self, site: &str) -> Vec<Url> {
let mut res = Vec::new();
let conn = self.search_conn().await;
- let mut offset = 0;
-
- while let Ok(resp) = conn
- .send(
- search_server::GetSiteUrls {
- site: site.to_string(),
- offset,
- limit: SITE_URL_BATCH_SIZE,
- },
- &AllShardsSelector,
- &RandomReplicaSelector,
- )
- .await
- {
- let urls: Vec<_> = resp
- .into_iter()
- .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))
- .collect();
+ let mut stream = SiteUrlStream::new(site.to_string(), conn).stream();
- if urls.is_empty() {
- break;
- }
-
- res.extend(urls);
- offset += SITE_URL_BATCH_SIZE;
+ while let Some(url) = stream.next().await {
+ res.push(url);
}
- Ok(res)
+ res
}
- async fn get_site_urls_from_live(&self, site: &str) -> Result<Vec<Url>> {
+ async fn get_site_urls_from_live(&self, site: &str) -> Vec<Url> {
let mut res = Vec::new();
let conn = self.live_conn().await;
- let mut offset = 0;
-
- while let Ok(resp) = conn
- .send(
- search_server::GetSiteUrls {
- site: site.to_string(),
- offset,
- limit: SITE_URL_BATCH_SIZE,
- },
- &AllShardsSelector,
- &RandomReplicaSelector,
- )
- .await
- {
- let urls: Vec<_> = resp
- .into_iter()
- .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))
- .collect();
- if urls.is_empty() {
- break;
- }
+ let mut stream = SiteUrlStream::new(site.to_string(), conn).stream();
- res.extend(urls);
- offset += SITE_URL_BATCH_SIZE;
+ while let Some(url) = stream.next().await {
+ res.push(url);
}
- Ok(res)
+ res
}
pub async fn get_site_urls(&self, site: &str) -> Vec<Url> {
let mut res = Vec::new();
- if let Ok(urls) = self.get_site_urls_from_backbone(site).await {
- res.extend(urls);
- }
-
- if let Ok(urls) = self.get_site_urls_from_live(site).await {
- res.extend(urls);
- }
+ res.extend(self.get_site_urls_from_backbone(site).await);
+ res.extend(self.get_site_urls_from_live(site).await);
res
}
diff --git a/crates/core/src/live_index/crawler/site_url_stream.rs b/crates/core/src/live_index/crawler/site_url_stream.rs
new file mode 100644
index 00000000..3e8d307c
--- /dev/null
+++ b/crates/core/src/live_index/crawler/site_url_stream.rs
@@ -0,0 +1,94 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+const SITE_URL_BATCH_SIZE: usize = 100;
+
+use std::sync::Arc;
+
+use url::Url;
+
+use crate::{
+ ampc::dht::ShardId,
+ distributed::{
+ sonic::{
+ replication::{AllShardsSelector, RandomReplicaSelector, ShardedClient},
+ service::Service,
+ },
+ streaming_response::StreamingResponse,
+ },
+ entrypoint::{live_index, search_server},
+ Result,
+};
+
+pub struct SiteUrlStream<S: Service> {
+ site: String,
+ offset: usize,
+ conn: Arc<ShardedClient<S, ShardId>>,
+}
+
+impl<S: Service> SiteUrlStream<S> {
+ pub fn new(site: String, conn: Arc<ShardedClient<S, ShardId>>) -> Self {
+ Self {
+ site,
+ offset: 0,
+ conn,
+ }
+ }
+}
+
+impl StreamingResponse for SiteUrlStream<search_server::SearchService> {
+ type Item = Url;
+
+ async fn next_batch(&mut self) -> Result<Vec<Self::Item>> {
+ let req = search_server::GetSiteUrls {
+ site: self.site.clone(),
+ offset: self.offset as u64,
+ limit: SITE_URL_BATCH_SIZE as u64,
+ };
+
+ let res = self
+ .conn
+ .send(req, &AllShardsSelector, &RandomReplicaSelector)
+ .await?;
+
+ Ok(res
+ .into_iter()
+ .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))
+ .collect())
+ }
+}
+
+impl StreamingResponse for SiteUrlStream<live_index::LiveIndexService> {
+ type Item = Url;
+
+ async fn next_batch(&mut self) -> Result<Vec<Self::Item>> {
+ let req = search_server::GetSiteUrls {
+ site: self.site.clone(),
+ offset: self.offset as u64,
+ limit: SITE_URL_BATCH_SIZE as u64,
+ };
+
+ let res = self
+ .conn
+ .send(req, &AllShardsSelector, &RandomReplicaSelector)
+ .await?;
+
+ Ok(res
+ .into_iter()
+ .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))
+ .collect())
+ }
+}