web search done right
refactor GetSiteUrls into a response stream for reusability
Mikkel Denker9fc02932d980
diff --git a/crates/core/src/distributed/mod.rs b/crates/core/src/distributed/mod.rsindex 74b51726..1bb6b27d 100644--- a/crates/core/src/distributed/mod.rs+++ b/crates/core/src/distributed/mod.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.pub mod cluster;pub mod member;+pub mod remote_cp;pub mod retry_strategy;pub mod sonic;-pub mod remote_cp;+pub mod streaming_response;diff --git a/crates/core/src/distributed/streaming_response.rs b/crates/core/src/distributed/streaming_response.rsnew file mode 100644index 00000000..4a27b858--- /dev/null+++ b/crates/core/src/distributed/streaming_response.rs@@ -0,0 +1,161 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use futures::{pin_mut, Stream};++use crate::Result;+use std::{+ future::Future,+ pin::Pin,+ task::{Context, Poll},+};++pub trait StreamingResponse: Unpin + Sized {+ type Item: Unpin;++ fn next_batch(&mut self) -> impl Future<Output = Result<Vec<Self::Item>>>;++ fn stream(self) -> impl Stream<Item = Self::Item> {+ StreamingResponseStream::new(self)+ }+}++pub struct StreamingResponseStream<T>+where+ T: StreamingResponse,+{+ inner: T,+ batch: Option<Vec<T::Item>>,+}++impl<T> StreamingResponseStream<T>+where+ T: StreamingResponse,+{+ fn new(inner: T) -> Self {+ Self { inner, batch: None }+ }+}++impl<T> Stream for StreamingResponseStream<T>+where+ T: StreamingResponse,+{+ type Item = T::Item;++ fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {+ let this = self.get_mut();++ if this.batch.is_none() {+ let fut = this.inner.next_batch();+ pin_mut!(fut);++ match fut.poll(cx) {+ Poll::Pending => return Poll::Pending,+ Poll::Ready(batch) => match batch {+ Ok(batch) => {+ if batch.is_empty() {+ return Poll::Ready(None);+ }+ this.batch = Some(batch);+ }+ Err(_) => return Poll::Ready(None),+ },+ }+ }++ match this.batch.as_mut() {+ Some(batch) => {+ if batch.is_empty() {+ let fut = this.inner.next_batch();+ pin_mut!(fut);++ match fut.poll(cx) {+ Poll::Pending => return Poll::Pending,+ Poll::Ready(next_batch) => match next_batch {+ Ok(next_batch) => {+ if next_batch.is_empty() {+ return Poll::Ready(None);+ }++ batch.extend(next_batch);+ }+ Err(_) => return Poll::Ready(None),+ },+ }+ }++ Poll::Ready(batch.pop())+ }+ None => Poll::Ready(None),+ }+ }+}++#[cfg(test)]+mod tests {+ use futures::StreamExt;++ use super::*;++ struct TestStreamingResponse {+ items: Vec<String>,+ index: usize,+ }++ impl TestStreamingResponse {+ fn new(items: Vec<String>) -> Self {+ Self { items, index: 0 }+ }+ }++ impl StreamingResponse for TestStreamingResponse {+ type Item = String;++ async fn next_batch(&mut self) -> Result<Vec<Self::Item>> {+ if self.index >= self.items.len() {+ return Ok(Vec::new());+ }++ let mut res = Vec::new();++ res.push(self.items[self.index].clone());++ self.index += 1;++ Ok(res)+ }+ }++ #[tokio::test]+ async fn test_streaming_response_stream() {+ let mut stream =+ TestStreamingResponse::new(vec!["a".to_string(), "b".to_string(), "c".to_string()])+ .stream();++ assert_eq!(stream.next().await, Some("a".to_string()));+ assert_eq!(stream.next().await, Some("b".to_string()));+ assert_eq!(stream.next().await, Some("c".to_string()));+ assert_eq!(stream.next().await, None);+ }++ #[tokio::test]+ async fn test_empty_stream() {+ let mut stream = TestStreamingResponse::new(Vec::new()).stream();++ assert_eq!(stream.next().await, None);+ }+}diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rsindex c0e598a8..ab07c2ad 100644--- a/crates/core/src/entrypoint/live_index/search_server.rs+++ b/crates/core/src/entrypoint/live_index/search_server.rs@@ -343,9 +343,11 @@ impl sonic::service::Message<LiveIndexService> for Search {impl sonic::service::Message<LiveIndexService> for GetSiteUrls {type Response = SiteUrls;async fn handle(self, server: &LiveIndexService) -> Self::Response {- let urls = server- .local_searcher- .get_site_urls(&self.site, self.offset, self.limit);+ let urls = server.local_searcher.get_site_urls(+ &self.site,+ self.offset as usize,+ self.limit as usize,+ );SiteUrls { urls }}diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex f7e92786..9316f633 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -190,8 +190,8 @@ impl sonic::service::Message<SearchService> for Size {#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct GetSiteUrls {pub site: String,- pub offset: usize,- pub limit: usize,+ pub offset: u64,+ pub limit: u64,}#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]@@ -203,9 +203,11 @@ pub struct SiteUrls {impl sonic::service::Message<SearchService> for GetSiteUrls {type Response = SiteUrls;async fn handle(self, server: &SearchService) -> Self::Response {- let urls = server- .local_searcher- .get_site_urls(&self.site, self.offset, self.limit);+ let urls = server.local_searcher.get_site_urls(+ &self.site,+ self.offset as usize,+ self.limit as usize,+ );SiteUrls { urls }}diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rsindex f1209505..c003f053 100644--- a/crates/core/src/live_index/crawler/mod.rs+++ b/crates/core/src/live_index/crawler/mod.rs@@ -18,6 +18,7 @@ mod budgets;mod checker;mod crawlable_site;mod crawled_db;+mod site_url_stream;use std::fs::File;use std::path::PathBuf;use std::sync::Arc;@@ -27,8 +28,9 @@ use crate::ampc::dht::ShardId;use crate::config::{CheckIntervals, CrawlerConfig, LiveCrawlerConfig};use crate::distributed::cluster::Cluster;use crate::distributed::sonic::replication::{- AllShardsSelector, RandomReplicaSelector, RandomShardSelector, ShardedClient,+ RandomReplicaSelector, RandomShardSelector, ShardedClient,};+use crate::distributed::streaming_response::StreamingResponse;use crate::entrypoint::search_server;use crate::entrypoint::site_stats::FinalSiteStats;use crate::{crawler, Result};@@ -38,11 +40,12 @@ use crate::{};use crawlable_site::{CrawlableSite, CrawlableSiteGuard};use crawled_db::ShardedCrawledDb;+use futures::StreamExt;+use site_url_stream::SiteUrlStream;use tokio::sync::Mutex;use tokio::sync::Semaphore;use url::Url;-const SITE_URL_BATCH_SIZE: usize = 100;const DEFAULT_CONSISTENCY_FRACTION: f64 = 0.5;const BLOG_FRACTION_THRESHOLD: f64 = 0.5;const NEWS_FRACTION_THRESHOLD: f64 = 0.5;@@ -99,82 +102,36 @@ impl Client {Ok(())}- async fn get_site_urls_from_backbone(&self, site: &str) -> Result<Vec<Url>> {+ async fn get_site_urls_from_backbone(&self, site: &str) -> Vec<Url> {let mut res = Vec::new();let conn = self.search_conn().await;- let mut offset = 0;-- while let Ok(resp) = conn- .send(- search_server::GetSiteUrls {- site: site.to_string(),- offset,- limit: SITE_URL_BATCH_SIZE,- },- &AllShardsSelector,- &RandomReplicaSelector,- )- .await- {- let urls: Vec<_> = resp- .into_iter()- .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))- .collect();+ let mut stream = SiteUrlStream::new(site.to_string(), conn).stream();- if urls.is_empty() {- break;- }-- res.extend(urls);- offset += SITE_URL_BATCH_SIZE;+ while let Some(url) = stream.next().await {+ res.push(url);}- Ok(res)+ res}- async fn get_site_urls_from_live(&self, site: &str) -> Result<Vec<Url>> {+ async fn get_site_urls_from_live(&self, site: &str) -> Vec<Url> {let mut res = Vec::new();let conn = self.live_conn().await;- let mut offset = 0;-- while let Ok(resp) = conn- .send(- search_server::GetSiteUrls {- site: site.to_string(),- offset,- limit: SITE_URL_BATCH_SIZE,- },- &AllShardsSelector,- &RandomReplicaSelector,- )- .await- {- let urls: Vec<_> = resp- .into_iter()- .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))- .collect();- if urls.is_empty() {- break;- }+ let mut stream = SiteUrlStream::new(site.to_string(), conn).stream();- res.extend(urls);- offset += SITE_URL_BATCH_SIZE;+ while let Some(url) = stream.next().await {+ res.push(url);}- Ok(res)+ res}pub async fn get_site_urls(&self, site: &str) -> Vec<Url> {let mut res = Vec::new();- if let Ok(urls) = self.get_site_urls_from_backbone(site).await {- res.extend(urls);- }-- if let Ok(urls) = self.get_site_urls_from_live(site).await {- res.extend(urls);- }+ res.extend(self.get_site_urls_from_backbone(site).await);+ res.extend(self.get_site_urls_from_live(site).await);res}diff --git a/crates/core/src/live_index/crawler/site_url_stream.rs b/crates/core/src/live_index/crawler/site_url_stream.rsnew file mode 100644index 00000000..3e8d307c--- /dev/null+++ b/crates/core/src/live_index/crawler/site_url_stream.rs@@ -0,0 +1,94 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++const SITE_URL_BATCH_SIZE: usize = 100;++use std::sync::Arc;++use url::Url;++use crate::{+ ampc::dht::ShardId,+ distributed::{+ sonic::{+ replication::{AllShardsSelector, RandomReplicaSelector, ShardedClient},+ service::Service,+ },+ streaming_response::StreamingResponse,+ },+ entrypoint::{live_index, search_server},+ Result,+};++pub struct SiteUrlStream<S: Service> {+ site: String,+ offset: usize,+ conn: Arc<ShardedClient<S, ShardId>>,+}++impl<S: Service> SiteUrlStream<S> {+ pub fn new(site: String, conn: Arc<ShardedClient<S, ShardId>>) -> Self {+ Self {+ site,+ offset: 0,+ conn,+ }+ }+}++impl StreamingResponse for SiteUrlStream<search_server::SearchService> {+ type Item = Url;++ async fn next_batch(&mut self) -> Result<Vec<Self::Item>> {+ let req = search_server::GetSiteUrls {+ site: self.site.clone(),+ offset: self.offset as u64,+ limit: SITE_URL_BATCH_SIZE as u64,+ };++ let res = self+ .conn+ .send(req, &AllShardsSelector, &RandomReplicaSelector)+ .await?;++ Ok(res+ .into_iter()+ .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))+ .collect())+ }+}++impl StreamingResponse for SiteUrlStream<live_index::LiveIndexService> {+ type Item = Url;++ async fn next_batch(&mut self) -> Result<Vec<Self::Item>> {+ let req = search_server::GetSiteUrls {+ site: self.site.clone(),+ offset: self.offset as u64,+ limit: SITE_URL_BATCH_SIZE as u64,+ };++ let res = self+ .conn+ .send(req, &AllShardsSelector, &RandomReplicaSelector)+ .await?;++ Ok(res+ .into_iter()+ .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))+ .collect())+ }+}