Skip to content

web search done right

[search_server] implement get homepage as GenericQuery

Mikkel Denker846a853969cb
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rs
index 03dbd2f4..135cf9a2 100644
--- a/crates/core/src/entrypoint/search_server.rs
+++ b/crates/core/src/entrypoint/search_server.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{collections::HashMap, sync::Arc};
+use std::sync::Arc;
use tracing::info;
use url::Url;
@@ -26,7 +26,7 @@ use crate::{
member::{Member, Service},
sonic::{self, service::sonic_service},
},
- generic_query::{self, GetWebpageQuery, SizeQuery, TopKeyPhrasesQuery},
+ generic_query::{self, GetHomepageQuery, GetWebpageQuery, SizeQuery, TopKeyPhrasesQuery},
index::Index,
inverted_index,
models::dual_encoder::DualEncoder,
@@ -116,7 +116,6 @@ macro_rules! impl_search {
sonic_service!(SearchService, [
RetrieveWebsites,
Search,
- GetHomepageDescriptions,
GetSiteUrls,
$(
$q,
@@ -128,7 +127,12 @@ macro_rules! impl_search {
}
}
-impl_search!([TopKeyPhrasesQuery, SizeQuery, GetWebpageQuery]);
+impl_search!([
+ TopKeyPhrasesQuery,
+ SizeQuery,
+ GetWebpageQuery,
+ GetHomepageQuery,
+]);
pub struct SearchService {
local_searcher: LocalSearcher<Arc<Index>>,
@@ -205,28 +209,6 @@ impl sonic::service::Message<SearchService> for Search {
}
}
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct GetHomepageDescriptions {
- #[bincode(with_serde)]
- pub urls: Vec<Url>,
-}
-impl sonic::service::Message<SearchService> for GetHomepageDescriptions {
- type Response = crate::bincode_utils::SerdeCompat<HashMap<Url, String>>;
- async fn handle(self, server: &SearchService) -> Self::Response {
- let mut result = HashMap::with_capacity(self.urls.len());
-
- for url in &self.urls {
- if let Some(homepage) = server.local_searcher.get_homepage(url).await {
- if let Some(desc) = homepage.description() {
- result.insert(url.clone(), desc.clone());
- }
- }
- }
-
- crate::bincode_utils::SerdeCompat(result)
- }
-}
-
pub async fn run(config: config::SearchServerConfig) -> Result<()> {
let addr = config.host;
let server = SearchService::new(config).await?.bind(addr).await.unwrap();
diff --git a/crates/core/src/generic_query/get_homepage.rs b/crates/core/src/generic_query/get_homepage.rs
new file mode 100644
index 00000000..e3bbc021
--- /dev/null
+++ b/crates/core/src/generic_query/get_homepage.rs
@@ -0,0 +1,107 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::TantivyDocument;
+use url::Url;
+
+use crate::{
+ inverted_index::RetrievedWebpage,
+ schema::text_field::{self, TextField},
+ search_ctx,
+ webpage::url_ext::UrlExt,
+};
+
+use super::{collector::FirstDocCollector, GenericQuery};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct GetHomepageQuery {
+ pub url: String,
+}
+
+impl GetHomepageQuery {
+ pub fn new(url: &str) -> Self {
+ Self {
+ url: url.to_string(),
+ }
+ }
+
+ fn query(&self, ctx: &search_ctx::Ctx) -> Option<Box<dyn tantivy::query::Query>> {
+ let url = Url::robust_parse(&self.url).ok()?;
+ let host = url.normalized_host().map(|s| s.to_string())?;
+
+ let field = ctx
+ .tv_searcher
+ .schema()
+ .get_field(text_field::SiteIfHomepageNoTokenizer.name())
+ .unwrap();
+
+ let term = tantivy::Term::from_field_text(field, &host);
+ Some(Box::new(tantivy::query::TermQuery::new(
+ term,
+ tantivy::schema::IndexRecordOption::Basic,
+ )) as Box<dyn tantivy::query::Query>)
+ }
+}
+
+impl GenericQuery for GetHomepageQuery {
+ type Collector = FirstDocCollector;
+ type TantivyQuery = Box<dyn tantivy::query::Query>;
+ type IntermediateOutput = Option<RetrievedWebpage>;
+ type Output = Option<RetrievedWebpage>;
+
+ fn tantivy_query(&self, ctx: &search_ctx::Ctx) -> Self::TantivyQuery {
+ self.query(ctx).unwrap_or_else(|| {
+ Box::new(tantivy::query::EmptyQuery) as Box<dyn tantivy::query::Query>
+ })
+ }
+
+ fn collector(&self, ctx: &search_ctx::Ctx) -> Self::Collector {
+ FirstDocCollector::with_shard_id(ctx.shard_id)
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ FirstDocCollector::without_shard_id()
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: crate::ampc::dht::ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ match fruit {
+ Some(doc_address) if doc_address.shard_id == shard_id => Some(doc_address),
+ _ => None,
+ }
+ }
+
+ fn retrieve(
+ &self,
+ ctx: &search_ctx::Ctx,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> crate::Result<Self::IntermediateOutput> {
+ match fruit {
+ Some(doc_address) => {
+ let doc: TantivyDocument = ctx.tv_searcher.doc(doc_address.into())?;
+ Ok(Some(RetrievedWebpage::from(doc)))
+ }
+ None => Ok(None),
+ }
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ results.into_iter().flatten().next()
+ }
+}
diff --git a/crates/core/src/generic_query/mod.rs b/crates/core/src/generic_query/mod.rs
index 28ce77b3..2ecc5365 100644
--- a/crates/core/src/generic_query/mod.rs
+++ b/crates/core/src/generic_query/mod.rs
@@ -25,6 +25,9 @@ pub use size::SizeQuery;
pub mod get_webpage;
pub use get_webpage::GetWebpageQuery;
+pub mod get_homepage;
+pub use get_homepage::GetHomepageQuery;
+
pub mod collector;
pub use collector::Collector;
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 3bdf2bff..840f01d4 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -255,12 +255,11 @@ pub struct SearchResult {
mod tests {
use candle_core::Tensor;
use maplit::hashmap;
- use url::Url;
use crate::{
collector::MainCollector,
config::CollectorConfig,
- generic_query::GetWebpageQuery,
+ generic_query::{GetHomepageQuery, GetWebpageQuery},
query::Query,
ranking::{LocalRanker, SignalComputer},
search_ctx::Ctx,
@@ -1009,7 +1008,8 @@ mod tests {
index.commit().expect("failed to commit index");
let webpage = index
- .get_homepage(&Url::parse("https://www.example.com").unwrap())
+ .search_generic(&GetHomepageQuery::new("https://www.example.com"))
+ .unwrap()
.unwrap();
assert_eq!(webpage.title, "News website".to_string());
assert_eq!(webpage.url, "https://www.example.com/".to_string());
diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rs
index d64adbde..c4a4fe4b 100644
--- a/crates/core/src/inverted_index/search.rs
+++ b/crates/core/src/inverted_index/search.rs
@@ -33,7 +33,7 @@ use crate::numericalfield_reader::NumericalFieldReader;
use crate::query::Query;
use crate::ranking::pipeline::LocalRecallRankingWebpage;
use crate::ranking::SignalComputer;
-use crate::schema::{numerical_field, text_field, Field, NumericalFieldEnum, TextFieldEnum};
+use crate::schema::{numerical_field, text_field, Field, NumericalFieldEnum};
use crate::search_ctx::Ctx;
use crate::snippet;
use crate::snippet::TextSnippet;
@@ -277,37 +277,6 @@ impl InvertedIndex {
Ok(RetrievedWebpage::from(doc))
}
- pub(crate) fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {
- let tv_searcher = self.reader.searcher();
- let field = tv_searcher
- .schema()
- .get_field(
- Field::Text(TextFieldEnum::from(text_field::SiteIfHomepageNoTokenizer)).name(),
- )
- .unwrap();
-
- let host = url.normalized_host().unwrap_or_default();
-
- let term = tantivy::Term::from_field_text(field, host);
-
- let query = tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic);
-
- let mut res = tv_searcher
- .search(&query, &tantivy::collector::TopDocs::with_limit(1))
- .unwrap();
-
- res.pop().map(|(_, doc)| {
- self.retrieve_doc(
- DocAddress::from_tantivy(
- doc,
- self.shard_id.expect("Shard ID should be set for searches"),
- ),
- &tv_searcher,
- )
- .unwrap()
- })
- }
-
pub(crate) fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {
let ctx = self.local_search_ctx();
let tv_searcher = ctx.tv_searcher;
diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rs
index ab8ea624..1a44e3c2 100644
--- a/crates/core/src/searcher/distributed.rs
+++ b/crates/core/src/searcher/distributed.rs
@@ -49,7 +49,6 @@ use itertools::Itertools;
use std::future::Future;
use thiserror::Error;
use tokio::sync::Mutex;
-use url::Url;
use super::{InitialWebsiteResult, LocalSearcher, SearchQuery};
@@ -81,11 +80,6 @@ pub trait SearchClient {
fn search_entity(&self, query: &str) -> impl Future<Output = Option<EntityMatch>> + Send;
- fn get_homepage_descriptions(
- &self,
- urls: &[Url],
- ) -> impl Future<Output = HashMap<Url, String>> + Send;
-
fn get_entity_image(
&self,
image_id: &str,
@@ -410,33 +404,6 @@ impl SearchClient for DistributedSearcher {
retrieved_webpages
}
- async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {
- let client = self.conn().await;
-
- let res = client
- .send(
- search_server::GetHomepageDescriptions {
- urls: urls.to_vec(),
- },
- &AllShardsSelector,
- &RandomReplicaSelector,
- )
- .await;
-
- match res {
- Ok(v) => v
- .into_iter()
- .flatten()
- .flat_map(|(_, v)| {
- v.into_iter()
- .map(|(_, crate::bincode_utils::SerdeCompat(v))| v)
- })
- .flatten()
- .collect(),
- _ => HashMap::new(),
- }
- }
-
async fn get_entity_image(
&self,
image_id: &str,
@@ -706,23 +673,6 @@ impl SearchClient for LocalSearchClient {
res
}
- async fn get_homepage_descriptions(
- &self,
- urls: &[url::Url],
- ) -> std::collections::HashMap<url::Url, String> {
- let mut res = std::collections::HashMap::new();
-
- for url in urls {
- if let Some(homepage) = self.0.get_homepage(url).await {
- if let Some(desc) = homepage.description() {
- res.insert(url.clone(), desc.clone());
- }
- }
- }
-
- res
- }
-
async fn get_entity_image(
&self,
_image_id: &str,
diff --git a/crates/core/src/searcher/local/mod.rs b/crates/core/src/searcher/local/mod.rs
index 4cfa36ca..53c69677 100644
--- a/crates/core/src/searcher/local/mod.rs
+++ b/crates/core/src/searcher/local/mod.rs
@@ -31,7 +31,6 @@ use crate::collector::approx_count;
use crate::config::CollectorConfig;
use crate::generic_query::{self, GenericQuery};
use crate::index::Index;
-use crate::inverted_index::RetrievedWebpage;
use crate::models::dual_encoder::DualEncoder;
use crate::ranking::models::linear::LinearRegression;
use crate::ranking::pipeline::{
@@ -216,10 +215,6 @@ where
crate::block_on(self.search(query))
}
- pub async fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {
- self.inner.guard().await.inverted_index().get_homepage(url)
- }
-
pub async fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {
self.inner
.guard()