web search done right
[search_server] implement get homepage as GenericQuery
Mikkel Denker846a853969cb
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex 03dbd2f4..135cf9a2 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{collections::HashMap, sync::Arc};+use std::sync::Arc;use tracing::info;use url::Url;@@ -26,7 +26,7 @@ use crate::{member::{Member, Service},sonic::{self, service::sonic_service},},- generic_query::{self, GetWebpageQuery, SizeQuery, TopKeyPhrasesQuery},+ generic_query::{self, GetHomepageQuery, GetWebpageQuery, SizeQuery, TopKeyPhrasesQuery},index::Index,inverted_index,models::dual_encoder::DualEncoder,@@ -116,7 +116,6 @@ macro_rules! impl_search {sonic_service!(SearchService, [RetrieveWebsites,Search,- GetHomepageDescriptions,GetSiteUrls,$($q,@@ -128,7 +127,12 @@ macro_rules! impl_search {}}-impl_search!([TopKeyPhrasesQuery, SizeQuery, GetWebpageQuery]);+impl_search!([+ TopKeyPhrasesQuery,+ SizeQuery,+ GetWebpageQuery,+ GetHomepageQuery,+]);pub struct SearchService {local_searcher: LocalSearcher<Arc<Index>>,@@ -205,28 +209,6 @@ impl sonic::service::Message<SearchService> for Search {}}-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]-pub struct GetHomepageDescriptions {- #[bincode(with_serde)]- pub urls: Vec<Url>,-}-impl sonic::service::Message<SearchService> for GetHomepageDescriptions {- type Response = crate::bincode_utils::SerdeCompat<HashMap<Url, String>>;- async fn handle(self, server: &SearchService) -> Self::Response {- let mut result = HashMap::with_capacity(self.urls.len());-- for url in &self.urls {- if let Some(homepage) = server.local_searcher.get_homepage(url).await {- if let Some(desc) = homepage.description() {- result.insert(url.clone(), desc.clone());- }- }- }-- crate::bincode_utils::SerdeCompat(result)- }-}-pub async fn run(config: config::SearchServerConfig) -> Result<()> {let addr = config.host;let server = SearchService::new(config).await?.bind(addr).await.unwrap();diff --git a/crates/core/src/generic_query/get_homepage.rs b/crates/core/src/generic_query/get_homepage.rsnew file mode 100644index 00000000..e3bbc021--- /dev/null+++ b/crates/core/src/generic_query/get_homepage.rs@@ -0,0 +1,107 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::TantivyDocument;+use url::Url;++use crate::{+ inverted_index::RetrievedWebpage,+ schema::text_field::{self, TextField},+ search_ctx,+ webpage::url_ext::UrlExt,+};++use super::{collector::FirstDocCollector, GenericQuery};++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct GetHomepageQuery {+ pub url: String,+}++impl GetHomepageQuery {+ pub fn new(url: &str) -> Self {+ Self {+ url: url.to_string(),+ }+ }++ fn query(&self, ctx: &search_ctx::Ctx) -> Option<Box<dyn tantivy::query::Query>> {+ let url = Url::robust_parse(&self.url).ok()?;+ let host = url.normalized_host().map(|s| s.to_string())?;++ let field = ctx+ .tv_searcher+ .schema()+ .get_field(text_field::SiteIfHomepageNoTokenizer.name())+ .unwrap();++ let term = tantivy::Term::from_field_text(field, &host);+ Some(Box::new(tantivy::query::TermQuery::new(+ term,+ tantivy::schema::IndexRecordOption::Basic,+ )) as Box<dyn tantivy::query::Query>)+ }+}++impl GenericQuery for GetHomepageQuery {+ type Collector = FirstDocCollector;+ type TantivyQuery = Box<dyn tantivy::query::Query>;+ type IntermediateOutput = Option<RetrievedWebpage>;+ type Output = Option<RetrievedWebpage>;++ fn tantivy_query(&self, ctx: &search_ctx::Ctx) -> Self::TantivyQuery {+ self.query(ctx).unwrap_or_else(|| {+ Box::new(tantivy::query::EmptyQuery) as Box<dyn tantivy::query::Query>+ })+ }++ fn collector(&self, ctx: &search_ctx::Ctx) -> Self::Collector {+ FirstDocCollector::with_shard_id(ctx.shard_id)+ }++ fn remote_collector(&self) -> Self::Collector {+ FirstDocCollector::without_shard_id()+ }++ fn filter_fruit_shards(+ &self,+ shard_id: crate::ampc::dht::ShardId,+ fruit: <Self::Collector as super::Collector>::Fruit,+ ) -> <Self::Collector as super::Collector>::Fruit {+ match fruit {+ Some(doc_address) if doc_address.shard_id == shard_id => Some(doc_address),+ _ => None,+ }+ }++ fn retrieve(+ &self,+ ctx: &search_ctx::Ctx,+ fruit: <Self::Collector as super::Collector>::Fruit,+ ) -> crate::Result<Self::IntermediateOutput> {+ match fruit {+ Some(doc_address) => {+ let doc: TantivyDocument = ctx.tv_searcher.doc(doc_address.into())?;+ Ok(Some(RetrievedWebpage::from(doc)))+ }+ None => Ok(None),+ }+ }++ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {+ results.into_iter().flatten().next()+ }+}diff --git a/crates/core/src/generic_query/mod.rs b/crates/core/src/generic_query/mod.rsindex 28ce77b3..2ecc5365 100644--- a/crates/core/src/generic_query/mod.rs+++ b/crates/core/src/generic_query/mod.rs@@ -25,6 +25,9 @@ pub use size::SizeQuery;pub mod get_webpage;pub use get_webpage::GetWebpageQuery;+pub mod get_homepage;+pub use get_homepage::GetHomepageQuery;+pub mod collector;pub use collector::Collector;diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 3bdf2bff..840f01d4 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -255,12 +255,11 @@ pub struct SearchResult {mod tests {use candle_core::Tensor;use maplit::hashmap;- use url::Url;use crate::{collector::MainCollector,config::CollectorConfig,- generic_query::GetWebpageQuery,+ generic_query::{GetHomepageQuery, GetWebpageQuery},query::Query,ranking::{LocalRanker, SignalComputer},search_ctx::Ctx,@@ -1009,7 +1008,8 @@ mod tests {index.commit().expect("failed to commit index");let webpage = index- .get_homepage(&Url::parse("https://www.example.com").unwrap())+ .search_generic(&GetHomepageQuery::new("https://www.example.com"))+ .unwrap().unwrap();assert_eq!(webpage.title, "News website".to_string());assert_eq!(webpage.url, "https://www.example.com/".to_string());diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rsindex d64adbde..c4a4fe4b 100644--- a/crates/core/src/inverted_index/search.rs+++ b/crates/core/src/inverted_index/search.rs@@ -33,7 +33,7 @@ use crate::numericalfield_reader::NumericalFieldReader;use crate::query::Query;use crate::ranking::pipeline::LocalRecallRankingWebpage;use crate::ranking::SignalComputer;-use crate::schema::{numerical_field, text_field, Field, NumericalFieldEnum, TextFieldEnum};+use crate::schema::{numerical_field, text_field, Field, NumericalFieldEnum};use crate::search_ctx::Ctx;use crate::snippet;use crate::snippet::TextSnippet;@@ -277,37 +277,6 @@ impl InvertedIndex {Ok(RetrievedWebpage::from(doc))}- pub(crate) fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {- let tv_searcher = self.reader.searcher();- let field = tv_searcher- .schema()- .get_field(- Field::Text(TextFieldEnum::from(text_field::SiteIfHomepageNoTokenizer)).name(),- )- .unwrap();-- let host = url.normalized_host().unwrap_or_default();-- let term = tantivy::Term::from_field_text(field, host);-- let query = tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic);-- let mut res = tv_searcher- .search(&query, &tantivy::collector::TopDocs::with_limit(1))- .unwrap();-- res.pop().map(|(_, doc)| {- self.retrieve_doc(- DocAddress::from_tantivy(- doc,- self.shard_id.expect("Shard ID should be set for searches"),- ),- &tv_searcher,- )- .unwrap()- })- }-pub(crate) fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {let ctx = self.local_search_ctx();let tv_searcher = ctx.tv_searcher;diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rsindex ab8ea624..1a44e3c2 100644--- a/crates/core/src/searcher/distributed.rs+++ b/crates/core/src/searcher/distributed.rs@@ -49,7 +49,6 @@ use itertools::Itertools;use std::future::Future;use thiserror::Error;use tokio::sync::Mutex;-use url::Url;use super::{InitialWebsiteResult, LocalSearcher, SearchQuery};@@ -81,11 +80,6 @@ pub trait SearchClient {fn search_entity(&self, query: &str) -> impl Future<Output = Option<EntityMatch>> + Send;- fn get_homepage_descriptions(- &self,- urls: &[Url],- ) -> impl Future<Output = HashMap<Url, String>> + Send;-fn get_entity_image(&self,image_id: &str,@@ -410,33 +404,6 @@ impl SearchClient for DistributedSearcher {retrieved_webpages}- async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {- let client = self.conn().await;-- let res = client- .send(- search_server::GetHomepageDescriptions {- urls: urls.to_vec(),- },- &AllShardsSelector,- &RandomReplicaSelector,- )- .await;-- match res {- Ok(v) => v- .into_iter()- .flatten()- .flat_map(|(_, v)| {- v.into_iter()- .map(|(_, crate::bincode_utils::SerdeCompat(v))| v)- })- .flatten()- .collect(),- _ => HashMap::new(),- }- }-async fn get_entity_image(&self,image_id: &str,@@ -706,23 +673,6 @@ impl SearchClient for LocalSearchClient {res}- async fn get_homepage_descriptions(- &self,- urls: &[url::Url],- ) -> std::collections::HashMap<url::Url, String> {- let mut res = std::collections::HashMap::new();-- for url in urls {- if let Some(homepage) = self.0.get_homepage(url).await {- if let Some(desc) = homepage.description() {- res.insert(url.clone(), desc.clone());- }- }- }-- res- }-async fn get_entity_image(&self,_image_id: &str,diff --git a/crates/core/src/searcher/local/mod.rs b/crates/core/src/searcher/local/mod.rsindex 4cfa36ca..53c69677 100644--- a/crates/core/src/searcher/local/mod.rs+++ b/crates/core/src/searcher/local/mod.rs@@ -31,7 +31,6 @@ use crate::collector::approx_count;use crate::config::CollectorConfig;use crate::generic_query::{self, GenericQuery};use crate::index::Index;-use crate::inverted_index::RetrievedWebpage;use crate::models::dual_encoder::DualEncoder;use crate::ranking::models::linear::LinearRegression;use crate::ranking::pipeline::{@@ -216,10 +215,6 @@ wherecrate::block_on(self.search(query))}- pub async fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {- self.inner.guard().await.inverted_index().get_homepage(url)- }-pub async fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {self.inner.guard()