web search done right
[search_server] implement get webpage as GenericQuery
Mikkel Denker2f14ed48980c
diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rsindex e92dc896..6d4340cb 100644--- a/crates/core/src/collector/top_docs.rs+++ b/crates/core/src/collector/top_docs.rs@@ -25,6 +25,7 @@ use tantivy::{};use crate::{+ ampc::dht::ShardId,config::CollectorConfig,inverted_index::{DocAddress, WebpagePointer},numericalfield_reader,@@ -43,6 +44,7 @@ pub struct TopDocs {columnfield_reader: numericalfield_reader::NumericalFieldReader,de_rank_similar: bool,collector_config: CollectorConfig,+ shard_id: Option<ShardId>,}impl TopDocs {@@ -57,6 +59,7 @@ impl TopDocs {de_rank_similar: false,columnfield_reader,collector_config: CollectorConfig::default(),+ shard_id: None,}}@@ -84,6 +87,11 @@ impl TopDocs {self}+ pub fn and_shard_id(mut self, shard_id: ShardId) -> Self {+ self.shard_id = Some(shard_id);+ self+ }+pub fn main_collector(self, score_tweaker: InitialScoreTweaker) -> MainCollector {MainCollector::new(score_tweaker, self)}@@ -112,6 +120,7 @@ impl TopDocs {self.top_n + self.offset,self.collector_config.clone(),),+ shard_id: self.shard_id,})}}@@ -122,6 +131,7 @@ pub struct TopSegmentCollector {num_docs_taken: usize,segment_ord: SegmentOrdinal,bucket_collector: BucketCollector<SegmentDoc>,+ shard_id: Option<ShardId>,}impl TopSegmentCollector {@@ -190,6 +200,7 @@ impl TopSegmentCollector {},id: doc,segment: self.segment_ord,+ shard_id: self.shard_id.expect("Shard ID should be set for searches"),score,});}@@ -357,6 +368,7 @@ pub struct SegmentDoc {id: DocId,segment: SegmentOrdinal,score: Score,+ shard_id: ShardId,}impl Doc for SegmentDoc {@@ -443,10 +455,7 @@ where.map(|doc| WebpagePointer {score: doc.score,hashes: doc.hashes,- address: DocAddress {- segment: doc.segment,- doc_id: doc.id,- },+ address: DocAddress::new(doc.segment, doc.id, doc.shard_id),}).collect())}@@ -494,6 +503,7 @@ mod tests {id: doc.1,score: Score { total: doc.2 },segment: 0,+ shard_id: ShardId::new(0),});}diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex 61b6b916..03dbd2f4 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -26,9 +26,9 @@ use crate::{member::{Member, Service},sonic::{self, service::sonic_service},},- generic_query::{self, SizeQuery, TopKeyPhrasesQuery},+ generic_query::{self, GetWebpageQuery, SizeQuery, TopKeyPhrasesQuery},index::Index,- inverted_index::{self, RetrievedWebpage},+ inverted_index,models::dual_encoder::DualEncoder,ranking::models::linear::LinearRegression,searcher::{InitialWebsiteResult, LocalSearcher, SearchQuery},@@ -116,7 +116,6 @@ macro_rules! impl_search {sonic_service!(SearchService, [RetrieveWebsites,Search,- GetWebpage,GetHomepageDescriptions,GetSiteUrls,$(@@ -129,7 +128,7 @@ macro_rules! impl_search {}}-impl_search!([TopKeyPhrasesQuery, SizeQuery,]);+impl_search!([TopKeyPhrasesQuery, SizeQuery, GetWebpageQuery]);pub struct SearchService {local_searcher: LocalSearcher<Arc<Index>>,@@ -206,17 +205,6 @@ impl sonic::service::Message<SearchService> for Search {}}-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]-pub struct GetWebpage {- pub url: String,-}-impl sonic::service::Message<SearchService> for GetWebpage {- type Response = Option<RetrievedWebpage>;- async fn handle(self, server: &SearchService) -> Self::Response {- server.local_searcher.get_webpage(&self.url).await- }-}-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct GetHomepageDescriptions {#[bincode(with_serde)]diff --git a/crates/core/src/generic_query/collector/first_doc.rs b/crates/core/src/generic_query/collector/first_doc.rsnew file mode 100644index 00000000..b4160fbe--- /dev/null+++ b/crates/core/src/generic_query/collector/first_doc.rs@@ -0,0 +1,89 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::{collector::SegmentCollector, DocId, SegmentOrdinal};++use super::Collector;+use crate::{ampc::dht::ShardId, inverted_index::DocAddress};++pub struct FirstDocCollector {+ shard_id: Option<ShardId>,+}++impl FirstDocCollector {+ pub fn with_shard_id(shard_id: ShardId) -> Self {+ Self {+ shard_id: Some(shard_id),+ }+ }++ pub fn without_shard_id() -> Self {+ Self { shard_id: None }+ }+}++impl Collector for FirstDocCollector {+ type Fruit = Option<DocAddress>;+ type Child = FirstDocSegmentCollector;++ fn for_segment(+ &self,+ segment_id: SegmentOrdinal,+ _: &tantivy::SegmentReader,+ ) -> crate::Result<Self::Child> {+ Ok(FirstDocSegmentCollector::new(+ segment_id,+ self.shard_id.unwrap(),+ ))+ }++ fn merge_fruits(+ &self,+ fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,+ ) -> crate::Result<Self::Fruit> {+ Ok(fruits.into_iter().flatten().next())+ }+}++pub struct FirstDocSegmentCollector {+ first_doc: Option<DocAddress>,+ segment_id: SegmentOrdinal,+ shard_id: ShardId,+}++impl FirstDocSegmentCollector {+ pub fn new(segment_id: SegmentOrdinal, shard_id: ShardId) -> Self {+ Self {+ first_doc: None,+ segment_id,+ shard_id,+ }+ }+}++impl SegmentCollector for FirstDocSegmentCollector {+ type Fruit = Option<DocAddress>;++ fn collect(&mut self, doc: DocId, _: tantivy::Score) {+ if self.first_doc.is_none() {+ self.first_doc = Some(DocAddress::new(self.segment_id, doc, self.shard_id));+ }+ }++ fn harvest(self) -> Self::Fruit {+ self.first_doc+ }+}diff --git a/crates/core/src/generic_query/collector/mod.rs b/crates/core/src/generic_query/collector/mod.rsindex 35425c6b..a0b407ef 100644--- a/crates/core/src/generic_query/collector/mod.rs+++ b/crates/core/src/generic_query/collector/mod.rs@@ -20,6 +20,9 @@ pub use top_key_phrases::TopKeyPhrasesCollector;mod size;pub use size::SizeCollector;+mod first_doc;+pub use first_doc::FirstDocCollector;+use tantivy::{collector::{Fruit, SegmentCollector},query::Weight,diff --git a/crates/core/src/generic_query/get_webpage.rs b/crates/core/src/generic_query/get_webpage.rsnew file mode 100644index 00000000..19c0599b--- /dev/null+++ b/crates/core/src/generic_query/get_webpage.rs@@ -0,0 +1,103 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::TantivyDocument;+use url::Url;++use crate::{+ inverted_index::RetrievedWebpage,+ schema::text_field::{self, TextField},+ search_ctx,+ webpage::url_ext::UrlExt,+};++use super::{collector::FirstDocCollector, GenericQuery};++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct GetWebpageQuery {+ pub url: String,+}++impl GetWebpageQuery {+ pub fn new(url: &str) -> Self {+ Self {+ url: url.to_string(),+ }+ }+}++impl GenericQuery for GetWebpageQuery {+ type Collector = FirstDocCollector;+ type TantivyQuery = Box<dyn tantivy::query::Query>;+ type IntermediateOutput = Option<RetrievedWebpage>;+ type Output = Option<RetrievedWebpage>;++ fn tantivy_query(&self, ctx: &search_ctx::Ctx) -> Self::TantivyQuery {+ match Url::robust_parse(&self.url) {+ Ok(url) => {+ let field = ctx+ .tv_searcher+ .schema()+ .get_field(text_field::UrlNoTokenizer.name())+ .unwrap();++ let term = tantivy::Term::from_field_text(field, url.as_str());+ Box::new(tantivy::query::TermQuery::new(+ term,+ tantivy::schema::IndexRecordOption::Basic,+ ))+ }+ Err(_) => Box::new(tantivy::query::EmptyQuery),+ }+ }++ fn collector(&self, ctx: &search_ctx::Ctx) -> Self::Collector {+ FirstDocCollector::with_shard_id(ctx.shard_id)+ }++ fn remote_collector(&self) -> Self::Collector {+ FirstDocCollector::without_shard_id()+ }++ fn filter_fruit_shards(+ &self,+ shard_id: crate::ampc::dht::ShardId,+ fruit: <Self::Collector as super::Collector>::Fruit,+ ) -> <Self::Collector as super::Collector>::Fruit {+ match fruit {+ Some(doc_address) if doc_address.shard_id == shard_id => Some(doc_address),+ _ => None,+ }+ }++ fn retrieve(+ &self,+ ctx: &search_ctx::Ctx,+ fruit: <Self::Collector as super::Collector>::Fruit,+ ) -> crate::Result<Self::IntermediateOutput> {+ match fruit {+ Some(doc_address) => {+ let doc: TantivyDocument = ctx.tv_searcher.doc(doc_address.into())?;+ Ok(Some(RetrievedWebpage::from(doc)))+ }+ None => Ok(None),+ }+ }++ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {+ results.into_iter().flatten().next()+ }+}diff --git a/crates/core/src/generic_query/mod.rs b/crates/core/src/generic_query/mod.rsindex 6b988657..28ce77b3 100644--- a/crates/core/src/generic_query/mod.rs+++ b/crates/core/src/generic_query/mod.rs@@ -22,6 +22,9 @@ pub use top_key_phrases::TopKeyPhrasesQuery;pub mod size;pub use size::SizeQuery;+pub mod get_webpage;+pub use get_webpage::GetWebpageQuery;+pub mod collector;pub use collector::Collector;diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex f0553d64..3bdf2bff 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -87,13 +87,23 @@ pub struct WebpagePointer {pub struct DocAddress {pub segment: u32,pub doc_id: u32,+ pub shard_id: ShardId,}-impl From<tantivy::DocAddress> for DocAddress {- fn from(address: tantivy::DocAddress) -> Self {+impl DocAddress {+ pub fn new(segment: u32, doc_id: u32, shard_id: ShardId) -> Self {+ Self {+ segment,+ doc_id,+ shard_id,+ }+ }++ pub fn from_tantivy(address: tantivy::DocAddress, shard_id: ShardId) -> Self {Self {segment: address.segment_ord,doc_id: address.doc_id,+ shard_id,}}}@@ -227,6 +237,7 @@ impl InvertedIndex {pub fn temporary() -> Result<(Self, file_store::temp::TempDir)> {let dir = crate::gen_temp_dir()?;let mut s = Self::open(dir.as_ref().join("index"))?;+ s.set_shard_id(ShardId::new(0));s.prepare_writer()?;@@ -249,6 +260,7 @@ mod tests {use crate::{collector::MainCollector,config::CollectorConfig,+ generic_query::GetWebpageQuery,query::Query,ranking::{LocalRanker, SignalComputer},search_ctx::Ctx,@@ -957,7 +969,11 @@ mod tests {index.commit().expect("failed to commit index");- let webpage = index.get_webpage("https://www.example.com").unwrap();+ let webpage = index+ .search_generic(&GetWebpageQuery::new("https://www.example.com"))+ .unwrap()+ .unwrap();+assert_eq!(webpage.title, "News website".to_string());assert_eq!(webpage.url, "https://www.example.com/".to_string());}diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rsindex 447ff0d6..d64adbde 100644--- a/crates/core/src/inverted_index/search.rs+++ b/crates/core/src/inverted_index/search.rs@@ -277,26 +277,6 @@ impl InvertedIndex {Ok(RetrievedWebpage::from(doc))}- pub(crate) fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {- let url = Url::parse(url).ok()?;- let tv_searcher = self.reader.searcher();- let field = tv_searcher- .schema()- .get_field(Field::Text(TextFieldEnum::from(text_field::UrlNoTokenizer)).name())- .unwrap();-- let term = tantivy::Term::from_field_text(field, url.as_str());-- let query = tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic);-- let mut res = tv_searcher- .search(&query, &tantivy::collector::TopDocs::with_limit(1))- .unwrap();-- res.pop()- .map(|(_, doc)| self.retrieve_doc(doc.into(), &tv_searcher).unwrap())- }-pub(crate) fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {let tv_searcher = self.reader.searcher();let field = tv_searcher@@ -316,8 +296,16 @@ impl InvertedIndex {.search(&query, &tantivy::collector::TopDocs::with_limit(1)).unwrap();- res.pop()- .map(|(_, doc)| self.retrieve_doc(doc.into(), &tv_searcher).unwrap())+ res.pop().map(|(_, doc)| {+ self.retrieve_doc(+ DocAddress::from_tantivy(+ doc,+ self.shard_id.expect("Shard ID should be set for searches"),+ ),+ &tv_searcher,+ )+ .unwrap()+ })}pub(crate) fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {@@ -339,7 +327,16 @@ impl InvertedIndex {) {Ok(res) => res.into_iter()- .filter_map(|(_, doc)| self.retrieve_doc(doc.into(), &tv_searcher).ok())+ .filter_map(|(_, doc)| {+ self.retrieve_doc(+ DocAddress::from_tantivy(+ doc,+ self.shard_id.expect("Shard ID should be set for searches"),+ ),+ &tv_searcher,+ )+ .ok()+ }).filter_map(|page| Url::parse(&page.url).ok()).collect(),Err(_) => vec![],@@ -376,7 +373,12 @@ impl InvertedIndex {From<<Q::Collector as generic_query::Collector>::Fruit>,{let fruit = self.search_initial_generic(query)?;- let fruit = query.remote_collector().merge_fruits(vec![fruit.into()])?;+ let mut fruit = query.remote_collector().merge_fruits(vec![fruit.into()])?;++ if let Some(shard_id) = self.shard_id {+ fruit = query.filter_fruit_shards(shard_id, fruit);+ }+let res = self.retrieve_generic(query, fruit)?;Ok(Q::merge_results(vec![res]))}diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex e1e764ee..ef670280 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -118,7 +118,9 @@ impl LocalRanker {collector = collector.and_max_docs(max_docs.clone());}- collector = collector.and_collector_config(self.collector_config.clone());+ collector = collector+ .and_collector_config(self.collector_config.clone())+ .and_shard_id(ctx.shard_id);collector.main_collector(score_tweaker)}diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rsindex d95c73d8..ff8fcee5 100644--- a/crates/core/src/ranking/pipeline/mod.rs+++ b/crates/core/src/ranking/pipeline/mod.rs@@ -195,10 +195,7 @@ mod tests {url_without_tld: Prehashed(0),simhash: 0,},- address: DocAddress {- segment: 0,- doc_id: i as u32,- },+ address: DocAddress::new(0, i as u32, ShardId::new(0)),};let mut signals = EnumMap::new();diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex fc6f850c..7bff888e 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -31,7 +31,6 @@ use crate::collector::{self, approx_count};use crate::config::{ApiConfig, ApiSpellCheck, ApiThresholds, CollectorConfig, WidgetsConfig};use crate::enum_map::EnumMap;use crate::image_store::Image;-use crate::inverted_index::RetrievedWebpage;use crate::models::dual_encoder::DualEncoder;use crate::ranking::models::cross_encoder::CrossEncoderModel;use crate::ranking::pipeline::{PrecisionRankingWebpage, RankableWebpage, RecallRankingWebpage};@@ -773,10 +772,6 @@ whereOk(SearchResult::Websites(self.search_websites(query).await?))}- pub async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {- self.distributed_searcher.get_webpage(url).await- }-pub async fn get_entity_image(&self,image_id: &str,diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rsindex e1f8a566..ab8ea624 100644--- a/crates/core/src/searcher/distributed.rs+++ b/crates/core/src/searcher/distributed.rs@@ -81,11 +81,6 @@ pub trait SearchClient {fn search_entity(&self, query: &str) -> impl Future<Output = Option<EntityMatch>> + Send;- fn get_webpage(- &self,- url: &str,- ) -> impl Future<Output = Result<Option<RetrievedWebpage>>> + Send;-fn get_homepage_descriptions(&self,urls: &[Url],@@ -415,33 +410,6 @@ impl SearchClient for DistributedSearcher {retrieved_webpages}- async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {- let client = self.conn().await;-- let res = client- .send(- search_server::GetWebpage {- url: url.to_string(),- },- &AllShardsSelector,- &RandomReplicaSelector,- )- .await- .map_err(|_| Error::SearchFailed)?;-- if let Some(res) = res- .into_iter()- .flatten()- .flat_map(|(_, v)| v.into_iter().map(|(_, v)| v))- .flatten()- .next()- {- Ok(Some(res))- } else {- Err(Error::WebpageNotFound.into())- }- }-async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {let client = self.conn().await;@@ -738,10 +706,6 @@ impl SearchClient for LocalSearchClient {res}- async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {- Ok(self.0.get_webpage(url).await)- }-async fn get_homepage_descriptions(&self,urls: &[url::Url],diff --git a/crates/core/src/searcher/local/mod.rs b/crates/core/src/searcher/local/mod.rsindex 39185b3f..4cfa36ca 100644--- a/crates/core/src/searcher/local/mod.rs+++ b/crates/core/src/searcher/local/mod.rs@@ -216,10 +216,6 @@ wherecrate::block_on(self.search(query))}- pub async fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {- self.inner.guard().await.inverted_index().get_webpage(url)- }-pub async fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {self.inner.guard().await.inverted_index().get_homepage(url)}