Skip to content

web search done right

[search_server] implement get webpage as GenericQuery

Mikkel Denker2f14ed48980c
diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rs
index e92dc896..6d4340cb 100644
--- a/crates/core/src/collector/top_docs.rs
+++ b/crates/core/src/collector/top_docs.rs
@@ -25,6 +25,7 @@ use tantivy::{
};
use crate::{
+ ampc::dht::ShardId,
config::CollectorConfig,
inverted_index::{DocAddress, WebpagePointer},
numericalfield_reader,
@@ -43,6 +44,7 @@ pub struct TopDocs {
columnfield_reader: numericalfield_reader::NumericalFieldReader,
de_rank_similar: bool,
collector_config: CollectorConfig,
+ shard_id: Option<ShardId>,
}
impl TopDocs {
@@ -57,6 +59,7 @@ impl TopDocs {
de_rank_similar: false,
columnfield_reader,
collector_config: CollectorConfig::default(),
+ shard_id: None,
}
}
@@ -84,6 +87,11 @@ impl TopDocs {
self
}
+ pub fn and_shard_id(mut self, shard_id: ShardId) -> Self {
+ self.shard_id = Some(shard_id);
+ self
+ }
+
pub fn main_collector(self, score_tweaker: InitialScoreTweaker) -> MainCollector {
MainCollector::new(score_tweaker, self)
}
@@ -112,6 +120,7 @@ impl TopDocs {
self.top_n + self.offset,
self.collector_config.clone(),
),
+ shard_id: self.shard_id,
})
}
}
@@ -122,6 +131,7 @@ pub struct TopSegmentCollector {
num_docs_taken: usize,
segment_ord: SegmentOrdinal,
bucket_collector: BucketCollector<SegmentDoc>,
+ shard_id: Option<ShardId>,
}
impl TopSegmentCollector {
@@ -190,6 +200,7 @@ impl TopSegmentCollector {
},
id: doc,
segment: self.segment_ord,
+ shard_id: self.shard_id.expect("Shard ID should be set for searches"),
score,
});
}
@@ -357,6 +368,7 @@ pub struct SegmentDoc {
id: DocId,
segment: SegmentOrdinal,
score: Score,
+ shard_id: ShardId,
}
impl Doc for SegmentDoc {
@@ -443,10 +455,7 @@ where
.map(|doc| WebpagePointer {
score: doc.score,
hashes: doc.hashes,
- address: DocAddress {
- segment: doc.segment,
- doc_id: doc.id,
- },
+ address: DocAddress::new(doc.segment, doc.id, doc.shard_id),
})
.collect())
}
@@ -494,6 +503,7 @@ mod tests {
id: doc.1,
score: Score { total: doc.2 },
segment: 0,
+ shard_id: ShardId::new(0),
});
}
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rs
index 61b6b916..03dbd2f4 100644
--- a/crates/core/src/entrypoint/search_server.rs
+++ b/crates/core/src/entrypoint/search_server.rs
@@ -26,9 +26,9 @@ use crate::{
member::{Member, Service},
sonic::{self, service::sonic_service},
},
- generic_query::{self, SizeQuery, TopKeyPhrasesQuery},
+ generic_query::{self, GetWebpageQuery, SizeQuery, TopKeyPhrasesQuery},
index::Index,
- inverted_index::{self, RetrievedWebpage},
+ inverted_index,
models::dual_encoder::DualEncoder,
ranking::models::linear::LinearRegression,
searcher::{InitialWebsiteResult, LocalSearcher, SearchQuery},
@@ -116,7 +116,6 @@ macro_rules! impl_search {
sonic_service!(SearchService, [
RetrieveWebsites,
Search,
- GetWebpage,
GetHomepageDescriptions,
GetSiteUrls,
$(
@@ -129,7 +128,7 @@ macro_rules! impl_search {
}
}
-impl_search!([TopKeyPhrasesQuery, SizeQuery,]);
+impl_search!([TopKeyPhrasesQuery, SizeQuery, GetWebpageQuery]);
pub struct SearchService {
local_searcher: LocalSearcher<Arc<Index>>,
@@ -206,17 +205,6 @@ impl sonic::service::Message<SearchService> for Search {
}
}
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct GetWebpage {
- pub url: String,
-}
-impl sonic::service::Message<SearchService> for GetWebpage {
- type Response = Option<RetrievedWebpage>;
- async fn handle(self, server: &SearchService) -> Self::Response {
- server.local_searcher.get_webpage(&self.url).await
- }
-}
-
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct GetHomepageDescriptions {
#[bincode(with_serde)]
diff --git a/crates/core/src/generic_query/collector/first_doc.rs b/crates/core/src/generic_query/collector/first_doc.rs
new file mode 100644
index 00000000..b4160fbe
--- /dev/null
+++ b/crates/core/src/generic_query/collector/first_doc.rs
@@ -0,0 +1,89 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::{collector::SegmentCollector, DocId, SegmentOrdinal};
+
+use super::Collector;
+use crate::{ampc::dht::ShardId, inverted_index::DocAddress};
+
+pub struct FirstDocCollector {
+ shard_id: Option<ShardId>,
+}
+
+impl FirstDocCollector {
+ pub fn with_shard_id(shard_id: ShardId) -> Self {
+ Self {
+ shard_id: Some(shard_id),
+ }
+ }
+
+ pub fn without_shard_id() -> Self {
+ Self { shard_id: None }
+ }
+}
+
+impl Collector for FirstDocCollector {
+ type Fruit = Option<DocAddress>;
+ type Child = FirstDocSegmentCollector;
+
+ fn for_segment(
+ &self,
+ segment_id: SegmentOrdinal,
+ _: &tantivy::SegmentReader,
+ ) -> crate::Result<Self::Child> {
+ Ok(FirstDocSegmentCollector::new(
+ segment_id,
+ self.shard_id.unwrap(),
+ ))
+ }
+
+ fn merge_fruits(
+ &self,
+ fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> crate::Result<Self::Fruit> {
+ Ok(fruits.into_iter().flatten().next())
+ }
+}
+
+pub struct FirstDocSegmentCollector {
+ first_doc: Option<DocAddress>,
+ segment_id: SegmentOrdinal,
+ shard_id: ShardId,
+}
+
+impl FirstDocSegmentCollector {
+ pub fn new(segment_id: SegmentOrdinal, shard_id: ShardId) -> Self {
+ Self {
+ first_doc: None,
+ segment_id,
+ shard_id,
+ }
+ }
+}
+
+impl SegmentCollector for FirstDocSegmentCollector {
+ type Fruit = Option<DocAddress>;
+
+ fn collect(&mut self, doc: DocId, _: tantivy::Score) {
+ if self.first_doc.is_none() {
+ self.first_doc = Some(DocAddress::new(self.segment_id, doc, self.shard_id));
+ }
+ }
+
+ fn harvest(self) -> Self::Fruit {
+ self.first_doc
+ }
+}
diff --git a/crates/core/src/generic_query/collector/mod.rs b/crates/core/src/generic_query/collector/mod.rs
index 35425c6b..a0b407ef 100644
--- a/crates/core/src/generic_query/collector/mod.rs
+++ b/crates/core/src/generic_query/collector/mod.rs
@@ -20,6 +20,9 @@ pub use top_key_phrases::TopKeyPhrasesCollector;
mod size;
pub use size::SizeCollector;
+mod first_doc;
+pub use first_doc::FirstDocCollector;
+
use tantivy::{
collector::{Fruit, SegmentCollector},
query::Weight,
diff --git a/crates/core/src/generic_query/get_webpage.rs b/crates/core/src/generic_query/get_webpage.rs
new file mode 100644
index 00000000..19c0599b
--- /dev/null
+++ b/crates/core/src/generic_query/get_webpage.rs
@@ -0,0 +1,103 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::TantivyDocument;
+use url::Url;
+
+use crate::{
+ inverted_index::RetrievedWebpage,
+ schema::text_field::{self, TextField},
+ search_ctx,
+ webpage::url_ext::UrlExt,
+};
+
+use super::{collector::FirstDocCollector, GenericQuery};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct GetWebpageQuery {
+ pub url: String,
+}
+
+impl GetWebpageQuery {
+ pub fn new(url: &str) -> Self {
+ Self {
+ url: url.to_string(),
+ }
+ }
+}
+
+impl GenericQuery for GetWebpageQuery {
+ type Collector = FirstDocCollector;
+ type TantivyQuery = Box<dyn tantivy::query::Query>;
+ type IntermediateOutput = Option<RetrievedWebpage>;
+ type Output = Option<RetrievedWebpage>;
+
+ fn tantivy_query(&self, ctx: &search_ctx::Ctx) -> Self::TantivyQuery {
+ match Url::robust_parse(&self.url) {
+ Ok(url) => {
+ let field = ctx
+ .tv_searcher
+ .schema()
+ .get_field(text_field::UrlNoTokenizer.name())
+ .unwrap();
+
+ let term = tantivy::Term::from_field_text(field, url.as_str());
+ Box::new(tantivy::query::TermQuery::new(
+ term,
+ tantivy::schema::IndexRecordOption::Basic,
+ ))
+ }
+ Err(_) => Box::new(tantivy::query::EmptyQuery),
+ }
+ }
+
+ fn collector(&self, ctx: &search_ctx::Ctx) -> Self::Collector {
+ FirstDocCollector::with_shard_id(ctx.shard_id)
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ FirstDocCollector::without_shard_id()
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: crate::ampc::dht::ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ match fruit {
+ Some(doc_address) if doc_address.shard_id == shard_id => Some(doc_address),
+ _ => None,
+ }
+ }
+
+ fn retrieve(
+ &self,
+ ctx: &search_ctx::Ctx,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> crate::Result<Self::IntermediateOutput> {
+ match fruit {
+ Some(doc_address) => {
+ let doc: TantivyDocument = ctx.tv_searcher.doc(doc_address.into())?;
+ Ok(Some(RetrievedWebpage::from(doc)))
+ }
+ None => Ok(None),
+ }
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ results.into_iter().flatten().next()
+ }
+}
diff --git a/crates/core/src/generic_query/mod.rs b/crates/core/src/generic_query/mod.rs
index 6b988657..28ce77b3 100644
--- a/crates/core/src/generic_query/mod.rs
+++ b/crates/core/src/generic_query/mod.rs
@@ -22,6 +22,9 @@ pub use top_key_phrases::TopKeyPhrasesQuery;
pub mod size;
pub use size::SizeQuery;
+pub mod get_webpage;
+pub use get_webpage::GetWebpageQuery;
+
pub mod collector;
pub use collector::Collector;
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index f0553d64..3bdf2bff 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -87,13 +87,23 @@ pub struct WebpagePointer {
pub struct DocAddress {
pub segment: u32,
pub doc_id: u32,
+ pub shard_id: ShardId,
}
-impl From<tantivy::DocAddress> for DocAddress {
- fn from(address: tantivy::DocAddress) -> Self {
+impl DocAddress {
+ pub fn new(segment: u32, doc_id: u32, shard_id: ShardId) -> Self {
+ Self {
+ segment,
+ doc_id,
+ shard_id,
+ }
+ }
+
+ pub fn from_tantivy(address: tantivy::DocAddress, shard_id: ShardId) -> Self {
Self {
segment: address.segment_ord,
doc_id: address.doc_id,
+ shard_id,
}
}
}
@@ -227,6 +237,7 @@ impl InvertedIndex {
pub fn temporary() -> Result<(Self, file_store::temp::TempDir)> {
let dir = crate::gen_temp_dir()?;
let mut s = Self::open(dir.as_ref().join("index"))?;
+ s.set_shard_id(ShardId::new(0));
s.prepare_writer()?;
@@ -249,6 +260,7 @@ mod tests {
use crate::{
collector::MainCollector,
config::CollectorConfig,
+ generic_query::GetWebpageQuery,
query::Query,
ranking::{LocalRanker, SignalComputer},
search_ctx::Ctx,
@@ -957,7 +969,11 @@ mod tests {
index.commit().expect("failed to commit index");
- let webpage = index.get_webpage("https://www.example.com").unwrap();
+ let webpage = index
+ .search_generic(&GetWebpageQuery::new("https://www.example.com"))
+ .unwrap()
+ .unwrap();
+
assert_eq!(webpage.title, "News website".to_string());
assert_eq!(webpage.url, "https://www.example.com/".to_string());
}
diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rs
index 447ff0d6..d64adbde 100644
--- a/crates/core/src/inverted_index/search.rs
+++ b/crates/core/src/inverted_index/search.rs
@@ -277,26 +277,6 @@ impl InvertedIndex {
Ok(RetrievedWebpage::from(doc))
}
- pub(crate) fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {
- let url = Url::parse(url).ok()?;
- let tv_searcher = self.reader.searcher();
- let field = tv_searcher
- .schema()
- .get_field(Field::Text(TextFieldEnum::from(text_field::UrlNoTokenizer)).name())
- .unwrap();
-
- let term = tantivy::Term::from_field_text(field, url.as_str());
-
- let query = tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic);
-
- let mut res = tv_searcher
- .search(&query, &tantivy::collector::TopDocs::with_limit(1))
- .unwrap();
-
- res.pop()
- .map(|(_, doc)| self.retrieve_doc(doc.into(), &tv_searcher).unwrap())
- }
-
pub(crate) fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {
let tv_searcher = self.reader.searcher();
let field = tv_searcher
@@ -316,8 +296,16 @@ impl InvertedIndex {
.search(&query, &tantivy::collector::TopDocs::with_limit(1))
.unwrap();
- res.pop()
- .map(|(_, doc)| self.retrieve_doc(doc.into(), &tv_searcher).unwrap())
+ res.pop().map(|(_, doc)| {
+ self.retrieve_doc(
+ DocAddress::from_tantivy(
+ doc,
+ self.shard_id.expect("Shard ID should be set for searches"),
+ ),
+ &tv_searcher,
+ )
+ .unwrap()
+ })
}
pub(crate) fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {
@@ -339,7 +327,16 @@ impl InvertedIndex {
) {
Ok(res) => res
.into_iter()
- .filter_map(|(_, doc)| self.retrieve_doc(doc.into(), &tv_searcher).ok())
+ .filter_map(|(_, doc)| {
+ self.retrieve_doc(
+ DocAddress::from_tantivy(
+ doc,
+ self.shard_id.expect("Shard ID should be set for searches"),
+ ),
+ &tv_searcher,
+ )
+ .ok()
+ })
.filter_map(|page| Url::parse(&page.url).ok())
.collect(),
Err(_) => vec![],
@@ -376,7 +373,12 @@ impl InvertedIndex {
From<<Q::Collector as generic_query::Collector>::Fruit>,
{
let fruit = self.search_initial_generic(query)?;
- let fruit = query.remote_collector().merge_fruits(vec![fruit.into()])?;
+ let mut fruit = query.remote_collector().merge_fruits(vec![fruit.into()])?;
+
+ if let Some(shard_id) = self.shard_id {
+ fruit = query.filter_fruit_shards(shard_id, fruit);
+ }
+
let res = self.retrieve_generic(query, fruit)?;
Ok(Q::merge_results(vec![res]))
}
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index e1e764ee..ef670280 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -118,7 +118,9 @@ impl LocalRanker {
collector = collector.and_max_docs(max_docs.clone());
}
- collector = collector.and_collector_config(self.collector_config.clone());
+ collector = collector
+ .and_collector_config(self.collector_config.clone())
+ .and_shard_id(ctx.shard_id);
collector.main_collector(score_tweaker)
}
diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rs
index d95c73d8..ff8fcee5 100644
--- a/crates/core/src/ranking/pipeline/mod.rs
+++ b/crates/core/src/ranking/pipeline/mod.rs
@@ -195,10 +195,7 @@ mod tests {
url_without_tld: Prehashed(0),
simhash: 0,
},
- address: DocAddress {
- segment: 0,
- doc_id: i as u32,
- },
+ address: DocAddress::new(0, i as u32, ShardId::new(0)),
};
let mut signals = EnumMap::new();
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index fc6f850c..7bff888e 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -31,7 +31,6 @@ use crate::collector::{self, approx_count};
use crate::config::{ApiConfig, ApiSpellCheck, ApiThresholds, CollectorConfig, WidgetsConfig};
use crate::enum_map::EnumMap;
use crate::image_store::Image;
-use crate::inverted_index::RetrievedWebpage;
use crate::models::dual_encoder::DualEncoder;
use crate::ranking::models::cross_encoder::CrossEncoderModel;
use crate::ranking::pipeline::{PrecisionRankingWebpage, RankableWebpage, RecallRankingWebpage};
@@ -773,10 +772,6 @@ where
Ok(SearchResult::Websites(self.search_websites(query).await?))
}
- pub async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {
- self.distributed_searcher.get_webpage(url).await
- }
-
pub async fn get_entity_image(
&self,
image_id: &str,
diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rs
index e1f8a566..ab8ea624 100644
--- a/crates/core/src/searcher/distributed.rs
+++ b/crates/core/src/searcher/distributed.rs
@@ -81,11 +81,6 @@ pub trait SearchClient {
fn search_entity(&self, query: &str) -> impl Future<Output = Option<EntityMatch>> + Send;
- fn get_webpage(
- &self,
- url: &str,
- ) -> impl Future<Output = Result<Option<RetrievedWebpage>>> + Send;
-
fn get_homepage_descriptions(
&self,
urls: &[Url],
@@ -415,33 +410,6 @@ impl SearchClient for DistributedSearcher {
retrieved_webpages
}
- async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {
- let client = self.conn().await;
-
- let res = client
- .send(
- search_server::GetWebpage {
- url: url.to_string(),
- },
- &AllShardsSelector,
- &RandomReplicaSelector,
- )
- .await
- .map_err(|_| Error::SearchFailed)?;
-
- if let Some(res) = res
- .into_iter()
- .flatten()
- .flat_map(|(_, v)| v.into_iter().map(|(_, v)| v))
- .flatten()
- .next()
- {
- Ok(Some(res))
- } else {
- Err(Error::WebpageNotFound.into())
- }
- }
-
async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {
let client = self.conn().await;
@@ -738,10 +706,6 @@ impl SearchClient for LocalSearchClient {
res
}
- async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {
- Ok(self.0.get_webpage(url).await)
- }
-
async fn get_homepage_descriptions(
&self,
urls: &[url::Url],
diff --git a/crates/core/src/searcher/local/mod.rs b/crates/core/src/searcher/local/mod.rs
index 39185b3f..4cfa36ca 100644
--- a/crates/core/src/searcher/local/mod.rs
+++ b/crates/core/src/searcher/local/mod.rs
@@ -216,10 +216,6 @@ where
crate::block_on(self.search(query))
}
- pub async fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {
- self.inner.guard().await.inverted_index().get_webpage(url)
- }
-
pub async fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {
self.inner.guard().await.inverted_index().get_homepage(url)
}