web search done right
Move entity index out of normal search index and have dedicated search server for it
Mikkel Denkercc91935d0a52
diff --git a/configs/entity_search_server.toml b/configs/entity_search_server.tomlnew file mode 100644index 00000000..40ae01ca--- /dev/null+++ b/configs/entity_search_server.toml@@ -0,0 +1,5 @@+cluster_id = "dev_search"+gossip_addr = "0.0.0.0:3010"+gossip_seed_nodes = ["0.0.0.0:3005"]+host = "0.0.0.0:3009"+index_path = "data/entity"diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex c1f7121b..15001055 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -250,7 +250,6 @@ pub struct SearchServerConfig {pub gossip_addr: SocketAddr,pub shard_id: ShardId,pub index_path: String,- pub entity_index_path: Option<String>,pub host_centrality_store_path: Option<String>,pub linear_model_path: Option<String>,pub lambda_model_path: Option<String>,@@ -263,6 +262,15 @@ pub struct SearchServerConfig {pub snippet: SnippetConfig,}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct EntitySearchServerConfig {+ pub cluster_id: String,+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,+ pub gossip_addr: SocketAddr,+ pub index_path: String,+ pub host: SocketAddr,+}+#[derive(Debug, Serialize, Deserialize, Clone)]pub struct CrawlCoordinatorConfig {pub job_queue: String,diff --git a/core/src/distributed/member.rs b/core/src/distributed/member.rsindex 48cfae15..205c9974 100644--- a/core/src/distributed/member.rs+++ b/core/src/distributed/member.rs@@ -26,6 +26,9 @@ pub enum Service {host: SocketAddr,shard: ShardId,},+ EntitySearcher {+ host: SocketAddr,+ },LiveIndex {host: SocketAddr,split_id: crate::feed::scheduler::SplitId,diff --git a/core/src/entrypoint/entity_search_server.rs b/core/src/entrypoint/entity_search_server.rsnew file mode 100644index 00000000..e1fb1da4--- /dev/null+++ b/core/src/entrypoint/entity_search_server.rs@@ -0,0 +1,105 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use serde::{Deserialize, Serialize};++use crate::{+ config,+ distributed::{+ cluster::Cluster,+ member::{Member, Service},+ sonic,+ },+ entity_index::EntityIndex,+ image_store::Image,+ sonic_service,+};+use anyhow::Result;++sonic_service!(SearchService, [Search, GetEntityImage]);++pub struct SearchService {+ index: EntityIndex,+ // dropping the handle leaves the cluster+ #[allow(unused)]+ cluster_handle: Cluster,+}+impl SearchService {+ async fn new(config: config::EntitySearchServerConfig) -> Result<Self> {+ let index = EntityIndex::open(config.index_path)?;++ let cluster_handle = Cluster::join(+ Member {+ id: config.cluster_id,+ service: Service::EntitySearcher { host: config.host },+ },+ config.gossip_addr,+ config.gossip_seed_nodes.unwrap_or_default(),+ )+ .await?;++ Ok(SearchService {+ index,+ cluster_handle,+ })+ }+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct Search {+ pub query: String,+}++impl sonic::service::Message<SearchService> for Search {+ type Response = Option<crate::entity_index::EntityMatch>;+ async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {+ Ok(server.index.search(&self.query))+ }+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct GetEntityImage {+ pub image_id: String,+ pub max_width: Option<u64>,+ pub max_height: Option<u64>,+}+impl sonic::service::Message<SearchService> for GetEntityImage {+ type Response = Option<Image>;+ async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {+ Ok(server.index.retrieve_image(&self.image_id).map(|img| {+ let max_width = self.max_width.unwrap_or(u64::MAX) as u32;+ let max_height = self.max_height.unwrap_or(u64::MAX) as u32;++ img.resize_max(max_width, max_height)+ }))+ }+}++pub async fn run(config: config::EntitySearchServerConfig) -> Result<()> {+ let addr = config.host;+ let server = SearchService::new(config).await?.bind(addr).await.unwrap();++ tracing::info!(+ "entity search server is ready to accept requests on {}",+ addr+ );++ loop {+ if let Err(e) = server.accept().await {+ tracing::error!("{:?}", e);+ }+ }+}diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rsindex 8f40c4db..ee699d68 100644--- a/core/src/entrypoint/mod.rs+++ b/core/src/entrypoint/mod.rs@@ -25,6 +25,7 @@ pub mod configure;pub mod crawler;pub mod dmoz_parser;mod entity;+pub mod entity_search_server;pub mod feed_indexer;pub mod indexer;pub mod safety_classifier;diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex a20995fa..b3bab405 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -27,8 +27,6 @@ use crate::{member::{Member, Service},sonic,},- entity_index::{EntityIndex, EntityMatch},- image_store::Image,index::Index,inverted_index::{self, RetrievedWebpage},ranking::{@@ -44,10 +42,8 @@ sonic_service!([RetrieveWebsites,Search,- SearchEntity,GetWebpage,GetHomepageDescriptions,- GetEntityImage,]);@@ -60,9 +56,6 @@ pub struct SearchService {impl SearchService {async fn new(config: config::SearchServerConfig) -> Result<Self> {- let entity_index = config- .entity_index_path- .map(|path| EntityIndex::open(path).unwrap());let centrality_store = config.host_centrality_store_path.map(|p| InboundSimilarity::open(Path::new(&p).join("inbound_similarity")).unwrap());@@ -70,10 +63,6 @@ impl SearchService {let mut local_searcher = LocalSearcher::new(search_index);- if let Some(entity_index) = entity_index {- local_searcher.set_entity_index(entity_index);- }-if let Some(centrality_store) = centrality_store {local_searcher.set_inbound_similarity(centrality_store);}@@ -173,39 +162,6 @@ impl sonic::service::Message<SearchService> for GetHomepageDescriptions {}}-#[derive(Debug, Clone, Serialize, Deserialize)]-pub struct GetEntityImage {- pub image_id: String,- pub max_width: Option<u64>,- pub max_height: Option<u64>,-}-impl sonic::service::Message<SearchService> for GetEntityImage {- type Response = Option<Image>;- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {- Ok(server- .local_searcher- .get_entity_image(&self.image_id)- .map(|img| {- let max_width = self.max_width.unwrap_or(u64::MAX) as u32;- let max_height = self.max_height.unwrap_or(u64::MAX) as u32;-- img.resize_max(max_width, max_height)- }))- }-}--#[derive(Debug, Clone, Serialize, Deserialize)]-pub struct SearchEntity {- pub query: String,-}--impl sonic::service::Message<SearchService> for SearchEntity {- type Response = Option<EntityMatch>;- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {- Ok(server.local_searcher.search_entity(&self.query))- }-}-pub async fn run(config: config::SearchServerConfig) -> Result<()> {let addr = config.host;let server = SearchService::new(config).await?.bind(addr).await.unwrap();diff --git a/core/src/main.rs b/core/src/main.rsindex c9089a97..8e827445 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -24,7 +24,9 @@ use stract::entrypoint::autosuggest_scrape::{self, Gl};#[cfg(feature = "dev")]use stract::entrypoint::configure;-use stract::entrypoint::{self, api, safety_classifier, search_server, webgraph_server};+use stract::entrypoint::{+ self, api, entity_search_server, safety_classifier, search_server, webgraph_server,+};use stract::webgraph::WebgraphBuilder;use tracing_subscriber::prelude::*;@@ -75,6 +77,11 @@ enum Commands {config_path: String,},+ /// Deploy the entity search server.+ EntitySearchServer {+ config_path: String,+ },+/// Deploy the json http api. The api interacts with/// the search servers, webgraph servers etc. to provide the necesarry functionality.Api {@@ -314,6 +321,14 @@ fn main() -> Result<()> {.build()?.block_on(search_server::run(config))?}+ Commands::EntitySearchServer { config_path } => {+ let config: config::EntitySearchServerConfig = load_toml_config(config_path);++ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?+ .block_on(entity_search_server::run(config))?+ }Commands::AutosuggestScrape {num_queries: queries_to_scrape,gl,diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex da2f0403..37705b9d 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -19,12 +19,15 @@ use crate::{cluster::Cluster,member::Service,sonic::replication::{- AllShardsSelector, RandomReplicaSelector, RandomShardSelector, RemoteClient,- ReplicatedClient, Shard, ShardIdentifier, ShardedClient, SpecificShardSelector,+ AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient, Shard,+ ShardIdentifier, ShardedClient, SpecificShardSelector,},},entity_index::EntityMatch,- entrypoint::search_server::{self, SearchService},+ entrypoint::{+ entity_search_server,+ search_server::{self, SearchService},+ },image_store::Image,inverted_index::{RetrievedWebpage, WebsitePointer},ranking::pipeline::{RankingWebsite, RetrievedWebpageRanking},@@ -107,6 +110,17 @@ impl DistributedSearcher {ShardedClient::new(shard_clients)}+ async fn entity_client(&self) -> ReplicatedClient<entity_search_server::SearchService> {+ let mut replicas = Vec::new();+ for member in self.cluster.members().await {+ if let Service::EntitySearcher { host } = member.service {+ replicas.push(RemoteClient::new(host));+ }+ }++ ReplicatedClient::new(replicas)+ }+async fn retrieve_webpages_from_shard(&self,shard: ShardId,@@ -252,43 +266,37 @@ impl SearchClient for DistributedSearcher {max_height: Option<u64>,max_width: Option<u64>,) -> Result<Option<Image>> {- let client = self.client().await;+ let client = self.entity_client().await;- let (_, mut res) = client+ Ok(client.send(- &search_server::GetEntityImage {+ &entity_search_server::GetEntityImage {image_id: image_id.to_string(),max_height,max_width,},- &RandomShardSelector,&RandomReplicaSelector,).await.map_err(|_| Error::SearchFailed)?.pop()- .unwrap();-- Ok(res.pop().flatten())+ .unwrap())}async fn search_entity(&self, query: &str) -> Option<EntityMatch> {- let client = self.client().await;+ let client = self.entity_client().await;- let (_, mut res) = client+ client.send(- &search_server::SearchEntity {+ &entity_search_server::Search {query: query.to_string(),},- &RandomShardSelector,&RandomReplicaSelector,).await.ok()?.pop()- .unwrap();-- res.pop().flatten()+ .unwrap()}}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 5d32c8e0..c25af51c 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -20,8 +20,6 @@ use std::sync::{Arc, RwLockReadGuard};use url::Url;use crate::config::{CollectorConfig, SnippetConfig};-use crate::entity_index::{EntityIndex, EntityMatch};-use crate::image_store::Image;use crate::index::Index;use crate::inverted_index::{InvertedIndex, RetrievedWebpage};use crate::query::Query;@@ -113,7 +111,6 @@ impl<'a> SearchGuard<'a> for LiveIndexSearchGuard<'a> {pub struct LocalSearcher<I: SearchableIndex> {index: I,- entity_index: Option<EntityIndex>,inbound_similarity: Option<InboundSimilarity>,linear_regression: Option<Arc<LinearRegression>>,lambda_model: Option<Arc<LambdaMART>>,@@ -145,8 +142,6 @@ whereLocalSearcher {index,- // spell: None,- entity_index: None,inbound_similarity: None,linear_regression: None,lambda_model: None,@@ -154,10 +149,6 @@ where}}- pub fn set_entity_index(&mut self, entity_index: EntityIndex) {- self.entity_index = Some(entity_index);- }-pub fn set_inbound_similarity(&mut self, inbound: InboundSimilarity) {self.inbound_similarity = Some(inbound);}@@ -322,12 +313,6 @@ where})}- pub fn search_entity(&self, query: &str) -> Option<EntityMatch> {- self.entity_index- .as_ref()- .and_then(|index| index.search(query))- }-pub fn search_initial(&self,query: &SearchQuery,@@ -453,12 +438,6 @@ wherepub fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {self.index.guard().inverted_index().get_homepage(url)}-- pub fn get_entity_image(&self, image_id: &str) -> Option<Image> {- self.entity_index- .as_ref()- .and_then(|index| index.retrieve_image(image_id))- }}#[cfg(test)]diff --git a/justfile b/justfileindex bc9d3fd2..04abba66 100644--- a/justfile+++ b/justfile@@ -10,6 +10,8 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- api configs/api.toml"@dev-search-server:cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- search-server configs/search_server.toml"+@dev-entity-search-server:+ cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- entity-search-server configs/entity_search_server.toml"@dev-webgraph:cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/server.toml"@dev-llm:diff --git a/scripts/run_dev.py b/scripts/run_dev.pyindex 24727eb9..dab9d148 100755--- a/scripts/run_dev.py+++ b/scripts/run_dev.py@@ -20,6 +20,7 @@ processes = []processes.append(subprocess.Popen(["just", "dev-api"]))processes.append(subprocess.Popen(["just", "dev-search-server"]))+processes.append(subprocess.Popen(["just", "dev-entity-search-server"]))processes.append(subprocess.Popen(["just", "dev-webgraph"]))processes.append(subprocess.Popen(["just", "dev-frontend"]))processes.append(subprocess.Popen(["just", "dev-llm"]))