Skip to content

web search done right

Move entity index out of normal search index and have dedicated search server for it

Mikkel Denkercc91935d0a52
diff --git a/configs/entity_search_server.toml b/configs/entity_search_server.toml
new file mode 100644
index 00000000..40ae01ca
--- /dev/null
+++ b/configs/entity_search_server.toml
@@ -0,0 +1,5 @@
+cluster_id = "dev_search"
+gossip_addr = "0.0.0.0:3010"
+gossip_seed_nodes = ["0.0.0.0:3005"]
+host = "0.0.0.0:3009"
+index_path = "data/entity"
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index c1f7121b..15001055 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -250,7 +250,6 @@ pub struct SearchServerConfig {
pub gossip_addr: SocketAddr,
pub shard_id: ShardId,
pub index_path: String,
- pub entity_index_path: Option<String>,
pub host_centrality_store_path: Option<String>,
pub linear_model_path: Option<String>,
pub lambda_model_path: Option<String>,
@@ -263,6 +262,15 @@ pub struct SearchServerConfig {
pub snippet: SnippetConfig,
}
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct EntitySearchServerConfig {
+ pub cluster_id: String,
+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
+ pub gossip_addr: SocketAddr,
+ pub index_path: String,
+ pub host: SocketAddr,
+}
+
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct CrawlCoordinatorConfig {
pub job_queue: String,
diff --git a/core/src/distributed/member.rs b/core/src/distributed/member.rs
index 48cfae15..205c9974 100644
--- a/core/src/distributed/member.rs
+++ b/core/src/distributed/member.rs
@@ -26,6 +26,9 @@ pub enum Service {
host: SocketAddr,
shard: ShardId,
},
+ EntitySearcher {
+ host: SocketAddr,
+ },
LiveIndex {
host: SocketAddr,
split_id: crate::feed::scheduler::SplitId,
diff --git a/core/src/entrypoint/entity_search_server.rs b/core/src/entrypoint/entity_search_server.rs
new file mode 100644
index 00000000..e1fb1da4
--- /dev/null
+++ b/core/src/entrypoint/entity_search_server.rs
@@ -0,0 +1,105 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use serde::{Deserialize, Serialize};
+
+use crate::{
+ config,
+ distributed::{
+ cluster::Cluster,
+ member::{Member, Service},
+ sonic,
+ },
+ entity_index::EntityIndex,
+ image_store::Image,
+ sonic_service,
+};
+use anyhow::Result;
+
+sonic_service!(SearchService, [Search, GetEntityImage]);
+
+pub struct SearchService {
+ index: EntityIndex,
+ // dropping the handle leaves the cluster
+ #[allow(unused)]
+ cluster_handle: Cluster,
+}
+impl SearchService {
+ async fn new(config: config::EntitySearchServerConfig) -> Result<Self> {
+ let index = EntityIndex::open(config.index_path)?;
+
+ let cluster_handle = Cluster::join(
+ Member {
+ id: config.cluster_id,
+ service: Service::EntitySearcher { host: config.host },
+ },
+ config.gossip_addr,
+ config.gossip_seed_nodes.unwrap_or_default(),
+ )
+ .await?;
+
+ Ok(SearchService {
+ index,
+ cluster_handle,
+ })
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Search {
+ pub query: String,
+}
+
+impl sonic::service::Message<SearchService> for Search {
+ type Response = Option<crate::entity_index::EntityMatch>;
+ async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
+ Ok(server.index.search(&self.query))
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GetEntityImage {
+ pub image_id: String,
+ pub max_width: Option<u64>,
+ pub max_height: Option<u64>,
+}
+impl sonic::service::Message<SearchService> for GetEntityImage {
+ type Response = Option<Image>;
+ async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
+ Ok(server.index.retrieve_image(&self.image_id).map(|img| {
+ let max_width = self.max_width.unwrap_or(u64::MAX) as u32;
+ let max_height = self.max_height.unwrap_or(u64::MAX) as u32;
+
+ img.resize_max(max_width, max_height)
+ }))
+ }
+}
+
+pub async fn run(config: config::EntitySearchServerConfig) -> Result<()> {
+ let addr = config.host;
+ let server = SearchService::new(config).await?.bind(addr).await.unwrap();
+
+ tracing::info!(
+ "entity search server is ready to accept requests on {}",
+ addr
+ );
+
+ loop {
+ if let Err(e) = server.accept().await {
+ tracing::error!("{:?}", e);
+ }
+ }
+}
diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rs
index 8f40c4db..ee699d68 100644
--- a/core/src/entrypoint/mod.rs
+++ b/core/src/entrypoint/mod.rs
@@ -25,6 +25,7 @@ pub mod configure;
pub mod crawler;
pub mod dmoz_parser;
mod entity;
+pub mod entity_search_server;
pub mod feed_indexer;
pub mod indexer;
pub mod safety_classifier;
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index a20995fa..b3bab405 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -27,8 +27,6 @@ use crate::{
member::{Member, Service},
sonic,
},
- entity_index::{EntityIndex, EntityMatch},
- image_store::Image,
index::Index,
inverted_index::{self, RetrievedWebpage},
ranking::{
@@ -44,10 +42,8 @@ sonic_service!(
[
RetrieveWebsites,
Search,
- SearchEntity,
GetWebpage,
GetHomepageDescriptions,
- GetEntityImage,
]
);
@@ -60,9 +56,6 @@ pub struct SearchService {
impl SearchService {
async fn new(config: config::SearchServerConfig) -> Result<Self> {
- let entity_index = config
- .entity_index_path
- .map(|path| EntityIndex::open(path).unwrap());
let centrality_store = config
.host_centrality_store_path
.map(|p| InboundSimilarity::open(Path::new(&p).join("inbound_similarity")).unwrap());
@@ -70,10 +63,6 @@ impl SearchService {
let mut local_searcher = LocalSearcher::new(search_index);
- if let Some(entity_index) = entity_index {
- local_searcher.set_entity_index(entity_index);
- }
-
if let Some(centrality_store) = centrality_store {
local_searcher.set_inbound_similarity(centrality_store);
}
@@ -173,39 +162,6 @@ impl sonic::service::Message<SearchService> for GetHomepageDescriptions {
}
}
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct GetEntityImage {
- pub image_id: String,
- pub max_width: Option<u64>,
- pub max_height: Option<u64>,
-}
-impl sonic::service::Message<SearchService> for GetEntityImage {
- type Response = Option<Image>;
- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
- Ok(server
- .local_searcher
- .get_entity_image(&self.image_id)
- .map(|img| {
- let max_width = self.max_width.unwrap_or(u64::MAX) as u32;
- let max_height = self.max_height.unwrap_or(u64::MAX) as u32;
-
- img.resize_max(max_width, max_height)
- }))
- }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct SearchEntity {
- pub query: String,
-}
-
-impl sonic::service::Message<SearchService> for SearchEntity {
- type Response = Option<EntityMatch>;
- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
- Ok(server.local_searcher.search_entity(&self.query))
- }
-}
-
pub async fn run(config: config::SearchServerConfig) -> Result<()> {
let addr = config.host;
let server = SearchService::new(config).await?.bind(addr).await.unwrap();
diff --git a/core/src/main.rs b/core/src/main.rs
index c9089a97..8e827445 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -24,7 +24,9 @@ use stract::entrypoint::autosuggest_scrape::{self, Gl};
#[cfg(feature = "dev")]
use stract::entrypoint::configure;
-use stract::entrypoint::{self, api, safety_classifier, search_server, webgraph_server};
+use stract::entrypoint::{
+ self, api, entity_search_server, safety_classifier, search_server, webgraph_server,
+};
use stract::webgraph::WebgraphBuilder;
use tracing_subscriber::prelude::*;
@@ -75,6 +77,11 @@ enum Commands {
config_path: String,
},
+ /// Deploy the entity search server.
+ EntitySearchServer {
+ config_path: String,
+ },
+
/// Deploy the json http api. The api interacts with
/// the search servers, webgraph servers etc. to provide the necesarry functionality.
Api {
@@ -314,6 +321,14 @@ fn main() -> Result<()> {
.build()?
.block_on(search_server::run(config))?
}
+ Commands::EntitySearchServer { config_path } => {
+ let config: config::EntitySearchServerConfig = load_toml_config(config_path);
+
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()?
+ .block_on(entity_search_server::run(config))?
+ }
Commands::AutosuggestScrape {
num_queries: queries_to_scrape,
gl,
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index da2f0403..37705b9d 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -19,12 +19,15 @@ use crate::{
cluster::Cluster,
member::Service,
sonic::replication::{
- AllShardsSelector, RandomReplicaSelector, RandomShardSelector, RemoteClient,
- ReplicatedClient, Shard, ShardIdentifier, ShardedClient, SpecificShardSelector,
+ AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient, Shard,
+ ShardIdentifier, ShardedClient, SpecificShardSelector,
},
},
entity_index::EntityMatch,
- entrypoint::search_server::{self, SearchService},
+ entrypoint::{
+ entity_search_server,
+ search_server::{self, SearchService},
+ },
image_store::Image,
inverted_index::{RetrievedWebpage, WebsitePointer},
ranking::pipeline::{RankingWebsite, RetrievedWebpageRanking},
@@ -107,6 +110,17 @@ impl DistributedSearcher {
ShardedClient::new(shard_clients)
}
+ async fn entity_client(&self) -> ReplicatedClient<entity_search_server::SearchService> {
+ let mut replicas = Vec::new();
+ for member in self.cluster.members().await {
+ if let Service::EntitySearcher { host } = member.service {
+ replicas.push(RemoteClient::new(host));
+ }
+ }
+
+ ReplicatedClient::new(replicas)
+ }
+
async fn retrieve_webpages_from_shard(
&self,
shard: ShardId,
@@ -252,43 +266,37 @@ impl SearchClient for DistributedSearcher {
max_height: Option<u64>,
max_width: Option<u64>,
) -> Result<Option<Image>> {
- let client = self.client().await;
+ let client = self.entity_client().await;
- let (_, mut res) = client
+ Ok(client
.send(
- &search_server::GetEntityImage {
+ &entity_search_server::GetEntityImage {
image_id: image_id.to_string(),
max_height,
max_width,
},
- &RandomShardSelector,
&RandomReplicaSelector,
)
.await
.map_err(|_| Error::SearchFailed)?
.pop()
- .unwrap();
-
- Ok(res.pop().flatten())
+ .unwrap())
}
async fn search_entity(&self, query: &str) -> Option<EntityMatch> {
- let client = self.client().await;
+ let client = self.entity_client().await;
- let (_, mut res) = client
+ client
.send(
- &search_server::SearchEntity {
+ &entity_search_server::Search {
query: query.to_string(),
},
- &RandomShardSelector,
&RandomReplicaSelector,
)
.await
.ok()?
.pop()
- .unwrap();
-
- res.pop().flatten()
+ .unwrap()
}
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 5d32c8e0..c25af51c 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -20,8 +20,6 @@ use std::sync::{Arc, RwLockReadGuard};
use url::Url;
use crate::config::{CollectorConfig, SnippetConfig};
-use crate::entity_index::{EntityIndex, EntityMatch};
-use crate::image_store::Image;
use crate::index::Index;
use crate::inverted_index::{InvertedIndex, RetrievedWebpage};
use crate::query::Query;
@@ -113,7 +111,6 @@ impl<'a> SearchGuard<'a> for LiveIndexSearchGuard<'a> {
pub struct LocalSearcher<I: SearchableIndex> {
index: I,
- entity_index: Option<EntityIndex>,
inbound_similarity: Option<InboundSimilarity>,
linear_regression: Option<Arc<LinearRegression>>,
lambda_model: Option<Arc<LambdaMART>>,
@@ -145,8 +142,6 @@ where
LocalSearcher {
index,
- // spell: None,
- entity_index: None,
inbound_similarity: None,
linear_regression: None,
lambda_model: None,
@@ -154,10 +149,6 @@ where
}
}
- pub fn set_entity_index(&mut self, entity_index: EntityIndex) {
- self.entity_index = Some(entity_index);
- }
-
pub fn set_inbound_similarity(&mut self, inbound: InboundSimilarity) {
self.inbound_similarity = Some(inbound);
}
@@ -322,12 +313,6 @@ where
})
}
- pub fn search_entity(&self, query: &str) -> Option<EntityMatch> {
- self.entity_index
- .as_ref()
- .and_then(|index| index.search(query))
- }
-
pub fn search_initial(
&self,
query: &SearchQuery,
@@ -453,12 +438,6 @@ where
pub fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {
self.index.guard().inverted_index().get_homepage(url)
}
-
- pub fn get_entity_image(&self, image_id: &str) -> Option<Image> {
- self.entity_index
- .as_ref()
- .and_then(|index| index.retrieve_image(image_id))
- }
}
#[cfg(test)]
diff --git a/justfile b/justfile
index bc9d3fd2..04abba66 100644
--- a/justfile
+++ b/justfile
@@ -10,6 +10,8 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")
cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- api configs/api.toml"
@dev-search-server:
cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- search-server configs/search_server.toml"
+@dev-entity-search-server:
+ cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- entity-search-server configs/entity_search_server.toml"
@dev-webgraph:
cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/server.toml"
@dev-llm:
diff --git a/scripts/run_dev.py b/scripts/run_dev.py
index 24727eb9..dab9d148 100755
--- a/scripts/run_dev.py
+++ b/scripts/run_dev.py
@@ -20,6 +20,7 @@ processes = []
processes.append(subprocess.Popen(["just", "dev-api"]))
processes.append(subprocess.Popen(["just", "dev-search-server"]))
+processes.append(subprocess.Popen(["just", "dev-entity-search-server"]))
processes.append(subprocess.Popen(["just", "dev-webgraph"]))
processes.append(subprocess.Popen(["just", "dev-frontend"]))
processes.append(subprocess.Popen(["just", "dev-llm"]))