web search done right
split webgraph server into host and page. allows us to host each graph on separate sets of servers.
Mikkel Denker788f92c8f4c0
diff --git a/configs/webgraph/server.toml b/configs/webgraph/host_server.tomlsimilarity index 60%rename from configs/webgraph/server.tomlrename to configs/webgraph/host_server.tomlindex 1ff98aa4..d21e5b2a 100644--- a/configs/webgraph/server.toml+++ b/configs/webgraph/host_server.toml@@ -1,6 +1,7 @@-host = "0.0.0.0:3003"cluster_id = "dev_webgraph"gossip_addr = "0.0.0.0:3007"-host_graph_path = "data/webgraph_host"-page_graph_path = "data/webgraph_page"+gossip_seed_nodes = ["0.0.0.0:3005"]+granularity = "host"+graph_path = "data/webgraph_host"+host = "0.0.0.0:3003"inbound_similarity_path = "data/centrality/inbound_similarity"diff --git a/configs/webgraph/page_server.toml b/configs/webgraph/page_server.tomlnew file mode 100644index 00000000..46a58021--- /dev/null+++ b/configs/webgraph/page_server.toml@@ -0,0 +1,7 @@+cluster_id = "dev_webgraph"+gossip_addr = "0.0.0.0:3012"+gossip_seed_nodes = ["0.0.0.0:3005"]+granularity = "page"+graph_path = "data/webgraph_page"+host = "0.0.0.0:3011"+inbound_similarity_path = "data/centrality/inbound_similarity"diff --git a/core/src/api/webgraph.rs b/core/src/api/webgraph.rsindex a4db60db..771b7ac1 100644--- a/core/src/api/webgraph.rs+++ b/core/src/api/webgraph.rs@@ -21,8 +21,8 @@ use http::StatusCode;use utoipa::{IntoParams, ToSchema};use crate::{+ config::WebgraphGranularity,distributed::{cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff, sonic},- entrypoint::webgraph_server::GraphLevel,webgraph::{FullEdge, Node},};@@ -37,13 +37,13 @@ impl RemoteWebgraph {Self { cluster }}- async fn host(&self) -> Option<SocketAddr> {+ async fn host(&self, level: WebgraphGranularity) -> Option<SocketAddr> {self.cluster.members().await.iter().find_map(|member| match member.service {- Service::Webgraph { host } => Some(host),+ Service::Webgraph { host, granularity } if granularity == level => Some(host),_ => None,})}@@ -87,7 +87,7 @@ pub mod host {state.counters.explore_counter.inc();let host = state.remote_webgraph- .host()+ .host(WebgraphGranularity::Host).await.ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;@@ -134,7 +134,7 @@ pub mod host {) -> std::result::Result<impl IntoResponse, StatusCode> {let host = state.remote_webgraph- .host()+ .host(WebgraphGranularity::Host).await.ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;@@ -180,7 +180,7 @@ pub mod host {let url = Url::parse(&("http://".to_string() + params.host.as_str())).map_err(|_| StatusCode::BAD_REQUEST)?;let node = Node::from(url).into_host();- let links = ingoing_links(state, node, GraphLevel::Host)+ let links = ingoing_links(state, node, WebgraphGranularity::Host).await.map_err(|_| {tracing::error!("Failed to send request to webgraph");@@ -204,7 +204,7 @@ pub mod host {let url = Url::parse(&("http://".to_string() + params.host.as_str())).map_err(|_| StatusCode::BAD_REQUEST)?;let node = Node::from(url).into_host();- let links = outgoing_links(state, node, GraphLevel::Host)+ let links = outgoing_links(state, node, WebgraphGranularity::Host).await.map_err(|_| {tracing::error!("Failed to send request to webgraph");@@ -236,7 +236,7 @@ pub mod page {extract::Query(params): extract::Query<PageLinksParams>,) -> std::result::Result<impl IntoResponse, StatusCode> {let node = Node::from(params.page);- let links = ingoing_links(state, node, GraphLevel::Page)+ let links = ingoing_links(state, node, WebgraphGranularity::Page).await.map_err(|_| {tracing::error!("Failed to send request to webgraph");@@ -258,7 +258,7 @@ pub mod page {extract::Query(params): extract::Query<PageLinksParams>,) -> std::result::Result<impl IntoResponse, StatusCode> {let node = Node::from(params.page);- let links = outgoing_links(state, node, GraphLevel::Page)+ let links = outgoing_links(state, node, WebgraphGranularity::Page).await.map_err(|_| {tracing::error!("Failed to send request to webgraph");@@ -272,13 +272,15 @@ pub mod page {async fn ingoing_links(state: Arc<State>,node: Node,- level: GraphLevel,+ level: WebgraphGranularity,) -> anyhow::Result<Vec<FullEdge>> {let host = state.remote_webgraph- .host()+ .host(level).await- .ok_or(anyhow::anyhow!("no remote webgraph"))?;+ .ok_or(anyhow::anyhow!(+ "no remote webgraph for granularity {level:?}"+ ))?;let retry = ExponentialBackoff::from_millis(30).with_limit(Duration::from_millis(200))@@ -293,7 +295,7 @@ async fn ingoing_links(Ok(conn.send_with_timeout(- &crate::entrypoint::webgraph_server::IngoingLinks { node, level },+ &crate::entrypoint::webgraph_server::IngoingLinks { node },Duration::from_secs(60),).await?)@@ -302,13 +304,15 @@ async fn ingoing_links(async fn outgoing_links(state: Arc<State>,node: Node,- level: GraphLevel,+ level: WebgraphGranularity,) -> anyhow::Result<Vec<FullEdge>> {let host = state.remote_webgraph- .host()+ .host(level).await- .ok_or(anyhow::anyhow!("no remote webgraph"))?;+ .ok_or(anyhow::anyhow!(+ "no remote webgraph for granularity {level:?}"+ ))?;let retry = ExponentialBackoff::from_millis(30).with_limit(Duration::from_millis(200))@@ -323,7 +327,7 @@ async fn outgoing_links(Ok(conn.send_with_timeout(- &crate::entrypoint::webgraph_server::OutgoingLinks { node, level },+ &crate::entrypoint::webgraph_server::OutgoingLinks { node },Duration::from_secs(60),).await?)diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 15001055..3b616607 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -360,12 +360,20 @@ pub struct AliceLocalConfig {pub encryption_key: String,}+#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy, Debug)]+#[serde(rename_all = "snake_case")]+pub enum WebgraphGranularity {+ Host,+ Page,+}+#[derive(Debug, Serialize, Deserialize, Clone)]pub struct WebgraphServerConfig {pub host: SocketAddr,- pub host_graph_path: String,- pub page_graph_path: String,- pub inbound_similarity_path: String,+ pub graph_path: String,+ pub granularity: WebgraphGranularity,+ pub inbound_similarity_path: Option<String>,+pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,diff --git a/core/src/distributed/member.rs b/core/src/distributed/member.rsindex 205c9974..dfc7e2ac 100644--- a/core/src/distributed/member.rs+++ b/core/src/distributed/member.rs@@ -18,7 +18,7 @@ use std::net::SocketAddr;use serde::{Deserialize, Serialize};-use crate::searcher::ShardId;+use crate::{config::WebgraphGranularity, searcher::ShardId};#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]pub enum Service {@@ -38,6 +38,7 @@ pub enum Service {},Webgraph {host: SocketAddr,+ granularity: WebgraphGranularity,},Alice {host: SocketAddr,diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rsindex e4607fe9..58e9cc95 100644--- a/core/src/entrypoint/webgraph_server.rs+++ b/core/src/entrypoint/webgraph_server.rs@@ -25,6 +25,7 @@ use url::Url;use utoipa::ToSchema;use crate::config;+use crate::config::WebgraphGranularity;use crate::distributed::cluster::Cluster;use crate::distributed::member::Member;use crate::distributed::member::Service;@@ -53,10 +54,10 @@ pub struct ScoredHost {const MAX_HOSTS: usize = 20;pub struct WebGraphService {+ granularity: WebgraphGranularity,searcher: DistributedSearcher,- similar_hosts_finder: SimilarHostsFinder,- host_graph: Arc<Webgraph>,- page_graph: Arc<Webgraph>,+ similar_hosts_finder: Option<SimilarHostsFinder>,+ graph: Arc<Webgraph>,}sonic_service!(@@ -79,9 +80,15 @@ impl Message<WebGraphService> for SimilarHosts {type Response = Vec<ScoredHost>;async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {+ if server.similar_hosts_finder.is_none() {+ return Ok(vec![]);+ }+let sites = &self.hosts[..std::cmp::min(self.hosts.len(), MAX_HOSTS)];let similar_hosts = server.similar_hosts_finder+ .as_ref()+ .unwrap().find_similar_hosts(sites, self.top_n);let urls = similar_hosts@@ -119,7 +126,12 @@ impl Message<WebGraphService> for Knows {let node = Node::from(url).into_host();- if server.similar_hosts_finder.knows_about(&node) {+ if server+ .similar_hosts_finder+ .as_ref()+ .map(|finder| finder.knows_about(&node))+ .unwrap_or(false)+ {Ok(Some(node))} else {Ok(None)@@ -127,28 +139,21 @@ impl Message<WebGraphService> for Knows {}}-#[derive(Debug, Clone, Serialize, Deserialize)]-pub enum GraphLevel {- Host,- Page,-}-#[derive(Debug, Clone, Serialize, Deserialize)]pub struct IngoingLinks {pub node: Node,- pub level: GraphLevel,}impl Message<WebGraphService> for IngoingLinks {type Response = Vec<FullEdge>;async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {- match self.level {- GraphLevel::Host => {+ match server.granularity {+ WebgraphGranularity::Host => {let node = self.node.into_host();- Ok(server.host_graph.ingoing_edges(node))+ Ok(server.graph.ingoing_edges(node))}- GraphLevel::Page => Ok(server.page_graph.ingoing_edges(self.node)),+ WebgraphGranularity::Page => Ok(server.graph.ingoing_edges(self.node)),}}}@@ -156,19 +161,18 @@ impl Message<WebGraphService> for IngoingLinks {#[derive(Debug, Clone, Serialize, Deserialize)]pub struct OutgoingLinks {pub node: Node,- pub level: GraphLevel,}impl Message<WebGraphService> for OutgoingLinks {type Response = Vec<FullEdge>;async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {- match self.level {- GraphLevel::Host => {+ match server.granularity {+ WebgraphGranularity::Host => {let node = self.node.into_host();- Ok(server.host_graph.outgoing_edges(node))+ Ok(server.graph.outgoing_edges(node))}- GraphLevel::Page => Ok(server.page_graph.outgoing_edges(self.node)),+ WebgraphGranularity::Page => Ok(server.graph.outgoing_edges(self.node)),}}}@@ -181,7 +185,10 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {Cluster::join(Member {id: config.cluster_id,- service: Service::Webgraph { host: addr },+ service: Service::Webgraph {+ host: addr,+ granularity: config.granularity,+ },},config.gossip_addr,config.gossip_seed_nodes.unwrap_or_default(),@@ -190,29 +197,26 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {);let searcher = DistributedSearcher::new(cluster);- let host_graph = Arc::new(- WebgraphBuilder::new(config.host_graph_path)+ let graph = Arc::new(+ WebgraphBuilder::new(config.graph_path).compression(Compression::Lz4).open(),);- let page_graph = Arc::new(- WebgraphBuilder::new(config.page_graph_path)- .compression(Compression::Lz4)- .open(),- );- let inbound_similarity = InboundSimilarity::open(config.inbound_similarity_path)?;- let similar_hosts_finder = SimilarHostsFinder::new(- Arc::clone(&host_graph),- inbound_similarity,- config.max_similar_hosts,- );+ let similar_hosts_finder = config.inbound_similarity_path.map(|path| {+ let inbound_similarity = InboundSimilarity::open(path).unwrap();+ SimilarHostsFinder::new(+ Arc::clone(&graph),+ inbound_similarity,+ config.max_similar_hosts,+ )+ });let server = WebGraphService {- host_graph,- page_graph,+ graph,searcher,similar_hosts_finder,+ granularity: config.granularity,}.bind(addr).awaitdiff --git a/justfile b/justfileindex 04abba66..60c958d8 100644--- a/justfile+++ b/justfile@@ -13,7 +13,7 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")@dev-entity-search-server:cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- entity-search-server configs/entity_search_server.toml"@dev-webgraph:- cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/server.toml"+ cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/host_server.toml" && cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/page_server.toml"@dev-llm:./.venv/bin/python3 -m llama_cpp.server --model data/mistral-7b-instruct-v0.2.Q4_K_M.gguf --port 4000@dev-frontend: