Skip to content

web search done right

split webgraph server into host and page. allows us to host each graph on separate sets of servers.

Mikkel Denker788f92c8f4c0
diff --git a/configs/webgraph/server.toml b/configs/webgraph/host_server.toml
similarity index 60%
rename from configs/webgraph/server.toml
rename to configs/webgraph/host_server.toml
index 1ff98aa4..d21e5b2a 100644
--- a/configs/webgraph/server.toml
+++ b/configs/webgraph/host_server.toml
@@ -1,6 +1,7 @@
-host = "0.0.0.0:3003"
cluster_id = "dev_webgraph"
gossip_addr = "0.0.0.0:3007"
-host_graph_path = "data/webgraph_host"
-page_graph_path = "data/webgraph_page"
+gossip_seed_nodes = ["0.0.0.0:3005"]
+granularity = "host"
+graph_path = "data/webgraph_host"
+host = "0.0.0.0:3003"
inbound_similarity_path = "data/centrality/inbound_similarity"
diff --git a/configs/webgraph/page_server.toml b/configs/webgraph/page_server.toml
new file mode 100644
index 00000000..46a58021
--- /dev/null
+++ b/configs/webgraph/page_server.toml
@@ -0,0 +1,7 @@
+cluster_id = "dev_webgraph"
+gossip_addr = "0.0.0.0:3012"
+gossip_seed_nodes = ["0.0.0.0:3005"]
+granularity = "page"
+graph_path = "data/webgraph_page"
+host = "0.0.0.0:3011"
+inbound_similarity_path = "data/centrality/inbound_similarity"
diff --git a/core/src/api/webgraph.rs b/core/src/api/webgraph.rs
index a4db60db..771b7ac1 100644
--- a/core/src/api/webgraph.rs
+++ b/core/src/api/webgraph.rs
@@ -21,8 +21,8 @@ use http::StatusCode;
use utoipa::{IntoParams, ToSchema};
use crate::{
+ config::WebgraphGranularity,
distributed::{cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff, sonic},
- entrypoint::webgraph_server::GraphLevel,
webgraph::{FullEdge, Node},
};
@@ -37,13 +37,13 @@ impl RemoteWebgraph {
Self { cluster }
}
- async fn host(&self) -> Option<SocketAddr> {
+ async fn host(&self, level: WebgraphGranularity) -> Option<SocketAddr> {
self.cluster
.members()
.await
.iter()
.find_map(|member| match member.service {
- Service::Webgraph { host } => Some(host),
+ Service::Webgraph { host, granularity } if granularity == level => Some(host),
_ => None,
})
}
@@ -87,7 +87,7 @@ pub mod host {
state.counters.explore_counter.inc();
let host = state
.remote_webgraph
- .host()
+ .host(WebgraphGranularity::Host)
.await
.ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;
@@ -134,7 +134,7 @@ pub mod host {
) -> std::result::Result<impl IntoResponse, StatusCode> {
let host = state
.remote_webgraph
- .host()
+ .host(WebgraphGranularity::Host)
.await
.ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;
@@ -180,7 +180,7 @@ pub mod host {
let url = Url::parse(&("http://".to_string() + params.host.as_str()))
.map_err(|_| StatusCode::BAD_REQUEST)?;
let node = Node::from(url).into_host();
- let links = ingoing_links(state, node, GraphLevel::Host)
+ let links = ingoing_links(state, node, WebgraphGranularity::Host)
.await
.map_err(|_| {
tracing::error!("Failed to send request to webgraph");
@@ -204,7 +204,7 @@ pub mod host {
let url = Url::parse(&("http://".to_string() + params.host.as_str()))
.map_err(|_| StatusCode::BAD_REQUEST)?;
let node = Node::from(url).into_host();
- let links = outgoing_links(state, node, GraphLevel::Host)
+ let links = outgoing_links(state, node, WebgraphGranularity::Host)
.await
.map_err(|_| {
tracing::error!("Failed to send request to webgraph");
@@ -236,7 +236,7 @@ pub mod page {
extract::Query(params): extract::Query<PageLinksParams>,
) -> std::result::Result<impl IntoResponse, StatusCode> {
let node = Node::from(params.page);
- let links = ingoing_links(state, node, GraphLevel::Page)
+ let links = ingoing_links(state, node, WebgraphGranularity::Page)
.await
.map_err(|_| {
tracing::error!("Failed to send request to webgraph");
@@ -258,7 +258,7 @@ pub mod page {
extract::Query(params): extract::Query<PageLinksParams>,
) -> std::result::Result<impl IntoResponse, StatusCode> {
let node = Node::from(params.page);
- let links = outgoing_links(state, node, GraphLevel::Page)
+ let links = outgoing_links(state, node, WebgraphGranularity::Page)
.await
.map_err(|_| {
tracing::error!("Failed to send request to webgraph");
@@ -272,13 +272,15 @@ pub mod page {
async fn ingoing_links(
state: Arc<State>,
node: Node,
- level: GraphLevel,
+ level: WebgraphGranularity,
) -> anyhow::Result<Vec<FullEdge>> {
let host = state
.remote_webgraph
- .host()
+ .host(level)
.await
- .ok_or(anyhow::anyhow!("no remote webgraph"))?;
+ .ok_or(anyhow::anyhow!(
+ "no remote webgraph for granularity {level:?}"
+ ))?;
let retry = ExponentialBackoff::from_millis(30)
.with_limit(Duration::from_millis(200))
@@ -293,7 +295,7 @@ async fn ingoing_links(
Ok(conn
.send_with_timeout(
- &crate::entrypoint::webgraph_server::IngoingLinks { node, level },
+ &crate::entrypoint::webgraph_server::IngoingLinks { node },
Duration::from_secs(60),
)
.await?)
@@ -302,13 +304,15 @@ async fn ingoing_links(
async fn outgoing_links(
state: Arc<State>,
node: Node,
- level: GraphLevel,
+ level: WebgraphGranularity,
) -> anyhow::Result<Vec<FullEdge>> {
let host = state
.remote_webgraph
- .host()
+ .host(level)
.await
- .ok_or(anyhow::anyhow!("no remote webgraph"))?;
+ .ok_or(anyhow::anyhow!(
+ "no remote webgraph for granularity {level:?}"
+ ))?;
let retry = ExponentialBackoff::from_millis(30)
.with_limit(Duration::from_millis(200))
@@ -323,7 +327,7 @@ async fn outgoing_links(
Ok(conn
.send_with_timeout(
- &crate::entrypoint::webgraph_server::OutgoingLinks { node, level },
+ &crate::entrypoint::webgraph_server::OutgoingLinks { node },
Duration::from_secs(60),
)
.await?)
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 15001055..3b616607 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -360,12 +360,20 @@ pub struct AliceLocalConfig {
pub encryption_key: String,
}
+#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy, Debug)]
+#[serde(rename_all = "snake_case")]
+pub enum WebgraphGranularity {
+ Host,
+ Page,
+}
+
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct WebgraphServerConfig {
pub host: SocketAddr,
- pub host_graph_path: String,
- pub page_graph_path: String,
- pub inbound_similarity_path: String,
+ pub graph_path: String,
+ pub granularity: WebgraphGranularity,
+ pub inbound_similarity_path: Option<String>,
+
pub cluster_id: String,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
diff --git a/core/src/distributed/member.rs b/core/src/distributed/member.rs
index 205c9974..dfc7e2ac 100644
--- a/core/src/distributed/member.rs
+++ b/core/src/distributed/member.rs
@@ -18,7 +18,7 @@ use std::net::SocketAddr;
use serde::{Deserialize, Serialize};
-use crate::searcher::ShardId;
+use crate::{config::WebgraphGranularity, searcher::ShardId};
#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]
pub enum Service {
@@ -38,6 +38,7 @@ pub enum Service {
},
Webgraph {
host: SocketAddr,
+ granularity: WebgraphGranularity,
},
Alice {
host: SocketAddr,
diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rs
index e4607fe9..58e9cc95 100644
--- a/core/src/entrypoint/webgraph_server.rs
+++ b/core/src/entrypoint/webgraph_server.rs
@@ -25,6 +25,7 @@ use url::Url;
use utoipa::ToSchema;
use crate::config;
+use crate::config::WebgraphGranularity;
use crate::distributed::cluster::Cluster;
use crate::distributed::member::Member;
use crate::distributed::member::Service;
@@ -53,10 +54,10 @@ pub struct ScoredHost {
const MAX_HOSTS: usize = 20;
pub struct WebGraphService {
+ granularity: WebgraphGranularity,
searcher: DistributedSearcher,
- similar_hosts_finder: SimilarHostsFinder,
- host_graph: Arc<Webgraph>,
- page_graph: Arc<Webgraph>,
+ similar_hosts_finder: Option<SimilarHostsFinder>,
+ graph: Arc<Webgraph>,
}
sonic_service!(
@@ -79,9 +80,15 @@ impl Message<WebGraphService> for SimilarHosts {
type Response = Vec<ScoredHost>;
async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {
+ if server.similar_hosts_finder.is_none() {
+ return Ok(vec![]);
+ }
+
let sites = &self.hosts[..std::cmp::min(self.hosts.len(), MAX_HOSTS)];
let similar_hosts = server
.similar_hosts_finder
+ .as_ref()
+ .unwrap()
.find_similar_hosts(sites, self.top_n);
let urls = similar_hosts
@@ -119,7 +126,12 @@ impl Message<WebGraphService> for Knows {
let node = Node::from(url).into_host();
- if server.similar_hosts_finder.knows_about(&node) {
+ if server
+ .similar_hosts_finder
+ .as_ref()
+ .map(|finder| finder.knows_about(&node))
+ .unwrap_or(false)
+ {
Ok(Some(node))
} else {
Ok(None)
@@ -127,28 +139,21 @@ impl Message<WebGraphService> for Knows {
}
}
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum GraphLevel {
- Host,
- Page,
-}
-
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IngoingLinks {
pub node: Node,
- pub level: GraphLevel,
}
impl Message<WebGraphService> for IngoingLinks {
type Response = Vec<FullEdge>;
async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {
- match self.level {
- GraphLevel::Host => {
+ match server.granularity {
+ WebgraphGranularity::Host => {
let node = self.node.into_host();
- Ok(server.host_graph.ingoing_edges(node))
+ Ok(server.graph.ingoing_edges(node))
}
- GraphLevel::Page => Ok(server.page_graph.ingoing_edges(self.node)),
+ WebgraphGranularity::Page => Ok(server.graph.ingoing_edges(self.node)),
}
}
}
@@ -156,19 +161,18 @@ impl Message<WebGraphService> for IngoingLinks {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OutgoingLinks {
pub node: Node,
- pub level: GraphLevel,
}
impl Message<WebGraphService> for OutgoingLinks {
type Response = Vec<FullEdge>;
async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {
- match self.level {
- GraphLevel::Host => {
+ match server.granularity {
+ WebgraphGranularity::Host => {
let node = self.node.into_host();
- Ok(server.host_graph.outgoing_edges(node))
+ Ok(server.graph.outgoing_edges(node))
}
- GraphLevel::Page => Ok(server.page_graph.outgoing_edges(self.node)),
+ WebgraphGranularity::Page => Ok(server.graph.outgoing_edges(self.node)),
}
}
}
@@ -181,7 +185,10 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
Cluster::join(
Member {
id: config.cluster_id,
- service: Service::Webgraph { host: addr },
+ service: Service::Webgraph {
+ host: addr,
+ granularity: config.granularity,
+ },
},
config.gossip_addr,
config.gossip_seed_nodes.unwrap_or_default(),
@@ -190,29 +197,26 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
);
let searcher = DistributedSearcher::new(cluster);
- let host_graph = Arc::new(
- WebgraphBuilder::new(config.host_graph_path)
+ let graph = Arc::new(
+ WebgraphBuilder::new(config.graph_path)
.compression(Compression::Lz4)
.open(),
);
- let page_graph = Arc::new(
- WebgraphBuilder::new(config.page_graph_path)
- .compression(Compression::Lz4)
- .open(),
- );
- let inbound_similarity = InboundSimilarity::open(config.inbound_similarity_path)?;
- let similar_hosts_finder = SimilarHostsFinder::new(
- Arc::clone(&host_graph),
- inbound_similarity,
- config.max_similar_hosts,
- );
+ let similar_hosts_finder = config.inbound_similarity_path.map(|path| {
+ let inbound_similarity = InboundSimilarity::open(path).unwrap();
+ SimilarHostsFinder::new(
+ Arc::clone(&graph),
+ inbound_similarity,
+ config.max_similar_hosts,
+ )
+ });
let server = WebGraphService {
- host_graph,
- page_graph,
+ graph,
searcher,
similar_hosts_finder,
+ granularity: config.granularity,
}
.bind(addr)
.await
diff --git a/justfile b/justfile
index 04abba66..60c958d8 100644
--- a/justfile
+++ b/justfile
@@ -13,7 +13,7 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")
@dev-entity-search-server:
cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- entity-search-server configs/entity_search_server.toml"
@dev-webgraph:
- cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/server.toml"
+ cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/host_server.toml" && cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/page_server.toml"
@dev-llm:
./.venv/bin/python3 -m llama_cpp.server --model data/mistral-7b-instruct-v0.2.Q4_K_M.gguf --port 4000
@dev-frontend: