web search done right
Fixed a bug where distance calculation in online-harmonic used the wrong node from the edge
Mikkel Denkercb64b49ad9d9
diff --git a/configs/frontend.toml b/configs/frontend.tomlindex 79048ff0..c263b2a9 100644--- a/configs/frontend.toml+++ b/configs/frontend.toml@@ -2,8 +2,8 @@ cluster_id = "dev_frontend"queries_csv_path = "data/queries_us.csv"host = "0.0.0.0:3000"prometheus_host = "0.0.0.0:3001"-gossip_addr = "0.0.0.0:3003"-gossip_seed_nodes = ["0.0.0.0:3004"]+gossip_addr = "0.0.0.0:3004"+gossip_seed_nodes = ["0.0.0.0:3005", "0.0.0.0:3006"]crossencoder_model_path = "data/cross_encoder"# lambda_model_path = "data/lambdamart.txt"# qa_model_path = "data/qa_model"diff --git a/configs/search_server.toml b/configs/search_server.tomlindex f143b8ea..4428732e 100644--- a/configs/search_server.toml+++ b/configs/search_server.toml@@ -1,5 +1,5 @@cluster_id = "dev_search"-gossip_addr = "0.0.0.0:3004"+gossip_addr = "0.0.0.0:3005"shard_id = 0host = "0.0.0.0:3002"index_path = "data/index"diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.tomlindex 9d3a11c2..1074ec96 100644--- a/configs/webgraph/local.toml+++ b/configs/webgraph/local.toml@@ -1,8 +1,8 @@-limit_warc_files = 10+limit_warc_files = 5graph_base_path = "data/webgraph"batch_size = 1[warc_source]type = "HTTP"base_url = "https://data.commoncrawl.org/"-warc_paths_file = "data/warc.paths"\ No newline at end of file+warc_paths_file = "data/warc.paths"diff --git a/configs/webgraph/server.toml b/configs/webgraph/server.tomlnew file mode 100644index 00000000..472ae19b--- /dev/null+++ b/configs/webgraph/server.toml@@ -0,0 +1,5 @@+host = "0.0.0.0:3003"+cluster_id = "dev_webgraph"+gossip_addr = "0.0.0.0:3006"+graph_path = "data/webgraph"+inbound_similarity_path = "data/centrality/inbound_similarity"\ No newline at end of filediff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rsindex ccff3424..f984edc0 100644--- a/core/examples/print_inbound_links.rs+++ b/core/examples/print_inbound_links.rs@@ -23,7 +23,7 @@ pub fn main() {let graph = WebgraphBuilder::new("data/webgraph").open();let inbound = InboundSimilarity::open("data/centrality/inbound_similarity").unwrap();- for host in ["www.homedepot.com", "doc.rust-lang.org"] {+ for host in ["www.homedepot.com"] {println!("{host}:");let node = Node::from(host).into_host();diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex 7cea00ee..e8bb821f 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -21,9 +21,9 @@ use tower_http::{compression::CompressionLayer, services::ServeDir};use crate::{autosuggest::Autosuggest,bangs::Bangs,- cluster::{+ distributed::{+ cluster::Cluster,member::{Member, Service},- Cluster,},leaky_queue::LeakyQueue,qa_model::QaModel,@@ -44,6 +44,8 @@ use axum::{routing::post,};+use self::webgraph::RemoteWebgraph;+mod about;mod autosuggest;mod improvement;@@ -55,11 +57,13 @@ mod privacy;pub mod search;mod sites;mod summarize;+mod webgraph;pub struct HtmlTemplate<T>(T);pub struct State {pub searcher: DistributedSearcher,+ pub remote_webgraph: RemoteWebgraph,pub autosuggest: Autosuggest,pub search_counter_success: crate::metrics::Counter,pub search_counter_fail: crate::metrics::Counter,@@ -122,15 +126,18 @@ pub async fn router(let bangs = Bangs::from_path(&config.bangs_path);- let cluster = Cluster::join(- Member {- id: config.cluster_id.clone(),- service: Service::Frontend { host: config.host },- },- config.gossip_addr,- config.gossip_seed_nodes.clone().unwrap_or_default(),- )- .await?;+ let cluster = Arc::new(+ Cluster::join(+ Member {+ id: config.cluster_id.clone(),+ service: Service::Frontend { host: config.host },+ },+ config.gossip_addr,+ config.gossip_seed_nodes.clone().unwrap_or_default(),+ )+ .await?,+ );+ let remote_webgraph = RemoteWebgraph::new(cluster.clone());let searcher = DistributedSearcher::new(cluster, crossencoder, lambda_model, qa_model, bangs);let state = Arc::new(State {@@ -138,6 +145,7 @@ pub async fn router(autosuggest,search_counter_success,search_counter_fail,+ remote_webgraph,summarizer: Arc::new(Summarizer::open(&config.summarizer_path)?),improvement_queue: query_store_queue,});@@ -164,7 +172,11 @@ pub async fn router(.route("/improvement/store", post(improvement::store)).fallback(get_service(ServeDir::new("frontend/dist/"))).layer(CompressionLayer::new())- .merge(Router::new().route("/summarize", get(summarize::route)))+ .merge(+ Router::new()+ .route("/beta/api/summarize", get(summarize::route))+ .route("/beta/api/similar_sites", post(webgraph::similar_sites)),+ ).with_state(state))}diff --git a/core/src/api/webgraph.rs b/core/src/api/webgraph.rsnew file mode 100644index 00000000..607735ea--- /dev/null+++ b/core/src/api/webgraph.rs@@ -0,0 +1,104 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{net::SocketAddr, sync::Arc, time::Duration};++use axum::{extract, response::IntoResponse, Json};+use http::StatusCode;+use itertools::Itertools;++use crate::{+ distributed::{cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff, sonic},+ similar_sites::ScoredNode,+};++use super::State;++pub struct RemoteWebgraph {+ cluster: Arc<Cluster>,+}++impl RemoteWebgraph {+ pub fn new(cluster: Arc<Cluster>) -> Self {+ Self { cluster }+ }++ async fn host(&self) -> SocketAddr {+ self.cluster+ .members()+ .await+ .iter()+ .find_map(|member| match member.service {+ Service::Webgraph { host } => Some(host),+ _ => None,+ })+ .unwrap()+ }+}++#[derive(serde::Deserialize)]+pub struct Params {+ pub sites: Vec<String>,+ pub top_n: usize,+}++#[derive(serde::Serialize, serde::Deserialize)]+struct ScoredSite {+ pub site: String,+ pub score: f64,+}++#[allow(clippy::unused_async)]+pub async fn similar_sites(+ extract::State(state): extract::State<Arc<State>>,+ extract::Json(params): extract::Json<Params>,+) -> std::result::Result<impl IntoResponse, StatusCode> {+ let host = state.remote_webgraph.host().await;++ let retry = ExponentialBackoff::from_millis(30)+ .with_limit(Duration::from_millis(200))+ .take(5);++ let mut conn = sonic::ResilientConnection::create(host, retry);++ let top_n = params.top_n.min(1000);++ match conn+ .send_with_timeout::<_, Vec<ScoredNode>>(+ &crate::entrypoint::webgraph_server::Request::SimilarSites {+ sites: params.sites,+ top_n,+ },+ Duration::from_secs(30),+ )+ .await+ {+ Ok(sonic::Response::Content(nodes)) => Ok(Json(+ nodes+ .into_iter()+ .map(|node| ScoredSite {+ site: node.node.name,+ score: node.score,+ })+ .collect_vec(),+ )),+ Err(err) => {+ tracing::error!("Failed to send request to webgraph: {}", err);+ Err(StatusCode::INTERNAL_SERVER_ERROR)+ }+ _ => Err(StatusCode::INTERNAL_SERVER_ERROR),+ }+}diff --git a/core/src/cluster/mod.rs b/core/src/distributed/cluster.rssimilarity index 95%rename from core/src/cluster/mod.rsrename to core/src/distributed/cluster.rsindex 682508be..ee4ebee7 100644--- a/core/src/cluster/mod.rs+++ b/core/src/distributed/cluster.rs@@ -22,14 +22,12 @@ use tokio::sync::RwLock;use tokio_stream::StreamExt;use tracing::error;-use self::member::{Member, Service};+use crate::distributed::member::{Member, Service};const CLUSTER_ID: &str = "stract-cluster";const GOSSIP_INTERVAL: Duration = Duration::from_secs(1);const SERVICE_KEY: &str = "service";-pub mod member;-#[derive(Debug, thiserror::Error)]pub enum Error {#[error("Anyhow")]@@ -55,13 +53,14 @@ impl Cluster {) -> Result<Self> {let transport = UdpTransport;let failure_detector_config = FailureDetectorConfig {- phi_threshold: 10.0, // TODO: choose this value based on testsinitial_interval: GOSSIP_INTERVAL,..Default::default()};+ let uuid = uuid::Uuid::new_v4().to_string();+let node_id = NodeId {- id: self_node.id.clone(),+ id: format!("{}_{}", self_node.id, uuid),gossip_public_address: gossip_addr,};@@ -125,6 +124,7 @@ impl Cluster {}}+ tracing::info!("new members: {:#?}", new_members);let mut write = alive_nodes_ref.write().await;write.clear();diff --git a/core/src/cluster/member.rs b/core/src/distributed/member.rssimilarity index 97%rename from core/src/cluster/member.rsrename to core/src/distributed/member.rsindex f1c470bb..fd9e0a18 100644--- a/core/src/cluster/member.rs+++ b/core/src/distributed/member.rs@@ -24,6 +24,7 @@ use crate::searcher::ShardId;pub enum Service {Searcher { host: SocketAddr, shard: ShardId },Frontend { host: SocketAddr },+ Webgraph { host: SocketAddr },}#[derive(PartialEq, Eq, Hash, Clone, Debug)]diff --git a/core/src/distributed/mod.rs b/core/src/distributed/mod.rsnew file mode 100644index 00000000..9d64573f--- /dev/null+++ b/core/src/distributed/mod.rs@@ -0,0 +1,19 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+pub mod cluster;+pub mod member;+pub mod retry_strategy;+pub mod sonic;diff --git a/core/src/exponential_backoff.rs b/core/src/distributed/retry_strategy.rssimilarity index 100%rename from core/src/exponential_backoff.rsrename to core/src/distributed/retry_strategy.rsdiff --git a/core/src/sonic.rs b/core/src/distributed/sonic.rssimilarity index 79%rename from core/src/sonic.rsrename to core/src/distributed/sonic.rsindex 10073cf1..62d77092 100644--- a/core/src/sonic.rs+++ b/core/src/distributed/sonic.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::time::Duration;+use std::{net::SocketAddr, time::Duration};use serde::{de::DeserializeOwned, Deserialize, Serialize};use thiserror::Error;@@ -132,7 +132,7 @@ impl Connection {pub async fn send_without_timeout<T: Serialize, R: DeserializeOwned + Serialize>(mut self,- request: T,+ request: &T,) -> Result<Response<R>> {let bytes = bincode::serialize(&request).unwrap();@@ -160,7 +160,7 @@ impl Connection {pub async fn send<T: Serialize, R: DeserializeOwned + Serialize>(self,- request: T,+ request: &T,) -> Result<Response<R>> {self.send_with_timeout(request, Duration::from_secs(30)).await@@ -168,7 +168,7 @@ impl Connection {pub async fn send_with_timeout<T: Serialize, R: DeserializeOwned + Serialize>(self,- request: T,+ request: &T,timeout: Duration,) -> Result<Response<R>> {match tokio::time::timeout(timeout, self.send_without_timeout(request)).await {@@ -177,3 +177,41 @@ impl Connection {}}}++pub struct ResilientConnection<Rt: Iterator<Item = Duration>> {+ addr: SocketAddr,+ retry: Rt,+}++impl<Rt: Iterator<Item = Duration>> ResilientConnection<Rt> {+ pub fn create(addr: SocketAddr, retry: Rt) -> Self {+ Self { addr, retry }+ }++ pub async fn send_with_timeout<T: Serialize, R: DeserializeOwned + Serialize>(+ &mut self,+ request: &T,+ timeout: Duration,+ ) -> Result<Response<R>> {+ loop {+ match Connection::create_with_timeout(&self.addr, timeout).await {+ Ok(conn) => {+ let response = conn.send_with_timeout(request, timeout).await;+ if let Err(Error::ConnectionTimeout) = response {+ continue;+ }+ return response;+ }+ Err(Error::ConnectionTimeout) => {+ if let Some(timeout) = self.retry.next() {+ tokio::time::sleep(timeout).await;+ continue;+ } else {+ return Err(Error::ConnectionTimeout);+ }+ }+ Err(e) => return Err(e),+ }+ }+ }+}diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rsindex f5dff68c..374c9dfb 100644--- a/core/src/entrypoint/mod.rs+++ b/core/src/entrypoint/mod.rs@@ -25,6 +25,7 @@ pub mod indexer;pub mod search_server;pub mod topic_centrality;mod webgraph;+pub mod webgraph_server;use std::{fs::File, path::Path};diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex 5812e0d7..284919c4 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -19,9 +19,10 @@ use std::net::SocketAddr;use tracing::info;use crate::{- cluster::{+ distributed::{+ cluster::Cluster,member::{Member, Service},- Cluster,+ sonic,},entity_index::EntityIndex,index::Index,@@ -31,7 +32,7 @@ use crate::{models::{lambdamart::LambdaMART, linear::LinearRegression},},searcher::{self, LocalSearcher},- sonic, Result, SearchServerConfig,+ Result, SearchServerConfig,};pub async fn run(config: SearchServerConfig) -> Result<()> {diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rsnew file mode 100644index 00000000..6517b3e9--- /dev/null+++ b/core/src/entrypoint/webgraph_server.rs@@ -0,0 +1,73 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::net::SocketAddr;++use serde::Deserialize;+use serde::Serialize;+use tracing::info;++use crate::distributed::cluster::Cluster;+use crate::distributed::member::Member;+use crate::distributed::member::Service;+use crate::distributed::sonic;+use crate::ranking::inbound_similarity::InboundSimilarity;+use crate::similar_sites::SimilarSitesFinder;+use crate::webgraph::WebgraphBuilder;+use crate::Result;+use crate::WebgraphServerConfig;++#[derive(Serialize, Deserialize)]+pub enum Request {+ SimilarSites { sites: Vec<String>, top_n: usize },+}++pub async fn run(config: WebgraphServerConfig) -> Result<()> {+ let addr: SocketAddr = config.host;++ // dropping the handle leaves the cluster+ let _cluster_handle = Cluster::join(+ Member {+ id: config.cluster_id,+ service: Service::Webgraph { host: addr },+ },+ config.gossip_addr,+ config.gossip_seed_nodes.unwrap_or_default(),+ )+ .await?;++ let graph = WebgraphBuilder::new(config.graph_path).open();+ let inbound_similarity = InboundSimilarity::open(config.inbound_similarity_path)?;++ let similar_sites_finder = SimilarSitesFinder::new(graph, inbound_similarity);++ let server = sonic::Server::bind(addr).await.unwrap();++ info!("webgraph server is ready to accept requests on {}", addr);++ loop {+ if let Ok(req) = server.accept::<Request>().await {+ match &req.body {+ Request::SimilarSites { sites, top_n } => {+ let similar_sites = similar_sites_finder.find_similar_sites(sites, *top_n);+ req.respond(sonic::Response::Content(similar_sites))+ .await+ .ok();+ }+ }+ }+ }+}diff --git a/core/src/image_downloader.rs b/core/src/image_downloader.rsindex 011e8918..8530c493 100644--- a/core/src/image_downloader.rs+++ b/core/src/image_downloader.rs@@ -19,7 +19,7 @@ use std::{collections::HashSet, hash::Hash, time::Duration};use serde::Serialize;use crate::{- exponential_backoff::ExponentialBackoff,+ distributed::retry_strategy::ExponentialBackoff,image_store::{Image, ImageStore},webpage::Url,};diff --git a/core/src/lib.rs b/core/src/lib.rsindex 53427959..7c3ae9cc 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -18,7 +18,7 @@#![allow(clippy::cast_precision_loss)]#![allow(clippy::missing_errors_doc)]-use searcher::{distributed, ShardId};+use searcher::ShardId;use serde::{Deserialize, Serialize};use std::fs::File;use std::io::{self, BufRead};@@ -36,13 +36,12 @@ pub mod mapreduce;mod api;mod autosuggest;mod bangs;-mod cluster;mod collector;mod directory;+mod distributed;mod entity_index;mod enum_map;mod executor;-mod exponential_backoff;mod fastfield_reader;mod human_website_annotations;mod hyperloglog;@@ -66,7 +65,6 @@ pub mod searcher;mod simhash;pub mod similar_sites;mod snippet;-mod sonic;pub mod spell;mod subdomain_count;pub mod summarizer;@@ -164,7 +162,6 @@ pub struct HttpConfig {#[derive(Debug, Serialize, Deserialize, Clone)]pub struct FrontendConfig {- pub cluster_id: String,pub queries_csv_path: String,pub host: SocketAddr,pub prometheus_host: SocketAddr,@@ -174,6 +171,7 @@ pub struct FrontendConfig {pub bangs_path: String,pub summarizer_path: String,pub query_store_db_host: Option<String>,+ pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,}@@ -192,6 +190,16 @@ pub struct SearchServerConfig {pub host: SocketAddr,}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct WebgraphServerConfig {+ pub host: SocketAddr,+ pub graph_path: String,+ pub inbound_similarity_path: String,+ pub cluster_id: String,+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,+ pub gossip_addr: SocketAddr,+}+#[derive(Error, Debug)]pub enum Error {#[error("Failed to download object from HTTP")]@@ -273,10 +281,10 @@ pub enum Error {InternalError(String),#[error("Distributed searcher")]- DistributedSearcher(#[from] distributed::Error),+ DistributedSearcher(#[from] searcher::distributed::Error),#[error("Cluster")]- Cluster(#[from] cluster::Error),+ Cluster(#[from] crate::distributed::cluster::Error),#[error("Lambdamart")]LambdaMART(#[from] ranking::models::lambdamart::Error),diff --git a/core/src/main.rs b/core/src/main.rsindex 39a8e153..3a8697b8 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -22,9 +22,9 @@ use stract::entrypoint::autosuggest_scrape::{self, Gl};#[cfg(feature = "dev")]use stract::entrypoint::configure;use stract::entrypoint::indexer::IndexPointer;-use stract::entrypoint::{self, frontend, search_server};+use stract::entrypoint::{self, frontend, search_server, webgraph_server};use stract::webgraph::WebgraphBuilder;-use stract::{FrontendConfig, SearchServerConfig};+use stract::{FrontendConfig, SearchServerConfig, WebgraphServerConfig};use tracing::Level;use tracing_subscriber::FmtSubscriber;@@ -111,6 +111,9 @@ enum WebgraphOptions {#[clap(long)]num_segments: Option<usize>,},+ Server {+ config_path: String,+ },}#[derive(Subcommand)]@@ -231,6 +234,14 @@ fn main() -> Result<()> {webgraph.merge_segments(num_segments)}}+ WebgraphOptions::Server { config_path } => {+ let config: WebgraphServerConfig = load_toml_config(config_path);++ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?+ .block_on(webgraph_server::run(config))?+ }},Commands::Frontend { config_path } => {let config: FrontendConfig = load_toml_config(config_path);diff --git a/core/src/mapreduce/manager.rs b/core/src/mapreduce/manager.rsindex f62767de..96f88f0b 100644--- a/core/src/mapreduce/manager.rs+++ b/core/src/mapreduce/manager.rs@@ -1,8 +1,8 @@use super::{Error, Result, Worker};use super::{Map, Reduce};-use crate::exponential_backoff::ExponentialBackoff;+use crate::distributed::retry_strategy::ExponentialBackoff;+use crate::distributed::sonic;use crate::mapreduce::Task;-use crate::sonic;use futures::StreamExt;use itertools::Itertools;use serde::de::DeserializeOwned;@@ -48,7 +48,7 @@ impl RemoteWorker {O: Serialize + DeserializeOwned + Send,{let conn = self.connect().await?;- match conn.send(Task::Job(job)).await {+ match conn.send(&Task::Job(job)).await {Ok(sonic::Response::Content(res)) => Ok(res),_ => Err(Error::NoResponse),}@@ -62,7 +62,7 @@ impl RemoteWorker {{debug!("closing worker {:}", self.addr);let conn = self.connect().await?;- let res: sonic::Response<O> = conn.send(Task::<I>::AllFinished).await?;+ let res: sonic::Response<O> = conn.send(&Task::<I>::AllFinished).await?;debug_assert!(matches!(res, sonic::Response::Empty));diff --git a/core/src/mapreduce/mod.rs b/core/src/mapreduce/mod.rsindex 5c62ad35..16808ddf 100644--- a/core/src/mapreduce/mod.rs+++ b/core/src/mapreduce/mod.rs@@ -24,7 +24,7 @@ use thiserror::Error;pub use worker::StatelessWorker;pub use worker::Worker;-use crate::sonic;+use crate::distributed::sonic;pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/core/src/mapreduce/worker.rs b/core/src/mapreduce/worker.rsindex fa357820..fd4ff2f1 100644--- a/core/src/mapreduce/worker.rs+++ b/core/src/mapreduce/worker.rs@@ -16,7 +16,7 @@use std::net::SocketAddr;-use crate::sonic;+use crate::distributed::sonic;use super::{Map, Result, Task};use async_trait::async_trait;diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 264bb8dc..38b59335 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -612,8 +612,13 @@ mod tests {let mut graph = WebgraphBuilder::new_memory().open();graph.insert(+ Node::from("https://www.e.com").into_host(),Node::from("https://www.a.com").into_host(),- Node::from("https://www.b.com").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("https://www.a.com").into_host(),+ Node::from("https://www.e.com").into_host(),String::new(),);@@ -623,6 +628,17 @@ mod tests {String::new(),);+ graph.insert(+ Node::from("https://www.b.com").into_host(),+ Node::from("https://www.e.com").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("https://www.e.com").into_host(),+ Node::from("https://www.b.com").into_host(),+ String::new(),+ );+graph.commit();let centrality_store = CentralityStore::build(&graph, gen_temp_path());@@ -695,7 +711,7 @@ mod tests {r#"<html><head>- <title>Website B</title>+ <title>Website C</title></head><body>{CONTENT} {}diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 4e354d27..c618b5af 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -281,9 +281,9 @@ mod tests {fn it_ranks_search_results() {let mut graph = WebgraphBuilder::new_memory().open();- graph.insert(Node::from("a.com"), Node::from("b.com"), String::new());+ graph.insert(Node::from("b.com"), Node::from("a.com"), String::new());graph.insert(Node::from("c.com"), Node::from("d.com"), String::new());- graph.insert(Node::from("a.com"), Node::from("e.com"), String::new());+ graph.insert(Node::from("b.com"), Node::from("e.com"), String::new());graph.commit();diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 63fc8ad4..f925e133 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -36,7 +36,12 @@ mod tests {graph.insert(Node::from("https://www.first.com").into_host(),- Node::from("https://www.second.com").into_host(),+ Node::from("https://www.nan.com").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("https://www.nan.com").into_host(),+ Node::from("https://www.first.com").into_host(),String::new(),);graph.insert(@@ -44,6 +49,16 @@ mod tests {Node::from("https://www.third.com").into_host(),String::new(),);+ graph.insert(+ Node::from("https://www.nan.com").into_host(),+ Node::from("https://www.second.com").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("https://www.second.com").into_host(),+ Node::from("https://www.nan.com").into_host(),+ String::new(),+ );graph.commit();diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 764ab4cc..7aae15e4 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -17,9 +17,8 @@use crate::{bangs::{Bang, BangHit, Bangs},ceil_char_boundary,- cluster::{member::Service, Cluster},collector::BucketCollector,- exponential_backoff::ExponentialBackoff,+ distributed::{cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff},floor_char_boundary,inverted_index::{self, RetrievedWebpage},qa_model::QaModel,@@ -54,7 +53,7 @@ use optics::Optic;use serde::{Deserialize, Serialize};use thiserror::Error;-use crate::sonic;+use crate::distributed::sonic;use super::{InitialWebsiteResult, SearchQuery, SearchResult};@@ -79,18 +78,13 @@ pub enum Error {impl RemoteSearcher {async fn search(&self, query: &SearchQuery) -> Result<InitialWebsiteResult> {- for timeout in ExponentialBackoff::from_millis(30)- .with_limit(Duration::from_millis(200))- .take(5)+ let mut conn = self.conn();++ if let Ok(sonic::Response::Content(body)) = conn+ .send_with_timeout(&Request::Search(query.clone()), Duration::from_secs(1))+ .await{- if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await- {- if let Ok(sonic::Response::Content(body)) =- connection.send(Request::Search(query.clone())).await- {- return Ok(body);- }- }+ return Ok(body);}Err(Error::SearchFailed.into())@@ -101,47 +95,48 @@ impl RemoteSearcher {pointers: &[inverted_index::WebsitePointer],original_query: &str,) -> Result<Vec<RetrievedWebpage>> {- for timeout in ExponentialBackoff::from_millis(30)- .with_limit(Duration::from_millis(200))- .take(5)+ let mut conn = self.conn();++ if let Ok(sonic::Response::Content(body)) = conn+ .send_with_timeout(+ &Request::RetrieveWebsites {+ websites: pointers.to_vec(),+ query: original_query.to_string(),+ },+ Duration::from_secs(1),+ )+ .await{- if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await- {- if let Ok(sonic::Response::Content(body)) = connection- .send(Request::RetrieveWebsites {- websites: pointers.to_vec(),- query: original_query.to_string(),- })- .await- {- return Ok(body);- }- }+ return Ok(body);}-Err(Error::SearchFailed.into())}async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {- for timeout in ExponentialBackoff::from_millis(30)- .with_limit(Duration::from_millis(200))- .take(5)+ let mut conn = self.conn();++ if let Ok(sonic::Response::Content(body)) = conn+ .send_with_timeout(+ &Request::GetWebpage {+ url: url.to_string(),+ },+ Duration::from_secs(1),+ )+ .await{- if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await- {- if let Ok(sonic::Response::Content(body)) = connection- .send(Request::GetWebpage {- url: url.to_string(),- })- .await- {- return Ok(body);- }- }+ return Ok(body);}Err(Error::WebpageNotFound.into())}++ fn conn(&self) -> sonic::ResilientConnection<impl Iterator<Item = Duration>> {+ let retry = ExponentialBackoff::from_millis(30)+ .with_limit(Duration::from_millis(200))+ .take(5);++ sonic::ResilientConnection::create(self.addr, retry)+ }}#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]@@ -237,7 +232,7 @@ pub enum Request {}pub struct DistributedSearcher {- cluster: Cluster,+ cluster: Arc<Cluster>,cross_encoder: Arc<CrossEncoderModel>,lambda_model: Option<Arc<LambdaMART>>,qa_model: Option<Arc<QaModel>>,@@ -262,7 +257,7 @@ impl AsRankingWebsite for ScoredWebsitePointer {impl DistributedSearcher {pub fn new(- cluster: Cluster,+ cluster: Arc<Cluster>,cross_encoder: CrossEncoderModel,lambda_model: Option<LambdaMART>,qa_model: Option<QaModel>,diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rsindex 6a51f1fe..417ce2fb 100644--- a/core/src/similar_sites.rs+++ b/core/src/similar_sites.rs@@ -24,6 +24,7 @@ use crate::{webgraph::{Node, NodeID, Webgraph},};+#[derive(serde::Serialize, serde::Deserialize, Debug)]pub struct ScoredNode {pub node: Node,pub score: f64,diff --git a/core/src/warc.rs b/core/src/warc.rsindex 16888558..7cdf9724 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::exponential_backoff::ExponentialBackoff;+use crate::distributed::retry_strategy::ExponentialBackoff;use crate::{Error, Result, WarcSource};use std::collections::BTreeMap;use std::fs::File;diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rsindex 249ee7b3..b29f1cc7 100644--- a/core/src/webgraph/centrality/online_harmonic.rs+++ b/core/src/webgraph/centrality/online_harmonic.rs@@ -39,21 +39,25 @@ use crate::webgraph::{Edge, Node};use crate::webgraph::{NodeID, Webgraph};use crate::Result;-const NUM_PROXY_NODES: usize = 500;+const NUM_PROXY_NODES: usize = 1000;const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;const USER_NODES_LIMIT: usize = 100; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existing-const MAX_DIST_PROXY: u8 = 6;+const MAX_DIST_PROXY: u8 = 10;const MAX_NUM_OUT_EDGES: usize = 1000;#[derive(Serialize, Deserialize)]pub struct ProxyNode {pub id: NodeID,- dist_from_node: IntMap<u8>, // from node to proxy node- dist_to_node: IntMap<u8>, // from proxy node to other node+ pub dist_from_node: IntMap<u8>, // from node to proxy node+ pub dist_to_node: IntMap<u8>, // from proxy node to other node}impl ProxyNode {fn dist(&self, from: &NodeID, to: &NodeID) -> Option<u8> {+ if self.id == *from || self.id == *to {+ return None;+ }+if let Some(from_node_to_proxy) = self.dist_from_node.get(&from.0) {if let Some(from_proxy_to_node) = self.dist_to_node.get(&to.0) {return Some(from_node_to_proxy + from_proxy_to_node);@@ -139,7 +143,6 @@ impl Scorer {if let Some(cached) = self.cache.get(&node) {return *cached;}-let res = (self.disliked_nodes.len() as f64+ (self.liked_nodes@@ -208,10 +211,6 @@ impl UserNode {}fn best_dist(&self, node: &NodeID) -> Option<u8> {- if *node == self.id {- return Some(0);- }-let mut best = None;for proxy in &self.proxy_nodes {@@ -332,7 +331,7 @@ fn reversed_distances(graph: &Webgraph, source: Node) -> BTreeMap<NodeID, u8> {modified_dijkstra(source,|node| graph.raw_ingoing_edges(&node),- |edge| edge.to,+ |edge| edge.from,graph,)}@@ -530,7 +529,7 @@ mod tests {let scorer = centrality.scorer(&[], &disliked_nodes);for node in &disliked_nodes {- assert!(scorer.score(*node) < 1.0);+ assert!(scorer.score(*node) < disliked_nodes.len() as f64);}}@@ -550,7 +549,7 @@ mod tests {let centrality =OnlineHarmonicCentrality::new_with_num_proxy(&graph, &harmonic_centrality_store, 5);- let liked_nodes: Vec<_> = vec![Node::from("B".to_string()), Node::from("E".to_string())]+ let liked_nodes: Vec<_> = vec![Node::from("B".to_string())].into_iter().filter_map(|node| graph.node2id(&node).copied()).collect();@@ -568,13 +567,13 @@ mod tests {);assert!(- scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())- > scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())+ scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())+ > scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap()));assert!(- scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())- > scorer.score(*graph.node2id(&Node::from("A".to_string())).unwrap())+ scorer.score(*graph.node2id(&Node::from("A".to_string())).unwrap())+ > scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap()));}}diff --git a/frontend/src/components/SearchModal.astro b/frontend/src/components/SearchModal.astroindex b07e6f29..509dbbd9 100644--- a/frontend/src/components/SearchModal.astro+++ b/frontend/src/components/SearchModal.astro@@ -265,7 +265,7 @@ import Hero from "./Hero.astro";if (!snip["summary"]["hasStarted"]) {snip["summary"]["hasStarted"] = true;- fetch("/summarize?" + queryData, {+ fetch("/beta/api/summarize?" + queryData, {method: "GET",}).then((e) => {snippetBubble.classList.add("hidden");diff --git a/justfile b/justfileindex caa81dd0..9e8ca4e7 100644--- a/justfile+++ b/justfile@@ -22,6 +22,7 @@./scripts/export_crossencoder./scripts/export_qa_model./scripts/export_abstractive_summary_model+ ./scripts/export_dual_encodercd frontend; npm install; npm run buildcargo run --release --all-features -- configure {{ARGS}}diff --git a/scripts/run_frontend.sh b/scripts/run_frontend.shindex eb3aab55..c4792f3a 100644--- a/scripts/run_frontend.sh+++ b/scripts/run_frontend.sh@@ -1,4 +1,5 @@#!/bin/bashtrap 'kill $(jobs -p)' EXITcargo run $1 -- frontend configs/frontend.toml &-cargo run $1 -- search-server configs/search_server.toml+cargo run $1 -- search-server configs/search_server.toml &+cargo run $1 -- webgraph server configs/webgraph/server.toml