Skip to content

web search done right

Fixed a bug where distance calculation in online-harmonic used the wrong node from the edge

Mikkel Denkercb64b49ad9d9
diff --git a/configs/frontend.toml b/configs/frontend.toml
index 79048ff0..c263b2a9 100644
--- a/configs/frontend.toml
+++ b/configs/frontend.toml
@@ -2,8 +2,8 @@ cluster_id = "dev_frontend"
queries_csv_path = "data/queries_us.csv"
host = "0.0.0.0:3000"
prometheus_host = "0.0.0.0:3001"
-gossip_addr = "0.0.0.0:3003"
-gossip_seed_nodes = ["0.0.0.0:3004"]
+gossip_addr = "0.0.0.0:3004"
+gossip_seed_nodes = ["0.0.0.0:3005", "0.0.0.0:3006"]
crossencoder_model_path = "data/cross_encoder"
# lambda_model_path = "data/lambdamart.txt"
# qa_model_path = "data/qa_model"
diff --git a/configs/search_server.toml b/configs/search_server.toml
index f143b8ea..4428732e 100644
--- a/configs/search_server.toml
+++ b/configs/search_server.toml
@@ -1,5 +1,5 @@
cluster_id = "dev_search"
-gossip_addr = "0.0.0.0:3004"
+gossip_addr = "0.0.0.0:3005"
shard_id = 0
host = "0.0.0.0:3002"
index_path = "data/index"
diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.toml
index 9d3a11c2..1074ec96 100644
--- a/configs/webgraph/local.toml
+++ b/configs/webgraph/local.toml
@@ -1,8 +1,8 @@
-limit_warc_files = 10
+limit_warc_files = 5
graph_base_path = "data/webgraph"
batch_size = 1
[warc_source]
type = "HTTP"
base_url = "https://data.commoncrawl.org/"
-warc_paths_file = "data/warc.paths"
\ No newline at end of file
+warc_paths_file = "data/warc.paths"
diff --git a/configs/webgraph/server.toml b/configs/webgraph/server.toml
new file mode 100644
index 00000000..472ae19b
--- /dev/null
+++ b/configs/webgraph/server.toml
@@ -0,0 +1,5 @@
+host = "0.0.0.0:3003"
+cluster_id = "dev_webgraph"
+gossip_addr = "0.0.0.0:3006"
+graph_path = "data/webgraph"
+inbound_similarity_path = "data/centrality/inbound_similarity"
\ No newline at end of file
diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rs
index ccff3424..f984edc0 100644
--- a/core/examples/print_inbound_links.rs
+++ b/core/examples/print_inbound_links.rs
@@ -23,7 +23,7 @@ pub fn main() {
let graph = WebgraphBuilder::new("data/webgraph").open();
let inbound = InboundSimilarity::open("data/centrality/inbound_similarity").unwrap();
- for host in ["www.homedepot.com", "doc.rust-lang.org"] {
+ for host in ["www.homedepot.com"] {
println!("{host}:");
let node = Node::from(host).into_host();
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index 7cea00ee..e8bb821f 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -21,9 +21,9 @@ use tower_http::{compression::CompressionLayer, services::ServeDir};
use crate::{
autosuggest::Autosuggest,
bangs::Bangs,
- cluster::{
+ distributed::{
+ cluster::Cluster,
member::{Member, Service},
- Cluster,
},
leaky_queue::LeakyQueue,
qa_model::QaModel,
@@ -44,6 +44,8 @@ use axum::{
routing::post,
};
+use self::webgraph::RemoteWebgraph;
+
mod about;
mod autosuggest;
mod improvement;
@@ -55,11 +57,13 @@ mod privacy;
pub mod search;
mod sites;
mod summarize;
+mod webgraph;
pub struct HtmlTemplate<T>(T);
pub struct State {
pub searcher: DistributedSearcher,
+ pub remote_webgraph: RemoteWebgraph,
pub autosuggest: Autosuggest,
pub search_counter_success: crate::metrics::Counter,
pub search_counter_fail: crate::metrics::Counter,
@@ -122,15 +126,18 @@ pub async fn router(
let bangs = Bangs::from_path(&config.bangs_path);
- let cluster = Cluster::join(
- Member {
- id: config.cluster_id.clone(),
- service: Service::Frontend { host: config.host },
- },
- config.gossip_addr,
- config.gossip_seed_nodes.clone().unwrap_or_default(),
- )
- .await?;
+ let cluster = Arc::new(
+ Cluster::join(
+ Member {
+ id: config.cluster_id.clone(),
+ service: Service::Frontend { host: config.host },
+ },
+ config.gossip_addr,
+ config.gossip_seed_nodes.clone().unwrap_or_default(),
+ )
+ .await?,
+ );
+ let remote_webgraph = RemoteWebgraph::new(cluster.clone());
let searcher = DistributedSearcher::new(cluster, crossencoder, lambda_model, qa_model, bangs);
let state = Arc::new(State {
@@ -138,6 +145,7 @@ pub async fn router(
autosuggest,
search_counter_success,
search_counter_fail,
+ remote_webgraph,
summarizer: Arc::new(Summarizer::open(&config.summarizer_path)?),
improvement_queue: query_store_queue,
});
@@ -164,7 +172,11 @@ pub async fn router(
.route("/improvement/store", post(improvement::store))
.fallback(get_service(ServeDir::new("frontend/dist/")))
.layer(CompressionLayer::new())
- .merge(Router::new().route("/summarize", get(summarize::route)))
+ .merge(
+ Router::new()
+ .route("/beta/api/summarize", get(summarize::route))
+ .route("/beta/api/similar_sites", post(webgraph::similar_sites)),
+ )
.with_state(state))
}
diff --git a/core/src/api/webgraph.rs b/core/src/api/webgraph.rs
new file mode 100644
index 00000000..607735ea
--- /dev/null
+++ b/core/src/api/webgraph.rs
@@ -0,0 +1,104 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{net::SocketAddr, sync::Arc, time::Duration};
+
+use axum::{extract, response::IntoResponse, Json};
+use http::StatusCode;
+use itertools::Itertools;
+
+use crate::{
+ distributed::{cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff, sonic},
+ similar_sites::ScoredNode,
+};
+
+use super::State;
+
+pub struct RemoteWebgraph {
+ cluster: Arc<Cluster>,
+}
+
+impl RemoteWebgraph {
+ pub fn new(cluster: Arc<Cluster>) -> Self {
+ Self { cluster }
+ }
+
+ async fn host(&self) -> SocketAddr {
+ self.cluster
+ .members()
+ .await
+ .iter()
+ .find_map(|member| match member.service {
+ Service::Webgraph { host } => Some(host),
+ _ => None,
+ })
+ .unwrap()
+ }
+}
+
+#[derive(serde::Deserialize)]
+pub struct Params {
+ pub sites: Vec<String>,
+ pub top_n: usize,
+}
+
+#[derive(serde::Serialize, serde::Deserialize)]
+struct ScoredSite {
+ pub site: String,
+ pub score: f64,
+}
+
+#[allow(clippy::unused_async)]
+pub async fn similar_sites(
+ extract::State(state): extract::State<Arc<State>>,
+ extract::Json(params): extract::Json<Params>,
+) -> std::result::Result<impl IntoResponse, StatusCode> {
+ let host = state.remote_webgraph.host().await;
+
+ let retry = ExponentialBackoff::from_millis(30)
+ .with_limit(Duration::from_millis(200))
+ .take(5);
+
+ let mut conn = sonic::ResilientConnection::create(host, retry);
+
+ let top_n = params.top_n.min(1000);
+
+ match conn
+ .send_with_timeout::<_, Vec<ScoredNode>>(
+ &crate::entrypoint::webgraph_server::Request::SimilarSites {
+ sites: params.sites,
+ top_n,
+ },
+ Duration::from_secs(30),
+ )
+ .await
+ {
+ Ok(sonic::Response::Content(nodes)) => Ok(Json(
+ nodes
+ .into_iter()
+ .map(|node| ScoredSite {
+ site: node.node.name,
+ score: node.score,
+ })
+ .collect_vec(),
+ )),
+ Err(err) => {
+ tracing::error!("Failed to send request to webgraph: {}", err);
+ Err(StatusCode::INTERNAL_SERVER_ERROR)
+ }
+ _ => Err(StatusCode::INTERNAL_SERVER_ERROR),
+ }
+}
diff --git a/core/src/cluster/mod.rs b/core/src/distributed/cluster.rs
similarity index 95%
rename from core/src/cluster/mod.rs
rename to core/src/distributed/cluster.rs
index 682508be..ee4ebee7 100644
--- a/core/src/cluster/mod.rs
+++ b/core/src/distributed/cluster.rs
@@ -22,14 +22,12 @@ use tokio::sync::RwLock;
use tokio_stream::StreamExt;
use tracing::error;
-use self::member::{Member, Service};
+use crate::distributed::member::{Member, Service};
const CLUSTER_ID: &str = "stract-cluster";
const GOSSIP_INTERVAL: Duration = Duration::from_secs(1);
const SERVICE_KEY: &str = "service";
-pub mod member;
-
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("Anyhow")]
@@ -55,13 +53,14 @@ impl Cluster {
) -> Result<Self> {
let transport = UdpTransport;
let failure_detector_config = FailureDetectorConfig {
- phi_threshold: 10.0, // TODO: choose this value based on tests
initial_interval: GOSSIP_INTERVAL,
..Default::default()
};
+ let uuid = uuid::Uuid::new_v4().to_string();
+
let node_id = NodeId {
- id: self_node.id.clone(),
+ id: format!("{}_{}", self_node.id, uuid),
gossip_public_address: gossip_addr,
};
@@ -125,6 +124,7 @@ impl Cluster {
}
}
+ tracing::info!("new members: {:#?}", new_members);
let mut write = alive_nodes_ref.write().await;
write.clear();
diff --git a/core/src/cluster/member.rs b/core/src/distributed/member.rs
similarity index 97%
rename from core/src/cluster/member.rs
rename to core/src/distributed/member.rs
index f1c470bb..fd9e0a18 100644
--- a/core/src/cluster/member.rs
+++ b/core/src/distributed/member.rs
@@ -24,6 +24,7 @@ use crate::searcher::ShardId;
pub enum Service {
Searcher { host: SocketAddr, shard: ShardId },
Frontend { host: SocketAddr },
+ Webgraph { host: SocketAddr },
}
#[derive(PartialEq, Eq, Hash, Clone, Debug)]
diff --git a/core/src/distributed/mod.rs b/core/src/distributed/mod.rs
new file mode 100644
index 00000000..9d64573f
--- /dev/null
+++ b/core/src/distributed/mod.rs
@@ -0,0 +1,19 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+pub mod cluster;
+pub mod member;
+pub mod retry_strategy;
+pub mod sonic;
diff --git a/core/src/exponential_backoff.rs b/core/src/distributed/retry_strategy.rs
similarity index 100%
rename from core/src/exponential_backoff.rs
rename to core/src/distributed/retry_strategy.rs
diff --git a/core/src/sonic.rs b/core/src/distributed/sonic.rs
similarity index 79%
rename from core/src/sonic.rs
rename to core/src/distributed/sonic.rs
index 10073cf1..62d77092 100644
--- a/core/src/sonic.rs
+++ b/core/src/distributed/sonic.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::time::Duration;
+use std::{net::SocketAddr, time::Duration};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use thiserror::Error;
@@ -132,7 +132,7 @@ impl Connection {
pub async fn send_without_timeout<T: Serialize, R: DeserializeOwned + Serialize>(
mut self,
- request: T,
+ request: &T,
) -> Result<Response<R>> {
let bytes = bincode::serialize(&request).unwrap();
@@ -160,7 +160,7 @@ impl Connection {
pub async fn send<T: Serialize, R: DeserializeOwned + Serialize>(
self,
- request: T,
+ request: &T,
) -> Result<Response<R>> {
self.send_with_timeout(request, Duration::from_secs(30))
.await
@@ -168,7 +168,7 @@ impl Connection {
pub async fn send_with_timeout<T: Serialize, R: DeserializeOwned + Serialize>(
self,
- request: T,
+ request: &T,
timeout: Duration,
) -> Result<Response<R>> {
match tokio::time::timeout(timeout, self.send_without_timeout(request)).await {
@@ -177,3 +177,41 @@ impl Connection {
}
}
}
+
+pub struct ResilientConnection<Rt: Iterator<Item = Duration>> {
+ addr: SocketAddr,
+ retry: Rt,
+}
+
+impl<Rt: Iterator<Item = Duration>> ResilientConnection<Rt> {
+ pub fn create(addr: SocketAddr, retry: Rt) -> Self {
+ Self { addr, retry }
+ }
+
+ pub async fn send_with_timeout<T: Serialize, R: DeserializeOwned + Serialize>(
+ &mut self,
+ request: &T,
+ timeout: Duration,
+ ) -> Result<Response<R>> {
+ loop {
+ match Connection::create_with_timeout(&self.addr, timeout).await {
+ Ok(conn) => {
+ let response = conn.send_with_timeout(request, timeout).await;
+ if let Err(Error::ConnectionTimeout) = response {
+ continue;
+ }
+ return response;
+ }
+ Err(Error::ConnectionTimeout) => {
+ if let Some(timeout) = self.retry.next() {
+ tokio::time::sleep(timeout).await;
+ continue;
+ } else {
+ return Err(Error::ConnectionTimeout);
+ }
+ }
+ Err(e) => return Err(e),
+ }
+ }
+ }
+}
diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rs
index f5dff68c..374c9dfb 100644
--- a/core/src/entrypoint/mod.rs
+++ b/core/src/entrypoint/mod.rs
@@ -25,6 +25,7 @@ pub mod indexer;
pub mod search_server;
pub mod topic_centrality;
mod webgraph;
+pub mod webgraph_server;
use std::{fs::File, path::Path};
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index 5812e0d7..284919c4 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -19,9 +19,10 @@ use std::net::SocketAddr;
use tracing::info;
use crate::{
- cluster::{
+ distributed::{
+ cluster::Cluster,
member::{Member, Service},
- Cluster,
+ sonic,
},
entity_index::EntityIndex,
index::Index,
@@ -31,7 +32,7 @@ use crate::{
models::{lambdamart::LambdaMART, linear::LinearRegression},
},
searcher::{self, LocalSearcher},
- sonic, Result, SearchServerConfig,
+ Result, SearchServerConfig,
};
pub async fn run(config: SearchServerConfig) -> Result<()> {
diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rs
new file mode 100644
index 00000000..6517b3e9
--- /dev/null
+++ b/core/src/entrypoint/webgraph_server.rs
@@ -0,0 +1,73 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::net::SocketAddr;
+
+use serde::Deserialize;
+use serde::Serialize;
+use tracing::info;
+
+use crate::distributed::cluster::Cluster;
+use crate::distributed::member::Member;
+use crate::distributed::member::Service;
+use crate::distributed::sonic;
+use crate::ranking::inbound_similarity::InboundSimilarity;
+use crate::similar_sites::SimilarSitesFinder;
+use crate::webgraph::WebgraphBuilder;
+use crate::Result;
+use crate::WebgraphServerConfig;
+
+#[derive(Serialize, Deserialize)]
+pub enum Request {
+ SimilarSites { sites: Vec<String>, top_n: usize },
+}
+
+pub async fn run(config: WebgraphServerConfig) -> Result<()> {
+ let addr: SocketAddr = config.host;
+
+ // dropping the handle leaves the cluster
+ let _cluster_handle = Cluster::join(
+ Member {
+ id: config.cluster_id,
+ service: Service::Webgraph { host: addr },
+ },
+ config.gossip_addr,
+ config.gossip_seed_nodes.unwrap_or_default(),
+ )
+ .await?;
+
+ let graph = WebgraphBuilder::new(config.graph_path).open();
+ let inbound_similarity = InboundSimilarity::open(config.inbound_similarity_path)?;
+
+ let similar_sites_finder = SimilarSitesFinder::new(graph, inbound_similarity);
+
+ let server = sonic::Server::bind(addr).await.unwrap();
+
+ info!("webgraph server is ready to accept requests on {}", addr);
+
+ loop {
+ if let Ok(req) = server.accept::<Request>().await {
+ match &req.body {
+ Request::SimilarSites { sites, top_n } => {
+ let similar_sites = similar_sites_finder.find_similar_sites(sites, *top_n);
+ req.respond(sonic::Response::Content(similar_sites))
+ .await
+ .ok();
+ }
+ }
+ }
+ }
+}
diff --git a/core/src/image_downloader.rs b/core/src/image_downloader.rs
index 011e8918..8530c493 100644
--- a/core/src/image_downloader.rs
+++ b/core/src/image_downloader.rs
@@ -19,7 +19,7 @@ use std::{collections::HashSet, hash::Hash, time::Duration};
use serde::Serialize;
use crate::{
- exponential_backoff::ExponentialBackoff,
+ distributed::retry_strategy::ExponentialBackoff,
image_store::{Image, ImageStore},
webpage::Url,
};
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 53427959..7c3ae9cc 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -18,7 +18,7 @@
#![allow(clippy::cast_precision_loss)]
#![allow(clippy::missing_errors_doc)]
-use searcher::{distributed, ShardId};
+use searcher::ShardId;
use serde::{Deserialize, Serialize};
use std::fs::File;
use std::io::{self, BufRead};
@@ -36,13 +36,12 @@ pub mod mapreduce;
mod api;
mod autosuggest;
mod bangs;
-mod cluster;
mod collector;
mod directory;
+mod distributed;
mod entity_index;
mod enum_map;
mod executor;
-mod exponential_backoff;
mod fastfield_reader;
mod human_website_annotations;
mod hyperloglog;
@@ -66,7 +65,6 @@ pub mod searcher;
mod simhash;
pub mod similar_sites;
mod snippet;
-mod sonic;
pub mod spell;
mod subdomain_count;
pub mod summarizer;
@@ -164,7 +162,6 @@ pub struct HttpConfig {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct FrontendConfig {
- pub cluster_id: String,
pub queries_csv_path: String,
pub host: SocketAddr,
pub prometheus_host: SocketAddr,
@@ -174,6 +171,7 @@ pub struct FrontendConfig {
pub bangs_path: String,
pub summarizer_path: String,
pub query_store_db_host: Option<String>,
+ pub cluster_id: String,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
}
@@ -192,6 +190,16 @@ pub struct SearchServerConfig {
pub host: SocketAddr,
}
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct WebgraphServerConfig {
+ pub host: SocketAddr,
+ pub graph_path: String,
+ pub inbound_similarity_path: String,
+ pub cluster_id: String,
+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
+ pub gossip_addr: SocketAddr,
+}
+
#[derive(Error, Debug)]
pub enum Error {
#[error("Failed to download object from HTTP")]
@@ -273,10 +281,10 @@ pub enum Error {
InternalError(String),
#[error("Distributed searcher")]
- DistributedSearcher(#[from] distributed::Error),
+ DistributedSearcher(#[from] searcher::distributed::Error),
#[error("Cluster")]
- Cluster(#[from] cluster::Error),
+ Cluster(#[from] crate::distributed::cluster::Error),
#[error("Lambdamart")]
LambdaMART(#[from] ranking::models::lambdamart::Error),
diff --git a/core/src/main.rs b/core/src/main.rs
index 39a8e153..3a8697b8 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -22,9 +22,9 @@ use stract::entrypoint::autosuggest_scrape::{self, Gl};
#[cfg(feature = "dev")]
use stract::entrypoint::configure;
use stract::entrypoint::indexer::IndexPointer;
-use stract::entrypoint::{self, frontend, search_server};
+use stract::entrypoint::{self, frontend, search_server, webgraph_server};
use stract::webgraph::WebgraphBuilder;
-use stract::{FrontendConfig, SearchServerConfig};
+use stract::{FrontendConfig, SearchServerConfig, WebgraphServerConfig};
use tracing::Level;
use tracing_subscriber::FmtSubscriber;
@@ -111,6 +111,9 @@ enum WebgraphOptions {
#[clap(long)]
num_segments: Option<usize>,
},
+ Server {
+ config_path: String,
+ },
}
#[derive(Subcommand)]
@@ -231,6 +234,14 @@ fn main() -> Result<()> {
webgraph.merge_segments(num_segments)
}
}
+ WebgraphOptions::Server { config_path } => {
+ let config: WebgraphServerConfig = load_toml_config(config_path);
+
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()?
+ .block_on(webgraph_server::run(config))?
+ }
},
Commands::Frontend { config_path } => {
let config: FrontendConfig = load_toml_config(config_path);
diff --git a/core/src/mapreduce/manager.rs b/core/src/mapreduce/manager.rs
index f62767de..96f88f0b 100644
--- a/core/src/mapreduce/manager.rs
+++ b/core/src/mapreduce/manager.rs
@@ -1,8 +1,8 @@
use super::{Error, Result, Worker};
use super::{Map, Reduce};
-use crate::exponential_backoff::ExponentialBackoff;
+use crate::distributed::retry_strategy::ExponentialBackoff;
+use crate::distributed::sonic;
use crate::mapreduce::Task;
-use crate::sonic;
use futures::StreamExt;
use itertools::Itertools;
use serde::de::DeserializeOwned;
@@ -48,7 +48,7 @@ impl RemoteWorker {
O: Serialize + DeserializeOwned + Send,
{
let conn = self.connect().await?;
- match conn.send(Task::Job(job)).await {
+ match conn.send(&Task::Job(job)).await {
Ok(sonic::Response::Content(res)) => Ok(res),
_ => Err(Error::NoResponse),
}
@@ -62,7 +62,7 @@ impl RemoteWorker {
{
debug!("closing worker {:}", self.addr);
let conn = self.connect().await?;
- let res: sonic::Response<O> = conn.send(Task::<I>::AllFinished).await?;
+ let res: sonic::Response<O> = conn.send(&Task::<I>::AllFinished).await?;
debug_assert!(matches!(res, sonic::Response::Empty));
diff --git a/core/src/mapreduce/mod.rs b/core/src/mapreduce/mod.rs
index 5c62ad35..16808ddf 100644
--- a/core/src/mapreduce/mod.rs
+++ b/core/src/mapreduce/mod.rs
@@ -24,7 +24,7 @@ use thiserror::Error;
pub use worker::StatelessWorker;
pub use worker::Worker;
-use crate::sonic;
+use crate::distributed::sonic;
pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/core/src/mapreduce/worker.rs b/core/src/mapreduce/worker.rs
index fa357820..fd4ff2f1 100644
--- a/core/src/mapreduce/worker.rs
+++ b/core/src/mapreduce/worker.rs
@@ -16,7 +16,7 @@
use std::net::SocketAddr;
-use crate::sonic;
+use crate::distributed::sonic;
use super::{Map, Result, Task};
use async_trait::async_trait;
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 264bb8dc..38b59335 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -612,8 +612,13 @@ mod tests {
let mut graph = WebgraphBuilder::new_memory().open();
graph.insert(
+ Node::from("https://www.e.com").into_host(),
Node::from("https://www.a.com").into_host(),
- Node::from("https://www.b.com").into_host(),
+ String::new(),
+ );
+ graph.insert(
+ Node::from("https://www.a.com").into_host(),
+ Node::from("https://www.e.com").into_host(),
String::new(),
);
@@ -623,6 +628,17 @@ mod tests {
String::new(),
);
+ graph.insert(
+ Node::from("https://www.b.com").into_host(),
+ Node::from("https://www.e.com").into_host(),
+ String::new(),
+ );
+ graph.insert(
+ Node::from("https://www.e.com").into_host(),
+ Node::from("https://www.b.com").into_host(),
+ String::new(),
+ );
+
graph.commit();
let centrality_store = CentralityStore::build(&graph, gen_temp_path());
@@ -695,7 +711,7 @@ mod tests {
r#"
<html>
<head>
- <title>Website B</title>
+ <title>Website C</title>
</head>
<body>
{CONTENT} {}
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 4e354d27..c618b5af 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -281,9 +281,9 @@ mod tests {
fn it_ranks_search_results() {
let mut graph = WebgraphBuilder::new_memory().open();
- graph.insert(Node::from("a.com"), Node::from("b.com"), String::new());
+ graph.insert(Node::from("b.com"), Node::from("a.com"), String::new());
graph.insert(Node::from("c.com"), Node::from("d.com"), String::new());
- graph.insert(Node::from("a.com"), Node::from("e.com"), String::new());
+ graph.insert(Node::from("b.com"), Node::from("e.com"), String::new());
graph.commit();
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 63fc8ad4..f925e133 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -36,7 +36,12 @@ mod tests {
graph.insert(
Node::from("https://www.first.com").into_host(),
- Node::from("https://www.second.com").into_host(),
+ Node::from("https://www.nan.com").into_host(),
+ String::new(),
+ );
+ graph.insert(
+ Node::from("https://www.nan.com").into_host(),
+ Node::from("https://www.first.com").into_host(),
String::new(),
);
graph.insert(
@@ -44,6 +49,16 @@ mod tests {
Node::from("https://www.third.com").into_host(),
String::new(),
);
+ graph.insert(
+ Node::from("https://www.nan.com").into_host(),
+ Node::from("https://www.second.com").into_host(),
+ String::new(),
+ );
+ graph.insert(
+ Node::from("https://www.second.com").into_host(),
+ Node::from("https://www.nan.com").into_host(),
+ String::new(),
+ );
graph.commit();
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 764ab4cc..7aae15e4 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -17,9 +17,8 @@
use crate::{
bangs::{Bang, BangHit, Bangs},
ceil_char_boundary,
- cluster::{member::Service, Cluster},
collector::BucketCollector,
- exponential_backoff::ExponentialBackoff,
+ distributed::{cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff},
floor_char_boundary,
inverted_index::{self, RetrievedWebpage},
qa_model::QaModel,
@@ -54,7 +53,7 @@ use optics::Optic;
use serde::{Deserialize, Serialize};
use thiserror::Error;
-use crate::sonic;
+use crate::distributed::sonic;
use super::{InitialWebsiteResult, SearchQuery, SearchResult};
@@ -79,18 +78,13 @@ pub enum Error {
impl RemoteSearcher {
async fn search(&self, query: &SearchQuery) -> Result<InitialWebsiteResult> {
- for timeout in ExponentialBackoff::from_millis(30)
- .with_limit(Duration::from_millis(200))
- .take(5)
+ let mut conn = self.conn();
+
+ if let Ok(sonic::Response::Content(body)) = conn
+ .send_with_timeout(&Request::Search(query.clone()), Duration::from_secs(1))
+ .await
{
- if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
- {
- if let Ok(sonic::Response::Content(body)) =
- connection.send(Request::Search(query.clone())).await
- {
- return Ok(body);
- }
- }
+ return Ok(body);
}
Err(Error::SearchFailed.into())
@@ -101,47 +95,48 @@ impl RemoteSearcher {
pointers: &[inverted_index::WebsitePointer],
original_query: &str,
) -> Result<Vec<RetrievedWebpage>> {
- for timeout in ExponentialBackoff::from_millis(30)
- .with_limit(Duration::from_millis(200))
- .take(5)
+ let mut conn = self.conn();
+
+ if let Ok(sonic::Response::Content(body)) = conn
+ .send_with_timeout(
+ &Request::RetrieveWebsites {
+ websites: pointers.to_vec(),
+ query: original_query.to_string(),
+ },
+ Duration::from_secs(1),
+ )
+ .await
{
- if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
- {
- if let Ok(sonic::Response::Content(body)) = connection
- .send(Request::RetrieveWebsites {
- websites: pointers.to_vec(),
- query: original_query.to_string(),
- })
- .await
- {
- return Ok(body);
- }
- }
+ return Ok(body);
}
-
Err(Error::SearchFailed.into())
}
async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {
- for timeout in ExponentialBackoff::from_millis(30)
- .with_limit(Duration::from_millis(200))
- .take(5)
+ let mut conn = self.conn();
+
+ if let Ok(sonic::Response::Content(body)) = conn
+ .send_with_timeout(
+ &Request::GetWebpage {
+ url: url.to_string(),
+ },
+ Duration::from_secs(1),
+ )
+ .await
{
- if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
- {
- if let Ok(sonic::Response::Content(body)) = connection
- .send(Request::GetWebpage {
- url: url.to_string(),
- })
- .await
- {
- return Ok(body);
- }
- }
+ return Ok(body);
}
Err(Error::WebpageNotFound.into())
}
+
+ fn conn(&self) -> sonic::ResilientConnection<impl Iterator<Item = Duration>> {
+ let retry = ExponentialBackoff::from_millis(30)
+ .with_limit(Duration::from_millis(200))
+ .take(5);
+
+ sonic::ResilientConnection::create(self.addr, retry)
+ }
}
#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]
@@ -237,7 +232,7 @@ pub enum Request {
}
pub struct DistributedSearcher {
- cluster: Cluster,
+ cluster: Arc<Cluster>,
cross_encoder: Arc<CrossEncoderModel>,
lambda_model: Option<Arc<LambdaMART>>,
qa_model: Option<Arc<QaModel>>,
@@ -262,7 +257,7 @@ impl AsRankingWebsite for ScoredWebsitePointer {
impl DistributedSearcher {
pub fn new(
- cluster: Cluster,
+ cluster: Arc<Cluster>,
cross_encoder: CrossEncoderModel,
lambda_model: Option<LambdaMART>,
qa_model: Option<QaModel>,
diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rs
index 6a51f1fe..417ce2fb 100644
--- a/core/src/similar_sites.rs
+++ b/core/src/similar_sites.rs
@@ -24,6 +24,7 @@ use crate::{
webgraph::{Node, NodeID, Webgraph},
};
+#[derive(serde::Serialize, serde::Deserialize, Debug)]
pub struct ScoredNode {
pub node: Node,
pub score: f64,
diff --git a/core/src/warc.rs b/core/src/warc.rs
index 16888558..7cdf9724 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::exponential_backoff::ExponentialBackoff;
+use crate::distributed::retry_strategy::ExponentialBackoff;
use crate::{Error, Result, WarcSource};
use std::collections::BTreeMap;
use std::fs::File;
diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rs
index 249ee7b3..b29f1cc7 100644
--- a/core/src/webgraph/centrality/online_harmonic.rs
+++ b/core/src/webgraph/centrality/online_harmonic.rs
@@ -39,21 +39,25 @@ use crate::webgraph::{Edge, Node};
use crate::webgraph::{NodeID, Webgraph};
use crate::Result;
-const NUM_PROXY_NODES: usize = 500;
+const NUM_PROXY_NODES: usize = 1000;
const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;
const USER_NODES_LIMIT: usize = 100; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existing
-const MAX_DIST_PROXY: u8 = 6;
+const MAX_DIST_PROXY: u8 = 10;
const MAX_NUM_OUT_EDGES: usize = 1000;
#[derive(Serialize, Deserialize)]
pub struct ProxyNode {
pub id: NodeID,
- dist_from_node: IntMap<u8>, // from node to proxy node
- dist_to_node: IntMap<u8>, // from proxy node to other node
+ pub dist_from_node: IntMap<u8>, // from node to proxy node
+ pub dist_to_node: IntMap<u8>, // from proxy node to other node
}
impl ProxyNode {
fn dist(&self, from: &NodeID, to: &NodeID) -> Option<u8> {
+ if self.id == *from || self.id == *to {
+ return None;
+ }
+
if let Some(from_node_to_proxy) = self.dist_from_node.get(&from.0) {
if let Some(from_proxy_to_node) = self.dist_to_node.get(&to.0) {
return Some(from_node_to_proxy + from_proxy_to_node);
@@ -139,7 +143,6 @@ impl Scorer {
if let Some(cached) = self.cache.get(&node) {
return *cached;
}
-
let res = (self.disliked_nodes.len() as f64
+ (self
.liked_nodes
@@ -208,10 +211,6 @@ impl UserNode {
}
fn best_dist(&self, node: &NodeID) -> Option<u8> {
- if *node == self.id {
- return Some(0);
- }
-
let mut best = None;
for proxy in &self.proxy_nodes {
@@ -332,7 +331,7 @@ fn reversed_distances(graph: &Webgraph, source: Node) -> BTreeMap<NodeID, u8> {
modified_dijkstra(
source,
|node| graph.raw_ingoing_edges(&node),
- |edge| edge.to,
+ |edge| edge.from,
graph,
)
}
@@ -530,7 +529,7 @@ mod tests {
let scorer = centrality.scorer(&[], &disliked_nodes);
for node in &disliked_nodes {
- assert!(scorer.score(*node) < 1.0);
+ assert!(scorer.score(*node) < disliked_nodes.len() as f64);
}
}
@@ -550,7 +549,7 @@ mod tests {
let centrality =
OnlineHarmonicCentrality::new_with_num_proxy(&graph, &harmonic_centrality_store, 5);
- let liked_nodes: Vec<_> = vec![Node::from("B".to_string()), Node::from("E".to_string())]
+ let liked_nodes: Vec<_> = vec![Node::from("B".to_string())]
.into_iter()
.filter_map(|node| graph.node2id(&node).copied())
.collect();
@@ -568,13 +567,13 @@ mod tests {
);
assert!(
- scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())
- > scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())
+ scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())
+ > scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())
);
assert!(
- scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())
- > scorer.score(*graph.node2id(&Node::from("A".to_string())).unwrap())
+ scorer.score(*graph.node2id(&Node::from("A".to_string())).unwrap())
+ > scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())
);
}
}
diff --git a/frontend/src/components/SearchModal.astro b/frontend/src/components/SearchModal.astro
index b07e6f29..509dbbd9 100644
--- a/frontend/src/components/SearchModal.astro
+++ b/frontend/src/components/SearchModal.astro
@@ -265,7 +265,7 @@ import Hero from "./Hero.astro";
if (!snip["summary"]["hasStarted"]) {
snip["summary"]["hasStarted"] = true;
- fetch("/summarize?" + queryData, {
+ fetch("/beta/api/summarize?" + queryData, {
method: "GET",
}).then((e) => {
snippetBubble.classList.add("hidden");
diff --git a/justfile b/justfile
index caa81dd0..9e8ca4e7 100644
--- a/justfile
+++ b/justfile
@@ -22,6 +22,7 @@
./scripts/export_crossencoder
./scripts/export_qa_model
./scripts/export_abstractive_summary_model
+ ./scripts/export_dual_encoder
cd frontend; npm install; npm run build
cargo run --release --all-features -- configure {{ARGS}}
diff --git a/scripts/run_frontend.sh b/scripts/run_frontend.sh
index eb3aab55..c4792f3a 100644
--- a/scripts/run_frontend.sh
+++ b/scripts/run_frontend.sh
@@ -1,4 +1,5 @@
#!/bin/bash
trap 'kill $(jobs -p)' EXIT
cargo run $1 -- frontend configs/frontend.toml &
-cargo run $1 -- search-server configs/search_server.toml
+cargo run $1 -- search-server configs/search_server.toml &
+cargo run $1 -- webgraph server configs/webgraph/server.toml