Skip to content

web search done right

Fix sonic broken pipe due to low timeouts

Mikkel Denker4c0b5e4d88de
diff --git a/core/src/api/webgraph.rs b/core/src/api/webgraph.rs
index 4d803503..fec808fd 100644
--- a/core/src/api/webgraph.rs
+++ b/core/src/api/webgraph.rs
@@ -68,12 +68,13 @@ pub async fn similar_sites(
let conn = sonic::service::ResilientConnection::create(host, retry);
match conn
- .send_with_timeout(
+ .send(
&crate::entrypoint::webgraph_server::SimilarSites {
sites: params.sites,
top_n: params.top_n,
},
- Duration::from_secs(30),
+ Duration::from_secs(5),
+ Some(Duration::from_secs(60)),
)
.await
{
@@ -111,9 +112,10 @@ pub async fn knows_site(
let conn = sonic::service::ResilientConnection::create(host, retry);
match conn
- .send_with_timeout(
+ .send(
&crate::entrypoint::webgraph_server::Knows { site: params.site },
- Duration::from_secs(2),
+ Duration::from_secs(5),
+ Some(Duration::from_secs(60)),
)
.await
{
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 0ccb3bb6..27fa39d4 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -126,12 +126,13 @@ impl WorkerThread {
let results = self.results.lock().await.drain(..).collect::<Vec<_>>();
let res = conn
- .send_with_timeout(
+ .send(
&NewJobs {
responses: results,
num_jobs: 2 * self.num_jobs_per_fetch,
},
- Duration::from_secs(60 * 60),
+ Duration::from_secs(60),
+ Some(Duration::from_secs(60 * 60)),
)
.await;
diff --git a/core/src/distributed/sonic/mod.rs b/core/src/distributed/sonic/mod.rs
index e59d0c0a..2b55c60a 100644
--- a/core/src/distributed/sonic/mod.rs
+++ b/core/src/distributed/sonic/mod.rs
@@ -37,6 +37,9 @@ pub enum Error {
#[error("Failed to connect to peer: connection timeout")]
ConnectionTimeout,
+ #[error("Failed to get response for request: connection timeout")]
+ RequestTimeout,
+
#[error("Other")]
Other(#[from] anyhow::Error),
}
@@ -144,14 +147,14 @@ where
}
pub async fn send(self, request: &Req) -> Result<Res> {
- self.send_with_timeout(request, Duration::from_secs(30))
+ self.send_with_timeout(request, Duration::from_secs(90))
.await
}
pub async fn send_with_timeout(self, request: &Req, timeout: Duration) -> Result<Res> {
match tokio::time::timeout(timeout, self.send_without_timeout(request)).await {
Ok(res) => res,
- Err(_) => Err(Error::ConnectionTimeout),
+ Err(_) => Err(Error::RequestTimeout),
}
}
}
diff --git a/core/src/distributed/sonic/service.rs b/core/src/distributed/sonic/service.rs
index 07f4084f..90862096 100644
--- a/core/src/distributed/sonic/service.rs
+++ b/core/src/distributed/sonic/service.rs
@@ -105,19 +105,21 @@ impl<S: Service, Rt: Iterator<Item = Duration>> ResilientConnection<S, Rt> {
}
}
- pub async fn send_with_timeout<R: Wrapper<S>>(
+ /// If `req_timeout` is None, send with default timeout of 90 seconds.
+ pub async fn send<R: Wrapper<S>>(
mut self,
request: &R,
- timeout: Duration,
+ conn_timeout: Duration,
+ req_timeout: Option<Duration>,
) -> Result<R::Response> {
loop {
- match Connection::create_with_timeout(&self.addr, timeout).await {
+ match Connection::create_with_timeout(&self.addr, conn_timeout).await {
Ok(conn) => {
- let response = conn.send_with_timeout(request, timeout).await;
- if let Err(Error::ConnectionTimeout) = response {
- continue;
+ if let Some(req_timeout) = req_timeout {
+ return conn.send_with_timeout(request, req_timeout).await;
+ } else {
+ return conn.send(request).await;
}
- return response;
}
Err(Error::ConnectionTimeout) => {
if let Some(timeout) = self.retry.next() {
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index 53814954..16070e4a 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -181,6 +181,8 @@ pub async fn run(config: config::SearchServerConfig) -> Result<()> {
info!("search server is ready to accept requests on {}", addr);
loop {
- server.accept().await?;
+ if let Err(e) = server.accept().await {
+ tracing::error!("{}", e);
+ }
}
}
diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rs
index 0ecd30e8..eeaba9fa 100644
--- a/core/src/entrypoint/webgraph_server.rs
+++ b/core/src/entrypoint/webgraph_server.rs
@@ -75,7 +75,7 @@ impl Message<WebGraphService> for SimilarSites {
let urls = similar_sites
.iter()
- .map(|s| Url::parse(&("http://".to_string() + s.node.name.as_str())).unwrap())
+ .filter_map(|s| Url::parse(&("http://".to_string() + s.node.name.as_str())).ok())
.collect_vec();
let descriptions = server.searcher.get_homepage_descriptions(&urls).await;
@@ -83,9 +83,9 @@ impl Message<WebGraphService> for SimilarSites {
let similar_sites = similar_sites
.into_iter()
.map(|site| {
- let description = descriptions
- .get(&Url::parse(&("http://".to_string() + site.node.name.as_str())).unwrap())
- .cloned();
+ let description = Url::parse(&("http://".to_string() + site.node.name.as_str()))
+ .ok()
+ .and_then(|url| descriptions.get(&url).cloned());
ScoredSite {
site: site.node.name,
@@ -151,6 +151,8 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
info!("webgraph server is ready to accept requests on {}", addr);
loop {
- let _ = server.accept().await;
+ if let Err(e) = server.accept().await {
+ tracing::error!("{}", e);
+ }
}
}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 3b3695bd..33520426 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -58,11 +58,12 @@ impl RemoteSearcher {
let conn = self.conn();
if let Ok(Some(body)) = conn
- .send_with_timeout(
+ .send(
&search_server::Search {
query: query.clone(),
},
- Duration::from_secs(1),
+ Duration::from_secs(5),
+ Some(Duration::from_secs(60)),
)
.await
{
@@ -80,12 +81,13 @@ impl RemoteSearcher {
let conn = self.conn();
if let Ok(Some(body)) = conn
- .send_with_timeout(
+ .send(
&search_server::RetrieveWebsites {
websites: pointers.to_vec(),
query: original_query.to_string(),
},
- Duration::from_secs(1),
+ Duration::from_secs(5),
+ Some(Duration::from_secs(60)),
)
.await
{
@@ -98,11 +100,12 @@ impl RemoteSearcher {
let conn = self.conn();
if let Ok(body) = conn
- .send_with_timeout(
+ .send(
&search_server::GetWebpage {
url: url.to_string(),
},
- Duration::from_secs(1),
+ Duration::from_secs(5),
+ Some(Duration::from_secs(60)),
)
.await
{
@@ -116,11 +119,12 @@ impl RemoteSearcher {
let conn = self.conn();
if let Ok(body) = conn
- .send_with_timeout(
+ .send(
&search_server::GetHomepageDescriptions {
urls: urls.to_vec(),
},
- Duration::from_secs(1),
+ Duration::from_secs(5),
+ Some(Duration::from_secs(60)),
)
.await
{
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 9f194a1f..5fdb1d67 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -1747,15 +1747,6 @@ mod tests {
);
}
- #[test]
- fn domain_from_domain_url() {
- let url: Url = Url::parse("http://example.com").unwrap();
- assert_eq!(url.root_domain().unwrap(), "example.com");
-
- let url: Url = Url::parse("http://test.example.com").unwrap();
- assert_eq!(url.root_domain().unwrap(), "example.com");
- }
-
#[test]
fn metadata_updated_time() {
let html = r#"
diff --git a/core/src/webpage/url_ext.rs b/core/src/webpage/url_ext.rs
index 5d09f632..19132092 100644
--- a/core/src/webpage/url_ext.rs
+++ b/core/src/webpage/url_ext.rs
@@ -36,10 +36,33 @@ impl UrlExt for url::Url {
}
fn subdomain(&self) -> Option<&str> {
- let host = self.host_str()?;
- let suffix = std::str::from_utf8(LIST.domain(host.as_bytes())?.as_bytes()).ok()?;
- let suffix = suffix.strip_prefix(".")?;
- let subdomain = host.strip_suffix(suffix)?;
+ let domain = dbg!(self.root_domain()?);
+ let host = dbg!(self.host_str()?);
+
+ let mut subdomain = host.strip_suffix(domain)?;
+
+ if let Some(s) = subdomain.strip_suffix('.') {
+ subdomain = s;
+ }
+
Some(subdomain)
}
}
+
+#[cfg(test)]
+mod tests {
+ use url::Url;
+
+ use super::*;
+
+ #[test]
+ fn domain_from_domain_url() {
+ let url: Url = Url::parse("http://example.com").unwrap();
+ assert_eq!(url.root_domain().unwrap(), "example.com");
+
+ let url: Url = Url::parse("http://test.example.com").unwrap();
+ assert_eq!(url.root_domain().unwrap(), "example.com");
+
+ assert_eq!(url.subdomain().unwrap(), "test");
+ }
+}
diff --git a/docs/src/index.md b/docs/src/index.md
index 2373a69e..9340cb25 100644
--- a/docs/src/index.md
+++ b/docs/src/index.md
@@ -1,7 +1,7 @@
# Introduction
Stract is an open source web search engine written in Rust. It is designed to be fast, customizable, easy to use and scalable.
-This documentation provides a high-level overview of the project and its components. For more detailed information, please refer to the documentation directly in the source code.
+This documentation is intended to be at a high level of abstraction to give an overview of the project and its components. For more detailed information, please refer to the documentation directly in the source code.
# Contributing
We welcome and greatly appreciate contributions of all kinds. Please refer to [CONTRIBUTING.md](https://github.com/StractOrg/stract/blob/main/CONTRIBUTING.md) for more information on how you can contribute to the project.
diff --git a/docs/src/search_index.md b/docs/src/search_index.md
index 8df56ade..2ea24f74 100644
--- a/docs/src/search_index.md
+++ b/docs/src/search_index.md
@@ -1 +1,30 @@
-# Search Index
\ No newline at end of file
+# Search Index
+Imagine you are a librarian at a large library. You have a vast collection of books, but they are all unorganized. Let's say a visitor comes by and asks you about a cooking book that has recipes for pasta. How would you find the books? You could start by looking at the table-of-content for each book, but that would take a long time. What if you had a list of all the words in each book and the pages on which they appear? That would make it much easier to find the books the visitor is looking for. This is essentially what a search index is: a list of words and the documents in which they appear. This is called an inverted index, since it inverts the relationship between words and documents.
+
+We use [tantivy](https://github.com/quickwit-oss/tantivy/) as our search index. Tantivy is a full-text search engine library written in Rust and inspired by [Lucene](https://lucene.apache.org/). An excellent overview of how tantivy works can be found [here](https://github.com/quickwit-oss/tantivy/blob/main/ARCHITECTURE.md).
+
+## Indexing
+- Can be done distributed, since a tantivy index consists of multiple segments.
+
+- Tokenization
+
+- Ordered by all ranking signals that are computable before the query is known.
+
+## Searching
+- Distributed searcher
+ - Lookup a local searcher at each search server (one per shard)
+
+- intersection query
+
+- Search operators
+
+## Ranking
+- We don't use tantivy ranking
+
+- Bm25 for text combined with all other signals
+
+- Multiple stages
+ - "Initial guess" based on index sorting
+ - Linear regression
+ - LambdaMART
+ - Top 20 gets scored with a cross encoder and ranked again