web search done right
Sonic connection pool (#200)
Mikkel Denkere026fe55480d
diff --git a/Cargo.lock b/Cargo.lockindex bc99be0a..52a72701 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1225,6 +1225,23 @@ dependencies = ["rayon",]+[[package]]+name = "deadpool"+version = "0.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6541a3916932fe57768d4be0b1ffb5ec7cbf74ca8c903fdfd5c0fe8aa958f0ed"+dependencies = [+ "deadpool-runtime",+ "num_cpus",+ "tokio",+]++[[package]]+name = "deadpool-runtime"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"+[[package]]name = "deranged"version = "0.3.11"@@ -4627,6 +4644,7 @@ dependencies = ["crossbeam-channel","csv","dashmap",+ "deadpool","encoding_rs","enum_dispatch","eventsource-stream",diff --git a/Cargo.toml b/Cargo.tomlindex e28763ce..7534ef40 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -51,6 +51,7 @@ criterion = "0.5.1"crossbeam-channel = "0.5.6"csv = "1.1.6"dashmap = {version = "5.4.0", features = ["rayon"]}+deadpool = "0.12.1"encoding_rs = "0.8.31"enum_dispatch = "0.3.12"eventsource-stream = "0.2.3"diff --git a/assets/licenses.html b/assets/licenses.htmlindex b0c12ad8..fc618c17 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -44,7 +44,7 @@<h2>Overview of licenses:</h2><ul class="licenses-overview">- <li><a href="#Apache-2.0">Apache License 2.0</a> (391)</li>+ <li><a href="#Apache-2.0">Apache License 2.0</a> (393)</li><li><a href="#MIT">MIT License</a> (192)</li><li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (8)</li><li><a href="#BSD-3-Clause">BSD 3-Clause "New" or "Revised" License</a> (7)</li>@@ -3660,6 +3660,217 @@ You should also get your employer (if you work as a programmer) or school, if anWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.See the License for the specific language governing permissions andlimitations under the License.+</pre>+ </li>+ <li class="license">+ <h3 id="Apache-2.0">Apache License 2.0</h3>+ <h4>Used by:</h4>+ <ul class="license-used-by">+ <li><a href=" https://github.com/bikeshedder/deadpool ">deadpool-runtime 0.1.4</a></li>+ <li><a href=" https://github.com/bikeshedder/deadpool ">deadpool 0.12.1</a></li>+ </ul>+ <pre class="license-text"> Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for reasonable and customary use in describing the+ origin of the Work and reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may choose to offer,+ and charge a fee for, acceptance of support, warranty, indemnity,+ or other liability obligations and/or rights consistent with this+ License. However, in accepting such obligations, You may act only+ on Your own behalf and on Your sole responsibility, not on behalf+ of any other Contributor, and only if You agree to indemnify,+ defend, and hold each Contributor harmless for any liability+ incurred by, or claims asserted against, such Contributor by reason+ of your accepting any such warranty or additional liability.++ END OF TERMS AND CONDITIONS++ APPENDIX: How to apply the Apache License to your work.++ To apply the Apache License to your work, attach the following+ boilerplate notice, with the fields enclosed by brackets "{}"+ replaced with your own identifying information. (Don't include+ the brackets!) The text should be enclosed in the appropriate+ comment syntax for the file format. We also recommend that a+ file or class name and description of purpose be included on the+ same "printed page" as the copyright notice for easier+ identification within third-party archives.++ Copyright 2019 Michael P. Jung++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+</pre></li><li class="license">diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 6724c6a3..92101078 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -46,6 +46,7 @@ clap = {workspace = true}crossbeam-channel = {workspace = true}csv = {workspace = true}dashmap = {workspace = true}+deadpool = {workspace = true}encoding_rs = {workspace = true}enum_dispatch = {workspace = true}eventsource-stream = {workspace = true}diff --git a/crates/core/src/ampc/dht/client.rs b/crates/core/src/ampc/dht/client.rsindex 61743855..39d2da8f 100644--- a/crates/core/src/ampc/dht/client.rs+++ b/crates/core/src/ampc/dht/client.rs@@ -33,11 +33,59 @@ use super::{UpsertAction,};-#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+#[derive(Debug)]pub struct Node {api: api::RemoteClient,}+impl bincode::Encode for Node {+ fn encode<E: bincode::enc::Encoder>(+ &self,+ encoder: &mut E,+ ) -> std::prelude::v1::Result<(), bincode::error::EncodeError> {+ let addr = self.api.addr();+ addr.encode(encoder)+ }+}++impl bincode::Decode for Node {+ fn decode<D: bincode::de::Decoder>(+ decoder: &mut D,+ ) -> std::prelude::v1::Result<Self, bincode::error::DecodeError> {+ let addr = SocketAddr::decode(decoder)?;+ Ok(Self::new(addr))+ }+}++impl<'de> bincode::BorrowDecode<'de> for Node {+ fn borrow_decode<D: bincode::de::BorrowDecoder<'de>>(+ decoder: &mut D,+ ) -> std::prelude::v1::Result<Self, bincode::error::DecodeError> {+ let addr = SocketAddr::borrow_decode(decoder)?;+ Ok(Self::new(addr))+ }+}++impl serde::Serialize for Node {+ fn serialize<S>(&self, serializer: S) -> std::prelude::v1::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ let addr = self.api.addr();+ addr.serialize(serializer)+ }+}++impl<'de> serde::Deserialize<'de> for Node {+ fn deserialize<D>(deserializer: D) -> std::prelude::v1::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ let addr = SocketAddr::deserialize(deserializer)?;+ Ok(Self::new(addr))+ }+}+impl Clone for Node {fn clone(&self) -> Self {Self {diff --git a/crates/core/src/ampc/dht/mod.rs b/crates/core/src/ampc/dht/mod.rsindex 483f16dc..13e3f5db 100644--- a/crates/core/src/ampc/dht/mod.rs+++ b/crates/core/src/ampc/dht/mod.rs@@ -184,7 +184,7 @@ pub mod tests {#[traced_test]async fn test_simple_set_get() -> anyhow::Result<()> {let (raft1, server1, addr1) = server(1).await?;- let (raft2, server2, addr2) = server(2).await?;+ let (_, server2, addr2) = server(2).await?;let servers = vec![server1, server2];@@ -211,16 +211,6 @@ pub mod tests {}};- if let Err(e) = raft2.initialize(members.clone()).await {- match e {- openraft::error::RaftError::APIError(e) => match e {- InitializeError::NotAllowed(_) => {}- InitializeError::NotInMembers(_) => panic!("{:?}", e),- },- openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),- }- };-let c1 = RemoteClient::new(addr1);let c2 = RemoteClient::new(addr2);@@ -236,6 +226,14 @@ pub mod tests {let res = c1.get(table.clone(), "hello".as_bytes().into()).await?;assert_eq!(res, Some("world".as_bytes().into()));+ c2.set(+ table.clone(),+ "ensure-linearized-read".as_bytes().into(),+ vec![].into(),+ )+ .await?;+ tokio::time::sleep(std::time::Duration::from_secs(1)).await;+let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;assert_eq!(res, Some("world".as_bytes().into()));@@ -246,6 +244,14 @@ pub mod tests {).await?;+ c1.set(+ table.clone(),+ "ensure-linearized-read".as_bytes().into(),+ vec![].into(),+ )+ .await?;+ tokio::time::sleep(std::time::Duration::from_secs(1)).await;+let res = c1.get(table.clone(), "hello".as_bytes().into()).await?;assert_eq!(res, Some("world2".as_bytes().into()));@@ -299,7 +305,14 @@ pub mod tests {"world".as_bytes().into(),).await?;- tokio::time::sleep(std::time::Duration::from_secs(1)).await; // reads are not linearized (only eventual consistency)++ c2.set(+ table.clone(),+ "ensure-linearized-read".as_bytes().into(),+ vec![].into(),+ )+ .await?;+ tokio::time::sleep(std::time::Duration::from_secs(1)).await;let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;diff --git a/crates/core/src/ampc/dht/network/api.rs b/crates/core/src/ampc/dht/network/api.rsindex 73430344..169ac692 100644--- a/crates/core/src/ampc/dht/network/api.rs+++ b/crates/core/src/ampc/dht/network/api.rs@@ -283,46 +283,11 @@ impl std::fmt::Debug for RemoteClient {}}-#[derive(serde::Serialize, serde::Deserialize)]pub struct RemoteClient {self_remote: sonic::replication::RemoteClient<Server>,- #[serde(skip)]likely_leader: RwLock<Option<sonic::replication::RemoteClient<Server>>>,}-impl bincode::Encode for RemoteClient {- fn encode<E: bincode::enc::Encoder>(- &self,- encoder: &mut E,- ) -> Result<(), bincode::error::EncodeError> {- self.self_remote.encode(encoder)- }-}--impl bincode::Decode for RemoteClient {- fn decode<D: bincode::de::Decoder>(- decoder: &mut D,- ) -> Result<Self, bincode::error::DecodeError> {- let self_remote = sonic::replication::RemoteClient::decode(decoder)?;- Ok(Self {- self_remote,- likely_leader: RwLock::new(None),- })- }-}--impl<'de> bincode::BorrowDecode<'de> for RemoteClient {- fn borrow_decode<D: bincode::de::BorrowDecoder<'de>>(- decoder: &mut D,- ) -> Result<Self, bincode::error::DecodeError> {- let self_remote = sonic::replication::RemoteClient::borrow_decode(decoder)?;- Ok(Self {- self_remote,- likely_leader: RwLock::new(None),- })- }-}-impl RemoteClient {pub fn new(addr: SocketAddr) -> Self {Self {@@ -351,7 +316,7 @@ impl RemoteClient {.as_ref().unwrap_or(&self.self_remote).send_with_timeout(- &Set {+ Set {table: table.clone(),key: key.clone(),value: value.clone(),@@ -393,7 +358,7 @@ impl RemoteClient {sonic::Error::IO(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {+ | sonic::Error::PoolGet => {tokio::time::sleep(backoff).await;}sonic::Error::BadRequest@@ -420,7 +385,7 @@ impl RemoteClient {.as_ref().unwrap_or(&self.self_remote).send_with_timeout(- &BatchSet {+ BatchSet {table: table.clone(),values: values.clone(),},@@ -461,7 +426,7 @@ impl RemoteClient {sonic::Error::IO(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {+ | sonic::Error::PoolGet => {tokio::time::sleep(backoff).await;}sonic::Error::BadRequest@@ -482,7 +447,7 @@ impl RemoteClient {match self.self_remote.send_with_timeout(- &Get {+ Get {table: table.clone(),key: key.clone(),},@@ -495,7 +460,7 @@ impl RemoteClient {sonic::Error::IO(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {+ | sonic::Error::PoolGet => {tokio::time::sleep(backoff).await;}sonic::Error::BadRequest@@ -518,7 +483,7 @@ impl RemoteClient {match self.self_remote.send_with_timeout(- &BatchGet {+ BatchGet {table: table.clone(),keys: keys.clone(),},@@ -531,7 +496,7 @@ impl RemoteClient {sonic::Error::IO(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {+ | sonic::Error::PoolGet => {tokio::time::sleep(backoff).await;}sonic::Error::BadRequest@@ -556,7 +521,7 @@ impl RemoteClient {.as_ref().unwrap_or(&self.self_remote).send_with_timeout(- &DropTable {+ DropTable {table: table.clone(),},Duration::from_secs(5),@@ -596,7 +561,7 @@ impl RemoteClient {sonic::Error::IO(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {+ | sonic::Error::PoolGet => {tokio::time::sleep(backoff).await;}sonic::Error::BadRequest@@ -621,7 +586,7 @@ impl RemoteClient {.as_ref().unwrap_or(&self.self_remote).send_with_timeout(- &CreateTable {+ CreateTable {table: table.clone(),},Duration::from_secs(5),@@ -661,7 +626,7 @@ impl RemoteClient {sonic::Error::IO(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {+ | sonic::Error::PoolGet => {tokio::time::sleep(backoff).await;}sonic::Error::BadRequest@@ -685,7 +650,7 @@ impl RemoteClient {.await.as_ref().unwrap_or(&self.self_remote)- .send_with_timeout(&AllTables, Duration::from_secs(5))+ .send_with_timeout(AllTables, Duration::from_secs(5)).await;match res {@@ -721,7 +686,7 @@ impl RemoteClient {sonic::Error::IO(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {+ | sonic::Error::PoolGet => {tokio::time::sleep(backoff).await;}sonic::Error::BadRequest@@ -746,7 +711,7 @@ impl RemoteClient {.as_ref().unwrap_or(&self.self_remote).send_with_timeout(- &CloneTable {+ CloneTable {from: from.clone(),to: to.clone(),},@@ -787,7 +752,7 @@ impl RemoteClient {sonic::Error::IO(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {+ | sonic::Error::PoolGet => {tokio::time::sleep(backoff).await;}sonic::Error::BadRequest@@ -820,7 +785,7 @@ impl RemoteClient {.as_ref().unwrap_or(&self.self_remote).send_with_timeout(- &Upsert {+ Upsert {table: table.clone(),key: key.clone(),value: value.clone(),@@ -863,7 +828,7 @@ impl RemoteClient {sonic::Error::IO(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {+ | sonic::Error::PoolGet => {tokio::time::sleep(backoff).await;}sonic::Error::BadRequest@@ -896,7 +861,7 @@ impl RemoteClient {.as_ref().unwrap_or(&self.self_remote).send_with_timeout(- &BatchUpsert {+ BatchUpsert {table: table.clone(),upsert_fn: upsert.clone(),values: values.clone(),@@ -938,7 +903,7 @@ impl RemoteClient {sonic::Error::IO(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {+ | sonic::Error::PoolGet => {tokio::time::sleep(backoff).await;}sonic::Error::BadRequest@@ -964,7 +929,7 @@ impl RemoteClient {let res = self.self_remote.send_with_timeout(- &RangeGet {+ RangeGet {table: table.clone(),range: range.clone(),limit,@@ -981,7 +946,7 @@ impl RemoteClient {sonic::Error::IO(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {+ | sonic::Error::PoolGet => {tokio::time::sleep(backoff).await;}sonic::Error::BadRequestdiff --git a/crates/core/src/ampc/dht/network/raft.rs b/crates/core/src/ampc/dht/network/raft.rsindex 3e1a13d2..401dc65e 100644--- a/crates/core/src/ampc/dht/network/raft.rs+++ b/crates/core/src/ampc/dht/network/raft.rs@@ -17,6 +17,7 @@use std::{collections::{BTreeMap, BTreeSet},net::SocketAddr,+ ops::DerefMut,time::Duration,};@@ -169,7 +170,7 @@ async fn metrics(for backoff in retry {let res = client- .send_with_timeout(&rpc, Duration::from_secs(30))+ .send_with_timeout(rpc.clone(), Duration::from_secs(30)).await;match res {@@ -203,7 +204,10 @@ impl RemoteClient {}async fn raft_conn<E: std::error::Error>(&self,- ) -> Result<Connection<Server>, crate::bincode_utils::SerdeCompat<RPCError<E>>> {+ ) -> Result<+ impl DerefMut<Target = Connection<Server>>,+ crate::bincode_utils::SerdeCompat<RPCError<E>>,+ > {self.inner.conn().await.map_err(|e| {crate::bincode_utils::SerdeCompat(RPCError::Unreachable(openraft::error::Unreachable::new(&e),@@ -220,8 +224,8 @@ impl RemoteClient {R: sonic::service::Wrapper<Server>,E: std::error::Error,{- let conn = self.raft_conn().await?;- conn.send_with_timeout(&rpc, option.soft_ttl())+ let mut conn = self.raft_conn().await?;+ conn.send_with_timeout(rpc, option.soft_ttl()).await.map_err(|e| match e {sonic::Error::ConnectionTimeout | sonic::Error::RequestTimeout => {@@ -250,7 +254,7 @@ impl RemoteClient {.likely_leader.read().await- .send_with_timeout(&rpc, Duration::from_secs(30))+ .send_with_timeout(rpc.clone(), Duration::from_secs(30)).await;match res {@@ -284,7 +288,7 @@ impl RemoteClient {sonic::Error::IO(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {+ | sonic::Error::PoolGet => {tokio::time::sleep(backoff).await;}sonic::Error::BadRequest@@ -309,7 +313,7 @@ impl RemoteClient {.likely_leader.read().await- .send_with_timeout(&rpc, Duration::from_secs(30))+ .send_with_timeout(rpc.clone(), Duration::from_secs(30)).await;match res {@@ -343,7 +347,7 @@ impl RemoteClient {sonic::Error::IO(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {+ | sonic::Error::PoolGet => {tokio::time::sleep(backoff).await;}sonic::Error::BadRequestdiff --git a/crates/core/src/ampc/server.rs b/crates/core/src/ampc/server.rsindex 5da586a5..e0b0cf98 100644--- a/crates/core/src/ampc/server.rs+++ b/crates/core/src/ampc/server.rs@@ -38,49 +38,66 @@ impl<W> Server<W>whereW: Worker + 'static,{- fn handle(&mut self) -> Result<()> {- let req = block_on(self.conn.accept())?;-- match req.body().clone() {- Req::Coordinator(coord_req) => {- let res = match coord_req {- CoordReq::CurrentJob => Resp::Coordinator(CoordResp::CurrentJob(- self.current_job.lock().unwrap().clone(),- )),- CoordReq::ScheduleJob { job, mapper } => {- *self.current_job.lock().unwrap() = Some(job.clone());- let worker = Arc::clone(&self.worker);- let dht = self.dht.clone();-- let current_job = Arc::clone(&self.current_job);- std::thread::spawn(move || {- mapper.map(job.clone(), &worker, dht.as_ref().expect("DHT not set"));- current_job.lock().unwrap().take();- });-- Resp::Coordinator(CoordResp::ScheduleJob(()))- }- CoordReq::Setup { dht } => {- self.dht = Some(Arc::new(dht));- Resp::Coordinator(CoordResp::Setup(()))- }- };-- block_on(req.respond(res))?;- }- Req::User(user_req) => {- let worker = Arc::clone(&self.worker);-- std::thread::spawn(move || {- let res = Resp::User(worker.handle(user_req.clone()));- block_on(req.respond(res)).unwrap();- });- }- };+ async fn async_handle(&mut self) -> Result<()> {+ let mut conn = self.conn.accept().await?;+ while let Ok(req) = conn.request().await {+ match req.body().clone() {+ Req::Coordinator(coord_req) => {+ let res = match coord_req {+ CoordReq::CurrentJob => Resp::Coordinator(CoordResp::CurrentJob(+ self.current_job.lock().unwrap().clone(),+ )),+ CoordReq::ScheduleJob { job, mapper } => {+ *self.current_job.lock().unwrap() = Some(job.clone());+ let worker = Arc::clone(&self.worker);+ let dht = self.dht.clone();++ let current_job = Arc::clone(&self.current_job);+ std::thread::spawn(move || {+ mapper.map(+ job.clone(),+ &worker,+ dht.as_ref().expect("DHT not set"),+ );+ current_job.lock().unwrap().take();+ });++ Resp::Coordinator(CoordResp::ScheduleJob(()))+ }+ CoordReq::Setup { dht } => {+ self.dht = Some(Arc::new(dht));+ Resp::Coordinator(CoordResp::Setup(()))+ }+ };++ req.respond(res).await?;+ }+ Req::User(user_req) => {+ let worker = Arc::clone(&self.worker);++ let (tx, rx) = crossbeam_channel::bounded(1);++ std::thread::spawn(move || {+ let res = Resp::User(worker.handle(user_req.clone()));+ tx.send(res).unwrap();+ });++ let res = tokio::task::spawn_blocking(move || rx.recv())+ .await+ .unwrap()?;++ req.respond(res).await?;+ }+ };+ }Ok(())}+ fn handle(&mut self) -> Result<()> {+ block_on(self.async_handle())+ }+pub fn bind(addr: impl ToSocketAddrs, worker: W) -> Result<Server<W>> {let worker = Arc::new(worker);let conn = block_on(sonic::Server::bind(addr))?;diff --git a/crates/core/src/ampc/worker.rs b/crates/core/src/ampc/worker.rsindex 05fedacd..249371a6 100644--- a/crates/core/src/ampc/worker.rs+++ b/crates/core/src/ampc/worker.rs@@ -90,7 +90,7 @@ where}fn send_raw(&self, req: &JobReq<Self::Job>) -> Result<JobResp<Self::Job>> {- let conn = self.conn()?;+ let mut conn = self.conn()?;let res = block_on(conn.send(req))?;Ok(res)}diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rsindex a76717f2..ece870a5 100644--- a/crates/core/src/api/mod.rs+++ b/crates/core/src/api/mod.rs@@ -184,11 +184,11 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {);let host_webgraph =- RemoteWebgraph::new(cluster.clone(), crate::config::WebgraphGranularity::Host);+ RemoteWebgraph::new(cluster.clone(), crate::config::WebgraphGranularity::Host).await;let page_webgraph =- RemoteWebgraph::new(cluster.clone(), crate::config::WebgraphGranularity::Page);+ RemoteWebgraph::new(cluster.clone(), crate::config::WebgraphGranularity::Page).await;- let dist_searcher = DistributedSearcher::new(Arc::clone(&cluster));+ let dist_searcher = DistributedSearcher::new(Arc::clone(&cluster)).await;let live_searcher = LiveSearcher::new(Arc::clone(&cluster));let state = {diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex c5ff1a33..d2de173b 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -49,7 +49,7 @@ pub struct IndexingLocalConfig {}#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]-#[serde(tag = "_type")]+#[serde(tag = "type")]pub enum IndexingGraphConfig {Local { path: String },Remote { gossip: GossipConfig },@@ -77,7 +77,7 @@ pub struct WebgraphConstructConfig {}#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]-#[serde(tag = "_type")]+#[serde(tag = "type")]pub enum WarcSource {HTTP(HttpConfig),Local(LocalConfig),@@ -382,7 +382,7 @@ pub struct CrawlRouterConfig {}#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]-#[serde(tag = "_type", content = "args", rename_all = "snake_case")]+#[serde(tag = "type", content = "args", rename_all = "snake_case")]pub enum AcceleratorDevice {Cpu,Cuda(usize),diff --git a/crates/core/src/crawler/router.rs b/crates/core/src/crawler/router.rsindex a0a14c8d..8eb3607d 100644--- a/crates/core/src/crawler/router.rs+++ b/crates/core/src/crawler/router.rs@@ -27,10 +27,10 @@ impl RemoteCoordinator {}async fn sample_job(&self) -> Result<Option<Job>> {- let conn = self.conn().await?;+ let mut conn = self.conn().await?;let response = conn- .send_with_timeout(&GetJob {}, Duration::from_secs(90))+ .send_with_timeout(GetJob {}, Duration::from_secs(90)).await?;Ok(response)diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex de39fe06..cb0a420b 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -96,9 +96,9 @@ impl WorkerThread {pub async fn run(self) {loop {- let conn = self.router_conn().await.unwrap();+ let mut conn = self.router_conn().await.unwrap();let res = conn- .send_with_timeout(&NewJob {}, Duration::from_secs(90))+ .send_with_timeout(NewJob {}, Duration::from_secs(90)).await;match res {diff --git a/crates/core/src/distributed/sonic/connection_pool.rs b/crates/core/src/distributed/sonic/connection_pool.rsnew file mode 100644index 00000000..1924bf82--- /dev/null+++ b/crates/core/src/distributed/sonic/connection_pool.rs@@ -0,0 +1,170 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::net::SocketAddr;++use crate::Result;+use deadpool::managed;++use super::service::Service;++pub trait Connection {+ type Manager: managed::Manager;++ fn new_manager(addr: SocketAddr) -> Self::Manager;+}++pub struct ConnectionPool<C>+where+ C: Connection,+{+ addr: SocketAddr,+ pool: managed::Pool<C::Manager>,+}++impl<C> std::fmt::Debug for ConnectionPool<C>+where+ C: Connection,+{+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("ConnectionPool")+ .field("addr", &self.addr)+ .finish()+ }+}++impl<C> ConnectionPool<C>+where+ C: Connection,+{+ pub fn new(addr: SocketAddr) -> Result<Self> {+ let manager = C::new_manager(addr);+ let pool = managed::Pool::builder(manager).build()?;++ Ok(Self { addr, pool })+ }++ pub async fn get(&self) -> Result<managed::Object<C::Manager>> {+ self.pool+ .get()+ .await+ .map_err(|_| anyhow::anyhow!("Failed to get connection from pool"))+ }+}++pub struct Manager<Req, Res> {+ addr: SocketAddr,+ _marker: std::marker::PhantomData<(Req, Res)>,+}++impl<Req, Res> Manager<Req, Res> {+ pub fn new(addr: SocketAddr) -> Self {+ Self {+ addr,+ _marker: std::marker::PhantomData,+ }+ }+}++impl<Req, Res> Connection for super::Connection<Req, Res>+where+ Req: Send + Sync + bincode::Encode,+ Res: Send + Sync + bincode::Decode,+{+ type Manager = Manager<Req, Res>;++ fn new_manager(addr: SocketAddr) -> Self::Manager {+ Manager::new(addr)+ }+}++impl<Req, Res> managed::Manager for Manager<Req, Res>+where+ Req: Send + Sync + bincode::Encode,+ Res: Send + Sync + bincode::Decode,+{+ type Type = super::Connection<Req, Res>;+ type Error = anyhow::Error;++ async fn create(&self) -> Result<Self::Type, Self::Error> {+ Ok(super::Connection::connect(self.addr).await?)+ }++ async fn recycle(+ &self,+ obj: &mut Self::Type,+ _: &managed::Metrics,+ ) -> managed::RecycleResult<Self::Error> {+ if obj.is_closed().await {+ Err(managed::RecycleError::Message(+ "Connection is closed".into(),+ ))+ } else {+ Ok(())+ }+ }+}++pub struct ServiceManager<S> {+ addr: SocketAddr,+ _marker: std::marker::PhantomData<S>,+}++impl<S> ServiceManager<S> {+ pub fn new(addr: SocketAddr) -> Self {+ Self {+ addr,+ _marker: std::marker::PhantomData,+ }+ }+}++impl<S> Connection for super::service::Connection<S>+where+ S: Send + Sync + Service,+{+ type Manager = ServiceManager<S>;++ fn new_manager(addr: SocketAddr) -> Self::Manager {+ ServiceManager::new(addr)+ }+}++impl<S> managed::Manager for ServiceManager<S>+where+ S: Send + Sync + Service,+{+ type Type = super::service::Connection<S>;+ type Error = anyhow::Error;++ async fn create(&self) -> Result<Self::Type, Self::Error> {+ Ok(super::service::Connection::create(self.addr).await?)+ }++ async fn recycle(+ &self,+ obj: &mut Self::Type,+ _: &managed::Metrics,+ ) -> managed::RecycleResult<Self::Error> {+ if obj.is_closed().await {+ Err(managed::RecycleError::Message(+ "Connection is closed".into(),+ ))+ } else {+ Ok(())+ }+ }+}diff --git a/crates/core/src/distributed/sonic/mod.rs b/crates/core/src/distributed/sonic/mod.rsindex 08fb1e96..06ca6dfd 100644--- a/crates/core/src/distributed/sonic/mod.rs+++ b/crates/core/src/distributed/sonic/mod.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -14,13 +14,17 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+pub mod connection_pool;pub mod replication;pub mod service;-use std::{marker::PhantomData, time::Duration};+pub use connection_pool::ConnectionPool;+use std::{marker::PhantomData, task::Poll, time::Duration};++use futures::future::poll_fn;use tokio::{- io::{AsyncReadExt, AsyncWriteExt},+ io::{AsyncReadExt, AsyncWriteExt, ReadBuf},net::{TcpListener, TcpStream, ToSocketAddrs},};@@ -39,8 +43,8 @@ pub enum Error {#[error("Failed to get response for request: connection timeout")]RequestTimeout,- #[error("Could not build connection pool")]- PoolCreation,+ #[error("Could not get connection from pool")]+ PoolGet,#[error("The request could not be processed")]BadRequest,@@ -109,7 +113,7 @@ where}}- async fn send_without_timeout(mut self, request: &Req) -> Result<Res> {+ async fn send_without_timeout(&mut self, request: &Req) -> Result<Res> {let bytes = bincode::encode_to_vec(request, bincode::config::standard()).unwrap();// disable linger to avoid TIME_WAIT.@@ -139,7 +143,6 @@ wherelet mut buf = vec![0; header.body_size];self.stream.read_exact(&mut buf).await?;self.stream.flush().await?;- self.stream.shutdown().await?;tracing::debug!("deserializing {:?}", std::any::type_name::<(Req, Res)>());let (res, _) = bincode::decode_from_slice(&buf, bincode::config::standard()).unwrap();@@ -147,17 +150,27 @@ whereOk(res)}- pub async fn send(self, request: &Req) -> Result<Res> {+ pub async fn send(&mut self, request: &Req) -> Result<Res> {self.send_with_timeout(request, Duration::from_secs(90)).await}- pub async fn send_with_timeout(self, request: &Req, timeout: Duration) -> Result<Res> {+ pub async fn send_with_timeout(&mut self, request: &Req, timeout: Duration) -> Result<Res> {match tokio::time::timeout(timeout, self.send_without_timeout(request)).await {Ok(res) => res,Err(_) => Err(Error::RequestTimeout),}}++ pub async fn is_closed(&self) -> bool {+ poll_fn(+ |cx| match self.stream.poll_peek(cx, &mut ReadBuf::new(&mut [0u8])) {+ Poll::Ready(t) => Poll::Ready(t.is_err()),+ Poll::Pending => Poll::Ready(false),+ },+ )+ .await+ }}#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)]@@ -183,9 +196,33 @@ where})}- async fn parse_incoming_stream(&self, mut stream: TcpStream) -> Result<Request<Req, Res>> {+ pub async fn accept(&self) -> Result<ServerConnection<Req, Res>> {+ let (stream, client) = self.listener.accept().await?;+ tracing::debug!(?client, "accepted connection");++ Ok(ServerConnection::new(stream))+ }+}++pub struct ServerConnection<Req, Res> {+ stream: TcpStream,+ marker: PhantomData<(Req, Res)>,+}++impl<Req, Res> ServerConnection<Req, Res>+where+ Req: bincode::Decode,+{+ fn new(stream: TcpStream) -> Self {+ ServerConnection {+ stream,+ marker: PhantomData,+ }+ }++ pub async fn request(&mut self) -> Result<Request<'_, Req, Res>> {let mut header_buf = vec![0; std::mem::size_of::<Header>()];- stream.read_exact(&mut header_buf).await?;+ self.stream.read_exact(&mut header_buf).await?;let header: Header = *bytemuck::from_bytes(&header_buf);if header.body_size > MAX_BODY_SIZE_BYTES {@@ -197,52 +234,38 @@ wherelet mut buf = vec![0; header.body_size];- stream.read_exact(&mut buf).await?;+ self.stream.read_exact(&mut buf).await?;let (body, _) = bincode::decode_from_slice(&buf, bincode::config::standard()).unwrap();Ok(Request {- stream,+ conn: self,body: Some(body),- marker: PhantomData,})}-- pub async fn accept(&self) -> Result<Request<Req, Res>> {- let (stream, client) = self.listener.accept().await?;- tracing::debug!(?client, "accepted connection");-- tokio::time::timeout(Duration::from_secs(60), self.parse_incoming_stream(stream))- .await- .map_err(|_| Error::ConnectionTimeout)?- }}-pub struct Request<Req, Res> {- stream: TcpStream,+pub struct Request<'a, Req, Res> {+ conn: &'a mut ServerConnection<Req, Res>,body: Option<Req>,- marker: PhantomData<(Req, Res)>,}-impl<Req, Res> Request<Req, Res>+impl<'a, Req, Res> Request<'a, Req, Res>whereRes: bincode::Encode,{- async fn respond_without_timeout(mut self, response: Res) -> Result<()> {+ async fn respond_without_timeout(self, response: Res) -> Result<()> {let bytes = bincode::encode_to_vec(&response, bincode::config::standard()).unwrap();let header = Header {body_size: bytes.len(),};- self.stream.write_all(bytemuck::bytes_of(&header)).await?;- self.stream.write_all(&bytes).await?;- self.stream.flush().await?;-- // wait for client to close connection- let mut buf: [u8; 1] = [0];- tokio::time::timeout(Duration::from_secs(5), self.stream.read_exact(&mut buf))- .await- .ok();+ self.conn+ .stream+ .write_all(bytemuck::bytes_of(&header))+ .await?;+ self.conn.stream.write_all(&bytes).await?;+ self.conn.stream.flush().await?;Ok(())}@@ -319,12 +342,13 @@ mod tests {let (a2, b2) = (a1.clone(), b1.clone());let (svr_res, con_res) = fixture(|svr| async move {- let req = svr.accept().await?;+ let mut conn = svr.accept().await?;+ let req = conn.request().await?;prop_assert_eq!(req.body(), &a1);req.respond(b1).await?;Ok(())},- |con| async move {+ |mut con| async move {let res = con.send(&a2).await?;prop_assert_eq!(res, b2);Ok(())diff --git a/crates/core/src/distributed/sonic/replication.rs b/crates/core/src/distributed/sonic/replication.rsindex bfafb881..58cb54bc 100644--- a/crates/core/src/distributed/sonic/replication.rs+++ b/crates/core/src/distributed/sonic/replication.rs@@ -18,46 +18,19 @@ use futures::future::join_all;use rand::seq::IteratorRandom;use super::Result;-use crate::distributed::{retry_strategy::ExponentialBackoff, sonic};-use std::{net::SocketAddr, time::Duration};+use crate::distributed::{cluster::Cluster, retry_strategy::ExponentialBackoff, sonic};+use std::{net::SocketAddr, ops::DerefMut, sync::Arc, time::Duration};-#[derive(Debug, serde::Serialize, serde::Deserialize)]-pub struct RemoteClient<S> {+#[derive(Debug)]+pub struct RemoteClient<S>+where+ S: sonic::service::Service,+{addr: SocketAddr,+ pool: sonic::ConnectionPool<sonic::service::Connection<S>>,_phantom: std::marker::PhantomData<S>,}-impl<S> bincode::Encode for RemoteClient<S> {- fn encode<E: bincode::enc::Encoder>(- &self,- encoder: &mut E,- ) -> Result<(), bincode::error::EncodeError> {- self.addr.encode(encoder)- }-}--impl<S> bincode::Decode for RemoteClient<S> {- fn decode<D: bincode::de::Decoder>(- decoder: &mut D,- ) -> Result<Self, bincode::error::DecodeError> {- Ok(Self {- addr: SocketAddr::decode(decoder)?,- _phantom: std::marker::PhantomData,- })- }-}--impl<'de, S> bincode::BorrowDecode<'de> for RemoteClient<S> {- fn borrow_decode<D: bincode::de::BorrowDecoder<'de>>(- decoder: &mut D,- ) -> Result<Self, bincode::error::DecodeError> {- Ok(Self {- addr: SocketAddr::borrow_decode(decoder)?,- _phantom: std::marker::PhantomData,- })- }-}-impl<S> Clone for RemoteClient<S>whereS: sonic::service::Service,@@ -78,6 +51,7 @@ wherepub fn create(addr: SocketAddr) -> Self {Self {addr,+ pool: sonic::ConnectionPool::new(addr).unwrap(),_phantom: std::marker::PhantomData,}}@@ -91,20 +65,14 @@ impl<S> RemoteClient<S>whereS: sonic::service::Service,{- pub async fn conn(&self) -> Result<sonic::service::Connection<S>> {- let retry = ExponentialBackoff::from_millis(30)- .with_limit(Duration::from_millis(200))- .take(5);-- sonic::service::Connection::create_with_timeout_retry(- self.addr,- Duration::from_secs(30),- retry,- )- .await+ pub async fn conn(&self) -> Result<impl DerefMut<Target = sonic::service::Connection<S>>> {+ self.pool+ .get()+ .await+ .map_err(|_| crate::distributed::sonic::Error::PoolGet)}- pub async fn send<R: sonic::service::Wrapper<S>>(&self, req: &R) -> Result<R::Response> {+ pub async fn send<R: sonic::service::Wrapper<S> + Clone>(&self, req: R) -> Result<R::Response> {self.send_with_timeout_retry(req,Duration::from_secs(60),@@ -113,7 +81,7 @@ where.await}- pub async fn batch_send<R: sonic::service::Wrapper<S>>(+ pub async fn batch_send<R: sonic::service::Wrapper<S> + Clone>(&self,reqs: &[R],) -> Result<Vec<R::Response>> {@@ -125,33 +93,33 @@ where.await}- pub async fn send_with_timeout<R: sonic::service::Wrapper<S>>(+ pub async fn send_with_timeout<R: sonic::service::Wrapper<S> + Clone>(&self,- req: &R,+ req: R,timeout: Duration,) -> Result<R::Response> {- let conn = self.conn().await?;+ let mut conn = self.conn().await?;conn.send_with_timeout(req, timeout).await}- pub async fn batch_send_with_timeout<R: sonic::service::Wrapper<S>>(+ pub async fn batch_send_with_timeout<R: sonic::service::Wrapper<S> + Clone>(&self,reqs: &[R],timeout: Duration,) -> Result<Vec<R::Response>> {- let conn = self.conn().await?;+ let mut conn = self.conn().await?;conn.batch_send_with_timeout(reqs, timeout).await}- pub async fn send_with_timeout_retry<R: sonic::service::Wrapper<S>>(+ pub async fn send_with_timeout_retry<R: sonic::service::Wrapper<S> + Clone>(&self,- req: &R,+ req: R,timeout: Duration,retry: impl Iterator<Item = Duration>,) -> Result<R::Response> {let mut er = None;for backoff in retry {- match self.send_with_timeout(req, timeout).await {+ match self.send_with_timeout(req.clone(), timeout).await {Ok(r) => return Ok(r),Err(e) => {tracing::error!("Failed to send request: {:?}", e);@@ -164,7 +132,7 @@ whereErr(er.unwrap())}- pub async fn batch_send_with_timeout_retry<R: sonic::service::Wrapper<S>>(+ pub async fn batch_send_with_timeout_retry<R: sonic::service::Wrapper<S> + Clone>(&self,reqs: &[R],timeout: Duration,@@ -225,29 +193,33 @@ whereSelf { clients }}+ pub fn is_empty(&self) -> bool {+ self.clients.is_empty()+ }+async fn send_single<Req>(&self,- req: &Req,+ req: Req,client: &RemoteClient<S>,) -> Result<(SocketAddr, Req::Response)>where- Req: sonic::service::Wrapper<S>,+ Req: sonic::service::Wrapper<S> + Clone,{Ok((client.addr(), client.send(req).await?))}pub async fn send<Req, Sel>(&self,- req: &Req,+ req: Req,selector: &Sel,) -> Result<Vec<(SocketAddr, Req::Response)>>where- Req: sonic::service::Wrapper<S>,+ Req: sonic::service::Wrapper<S> + Clone,Sel: ReplicaSelector<S>,{let mut futures = Vec::new();for client in selector.select(&self.clients) {- futures.push(self.send_single(req, client));+ futures.push(self.send_single(req.clone(), client));}let mut results = Vec::new();@@ -269,7 +241,7 @@ whereclient: &RemoteClient<S>,) -> Result<(SocketAddr, Vec<Req::Response>)>where- Req: sonic::service::Wrapper<S>,+ Req: sonic::service::Wrapper<S> + Clone,{Ok((client.addr(), client.batch_send(reqs).await?))}@@ -280,7 +252,7 @@ whereselector: &Sel,) -> Result<Vec<(SocketAddr, Vec<Req::Response>)>>where- Req: sonic::service::Wrapper<S>,+ Req: sonic::service::Wrapper<S> + Clone,Sel: ReplicaSelector<S>,{let mut futures = Vec::new();@@ -304,6 +276,8 @@ wherepub trait ShardIdentifier: PartialEq + Eq + Clone {}+impl ShardIdentifier for () {}+pub trait ShardSelector<S: sonic::service::Service, Id: ShardIdentifier> {fn select<'a>(&self, shards: &'a [Shard<S, Id>]) -> Vec<&'a Shard<S, Id>>;}@@ -373,14 +347,18 @@ whereSelf { shards }}+ pub fn is_empty(&self) -> bool {+ self.shards.is_empty()+ }+async fn send_single<Req, Sel>(&self,- req: &Req,+ req: Req,shard: &Shard<S, Id>,replica_selector: &Sel,) -> Result<(Id, Vec<(SocketAddr, Req::Response)>)>where- Req: sonic::service::Wrapper<S>,+ Req: sonic::service::Wrapper<S> + Clone,Sel: ReplicaSelector<S>,{Ok((@@ -391,18 +369,18 @@ wherepub async fn send<Req, SSel, RSel>(&self,- req: &Req,+ req: Req,shard_selector: &SSel,replica_selector: &RSel,) -> Result<Vec<(Id, Vec<(SocketAddr, Req::Response)>)>>where- Req: sonic::service::Wrapper<S>,+ Req: sonic::service::Wrapper<S> + Clone,SSel: ShardSelector<S, Id>,RSel: ReplicaSelector<S>,{let mut futures = Vec::new();for shard in shard_selector.select(&self.shards) {- futures.push(self.send_single(req, shard, replica_selector));+ futures.push(self.send_single(req.clone(), shard, replica_selector));}let mut results = Vec::new();@@ -425,7 +403,7 @@ wherereplica_selector: &Sel,) -> Result<(Id, Vec<(SocketAddr, Vec<Req::Response>)>)>where- Req: sonic::service::Wrapper<S>,+ Req: sonic::service::Wrapper<S> + Clone,Sel: ReplicaSelector<S>,{Ok((@@ -441,7 +419,7 @@ wherereplica_selector: &RSel,) -> Result<Vec<(Id, Vec<(SocketAddr, Vec<Req::Response>)>)>>where- Req: sonic::service::Wrapper<S>,+ Req: sonic::service::Wrapper<S> + Clone,SSel: ShardSelector<S, Id>,RSel: ReplicaSelector<S>,{@@ -463,3 +441,52 @@ whereOk(results)}}++pub trait ReusableClientManager {+ const CLIENT_REFRESH_INTERVAL: Duration;++ type Service: sonic::service::Service;+ type ShardId: ShardIdentifier;++ fn new_client(+ &self,+ cluster: &Cluster,+ ) -> impl std::future::Future<Output = ShardedClient<Self::Service, Self::ShardId>>;+}++pub struct ReusableShardedClient<M>+where+ M: ReusableClientManager,+{+ cluster: Arc<Cluster>,+ client: Arc<sonic::replication::ShardedClient<M::Service, M::ShardId>>,+ last_client_update: std::time::Instant,+ manager: M,+}++impl<M> ReusableShardedClient<M>+where+ M: ReusableClientManager,+{+ pub async fn new(cluster: Arc<Cluster>, manager: M) -> Self {+ let client = Arc::new(manager.new_client(&cluster).await);+ let last_client_update = std::time::Instant::now();++ Self {+ cluster,+ client,+ last_client_update,+ manager,+ }+ }++ pub async fn conn(&mut self) -> Arc<sonic::replication::ShardedClient<M::Service, M::ShardId>> {+ if self.client.is_empty() || self.last_client_update.elapsed() > M::CLIENT_REFRESH_INTERVAL+ {+ self.client = Arc::new(self.manager.new_client(&self.cluster).await);+ self.last_client_update = std::time::Instant::now();+ }++ self.client.clone()+ }+}diff --git a/crates/core/src/distributed/sonic/service.rs b/crates/core/src/distributed/sonic/service.rsindex 2857b2f2..01a40715 100644--- a/crates/core/src/distributed/sonic/service.rs+++ b/crates/core/src/distributed/sonic/service.rs@@ -23,8 +23,7 @@ use crate::OneOrMany;use super::Result;pub trait Service: Sized + Send + Sync + 'static {- type Request: bincode::Decode + Send + Sync;- type RequestRef<'a>: bincode::Encode + Send + Sync;+ type Request: bincode::Encode + bincode::Decode + Send + Sync;type Response: bincode::Encode + bincode::Decode + Send + Sync;fn handle(@@ -38,7 +37,7 @@ pub trait Message<S: Service> {fn handle(self, server: &S) -> impl std::future::Future<Output = Self::Response>;}pub trait Wrapper<S: Service>: Message<S> {- fn wrap_request_ref(req: &Self) -> S::RequestRef<'_>;+ fn wrap_request(req: Self) -> S::Request;fn unwrap_response(res: S::Response) -> Option<Self::Response>;}@@ -55,27 +54,29 @@ impl<S: Service> Server<S> {})}pub async fn accept(&self) -> Result<()> {- let mut req = self.inner.accept().await?;+ let mut conn = self.inner.accept().await?;let service = Arc::clone(&self.service);tokio::spawn(async move {- match req.take_body() {- OneOrMany::One(body) => {- let res = S::handle(body, &service).await;+ while let Ok(mut req) = conn.request().await {+ match req.take_body() {+ OneOrMany::One(body) => {+ let res = S::handle(body, &service).await;- if let Err(e) = req.respond(OneOrMany::One(res)).await {- tracing::error!("failed to respond to request: {}", e);+ if let Err(e) = req.respond(OneOrMany::One(res)).await {+ tracing::error!("failed to respond to request: {}", e);+ }}- }- OneOrMany::Many(bodies) => {- let mut res = Vec::new();+ OneOrMany::Many(bodies) => {+ let mut res = Vec::new();- for req in bodies {- res.push(S::handle(req, &service).await);- }+ for req in bodies {+ res.push(S::handle(req, &service).await);+ }- if let Err(e) = req.respond(OneOrMany::Many(res)).await {- tracing::error!("failed to respond to request: {}", e);+ if let Err(e) = req.respond(OneOrMany::Many(res)).await {+ tracing::error!("failed to respond to request: {}", e);+ }}}}@@ -85,12 +86,12 @@ impl<S: Service> Server<S> {}}-pub struct Connection<'a, S: Service> {- inner: super::Connection<OneOrMany<S::RequestRef<'a>>, OneOrMany<S::Response>>,+pub struct Connection<S: Service> {+ inner: super::Connection<OneOrMany<S::Request>, OneOrMany<S::Response>>,}-impl<'a, S: Service> Connection<'a, S> {- pub async fn create(server: impl ToSocketAddrs) -> Result<Connection<'a, S>> {+impl<S: Service> Connection<S> {+ pub async fn create(server: impl ToSocketAddrs) -> Result<Connection<S>> {Ok(Connection {inner: super::Connection::create(server).await?,})@@ -99,7 +100,7 @@ impl<'a, S: Service> Connection<'a, S> {pub async fn create_with_timeout(server: impl ToSocketAddrs,timeout: Duration,- ) -> Result<Connection<'a, S>> {+ ) -> Result<Connection<S>> {Ok(Connection {inner: super::Connection::create_with_timeout(server, timeout).await?,})@@ -109,16 +110,16 @@ impl<'a, S: Service> Connection<'a, S> {server: impl ToSocketAddrs + Clone,timeout: Duration,retry: impl Iterator<Item = Duration>,- ) -> Result<Connection<'a, S>> {+ ) -> Result<Connection<S>> {Ok(Connection {inner: super::Connection::create_with_timeout_retry(server, timeout, retry).await?,})}- pub async fn send_without_timeout<R: Wrapper<S>>(self, request: &'a R) -> Result<R::Response> {+ pub async fn send_without_timeout<R: Wrapper<S>>(&mut self, request: R) -> Result<R::Response> {Ok(R::unwrap_response(self.inner- .send_without_timeout(&OneOrMany::One(R::wrap_request_ref(request)))+ .send_without_timeout(&OneOrMany::One(R::wrap_request(request))).await?.one().expect("response is missing"),@@ -126,10 +127,10 @@ impl<'a, S: Service> Connection<'a, S> {.unwrap())}- pub async fn send<R: Wrapper<S>>(self, request: &'a R) -> Result<R::Response> {+ pub async fn send<R: Wrapper<S>>(&mut self, request: R) -> Result<R::Response> {Ok(R::unwrap_response(self.inner- .send(&OneOrMany::One(R::wrap_request_ref(request)))+ .send(&OneOrMany::One(R::wrap_request(request))).await?.one().expect("response is missing"),@@ -138,13 +139,13 @@ impl<'a, S: Service> Connection<'a, S> {}pub async fn send_with_timeout<R: Wrapper<S>>(- self,- request: &'a R,+ &mut self,+ request: R,timeout: Duration,) -> Result<R::Response> {Ok(R::unwrap_response(self.inner- .send_with_timeout(&OneOrMany::One(R::wrap_request_ref(request)), timeout)+ .send_with_timeout(&OneOrMany::One(R::wrap_request(request)), timeout).await?.one().expect("response is missing"),@@ -152,9 +153,9 @@ impl<'a, S: Service> Connection<'a, S> {.unwrap())}- pub async fn batch_send_with_timeout<R: Wrapper<S>>(- self,- requests: &'a [R],+ pub async fn batch_send_with_timeout<R: Wrapper<S> + Clone>(+ &mut self,+ requests: &[R],timeout: Duration,) -> Result<Vec<R::Response>> {Ok(self@@ -163,7 +164,7 @@ impl<'a, S: Service> Connection<'a, S> {&OneOrMany::Many(requests.iter()- .map(|req| R::wrap_request_ref(req))+ .map(|req| R::wrap_request(req.clone())).collect::<Vec<_>>(),),timeout,@@ -174,6 +175,10 @@ impl<'a, S: Service> Connection<'a, S> {.map(|res| R::unwrap_response(res).unwrap()).collect())}++ pub async fn is_closed(&self) -> bool {+ self.inner.is_closed().await+ }}macro_rules! sonic_service {@@ -185,22 +190,18 @@ macro_rules! sonic_service {use $crate::distributed::sonic;- #[derive(Debug, Clone, ::bincode::Decode)]+ #[derive(Debug, Clone, ::bincode::Encode, ::bincode::Decode)]pub enum Request {$($req(Box<$req>),)*}- #[derive(Debug, Clone, ::bincode::Encode)]- pub enum RequestRef<'a> {- $($req(&'a $req),)*- }#[derive(::bincode::Encode, ::bincode::Decode)]pub enum Response {$($req(Box<<$req as sonic::service::Message<$service>>::Response>),)*}$(impl sonic::service::Wrapper<$service> for $req {- fn wrap_request_ref(req: &Self) -> RequestRef {- RequestRef::$req(req)+ fn wrap_request(req: Self) -> Request {+ Request::$req(Box::new(req))}fn unwrap_response(res: <$service as sonic::service::Service>::Response) -> Option<Self::Response> {#[allow(irrefutable_let_patterns)]@@ -214,7 +215,6 @@ macro_rules! sonic_service {)*impl sonic::service::Service for $service {type Request = Request;- type RequestRef<'a> = RequestRef<'a>;type Response = Response;// NOTE: This is a workaround for the fact that async functions@@ -248,6 +248,8 @@ mod tests {use std::{marker::PhantomData, net::SocketAddr, sync::atomic::AtomicI32};+ use crate::distributed::sonic::{service, ConnectionPool};+use super::{Server, Service, Wrapper};use futures::Future;@@ -257,11 +259,16 @@ mod tests {}impl<S: Service> ConnectionBuilder<S> {- async fn send<R: Wrapper<S>>(&self, req: &R) -> Result<R::Response, anyhow::Error> {- Ok(super::Connection::create(self.addr)- .await?- .send(req)- .await?)+ async fn conn(&self) -> Result<super::Connection<S>, anyhow::Error> {+ Ok(super::Connection::create(self.addr).await?)+ }++ async fn send<R: Wrapper<S>>(&self, req: R) -> Result<R::Response, anyhow::Error> {+ Ok(self.conn().await?.send(req).await?)+ }++ fn addr(&self) -> SocketAddr {+ self.addr}}@@ -365,20 +372,20 @@ mod tests {},|b| async move {let val = b- .send(&Change { amount: 15 })+ .send(Change { amount: 15 }).await.map_err(|e| TestCaseError::Fail(e.to_string().into()))?;assert_eq!(val, 15);let val = b- .send(&Change { amount: 15 })+ .send(Change { amount: 15 }).await.map_err(|e| TestCaseError::Fail(e.to_string().into()))?;assert_eq!(val, 30);- b.send(&Reset)+ b.send(Reset).await.map_err(|e| TestCaseError::Fail(e.to_string().into()))?;let val = b- .send(&Change { amount: 15 })+ .send(Change { amount: 15 }).await.map_err(|e| TestCaseError::Fail(e.to_string().into()))?;assert_eq!(val, 15);@@ -389,12 +396,84 @@ mod tests {Ok(())}+ #[test]+ fn test_connection_reuse() {+ fixture(+ CounterService {+ counter: AtomicI32::new(0),+ },+ |b| async move {+ let mut conn = b.conn().await.unwrap();++ let val = conn+ .send(Change { amount: 15 })+ .await+ .map_err(|e| TestCaseError::Fail(e.to_string().into()))?;+ assert_eq!(val, 15);++ let val = conn+ .send(Change { amount: 15 })+ .await+ .map_err(|e| TestCaseError::Fail(e.to_string().into()))?;+ assert_eq!(val, 30);++ conn.send(Reset)+ .await+ .map_err(|e| TestCaseError::Fail(e.to_string().into()))?;++ // send in a new connection+ let val = b+ .send(Change { amount: 15 })+ .await+ .map_err(|e| TestCaseError::Fail(e.to_string().into()))?;+ assert_eq!(val, 15);++ Ok(())+ },+ )+ .unwrap();+ }++ #[test]+ fn test_connection_pool() {+ fixture(+ CounterService {+ counter: AtomicI32::new(0),+ },+ |b| async move {+ let pool: ConnectionPool<service::Connection<CounterService>> =+ ConnectionPool::new(b.addr()).unwrap();++ let val = pool+ .get()+ .await+ .unwrap()+ .send(Change { amount: 15 })+ .await+ .map_err(|e| TestCaseError::Fail(e.to_string().into()))?;+ assert_eq!(val, 15);++ let val = pool+ .get()+ .await+ .unwrap()+ .send(Change { amount: 15 })+ .await+ .map_err(|e| TestCaseError::Fail(e.to_string().into()))?;+ assert_eq!(val, 30);++ Ok(())+ },+ )+ .unwrap();+ }+proptest! {#[test]fn ref_serialization(a: Change) {fixture(CounterService { counter: AtomicI32::new(0) }, |conn| async move {- conn.send(&Reset).await.map_err(|e| TestCaseError::Fail(e.to_string().into()))?;- let val = conn.send(&a).await.map_err(|e| TestCaseError::Fail(e.to_string().into()))?;+ conn.send(Reset).await.map_err(|e| TestCaseError::Fail(e.to_string().into()))?;+ let val = conn.send(a.clone()).await.map_err(|e| TestCaseError::Fail(e.to_string().into()))?;prop_assert_eq!(val, a.amount);Ok(())})?;diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsindex d20558a3..91641771 100644--- a/crates/core/src/entrypoint/indexer/worker.rs+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -150,7 +150,10 @@ impl Webgraph {),IndexingGraphConfig::Remote { gossip } => {let cluster = crate::start_gossip_cluster_thread(gossip.clone(), None);- Self::Remote(RemoteWebgraph::new(cluster, WebgraphGranularity::Page))+ Self::Remote(crate::block_on(RemoteWebgraph::new(+ cluster,+ WebgraphGranularity::Page,+ )))}}}diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rsindex 43548392..22a4a17d 100644--- a/crates/core/src/searcher/distributed.rs+++ b/crates/core/src/searcher/distributed.rs@@ -19,8 +19,9 @@ use crate::{cluster::Cluster,member::{Service, ShardId},sonic::replication::{- AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient, Shard,- ShardIdentifier, ShardedClient, SpecificShardSelector,+ AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient,+ ReusableClientManager, ReusableShardedClient, Shard, ShardIdentifier, ShardedClient,+ SpecificShardSelector,},},entity_index::EntityMatch,@@ -42,10 +43,13 @@ use futures::future::join_all;use itertools::Itertools;use std::future::Future;use thiserror::Error;+use tokio::sync::Mutex;use url::Url;use super::{InitialWebsiteResult, LocalSearcher, SearchQuery};+const CLIENT_REFRESH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);+#[derive(Error, Debug)]pub enum Error {#[error("Failed to get search result")]@@ -104,18 +108,18 @@ pub struct InitialSearchResultShard {pub shard: ShardId,}-pub struct DistributedSearcher {- cluster: Arc<Cluster>,-}+struct SearchClientManager;-impl DistributedSearcher {- pub fn new(cluster: Arc<Cluster>) -> Self {- Self { cluster }- }+impl ReusableClientManager for SearchClientManager {+ const CLIENT_REFRESH_INTERVAL: std::time::Duration = CLIENT_REFRESH_INTERVAL;++ type Service = SearchService;++ type ShardId = ShardId;- async fn client(&self) -> ShardedClient<SearchService, ShardId> {+ async fn new_client(&self, cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {let mut shards = HashMap::new();- for member in self.cluster.members().await {+ for member in cluster.members().await {if let Service::Searcher { host, shard } = member.service {shards.entry(shard).or_insert_with(Vec::new).push(host);}@@ -132,16 +136,57 @@ impl DistributedSearcher {ShardedClient::new(shard_clients)}+}++struct EntitySearchClientManager;++impl ReusableClientManager for EntitySearchClientManager {+ const CLIENT_REFRESH_INTERVAL: std::time::Duration = CLIENT_REFRESH_INTERVAL;++ type Service = entity_search_server::SearchService;+ type ShardId = ();- async fn entity_client(&self) -> ReplicatedClient<entity_search_server::SearchService> {+ async fn new_client(&self, cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {let mut replicas = Vec::new();- for member in self.cluster.members().await {+ for member in cluster.members().await {if let Service::EntitySearcher { host } = member.service {replicas.push(RemoteClient::new(host));}}- ReplicatedClient::new(replicas)+ let rep = ReplicatedClient::new(replicas);++ if !rep.is_empty() {+ ShardedClient::new(vec![Shard::new((), rep)])+ } else {+ ShardedClient::new(vec![])+ }+ }+}++pub struct DistributedSearcher {+ client: Mutex<ReusableShardedClient<SearchClientManager>>,+ entiy_client: Mutex<ReusableShardedClient<EntitySearchClientManager>>,+}++impl DistributedSearcher {+ pub async fn new(cluster: Arc<Cluster>) -> Self {+ Self {+ client: Mutex::new(+ ReusableShardedClient::new(cluster.clone(), SearchClientManager).await,+ ),+ entiy_client: Mutex::new(+ ReusableShardedClient::new(cluster, EntitySearchClientManager).await,+ ),+ }+ }++ async fn conn(&self) -> Arc<ShardedClient<SearchService, ShardId>> {+ self.client.lock().await.conn().await+ }++ async fn entity_conn(&self) -> Arc<ShardedClient<entity_search_server::SearchService, ()>> {+ self.entiy_client.lock().await.conn().await}async fn retrieve_webpages_from_shard(@@ -155,7 +200,7 @@ impl DistributedSearcher {match client.send(- &search_server::RetrieveWebsites {+ search_server::RetrieveWebsites {websites: pointers,query: query.to_string(),},@@ -179,12 +224,12 @@ impl DistributedSearcher {impl SearchClient for DistributedSearcher {async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {- let client = self.client().await;+ let client = self.conn().await;let mut results = Vec::new();if let Ok(res) = client.send(- &search_server::Search {+ search_server::Search {query: query.clone(),},&AllShardsSelector,@@ -222,7 +267,7 @@ impl SearchClient for DistributedSearcher {rankings.insert(*i, pointer.website.clone());}- let client = self.client().await;+ let client = self.conn().await;let mut futures = Vec::new();for (shard, pointers) in pointers {futures.push(self.retrieve_webpages_from_shard(shard, &client, query, pointers));@@ -244,11 +289,11 @@ impl SearchClient for DistributedSearcher {}async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {- let client = self.client().await;+ let client = self.conn().await;let res = client.send(- &search_server::GetWebpage {+ search_server::GetWebpage {url: url.to_string(),},&AllShardsSelector,@@ -270,11 +315,11 @@ impl SearchClient for DistributedSearcher {}async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {- let client = self.client().await;+ let client = self.conn().await;let res = client.send(- &search_server::GetHomepageDescriptions {+ search_server::GetHomepageDescriptions {urls: urls.to_vec(),},&AllShardsSelector,@@ -301,36 +346,40 @@ impl SearchClient for DistributedSearcher {max_height: Option<u64>,max_width: Option<u64>,) -> Result<Option<Image>> {- let client = self.entity_client().await;+ let client = self.entity_conn().await;Ok(client.send(- &entity_search_server::GetEntityImage {+ entity_search_server::GetEntityImage {image_id: image_id.to_string(),max_height,max_width,},+ &AllShardsSelector,&RandomReplicaSelector,).await.map_err(|_| Error::SearchFailed)?.pop()+ .and_then(|(_, mut v)| v.pop()).and_then(|(_, v)| v))}async fn search_entity(&self, query: &str) -> Option<EntityMatch> {- let client = self.entity_client().await;+ let client = self.entity_conn().await;client.send(- &entity_search_server::Search {+ entity_search_server::Search {query: query.to_string(),},+ &AllShardsSelector,&RandomReplicaSelector,).await.ok()?.pop()+ .and_then(|(_, mut v)| v.pop()).and_then(|(_, v)| v)}}diff --git a/crates/core/src/searcher/live.rs b/crates/core/src/searcher/live.rsindex f31552b0..3e1eaa52 100644--- a/crates/core/src/searcher/live.rs+++ b/crates/core/src/searcher/live.rs@@ -92,7 +92,7 @@ impl LiveSearcher {match client.send(- &search_server::RetrieveWebsites {+ search_server::RetrieveWebsites {websites: pointers,query: query.to_string(),},@@ -121,7 +121,7 @@ impl SearchClient for LiveSearcher {if let Ok(res) = client.send(- &search_server::Search {+ search_server::Search {query: query.clone(),},&AllShardsSelector,diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rsindex a363d794..facadb11 100644--- a/crates/core/src/webgraph/remote.rs+++ b/crates/core/src/webgraph/remote.rs@@ -17,6 +17,7 @@use std::sync::Arc;use itertools::Itertools;+use tokio::sync::Mutex;use url::Url;use crate::{@@ -40,22 +41,21 @@ use crate::{use super::{Edge, EdgeLimit, FullEdge, Node, NodeID};-pub struct RemoteWebgraph {+struct WebgraphClientManager {granularity: WebgraphGranularity,- cluster: Arc<Cluster>,}-impl RemoteWebgraph {- pub fn new(cluster: Arc<Cluster>, granularity: WebgraphGranularity) -> Self {- Self {- cluster,- granularity,- }- }+impl sonic::replication::ReusableClientManager for WebgraphClientManager {+ const CLIENT_REFRESH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);- async fn conn(&self) -> sonic::replication::ShardedClient<WebGraphService, ShardId> {- let shards = self- .cluster+ type Service = WebGraphService;+ type ShardId = ShardId;++ async fn new_client(+ &self,+ cluster: &Cluster,+ ) -> sonic::replication::ShardedClient<Self::Service, Self::ShardId> {+ let shards = cluster.members().await.into_iter()@@ -87,6 +87,26 @@ impl RemoteWebgraph {sonic::replication::ShardedClient::new(shards)}+}++pub struct RemoteWebgraph {+ client: Mutex<sonic::replication::ReusableShardedClient<WebgraphClientManager>>,+}++impl RemoteWebgraph {+ pub async fn new(cluster: Arc<Cluster>, granularity: WebgraphGranularity) -> Self {+ let manager = WebgraphClientManager { granularity };++ Self {+ client: Mutex::new(+ sonic::replication::ReusableShardedClient::new(cluster, manager).await,+ ),+ }+ }++ async fn conn(&self) -> Arc<sonic::replication::ShardedClient<WebGraphService, ShardId>> {+ self.client.lock().await.conn().await+ }pub async fn knows(&self, mut host: String) -> Result<Option<Node>> {if let Some(suf) = host.strip_prefix("http://") {@@ -113,7 +133,7 @@ impl RemoteWebgraph {.conn().await.send(- &GetNode { node: id },+ GetNode { node: id },&AllShardsSelector,&RandomReplicaSelector,)@@ -158,7 +178,7 @@ impl RemoteWebgraph {.conn().await.send(- &IngoingEdges { node, limit },+ IngoingEdges { node, limit },&AllShardsSelector,&RandomReplicaSelector,)@@ -178,7 +198,7 @@ impl RemoteWebgraph {.conn().await.send(- &RawIngoingEdges { node: id, limit },+ RawIngoingEdges { node: id, limit },&AllShardsSelector,&RandomReplicaSelector,)@@ -202,7 +222,7 @@ impl RemoteWebgraph {.conn().await.send(- &RawIngoingEdgesWithLabels { node: id, limit },+ RawIngoingEdgesWithLabels { node: id, limit },&AllShardsSelector,&RandomReplicaSelector,)@@ -284,7 +304,7 @@ impl RemoteWebgraph {.conn().await.send(- &OutgoingEdges { node, limit },+ OutgoingEdges { node, limit },&AllShardsSelector,&RandomReplicaSelector,)@@ -304,7 +324,7 @@ impl RemoteWebgraph {.conn().await.send(- &RawOutgoingEdges { node: id, limit },+ RawOutgoingEdges { node: id, limit },&AllShardsSelector,&RandomReplicaSelector,)@@ -328,7 +348,7 @@ impl RemoteWebgraph {.conn().await.send(- &RawOutgoingEdgesWithLabels { node: id, limit },+ RawOutgoingEdgesWithLabels { node: id, limit },&AllShardsSelector,&RandomReplicaSelector,)