Skip to content

web search done right

Distributed in-memory key/value store for mapreduce (#181)

Mikkel Denker37b6c7d86c34
diff --git a/Cargo.lock b/Cargo.lock
index 3ba9853e..35d4a147 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -168,6 +168,15 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "anyerror"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bcd04a72664a65fb9adeae7ced0c98efd68a2b7a45adda8319b3bb36538404b8"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "anyhow"
version = "1.0.80"
@@ -531,6 +540,16 @@ version = "3.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b"
+[[package]]
+name = "byte-unit"
+version = "4.0.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da78b32057b8fdfc352504708feeba7216dcd65a2c9ab02978cbd288d1279b6c"
+dependencies = [
+ "serde",
+ "utf8-width",
+]
+
[[package]]
name = "bytecount"
version = "0.6.7"
@@ -2997,6 +3016,39 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
+[[package]]
+name = "openraft"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "863a31dad5014893c47cdb4291c1f14cd901304b548c739004307fdd2cd37454"
+dependencies = [
+ "anyerror",
+ "byte-unit",
+ "clap",
+ "derive_more",
+ "futures",
+ "maplit",
+ "openraft-macros",
+ "rand 0.8.5",
+ "serde",
+ "thiserror",
+ "tokio",
+ "tracing",
+ "tracing-futures",
+ "validit",
+]
+
+[[package]]
+name = "openraft-macros"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c5f384d67b15e283f1cf6bb778c8f29ab2f8e8729b4bc82e8a1c21f2bf767d60"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.52",
+]
+
[[package]]
name = "openssl"
version = "0.10.64"
@@ -4511,6 +4563,7 @@ dependencies = [
"min-max-heap",
"num_cpus",
"once_cell",
+ "openraft",
"optics",
"proptest",
"proptest-derive",
@@ -4542,6 +4595,7 @@ dependencies = [
"tower-http",
"tracing",
"tracing-subscriber",
+ "tracing-test",
"url",
"urlencoding",
"utoipa",
@@ -5232,6 +5286,16 @@ dependencies = [
"valuable",
]
+[[package]]
+name = "tracing-futures"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
+dependencies = [
+ "pin-project",
+ "tracing",
+]
+
[[package]]
name = "tracing-log"
version = "0.2.0"
@@ -5261,6 +5325,29 @@ dependencies = [
"tracing-log",
]
+[[package]]
+name = "tracing-test"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a2c0ff408fe918a94c428a3f2ad04e4afd5c95bbc08fcf868eff750c15728a4"
+dependencies = [
+ "lazy_static",
+ "tracing-core",
+ "tracing-subscriber",
+ "tracing-test-macro",
+]
+
+[[package]]
+name = "tracing-test-macro"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "258bc1c4f8e2e73a977812ab339d503e6feeb92700f6d07a6de4d321522d5c08"
+dependencies = [
+ "lazy_static",
+ "quote",
+ "syn 1.0.109",
+]
+
[[package]]
name = "try-lock"
version = "0.2.5"
@@ -5398,6 +5485,12 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba"
+[[package]]
+name = "utf8-width"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3"
+
[[package]]
name = "utf8parse"
version = "0.2.1"
@@ -5455,6 +5548,15 @@ dependencies = [
"serde",
]
+[[package]]
+name = "validit"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1fad49f3eae9c160c06b4d49700a99e75817f127cf856e494b56d5e23170020"
+dependencies = [
+ "anyerror",
+]
+
[[package]]
name = "valuable"
version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 284a21cd..94d7587c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,10 +1,5 @@
[workspace]
-members = [
- "crates/core",
- "crates/optics",
- "crates/kuchiki",
- "crates/zimba",
-]
+members = ["crates/core", "crates/optics", "crates/kuchiki", "crates/zimba"]
resolver = "2"
[profile.release]
@@ -16,51 +11,52 @@ debug = true
[workspace.dependencies]
aes-gcm = "0.10.2"
-anyhow = {version = "1.0.72", features = ["backtrace"]}
+anyhow = { version = "1.0.72", features = ["backtrace"] }
async-channel = "1.8.0"
async-stream = "0.3.3"
axum = "0.7.2"
-axum-extra = {version = "0.9.0"}
+axum-extra = { version = "0.9.0" }
axum-macros = "0.4.0"
base64 = "0.21.4"
bincode = "1.3.3"
bindgen = "0.69.2"
bitvec = "1.0.1"
bytecount = "0.6.7"
-bytemuck = {version = "1.13.1", features = ["derive"]}
+bytemuck = { version = "1.13.1", features = ["derive"] }
byteorder = "1.4.3"
bzip2 = "0.4.4"
candle-core = "0.3.3"
candle-nn = "0.3.3"
+openraft = { version = "0.9.1", features = ["storage-v2", "serde"] }
candle-transformers = "0.3.3"
-cc = {version = "1", features = ["parallel"]}
+cc = { version = "1", features = ["parallel"] }
chardetng = "0.1.17"
chitchat = "0.5.0"
-chrono = {version = "0.4.23", features = ["serde"]}
-clap = {version = "4.4.6", features = ["derive"]}
+chrono = { version = "0.4.23", features = ["serde"] }
+clap = { version = "4.4.6", features = ["derive"] }
cmake = "0.1"
criterion = "0.5.1"
crossbeam-channel = "0.5.6"
csv = "1.1.6"
-dashmap = {version = "5.4.0", features = ["rayon"]}
+dashmap = { version = "5.4.0", features = ["rayon"] }
encoding_rs = "0.8.31"
eventsource-stream = "0.2.3"
fend-core = "1.2.2"
flate2 = "1.0.28"
fnv = "1.0.3"
-fst = {version = "0.4.7", features = ["levenshtein"]}
+fst = { version = "0.4.7", features = ["levenshtein"] }
futures = "0.3.21"
-half = {version = "2.2.1", features = ["serde"]}
-hashbrown = {version = "0.14.0", features = ["serde"]}
+half = { version = "2.2.1", features = ["serde"] }
+hashbrown = { version = "0.14.0", features = ["serde"] }
http = "1.0.0"
image = "0.24.3"
-indicatif = {version = "0.17.7", features = ["rayon"]}
+indicatif = { version = "0.17.7", features = ["rayon"] }
insta = "1.31"
itertools = "0.11.0"
-lalrpop = {version = "0.20.0", features = ["lexer"]}
-lalrpop-util = {version = "0.20.0", features = ["lexer"]}
+lalrpop = { version = "0.20.0", features = ["lexer"] }
+lalrpop-util = { version = "0.20.0", features = ["lexer"] }
libc = "0.2.142"
-log = {version = "0.4", features = ["release_max_level_info"]}
+log = { version = "0.4", features = ["release_max_level_info"] }
logos = "0.13.0"
lz-str = "0.2.1"
lz4_flex = "0.11.1"
@@ -80,41 +76,42 @@ quick-xml = "0.30.0"
rand = "0.8.5"
rayon = "1.5.3"
regex = "1.6.0"
-reqwest = {version = "0.11.16", features = ["blocking", "stream", "json"]}
+reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }
ring = "0.17.3"
rio_api = "0.8.4"
rio_turtle = "0.8.4"
robotstxt-with-cache = "0.4.0"
-rocksdb = {version = "0.21.0", features = ["default", "jemalloc", "io-uring"]}
-rusqlite = {version = "0.29.0", features = [
- "bundled",
- "modern-full",
- "buildtime_bindgen",
-]}
-rust-s3 = {version = "0.33.0", features = ["blocking", "tokio"]}
+rocksdb = { version = "0.21.0", features = ["default", "jemalloc", "io-uring"] }
+rusqlite = { version = "0.29.0", features = [
+ "bundled",
+ "modern-full",
+ "buildtime_bindgen",
+] }
+rust-s3 = { version = "0.33.0", features = ["blocking", "tokio"] }
rust-stemmers = "1.2.0"
safetensors = "0.3.1"
-scylla = {version = "0.12.0", features = ["chrono"]}
-serde = {version = "1.0.137", features = ["rc", "derive"]}
+scylla = { version = "0.12.0", features = ["chrono"] }
+serde = { version = "1.0.137", features = ["rc", "derive"] }
serde_json = "1.0.81"
serde_urlencoded = "0.7.1"
-tantivy = {git = "https://github.com/quickwit-oss/tantivy", rev = "182f58cea"}
+tantivy = { git = "https://github.com/quickwit-oss/tantivy", rev = "182f58cea" }
thiserror = "1.0.31"
tikv-jemallocator = "0.5"
tokenizers = "0.13.2"
-tokio = {version = "1.23.1", features = ["full"]}
+tokio = { version = "1.23.1", features = ["full"] }
tokio-stream = "0.1.11"
toml = "0.8.2"
-tower-http = {version = "0.5.0", features = ["compression-gzip", "cors"]}
-tracing = {version = "0.1.34", features = ["release_max_level_info"]}
-tracing-subscriber = {version = "0.3.11", features = ["env-filter"]}
-url = {version = "2.4.0", features = ["serde"]}
+tower-http = { version = "0.5.0", features = ["compression-gzip", "cors"] }
+tracing = { version = "0.1.34", features = ["release_max_level_info"] }
+tracing-subscriber = { version = "0.3.11", features = ["env-filter"] }
+url = { version = "2.4.0", features = ["serde"] }
urlencoding = "2.1.2"
-utoipa = {version = "4.0.0", features = ["axum_extras"]}
-utoipa-swagger-ui = {version = "5.0.0", features = ["axum"]}
+utoipa = { version = "4.0.0", features = ["axum_extras"] }
+utoipa-swagger-ui = { version = "5.0.0", features = ["axum"] }
uuid = "1.1.2"
-whatlang = {version = "0.16.0", features = ["serde"]}
+whatlang = { version = "0.16.0", features = ["serde"] }
zstd = "0.13"
+tracing-test = "0.2.4"
[profile.test.package]
flate2.opt-level = 3
diff --git a/assets/licenses.html b/assets/licenses.html
index a2e6042f..f9d0aa04 100644
--- a/assets/licenses.html
+++ b/assets/licenses.html
@@ -44,8 +44,8 @@
<h2>Overview of licenses:</h2>
<ul class="licenses-overview">
- <li><a href="#Apache-2.0">Apache License 2.0</a> (390)</li>
- <li><a href="#MIT">MIT License</a> (175)</li>
+ <li><a href="#Apache-2.0">Apache License 2.0</a> (394)</li>
+ <li><a href="#MIT">MIT License</a> (180)</li>
<li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li>
<li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (8)</li>
<li><a href="#Unicode-DFS-2016">Unicode License Agreement - Data Files and Software (2016)</a> (5)</li>
@@ -2595,6 +2595,7 @@
<h3 id="Apache-2.0">Apache License 2.0</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
+ <li><a href=" https://github.com/drmingdrmer/anyerror ">anyerror 0.1.12</a></li>
<li><a href=" https://github.com/epwalsh/rust-cached-path ">cached-path 0.6.1</a></li>
<li><a href=" https://github.com/huggingface/candle ">candle-core 0.3.3</a></li>
<li><a href=" https://github.com/enarx/ciborium ">ciborium-io 0.2.2</a></li>
@@ -7157,6 +7158,7 @@ limitations under the License.</pre>
<li><a href=" https://github.com/unicode-rs/unicode-xid ">unicode-xid 0.2.4</a></li>
<li><a href=" https://github.com/servo/rust-url ">url 2.5.0</a></li>
<li><a href=" https://github.com/uuid-rs/uuid ">uuid 1.7.0</a></li>
+ <li><a href=" https://github.com/drmingdrmer/validit ">validit 0.2.4</a></li>
<li><a href=" https://github.com/SergioBenitez/version_check ">version_check 0.9.4</a></li>
<li><a href=" https://github.com/alexcrichton/wait-timeout ">wait-timeout 0.2.0</a></li>
<li><a href=" https://github.com/bytecodealliance/wasi ">wasi 0.11.0+wasi-snapshot-preview1</a></li>
@@ -9898,6 +9900,8 @@ limitations under the License.
<li><a href=" https://github.com/reem/rust-mac.git ">mac 0.1.1</a></li>
<li><a href=" https://github.com/stainless-steel/md5 ">md5 0.7.0</a></li>
<li><a href=" https://github.com/faern/oneshot ">oneshot 0.1.6</a></li>
+ <li><a href=" https://github.com/datafuselabs/openraft ">openraft-macros 0.9.1</a></li>
+ <li><a href=" https://github.com/datafuselabs/openraft ">openraft 0.9.1</a></li>
<li><a href=" https://github.com/sfackler/rust-openssl ">openssl 0.10.64</a></li>
<li><a href=" https://github.com/oxigraph/rio ">rio_api 0.8.4</a></li>
<li><a href=" https://github.com/oxigraph/rio ">rio_turtle 0.8.4</a></li>
@@ -12510,6 +12514,7 @@ DEALINGS IN THE SOFTWARE.
<li><a href=" https://github.com/tokio-rs/tls ">tokio-native-tls 0.3.1</a></li>
<li><a href=" https://github.com/tokio-rs/tracing ">tracing-attributes 0.1.27</a></li>
<li><a href=" https://github.com/tokio-rs/tracing ">tracing-core 0.1.32</a></li>
+ <li><a href=" https://github.com/tokio-rs/tracing ">tracing-futures 0.2.5</a></li>
<li><a href=" https://github.com/tokio-rs/tracing ">tracing-log 0.2.0</a></li>
<li><a href=" https://github.com/tokio-rs/tracing ">tracing-subscriber 0.3.18</a></li>
<li><a href=" https://github.com/tokio-rs/tracing ">tracing 0.1.40</a></li>
@@ -13024,6 +13029,35 @@ furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+</pre>
+ </li>
+ <li class="license">
+ <h3 id="MIT">MIT License</h3>
+ <h4>Used by:</h4>
+ <ul class="license-used-by">
+ <li><a href=" https://github.com/magiclen/byte-unit ">byte-unit 4.0.19</a></li>
+ </ul>
+ <pre class="license-text">MIT License
+
+Copyright (c) 2018 magiclen.org (Ron Li)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the &quot;Software&quot;), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
@@ -13201,6 +13235,35 @@ furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+</pre>
+ </li>
+ <li class="license">
+ <h3 id="MIT">MIT License</h3>
+ <h4>Used by:</h4>
+ <ul class="license-used-by">
+ <li><a href=" https://github.com/magiclen/utf8-width ">utf8-width 0.1.7</a></li>
+ </ul>
+ <pre class="license-text">MIT License
+
+Copyright (c) 2020 magiclen.org (Ron Li)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the &quot;Software&quot;), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
@@ -13402,6 +13465,8 @@ SOFTWARE.
<li><a href=" https://github.com/plotters-rs/plotters ">plotters 0.3.5</a></li>
<li><a href=" https://github.com/durch/rust-s3 ">rust-s3 0.33.0</a></li>
<li><a href=" https://github.com/tokio-rs/tokio ">tokio-macros 2.2.0</a></li>
+ <li><a href=" https://github.com/dbrgn/tracing-test ">tracing-test-macro 0.2.4</a></li>
+ <li><a href=" https://github.com/dbrgn/tracing-test ">tracing-test 0.2.4</a></li>
<li><a href=" https://github.com/tokio-rs/valuable ">valuable 0.1.0</a></li>
<li><a href=" https://github.com/greyblake/whatlang-rs ">whatlang 0.16.4</a></li>
</ul>
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 8e65aaa6..5c15455f 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -20,101 +20,103 @@ name = "stract"
path = "src/main.rs"
[dependencies]
-aes-gcm = {workspace = true}
-anyhow = {workspace = true}
-async-channel = {workspace = true}
-async-stream = {workspace = true}
-axum = {workspace = true}
-axum-extra = {workspace = true}
-axum-macros = {workspace = true}
-base64 = {workspace = true}
-bincode = {workspace = true}
-bitvec = {workspace = true}
-bytecount = {workspace = true}
-bytemuck = {workspace = true}
-byteorder = {workspace = true}
-bzip2 = {workspace = true}
-candle-core = {workspace = true}
-candle-nn = {workspace = true}
-candle-transformers = {workspace = true}
-chardetng = {workspace = true}
-chitchat = {workspace = true}
-chrono = {workspace = true}
-clap = {workspace = true}
-crossbeam-channel = {workspace = true}
-csv = {workspace = true}
-dashmap = {workspace = true}
-encoding_rs = {workspace = true}
-eventsource-stream = {workspace = true}
-fend-core = {workspace = true}
-flate2 = {workspace = true}
-fnv = {workspace = true}
-fst = {workspace = true}
-futures = {workspace = true}
-half = {workspace = true}
-hashbrown = {workspace = true}
-http = {workspace = true}
-image = {workspace = true}
-indicatif = {workspace = true}
-itertools = {workspace = true}
-kuchiki = {path = "../kuchiki"}
-libc = {workspace = true}
-log = {workspace = true}
-logos = {workspace = true}
-lz-str = {workspace = true}
-lz4_flex = {workspace = true}
-md5 = {workspace = true}
-memmap = {workspace = true}
-memmap2 = {workspace = true}
-mime = {workspace = true}
-min-max-heap = {workspace = true}
-num_cpus = {workspace = true}
-once_cell = {workspace = true}
-optics = {path = "../optics"}
-publicsuffix = {workspace = true}
-quick-xml = {workspace = true}
-rand = {workspace = true}
-rayon = {workspace = true}
-regex = {workspace = true}
-reqwest = {workspace = true}
-ring = {workspace = true}
-rio_api = {workspace = true}
-rio_turtle = {workspace = true}
-robotstxt-with-cache = {workspace = true}
-rocksdb = {workspace = true}
-rust-s3 = {workspace = true}
-rust-stemmers = {workspace = true}
-safetensors = {workspace = true}
-scylla = {workspace = true}
-serde = {workspace = true}
-serde_json = {workspace = true}
-serde_urlencoded = {workspace = true}
-tantivy = {workspace = true}
-thiserror = {workspace = true}
-tokenizers = {workspace = true}
-tokio = {workspace = true}
-tokio-stream = {workspace = true}
-toml = {workspace = true}
-tower-http = {workspace = true}
-tracing = {workspace = true}
-tracing-subscriber = {workspace = true}
-url = {workspace = true}
-urlencoding = {workspace = true}
-utoipa = {workspace = true}
-utoipa-swagger-ui = {workspace = true}
-uuid = {workspace = true}
-whatlang = {workspace = true}
-zimba = {path = "../zimba"}
+aes-gcm = { workspace = true }
+anyhow = { workspace = true }
+async-channel = { workspace = true }
+async-stream = { workspace = true }
+axum = { workspace = true }
+axum-extra = { workspace = true }
+axum-macros = { workspace = true }
+base64 = { workspace = true }
+bincode = { workspace = true }
+bitvec = { workspace = true }
+bytecount = { workspace = true }
+bytemuck = { workspace = true }
+byteorder = { workspace = true }
+bzip2 = { workspace = true }
+candle-core = { workspace = true }
+candle-nn = { workspace = true }
+candle-transformers = { workspace = true }
+chardetng = { workspace = true }
+chitchat = { workspace = true }
+chrono = { workspace = true }
+clap = { workspace = true }
+crossbeam-channel = { workspace = true }
+csv = { workspace = true }
+dashmap = { workspace = true }
+encoding_rs = { workspace = true }
+eventsource-stream = { workspace = true }
+fend-core = { workspace = true }
+flate2 = { workspace = true }
+fnv = { workspace = true }
+fst = { workspace = true }
+futures = { workspace = true }
+half = { workspace = true }
+hashbrown = { workspace = true }
+http = { workspace = true }
+image = { workspace = true }
+indicatif = { workspace = true }
+itertools = { workspace = true }
+kuchiki = { path = "../kuchiki" }
+libc = { workspace = true }
+log = { workspace = true }
+logos = { workspace = true }
+lz-str = { workspace = true }
+lz4_flex = { workspace = true }
+md5 = { workspace = true }
+memmap = { workspace = true }
+memmap2 = { workspace = true }
+mime = { workspace = true }
+min-max-heap = { workspace = true }
+num_cpus = { workspace = true }
+once_cell = { workspace = true }
+openraft = { workspace = true }
+optics = { path = "../optics" }
+publicsuffix = { workspace = true }
+quick-xml = { workspace = true }
+rand = { workspace = true }
+rayon = { workspace = true }
+regex = { workspace = true }
+reqwest = { workspace = true }
+ring = { workspace = true }
+rio_api = { workspace = true }
+rio_turtle = { workspace = true }
+robotstxt-with-cache = { workspace = true }
+rocksdb = { workspace = true }
+rust-s3 = { workspace = true }
+rust-stemmers = { workspace = true }
+safetensors = { workspace = true }
+scylla = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+serde_urlencoded = { workspace = true }
+tantivy = { workspace = true }
+thiserror = { workspace = true }
+tokenizers = { workspace = true }
+tokio = { workspace = true }
+tokio-stream = { workspace = true }
+toml = { workspace = true }
+tower-http = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
+url = { workspace = true }
+urlencoding = { workspace = true }
+utoipa = { workspace = true }
+utoipa-swagger-ui = { workspace = true }
+uuid = { workspace = true }
+whatlang = { workspace = true }
+zimba = { path = "../zimba" }
[target.'cfg(not(target_env = "msvc"))'.dependencies]
-tikv-jemallocator = {workspace = true}
+tikv-jemallocator = { workspace = true }
[dev-dependencies]
-criterion = {workspace = true}
-insta = {workspace = true}
-maplit = {workspace = true}
-proptest = {workspace = true}
-proptest-derive = {workspace = true}
+criterion = { workspace = true }
+insta = { workspace = true }
+maplit = { workspace = true }
+proptest = { workspace = true }
+proptest-derive = { workspace = true }
+tracing-test = { workspace = true }
[[bench]]
harness = false
diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rs
index 1c04cb84..04ef9fd0 100644
--- a/crates/core/examples/search_preindexed.rs
+++ b/crates/core/examples/search_preindexed.rs
@@ -6,11 +6,12 @@ use stract::{
ApiConfig, ApiThresholds, CollectorConfig, CorrectionConfig, LLMConfig, SnippetConfig,
WidgetsConfig,
},
+ distributed::member::ShardId,
image_store::Image,
index::Index,
inverted_index::RetrievedWebpage,
ranking::{inbound_similarity::InboundSimilarity, pipeline::PrecisionRankingWebpage},
- searcher::{api::ApiSearcher, live::LiveSearcher, LocalSearcher, SearchQuery, ShardId},
+ searcher::{api::ApiSearcher, live::LiveSearcher, LocalSearcher, SearchQuery},
Result,
};
struct Searcher(LocalSearcher<Index>);
diff --git a/crates/core/src/api/hosts.rs b/crates/core/src/api/hosts.rs
index 4e53d9df..363fd8a7 100644
--- a/crates/core/src/api/hosts.rs
+++ b/crates/core/src/api/hosts.rs
@@ -14,7 +14,11 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use axum::{body::Body, extract, response::{IntoResponse, Response}};
+use axum::{
+ body::Body,
+ extract,
+ response::{IntoResponse, Response},
+};
use http::StatusCode;
use optics::{HostRankings, Optic};
use utoipa::ToSchema;
diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rs
index 771b7ac1..9cddf89a 100644
--- a/crates/core/src/api/webgraph.rs
+++ b/crates/core/src/api/webgraph.rs
@@ -95,7 +95,7 @@ pub mod host {
.with_limit(Duration::from_millis(200))
.take(5);
- let conn = sonic::service::ResilientConnection::create_with_timeout(
+ let conn = sonic::service::Connection::create_with_timeout_retry(
host,
Duration::from_secs(30),
retry,
@@ -142,7 +142,7 @@ pub mod host {
.with_limit(Duration::from_millis(200))
.take(5);
- let conn = sonic::service::ResilientConnection::create_with_timeout(
+ let conn = sonic::service::Connection::create_with_timeout_retry(
host,
Duration::from_secs(30),
retry,
@@ -286,12 +286,9 @@ async fn ingoing_links(
.with_limit(Duration::from_millis(200))
.take(5);
- let conn = sonic::service::ResilientConnection::create_with_timeout(
- host,
- Duration::from_secs(30),
- retry,
- )
- .await?;
+ let conn =
+ sonic::service::Connection::create_with_timeout_retry(host, Duration::from_secs(30), retry)
+ .await?;
Ok(conn
.send_with_timeout(
@@ -318,12 +315,9 @@ async fn outgoing_links(
.with_limit(Duration::from_millis(200))
.take(5);
- let conn = sonic::service::ResilientConnection::create_with_timeout(
- host,
- Duration::from_secs(30),
- retry,
- )
- .await?;
+ let conn =
+ sonic::service::Connection::create_with_timeout_retry(host, Duration::from_secs(30), retry)
+ .await?;
Ok(conn
.send_with_timeout(
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index cafecd78..4a246cf3 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -17,8 +17,8 @@
pub mod defaults;
use super::Result;
+use crate::distributed::member::ShardId;
use crate::feed::scheduler::SplitId;
-use crate::searcher::ShardId;
use serde::{Deserialize, Serialize};
use std::fs::File;
use std::io::{self, BufRead};
diff --git a/crates/core/src/crawler/router.rs b/crates/core/src/crawler/router.rs
index fa4fcec2..a0a14c8d 100644
--- a/crates/core/src/crawler/router.rs
+++ b/crates/core/src/crawler/router.rs
@@ -15,10 +15,10 @@ struct RemoteCoordinator {
}
impl RemoteCoordinator {
- async fn conn(&self) -> Result<sonic::service::ResilientConnection<CoordinatorService>> {
+ async fn conn(&self) -> Result<sonic::service::Connection<CoordinatorService>> {
let retry = ExponentialBackoff::from_millis(1_000).with_limit(Duration::from_secs(10));
- Ok(sonic::service::ResilientConnection::create_with_timeout(
+ Ok(sonic::service::Connection::create_with_timeout_retry(
self.addr,
Duration::from_secs(60),
retry,
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rs
index 089d851b..d96012a0 100644
--- a/crates/core/src/crawler/worker.rs
+++ b/crates/core/src/crawler/worker.rs
@@ -76,12 +76,12 @@ impl WorkerThread {
})
}
- async fn router_conn(&self) -> Result<sonic::service::ResilientConnection<RouterService>> {
+ async fn router_conn(&self) -> Result<sonic::service::Connection<RouterService>> {
let retry = ExponentialBackoff::from_millis(1_000).with_limit(Duration::from_secs(10));
let router = *self.router_hosts.choose(&mut rand::thread_rng()).unwrap();
- Ok(sonic::service::ResilientConnection::create_with_timeout(
+ Ok(sonic::service::Connection::create_with_timeout_retry(
router,
Duration::from_secs(90),
retry,
diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rs
index 0bb9065a..ce1a32d9 100644
--- a/crates/core/src/distributed/member.rs
+++ b/crates/core/src/distributed/member.rs
@@ -18,7 +18,22 @@ use std::net::SocketAddr;
use serde::{Deserialize, Serialize};
-use crate::{config::WebgraphGranularity, searcher::ShardId};
+use crate::config::WebgraphGranularity;
+
+#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy, Debug, PartialOrd, Ord)]
+pub struct ShardId(u64);
+
+impl ShardId {
+ pub fn new(id: u64) -> Self {
+ Self(id)
+ }
+}
+
+impl From<u64> for ShardId {
+ fn from(id: u64) -> Self {
+ Self(id)
+ }
+}
#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]
pub enum Service {
@@ -40,6 +55,10 @@ pub enum Service {
host: SocketAddr,
granularity: WebgraphGranularity,
},
+ Dht {
+ host: SocketAddr,
+ shard: ShardId,
+ },
}
#[derive(PartialEq, Eq, Hash, Clone, Debug)]
diff --git a/crates/core/src/distributed/retry_strategy.rs b/crates/core/src/distributed/retry_strategy.rs
index 04d9ea9e..ccc75a67 100644
--- a/crates/core/src/distributed/retry_strategy.rs
+++ b/crates/core/src/distributed/retry_strategy.rs
@@ -1,7 +1,8 @@
/*
- * This file is copied from [tokio_retry](https://github.com/srijs/rust-tokio-retry/blob/master/src/strategy/exponential_backoff.rs)
- * as it seems silly to pull-in a new dependency for a single file.
+ * This file is partially copied from [tokio_retry](https://github.com/srijs/rust-tokio-retry/blob/master/src/strategy/exponential_backoff.rs)
+ * and modified as it seems silly to pull-in a new dependency for a single file.
* */
+use rand::Rng;
use std::iter::Iterator;
use std::time::Duration;
use std::u64::MAX as U64_MAX;
@@ -66,6 +67,28 @@ impl Iterator for ExponentialBackoff {
}
}
+pub struct RandomBackoff {
+ min: Duration,
+ max: Duration,
+}
+
+impl RandomBackoff {
+ pub fn new(min: Duration, max: Duration) -> RandomBackoff {
+ RandomBackoff { min, max }
+ }
+}
+
+impl Iterator for RandomBackoff {
+ type Item = Duration;
+
+ fn next(&mut self) -> Option<Duration> {
+ let mut rng = rand::thread_rng();
+ let range = self.max - self.min;
+ let duration = rng.gen_range(0..range.as_millis()) + self.min.as_millis();
+ Some(Duration::from_millis(duration as u64))
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/core/src/distributed/sonic/mod.rs b/crates/core/src/distributed/sonic/mod.rs
index d6b6f382..203d5e4f 100644
--- a/crates/core/src/distributed/sonic/mod.rs
+++ b/crates/core/src/distributed/sonic/mod.rs
@@ -49,8 +49,11 @@ pub enum Error {
#[error("The request could not be processed")]
BadRequest,
- #[error("Other")]
- Other(#[from] anyhow::Error),
+ #[error("The body size ({body_size}) is larger than the maximum allowed ({max_size})")]
+ BodyTooLarge { body_size: usize, max_size: usize },
+
+ #[error("An application error occurred: {0}")]
+ Application(#[from] anyhow::Error),
}
pub struct Connection<Req, Res> {
@@ -83,6 +86,29 @@ where
}
}
+ pub async fn create_with_timeout_retry(
+ server: impl ToSocketAddrs + Clone,
+ timeout: Duration,
+ retry: impl Iterator<Item = Duration>,
+ ) -> Result<Self> {
+ let mut conn = Connection::create_with_timeout(server.clone(), timeout).await;
+ let mut retry = retry;
+
+ loop {
+ match conn {
+ Ok(conn) => return Ok(conn),
+ Err(_) => {
+ if let Some(timeout) = retry.next() {
+ tokio::time::sleep(timeout).await;
+ conn = Connection::create_with_timeout(server.clone(), timeout).await;
+ } else {
+ return Err(Error::ConnectionTimeout);
+ }
+ }
+ }
+ }
+ }
+
async fn send_without_timeout(mut self, request: &Req) -> Result<Res> {
let bytes = bincode::serialize(&request).unwrap();
@@ -104,11 +130,10 @@ where
let header: Header = *bytemuck::from_bytes(&header_buf);
if header.body_size > MAX_BODY_SIZE_BYTES {
- return Err(Error::Other(anyhow::anyhow!(
- "body size too large: {} (max: {})",
- header.body_size,
- MAX_BODY_SIZE_BYTES
- )));
+ return Err(Error::BodyTooLarge {
+ body_size: header.body_size,
+ max_size: MAX_BODY_SIZE_BYTES,
+ });
}
let mut buf = vec![0; header.body_size];
@@ -140,7 +165,7 @@ struct Header {
}
pub struct Server<Req, Res> {
- pub(super) listener: TcpListener,
+ listener: TcpListener,
marker: PhantomData<(Req, Res)>,
}
@@ -162,11 +187,10 @@ where
let header: Header = *bytemuck::from_bytes(&header_buf);
if header.body_size > MAX_BODY_SIZE_BYTES {
- return Err(Error::Other(anyhow::anyhow!(
- "body size too large: {} (max: {})",
- header.body_size,
- MAX_BODY_SIZE_BYTES
- )));
+ return Err(Error::BodyTooLarge {
+ body_size: header.body_size,
+ max_size: MAX_BODY_SIZE_BYTES,
+ });
}
let mut buf = vec![0; header.body_size];
@@ -239,43 +263,6 @@ where
}
}
-pub struct ResilientConnection<Req, Res> {
- conn: Connection<Req, Res>,
-}
-
-impl<Req, Res> ResilientConnection<Req, Res>
-where
- Req: Serialize,
- Res: DeserializeOwned,
-{
- pub async fn create_with_timeout(
- server: impl ToSocketAddrs + Clone,
- timeout: Duration,
- retry: impl Iterator<Item = Duration>,
- ) -> Result<Self> {
- let mut conn = Connection::create_with_timeout(server.clone(), timeout).await;
- let mut retry = retry;
-
- loop {
- match conn {
- Ok(conn) => return Ok(ResilientConnection { conn }),
- Err(_) => {
- if let Some(timeout) = retry.next() {
- tokio::time::sleep(timeout).await;
- conn = Connection::create_with_timeout(server.clone(), timeout).await;
- } else {
- return Err(Error::ConnectionTimeout);
- }
- }
- }
- }
- }
-
- pub async fn send_with_timeout(self, request: &Req, timeout: Duration) -> Result<Res> {
- self.conn.send_with_timeout(request, timeout).await
- }
-}
-
#[cfg(test)]
mod tests {
use std::{collections::HashMap, future::Future};
@@ -284,6 +271,8 @@ mod tests {
use proptest_derive::Arbitrary;
use serde::Deserialize;
+ use crate::free_socket_addr;
+
use super::*;
fn fixture<
@@ -302,8 +291,8 @@ mod tests {
.build()
.unwrap()
.block_on(async move {
- let server = Server::bind(("127.0.0.1", 0)).await.unwrap();
- let addr = server.listener.local_addr().unwrap();
+ let addr = free_socket_addr();
+ let server = Server::bind(addr.clone()).await.unwrap();
let connection = Connection::create(addr).await.unwrap();
let svr_task = tokio::spawn(async move { svr_fn(server).await });
diff --git a/crates/core/src/distributed/sonic/replication.rs b/crates/core/src/distributed/sonic/replication.rs
index c1f36bbf..0ca6e34d 100644
--- a/crates/core/src/distributed/sonic/replication.rs
+++ b/crates/core/src/distributed/sonic/replication.rs
@@ -21,17 +21,30 @@ use super::Result;
use crate::distributed::{retry_strategy::ExponentialBackoff, sonic};
use std::{net::SocketAddr, time::Duration};
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub struct RemoteClient<S: sonic::service::Service> {
addr: SocketAddr,
_phantom: std::marker::PhantomData<S>,
}
+impl<S> Clone for RemoteClient<S>
+where
+ S: sonic::service::Service,
+{
+ fn clone(&self) -> Self {
+ Self::create(self.addr)
+ }
+}
+
impl<S> RemoteClient<S>
where
S: sonic::service::Service,
{
pub fn new(addr: SocketAddr) -> Self {
+ Self::create(addr)
+ }
+
+ pub fn create(addr: SocketAddr) -> Self {
Self {
addr,
_phantom: std::marker::PhantomData,
@@ -43,12 +56,12 @@ impl<S> RemoteClient<S>
where
S: sonic::service::Service,
{
- async fn conn(&self) -> Result<sonic::service::ResilientConnection<S>> {
+ pub async fn conn(&self) -> Result<sonic::service::Connection<S>> {
let retry = ExponentialBackoff::from_millis(30)
.with_limit(Duration::from_millis(200))
.take(5);
- sonic::service::ResilientConnection::create_with_timeout(
+ sonic::service::Connection::create_with_timeout_retry(
self.addr,
Duration::from_secs(30),
retry,
@@ -56,9 +69,43 @@ where
.await
}
- async fn send<R: sonic::service::Wrapper<S>>(&self, req: &R) -> Result<R::Response> {
+ pub async fn send<R: sonic::service::Wrapper<S>>(&self, req: &R) -> Result<R::Response> {
+ self.send_with_timeout_retry(
+ req,
+ Duration::from_secs(60),
+ ExponentialBackoff::from_millis(500).with_limit(Duration::from_secs(3)),
+ )
+ .await
+ }
+
+ pub async fn send_with_timeout<R: sonic::service::Wrapper<S>>(
+ &self,
+ req: &R,
+ timeout: Duration,
+ ) -> Result<R::Response> {
let conn = self.conn().await?;
- conn.send_with_timeout(req, Duration::from_secs(60)).await
+ conn.send_with_timeout(req, timeout).await
+ }
+
+ pub async fn send_with_timeout_retry<R: sonic::service::Wrapper<S>>(
+ &self,
+ req: &R,
+ timeout: Duration,
+ retry: impl Iterator<Item = Duration>,
+ ) -> Result<R::Response> {
+ let mut er = None;
+ for backoff in retry {
+ match self.send_with_timeout(req, timeout).await {
+ Ok(r) => return Ok(r),
+ Err(e) => {
+ tracing::error!("Failed to send request: {:?}", e);
+ er = Some(e);
+ tokio::time::sleep(backoff).await;
+ }
+ }
+ }
+
+ Err(er.unwrap())
}
}
@@ -101,10 +148,10 @@ where
Self { clients }
}
- pub async fn send<Req, Rep>(&self, req: &Req, selector: &Rep) -> Result<Vec<Req::Response>>
+ pub async fn send<Req, Sel>(&self, req: &Req, selector: &Sel) -> Result<Vec<Req::Response>>
where
Req: sonic::service::Wrapper<S>,
- Rep: ReplicaSelector<S>,
+ Sel: ReplicaSelector<S>,
{
let mut futures = Vec::new();
for client in selector.select(&self.clients) {
@@ -196,15 +243,15 @@ where
Self { shards }
}
- async fn send_single<Req, RSel>(
+ async fn send_single<Req, Sel>(
&self,
req: &Req,
shard: &Shard<S, Id>,
- replica_selector: &RSel,
+ replica_selector: &Sel,
) -> Result<(Id, Vec<Req::Response>)>
where
Req: sonic::service::Wrapper<S>,
- RSel: ReplicaSelector<S>,
+ Sel: ReplicaSelector<S>,
{
Ok((
shard.id.clone(),
diff --git a/crates/core/src/distributed/sonic/service.rs b/crates/core/src/distributed/sonic/service.rs
index 7fe6b646..6efea7f9 100644
--- a/crates/core/src/distributed/sonic/service.rs
+++ b/crates/core/src/distributed/sonic/service.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{net::SocketAddr, sync::Arc, time::Duration};
+use std::{sync::Arc, time::Duration};
use tokio::net::ToSocketAddrs;
@@ -28,12 +28,12 @@ pub trait Service: Sized + Send + Sync + 'static {
fn handle(
req: Self::Request,
server: &Self,
- ) -> impl std::future::Future<Output = Result<Self::Response>> + Send + '_;
+ ) -> impl std::future::Future<Output = Self::Response> + Send + '_;
}
pub trait Message<S: Service> {
type Response;
- fn handle(self, server: &S) -> impl std::future::Future<Output = Result<Self::Response>>;
+ fn handle(self, server: &S) -> impl std::future::Future<Output = Self::Response>;
}
pub trait Wrapper<S: Service>: Message<S> {
fn wrap_request_ref(req: &Self) -> S::RequestRef<'_>;
@@ -57,15 +57,9 @@ impl<S: Service> Server<S> {
let service = Arc::clone(&self.service);
tokio::spawn(async move {
- match S::handle(req.take_body(), &service).await {
- Ok(res) => {
- if let Err(e) = req.respond(res).await {
- tracing::error!("failed to respond to request: {}", e);
- }
- }
- Err(e) => {
- tracing::error!("failed to handle request: {}", e);
- }
+ let res = S::handle(req.take_body(), &service).await;
+ if let Err(e) = req.respond(res).await {
+ tracing::error!("failed to respond to request: {}", e);
}
});
@@ -84,6 +78,7 @@ impl<'a, S: Service> Connection<'a, S> {
inner: super::Connection::create(server).await?,
})
}
+
#[allow(dead_code)]
pub async fn create_with_timeout(
server: impl ToSocketAddrs,
@@ -93,6 +88,17 @@ impl<'a, S: Service> Connection<'a, S> {
inner: super::Connection::create_with_timeout(server, timeout).await?,
})
}
+
+ pub async fn create_with_timeout_retry(
+ server: impl ToSocketAddrs + Clone,
+ timeout: Duration,
+ retry: impl Iterator<Item = Duration>,
+ ) -> Result<Connection<'a, S>> {
+ Ok(Connection {
+ inner: super::Connection::create_with_timeout_retry(server, timeout, retry).await?,
+ })
+ }
+
#[allow(dead_code)]
pub async fn send_without_timeout<R: Wrapper<S>>(self, request: &'a R) -> Result<R::Response> {
Ok(R::unwrap_response(
@@ -102,11 +108,12 @@ impl<'a, S: Service> Connection<'a, S> {
)
.unwrap())
}
+
#[allow(dead_code)]
pub async fn send<R: Wrapper<S>>(self, request: &'a R) -> Result<R::Response> {
Ok(R::unwrap_response(self.inner.send(&R::wrap_request_ref(request)).await?).unwrap())
}
- #[allow(dead_code)]
+
pub async fn send_with_timeout<R: Wrapper<S>>(
self,
request: &'a R,
@@ -121,31 +128,6 @@ impl<'a, S: Service> Connection<'a, S> {
}
}
-pub struct ResilientConnection<'a, S: Service> {
- inner: super::ResilientConnection<S::RequestRef<'a>, S::Response>,
-}
-
-impl<'a, S: Service> ResilientConnection<'a, S> {
- pub async fn create_with_timeout<Rt: Iterator<Item = Duration>>(
- addr: SocketAddr,
- timeout: Duration,
- retry: Rt,
- ) -> Result<ResilientConnection<'a, S>> {
- Ok(Self {
- inner: super::ResilientConnection::create_with_timeout(addr, timeout, retry).await?,
- })
- }
-
- pub async fn send_with_timeout<R: Wrapper<S>>(
- self,
- request: &'a R,
- timeout: Duration,
- ) -> Result<R::Response> {
- let req_ref = R::wrap_request_ref(request);
- Ok(R::unwrap_response(self.inner.send_with_timeout(&req_ref, timeout).await?).unwrap())
- }
-}
-
#[macro_export]
macro_rules! sonic_service {
($service:ident, [$($req:ident),*$(,)?]) => {
@@ -192,11 +174,11 @@ macro_rules! sonic_service {
// don't have a Send bound by default, and there's currently no
// way of specifying that.
#[allow(clippy::manual_async_fn)]
- fn handle(req: Request, server: &Self) -> impl std::future::Future<Output = sonic::Result<Self::Response>> + Send + '_ {
+ fn handle(req: Request, server: &Self) -> impl std::future::Future<Output = Self::Response> + Send + '_ {
async move {
match req {
$(
- Request::$req(value) => Ok(Response::$req(sonic::service::Message::handle(value, server).await?)),
+ Request::$req(value) => Response::$req(sonic::service::Message::handle(value, server).await),
)*
}
}
@@ -280,8 +262,6 @@ mod tests {
use proptest_derive::Arbitrary;
use serde::{Deserialize, Serialize};
- use crate::distributed::sonic;
-
use super::super::Message;
pub struct CounterService {
@@ -300,20 +280,19 @@ mod tests {
impl Message<CounterService> for Change {
type Response = i32;
- async fn handle(self, server: &CounterService) -> sonic::Result<Self::Response> {
+ async fn handle(self, server: &CounterService) -> Self::Response {
let prev = server
.counter
.fetch_add(self.amount, std::sync::atomic::Ordering::SeqCst);
- Ok(prev + self.amount)
+ prev + self.amount
}
}
impl Message<CounterService> for Reset {
type Response = ();
- async fn handle(self, server: &CounterService) -> sonic::Result<Self::Response> {
+ async fn handle(self, server: &CounterService) -> Self::Response {
server.counter.store(0, std::sync::atomic::Ordering::SeqCst);
- Ok(())
}
}
}
diff --git a/crates/core/src/entrypoint/crawler.rs b/crates/core/src/entrypoint/crawler.rs
index d56266b1..2e46552c 100644
--- a/crates/core/src/entrypoint/crawler.rs
+++ b/crates/core/src/entrypoint/crawler.rs
@@ -21,7 +21,7 @@ use serde::{Deserialize, Serialize};
use crate::{
config,
crawler::{self, planner::CrawlPlanner, CrawlCoordinator, Crawler},
- distributed::sonic::{self, service::Message},
+ distributed::sonic::service::Message,
kv::rocksdb_store::RocksDbStore,
sonic_service,
webgraph::WebgraphBuilder,
@@ -104,8 +104,8 @@ pub mod router {
impl Message<RouterService> for NewJob {
type Response = Option<Job>;
- async fn handle(self, server: &RouterService) -> sonic::Result<Self::Response> {
- Ok(server.router.sample_job().await?)
+ async fn handle(self, server: &RouterService) -> Self::Response {
+ server.router.sample_job().await.ok().flatten()
}
}
}
@@ -127,9 +127,8 @@ pub mod coordinator {
impl Message<CoordinatorService> for GetJob {
type Response = Option<Job>;
- async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {
- let job = server.coordinator.sample_job()?;
- Ok(job)
+ async fn handle(self, server: &CoordinatorService) -> Self::Response {
+ server.coordinator.sample_job().ok().flatten()
}
}
}
diff --git a/crates/core/src/entrypoint/entity_search_server.rs b/crates/core/src/entrypoint/entity_search_server.rs
index e1fb1da4..397d8bad 100644
--- a/crates/core/src/entrypoint/entity_search_server.rs
+++ b/crates/core/src/entrypoint/entity_search_server.rs
@@ -65,8 +65,8 @@ pub struct Search {
impl sonic::service::Message<SearchService> for Search {
type Response = Option<crate::entity_index::EntityMatch>;
- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
- Ok(server.index.search(&self.query))
+ async fn handle(self, server: &SearchService) -> Self::Response {
+ server.index.search(&self.query)
}
}
@@ -78,13 +78,13 @@ pub struct GetEntityImage {
}
impl sonic::service::Message<SearchService> for GetEntityImage {
type Response = Option<Image>;
- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
- Ok(server.index.retrieve_image(&self.image_id).map(|img| {
+ async fn handle(self, server: &SearchService) -> Self::Response {
+ server.index.retrieve_image(&self.image_id).map(|img| {
let max_width = self.max_width.unwrap_or(u64::MAX) as u32;
let max_height = self.max_height.unwrap_or(u64::MAX) as u32;
img.resize_max(max_width, max_height)
- }))
+ })
}
}
diff --git a/crates/core/src/entrypoint/live_index.rs b/crates/core/src/entrypoint/live_index.rs
index c758b5c5..c3a567a4 100644
--- a/crates/core/src/entrypoint/live_index.rs
+++ b/crates/core/src/entrypoint/live_index.rs
@@ -81,24 +81,18 @@ impl SearchService {
impl sonic::service::Message<SearchService> for RetrieveWebsites {
type Response = Option<Vec<inverted_index::RetrievedWebpage>>;
- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
- match server
+ async fn handle(self, server: &SearchService) -> Self::Response {
+ server
.local_searcher
.retrieve_websites(&self.websites, &self.query)
- {
- Ok(response) => Ok(Some(response)),
- Err(_) => Ok(None),
- }
+ .ok()
}
}
impl sonic::service::Message<SearchService> for Search {
type Response = Option<InitialWebsiteResult>;
- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
- match server.local_searcher.search_initial(&self.query, true) {
- Ok(result) => Ok(Some(result)),
- Err(_) => Ok(None),
- }
+ async fn handle(self, server: &SearchService) -> Self::Response {
+ server.local_searcher.search_initial(&self.query, true).ok()
}
}
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rs
index 864dc12b..0aba25db 100644
--- a/crates/core/src/entrypoint/search_server.rs
+++ b/crates/core/src/entrypoint/search_server.rs
@@ -110,14 +110,11 @@ pub struct RetrieveWebsites {
}
impl sonic::service::Message<SearchService> for RetrieveWebsites {
type Response = Option<Vec<inverted_index::RetrievedWebpage>>;
- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
- match server
+ async fn handle(self, server: &SearchService) -> Self::Response {
+ server
.local_searcher
.retrieve_websites(&self.websites, &self.query)
- {
- Ok(response) => Ok(Some(response)),
- Err(_) => Ok(None),
- }
+ .ok()
}
}
@@ -127,11 +124,8 @@ pub struct Search {
}
impl sonic::service::Message<SearchService> for Search {
type Response = Option<InitialWebsiteResult>;
- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
- match server.local_searcher.search_initial(&self.query, true) {
- Ok(result) => Ok(Some(result)),
- Err(_) => Ok(None),
- }
+ async fn handle(self, server: &SearchService) -> Self::Response {
+ server.local_searcher.search_initial(&self.query, true).ok()
}
}
@@ -141,8 +135,8 @@ pub struct GetWebpage {
}
impl sonic::service::Message<SearchService> for GetWebpage {
type Response = Option<RetrievedWebpage>;
- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
- Ok(server.local_searcher.get_webpage(&self.url))
+ async fn handle(self, server: &SearchService) -> Self::Response {
+ server.local_searcher.get_webpage(&self.url)
}
}
@@ -152,7 +146,7 @@ pub struct GetHomepageDescriptions {
}
impl sonic::service::Message<SearchService> for GetHomepageDescriptions {
type Response = HashMap<Url, String>;
- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
+ async fn handle(self, server: &SearchService) -> Self::Response {
let mut result = HashMap::with_capacity(self.urls.len());
for url in &self.urls {
@@ -163,7 +157,7 @@ impl sonic::service::Message<SearchService> for GetHomepageDescriptions {
}
}
- Ok(result)
+ result
}
}
diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rs
index 578cb576..d9a17cd4 100644
--- a/crates/core/src/entrypoint/webgraph_server.rs
+++ b/crates/core/src/entrypoint/webgraph_server.rs
@@ -29,7 +29,6 @@ use crate::config::WebgraphGranularity;
use crate::distributed::cluster::Cluster;
use crate::distributed::member::Member;
use crate::distributed::member::Service;
-use crate::distributed::sonic;
use crate::distributed::sonic::service::Message;
use crate::ranking::inbound_similarity::InboundSimilarity;
use crate::searcher::DistributedSearcher;
@@ -79,9 +78,9 @@ pub struct Knows {
impl Message<WebGraphService> for SimilarHosts {
type Response = Vec<ScoredHost>;
- async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {
+ async fn handle(self, server: &WebGraphService) -> Self::Response {
if server.similar_hosts_finder.is_none() {
- return Ok(vec![]);
+ return vec![];
}
let sites = &self.hosts[..std::cmp::min(self.hosts.len(), MAX_HOSTS)];
@@ -113,14 +112,14 @@ impl Message<WebGraphService> for SimilarHosts {
})
.collect_vec();
- Ok(similar_hosts)
+ similar_hosts
}
}
impl Message<WebGraphService> for Knows {
type Response = Option<Node>;
- async fn handle(mut self, server: &WebGraphService) -> sonic::Result<Self::Response> {
+ async fn handle(mut self, server: &WebGraphService) -> Self::Response {
if let Some(suf) = self.host.strip_prefix("http://") {
self.host = suf.to_string();
}
@@ -129,8 +128,7 @@ impl Message<WebGraphService> for Knows {
self.host = suf.to_string();
}
- let url = Url::parse(&("http://".to_string() + self.host.as_str()))
- .map_err(|_| sonic::Error::BadRequest)?;
+ let url = Url::parse(&("http://".to_string() + self.host.as_str())).ok()?;
let node = Node::from(url).into_host();
@@ -140,9 +138,9 @@ impl Message<WebGraphService> for Knows {
.map(|finder| finder.knows_about(&node))
.unwrap_or(false)
{
- Ok(Some(node))
+ Some(node)
} else {
- Ok(None)
+ None
}
}
}
@@ -155,13 +153,13 @@ pub struct IngoingLinks {
impl Message<WebGraphService> for IngoingLinks {
type Response = Vec<FullEdge>;
- async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {
+ async fn handle(self, server: &WebGraphService) -> Self::Response {
match server.granularity {
WebgraphGranularity::Host => {
let node = self.node.into_host();
- Ok(server.graph.ingoing_edges(node))
+ server.graph.ingoing_edges(node)
}
- WebgraphGranularity::Page => Ok(server.graph.ingoing_edges(self.node)),
+ WebgraphGranularity::Page => server.graph.ingoing_edges(self.node),
}
}
}
@@ -174,13 +172,13 @@ pub struct OutgoingLinks {
impl Message<WebGraphService> for OutgoingLinks {
type Response = Vec<FullEdge>;
- async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {
+ async fn handle(self, server: &WebGraphService) -> Self::Response {
match server.granularity {
WebgraphGranularity::Host => {
let node = self.node.into_host();
- Ok(server.graph.outgoing_edges(node))
+ server.graph.outgoing_edges(node)
}
- WebgraphGranularity::Page => Ok(server.graph.outgoing_edges(self.node)),
+ WebgraphGranularity::Page => server.graph.outgoing_edges(self.node),
}
}
}
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index e3d9f0c4..20dccb0b 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -44,7 +44,7 @@ mod bloom;
mod collector;
pub mod config;
pub mod crawler;
-mod distributed;
+pub mod distributed;
pub mod entity_index;
mod enum_map;
mod executor;
@@ -170,6 +170,16 @@ fn rand_words(num_words: usize) -> String {
res.trim().to_string()
}
+#[cfg(test)]
+fn free_socket_addr() -> std::net::SocketAddr {
+ use std::net::{Ipv4Addr, TcpListener};
+
+ let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap();
+ let addr = listener.local_addr().unwrap();
+
+ addr
+}
+
fn ceil_char_boundary(str: &str, index: usize) -> usize {
let mut res = index;
diff --git a/crates/core/src/mapreduce/dht/client.rs b/crates/core/src/mapreduce/dht/client.rs
new file mode 100644
index 00000000..e7661b45
--- /dev/null
+++ b/crates/core/src/mapreduce/dht/client.rs
@@ -0,0 +1,135 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use rand::seq::SliceRandom;
+use std::{collections::BTreeMap, net::SocketAddr};
+
+use crate::{
+ distributed::{
+ cluster::Cluster,
+ member::{Service, ShardId},
+ },
+ Result,
+};
+
+use super::network::api;
+
+struct Node {
+ api: api::RemoteClient,
+}
+
+impl Node {
+ fn new(addr: SocketAddr) -> Self {
+ let api = api::RemoteClient::new(addr);
+
+ Self { api }
+ }
+
+ async fn get(&self, key: Vec<u8>) -> Result<Option<Vec<u8>>> {
+ self.api.get(key).await
+ }
+
+ async fn set(&self, key: Vec<u8>, value: Vec<u8>) -> Result<()> {
+ self.api.set(key, value).await
+ }
+}
+
+struct Shard {
+ nodes: Vec<Node>,
+}
+
+impl Shard {
+ fn new() -> Self {
+ Self { nodes: Vec::new() }
+ }
+
+ fn add_node(&mut self, addr: SocketAddr) {
+ self.nodes.push(Node::new(addr));
+ }
+
+ fn node(&self) -> &Node {
+ self.nodes.choose(&mut rand::thread_rng()).unwrap()
+ }
+
+ async fn get(&self, key: Vec<u8>) -> Result<Option<Vec<u8>>> {
+ self.node().get(key).await
+ }
+
+ async fn set(&self, key: Vec<u8>, value: Vec<u8>) -> Result<()> {
+ self.node().set(key, value).await
+ }
+}
+
+pub struct Client {
+ shards: BTreeMap<ShardId, Shard>,
+}
+
+impl Client {
+ pub async fn new(cluster: &Cluster) -> Self {
+ let dht_members = cluster
+ .members()
+ .await
+ .into_iter()
+ .filter_map(|member| {
+ if let Service::Dht { shard, host } = member.service {
+ Some((shard, host))
+ } else {
+ None
+ }
+ })
+ .collect::<Vec<_>>();
+
+ let mut shards = BTreeMap::new();
+
+ for (shard, host) in dht_members {
+ shards
+ .entry(shard)
+ .or_insert_with(Shard::new)
+ .add_node(host);
+ }
+
+ Self { shards }
+ }
+
+ pub fn add_node(&mut self, shard_id: ShardId, addr: SocketAddr) {
+ self.shards
+ .entry(shard_id)
+ .or_insert_with(Shard::new)
+ .add_node(addr);
+ }
+
+ fn shard_for_key(&self, key: &[u8]) -> Result<&Shard> {
+ let ids = self.shards.keys().collect::<Vec<_>>();
+
+ if ids.is_empty() {
+ return Err(anyhow::anyhow!("No shards"));
+ }
+
+ let hash = md5::compute(key);
+ let hash = u64::from_le_bytes((&hash.0[..(u64::BITS / 8) as usize]).try_into().unwrap());
+
+ let shard_id = ids[hash as usize % ids.len()];
+ Ok(self.shards.get(shard_id).unwrap())
+ }
+
+ pub async fn get(&self, key: Vec<u8>) -> Result<Option<Vec<u8>>> {
+ self.shard_for_key(&key)?.get(key).await
+ }
+
+ pub async fn set(&self, key: Vec<u8>, value: Vec<u8>) -> Result<()> {
+ self.shard_for_key(&key)?.set(key, value).await
+ }
+}
diff --git a/crates/core/src/mapreduce/dht/log_store.rs b/crates/core/src/mapreduce/dht/log_store.rs
new file mode 100644
index 00000000..4ab52a9e
--- /dev/null
+++ b/crates/core/src/mapreduce/dht/log_store.rs
@@ -0,0 +1,270 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::collections::BTreeMap;
+use std::fmt::Debug;
+use std::ops::RangeBounds;
+use std::sync::Arc;
+
+use openraft::storage::LogFlushed;
+use openraft::LogId;
+use openraft::LogState;
+use openraft::RaftLogId;
+use openraft::RaftTypeConfig;
+use openraft::StorageError;
+use openraft::Vote;
+use tokio::sync::Mutex;
+
+/// RaftLogStore implementation with a in-memory storage
+#[derive(Clone, Debug, Default)]
+pub struct LogStore<C: RaftTypeConfig> {
+ inner: Arc<Mutex<LogStoreInner<C>>>,
+}
+
+#[derive(Debug)]
+pub struct LogStoreInner<C: RaftTypeConfig> {
+ /// The last purged log id.
+ last_purged_log_id: Option<LogId<C::NodeId>>,
+
+ /// The Raft log.
+ log: BTreeMap<u64, C::Entry>,
+
+ /// The commit log id.
+ committed: Option<LogId<C::NodeId>>,
+
+ /// The current granted vote.
+ vote: Option<Vote<C::NodeId>>,
+}
+
+impl<C: RaftTypeConfig> Default for LogStoreInner<C> {
+ fn default() -> Self {
+ Self {
+ last_purged_log_id: None,
+ log: BTreeMap::new(),
+ committed: None,
+ vote: None,
+ }
+ }
+}
+
+impl<C: RaftTypeConfig> LogStoreInner<C> {
+ async fn try_get_log_entries<RB: RangeBounds<u64> + Clone + Debug>(
+ &mut self,
+ range: RB,
+ ) -> Result<Vec<C::Entry>, StorageError<C::NodeId>>
+ where
+ C::Entry: Clone,
+ {
+ let response = self
+ .log
+ .range(range.clone())
+ .map(|(_, val)| val.clone())
+ .collect::<Vec<_>>();
+ Ok(response)
+ }
+
+ async fn get_log_state(&mut self) -> Result<LogState<C>, StorageError<C::NodeId>> {
+ let last = self
+ .log
+ .iter()
+ .next_back()
+ .map(|(_, ent)| *ent.get_log_id());
+
+ let last_purged = self.last_purged_log_id;
+
+ let last = match last {
+ None => last_purged,
+ Some(x) => Some(x),
+ };
+
+ Ok(LogState {
+ last_purged_log_id: last_purged,
+ last_log_id: last,
+ })
+ }
+
+ async fn save_committed(
+ &mut self,
+ committed: Option<LogId<C::NodeId>>,
+ ) -> Result<(), StorageError<C::NodeId>> {
+ self.committed = committed;
+ Ok(())
+ }
+
+ async fn read_committed(
+ &mut self,
+ ) -> Result<Option<LogId<C::NodeId>>, StorageError<C::NodeId>> {
+ Ok(self.committed)
+ }
+
+ async fn save_vote(&mut self, vote: &Vote<C::NodeId>) -> Result<(), StorageError<C::NodeId>> {
+ self.vote = Some(*vote);
+ Ok(())
+ }
+
+ async fn read_vote(&mut self) -> Result<Option<Vote<C::NodeId>>, StorageError<C::NodeId>> {
+ Ok(self.vote)
+ }
+
+ async fn append<I>(
+ &mut self,
+ entries: I,
+ callback: LogFlushed<C>,
+ ) -> Result<(), StorageError<C::NodeId>>
+ where
+ I: IntoIterator<Item = C::Entry>,
+ {
+ // Simple implementation that calls the flush-before-return `append_to_log`.
+ for entry in entries {
+ self.log.insert(entry.get_log_id().index, entry);
+ }
+ callback.log_io_completed(Ok(()));
+
+ Ok(())
+ }
+
+ async fn truncate(&mut self, log_id: LogId<C::NodeId>) -> Result<(), StorageError<C::NodeId>> {
+ let keys = self
+ .log
+ .range(log_id.index..)
+ .map(|(k, _v)| *k)
+ .collect::<Vec<_>>();
+ for key in keys {
+ self.log.remove(&key);
+ }
+
+ Ok(())
+ }
+
+ async fn purge(&mut self, log_id: LogId<C::NodeId>) -> Result<(), StorageError<C::NodeId>> {
+ {
+ let ld = &mut self.last_purged_log_id;
+ assert!(*ld <= Some(log_id));
+ *ld = Some(log_id);
+ }
+
+ {
+ let keys = self
+ .log
+ .range(..=log_id.index)
+ .map(|(k, _v)| *k)
+ .collect::<Vec<_>>();
+ for key in keys {
+ self.log.remove(&key);
+ }
+ }
+
+ Ok(())
+ }
+}
+
+mod impl_log_store {
+ use std::fmt::Debug;
+ use std::ops::RangeBounds;
+
+ use openraft::storage::LogFlushed;
+ use openraft::storage::RaftLogStorage;
+ use openraft::LogId;
+ use openraft::LogState;
+ use openraft::RaftLogReader;
+ use openraft::RaftTypeConfig;
+ use openraft::StorageError;
+ use openraft::Vote;
+
+ use super::LogStore;
+
+ impl<C: RaftTypeConfig> RaftLogReader<C> for LogStore<C>
+ where
+ C::Entry: Clone,
+ {
+ async fn try_get_log_entries<RB: RangeBounds<u64> + Clone + Debug>(
+ &mut self,
+ range: RB,
+ ) -> Result<Vec<C::Entry>, StorageError<C::NodeId>> {
+ let mut inner = self.inner.lock().await;
+ inner.try_get_log_entries(range).await
+ }
+ }
+
+ impl<C: RaftTypeConfig> RaftLogStorage<C> for LogStore<C>
+ where
+ C::Entry: Clone,
+ {
+ type LogReader = Self;
+
+ async fn get_log_state(&mut self) -> Result<LogState<C>, StorageError<C::NodeId>> {
+ let mut inner = self.inner.lock().await;
+ inner.get_log_state().await
+ }
+
+ async fn save_committed(
+ &mut self,
+ committed: Option<LogId<C::NodeId>>,
+ ) -> Result<(), StorageError<C::NodeId>> {
+ let mut inner = self.inner.lock().await;
+ inner.save_committed(committed).await
+ }
+
+ async fn read_committed(
+ &mut self,
+ ) -> Result<Option<LogId<C::NodeId>>, StorageError<C::NodeId>> {
+ let mut inner = self.inner.lock().await;
+ inner.read_committed().await
+ }
+
+ async fn save_vote(
+ &mut self,
+ vote: &Vote<C::NodeId>,
+ ) -> Result<(), StorageError<C::NodeId>> {
+ let mut inner = self.inner.lock().await;
+ inner.save_vote(vote).await
+ }
+
+ async fn read_vote(&mut self) -> Result<Option<Vote<C::NodeId>>, StorageError<C::NodeId>> {
+ let mut inner = self.inner.lock().await;
+ inner.read_vote().await
+ }
+
+ async fn append<I>(
+ &mut self,
+ entries: I,
+ callback: LogFlushed<C>,
+ ) -> Result<(), StorageError<C::NodeId>>
+ where
+ I: IntoIterator<Item = C::Entry>,
+ {
+ let mut inner = self.inner.lock().await;
+ inner.append(entries, callback).await
+ }
+
+ async fn truncate(
+ &mut self,
+ log_id: LogId<C::NodeId>,
+ ) -> Result<(), StorageError<C::NodeId>> {
+ let mut inner = self.inner.lock().await;
+ inner.truncate(log_id).await
+ }
+
+ async fn purge(&mut self, log_id: LogId<C::NodeId>) -> Result<(), StorageError<C::NodeId>> {
+ let mut inner = self.inner.lock().await;
+ inner.purge(log_id).await
+ }
+
+ async fn get_log_reader(&mut self) -> Self::LogReader {
+ self.clone()
+ }
+ }
+}
diff --git a/crates/core/src/mapreduce/dht/mod.rs b/crates/core/src/mapreduce/dht/mod.rs
new file mode 100644
index 00000000..08b8af32
--- /dev/null
+++ b/crates/core/src/mapreduce/dht/mod.rs
@@ -0,0 +1,494 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Simple in-memory key-value store with Raft consensus where keys
+//! and values are arbitrary bytes. It is intended to be deployed
+//! across multiple nodes with multiple shards. Each shard cluster
+//! is a Raft cluster, and each key is then routed to the correct
+//! cluster based on hash(key) % number_of_shards. The keys
+//! are currently *not* rebalanced if the number of shards change, so
+//! if an entire shard becomes unavailable or a new shard is added, all
+//! keys in the entire DHT is essentially lost as the
+//! keys might hash incorrectly.
+//!
+//! Heavily inspired by https://github.com/datafuselabs/openraft/blob/main/examples/raft-kv-memstore/
+
+mod client;
+pub mod log_store;
+mod network;
+pub mod store;
+
+use network::api::{Get, Set};
+
+use std::fmt::Debug;
+use std::io::Cursor;
+
+use openraft::BasicNode;
+use openraft::TokioRuntime;
+
+use self::network::Server;
+
+pub use network::api::RemoteClient as ApiClient;
+pub use network::raft::RemoteClient as RaftClient;
+
+pub use client::Client;
+
+pub type NodeId = u64;
+
+openraft::declare_raft_types!(
+ /// Declare the type configuration for example K/V store.
+ pub TypeConfig:
+ D = Request,
+ R = Response,
+ NodeId = NodeId,
+ Node = BasicNode,
+ Entry = openraft::Entry<TypeConfig>,
+ SnapshotData = Cursor<Vec<u8>>,
+ AsyncRuntime = TokioRuntime,
+);
+
+#[macro_export]
+macro_rules! raft_sonic_request_response {
+ ($service:ident, [$($req:ident),*$(,)?]) => {
+ #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
+ pub enum Request {
+ $(
+ $req($req),
+ )*
+ }
+
+ #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
+ pub enum Response {
+ $(
+ $req(<$req as crate::distributed::sonic::service::Message<$service>>::Response),
+ )*
+ Empty,
+ }
+
+ $(
+ impl TryFrom<Response> for <$req as crate::distributed::sonic::service::Message<$service>>::Response {
+ type Error = crate::distributed::sonic::Error;
+ fn try_from(res: Response) -> Result<Self, Self::Error> {
+ match res {
+ Response::$req(res) => Ok(res),
+ _ => Err(crate::distributed::sonic::Error::Application(anyhow::anyhow!("Invalid response for request from Raft"))),
+ }
+ }
+ }
+ )*
+
+ $(
+ impl From<$req> for Request {
+ fn from(req: $req) -> Self {
+ Request::$req(req)
+ }
+ }
+ )*
+ };
+}
+
+raft_sonic_request_response!(Server, [Get, Set]);
+
+#[cfg(test)]
+mod tests {
+ use std::{collections::BTreeMap, net::SocketAddr, sync::Arc};
+ use tests::network::api::RemoteClient;
+ use tokio::sync::Mutex;
+ use tracing_test::traced_test;
+
+ use crate::{distributed::sonic, free_socket_addr};
+ use openraft::{error::InitializeError, Config};
+
+ use proptest::prelude::*;
+ use proptest_derive::Arbitrary;
+
+ use rand::seq::SliceRandom;
+
+ use super::*;
+
+ async fn server(
+ id: u64,
+ ) -> anyhow::Result<(
+ openraft::Raft<TypeConfig>,
+ sonic::service::Server<Server>,
+ SocketAddr,
+ )> {
+ let config = Config {
+ heartbeat_interval: 500,
+ election_timeout_min: 1500,
+ election_timeout_max: 3000,
+ ..Default::default()
+ };
+
+ let config = Arc::new(config.validate().unwrap());
+
+ let log_store = log_store::LogStore::<TypeConfig>::default();
+ let state_machine_store = Arc::new(store::StateMachineStore::default());
+
+ let network = network::Network;
+
+ let raft = openraft::Raft::new(id, config, network, log_store, state_machine_store.clone())
+ .await?;
+
+ let addr = free_socket_addr();
+
+ let server = Server::new(raft.clone(), state_machine_store)
+ .bind(addr)
+ .await?;
+
+ Ok((raft, server, addr))
+ }
+
+ #[tokio::test]
+ #[traced_test]
+ async fn test_simple_set_get() -> anyhow::Result<()> {
+ let (raft1, server1, addr1) = server(1).await?;
+ let (raft2, server2, addr2) = server(2).await?;
+
+ let servers = vec![server1, server2];
+
+ for server in servers {
+ tokio::spawn(async move {
+ loop {
+ server.accept().await.unwrap();
+ }
+ });
+ }
+
+ let members: BTreeMap<u64, _> = vec![(1, addr1), (2, addr2)]
+ .into_iter()
+ .map(|(id, addr)| (id, BasicNode::new(addr)))
+ .collect();
+
+ if let Err(e) = raft1.initialize(members.clone()).await {
+ match e {
+ openraft::error::RaftError::APIError(e) => match e {
+ InitializeError::NotAllowed(_) => {}
+ InitializeError::NotInMembers(_) => panic!("{:?}", e),
+ },
+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),
+ }
+ };
+
+ if let Err(e) = raft2.initialize(members.clone()).await {
+ match e {
+ openraft::error::RaftError::APIError(e) => match e {
+ InitializeError::NotAllowed(_) => {}
+ InitializeError::NotInMembers(_) => panic!("{:?}", e),
+ },
+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),
+ }
+ };
+
+ let c1 = RemoteClient::new(addr1);
+ let c2 = RemoteClient::new(addr2);
+
+ c1.set("hello".as_bytes().to_vec(), "world".as_bytes().to_vec())
+ .await?;
+
+ let res = c1.get("hello".as_bytes().to_vec()).await?;
+ assert_eq!(res, Some("world".as_bytes().to_vec()));
+
+ let res = c2.get("hello".as_bytes().to_vec()).await?;
+ assert_eq!(res, Some("world".as_bytes().to_vec()));
+
+ c2.set("hello".as_bytes().to_vec(), "world2".as_bytes().to_vec())
+ .await?;
+
+ let res = c1.get("hello".as_bytes().to_vec()).await?;
+ assert_eq!(res, Some("world2".as_bytes().to_vec()));
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ #[traced_test]
+ async fn test_member_join() -> anyhow::Result<()> {
+ let (raft1, server1, addr1) = server(1).await?;
+ let (raft2, server2, addr2) = server(2).await?;
+ let (raft3, server3, addr3) = server(3).await?;
+
+ let servers = vec![server1, server2, server3];
+
+ for server in servers {
+ tokio::spawn(async move {
+ loop {
+ server.accept().await.unwrap();
+ }
+ });
+ }
+
+ let members: BTreeMap<u64, _> = vec![(1, addr1), (2, addr2)]
+ .into_iter()
+ .map(|(id, addr)| (id, BasicNode::new(addr)))
+ .collect();
+
+ if let Err(e) = raft1.initialize(members.clone()).await {
+ match e {
+ openraft::error::RaftError::APIError(e) => match e {
+ InitializeError::NotAllowed(_) => {}
+ InitializeError::NotInMembers(_) => panic!("{:?}", e),
+ },
+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),
+ }
+ };
+
+ if let Err(e) = raft2.initialize(members.clone()).await {
+ match e {
+ openraft::error::RaftError::APIError(e) => match e {
+ InitializeError::NotAllowed(_) => {}
+ InitializeError::NotInMembers(_) => panic!("{:?}", e),
+ },
+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),
+ }
+ };
+
+ let c1 = RemoteClient::new(addr1);
+ let c2 = RemoteClient::new(addr2);
+
+ c1.set("hello".as_bytes().to_vec(), "world".as_bytes().to_vec())
+ .await?;
+
+ let res = c2.get("hello".as_bytes().to_vec()).await?;
+
+ assert_eq!(res, Some("world".as_bytes().to_vec()));
+
+ let members: BTreeMap<u64, _> = vec![(1, addr1), (2, addr2), (3, addr3)]
+ .into_iter()
+ .map(|(id, addr)| (id, BasicNode::new(addr)))
+ .collect();
+
+ raft3.initialize(members.clone()).await?;
+ let rc1 = network::raft::RemoteClient::new(1, BasicNode::new(addr1));
+ rc1.join(3, addr3, members.clone()).await?;
+
+ let c3 = RemoteClient::new(addr3);
+ let res = c3.get("hello".as_bytes().to_vec()).await?;
+ assert_eq!(res, Some("world".as_bytes().to_vec()));
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ #[traced_test]
+ async fn test_node_crash() -> anyhow::Result<()> {
+ let (raft1, server1, addr1) = server(1).await?;
+ let (raft2, server2, addr2) = server(2).await?;
+ let (raft3, server3, addr3) = server(3).await?;
+
+ let servers = vec![server1, server2, server3];
+ let mut handles = Vec::new();
+
+ for server in servers {
+ handles.push(tokio::spawn(async move {
+ loop {
+ server.accept().await.unwrap();
+ }
+ }));
+ }
+
+ let members: BTreeMap<u64, _> = vec![(1, addr1), (2, addr2), (3, addr3)]
+ .into_iter()
+ .map(|(id, addr)| (id, BasicNode::new(addr)))
+ .collect();
+
+ if let Err(e) = raft1.initialize(members.clone()).await {
+ match e {
+ openraft::error::RaftError::APIError(e) => match e {
+ InitializeError::NotAllowed(_) => {}
+ InitializeError::NotInMembers(_) => panic!("{:?}", e),
+ },
+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),
+ }
+ };
+
+ if let Err(e) = raft2.initialize(members.clone()).await {
+ match e {
+ openraft::error::RaftError::APIError(e) => match e {
+ InitializeError::NotAllowed(_) => {}
+ InitializeError::NotInMembers(_) => panic!("{:?}", e),
+ },
+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),
+ }
+ };
+
+ if let Err(e) = raft3.initialize(members.clone()).await {
+ match e {
+ openraft::error::RaftError::APIError(e) => match e {
+ InitializeError::NotAllowed(_) => {}
+ InitializeError::NotInMembers(_) => panic!("{:?}", e),
+ },
+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),
+ }
+ };
+
+ let c1 = RemoteClient::new(addr1);
+ let c2 = RemoteClient::new(addr2);
+
+ let rc1 = network::raft::RemoteClient::new(1, BasicNode::new(addr1));
+
+ c1.set("hello".as_bytes().to_vec(), "world".as_bytes().to_vec())
+ .await?;
+
+ let res = c1.get("hello".as_bytes().to_vec()).await?;
+ assert_eq!(res, Some("world".as_bytes().to_vec()));
+
+ let res = c2.get("hello".as_bytes().to_vec()).await?;
+ assert_eq!(res, Some("world".as_bytes().to_vec()));
+
+ // crash node 2
+ handles[1].abort();
+ drop(raft2);
+
+ let (raft2, server2, addr2) = server(2).await?;
+ handles[1] = tokio::spawn(async move {
+ loop {
+ server2.accept().await.unwrap();
+ }
+ });
+
+ rc1.join(2, addr2, members.clone()).await?;
+
+ let c2 = RemoteClient::new(addr2);
+
+ let res = c2.get("hello".as_bytes().to_vec()).await?;
+ assert_eq!(res, Some("world".as_bytes().to_vec()));
+
+ // crash node 2 again
+ handles[1].abort();
+ drop(raft2);
+
+ c1.set("hello".as_bytes().to_vec(), "world2".as_bytes().to_vec())
+ .await?;
+
+ let (raft2, server2, addr2) = server(2).await?;
+ handles[1] = tokio::spawn(async move {
+ loop {
+ server2.accept().await.unwrap();
+ }
+ });
+ raft2.initialize(members.clone()).await?;
+ rc1.join(2, addr2, members.clone()).await?;
+
+ let c2 = RemoteClient::new(addr2);
+
+ let res = c2.get("hello".as_bytes().to_vec()).await?;
+ assert_eq!(res, Some("world2".as_bytes().to_vec()));
+
+ Ok(())
+ }
+
+ #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Arbitrary)]
+ enum Action {
+ Set { key: Vec<u8>, value: Vec<u8> },
+ // get actions[prev_key % actions.len()]
+ // if actions[prev_key % actions.len()] is a get, then get a non-existent key
+ Get { prev_key: usize },
+ }
+
+ proptest! {
+ #![proptest_config(ProptestConfig::with_cases(10))]
+
+ #[test]
+ #[traced_test]
+ fn proptest_chaos(actions: Vec<Action>) {
+ let ground_truth = Arc::new(Mutex::new(BTreeMap::<Vec<u8>, Vec<u8>>::new()));
+
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(async move {
+ let (raft1, server1, addr1) = server(1).await.unwrap();
+ let (raft2, server2, addr2) = server(2).await.unwrap();
+
+ let servers = vec![server1, server2];
+
+ let mut handles = Vec::new();
+ for server in servers {
+ handles.push(tokio::spawn(async move {
+ loop {
+ server.accept().await.unwrap();
+ }
+ }));
+ }
+
+ let members: BTreeMap<u64, _> = vec![(1, addr1), (2, addr2)]
+ .into_iter()
+ .map(|(id, addr)| (id, BasicNode::new(addr)))
+ .collect();
+
+ if let Err(e) = raft1.initialize(members.clone()).await {
+ match e {
+ openraft::error::RaftError::APIError(e) => match e {
+ InitializeError::NotAllowed(_) => {}
+ InitializeError::NotInMembers(_) => panic!("{:?}", e),
+ },
+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),
+ }
+ };
+
+ if let Err(e) = raft2.initialize(members.clone()).await {
+ match e {
+ openraft::error::RaftError::APIError(e) => match e {
+ InitializeError::NotAllowed(_) => {}
+ InitializeError::NotInMembers(_) => panic!("{:?}", e),
+ },
+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),
+ }
+ };
+
+ let c1 = RemoteClient::new(addr1);
+ let c2 = RemoteClient::new(addr2);
+
+ let clients = Arc::new(vec![c1, c2]);
+
+ let shared_actions = Arc::new(actions.clone());
+
+ for (i, action) in actions.into_iter().enumerate() {
+ match action {
+ Action::Set { key, value } => {
+ let client = clients.choose(&mut rand::thread_rng()).unwrap();
+
+ client.set(key.clone(), value.clone()).await.unwrap();
+ ground_truth.lock().await.insert(key.clone(), value.clone());
+ }
+ Action::Get { prev_key } => {
+ let client = clients.choose(&mut rand::thread_rng()).unwrap();
+ client.set(b"ensure-linearized-read".to_vec(), vec![]).await.unwrap();
+
+ let key = if i == 0 {
+ b"non-existent-key".to_vec()
+ } else {
+ match shared_actions[dbg!(prev_key % dbg!(i))] {
+ Action::Set { ref key, .. } => {
+ dbg!(key.clone())
+ },
+ Action::Get { .. } => b"non-existent-key".to_vec(),
+ }
+ };
+
+ let res = client.get(key.clone()).await.unwrap();
+ let expected = ground_truth.lock().await.get(&key).cloned();
+
+ assert_eq!(res, expected);
+ }
+ }
+ }
+ });
+ }
+ }
+}
diff --git a/crates/core/src/mapreduce/dht/network/api.rs b/crates/core/src/mapreduce/dht/network/api.rs
new file mode 100644
index 00000000..e4a111b6
--- /dev/null
+++ b/crates/core/src/mapreduce/dht/network/api.rs
@@ -0,0 +1,183 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use std::{net::SocketAddr, time::Duration};
+
+use crate::{distributed::retry_strategy::RandomBackoff, Result};
+use anyhow::anyhow;
+use openraft::{
+ error::{ClientWriteError, ForwardToLeader, RaftError},
+ BasicNode,
+};
+use tokio::sync::RwLock;
+
+use crate::{distributed::sonic, mapreduce::dht::NodeId};
+
+use super::Server;
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct Set {
+ pub key: Vec<u8>,
+ pub value: Vec<u8>,
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct Get {
+ pub key: Vec<u8>,
+}
+
+impl sonic::service::Message<Server> for Set {
+ type Response = Result<(), RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>;
+
+ async fn handle(self, server: &Server) -> Self::Response {
+ tracing::debug!("received set request: {:?}", self);
+
+ match server.raft.client_write(self.into()).await {
+ Ok(_) => Ok(()),
+ Err(e) => Err(e),
+ }
+ }
+}
+
+impl sonic::service::Message<Server> for Get {
+ type Response = Option<Vec<u8>>;
+
+ async fn handle(self, server: &Server) -> Self::Response {
+ server
+ .state_machine_store
+ .state_machine
+ .read()
+ .await
+ .data
+ .get(&self.key)
+ .cloned()
+ }
+}
+
+pub struct RemoteClient {
+ self_remote: sonic::replication::RemoteClient<Server>,
+ likely_leader: RwLock<sonic::replication::RemoteClient<Server>>,
+}
+
+impl RemoteClient {
+ pub fn new(addr: SocketAddr) -> Self {
+ Self {
+ self_remote: sonic::replication::RemoteClient::new(addr),
+ likely_leader: RwLock::new(sonic::replication::RemoteClient::new(addr)),
+ }
+ }
+
+ fn retry_strat() -> impl Iterator<Item = std::time::Duration> {
+ RandomBackoff::new(
+ std::time::Duration::from_millis(200),
+ std::time::Duration::from_secs(1),
+ )
+ }
+
+ pub async fn set(&self, key: Vec<u8>, value: Vec<u8>) -> Result<()> {
+ for backoff in Self::retry_strat() {
+ let res = self
+ .likely_leader
+ .read()
+ .await
+ .send_with_timeout(
+ &Set {
+ key: key.clone(),
+ value: value.clone(),
+ },
+ Duration::from_secs(5),
+ )
+ .await;
+
+ tracing::debug!(".set() got response: {res:?}");
+
+ match res {
+ Ok(res) => match res {
+ Ok(_) => return Ok(()),
+ Err(RaftError::APIError(e)) => match e {
+ ClientWriteError::ForwardToLeader(ForwardToLeader {
+ leader_id: _,
+ leader_node,
+ }) => match leader_node {
+ Some(leader_node) => {
+ let mut likely_leader = self.likely_leader.write().await;
+ *likely_leader = sonic::replication::RemoteClient::new(
+ leader_node
+ .addr
+ .parse()
+ .expect("node addr should always be valid addr"),
+ );
+ }
+ None => {
+ tokio::time::sleep(backoff).await;
+ }
+ },
+ ClientWriteError::ChangeMembershipError(_) => {
+ unreachable!(".set() should not change membership")
+ }
+ },
+ Err(RaftError::Fatal(e)) => return Err(e.into()),
+ },
+ Err(e) => match e {
+ sonic::Error::IO(_)
+ | sonic::Error::Serialization(_)
+ | sonic::Error::ConnectionTimeout
+ | sonic::Error::RequestTimeout
+ | sonic::Error::PoolCreation => {
+ tokio::time::sleep(backoff).await;
+ }
+ sonic::Error::BadRequest
+ | sonic::Error::BodyTooLarge {
+ body_size: _,
+ max_size: _,
+ }
+ | sonic::Error::Application(_) => return Err(e.into()),
+ },
+ }
+ }
+
+ Err(anyhow!("failed to set key"))
+ }
+
+ pub async fn get(&self, key: Vec<u8>) -> Result<Option<Vec<u8>>> {
+ for backoff in Self::retry_strat() {
+ match self
+ .self_remote
+ .send_with_timeout(&Get { key: key.clone() }, Duration::from_secs(5))
+ .await
+ {
+ Ok(res) => return Ok(res),
+ Err(e) => match e {
+ sonic::Error::IO(_)
+ | sonic::Error::Serialization(_)
+ | sonic::Error::ConnectionTimeout
+ | sonic::Error::RequestTimeout
+ | sonic::Error::PoolCreation => {
+ tokio::time::sleep(backoff).await;
+ }
+ sonic::Error::BadRequest
+ | sonic::Error::BodyTooLarge {
+ body_size: _,
+ max_size: _,
+ }
+ | sonic::Error::Application(_) => return Err(e.into()),
+ },
+ }
+ }
+
+ Err(anyhow!("failed to get key"))
+ }
+}
diff --git a/crates/core/src/mapreduce/dht/network/mod.rs b/crates/core/src/mapreduce/dht/network/mod.rs
new file mode 100644
index 00000000..b6efc5a1
--- /dev/null
+++ b/crates/core/src/mapreduce/dht/network/mod.rs
@@ -0,0 +1,87 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+pub mod api;
+pub mod raft;
+
+use api::{Get, Set};
+use std::{collections::BTreeMap, net::SocketAddr, sync::Arc};
+
+use openraft::{BasicNode, Raft, RaftNetworkFactory};
+
+use crate::sonic_service;
+
+use self::raft::RemoteClient;
+
+use super::{store::StateMachineStore, NodeId, TypeConfig};
+
+#[derive(Clone)]
+pub struct Network;
+
+impl RaftNetworkFactory<TypeConfig> for Network {
+ type Network = RemoteClient;
+
+ async fn new_client(&mut self, target: NodeId, node: &BasicNode) -> Self::Network {
+ RemoteClient::new(target, node.clone())
+ }
+}
+
+pub type AppendEntriesRequest = openraft::raft::AppendEntriesRequest<TypeConfig>;
+pub type AppendEntriesResponse = openraft::raft::AppendEntriesResponse<NodeId>;
+
+pub type InstallSnapshotRequest = openraft::raft::InstallSnapshotRequest<TypeConfig>;
+pub type InstallSnapshotResponse = openraft::raft::InstallSnapshotResponse<NodeId>;
+
+pub type VoteRequest = openraft::raft::VoteRequest<NodeId>;
+pub type VoteResponse = openraft::raft::VoteResponse<NodeId>;
+
+#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
+pub struct AddLearnerRequest {
+ pub id: NodeId,
+ pub addr: SocketAddr,
+}
+
+#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
+pub struct AddNodesRequest {
+ members: BTreeMap<NodeId, BasicNode>,
+}
+
+sonic_service!(
+ Server,
+ [
+ AppendEntriesRequest,
+ InstallSnapshotRequest,
+ VoteRequest,
+ AddLearnerRequest,
+ AddNodesRequest,
+ Get,
+ Set,
+ ]
+);
+
+pub struct Server {
+ raft: Raft<TypeConfig>,
+ state_machine_store: Arc<StateMachineStore>,
+}
+
+impl Server {
+ pub fn new(raft: Raft<TypeConfig>, state_machine_store: Arc<StateMachineStore>) -> Self {
+ Self {
+ raft,
+ state_machine_store,
+ }
+ }
+}
diff --git a/crates/core/src/mapreduce/dht/network/raft.rs b/crates/core/src/mapreduce/dht/network/raft.rs
new file mode 100644
index 00000000..bfac4b56
--- /dev/null
+++ b/crates/core/src/mapreduce/dht/network/raft.rs
@@ -0,0 +1,338 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use std::{
+ collections::{BTreeMap, BTreeSet},
+ net::SocketAddr,
+ time::Duration,
+};
+
+use openraft::{
+ error::{ClientWriteError, ForwardToLeader, InstallSnapshotError, RaftError},
+ network::RPCOption,
+ BasicNode, ChangeMembers, RaftNetwork,
+};
+use tokio::sync::RwLock;
+
+use crate::{
+ distributed::{
+ retry_strategy::ExponentialBackoff,
+ sonic::{self, service::Connection},
+ },
+ mapreduce::dht::{NodeId, TypeConfig},
+ Result,
+};
+
+use super::{
+ AddLearnerRequest, AddNodesRequest, AppendEntriesRequest, AppendEntriesResponse,
+ InstallSnapshotRequest, InstallSnapshotResponse, Server, VoteRequest, VoteResponse,
+};
+
+impl sonic::service::Message<Server> for AppendEntriesRequest {
+ type Response = Result<AppendEntriesResponse, RaftError<NodeId>>;
+
+ async fn handle(self, server: &Server) -> Self::Response {
+ tracing::debug!("received append entries request: {:?}", self);
+ server.raft.append_entries(self).await
+ }
+}
+
+impl sonic::service::Message<Server> for InstallSnapshotRequest {
+ type Response = Result<InstallSnapshotResponse, RaftError<NodeId, InstallSnapshotError>>;
+
+ async fn handle(self, server: &Server) -> Self::Response {
+ tracing::debug!("received install snapshot request: {:?}", self);
+ server.raft.install_snapshot(self).await
+ }
+}
+
+impl sonic::service::Message<Server> for VoteRequest {
+ type Response = Result<VoteResponse, RaftError<NodeId>>;
+
+ async fn handle(self, server: &Server) -> Self::Response {
+ tracing::debug!("received vote request: {:?}", self);
+ server.raft.vote(self).await
+ }
+}
+
+impl sonic::service::Message<Server> for AddLearnerRequest {
+ type Response = Result<(), RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>;
+
+ async fn handle(self, server: &Server) -> Self::Response {
+ tracing::debug!("received add learner request: {:?}", self);
+
+ let mut rem = BTreeSet::new();
+ rem.insert(self.id);
+
+ server
+ .raft
+ .change_membership(ChangeMembers::RemoveVoters(rem.clone()), true)
+ .await?;
+ server
+ .raft
+ .change_membership(ChangeMembers::RemoveNodes(rem.clone()), true)
+ .await?;
+
+ let node = BasicNode::new(self.addr);
+ server.raft.add_learner(self.id, node, false).await?;
+
+ Ok(())
+ }
+}
+
+impl sonic::service::Message<Server> for AddNodesRequest {
+ type Response = Result<(), RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>;
+
+ async fn handle(self, server: &Server) -> Self::Response {
+ tracing::debug!("received add nodes request: {:?}", self);
+ server
+ .raft
+ .change_membership(ChangeMembers::AddNodes(self.members), true)
+ .await?;
+
+ Ok(())
+ }
+}
+
+type RPCError<E = openraft::error::Infallible> =
+ openraft::error::RPCError<NodeId, BasicNode, RaftError<NodeId, E>>;
+
+pub struct RemoteClient {
+ target: NodeId,
+ node: BasicNode,
+ inner: sonic::replication::RemoteClient<Server>,
+ likely_leader: RwLock<sonic::replication::RemoteClient<Server>>,
+}
+
+impl RemoteClient {
+ pub fn new(target: NodeId, node: BasicNode) -> Self {
+ let addr: SocketAddr = node.addr.parse().expect("addr is not a valid address");
+ let inner = sonic::replication::RemoteClient::new(addr);
+ let likely_leader = RwLock::new(inner.clone());
+
+ Self {
+ target,
+ node,
+ inner,
+ likely_leader,
+ }
+ }
+ async fn raft_conn<E: std::error::Error>(&self) -> Result<Connection<Server>, RPCError<E>> {
+ self.inner
+ .conn()
+ .await
+ .map_err(|e| RPCError::Unreachable(openraft::error::Unreachable::new(&e)))
+ }
+
+ async fn send_raft_rpc<R, E>(
+ &self,
+ rpc: R,
+ option: RPCOption,
+ ) -> Result<R::Response, RPCError<E>>
+ where
+ R: sonic::service::Wrapper<Server>,
+ E: std::error::Error,
+ {
+ let conn = self.raft_conn().await?;
+ conn.send_with_timeout(&rpc, option.soft_ttl())
+ .await
+ .map_err(|e| match e {
+ sonic::Error::ConnectionTimeout | sonic::Error::RequestTimeout => {
+ RPCError::Unreachable(openraft::error::Unreachable::new(&e))
+ }
+ _ => {
+ panic!("unexpected error: {:?}", e)
+ }
+ })
+ }
+
+ async fn add_learner(&self, id: NodeId, addr: SocketAddr) -> Result<()> {
+ let rpc = AddLearnerRequest { id, addr };
+ let retry = ExponentialBackoff::from_millis(500)
+ .with_limit(Duration::from_secs(60))
+ .take(5);
+
+ for backoff in retry {
+ let res = self
+ .likely_leader
+ .read()
+ .await
+ .send_with_timeout(&rpc, Duration::from_secs(30))
+ .await;
+
+ match res {
+ Ok(res) => match res {
+ Ok(_) => return Ok(()),
+ Err(RaftError::APIError(e)) => match e {
+ ClientWriteError::ForwardToLeader(ForwardToLeader {
+ leader_id: _,
+ leader_node,
+ }) => match leader_node {
+ Some(leader_node) => {
+ let mut likely_leader = self.likely_leader.write().await;
+ *likely_leader = sonic::replication::RemoteClient::new(
+ leader_node
+ .addr
+ .parse()
+ .expect("node addr should always be valid addr"),
+ );
+ }
+ None => tokio::time::sleep(backoff).await,
+ },
+ ClientWriteError::ChangeMembershipError(_) => {
+ tokio::time::sleep(backoff).await
+ }
+ },
+ Err(RaftError::Fatal(e)) => return Err(e.into()),
+ },
+ Err(e) => match e {
+ sonic::Error::IO(_)
+ | sonic::Error::Serialization(_)
+ | sonic::Error::ConnectionTimeout
+ | sonic::Error::RequestTimeout
+ | sonic::Error::PoolCreation => {
+ tokio::time::sleep(backoff).await;
+ }
+ sonic::Error::BadRequest
+ | sonic::Error::BodyTooLarge {
+ body_size: _,
+ max_size: _,
+ }
+ | sonic::Error::Application(_) => return Err(e.into()),
+ },
+ }
+ }
+
+ Err(anyhow::anyhow!("failed to add learner"))
+ }
+
+ async fn add_nodes(&self, members: BTreeMap<NodeId, BasicNode>) -> Result<()> {
+ let rpc = AddNodesRequest { members };
+ let retry = ExponentialBackoff::from_millis(500).with_limit(Duration::from_secs(10));
+
+ for backoff in retry {
+ let res = self
+ .likely_leader
+ .read()
+ .await
+ .send_with_timeout(&rpc, Duration::from_secs(30))
+ .await;
+
+ match res {
+ Ok(res) => match res {
+ Ok(_) => return Ok(()),
+ Err(RaftError::APIError(e)) => match e {
+ ClientWriteError::ForwardToLeader(ForwardToLeader {
+ leader_id: _,
+ leader_node,
+ }) => match leader_node {
+ Some(leader_node) => {
+ let mut likely_leader = self.likely_leader.write().await;
+ *likely_leader = sonic::replication::RemoteClient::new(
+ leader_node
+ .addr
+ .parse()
+ .expect("node addr should always be valid addr"),
+ );
+ }
+ None => tokio::time::sleep(backoff).await,
+ },
+ ClientWriteError::ChangeMembershipError(_) => {
+ tokio::time::sleep(backoff).await
+ }
+ },
+ Err(RaftError::Fatal(e)) => return Err(e.into()),
+ },
+ Err(e) => match e {
+ sonic::Error::IO(_)
+ | sonic::Error::Serialization(_)
+ | sonic::Error::ConnectionTimeout
+ | sonic::Error::RequestTimeout
+ | sonic::Error::PoolCreation => {
+ tokio::time::sleep(backoff).await;
+ }
+ sonic::Error::BadRequest
+ | sonic::Error::BodyTooLarge {
+ body_size: _,
+ max_size: _,
+ }
+ | sonic::Error::Application(_) => return Err(e.into()),
+ },
+ }
+ }
+
+ unreachable!("should continue to retry");
+ }
+
+ pub async fn join(
+ &self,
+ id: NodeId,
+ addr: SocketAddr,
+ new_all_nodes: BTreeMap<NodeId, BasicNode>,
+ ) -> Result<()> {
+ self.add_learner(id, addr).await?;
+ self.add_nodes(new_all_nodes).await?;
+
+ Ok(())
+ }
+}
+
+impl RaftNetwork<TypeConfig> for RemoteClient {
+ async fn append_entries(
+ &mut self,
+ rpc: AppendEntriesRequest,
+ option: RPCOption,
+ ) -> Result<AppendEntriesResponse, RPCError> {
+ self.send_raft_rpc(rpc, option).await?.map_err(|e| {
+ openraft::error::RemoteError {
+ target: self.target,
+ target_node: Some(self.node.clone()),
+ source: e,
+ }
+ .into()
+ })
+ }
+
+ async fn install_snapshot(
+ &mut self,
+ rpc: InstallSnapshotRequest,
+ option: RPCOption,
+ ) -> Result<InstallSnapshotResponse, RPCError<InstallSnapshotError>> {
+ self.send_raft_rpc(rpc, option).await?.map_err(|e| {
+ openraft::error::RemoteError {
+ target: self.target,
+ target_node: Some(self.node.clone()),
+ source: e,
+ }
+ .into()
+ })
+ }
+
+ async fn vote(
+ &mut self,
+ rpc: VoteRequest,
+ option: RPCOption,
+ ) -> Result<VoteResponse, RPCError> {
+ self.send_raft_rpc(rpc, option).await?.map_err(|e| {
+ openraft::error::RemoteError {
+ target: self.target,
+ target_node: Some(self.node.clone()),
+ source: e,
+ }
+ .into()
+ })
+ }
+}
diff --git a/crates/core/src/mapreduce/dht/store.rs b/crates/core/src/mapreduce/dht/store.rs
new file mode 100644
index 00000000..6ad043b9
--- /dev/null
+++ b/crates/core/src/mapreduce/dht/store.rs
@@ -0,0 +1,266 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+use std::collections::BTreeMap;
+use std::fmt::Debug;
+use std::io::Cursor;
+use std::sync::Arc;
+use std::sync::Mutex;
+
+use openraft::storage::RaftStateMachine;
+use openraft::storage::Snapshot;
+use openraft::BasicNode;
+use openraft::Entry;
+use openraft::EntryPayload;
+use openraft::LogId;
+use openraft::RaftSnapshotBuilder;
+use openraft::RaftTypeConfig;
+use openraft::SnapshotMeta;
+use openraft::StorageError;
+use openraft::StorageIOError;
+use openraft::StoredMembership;
+use serde::Deserialize;
+use serde::Serialize;
+use tokio::sync::RwLock;
+
+use crate::mapreduce::dht::network::api;
+
+use super::NodeId;
+use super::TypeConfig;
+use super::{Request, Response};
+
+#[derive(Debug)]
+pub struct StoredSnapshot {
+ pub meta: SnapshotMeta<NodeId, BasicNode>,
+
+ /// The data of the state machine at the time of this snapshot.
+ pub data: Vec<u8>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Default, Clone)]
+pub struct StateMachineData {
+ pub last_applied_log: Option<LogId<NodeId>>,
+ pub last_membership: StoredMembership<NodeId, BasicNode>,
+
+ /// Application data.
+ pub data: BTreeMap<Vec<u8>, Vec<u8>>,
+}
+
+#[derive(Debug, Default)]
+pub struct StateMachineStore {
+ pub state_machine: RwLock<StateMachineData>,
+ snapshot_idx: Arc<Mutex<u64>>,
+
+ /// The last received snapshot.
+ current_snapshot: RwLock<Option<StoredSnapshot>>,
+}
+
+impl RaftSnapshotBuilder<TypeConfig> for Arc<StateMachineStore> {
+ #[tracing::instrument(level = "trace", skip(self))]
+ async fn build_snapshot(&mut self) -> Result<Snapshot<TypeConfig>, StorageError<NodeId>> {
+ let data;
+ let last_applied_log;
+ let last_membership;
+
+ {
+ // Serialize the data of the state machine.
+ let state_machine = self.state_machine.read().await;
+ data = bincode::serialize(&*state_machine)
+ .map_err(|e| StorageIOError::read_state_machine(&e))?;
+
+ last_applied_log = state_machine.last_applied_log;
+ last_membership = state_machine.last_membership.clone();
+ }
+
+ let snapshot_idx = {
+ let mut l = self.snapshot_idx.lock().unwrap();
+ *l += 1;
+ *l
+ };
+
+ let snapshot_id = if let Some(last) = last_applied_log {
+ format!("{}-{}-{}", last.leader_id, last.index, snapshot_idx)
+ } else {
+ format!("--{}", snapshot_idx)
+ };
+
+ let meta = SnapshotMeta {
+ last_log_id: last_applied_log,
+ last_membership,
+ snapshot_id,
+ };
+
+ let snapshot = StoredSnapshot {
+ meta: meta.clone(),
+ data: data.clone(),
+ };
+
+ {
+ let mut current_snapshot = self.current_snapshot.write().await;
+ *current_snapshot = Some(snapshot);
+ }
+
+ Ok(Snapshot {
+ meta,
+ snapshot: Box::new(Cursor::new(data)),
+ })
+ }
+}
+
+impl RaftStateMachine<TypeConfig> for Arc<StateMachineStore> {
+ type SnapshotBuilder = Self;
+
+ async fn applied_state(
+ &mut self,
+ ) -> Result<(Option<LogId<NodeId>>, StoredMembership<NodeId, BasicNode>), StorageError<NodeId>>
+ {
+ let state_machine = self.state_machine.read().await;
+ Ok((
+ state_machine.last_applied_log,
+ state_machine.last_membership.clone(),
+ ))
+ }
+
+ #[tracing::instrument(level = "trace", skip(self, entries))]
+ async fn apply<I>(&mut self, entries: I) -> Result<Vec<Response>, StorageError<NodeId>>
+ where
+ I: IntoIterator<Item = Entry<TypeConfig>> + Send,
+ {
+ let mut res = Vec::new();
+ let mut sm = self.state_machine.write().await;
+
+ for entry in entries {
+ tracing::debug!(%entry.log_id, "replicate to sm");
+
+ if let Some(ref last) = sm.last_applied_log {
+ if last >= &entry.log_id {
+ res.push(Response::Empty);
+ continue;
+ }
+ }
+
+ sm.last_applied_log = Some(entry.log_id);
+
+ match entry.payload {
+ EntryPayload::Blank => res.push(Response::Empty),
+ EntryPayload::Normal(ref req) => match req {
+ Request::Set(api::Set { key, value }) => {
+ sm.data.insert(key.clone(), value.clone());
+ res.push(Response::Set(Ok(())))
+ }
+
+ Request::Get(api::Get { key: _ }) => {
+ unreachable!("Get requests should not be replicated")
+ }
+ },
+ EntryPayload::Membership(ref mem) => {
+ sm.last_membership = StoredMembership::new(Some(entry.log_id), mem.clone());
+ res.push(Response::Empty)
+ }
+ };
+ }
+ Ok(res)
+ }
+
+ #[tracing::instrument(level = "trace", skip(self))]
+ async fn begin_receiving_snapshot(
+ &mut self,
+ ) -> Result<Box<<TypeConfig as RaftTypeConfig>::SnapshotData>, StorageError<NodeId>> {
+ Ok(Box::new(Cursor::new(Vec::new())))
+ }
+
+ #[tracing::instrument(level = "trace", skip(self, snapshot))]
+ async fn install_snapshot(
+ &mut self,
+ meta: &SnapshotMeta<NodeId, BasicNode>,
+ snapshot: Box<<TypeConfig as RaftTypeConfig>::SnapshotData>,
+ ) -> Result<(), StorageError<NodeId>> {
+ tracing::info!(
+ { snapshot_size = snapshot.get_ref().len() },
+ "decoding snapshot for installation"
+ );
+
+ let new_snapshot = StoredSnapshot {
+ meta: meta.clone(),
+ data: snapshot.into_inner(),
+ };
+
+ // Update the state machine.
+ {
+ let data: BTreeMap<Vec<u8>, Vec<u8>> = bincode::deserialize(&new_snapshot.data)
+ .map_err(|e| {
+ StorageIOError::read_snapshot(Some(new_snapshot.meta.signature()), &e)
+ })?;
+
+ let mut state_machine = self.state_machine.write().await;
+ state_machine.data = data;
+ state_machine.last_applied_log = meta.last_log_id;
+ state_machine.last_membership = meta.last_membership.clone();
+ }
+
+ // Update current snapshot.
+ let mut current_snapshot = self.current_snapshot.write().await;
+ *current_snapshot = Some(new_snapshot);
+ Ok(())
+ }
+
+ #[tracing::instrument(level = "trace", skip(self))]
+ async fn get_current_snapshot(
+ &mut self,
+ ) -> Result<Option<Snapshot<TypeConfig>>, StorageError<NodeId>> {
+ match &*self.current_snapshot.read().await {
+ Some(snapshot) => {
+ let data = snapshot.data.clone();
+ Ok(Some(Snapshot {
+ meta: snapshot.meta.clone(),
+ snapshot: Box::new(Cursor::new(data)),
+ }))
+ }
+ None => Ok(None),
+ }
+ }
+
+ async fn get_snapshot_builder(&mut self) -> Self::SnapshotBuilder {
+ Arc::clone(self)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use openraft::testing::{StoreBuilder, Suite};
+
+ type LogStore = crate::mapreduce::dht::log_store::LogStore<TypeConfig>;
+
+ use super::*;
+
+ struct MemStoreBuilder {}
+
+ impl StoreBuilder<TypeConfig, LogStore, Arc<StateMachineStore>, ()> for MemStoreBuilder {
+ async fn build(
+ &self,
+ ) -> Result<((), LogStore, Arc<StateMachineStore>), StorageError<NodeId>> {
+ let log_store = LogStore::default();
+ let sm = Arc::new(StateMachineStore::default());
+
+ Ok(((), log_store, sm))
+ }
+ }
+
+ #[test]
+ pub fn test_raft_impl() -> Result<(), StorageError<NodeId>> {
+ Suite::test_all(MemStoreBuilder {})?;
+ Ok(())
+ }
+}
diff --git a/crates/core/src/mapreduce/mod.rs b/crates/core/src/mapreduce/mod.rs
index 9069c1ed..a96d1e18 100644
--- a/crates/core/src/mapreduce/mod.rs
+++ b/crates/core/src/mapreduce/mod.rs
@@ -19,6 +19,7 @@
use serde::{de::DeserializeOwned, Deserialize, Serialize};
+pub mod dht;
mod manager;
mod worker;
diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rs
index 6888a51b..103f0868 100644
--- a/crates/core/src/searcher/distributed.rs
+++ b/crates/core/src/searcher/distributed.rs
@@ -17,7 +17,7 @@
use crate::{
distributed::{
cluster::Cluster,
- member::Service,
+ member::{Service, ShardId},
sonic::replication::{
AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient, Shard,
ShardIdentifier, ShardedClient, SpecificShardSelector,
@@ -39,7 +39,6 @@ use std::{collections::HashMap, sync::Arc};
use fnv::FnvHashMap;
use futures::future::join_all;
use itertools::Itertools;
-use serde::{Deserialize, Serialize};
use std::future::Future;
use thiserror::Error;
use url::Url;
@@ -64,15 +63,6 @@ pub struct ScoredWebpagePointer {
pub shard: ShardId,
}
-#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy, Debug)]
-pub struct ShardId(u64);
-
-impl ShardId {
- pub fn new(id: u64) -> Self {
- Self(id)
- }
-}
-
impl ShardIdentifier for ShardId {}
#[derive(Debug)]
diff --git a/crates/kuchiki/src/iter.rs b/crates/kuchiki/src/iter.rs
index 9af15bb1..83279d90 100644
--- a/crates/kuchiki/src/iter.rs
+++ b/crates/kuchiki/src/iter.rs
@@ -12,13 +12,15 @@ use crate::Result;
impl NodeRef {
/// Return an iterator of references to this node and its ancestors.
#[inline]
- #[must_use] pub fn inclusive_ancestors(&self) -> Ancestors {
+ #[must_use]
+ pub fn inclusive_ancestors(&self) -> Ancestors {
Ancestors(Some(self.clone()))
}
/// Return an iterator of references to this node’s ancestors.
#[inline]
- #[must_use] pub fn ancestors(&self) -> Ancestors {
+ #[must_use]
+ pub fn ancestors(&self) -> Ancestors {
Ancestors(self.parent())
}
@@ -63,7 +65,8 @@ impl NodeRef {
/// Return an iterator of references to this node and the siblings after it.
#[inline]
- #[must_use] pub fn inclusive_following_siblings(&self) -> Siblings {
+ #[must_use]
+ pub fn inclusive_following_siblings(&self) -> Siblings {
match self.parent() {
Some(parent) => {
let last_sibling = parent.last_child().unwrap();
@@ -85,7 +88,8 @@ impl NodeRef {
/// Return an iterator of references to this node’s siblings after it.
#[inline]
- #[must_use] pub fn following_siblings(&self) -> Siblings {
+ #[must_use]
+ pub fn following_siblings(&self) -> Siblings {
match (self.parent(), self.next_sibling()) {
(Some(parent), Some(next_sibling)) => {
let last_sibling = parent.last_child().unwrap();
@@ -100,7 +104,8 @@ impl NodeRef {
/// Return an iterator of references to this node’s children.
#[inline]
- #[must_use] pub fn children(&self) -> Siblings {
+ #[must_use]
+ pub fn children(&self) -> Siblings {
match (self.first_child(), self.last_child()) {
(Some(first_child), Some(last_child)) => Siblings(Some(State {
next: first_child,
@@ -117,7 +122,8 @@ impl NodeRef {
///
/// Note: this is the `NodeEdge::Start` items from `traverse()`.
#[inline]
- #[must_use] pub fn inclusive_descendants(&self) -> Descendants {
+ #[must_use]
+ pub fn inclusive_descendants(&self) -> Descendants {
Descendants(self.traverse_inclusive())
}
@@ -127,14 +133,16 @@ impl NodeRef {
///
/// Note: this is the `NodeEdge::Start` items from `traverse()`.
#[inline]
- #[must_use] pub fn descendants(&self) -> Descendants {
+ #[must_use]
+ pub fn descendants(&self) -> Descendants {
Descendants(self.traverse())
}
/// Return an iterator of the start and end edges of this node and its descendants,
/// in tree order.
#[inline]
- #[must_use] pub fn traverse_inclusive(&self) -> Traverse {
+ #[must_use]
+ pub fn traverse_inclusive(&self) -> Traverse {
Traverse(Some(State {
next: NodeEdge::Start(self.clone()),
next_back: NodeEdge::End(self.clone()),
@@ -144,7 +152,8 @@ impl NodeRef {
/// Return an iterator of the start and end edges of this node’s descendants,
/// in tree order.
#[inline]
- #[must_use] pub fn traverse(&self) -> Traverse {
+ #[must_use]
+ pub fn traverse(&self) -> Traverse {
match (self.first_child(), self.last_child()) {
(Some(first_child), Some(last_child)) => Traverse(Some(State {
next: NodeEdge::Start(first_child),
@@ -163,7 +172,8 @@ impl NodeRef {
/// Return the first inclusive descendants element that match the given selector list.
#[inline]
- #[must_use] pub fn select_first(&self, selectors: &str) -> Option<NodeDataRef<ElementData>> {
+ #[must_use]
+ pub fn select_first(&self, selectors: &str) -> Option<NodeDataRef<ElementData>> {
let mut elements = self.select(selectors).ok()?;
elements.next()
}
diff --git a/crates/kuchiki/src/node_data_ref.rs b/crates/kuchiki/src/node_data_ref.rs
index a5f5cbaa..53dd3b8a 100644
--- a/crates/kuchiki/src/node_data_ref.rs
+++ b/crates/kuchiki/src/node_data_ref.rs
@@ -61,15 +61,18 @@ impl<T> NodeDataRef<T> {
where
F: FnOnce(&Node) -> Option<&T>,
{
- f(&rc).map(|r| std::ptr::from_ref::<T>(r)).map(move |r| NodeDataRef {
- _reference: r,
- _keep_alive: rc,
- })
+ f(&rc)
+ .map(|r| std::ptr::from_ref::<T>(r))
+ .map(move |r| NodeDataRef {
+ _reference: r,
+ _keep_alive: rc,
+ })
}
/// Access the corresponding node.
#[inline]
- #[must_use] pub fn as_node(&self) -> &NodeRef {
+ #[must_use]
+ pub fn as_node(&self) -> &NodeRef {
&self._keep_alive
}
}
@@ -110,7 +113,8 @@ impl<T: fmt::Debug> fmt::Debug for NodeDataRef<T> {
impl NodeDataRef<ElementData> {
/// Return the concatenation of all text nodes in this subtree.
- #[must_use] pub fn text_contents(&self) -> String {
+ #[must_use]
+ pub fn text_contents(&self) -> String {
self.as_node().text_contents()
}
}
diff --git a/crates/kuchiki/src/parser.rs b/crates/kuchiki/src/parser.rs
index 1397d006..a69447c6 100644
--- a/crates/kuchiki/src/parser.rs
+++ b/crates/kuchiki/src/parser.rs
@@ -20,12 +20,14 @@ pub struct ParseOpts {
}
/// Parse an HTML document with html5ever and the default configuration.
-#[must_use] pub fn parse_html() -> html5ever::Parser<Sink> {
+#[must_use]
+pub fn parse_html() -> html5ever::Parser<Sink> {
parse_html_with_options(ParseOpts::default())
}
/// Parse an HTML document with html5ever with custom configuration.
-#[must_use] pub fn parse_html_with_options(opts: ParseOpts) -> html5ever::Parser<Sink> {
+#[must_use]
+pub fn parse_html_with_options(opts: ParseOpts) -> html5ever::Parser<Sink> {
let sink = Sink {
document_node: NodeRef::new_document(),
on_parse_error: opts.on_parse_error,
@@ -38,7 +40,8 @@ pub struct ParseOpts {
}
/// Parse an HTML fragment with html5ever and the default configuration.
-#[must_use] pub fn parse_fragment(ctx_name: QualName, ctx_attr: Vec<Attribute>) -> html5ever::Parser<Sink> {
+#[must_use]
+pub fn parse_fragment(ctx_name: QualName, ctx_attr: Vec<Attribute>) -> html5ever::Parser<Sink> {
parse_fragment_with_options(ParseOpts::default(), ctx_name, ctx_attr)
}
diff --git a/crates/kuchiki/src/select.rs b/crates/kuchiki/src/select.rs
index 9c1a7c73..05cde0e5 100644
--- a/crates/kuchiki/src/select.rs
+++ b/crates/kuchiki/src/select.rs
@@ -50,7 +50,9 @@ impl<'i> Parser<'i> for KuchikiParser {
location: SourceLocation,
name: CowRcStr<'i>,
) -> std::result::Result<PseudoClass, ParseError<'i, SelectorParseErrorKind<'i>>> {
- use self::PseudoClass::{Active, AnyLink, Checked, Disabled, Enabled, Focus, Hover, Indeterminate, Link, Visited};
+ use self::PseudoClass::{
+ Active, AnyLink, Checked, Disabled, Enabled, Focus, Hover, Indeterminate, Link, Visited,
+ };
if name.eq_ignore_ascii_case("any-link") {
Ok(AnyLink)
} else if name.eq_ignore_ascii_case("link") {
@@ -314,7 +316,9 @@ impl selectors::Element for NodeDataRef<ElementData> {
where
F: FnMut(&Self, matching::ElementSelectorFlags),
{
- use self::PseudoClass::{Active, AnyLink, Checked, Disabled, Enabled, Focus, Hover, Indeterminate, Link, Visited};
+ use self::PseudoClass::{
+ Active, AnyLink, Checked, Disabled, Enabled, Focus, Hover, Indeterminate, Link, Visited,
+ };
match *pseudo {
Active | Focus | Hover | Enabled | Disabled | Checked | Indeterminate | Visited => {
false
@@ -359,7 +363,8 @@ impl Selectors {
/// Returns whether the given element matches this list of selectors.
#[inline]
- #[must_use] pub fn matches(&self, element: &NodeDataRef<ElementData>) -> bool {
+ #[must_use]
+ pub fn matches(&self, element: &NodeDataRef<ElementData>) -> bool {
self.0.iter().any(|s| s.matches(element))
}
@@ -379,7 +384,8 @@ impl Selectors {
impl Selector {
/// Returns whether the given element matches this selector.
#[inline]
- #[must_use] pub fn matches(&self, element: &NodeDataRef<ElementData>) -> bool {
+ #[must_use]
+ pub fn matches(&self, element: &NodeDataRef<ElementData>) -> bool {
let mut context = matching::MatchingContext::new(
matching::MatchingMode::Normal,
None,
@@ -390,7 +396,8 @@ impl Selector {
}
/// Return the specificity of this selector.
- #[must_use] pub fn specificity(&self) -> Specificity {
+ #[must_use]
+ pub fn specificity(&self) -> Specificity {
Specificity(self.0.specificity())
}
}
diff --git a/crates/kuchiki/src/tree.rs b/crates/kuchiki/src/tree.rs
index 555e8861..e4d6f7c6 100644
--- a/crates/kuchiki/src/tree.rs
+++ b/crates/kuchiki/src/tree.rs
@@ -126,7 +126,12 @@ pub struct Node {
impl fmt::Debug for Node {
#[inline]
fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
- write!(f, "{:?} @ {:?}", self.data, std::ptr::from_ref::<Node>(self))
+ write!(
+ f,
+ "{:?} @ {:?}",
+ self.data,
+ std::ptr::from_ref::<Node>(self)
+ )
}
}
@@ -267,14 +272,16 @@ impl NodeRef {
/// Create a new document node.
#[inline]
- #[must_use] pub fn new_document() -> NodeRef {
+ #[must_use]
+ pub fn new_document() -> NodeRef {
NodeRef::new(NodeData::Document(DocumentData {
_quirks_mode: Cell::new(QuirksMode::NoQuirks),
}))
}
/// Return the concatenation of all text nodes in this subtree.
- #[must_use] pub fn text_contents(&self) -> String {
+ #[must_use]
+ pub fn text_contents(&self) -> String {
let mut s = String::new();
for text_node in self.inclusive_descendants().text_nodes() {
s.push_str(&text_node.borrow());
diff --git a/crates/optics/src/ast.rs b/crates/optics/src/ast.rs
index c70b2021..bf6f827b 100644
--- a/crates/optics/src/ast.rs
+++ b/crates/optics/src/ast.rs
@@ -77,7 +77,12 @@ impl From<Vec<RawOpticBlock>> for RawOptic {
}
}
- RawOptic { rules, rankings, host_preferences, discard_non_matching }
+ RawOptic {
+ rules,
+ rankings,
+ host_preferences,
+ discard_non_matching,
+ }
}
}
diff --git a/crates/optics/src/lexer.rs b/crates/optics/src/lexer.rs
index c65142bc..084d5ddf 100644
--- a/crates/optics/src/lexer.rs
+++ b/crates/optics/src/lexer.rs
@@ -359,7 +359,10 @@ mod tests {
let lexer = LexerBridge::new(s);
- let result: Vec<Token> = lexer.filter_map(std::result::Result::ok).map(|(_, t, _)| t).collect();
+ let result: Vec<Token> = lexer
+ .filter_map(std::result::Result::ok)
+ .map(|(_, t, _)| t)
+ .collect();
let expected = vec![
Token::Ranking,
@@ -405,7 +408,10 @@ mod tests {
let lexer = LexerBridge::new(s);
- let result: Vec<Token> = lexer.filter_map(std::result::Result::ok).map(|(_, t, _)| t).collect();
+ let result: Vec<Token> = lexer
+ .filter_map(std::result::Result::ok)
+ .map(|(_, t, _)| t)
+ .collect();
let expected = vec![
Token::Ranking,
diff --git a/crates/optics/src/lib.rs b/crates/optics/src/lib.rs
index 783e8892..92bc00eb 100644
--- a/crates/optics/src/lib.rs
+++ b/crates/optics/src/lib.rs
@@ -464,7 +464,8 @@ impl Display for HostRankings {
}
impl HostRankings {
- #[must_use] pub fn rules(&self) -> Rule {
+ #[must_use]
+ pub fn rules(&self) -> Rule {
let matches: Vec<_> = self
.blocked
.iter()
@@ -491,7 +492,8 @@ impl HostRankings {
}
}
- #[must_use] pub fn into_optic(self) -> Optic {
+ #[must_use]
+ pub fn into_optic(self) -> Optic {
Optic {
host_rankings: self,
..Default::default()
diff --git a/crates/zimba/src/lib.rs b/crates/zimba/src/lib.rs
index 16b8ea74..809bf314 100644
--- a/crates/zimba/src/lib.rs
+++ b/crates/zimba/src/lib.rs
@@ -490,7 +490,8 @@ impl Cluster {
})
}
- #[must_use] pub fn get_blob(&self, blob_number: usize) -> Option<&[u8]> {
+ #[must_use]
+ pub fn get_blob(&self, blob_number: usize) -> Option<&[u8]> {
if self.blob_offsets.is_empty() {
return None;
}
@@ -574,19 +575,23 @@ impl ZimFile {
Ok(Some(Cluster::from_bytes(&self.mmap[pointer..])?))
}
- #[must_use] pub fn mime_types(&self) -> &MimeTypes {
+ #[must_use]
+ pub fn mime_types(&self) -> &MimeTypes {
&self.mime_types
}
- #[must_use] pub fn url_pointers(&self) -> &UrlPointerList {
+ #[must_use]
+ pub fn url_pointers(&self) -> &UrlPointerList {
&self.url_pointers
}
- #[must_use] pub fn title_pointers(&self) -> &TitlePointerList {
+ #[must_use]
+ pub fn title_pointers(&self) -> &TitlePointerList {
&self.title_pointers
}
- #[must_use] pub fn dir_entries(&self) -> DirEntryIterator<'_> {
+ #[must_use]
+ pub fn dir_entries(&self) -> DirEntryIterator<'_> {
DirEntryIterator::new(&self.mmap, &self.url_pointers)
}
diff --git a/crates/zimba/src/wiki.rs b/crates/zimba/src/wiki.rs
index 390d18ae..d063998f 100644
--- a/crates/zimba/src/wiki.rs
+++ b/crates/zimba/src/wiki.rs
@@ -210,7 +210,8 @@ pub struct Image {
pub content: Vec<u8>,
}
impl Image {
- #[must_use] pub fn bytes(&self) -> &[u8] {
+ #[must_use]
+ pub fn bytes(&self) -> &[u8] {
&self.content
}
}