web search done right
Distributed in-memory key/value store for mapreduce (#181)
Mikkel Denker37b6c7d86c34
diff --git a/Cargo.lock b/Cargo.lockindex 3ba9853e..35d4a147 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -168,6 +168,15 @@ dependencies = ["windows-sys 0.52.0",]+[[package]]+name = "anyerror"+version = "0.1.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bcd04a72664a65fb9adeae7ced0c98efd68a2b7a45adda8319b3bb36538404b8"+dependencies = [+ "serde",+]+[[package]]name = "anyhow"version = "1.0.80"@@ -531,6 +540,16 @@ version = "3.15.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b"+[[package]]+name = "byte-unit"+version = "4.0.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "da78b32057b8fdfc352504708feeba7216dcd65a2c9ab02978cbd288d1279b6c"+dependencies = [+ "serde",+ "utf8-width",+]+[[package]]name = "bytecount"version = "0.6.7"@@ -2997,6 +3016,39 @@ version = "0.3.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"+[[package]]+name = "openraft"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "863a31dad5014893c47cdb4291c1f14cd901304b548c739004307fdd2cd37454"+dependencies = [+ "anyerror",+ "byte-unit",+ "clap",+ "derive_more",+ "futures",+ "maplit",+ "openraft-macros",+ "rand 0.8.5",+ "serde",+ "thiserror",+ "tokio",+ "tracing",+ "tracing-futures",+ "validit",+]++[[package]]+name = "openraft-macros"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c5f384d67b15e283f1cf6bb778c8f29ab2f8e8729b4bc82e8a1c21f2bf767d60"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.52",+]+[[package]]name = "openssl"version = "0.10.64"@@ -4511,6 +4563,7 @@ dependencies = ["min-max-heap","num_cpus","once_cell",+ "openraft","optics","proptest","proptest-derive",@@ -4542,6 +4595,7 @@ dependencies = ["tower-http","tracing","tracing-subscriber",+ "tracing-test","url","urlencoding","utoipa",@@ -5232,6 +5286,16 @@ dependencies = ["valuable",]+[[package]]+name = "tracing-futures"+version = "0.2.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"+dependencies = [+ "pin-project",+ "tracing",+]+[[package]]name = "tracing-log"version = "0.2.0"@@ -5261,6 +5325,29 @@ dependencies = ["tracing-log",]+[[package]]+name = "tracing-test"+version = "0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3a2c0ff408fe918a94c428a3f2ad04e4afd5c95bbc08fcf868eff750c15728a4"+dependencies = [+ "lazy_static",+ "tracing-core",+ "tracing-subscriber",+ "tracing-test-macro",+]++[[package]]+name = "tracing-test-macro"+version = "0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "258bc1c4f8e2e73a977812ab339d503e6feeb92700f6d07a6de4d321522d5c08"+dependencies = [+ "lazy_static",+ "quote",+ "syn 1.0.109",+]+[[package]]name = "try-lock"version = "0.2.5"@@ -5398,6 +5485,12 @@ version = "1.0.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba"+[[package]]+name = "utf8-width"+version = "0.1.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3"+[[package]]name = "utf8parse"version = "0.2.1"@@ -5455,6 +5548,15 @@ dependencies = ["serde",]+[[package]]+name = "validit"+version = "0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a1fad49f3eae9c160c06b4d49700a99e75817f127cf856e494b56d5e23170020"+dependencies = [+ "anyerror",+]+[[package]]name = "valuable"version = "0.1.0"diff --git a/Cargo.toml b/Cargo.tomlindex 284a21cd..94d7587c 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -1,10 +1,5 @@[workspace]-members = [- "crates/core",- "crates/optics",- "crates/kuchiki",- "crates/zimba",-]+members = ["crates/core", "crates/optics", "crates/kuchiki", "crates/zimba"]resolver = "2"[profile.release]@@ -16,51 +11,52 @@ debug = true[workspace.dependencies]aes-gcm = "0.10.2"-anyhow = {version = "1.0.72", features = ["backtrace"]}+anyhow = { version = "1.0.72", features = ["backtrace"] }async-channel = "1.8.0"async-stream = "0.3.3"axum = "0.7.2"-axum-extra = {version = "0.9.0"}+axum-extra = { version = "0.9.0" }axum-macros = "0.4.0"base64 = "0.21.4"bincode = "1.3.3"bindgen = "0.69.2"bitvec = "1.0.1"bytecount = "0.6.7"-bytemuck = {version = "1.13.1", features = ["derive"]}+bytemuck = { version = "1.13.1", features = ["derive"] }byteorder = "1.4.3"bzip2 = "0.4.4"candle-core = "0.3.3"candle-nn = "0.3.3"+openraft = { version = "0.9.1", features = ["storage-v2", "serde"] }candle-transformers = "0.3.3"-cc = {version = "1", features = ["parallel"]}+cc = { version = "1", features = ["parallel"] }chardetng = "0.1.17"chitchat = "0.5.0"-chrono = {version = "0.4.23", features = ["serde"]}-clap = {version = "4.4.6", features = ["derive"]}+chrono = { version = "0.4.23", features = ["serde"] }+clap = { version = "4.4.6", features = ["derive"] }cmake = "0.1"criterion = "0.5.1"crossbeam-channel = "0.5.6"csv = "1.1.6"-dashmap = {version = "5.4.0", features = ["rayon"]}+dashmap = { version = "5.4.0", features = ["rayon"] }encoding_rs = "0.8.31"eventsource-stream = "0.2.3"fend-core = "1.2.2"flate2 = "1.0.28"fnv = "1.0.3"-fst = {version = "0.4.7", features = ["levenshtein"]}+fst = { version = "0.4.7", features = ["levenshtein"] }futures = "0.3.21"-half = {version = "2.2.1", features = ["serde"]}-hashbrown = {version = "0.14.0", features = ["serde"]}+half = { version = "2.2.1", features = ["serde"] }+hashbrown = { version = "0.14.0", features = ["serde"] }http = "1.0.0"image = "0.24.3"-indicatif = {version = "0.17.7", features = ["rayon"]}+indicatif = { version = "0.17.7", features = ["rayon"] }insta = "1.31"itertools = "0.11.0"-lalrpop = {version = "0.20.0", features = ["lexer"]}-lalrpop-util = {version = "0.20.0", features = ["lexer"]}+lalrpop = { version = "0.20.0", features = ["lexer"] }+lalrpop-util = { version = "0.20.0", features = ["lexer"] }libc = "0.2.142"-log = {version = "0.4", features = ["release_max_level_info"]}+log = { version = "0.4", features = ["release_max_level_info"] }logos = "0.13.0"lz-str = "0.2.1"lz4_flex = "0.11.1"@@ -80,41 +76,42 @@ quick-xml = "0.30.0"rand = "0.8.5"rayon = "1.5.3"regex = "1.6.0"-reqwest = {version = "0.11.16", features = ["blocking", "stream", "json"]}+reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }ring = "0.17.3"rio_api = "0.8.4"rio_turtle = "0.8.4"robotstxt-with-cache = "0.4.0"-rocksdb = {version = "0.21.0", features = ["default", "jemalloc", "io-uring"]}-rusqlite = {version = "0.29.0", features = [- "bundled",- "modern-full",- "buildtime_bindgen",-]}-rust-s3 = {version = "0.33.0", features = ["blocking", "tokio"]}+rocksdb = { version = "0.21.0", features = ["default", "jemalloc", "io-uring"] }+rusqlite = { version = "0.29.0", features = [+ "bundled",+ "modern-full",+ "buildtime_bindgen",+] }+rust-s3 = { version = "0.33.0", features = ["blocking", "tokio"] }rust-stemmers = "1.2.0"safetensors = "0.3.1"-scylla = {version = "0.12.0", features = ["chrono"]}-serde = {version = "1.0.137", features = ["rc", "derive"]}+scylla = { version = "0.12.0", features = ["chrono"] }+serde = { version = "1.0.137", features = ["rc", "derive"] }serde_json = "1.0.81"serde_urlencoded = "0.7.1"-tantivy = {git = "https://github.com/quickwit-oss/tantivy", rev = "182f58cea"}+tantivy = { git = "https://github.com/quickwit-oss/tantivy", rev = "182f58cea" }thiserror = "1.0.31"tikv-jemallocator = "0.5"tokenizers = "0.13.2"-tokio = {version = "1.23.1", features = ["full"]}+tokio = { version = "1.23.1", features = ["full"] }tokio-stream = "0.1.11"toml = "0.8.2"-tower-http = {version = "0.5.0", features = ["compression-gzip", "cors"]}-tracing = {version = "0.1.34", features = ["release_max_level_info"]}-tracing-subscriber = {version = "0.3.11", features = ["env-filter"]}-url = {version = "2.4.0", features = ["serde"]}+tower-http = { version = "0.5.0", features = ["compression-gzip", "cors"] }+tracing = { version = "0.1.34", features = ["release_max_level_info"] }+tracing-subscriber = { version = "0.3.11", features = ["env-filter"] }+url = { version = "2.4.0", features = ["serde"] }urlencoding = "2.1.2"-utoipa = {version = "4.0.0", features = ["axum_extras"]}-utoipa-swagger-ui = {version = "5.0.0", features = ["axum"]}+utoipa = { version = "4.0.0", features = ["axum_extras"] }+utoipa-swagger-ui = { version = "5.0.0", features = ["axum"] }uuid = "1.1.2"-whatlang = {version = "0.16.0", features = ["serde"]}+whatlang = { version = "0.16.0", features = ["serde"] }zstd = "0.13"+tracing-test = "0.2.4"[profile.test.package]flate2.opt-level = 3diff --git a/assets/licenses.html b/assets/licenses.htmlindex a2e6042f..f9d0aa04 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -44,8 +44,8 @@<h2>Overview of licenses:</h2><ul class="licenses-overview">- <li><a href="#Apache-2.0">Apache License 2.0</a> (390)</li>- <li><a href="#MIT">MIT License</a> (175)</li>+ <li><a href="#Apache-2.0">Apache License 2.0</a> (394)</li>+ <li><a href="#MIT">MIT License</a> (180)</li><li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li><li><a href="#BSD-3-Clause">BSD 3-Clause "New" or "Revised" License</a> (8)</li><li><a href="#Unicode-DFS-2016">Unicode License Agreement - Data Files and Software (2016)</a> (5)</li>@@ -2595,6 +2595,7 @@<h3 id="Apache-2.0">Apache License 2.0</h3><h4>Used by:</h4><ul class="license-used-by">+ <li><a href=" https://github.com/drmingdrmer/anyerror ">anyerror 0.1.12</a></li><li><a href=" https://github.com/epwalsh/rust-cached-path ">cached-path 0.6.1</a></li><li><a href=" https://github.com/huggingface/candle ">candle-core 0.3.3</a></li><li><a href=" https://github.com/enarx/ciborium ">ciborium-io 0.2.2</a></li>@@ -7157,6 +7158,7 @@ limitations under the License.</pre><li><a href=" https://github.com/unicode-rs/unicode-xid ">unicode-xid 0.2.4</a></li><li><a href=" https://github.com/servo/rust-url ">url 2.5.0</a></li><li><a href=" https://github.com/uuid-rs/uuid ">uuid 1.7.0</a></li>+ <li><a href=" https://github.com/drmingdrmer/validit ">validit 0.2.4</a></li><li><a href=" https://github.com/SergioBenitez/version_check ">version_check 0.9.4</a></li><li><a href=" https://github.com/alexcrichton/wait-timeout ">wait-timeout 0.2.0</a></li><li><a href=" https://github.com/bytecodealliance/wasi ">wasi 0.11.0+wasi-snapshot-preview1</a></li>@@ -9898,6 +9900,8 @@ limitations under the License.<li><a href=" https://github.com/reem/rust-mac.git ">mac 0.1.1</a></li><li><a href=" https://github.com/stainless-steel/md5 ">md5 0.7.0</a></li><li><a href=" https://github.com/faern/oneshot ">oneshot 0.1.6</a></li>+ <li><a href=" https://github.com/datafuselabs/openraft ">openraft-macros 0.9.1</a></li>+ <li><a href=" https://github.com/datafuselabs/openraft ">openraft 0.9.1</a></li><li><a href=" https://github.com/sfackler/rust-openssl ">openssl 0.10.64</a></li><li><a href=" https://github.com/oxigraph/rio ">rio_api 0.8.4</a></li><li><a href=" https://github.com/oxigraph/rio ">rio_turtle 0.8.4</a></li>@@ -12510,6 +12514,7 @@ DEALINGS IN THE SOFTWARE.<li><a href=" https://github.com/tokio-rs/tls ">tokio-native-tls 0.3.1</a></li><li><a href=" https://github.com/tokio-rs/tracing ">tracing-attributes 0.1.27</a></li><li><a href=" https://github.com/tokio-rs/tracing ">tracing-core 0.1.32</a></li>+ <li><a href=" https://github.com/tokio-rs/tracing ">tracing-futures 0.2.5</a></li><li><a href=" https://github.com/tokio-rs/tracing ">tracing-log 0.2.0</a></li><li><a href=" https://github.com/tokio-rs/tracing ">tracing-subscriber 0.3.18</a></li><li><a href=" https://github.com/tokio-rs/tracing ">tracing 0.1.40</a></li>@@ -13024,6 +13029,35 @@ furnished to do so, subject to the following conditions:The above copyright notice and this permission notice shall be included in allcopies or substantial portions of the Software.+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.+</pre>+ </li>+ <li class="license">+ <h3 id="MIT">MIT License</h3>+ <h4>Used by:</h4>+ <ul class="license-used-by">+ <li><a href=" https://github.com/magiclen/byte-unit ">byte-unit 4.0.19</a></li>+ </ul>+ <pre class="license-text">MIT License++Copyright (c) 2018 magiclen.org (Ron Li)++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ORIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE@@ -13201,6 +13235,35 @@ furnished to do so, subject to the following conditions:The above copyright notice and this permission notice shall be included in allcopies or substantial portions of the Software.+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.+</pre>+ </li>+ <li class="license">+ <h3 id="MIT">MIT License</h3>+ <h4>Used by:</h4>+ <ul class="license-used-by">+ <li><a href=" https://github.com/magiclen/utf8-width ">utf8-width 0.1.7</a></li>+ </ul>+ <pre class="license-text">MIT License++Copyright (c) 2020 magiclen.org (Ron Li)++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ORIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE@@ -13402,6 +13465,8 @@ SOFTWARE.<li><a href=" https://github.com/plotters-rs/plotters ">plotters 0.3.5</a></li><li><a href=" https://github.com/durch/rust-s3 ">rust-s3 0.33.0</a></li><li><a href=" https://github.com/tokio-rs/tokio ">tokio-macros 2.2.0</a></li>+ <li><a href=" https://github.com/dbrgn/tracing-test ">tracing-test-macro 0.2.4</a></li>+ <li><a href=" https://github.com/dbrgn/tracing-test ">tracing-test 0.2.4</a></li><li><a href=" https://github.com/tokio-rs/valuable ">valuable 0.1.0</a></li><li><a href=" https://github.com/greyblake/whatlang-rs ">whatlang 0.16.4</a></li></ul>diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 8e65aaa6..5c15455f 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -20,101 +20,103 @@ name = "stract"path = "src/main.rs"[dependencies]-aes-gcm = {workspace = true}-anyhow = {workspace = true}-async-channel = {workspace = true}-async-stream = {workspace = true}-axum = {workspace = true}-axum-extra = {workspace = true}-axum-macros = {workspace = true}-base64 = {workspace = true}-bincode = {workspace = true}-bitvec = {workspace = true}-bytecount = {workspace = true}-bytemuck = {workspace = true}-byteorder = {workspace = true}-bzip2 = {workspace = true}-candle-core = {workspace = true}-candle-nn = {workspace = true}-candle-transformers = {workspace = true}-chardetng = {workspace = true}-chitchat = {workspace = true}-chrono = {workspace = true}-clap = {workspace = true}-crossbeam-channel = {workspace = true}-csv = {workspace = true}-dashmap = {workspace = true}-encoding_rs = {workspace = true}-eventsource-stream = {workspace = true}-fend-core = {workspace = true}-flate2 = {workspace = true}-fnv = {workspace = true}-fst = {workspace = true}-futures = {workspace = true}-half = {workspace = true}-hashbrown = {workspace = true}-http = {workspace = true}-image = {workspace = true}-indicatif = {workspace = true}-itertools = {workspace = true}-kuchiki = {path = "../kuchiki"}-libc = {workspace = true}-log = {workspace = true}-logos = {workspace = true}-lz-str = {workspace = true}-lz4_flex = {workspace = true}-md5 = {workspace = true}-memmap = {workspace = true}-memmap2 = {workspace = true}-mime = {workspace = true}-min-max-heap = {workspace = true}-num_cpus = {workspace = true}-once_cell = {workspace = true}-optics = {path = "../optics"}-publicsuffix = {workspace = true}-quick-xml = {workspace = true}-rand = {workspace = true}-rayon = {workspace = true}-regex = {workspace = true}-reqwest = {workspace = true}-ring = {workspace = true}-rio_api = {workspace = true}-rio_turtle = {workspace = true}-robotstxt-with-cache = {workspace = true}-rocksdb = {workspace = true}-rust-s3 = {workspace = true}-rust-stemmers = {workspace = true}-safetensors = {workspace = true}-scylla = {workspace = true}-serde = {workspace = true}-serde_json = {workspace = true}-serde_urlencoded = {workspace = true}-tantivy = {workspace = true}-thiserror = {workspace = true}-tokenizers = {workspace = true}-tokio = {workspace = true}-tokio-stream = {workspace = true}-toml = {workspace = true}-tower-http = {workspace = true}-tracing = {workspace = true}-tracing-subscriber = {workspace = true}-url = {workspace = true}-urlencoding = {workspace = true}-utoipa = {workspace = true}-utoipa-swagger-ui = {workspace = true}-uuid = {workspace = true}-whatlang = {workspace = true}-zimba = {path = "../zimba"}+aes-gcm = { workspace = true }+anyhow = { workspace = true }+async-channel = { workspace = true }+async-stream = { workspace = true }+axum = { workspace = true }+axum-extra = { workspace = true }+axum-macros = { workspace = true }+base64 = { workspace = true }+bincode = { workspace = true }+bitvec = { workspace = true }+bytecount = { workspace = true }+bytemuck = { workspace = true }+byteorder = { workspace = true }+bzip2 = { workspace = true }+candle-core = { workspace = true }+candle-nn = { workspace = true }+candle-transformers = { workspace = true }+chardetng = { workspace = true }+chitchat = { workspace = true }+chrono = { workspace = true }+clap = { workspace = true }+crossbeam-channel = { workspace = true }+csv = { workspace = true }+dashmap = { workspace = true }+encoding_rs = { workspace = true }+eventsource-stream = { workspace = true }+fend-core = { workspace = true }+flate2 = { workspace = true }+fnv = { workspace = true }+fst = { workspace = true }+futures = { workspace = true }+half = { workspace = true }+hashbrown = { workspace = true }+http = { workspace = true }+image = { workspace = true }+indicatif = { workspace = true }+itertools = { workspace = true }+kuchiki = { path = "../kuchiki" }+libc = { workspace = true }+log = { workspace = true }+logos = { workspace = true }+lz-str = { workspace = true }+lz4_flex = { workspace = true }+md5 = { workspace = true }+memmap = { workspace = true }+memmap2 = { workspace = true }+mime = { workspace = true }+min-max-heap = { workspace = true }+num_cpus = { workspace = true }+once_cell = { workspace = true }+openraft = { workspace = true }+optics = { path = "../optics" }+publicsuffix = { workspace = true }+quick-xml = { workspace = true }+rand = { workspace = true }+rayon = { workspace = true }+regex = { workspace = true }+reqwest = { workspace = true }+ring = { workspace = true }+rio_api = { workspace = true }+rio_turtle = { workspace = true }+robotstxt-with-cache = { workspace = true }+rocksdb = { workspace = true }+rust-s3 = { workspace = true }+rust-stemmers = { workspace = true }+safetensors = { workspace = true }+scylla = { workspace = true }+serde = { workspace = true }+serde_json = { workspace = true }+serde_urlencoded = { workspace = true }+tantivy = { workspace = true }+thiserror = { workspace = true }+tokenizers = { workspace = true }+tokio = { workspace = true }+tokio-stream = { workspace = true }+toml = { workspace = true }+tower-http = { workspace = true }+tracing = { workspace = true }+tracing-subscriber = { workspace = true }+url = { workspace = true }+urlencoding = { workspace = true }+utoipa = { workspace = true }+utoipa-swagger-ui = { workspace = true }+uuid = { workspace = true }+whatlang = { workspace = true }+zimba = { path = "../zimba" }[target.'cfg(not(target_env = "msvc"))'.dependencies]-tikv-jemallocator = {workspace = true}+tikv-jemallocator = { workspace = true }[dev-dependencies]-criterion = {workspace = true}-insta = {workspace = true}-maplit = {workspace = true}-proptest = {workspace = true}-proptest-derive = {workspace = true}+criterion = { workspace = true }+insta = { workspace = true }+maplit = { workspace = true }+proptest = { workspace = true }+proptest-derive = { workspace = true }+tracing-test = { workspace = true }[[bench]]harness = falsediff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rsindex 1c04cb84..04ef9fd0 100644--- a/crates/core/examples/search_preindexed.rs+++ b/crates/core/examples/search_preindexed.rs@@ -6,11 +6,12 @@ use stract::{ApiConfig, ApiThresholds, CollectorConfig, CorrectionConfig, LLMConfig, SnippetConfig,WidgetsConfig,},+ distributed::member::ShardId,image_store::Image,index::Index,inverted_index::RetrievedWebpage,ranking::{inbound_similarity::InboundSimilarity, pipeline::PrecisionRankingWebpage},- searcher::{api::ApiSearcher, live::LiveSearcher, LocalSearcher, SearchQuery, ShardId},+ searcher::{api::ApiSearcher, live::LiveSearcher, LocalSearcher, SearchQuery},Result,};struct Searcher(LocalSearcher<Index>);diff --git a/crates/core/src/api/hosts.rs b/crates/core/src/api/hosts.rsindex 4e53d9df..363fd8a7 100644--- a/crates/core/src/api/hosts.rs+++ b/crates/core/src/api/hosts.rs@@ -14,7 +14,11 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use axum::{body::Body, extract, response::{IntoResponse, Response}};+use axum::{+ body::Body,+ extract,+ response::{IntoResponse, Response},+};use http::StatusCode;use optics::{HostRankings, Optic};use utoipa::ToSchema;diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rsindex 771b7ac1..9cddf89a 100644--- a/crates/core/src/api/webgraph.rs+++ b/crates/core/src/api/webgraph.rs@@ -95,7 +95,7 @@ pub mod host {.with_limit(Duration::from_millis(200)).take(5);- let conn = sonic::service::ResilientConnection::create_with_timeout(+ let conn = sonic::service::Connection::create_with_timeout_retry(host,Duration::from_secs(30),retry,@@ -142,7 +142,7 @@ pub mod host {.with_limit(Duration::from_millis(200)).take(5);- let conn = sonic::service::ResilientConnection::create_with_timeout(+ let conn = sonic::service::Connection::create_with_timeout_retry(host,Duration::from_secs(30),retry,@@ -286,12 +286,9 @@ async fn ingoing_links(.with_limit(Duration::from_millis(200)).take(5);- let conn = sonic::service::ResilientConnection::create_with_timeout(- host,- Duration::from_secs(30),- retry,- )- .await?;+ let conn =+ sonic::service::Connection::create_with_timeout_retry(host, Duration::from_secs(30), retry)+ .await?;Ok(conn.send_with_timeout(@@ -318,12 +315,9 @@ async fn outgoing_links(.with_limit(Duration::from_millis(200)).take(5);- let conn = sonic::service::ResilientConnection::create_with_timeout(- host,- Duration::from_secs(30),- retry,- )- .await?;+ let conn =+ sonic::service::Connection::create_with_timeout_retry(host, Duration::from_secs(30), retry)+ .await?;Ok(conn.send_with_timeout(diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex cafecd78..4a246cf3 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -17,8 +17,8 @@pub mod defaults;use super::Result;+use crate::distributed::member::ShardId;use crate::feed::scheduler::SplitId;-use crate::searcher::ShardId;use serde::{Deserialize, Serialize};use std::fs::File;use std::io::{self, BufRead};diff --git a/crates/core/src/crawler/router.rs b/crates/core/src/crawler/router.rsindex fa4fcec2..a0a14c8d 100644--- a/crates/core/src/crawler/router.rs+++ b/crates/core/src/crawler/router.rs@@ -15,10 +15,10 @@ struct RemoteCoordinator {}impl RemoteCoordinator {- async fn conn(&self) -> Result<sonic::service::ResilientConnection<CoordinatorService>> {+ async fn conn(&self) -> Result<sonic::service::Connection<CoordinatorService>> {let retry = ExponentialBackoff::from_millis(1_000).with_limit(Duration::from_secs(10));- Ok(sonic::service::ResilientConnection::create_with_timeout(+ Ok(sonic::service::Connection::create_with_timeout_retry(self.addr,Duration::from_secs(60),retry,diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex 089d851b..d96012a0 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -76,12 +76,12 @@ impl WorkerThread {})}- async fn router_conn(&self) -> Result<sonic::service::ResilientConnection<RouterService>> {+ async fn router_conn(&self) -> Result<sonic::service::Connection<RouterService>> {let retry = ExponentialBackoff::from_millis(1_000).with_limit(Duration::from_secs(10));let router = *self.router_hosts.choose(&mut rand::thread_rng()).unwrap();- Ok(sonic::service::ResilientConnection::create_with_timeout(+ Ok(sonic::service::Connection::create_with_timeout_retry(router,Duration::from_secs(90),retry,diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rsindex 0bb9065a..ce1a32d9 100644--- a/crates/core/src/distributed/member.rs+++ b/crates/core/src/distributed/member.rs@@ -18,7 +18,22 @@ use std::net::SocketAddr;use serde::{Deserialize, Serialize};-use crate::{config::WebgraphGranularity, searcher::ShardId};+use crate::config::WebgraphGranularity;++#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy, Debug, PartialOrd, Ord)]+pub struct ShardId(u64);++impl ShardId {+ pub fn new(id: u64) -> Self {+ Self(id)+ }+}++impl From<u64> for ShardId {+ fn from(id: u64) -> Self {+ Self(id)+ }+}#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]pub enum Service {@@ -40,6 +55,10 @@ pub enum Service {host: SocketAddr,granularity: WebgraphGranularity,},+ Dht {+ host: SocketAddr,+ shard: ShardId,+ },}#[derive(PartialEq, Eq, Hash, Clone, Debug)]diff --git a/crates/core/src/distributed/retry_strategy.rs b/crates/core/src/distributed/retry_strategy.rsindex 04d9ea9e..ccc75a67 100644--- a/crates/core/src/distributed/retry_strategy.rs+++ b/crates/core/src/distributed/retry_strategy.rs@@ -1,7 +1,8 @@/*- * This file is copied from [tokio_retry](https://github.com/srijs/rust-tokio-retry/blob/master/src/strategy/exponential_backoff.rs)- * as it seems silly to pull-in a new dependency for a single file.+ * This file is partially copied from [tokio_retry](https://github.com/srijs/rust-tokio-retry/blob/master/src/strategy/exponential_backoff.rs)+ * and modified as it seems silly to pull-in a new dependency for a single file.* */+use rand::Rng;use std::iter::Iterator;use std::time::Duration;use std::u64::MAX as U64_MAX;@@ -66,6 +67,28 @@ impl Iterator for ExponentialBackoff {}}+pub struct RandomBackoff {+ min: Duration,+ max: Duration,+}++impl RandomBackoff {+ pub fn new(min: Duration, max: Duration) -> RandomBackoff {+ RandomBackoff { min, max }+ }+}++impl Iterator for RandomBackoff {+ type Item = Duration;++ fn next(&mut self) -> Option<Duration> {+ let mut rng = rand::thread_rng();+ let range = self.max - self.min;+ let duration = rng.gen_range(0..range.as_millis()) + self.min.as_millis();+ Some(Duration::from_millis(duration as u64))+ }+}+#[cfg(test)]mod tests {use super::*;diff --git a/crates/core/src/distributed/sonic/mod.rs b/crates/core/src/distributed/sonic/mod.rsindex d6b6f382..203d5e4f 100644--- a/crates/core/src/distributed/sonic/mod.rs+++ b/crates/core/src/distributed/sonic/mod.rs@@ -49,8 +49,11 @@ pub enum Error {#[error("The request could not be processed")]BadRequest,- #[error("Other")]- Other(#[from] anyhow::Error),+ #[error("The body size ({body_size}) is larger than the maximum allowed ({max_size})")]+ BodyTooLarge { body_size: usize, max_size: usize },++ #[error("An application error occurred: {0}")]+ Application(#[from] anyhow::Error),}pub struct Connection<Req, Res> {@@ -83,6 +86,29 @@ where}}+ pub async fn create_with_timeout_retry(+ server: impl ToSocketAddrs + Clone,+ timeout: Duration,+ retry: impl Iterator<Item = Duration>,+ ) -> Result<Self> {+ let mut conn = Connection::create_with_timeout(server.clone(), timeout).await;+ let mut retry = retry;++ loop {+ match conn {+ Ok(conn) => return Ok(conn),+ Err(_) => {+ if let Some(timeout) = retry.next() {+ tokio::time::sleep(timeout).await;+ conn = Connection::create_with_timeout(server.clone(), timeout).await;+ } else {+ return Err(Error::ConnectionTimeout);+ }+ }+ }+ }+ }+async fn send_without_timeout(mut self, request: &Req) -> Result<Res> {let bytes = bincode::serialize(&request).unwrap();@@ -104,11 +130,10 @@ wherelet header: Header = *bytemuck::from_bytes(&header_buf);if header.body_size > MAX_BODY_SIZE_BYTES {- return Err(Error::Other(anyhow::anyhow!(- "body size too large: {} (max: {})",- header.body_size,- MAX_BODY_SIZE_BYTES- )));+ return Err(Error::BodyTooLarge {+ body_size: header.body_size,+ max_size: MAX_BODY_SIZE_BYTES,+ });}let mut buf = vec![0; header.body_size];@@ -140,7 +165,7 @@ struct Header {}pub struct Server<Req, Res> {- pub(super) listener: TcpListener,+ listener: TcpListener,marker: PhantomData<(Req, Res)>,}@@ -162,11 +187,10 @@ wherelet header: Header = *bytemuck::from_bytes(&header_buf);if header.body_size > MAX_BODY_SIZE_BYTES {- return Err(Error::Other(anyhow::anyhow!(- "body size too large: {} (max: {})",- header.body_size,- MAX_BODY_SIZE_BYTES- )));+ return Err(Error::BodyTooLarge {+ body_size: header.body_size,+ max_size: MAX_BODY_SIZE_BYTES,+ });}let mut buf = vec![0; header.body_size];@@ -239,43 +263,6 @@ where}}-pub struct ResilientConnection<Req, Res> {- conn: Connection<Req, Res>,-}--impl<Req, Res> ResilientConnection<Req, Res>-where- Req: Serialize,- Res: DeserializeOwned,-{- pub async fn create_with_timeout(- server: impl ToSocketAddrs + Clone,- timeout: Duration,- retry: impl Iterator<Item = Duration>,- ) -> Result<Self> {- let mut conn = Connection::create_with_timeout(server.clone(), timeout).await;- let mut retry = retry;-- loop {- match conn {- Ok(conn) => return Ok(ResilientConnection { conn }),- Err(_) => {- if let Some(timeout) = retry.next() {- tokio::time::sleep(timeout).await;- conn = Connection::create_with_timeout(server.clone(), timeout).await;- } else {- return Err(Error::ConnectionTimeout);- }- }- }- }- }-- pub async fn send_with_timeout(self, request: &Req, timeout: Duration) -> Result<Res> {- self.conn.send_with_timeout(request, timeout).await- }-}-#[cfg(test)]mod tests {use std::{collections::HashMap, future::Future};@@ -284,6 +271,8 @@ mod tests {use proptest_derive::Arbitrary;use serde::Deserialize;+ use crate::free_socket_addr;+use super::*;fn fixture<@@ -302,8 +291,8 @@ mod tests {.build().unwrap().block_on(async move {- let server = Server::bind(("127.0.0.1", 0)).await.unwrap();- let addr = server.listener.local_addr().unwrap();+ let addr = free_socket_addr();+ let server = Server::bind(addr.clone()).await.unwrap();let connection = Connection::create(addr).await.unwrap();let svr_task = tokio::spawn(async move { svr_fn(server).await });diff --git a/crates/core/src/distributed/sonic/replication.rs b/crates/core/src/distributed/sonic/replication.rsindex c1f36bbf..0ca6e34d 100644--- a/crates/core/src/distributed/sonic/replication.rs+++ b/crates/core/src/distributed/sonic/replication.rs@@ -21,17 +21,30 @@ use super::Result;use crate::distributed::{retry_strategy::ExponentialBackoff, sonic};use std::{net::SocketAddr, time::Duration};-#[derive(Debug, Clone)]+#[derive(Debug)]pub struct RemoteClient<S: sonic::service::Service> {addr: SocketAddr,_phantom: std::marker::PhantomData<S>,}+impl<S> Clone for RemoteClient<S>+where+ S: sonic::service::Service,+{+ fn clone(&self) -> Self {+ Self::create(self.addr)+ }+}+impl<S> RemoteClient<S>whereS: sonic::service::Service,{pub fn new(addr: SocketAddr) -> Self {+ Self::create(addr)+ }++ pub fn create(addr: SocketAddr) -> Self {Self {addr,_phantom: std::marker::PhantomData,@@ -43,12 +56,12 @@ impl<S> RemoteClient<S>whereS: sonic::service::Service,{- async fn conn(&self) -> Result<sonic::service::ResilientConnection<S>> {+ pub async fn conn(&self) -> Result<sonic::service::Connection<S>> {let retry = ExponentialBackoff::from_millis(30).with_limit(Duration::from_millis(200)).take(5);- sonic::service::ResilientConnection::create_with_timeout(+ sonic::service::Connection::create_with_timeout_retry(self.addr,Duration::from_secs(30),retry,@@ -56,9 +69,43 @@ where.await}- async fn send<R: sonic::service::Wrapper<S>>(&self, req: &R) -> Result<R::Response> {+ pub async fn send<R: sonic::service::Wrapper<S>>(&self, req: &R) -> Result<R::Response> {+ self.send_with_timeout_retry(+ req,+ Duration::from_secs(60),+ ExponentialBackoff::from_millis(500).with_limit(Duration::from_secs(3)),+ )+ .await+ }++ pub async fn send_with_timeout<R: sonic::service::Wrapper<S>>(+ &self,+ req: &R,+ timeout: Duration,+ ) -> Result<R::Response> {let conn = self.conn().await?;- conn.send_with_timeout(req, Duration::from_secs(60)).await+ conn.send_with_timeout(req, timeout).await+ }++ pub async fn send_with_timeout_retry<R: sonic::service::Wrapper<S>>(+ &self,+ req: &R,+ timeout: Duration,+ retry: impl Iterator<Item = Duration>,+ ) -> Result<R::Response> {+ let mut er = None;+ for backoff in retry {+ match self.send_with_timeout(req, timeout).await {+ Ok(r) => return Ok(r),+ Err(e) => {+ tracing::error!("Failed to send request: {:?}", e);+ er = Some(e);+ tokio::time::sleep(backoff).await;+ }+ }+ }++ Err(er.unwrap())}}@@ -101,10 +148,10 @@ whereSelf { clients }}- pub async fn send<Req, Rep>(&self, req: &Req, selector: &Rep) -> Result<Vec<Req::Response>>+ pub async fn send<Req, Sel>(&self, req: &Req, selector: &Sel) -> Result<Vec<Req::Response>>whereReq: sonic::service::Wrapper<S>,- Rep: ReplicaSelector<S>,+ Sel: ReplicaSelector<S>,{let mut futures = Vec::new();for client in selector.select(&self.clients) {@@ -196,15 +243,15 @@ whereSelf { shards }}- async fn send_single<Req, RSel>(+ async fn send_single<Req, Sel>(&self,req: &Req,shard: &Shard<S, Id>,- replica_selector: &RSel,+ replica_selector: &Sel,) -> Result<(Id, Vec<Req::Response>)>whereReq: sonic::service::Wrapper<S>,- RSel: ReplicaSelector<S>,+ Sel: ReplicaSelector<S>,{Ok((shard.id.clone(),diff --git a/crates/core/src/distributed/sonic/service.rs b/crates/core/src/distributed/sonic/service.rsindex 7fe6b646..6efea7f9 100644--- a/crates/core/src/distributed/sonic/service.rs+++ b/crates/core/src/distributed/sonic/service.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{net::SocketAddr, sync::Arc, time::Duration};+use std::{sync::Arc, time::Duration};use tokio::net::ToSocketAddrs;@@ -28,12 +28,12 @@ pub trait Service: Sized + Send + Sync + 'static {fn handle(req: Self::Request,server: &Self,- ) -> impl std::future::Future<Output = Result<Self::Response>> + Send + '_;+ ) -> impl std::future::Future<Output = Self::Response> + Send + '_;}pub trait Message<S: Service> {type Response;- fn handle(self, server: &S) -> impl std::future::Future<Output = Result<Self::Response>>;+ fn handle(self, server: &S) -> impl std::future::Future<Output = Self::Response>;}pub trait Wrapper<S: Service>: Message<S> {fn wrap_request_ref(req: &Self) -> S::RequestRef<'_>;@@ -57,15 +57,9 @@ impl<S: Service> Server<S> {let service = Arc::clone(&self.service);tokio::spawn(async move {- match S::handle(req.take_body(), &service).await {- Ok(res) => {- if let Err(e) = req.respond(res).await {- tracing::error!("failed to respond to request: {}", e);- }- }- Err(e) => {- tracing::error!("failed to handle request: {}", e);- }+ let res = S::handle(req.take_body(), &service).await;+ if let Err(e) = req.respond(res).await {+ tracing::error!("failed to respond to request: {}", e);}});@@ -84,6 +78,7 @@ impl<'a, S: Service> Connection<'a, S> {inner: super::Connection::create(server).await?,})}+#[allow(dead_code)]pub async fn create_with_timeout(server: impl ToSocketAddrs,@@ -93,6 +88,17 @@ impl<'a, S: Service> Connection<'a, S> {inner: super::Connection::create_with_timeout(server, timeout).await?,})}++ pub async fn create_with_timeout_retry(+ server: impl ToSocketAddrs + Clone,+ timeout: Duration,+ retry: impl Iterator<Item = Duration>,+ ) -> Result<Connection<'a, S>> {+ Ok(Connection {+ inner: super::Connection::create_with_timeout_retry(server, timeout, retry).await?,+ })+ }+#[allow(dead_code)]pub async fn send_without_timeout<R: Wrapper<S>>(self, request: &'a R) -> Result<R::Response> {Ok(R::unwrap_response(@@ -102,11 +108,12 @@ impl<'a, S: Service> Connection<'a, S> {).unwrap())}+#[allow(dead_code)]pub async fn send<R: Wrapper<S>>(self, request: &'a R) -> Result<R::Response> {Ok(R::unwrap_response(self.inner.send(&R::wrap_request_ref(request)).await?).unwrap())}- #[allow(dead_code)]+pub async fn send_with_timeout<R: Wrapper<S>>(self,request: &'a R,@@ -121,31 +128,6 @@ impl<'a, S: Service> Connection<'a, S> {}}-pub struct ResilientConnection<'a, S: Service> {- inner: super::ResilientConnection<S::RequestRef<'a>, S::Response>,-}--impl<'a, S: Service> ResilientConnection<'a, S> {- pub async fn create_with_timeout<Rt: Iterator<Item = Duration>>(- addr: SocketAddr,- timeout: Duration,- retry: Rt,- ) -> Result<ResilientConnection<'a, S>> {- Ok(Self {- inner: super::ResilientConnection::create_with_timeout(addr, timeout, retry).await?,- })- }-- pub async fn send_with_timeout<R: Wrapper<S>>(- self,- request: &'a R,- timeout: Duration,- ) -> Result<R::Response> {- let req_ref = R::wrap_request_ref(request);- Ok(R::unwrap_response(self.inner.send_with_timeout(&req_ref, timeout).await?).unwrap())- }-}-#[macro_export]macro_rules! sonic_service {($service:ident, [$($req:ident),*$(,)?]) => {@@ -192,11 +174,11 @@ macro_rules! sonic_service {// don't have a Send bound by default, and there's currently no// way of specifying that.#[allow(clippy::manual_async_fn)]- fn handle(req: Request, server: &Self) -> impl std::future::Future<Output = sonic::Result<Self::Response>> + Send + '_ {+ fn handle(req: Request, server: &Self) -> impl std::future::Future<Output = Self::Response> + Send + '_ {async move {match req {$(- Request::$req(value) => Ok(Response::$req(sonic::service::Message::handle(value, server).await?)),+ Request::$req(value) => Response::$req(sonic::service::Message::handle(value, server).await),)*}}@@ -280,8 +262,6 @@ mod tests {use proptest_derive::Arbitrary;use serde::{Deserialize, Serialize};- use crate::distributed::sonic;-use super::super::Message;pub struct CounterService {@@ -300,20 +280,19 @@ mod tests {impl Message<CounterService> for Change {type Response = i32;- async fn handle(self, server: &CounterService) -> sonic::Result<Self::Response> {+ async fn handle(self, server: &CounterService) -> Self::Response {let prev = server.counter.fetch_add(self.amount, std::sync::atomic::Ordering::SeqCst);- Ok(prev + self.amount)+ prev + self.amount}}impl Message<CounterService> for Reset {type Response = ();- async fn handle(self, server: &CounterService) -> sonic::Result<Self::Response> {+ async fn handle(self, server: &CounterService) -> Self::Response {server.counter.store(0, std::sync::atomic::Ordering::SeqCst);- Ok(())}}}diff --git a/crates/core/src/entrypoint/crawler.rs b/crates/core/src/entrypoint/crawler.rsindex d56266b1..2e46552c 100644--- a/crates/core/src/entrypoint/crawler.rs+++ b/crates/core/src/entrypoint/crawler.rs@@ -21,7 +21,7 @@ use serde::{Deserialize, Serialize};use crate::{config,crawler::{self, planner::CrawlPlanner, CrawlCoordinator, Crawler},- distributed::sonic::{self, service::Message},+ distributed::sonic::service::Message,kv::rocksdb_store::RocksDbStore,sonic_service,webgraph::WebgraphBuilder,@@ -104,8 +104,8 @@ pub mod router {impl Message<RouterService> for NewJob {type Response = Option<Job>;- async fn handle(self, server: &RouterService) -> sonic::Result<Self::Response> {- Ok(server.router.sample_job().await?)+ async fn handle(self, server: &RouterService) -> Self::Response {+ server.router.sample_job().await.ok().flatten()}}}@@ -127,9 +127,8 @@ pub mod coordinator {impl Message<CoordinatorService> for GetJob {type Response = Option<Job>;- async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {- let job = server.coordinator.sample_job()?;- Ok(job)+ async fn handle(self, server: &CoordinatorService) -> Self::Response {+ server.coordinator.sample_job().ok().flatten()}}}diff --git a/crates/core/src/entrypoint/entity_search_server.rs b/crates/core/src/entrypoint/entity_search_server.rsindex e1fb1da4..397d8bad 100644--- a/crates/core/src/entrypoint/entity_search_server.rs+++ b/crates/core/src/entrypoint/entity_search_server.rs@@ -65,8 +65,8 @@ pub struct Search {impl sonic::service::Message<SearchService> for Search {type Response = Option<crate::entity_index::EntityMatch>;- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {- Ok(server.index.search(&self.query))+ async fn handle(self, server: &SearchService) -> Self::Response {+ server.index.search(&self.query)}}@@ -78,13 +78,13 @@ pub struct GetEntityImage {}impl sonic::service::Message<SearchService> for GetEntityImage {type Response = Option<Image>;- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {- Ok(server.index.retrieve_image(&self.image_id).map(|img| {+ async fn handle(self, server: &SearchService) -> Self::Response {+ server.index.retrieve_image(&self.image_id).map(|img| {let max_width = self.max_width.unwrap_or(u64::MAX) as u32;let max_height = self.max_height.unwrap_or(u64::MAX) as u32;img.resize_max(max_width, max_height)- }))+ })}}diff --git a/crates/core/src/entrypoint/live_index.rs b/crates/core/src/entrypoint/live_index.rsindex c758b5c5..c3a567a4 100644--- a/crates/core/src/entrypoint/live_index.rs+++ b/crates/core/src/entrypoint/live_index.rs@@ -81,24 +81,18 @@ impl SearchService {impl sonic::service::Message<SearchService> for RetrieveWebsites {type Response = Option<Vec<inverted_index::RetrievedWebpage>>;- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {- match server+ async fn handle(self, server: &SearchService) -> Self::Response {+ server.local_searcher.retrieve_websites(&self.websites, &self.query)- {- Ok(response) => Ok(Some(response)),- Err(_) => Ok(None),- }+ .ok()}}impl sonic::service::Message<SearchService> for Search {type Response = Option<InitialWebsiteResult>;- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {- match server.local_searcher.search_initial(&self.query, true) {- Ok(result) => Ok(Some(result)),- Err(_) => Ok(None),- }+ async fn handle(self, server: &SearchService) -> Self::Response {+ server.local_searcher.search_initial(&self.query, true).ok()}}diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex 864dc12b..0aba25db 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -110,14 +110,11 @@ pub struct RetrieveWebsites {}impl sonic::service::Message<SearchService> for RetrieveWebsites {type Response = Option<Vec<inverted_index::RetrievedWebpage>>;- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {- match server+ async fn handle(self, server: &SearchService) -> Self::Response {+ server.local_searcher.retrieve_websites(&self.websites, &self.query)- {- Ok(response) => Ok(Some(response)),- Err(_) => Ok(None),- }+ .ok()}}@@ -127,11 +124,8 @@ pub struct Search {}impl sonic::service::Message<SearchService> for Search {type Response = Option<InitialWebsiteResult>;- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {- match server.local_searcher.search_initial(&self.query, true) {- Ok(result) => Ok(Some(result)),- Err(_) => Ok(None),- }+ async fn handle(self, server: &SearchService) -> Self::Response {+ server.local_searcher.search_initial(&self.query, true).ok()}}@@ -141,8 +135,8 @@ pub struct GetWebpage {}impl sonic::service::Message<SearchService> for GetWebpage {type Response = Option<RetrievedWebpage>;- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {- Ok(server.local_searcher.get_webpage(&self.url))+ async fn handle(self, server: &SearchService) -> Self::Response {+ server.local_searcher.get_webpage(&self.url)}}@@ -152,7 +146,7 @@ pub struct GetHomepageDescriptions {}impl sonic::service::Message<SearchService> for GetHomepageDescriptions {type Response = HashMap<Url, String>;- async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {+ async fn handle(self, server: &SearchService) -> Self::Response {let mut result = HashMap::with_capacity(self.urls.len());for url in &self.urls {@@ -163,7 +157,7 @@ impl sonic::service::Message<SearchService> for GetHomepageDescriptions {}}- Ok(result)+ result}}diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rsindex 578cb576..d9a17cd4 100644--- a/crates/core/src/entrypoint/webgraph_server.rs+++ b/crates/core/src/entrypoint/webgraph_server.rs@@ -29,7 +29,6 @@ use crate::config::WebgraphGranularity;use crate::distributed::cluster::Cluster;use crate::distributed::member::Member;use crate::distributed::member::Service;-use crate::distributed::sonic;use crate::distributed::sonic::service::Message;use crate::ranking::inbound_similarity::InboundSimilarity;use crate::searcher::DistributedSearcher;@@ -79,9 +78,9 @@ pub struct Knows {impl Message<WebGraphService> for SimilarHosts {type Response = Vec<ScoredHost>;- async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {+ async fn handle(self, server: &WebGraphService) -> Self::Response {if server.similar_hosts_finder.is_none() {- return Ok(vec![]);+ return vec![];}let sites = &self.hosts[..std::cmp::min(self.hosts.len(), MAX_HOSTS)];@@ -113,14 +112,14 @@ impl Message<WebGraphService> for SimilarHosts {}).collect_vec();- Ok(similar_hosts)+ similar_hosts}}impl Message<WebGraphService> for Knows {type Response = Option<Node>;- async fn handle(mut self, server: &WebGraphService) -> sonic::Result<Self::Response> {+ async fn handle(mut self, server: &WebGraphService) -> Self::Response {if let Some(suf) = self.host.strip_prefix("http://") {self.host = suf.to_string();}@@ -129,8 +128,7 @@ impl Message<WebGraphService> for Knows {self.host = suf.to_string();}- let url = Url::parse(&("http://".to_string() + self.host.as_str()))- .map_err(|_| sonic::Error::BadRequest)?;+ let url = Url::parse(&("http://".to_string() + self.host.as_str())).ok()?;let node = Node::from(url).into_host();@@ -140,9 +138,9 @@ impl Message<WebGraphService> for Knows {.map(|finder| finder.knows_about(&node)).unwrap_or(false){- Ok(Some(node))+ Some(node)} else {- Ok(None)+ None}}}@@ -155,13 +153,13 @@ pub struct IngoingLinks {impl Message<WebGraphService> for IngoingLinks {type Response = Vec<FullEdge>;- async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {+ async fn handle(self, server: &WebGraphService) -> Self::Response {match server.granularity {WebgraphGranularity::Host => {let node = self.node.into_host();- Ok(server.graph.ingoing_edges(node))+ server.graph.ingoing_edges(node)}- WebgraphGranularity::Page => Ok(server.graph.ingoing_edges(self.node)),+ WebgraphGranularity::Page => server.graph.ingoing_edges(self.node),}}}@@ -174,13 +172,13 @@ pub struct OutgoingLinks {impl Message<WebGraphService> for OutgoingLinks {type Response = Vec<FullEdge>;- async fn handle(self, server: &WebGraphService) -> sonic::Result<Self::Response> {+ async fn handle(self, server: &WebGraphService) -> Self::Response {match server.granularity {WebgraphGranularity::Host => {let node = self.node.into_host();- Ok(server.graph.outgoing_edges(node))+ server.graph.outgoing_edges(node)}- WebgraphGranularity::Page => Ok(server.graph.outgoing_edges(self.node)),+ WebgraphGranularity::Page => server.graph.outgoing_edges(self.node),}}}diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex e3d9f0c4..20dccb0b 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -44,7 +44,7 @@ mod bloom;mod collector;pub mod config;pub mod crawler;-mod distributed;+pub mod distributed;pub mod entity_index;mod enum_map;mod executor;@@ -170,6 +170,16 @@ fn rand_words(num_words: usize) -> String {res.trim().to_string()}+#[cfg(test)]+fn free_socket_addr() -> std::net::SocketAddr {+ use std::net::{Ipv4Addr, TcpListener};++ let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap();+ let addr = listener.local_addr().unwrap();++ addr+}+fn ceil_char_boundary(str: &str, index: usize) -> usize {let mut res = index;diff --git a/crates/core/src/mapreduce/dht/client.rs b/crates/core/src/mapreduce/dht/client.rsnew file mode 100644index 00000000..e7661b45--- /dev/null+++ b/crates/core/src/mapreduce/dht/client.rs@@ -0,0 +1,135 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use rand::seq::SliceRandom;+use std::{collections::BTreeMap, net::SocketAddr};++use crate::{+ distributed::{+ cluster::Cluster,+ member::{Service, ShardId},+ },+ Result,+};++use super::network::api;++struct Node {+ api: api::RemoteClient,+}++impl Node {+ fn new(addr: SocketAddr) -> Self {+ let api = api::RemoteClient::new(addr);++ Self { api }+ }++ async fn get(&self, key: Vec<u8>) -> Result<Option<Vec<u8>>> {+ self.api.get(key).await+ }++ async fn set(&self, key: Vec<u8>, value: Vec<u8>) -> Result<()> {+ self.api.set(key, value).await+ }+}++struct Shard {+ nodes: Vec<Node>,+}++impl Shard {+ fn new() -> Self {+ Self { nodes: Vec::new() }+ }++ fn add_node(&mut self, addr: SocketAddr) {+ self.nodes.push(Node::new(addr));+ }++ fn node(&self) -> &Node {+ self.nodes.choose(&mut rand::thread_rng()).unwrap()+ }++ async fn get(&self, key: Vec<u8>) -> Result<Option<Vec<u8>>> {+ self.node().get(key).await+ }++ async fn set(&self, key: Vec<u8>, value: Vec<u8>) -> Result<()> {+ self.node().set(key, value).await+ }+}++pub struct Client {+ shards: BTreeMap<ShardId, Shard>,+}++impl Client {+ pub async fn new(cluster: &Cluster) -> Self {+ let dht_members = cluster+ .members()+ .await+ .into_iter()+ .filter_map(|member| {+ if let Service::Dht { shard, host } = member.service {+ Some((shard, host))+ } else {+ None+ }+ })+ .collect::<Vec<_>>();++ let mut shards = BTreeMap::new();++ for (shard, host) in dht_members {+ shards+ .entry(shard)+ .or_insert_with(Shard::new)+ .add_node(host);+ }++ Self { shards }+ }++ pub fn add_node(&mut self, shard_id: ShardId, addr: SocketAddr) {+ self.shards+ .entry(shard_id)+ .or_insert_with(Shard::new)+ .add_node(addr);+ }++ fn shard_for_key(&self, key: &[u8]) -> Result<&Shard> {+ let ids = self.shards.keys().collect::<Vec<_>>();++ if ids.is_empty() {+ return Err(anyhow::anyhow!("No shards"));+ }++ let hash = md5::compute(key);+ let hash = u64::from_le_bytes((&hash.0[..(u64::BITS / 8) as usize]).try_into().unwrap());++ let shard_id = ids[hash as usize % ids.len()];+ Ok(self.shards.get(shard_id).unwrap())+ }++ pub async fn get(&self, key: Vec<u8>) -> Result<Option<Vec<u8>>> {+ self.shard_for_key(&key)?.get(key).await+ }++ pub async fn set(&self, key: Vec<u8>, value: Vec<u8>) -> Result<()> {+ self.shard_for_key(&key)?.set(key, value).await+ }+}diff --git a/crates/core/src/mapreduce/dht/log_store.rs b/crates/core/src/mapreduce/dht/log_store.rsnew file mode 100644index 00000000..4ab52a9e--- /dev/null+++ b/crates/core/src/mapreduce/dht/log_store.rs@@ -0,0 +1,270 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::collections::BTreeMap;+use std::fmt::Debug;+use std::ops::RangeBounds;+use std::sync::Arc;++use openraft::storage::LogFlushed;+use openraft::LogId;+use openraft::LogState;+use openraft::RaftLogId;+use openraft::RaftTypeConfig;+use openraft::StorageError;+use openraft::Vote;+use tokio::sync::Mutex;++/// RaftLogStore implementation with a in-memory storage+#[derive(Clone, Debug, Default)]+pub struct LogStore<C: RaftTypeConfig> {+ inner: Arc<Mutex<LogStoreInner<C>>>,+}++#[derive(Debug)]+pub struct LogStoreInner<C: RaftTypeConfig> {+ /// The last purged log id.+ last_purged_log_id: Option<LogId<C::NodeId>>,++ /// The Raft log.+ log: BTreeMap<u64, C::Entry>,++ /// The commit log id.+ committed: Option<LogId<C::NodeId>>,++ /// The current granted vote.+ vote: Option<Vote<C::NodeId>>,+}++impl<C: RaftTypeConfig> Default for LogStoreInner<C> {+ fn default() -> Self {+ Self {+ last_purged_log_id: None,+ log: BTreeMap::new(),+ committed: None,+ vote: None,+ }+ }+}++impl<C: RaftTypeConfig> LogStoreInner<C> {+ async fn try_get_log_entries<RB: RangeBounds<u64> + Clone + Debug>(+ &mut self,+ range: RB,+ ) -> Result<Vec<C::Entry>, StorageError<C::NodeId>>+ where+ C::Entry: Clone,+ {+ let response = self+ .log+ .range(range.clone())+ .map(|(_, val)| val.clone())+ .collect::<Vec<_>>();+ Ok(response)+ }++ async fn get_log_state(&mut self) -> Result<LogState<C>, StorageError<C::NodeId>> {+ let last = self+ .log+ .iter()+ .next_back()+ .map(|(_, ent)| *ent.get_log_id());++ let last_purged = self.last_purged_log_id;++ let last = match last {+ None => last_purged,+ Some(x) => Some(x),+ };++ Ok(LogState {+ last_purged_log_id: last_purged,+ last_log_id: last,+ })+ }++ async fn save_committed(+ &mut self,+ committed: Option<LogId<C::NodeId>>,+ ) -> Result<(), StorageError<C::NodeId>> {+ self.committed = committed;+ Ok(())+ }++ async fn read_committed(+ &mut self,+ ) -> Result<Option<LogId<C::NodeId>>, StorageError<C::NodeId>> {+ Ok(self.committed)+ }++ async fn save_vote(&mut self, vote: &Vote<C::NodeId>) -> Result<(), StorageError<C::NodeId>> {+ self.vote = Some(*vote);+ Ok(())+ }++ async fn read_vote(&mut self) -> Result<Option<Vote<C::NodeId>>, StorageError<C::NodeId>> {+ Ok(self.vote)+ }++ async fn append<I>(+ &mut self,+ entries: I,+ callback: LogFlushed<C>,+ ) -> Result<(), StorageError<C::NodeId>>+ where+ I: IntoIterator<Item = C::Entry>,+ {+ // Simple implementation that calls the flush-before-return `append_to_log`.+ for entry in entries {+ self.log.insert(entry.get_log_id().index, entry);+ }+ callback.log_io_completed(Ok(()));++ Ok(())+ }++ async fn truncate(&mut self, log_id: LogId<C::NodeId>) -> Result<(), StorageError<C::NodeId>> {+ let keys = self+ .log+ .range(log_id.index..)+ .map(|(k, _v)| *k)+ .collect::<Vec<_>>();+ for key in keys {+ self.log.remove(&key);+ }++ Ok(())+ }++ async fn purge(&mut self, log_id: LogId<C::NodeId>) -> Result<(), StorageError<C::NodeId>> {+ {+ let ld = &mut self.last_purged_log_id;+ assert!(*ld <= Some(log_id));+ *ld = Some(log_id);+ }++ {+ let keys = self+ .log+ .range(..=log_id.index)+ .map(|(k, _v)| *k)+ .collect::<Vec<_>>();+ for key in keys {+ self.log.remove(&key);+ }+ }++ Ok(())+ }+}++mod impl_log_store {+ use std::fmt::Debug;+ use std::ops::RangeBounds;++ use openraft::storage::LogFlushed;+ use openraft::storage::RaftLogStorage;+ use openraft::LogId;+ use openraft::LogState;+ use openraft::RaftLogReader;+ use openraft::RaftTypeConfig;+ use openraft::StorageError;+ use openraft::Vote;++ use super::LogStore;++ impl<C: RaftTypeConfig> RaftLogReader<C> for LogStore<C>+ where+ C::Entry: Clone,+ {+ async fn try_get_log_entries<RB: RangeBounds<u64> + Clone + Debug>(+ &mut self,+ range: RB,+ ) -> Result<Vec<C::Entry>, StorageError<C::NodeId>> {+ let mut inner = self.inner.lock().await;+ inner.try_get_log_entries(range).await+ }+ }++ impl<C: RaftTypeConfig> RaftLogStorage<C> for LogStore<C>+ where+ C::Entry: Clone,+ {+ type LogReader = Self;++ async fn get_log_state(&mut self) -> Result<LogState<C>, StorageError<C::NodeId>> {+ let mut inner = self.inner.lock().await;+ inner.get_log_state().await+ }++ async fn save_committed(+ &mut self,+ committed: Option<LogId<C::NodeId>>,+ ) -> Result<(), StorageError<C::NodeId>> {+ let mut inner = self.inner.lock().await;+ inner.save_committed(committed).await+ }++ async fn read_committed(+ &mut self,+ ) -> Result<Option<LogId<C::NodeId>>, StorageError<C::NodeId>> {+ let mut inner = self.inner.lock().await;+ inner.read_committed().await+ }++ async fn save_vote(+ &mut self,+ vote: &Vote<C::NodeId>,+ ) -> Result<(), StorageError<C::NodeId>> {+ let mut inner = self.inner.lock().await;+ inner.save_vote(vote).await+ }++ async fn read_vote(&mut self) -> Result<Option<Vote<C::NodeId>>, StorageError<C::NodeId>> {+ let mut inner = self.inner.lock().await;+ inner.read_vote().await+ }++ async fn append<I>(+ &mut self,+ entries: I,+ callback: LogFlushed<C>,+ ) -> Result<(), StorageError<C::NodeId>>+ where+ I: IntoIterator<Item = C::Entry>,+ {+ let mut inner = self.inner.lock().await;+ inner.append(entries, callback).await+ }++ async fn truncate(+ &mut self,+ log_id: LogId<C::NodeId>,+ ) -> Result<(), StorageError<C::NodeId>> {+ let mut inner = self.inner.lock().await;+ inner.truncate(log_id).await+ }++ async fn purge(&mut self, log_id: LogId<C::NodeId>) -> Result<(), StorageError<C::NodeId>> {+ let mut inner = self.inner.lock().await;+ inner.purge(log_id).await+ }++ async fn get_log_reader(&mut self) -> Self::LogReader {+ self.clone()+ }+ }+}diff --git a/crates/core/src/mapreduce/dht/mod.rs b/crates/core/src/mapreduce/dht/mod.rsnew file mode 100644index 00000000..08b8af32--- /dev/null+++ b/crates/core/src/mapreduce/dht/mod.rs@@ -0,0 +1,494 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++//! Simple in-memory key-value store with Raft consensus where keys+//! and values are arbitrary bytes. It is intended to be deployed+//! across multiple nodes with multiple shards. Each shard cluster+//! is a Raft cluster, and each key is then routed to the correct+//! cluster based on hash(key) % number_of_shards. The keys+//! are currently *not* rebalanced if the number of shards change, so+//! if an entire shard becomes unavailable or a new shard is added, all+//! keys in the entire DHT is essentially lost as the+//! keys might hash incorrectly.+//!+//! Heavily inspired by https://github.com/datafuselabs/openraft/blob/main/examples/raft-kv-memstore/++mod client;+pub mod log_store;+mod network;+pub mod store;++use network::api::{Get, Set};++use std::fmt::Debug;+use std::io::Cursor;++use openraft::BasicNode;+use openraft::TokioRuntime;++use self::network::Server;++pub use network::api::RemoteClient as ApiClient;+pub use network::raft::RemoteClient as RaftClient;++pub use client::Client;++pub type NodeId = u64;++openraft::declare_raft_types!(+ /// Declare the type configuration for example K/V store.+ pub TypeConfig:+ D = Request,+ R = Response,+ NodeId = NodeId,+ Node = BasicNode,+ Entry = openraft::Entry<TypeConfig>,+ SnapshotData = Cursor<Vec<u8>>,+ AsyncRuntime = TokioRuntime,+);++#[macro_export]+macro_rules! raft_sonic_request_response {+ ($service:ident, [$($req:ident),*$(,)?]) => {+ #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]+ pub enum Request {+ $(+ $req($req),+ )*+ }++ #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]+ pub enum Response {+ $(+ $req(<$req as crate::distributed::sonic::service::Message<$service>>::Response),+ )*+ Empty,+ }++ $(+ impl TryFrom<Response> for <$req as crate::distributed::sonic::service::Message<$service>>::Response {+ type Error = crate::distributed::sonic::Error;+ fn try_from(res: Response) -> Result<Self, Self::Error> {+ match res {+ Response::$req(res) => Ok(res),+ _ => Err(crate::distributed::sonic::Error::Application(anyhow::anyhow!("Invalid response for request from Raft"))),+ }+ }+ }+ )*++ $(+ impl From<$req> for Request {+ fn from(req: $req) -> Self {+ Request::$req(req)+ }+ }+ )*+ };+}++raft_sonic_request_response!(Server, [Get, Set]);++#[cfg(test)]+mod tests {+ use std::{collections::BTreeMap, net::SocketAddr, sync::Arc};+ use tests::network::api::RemoteClient;+ use tokio::sync::Mutex;+ use tracing_test::traced_test;++ use crate::{distributed::sonic, free_socket_addr};+ use openraft::{error::InitializeError, Config};++ use proptest::prelude::*;+ use proptest_derive::Arbitrary;++ use rand::seq::SliceRandom;++ use super::*;++ async fn server(+ id: u64,+ ) -> anyhow::Result<(+ openraft::Raft<TypeConfig>,+ sonic::service::Server<Server>,+ SocketAddr,+ )> {+ let config = Config {+ heartbeat_interval: 500,+ election_timeout_min: 1500,+ election_timeout_max: 3000,+ ..Default::default()+ };++ let config = Arc::new(config.validate().unwrap());++ let log_store = log_store::LogStore::<TypeConfig>::default();+ let state_machine_store = Arc::new(store::StateMachineStore::default());++ let network = network::Network;++ let raft = openraft::Raft::new(id, config, network, log_store, state_machine_store.clone())+ .await?;++ let addr = free_socket_addr();++ let server = Server::new(raft.clone(), state_machine_store)+ .bind(addr)+ .await?;++ Ok((raft, server, addr))+ }++ #[tokio::test]+ #[traced_test]+ async fn test_simple_set_get() -> anyhow::Result<()> {+ let (raft1, server1, addr1) = server(1).await?;+ let (raft2, server2, addr2) = server(2).await?;++ let servers = vec![server1, server2];++ for server in servers {+ tokio::spawn(async move {+ loop {+ server.accept().await.unwrap();+ }+ });+ }++ let members: BTreeMap<u64, _> = vec![(1, addr1), (2, addr2)]+ .into_iter()+ .map(|(id, addr)| (id, BasicNode::new(addr)))+ .collect();++ if let Err(e) = raft1.initialize(members.clone()).await {+ match e {+ openraft::error::RaftError::APIError(e) => match e {+ InitializeError::NotAllowed(_) => {}+ InitializeError::NotInMembers(_) => panic!("{:?}", e),+ },+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),+ }+ };++ if let Err(e) = raft2.initialize(members.clone()).await {+ match e {+ openraft::error::RaftError::APIError(e) => match e {+ InitializeError::NotAllowed(_) => {}+ InitializeError::NotInMembers(_) => panic!("{:?}", e),+ },+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),+ }+ };++ let c1 = RemoteClient::new(addr1);+ let c2 = RemoteClient::new(addr2);++ c1.set("hello".as_bytes().to_vec(), "world".as_bytes().to_vec())+ .await?;++ let res = c1.get("hello".as_bytes().to_vec()).await?;+ assert_eq!(res, Some("world".as_bytes().to_vec()));++ let res = c2.get("hello".as_bytes().to_vec()).await?;+ assert_eq!(res, Some("world".as_bytes().to_vec()));++ c2.set("hello".as_bytes().to_vec(), "world2".as_bytes().to_vec())+ .await?;++ let res = c1.get("hello".as_bytes().to_vec()).await?;+ assert_eq!(res, Some("world2".as_bytes().to_vec()));++ Ok(())+ }++ #[tokio::test]+ #[traced_test]+ async fn test_member_join() -> anyhow::Result<()> {+ let (raft1, server1, addr1) = server(1).await?;+ let (raft2, server2, addr2) = server(2).await?;+ let (raft3, server3, addr3) = server(3).await?;++ let servers = vec![server1, server2, server3];++ for server in servers {+ tokio::spawn(async move {+ loop {+ server.accept().await.unwrap();+ }+ });+ }++ let members: BTreeMap<u64, _> = vec![(1, addr1), (2, addr2)]+ .into_iter()+ .map(|(id, addr)| (id, BasicNode::new(addr)))+ .collect();++ if let Err(e) = raft1.initialize(members.clone()).await {+ match e {+ openraft::error::RaftError::APIError(e) => match e {+ InitializeError::NotAllowed(_) => {}+ InitializeError::NotInMembers(_) => panic!("{:?}", e),+ },+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),+ }+ };++ if let Err(e) = raft2.initialize(members.clone()).await {+ match e {+ openraft::error::RaftError::APIError(e) => match e {+ InitializeError::NotAllowed(_) => {}+ InitializeError::NotInMembers(_) => panic!("{:?}", e),+ },+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),+ }+ };++ let c1 = RemoteClient::new(addr1);+ let c2 = RemoteClient::new(addr2);++ c1.set("hello".as_bytes().to_vec(), "world".as_bytes().to_vec())+ .await?;++ let res = c2.get("hello".as_bytes().to_vec()).await?;++ assert_eq!(res, Some("world".as_bytes().to_vec()));++ let members: BTreeMap<u64, _> = vec![(1, addr1), (2, addr2), (3, addr3)]+ .into_iter()+ .map(|(id, addr)| (id, BasicNode::new(addr)))+ .collect();++ raft3.initialize(members.clone()).await?;+ let rc1 = network::raft::RemoteClient::new(1, BasicNode::new(addr1));+ rc1.join(3, addr3, members.clone()).await?;++ let c3 = RemoteClient::new(addr3);+ let res = c3.get("hello".as_bytes().to_vec()).await?;+ assert_eq!(res, Some("world".as_bytes().to_vec()));++ Ok(())+ }++ #[tokio::test]+ #[traced_test]+ async fn test_node_crash() -> anyhow::Result<()> {+ let (raft1, server1, addr1) = server(1).await?;+ let (raft2, server2, addr2) = server(2).await?;+ let (raft3, server3, addr3) = server(3).await?;++ let servers = vec![server1, server2, server3];+ let mut handles = Vec::new();++ for server in servers {+ handles.push(tokio::spawn(async move {+ loop {+ server.accept().await.unwrap();+ }+ }));+ }++ let members: BTreeMap<u64, _> = vec![(1, addr1), (2, addr2), (3, addr3)]+ .into_iter()+ .map(|(id, addr)| (id, BasicNode::new(addr)))+ .collect();++ if let Err(e) = raft1.initialize(members.clone()).await {+ match e {+ openraft::error::RaftError::APIError(e) => match e {+ InitializeError::NotAllowed(_) => {}+ InitializeError::NotInMembers(_) => panic!("{:?}", e),+ },+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),+ }+ };++ if let Err(e) = raft2.initialize(members.clone()).await {+ match e {+ openraft::error::RaftError::APIError(e) => match e {+ InitializeError::NotAllowed(_) => {}+ InitializeError::NotInMembers(_) => panic!("{:?}", e),+ },+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),+ }+ };++ if let Err(e) = raft3.initialize(members.clone()).await {+ match e {+ openraft::error::RaftError::APIError(e) => match e {+ InitializeError::NotAllowed(_) => {}+ InitializeError::NotInMembers(_) => panic!("{:?}", e),+ },+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),+ }+ };++ let c1 = RemoteClient::new(addr1);+ let c2 = RemoteClient::new(addr2);++ let rc1 = network::raft::RemoteClient::new(1, BasicNode::new(addr1));++ c1.set("hello".as_bytes().to_vec(), "world".as_bytes().to_vec())+ .await?;++ let res = c1.get("hello".as_bytes().to_vec()).await?;+ assert_eq!(res, Some("world".as_bytes().to_vec()));++ let res = c2.get("hello".as_bytes().to_vec()).await?;+ assert_eq!(res, Some("world".as_bytes().to_vec()));++ // crash node 2+ handles[1].abort();+ drop(raft2);++ let (raft2, server2, addr2) = server(2).await?;+ handles[1] = tokio::spawn(async move {+ loop {+ server2.accept().await.unwrap();+ }+ });++ rc1.join(2, addr2, members.clone()).await?;++ let c2 = RemoteClient::new(addr2);++ let res = c2.get("hello".as_bytes().to_vec()).await?;+ assert_eq!(res, Some("world".as_bytes().to_vec()));++ // crash node 2 again+ handles[1].abort();+ drop(raft2);++ c1.set("hello".as_bytes().to_vec(), "world2".as_bytes().to_vec())+ .await?;++ let (raft2, server2, addr2) = server(2).await?;+ handles[1] = tokio::spawn(async move {+ loop {+ server2.accept().await.unwrap();+ }+ });+ raft2.initialize(members.clone()).await?;+ rc1.join(2, addr2, members.clone()).await?;++ let c2 = RemoteClient::new(addr2);++ let res = c2.get("hello".as_bytes().to_vec()).await?;+ assert_eq!(res, Some("world2".as_bytes().to_vec()));++ Ok(())+ }++ #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Arbitrary)]+ enum Action {+ Set { key: Vec<u8>, value: Vec<u8> },+ // get actions[prev_key % actions.len()]+ // if actions[prev_key % actions.len()] is a get, then get a non-existent key+ Get { prev_key: usize },+ }++ proptest! {+ #![proptest_config(ProptestConfig::with_cases(10))]++ #[test]+ #[traced_test]+ fn proptest_chaos(actions: Vec<Action>) {+ let ground_truth = Arc::new(Mutex::new(BTreeMap::<Vec<u8>, Vec<u8>>::new()));++ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()+ .unwrap()+ .block_on(async move {+ let (raft1, server1, addr1) = server(1).await.unwrap();+ let (raft2, server2, addr2) = server(2).await.unwrap();++ let servers = vec![server1, server2];++ let mut handles = Vec::new();+ for server in servers {+ handles.push(tokio::spawn(async move {+ loop {+ server.accept().await.unwrap();+ }+ }));+ }++ let members: BTreeMap<u64, _> = vec![(1, addr1), (2, addr2)]+ .into_iter()+ .map(|(id, addr)| (id, BasicNode::new(addr)))+ .collect();++ if let Err(e) = raft1.initialize(members.clone()).await {+ match e {+ openraft::error::RaftError::APIError(e) => match e {+ InitializeError::NotAllowed(_) => {}+ InitializeError::NotInMembers(_) => panic!("{:?}", e),+ },+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),+ }+ };++ if let Err(e) = raft2.initialize(members.clone()).await {+ match e {+ openraft::error::RaftError::APIError(e) => match e {+ InitializeError::NotAllowed(_) => {}+ InitializeError::NotInMembers(_) => panic!("{:?}", e),+ },+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),+ }+ };++ let c1 = RemoteClient::new(addr1);+ let c2 = RemoteClient::new(addr2);++ let clients = Arc::new(vec![c1, c2]);++ let shared_actions = Arc::new(actions.clone());++ for (i, action) in actions.into_iter().enumerate() {+ match action {+ Action::Set { key, value } => {+ let client = clients.choose(&mut rand::thread_rng()).unwrap();++ client.set(key.clone(), value.clone()).await.unwrap();+ ground_truth.lock().await.insert(key.clone(), value.clone());+ }+ Action::Get { prev_key } => {+ let client = clients.choose(&mut rand::thread_rng()).unwrap();+ client.set(b"ensure-linearized-read".to_vec(), vec![]).await.unwrap();++ let key = if i == 0 {+ b"non-existent-key".to_vec()+ } else {+ match shared_actions[dbg!(prev_key % dbg!(i))] {+ Action::Set { ref key, .. } => {+ dbg!(key.clone())+ },+ Action::Get { .. } => b"non-existent-key".to_vec(),+ }+ };++ let res = client.get(key.clone()).await.unwrap();+ let expected = ground_truth.lock().await.get(&key).cloned();++ assert_eq!(res, expected);+ }+ }+ }+ });+ }+ }+}diff --git a/crates/core/src/mapreduce/dht/network/api.rs b/crates/core/src/mapreduce/dht/network/api.rsnew file mode 100644index 00000000..e4a111b6--- /dev/null+++ b/crates/core/src/mapreduce/dht/network/api.rs@@ -0,0 +1,183 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use std::{net::SocketAddr, time::Duration};++use crate::{distributed::retry_strategy::RandomBackoff, Result};+use anyhow::anyhow;+use openraft::{+ error::{ClientWriteError, ForwardToLeader, RaftError},+ BasicNode,+};+use tokio::sync::RwLock;++use crate::{distributed::sonic, mapreduce::dht::NodeId};++use super::Server;++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+pub struct Set {+ pub key: Vec<u8>,+ pub value: Vec<u8>,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+pub struct Get {+ pub key: Vec<u8>,+}++impl sonic::service::Message<Server> for Set {+ type Response = Result<(), RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>;++ async fn handle(self, server: &Server) -> Self::Response {+ tracing::debug!("received set request: {:?}", self);++ match server.raft.client_write(self.into()).await {+ Ok(_) => Ok(()),+ Err(e) => Err(e),+ }+ }+}++impl sonic::service::Message<Server> for Get {+ type Response = Option<Vec<u8>>;++ async fn handle(self, server: &Server) -> Self::Response {+ server+ .state_machine_store+ .state_machine+ .read()+ .await+ .data+ .get(&self.key)+ .cloned()+ }+}++pub struct RemoteClient {+ self_remote: sonic::replication::RemoteClient<Server>,+ likely_leader: RwLock<sonic::replication::RemoteClient<Server>>,+}++impl RemoteClient {+ pub fn new(addr: SocketAddr) -> Self {+ Self {+ self_remote: sonic::replication::RemoteClient::new(addr),+ likely_leader: RwLock::new(sonic::replication::RemoteClient::new(addr)),+ }+ }++ fn retry_strat() -> impl Iterator<Item = std::time::Duration> {+ RandomBackoff::new(+ std::time::Duration::from_millis(200),+ std::time::Duration::from_secs(1),+ )+ }++ pub async fn set(&self, key: Vec<u8>, value: Vec<u8>) -> Result<()> {+ for backoff in Self::retry_strat() {+ let res = self+ .likely_leader+ .read()+ .await+ .send_with_timeout(+ &Set {+ key: key.clone(),+ value: value.clone(),+ },+ Duration::from_secs(5),+ )+ .await;++ tracing::debug!(".set() got response: {res:?}");++ match res {+ Ok(res) => match res {+ Ok(_) => return Ok(()),+ Err(RaftError::APIError(e)) => match e {+ ClientWriteError::ForwardToLeader(ForwardToLeader {+ leader_id: _,+ leader_node,+ }) => match leader_node {+ Some(leader_node) => {+ let mut likely_leader = self.likely_leader.write().await;+ *likely_leader = sonic::replication::RemoteClient::new(+ leader_node+ .addr+ .parse()+ .expect("node addr should always be valid addr"),+ );+ }+ None => {+ tokio::time::sleep(backoff).await;+ }+ },+ ClientWriteError::ChangeMembershipError(_) => {+ unreachable!(".set() should not change membership")+ }+ },+ Err(RaftError::Fatal(e)) => return Err(e.into()),+ },+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::Serialization(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to set key"))+ }++ pub async fn get(&self, key: Vec<u8>) -> Result<Option<Vec<u8>>> {+ for backoff in Self::retry_strat() {+ match self+ .self_remote+ .send_with_timeout(&Get { key: key.clone() }, Duration::from_secs(5))+ .await+ {+ Ok(res) => return Ok(res),+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::Serialization(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to get key"))+ }+}diff --git a/crates/core/src/mapreduce/dht/network/mod.rs b/crates/core/src/mapreduce/dht/network/mod.rsnew file mode 100644index 00000000..b6efc5a1--- /dev/null+++ b/crates/core/src/mapreduce/dht/network/mod.rs@@ -0,0 +1,87 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++pub mod api;+pub mod raft;++use api::{Get, Set};+use std::{collections::BTreeMap, net::SocketAddr, sync::Arc};++use openraft::{BasicNode, Raft, RaftNetworkFactory};++use crate::sonic_service;++use self::raft::RemoteClient;++use super::{store::StateMachineStore, NodeId, TypeConfig};++#[derive(Clone)]+pub struct Network;++impl RaftNetworkFactory<TypeConfig> for Network {+ type Network = RemoteClient;++ async fn new_client(&mut self, target: NodeId, node: &BasicNode) -> Self::Network {+ RemoteClient::new(target, node.clone())+ }+}++pub type AppendEntriesRequest = openraft::raft::AppendEntriesRequest<TypeConfig>;+pub type AppendEntriesResponse = openraft::raft::AppendEntriesResponse<NodeId>;++pub type InstallSnapshotRequest = openraft::raft::InstallSnapshotRequest<TypeConfig>;+pub type InstallSnapshotResponse = openraft::raft::InstallSnapshotResponse<NodeId>;++pub type VoteRequest = openraft::raft::VoteRequest<NodeId>;+pub type VoteResponse = openraft::raft::VoteResponse<NodeId>;++#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]+pub struct AddLearnerRequest {+ pub id: NodeId,+ pub addr: SocketAddr,+}++#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]+pub struct AddNodesRequest {+ members: BTreeMap<NodeId, BasicNode>,+}++sonic_service!(+ Server,+ [+ AppendEntriesRequest,+ InstallSnapshotRequest,+ VoteRequest,+ AddLearnerRequest,+ AddNodesRequest,+ Get,+ Set,+ ]+);++pub struct Server {+ raft: Raft<TypeConfig>,+ state_machine_store: Arc<StateMachineStore>,+}++impl Server {+ pub fn new(raft: Raft<TypeConfig>, state_machine_store: Arc<StateMachineStore>) -> Self {+ Self {+ raft,+ state_machine_store,+ }+ }+}diff --git a/crates/core/src/mapreduce/dht/network/raft.rs b/crates/core/src/mapreduce/dht/network/raft.rsnew file mode 100644index 00000000..bfac4b56--- /dev/null+++ b/crates/core/src/mapreduce/dht/network/raft.rs@@ -0,0 +1,338 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use std::{+ collections::{BTreeMap, BTreeSet},+ net::SocketAddr,+ time::Duration,+};++use openraft::{+ error::{ClientWriteError, ForwardToLeader, InstallSnapshotError, RaftError},+ network::RPCOption,+ BasicNode, ChangeMembers, RaftNetwork,+};+use tokio::sync::RwLock;++use crate::{+ distributed::{+ retry_strategy::ExponentialBackoff,+ sonic::{self, service::Connection},+ },+ mapreduce::dht::{NodeId, TypeConfig},+ Result,+};++use super::{+ AddLearnerRequest, AddNodesRequest, AppendEntriesRequest, AppendEntriesResponse,+ InstallSnapshotRequest, InstallSnapshotResponse, Server, VoteRequest, VoteResponse,+};++impl sonic::service::Message<Server> for AppendEntriesRequest {+ type Response = Result<AppendEntriesResponse, RaftError<NodeId>>;++ async fn handle(self, server: &Server) -> Self::Response {+ tracing::debug!("received append entries request: {:?}", self);+ server.raft.append_entries(self).await+ }+}++impl sonic::service::Message<Server> for InstallSnapshotRequest {+ type Response = Result<InstallSnapshotResponse, RaftError<NodeId, InstallSnapshotError>>;++ async fn handle(self, server: &Server) -> Self::Response {+ tracing::debug!("received install snapshot request: {:?}", self);+ server.raft.install_snapshot(self).await+ }+}++impl sonic::service::Message<Server> for VoteRequest {+ type Response = Result<VoteResponse, RaftError<NodeId>>;++ async fn handle(self, server: &Server) -> Self::Response {+ tracing::debug!("received vote request: {:?}", self);+ server.raft.vote(self).await+ }+}++impl sonic::service::Message<Server> for AddLearnerRequest {+ type Response = Result<(), RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>;++ async fn handle(self, server: &Server) -> Self::Response {+ tracing::debug!("received add learner request: {:?}", self);++ let mut rem = BTreeSet::new();+ rem.insert(self.id);++ server+ .raft+ .change_membership(ChangeMembers::RemoveVoters(rem.clone()), true)+ .await?;+ server+ .raft+ .change_membership(ChangeMembers::RemoveNodes(rem.clone()), true)+ .await?;++ let node = BasicNode::new(self.addr);+ server.raft.add_learner(self.id, node, false).await?;++ Ok(())+ }+}++impl sonic::service::Message<Server> for AddNodesRequest {+ type Response = Result<(), RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>;++ async fn handle(self, server: &Server) -> Self::Response {+ tracing::debug!("received add nodes request: {:?}", self);+ server+ .raft+ .change_membership(ChangeMembers::AddNodes(self.members), true)+ .await?;++ Ok(())+ }+}++type RPCError<E = openraft::error::Infallible> =+ openraft::error::RPCError<NodeId, BasicNode, RaftError<NodeId, E>>;++pub struct RemoteClient {+ target: NodeId,+ node: BasicNode,+ inner: sonic::replication::RemoteClient<Server>,+ likely_leader: RwLock<sonic::replication::RemoteClient<Server>>,+}++impl RemoteClient {+ pub fn new(target: NodeId, node: BasicNode) -> Self {+ let addr: SocketAddr = node.addr.parse().expect("addr is not a valid address");+ let inner = sonic::replication::RemoteClient::new(addr);+ let likely_leader = RwLock::new(inner.clone());++ Self {+ target,+ node,+ inner,+ likely_leader,+ }+ }+ async fn raft_conn<E: std::error::Error>(&self) -> Result<Connection<Server>, RPCError<E>> {+ self.inner+ .conn()+ .await+ .map_err(|e| RPCError::Unreachable(openraft::error::Unreachable::new(&e)))+ }++ async fn send_raft_rpc<R, E>(+ &self,+ rpc: R,+ option: RPCOption,+ ) -> Result<R::Response, RPCError<E>>+ where+ R: sonic::service::Wrapper<Server>,+ E: std::error::Error,+ {+ let conn = self.raft_conn().await?;+ conn.send_with_timeout(&rpc, option.soft_ttl())+ .await+ .map_err(|e| match e {+ sonic::Error::ConnectionTimeout | sonic::Error::RequestTimeout => {+ RPCError::Unreachable(openraft::error::Unreachable::new(&e))+ }+ _ => {+ panic!("unexpected error: {:?}", e)+ }+ })+ }++ async fn add_learner(&self, id: NodeId, addr: SocketAddr) -> Result<()> {+ let rpc = AddLearnerRequest { id, addr };+ let retry = ExponentialBackoff::from_millis(500)+ .with_limit(Duration::from_secs(60))+ .take(5);++ for backoff in retry {+ let res = self+ .likely_leader+ .read()+ .await+ .send_with_timeout(&rpc, Duration::from_secs(30))+ .await;++ match res {+ Ok(res) => match res {+ Ok(_) => return Ok(()),+ Err(RaftError::APIError(e)) => match e {+ ClientWriteError::ForwardToLeader(ForwardToLeader {+ leader_id: _,+ leader_node,+ }) => match leader_node {+ Some(leader_node) => {+ let mut likely_leader = self.likely_leader.write().await;+ *likely_leader = sonic::replication::RemoteClient::new(+ leader_node+ .addr+ .parse()+ .expect("node addr should always be valid addr"),+ );+ }+ None => tokio::time::sleep(backoff).await,+ },+ ClientWriteError::ChangeMembershipError(_) => {+ tokio::time::sleep(backoff).await+ }+ },+ Err(RaftError::Fatal(e)) => return Err(e.into()),+ },+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::Serialization(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow::anyhow!("failed to add learner"))+ }++ async fn add_nodes(&self, members: BTreeMap<NodeId, BasicNode>) -> Result<()> {+ let rpc = AddNodesRequest { members };+ let retry = ExponentialBackoff::from_millis(500).with_limit(Duration::from_secs(10));++ for backoff in retry {+ let res = self+ .likely_leader+ .read()+ .await+ .send_with_timeout(&rpc, Duration::from_secs(30))+ .await;++ match res {+ Ok(res) => match res {+ Ok(_) => return Ok(()),+ Err(RaftError::APIError(e)) => match e {+ ClientWriteError::ForwardToLeader(ForwardToLeader {+ leader_id: _,+ leader_node,+ }) => match leader_node {+ Some(leader_node) => {+ let mut likely_leader = self.likely_leader.write().await;+ *likely_leader = sonic::replication::RemoteClient::new(+ leader_node+ .addr+ .parse()+ .expect("node addr should always be valid addr"),+ );+ }+ None => tokio::time::sleep(backoff).await,+ },+ ClientWriteError::ChangeMembershipError(_) => {+ tokio::time::sleep(backoff).await+ }+ },+ Err(RaftError::Fatal(e)) => return Err(e.into()),+ },+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::Serialization(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ unreachable!("should continue to retry");+ }++ pub async fn join(+ &self,+ id: NodeId,+ addr: SocketAddr,+ new_all_nodes: BTreeMap<NodeId, BasicNode>,+ ) -> Result<()> {+ self.add_learner(id, addr).await?;+ self.add_nodes(new_all_nodes).await?;++ Ok(())+ }+}++impl RaftNetwork<TypeConfig> for RemoteClient {+ async fn append_entries(+ &mut self,+ rpc: AppendEntriesRequest,+ option: RPCOption,+ ) -> Result<AppendEntriesResponse, RPCError> {+ self.send_raft_rpc(rpc, option).await?.map_err(|e| {+ openraft::error::RemoteError {+ target: self.target,+ target_node: Some(self.node.clone()),+ source: e,+ }+ .into()+ })+ }++ async fn install_snapshot(+ &mut self,+ rpc: InstallSnapshotRequest,+ option: RPCOption,+ ) -> Result<InstallSnapshotResponse, RPCError<InstallSnapshotError>> {+ self.send_raft_rpc(rpc, option).await?.map_err(|e| {+ openraft::error::RemoteError {+ target: self.target,+ target_node: Some(self.node.clone()),+ source: e,+ }+ .into()+ })+ }++ async fn vote(+ &mut self,+ rpc: VoteRequest,+ option: RPCOption,+ ) -> Result<VoteResponse, RPCError> {+ self.send_raft_rpc(rpc, option).await?.map_err(|e| {+ openraft::error::RemoteError {+ target: self.target,+ target_node: Some(self.node.clone()),+ source: e,+ }+ .into()+ })+ }+}diff --git a/crates/core/src/mapreduce/dht/store.rs b/crates/core/src/mapreduce/dht/store.rsnew file mode 100644index 00000000..6ad043b9--- /dev/null+++ b/crates/core/src/mapreduce/dht/store.rs@@ -0,0 +1,266 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>+use std::collections::BTreeMap;+use std::fmt::Debug;+use std::io::Cursor;+use std::sync::Arc;+use std::sync::Mutex;++use openraft::storage::RaftStateMachine;+use openraft::storage::Snapshot;+use openraft::BasicNode;+use openraft::Entry;+use openraft::EntryPayload;+use openraft::LogId;+use openraft::RaftSnapshotBuilder;+use openraft::RaftTypeConfig;+use openraft::SnapshotMeta;+use openraft::StorageError;+use openraft::StorageIOError;+use openraft::StoredMembership;+use serde::Deserialize;+use serde::Serialize;+use tokio::sync::RwLock;++use crate::mapreduce::dht::network::api;++use super::NodeId;+use super::TypeConfig;+use super::{Request, Response};++#[derive(Debug)]+pub struct StoredSnapshot {+ pub meta: SnapshotMeta<NodeId, BasicNode>,++ /// The data of the state machine at the time of this snapshot.+ pub data: Vec<u8>,+}++#[derive(Serialize, Deserialize, Debug, Default, Clone)]+pub struct StateMachineData {+ pub last_applied_log: Option<LogId<NodeId>>,+ pub last_membership: StoredMembership<NodeId, BasicNode>,++ /// Application data.+ pub data: BTreeMap<Vec<u8>, Vec<u8>>,+}++#[derive(Debug, Default)]+pub struct StateMachineStore {+ pub state_machine: RwLock<StateMachineData>,+ snapshot_idx: Arc<Mutex<u64>>,++ /// The last received snapshot.+ current_snapshot: RwLock<Option<StoredSnapshot>>,+}++impl RaftSnapshotBuilder<TypeConfig> for Arc<StateMachineStore> {+ #[tracing::instrument(level = "trace", skip(self))]+ async fn build_snapshot(&mut self) -> Result<Snapshot<TypeConfig>, StorageError<NodeId>> {+ let data;+ let last_applied_log;+ let last_membership;++ {+ // Serialize the data of the state machine.+ let state_machine = self.state_machine.read().await;+ data = bincode::serialize(&*state_machine)+ .map_err(|e| StorageIOError::read_state_machine(&e))?;++ last_applied_log = state_machine.last_applied_log;+ last_membership = state_machine.last_membership.clone();+ }++ let snapshot_idx = {+ let mut l = self.snapshot_idx.lock().unwrap();+ *l += 1;+ *l+ };++ let snapshot_id = if let Some(last) = last_applied_log {+ format!("{}-{}-{}", last.leader_id, last.index, snapshot_idx)+ } else {+ format!("--{}", snapshot_idx)+ };++ let meta = SnapshotMeta {+ last_log_id: last_applied_log,+ last_membership,+ snapshot_id,+ };++ let snapshot = StoredSnapshot {+ meta: meta.clone(),+ data: data.clone(),+ };++ {+ let mut current_snapshot = self.current_snapshot.write().await;+ *current_snapshot = Some(snapshot);+ }++ Ok(Snapshot {+ meta,+ snapshot: Box::new(Cursor::new(data)),+ })+ }+}++impl RaftStateMachine<TypeConfig> for Arc<StateMachineStore> {+ type SnapshotBuilder = Self;++ async fn applied_state(+ &mut self,+ ) -> Result<(Option<LogId<NodeId>>, StoredMembership<NodeId, BasicNode>), StorageError<NodeId>>+ {+ let state_machine = self.state_machine.read().await;+ Ok((+ state_machine.last_applied_log,+ state_machine.last_membership.clone(),+ ))+ }++ #[tracing::instrument(level = "trace", skip(self, entries))]+ async fn apply<I>(&mut self, entries: I) -> Result<Vec<Response>, StorageError<NodeId>>+ where+ I: IntoIterator<Item = Entry<TypeConfig>> + Send,+ {+ let mut res = Vec::new();+ let mut sm = self.state_machine.write().await;++ for entry in entries {+ tracing::debug!(%entry.log_id, "replicate to sm");++ if let Some(ref last) = sm.last_applied_log {+ if last >= &entry.log_id {+ res.push(Response::Empty);+ continue;+ }+ }++ sm.last_applied_log = Some(entry.log_id);++ match entry.payload {+ EntryPayload::Blank => res.push(Response::Empty),+ EntryPayload::Normal(ref req) => match req {+ Request::Set(api::Set { key, value }) => {+ sm.data.insert(key.clone(), value.clone());+ res.push(Response::Set(Ok(())))+ }++ Request::Get(api::Get { key: _ }) => {+ unreachable!("Get requests should not be replicated")+ }+ },+ EntryPayload::Membership(ref mem) => {+ sm.last_membership = StoredMembership::new(Some(entry.log_id), mem.clone());+ res.push(Response::Empty)+ }+ };+ }+ Ok(res)+ }++ #[tracing::instrument(level = "trace", skip(self))]+ async fn begin_receiving_snapshot(+ &mut self,+ ) -> Result<Box<<TypeConfig as RaftTypeConfig>::SnapshotData>, StorageError<NodeId>> {+ Ok(Box::new(Cursor::new(Vec::new())))+ }++ #[tracing::instrument(level = "trace", skip(self, snapshot))]+ async fn install_snapshot(+ &mut self,+ meta: &SnapshotMeta<NodeId, BasicNode>,+ snapshot: Box<<TypeConfig as RaftTypeConfig>::SnapshotData>,+ ) -> Result<(), StorageError<NodeId>> {+ tracing::info!(+ { snapshot_size = snapshot.get_ref().len() },+ "decoding snapshot for installation"+ );++ let new_snapshot = StoredSnapshot {+ meta: meta.clone(),+ data: snapshot.into_inner(),+ };++ // Update the state machine.+ {+ let data: BTreeMap<Vec<u8>, Vec<u8>> = bincode::deserialize(&new_snapshot.data)+ .map_err(|e| {+ StorageIOError::read_snapshot(Some(new_snapshot.meta.signature()), &e)+ })?;++ let mut state_machine = self.state_machine.write().await;+ state_machine.data = data;+ state_machine.last_applied_log = meta.last_log_id;+ state_machine.last_membership = meta.last_membership.clone();+ }++ // Update current snapshot.+ let mut current_snapshot = self.current_snapshot.write().await;+ *current_snapshot = Some(new_snapshot);+ Ok(())+ }++ #[tracing::instrument(level = "trace", skip(self))]+ async fn get_current_snapshot(+ &mut self,+ ) -> Result<Option<Snapshot<TypeConfig>>, StorageError<NodeId>> {+ match &*self.current_snapshot.read().await {+ Some(snapshot) => {+ let data = snapshot.data.clone();+ Ok(Some(Snapshot {+ meta: snapshot.meta.clone(),+ snapshot: Box::new(Cursor::new(data)),+ }))+ }+ None => Ok(None),+ }+ }++ async fn get_snapshot_builder(&mut self) -> Self::SnapshotBuilder {+ Arc::clone(self)+ }+}++#[cfg(test)]+mod tests {+ use openraft::testing::{StoreBuilder, Suite};++ type LogStore = crate::mapreduce::dht::log_store::LogStore<TypeConfig>;++ use super::*;++ struct MemStoreBuilder {}++ impl StoreBuilder<TypeConfig, LogStore, Arc<StateMachineStore>, ()> for MemStoreBuilder {+ async fn build(+ &self,+ ) -> Result<((), LogStore, Arc<StateMachineStore>), StorageError<NodeId>> {+ let log_store = LogStore::default();+ let sm = Arc::new(StateMachineStore::default());++ Ok(((), log_store, sm))+ }+ }++ #[test]+ pub fn test_raft_impl() -> Result<(), StorageError<NodeId>> {+ Suite::test_all(MemStoreBuilder {})?;+ Ok(())+ }+}diff --git a/crates/core/src/mapreduce/mod.rs b/crates/core/src/mapreduce/mod.rsindex 9069c1ed..a96d1e18 100644--- a/crates/core/src/mapreduce/mod.rs+++ b/crates/core/src/mapreduce/mod.rs@@ -19,6 +19,7 @@use serde::{de::DeserializeOwned, Deserialize, Serialize};+pub mod dht;mod manager;mod worker;diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rsindex 6888a51b..103f0868 100644--- a/crates/core/src/searcher/distributed.rs+++ b/crates/core/src/searcher/distributed.rs@@ -17,7 +17,7 @@use crate::{distributed::{cluster::Cluster,- member::Service,+ member::{Service, ShardId},sonic::replication::{AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient, Shard,ShardIdentifier, ShardedClient, SpecificShardSelector,@@ -39,7 +39,6 @@ use std::{collections::HashMap, sync::Arc};use fnv::FnvHashMap;use futures::future::join_all;use itertools::Itertools;-use serde::{Deserialize, Serialize};use std::future::Future;use thiserror::Error;use url::Url;@@ -64,15 +63,6 @@ pub struct ScoredWebpagePointer {pub shard: ShardId,}-#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy, Debug)]-pub struct ShardId(u64);--impl ShardId {- pub fn new(id: u64) -> Self {- Self(id)- }-}-impl ShardIdentifier for ShardId {}#[derive(Debug)]diff --git a/crates/kuchiki/src/iter.rs b/crates/kuchiki/src/iter.rsindex 9af15bb1..83279d90 100644--- a/crates/kuchiki/src/iter.rs+++ b/crates/kuchiki/src/iter.rs@@ -12,13 +12,15 @@ use crate::Result;impl NodeRef {/// Return an iterator of references to this node and its ancestors.#[inline]- #[must_use] pub fn inclusive_ancestors(&self) -> Ancestors {+ #[must_use]+ pub fn inclusive_ancestors(&self) -> Ancestors {Ancestors(Some(self.clone()))}/// Return an iterator of references to this node’s ancestors.#[inline]- #[must_use] pub fn ancestors(&self) -> Ancestors {+ #[must_use]+ pub fn ancestors(&self) -> Ancestors {Ancestors(self.parent())}@@ -63,7 +65,8 @@ impl NodeRef {/// Return an iterator of references to this node and the siblings after it.#[inline]- #[must_use] pub fn inclusive_following_siblings(&self) -> Siblings {+ #[must_use]+ pub fn inclusive_following_siblings(&self) -> Siblings {match self.parent() {Some(parent) => {let last_sibling = parent.last_child().unwrap();@@ -85,7 +88,8 @@ impl NodeRef {/// Return an iterator of references to this node’s siblings after it.#[inline]- #[must_use] pub fn following_siblings(&self) -> Siblings {+ #[must_use]+ pub fn following_siblings(&self) -> Siblings {match (self.parent(), self.next_sibling()) {(Some(parent), Some(next_sibling)) => {let last_sibling = parent.last_child().unwrap();@@ -100,7 +104,8 @@ impl NodeRef {/// Return an iterator of references to this node’s children.#[inline]- #[must_use] pub fn children(&self) -> Siblings {+ #[must_use]+ pub fn children(&self) -> Siblings {match (self.first_child(), self.last_child()) {(Some(first_child), Some(last_child)) => Siblings(Some(State {next: first_child,@@ -117,7 +122,8 @@ impl NodeRef {////// Note: this is the `NodeEdge::Start` items from `traverse()`.#[inline]- #[must_use] pub fn inclusive_descendants(&self) -> Descendants {+ #[must_use]+ pub fn inclusive_descendants(&self) -> Descendants {Descendants(self.traverse_inclusive())}@@ -127,14 +133,16 @@ impl NodeRef {////// Note: this is the `NodeEdge::Start` items from `traverse()`.#[inline]- #[must_use] pub fn descendants(&self) -> Descendants {+ #[must_use]+ pub fn descendants(&self) -> Descendants {Descendants(self.traverse())}/// Return an iterator of the start and end edges of this node and its descendants,/// in tree order.#[inline]- #[must_use] pub fn traverse_inclusive(&self) -> Traverse {+ #[must_use]+ pub fn traverse_inclusive(&self) -> Traverse {Traverse(Some(State {next: NodeEdge::Start(self.clone()),next_back: NodeEdge::End(self.clone()),@@ -144,7 +152,8 @@ impl NodeRef {/// Return an iterator of the start and end edges of this node’s descendants,/// in tree order.#[inline]- #[must_use] pub fn traverse(&self) -> Traverse {+ #[must_use]+ pub fn traverse(&self) -> Traverse {match (self.first_child(), self.last_child()) {(Some(first_child), Some(last_child)) => Traverse(Some(State {next: NodeEdge::Start(first_child),@@ -163,7 +172,8 @@ impl NodeRef {/// Return the first inclusive descendants element that match the given selector list.#[inline]- #[must_use] pub fn select_first(&self, selectors: &str) -> Option<NodeDataRef<ElementData>> {+ #[must_use]+ pub fn select_first(&self, selectors: &str) -> Option<NodeDataRef<ElementData>> {let mut elements = self.select(selectors).ok()?;elements.next()}diff --git a/crates/kuchiki/src/node_data_ref.rs b/crates/kuchiki/src/node_data_ref.rsindex a5f5cbaa..53dd3b8a 100644--- a/crates/kuchiki/src/node_data_ref.rs+++ b/crates/kuchiki/src/node_data_ref.rs@@ -61,15 +61,18 @@ impl<T> NodeDataRef<T> {whereF: FnOnce(&Node) -> Option<&T>,{- f(&rc).map(|r| std::ptr::from_ref::<T>(r)).map(move |r| NodeDataRef {- _reference: r,- _keep_alive: rc,- })+ f(&rc)+ .map(|r| std::ptr::from_ref::<T>(r))+ .map(move |r| NodeDataRef {+ _reference: r,+ _keep_alive: rc,+ })}/// Access the corresponding node.#[inline]- #[must_use] pub fn as_node(&self) -> &NodeRef {+ #[must_use]+ pub fn as_node(&self) -> &NodeRef {&self._keep_alive}}@@ -110,7 +113,8 @@ impl<T: fmt::Debug> fmt::Debug for NodeDataRef<T> {impl NodeDataRef<ElementData> {/// Return the concatenation of all text nodes in this subtree.- #[must_use] pub fn text_contents(&self) -> String {+ #[must_use]+ pub fn text_contents(&self) -> String {self.as_node().text_contents()}}diff --git a/crates/kuchiki/src/parser.rs b/crates/kuchiki/src/parser.rsindex 1397d006..a69447c6 100644--- a/crates/kuchiki/src/parser.rs+++ b/crates/kuchiki/src/parser.rs@@ -20,12 +20,14 @@ pub struct ParseOpts {}/// Parse an HTML document with html5ever and the default configuration.-#[must_use] pub fn parse_html() -> html5ever::Parser<Sink> {+#[must_use]+pub fn parse_html() -> html5ever::Parser<Sink> {parse_html_with_options(ParseOpts::default())}/// Parse an HTML document with html5ever with custom configuration.-#[must_use] pub fn parse_html_with_options(opts: ParseOpts) -> html5ever::Parser<Sink> {+#[must_use]+pub fn parse_html_with_options(opts: ParseOpts) -> html5ever::Parser<Sink> {let sink = Sink {document_node: NodeRef::new_document(),on_parse_error: opts.on_parse_error,@@ -38,7 +40,8 @@ pub struct ParseOpts {}/// Parse an HTML fragment with html5ever and the default configuration.-#[must_use] pub fn parse_fragment(ctx_name: QualName, ctx_attr: Vec<Attribute>) -> html5ever::Parser<Sink> {+#[must_use]+pub fn parse_fragment(ctx_name: QualName, ctx_attr: Vec<Attribute>) -> html5ever::Parser<Sink> {parse_fragment_with_options(ParseOpts::default(), ctx_name, ctx_attr)}diff --git a/crates/kuchiki/src/select.rs b/crates/kuchiki/src/select.rsindex 9c1a7c73..05cde0e5 100644--- a/crates/kuchiki/src/select.rs+++ b/crates/kuchiki/src/select.rs@@ -50,7 +50,9 @@ impl<'i> Parser<'i> for KuchikiParser {location: SourceLocation,name: CowRcStr<'i>,) -> std::result::Result<PseudoClass, ParseError<'i, SelectorParseErrorKind<'i>>> {- use self::PseudoClass::{Active, AnyLink, Checked, Disabled, Enabled, Focus, Hover, Indeterminate, Link, Visited};+ use self::PseudoClass::{+ Active, AnyLink, Checked, Disabled, Enabled, Focus, Hover, Indeterminate, Link, Visited,+ };if name.eq_ignore_ascii_case("any-link") {Ok(AnyLink)} else if name.eq_ignore_ascii_case("link") {@@ -314,7 +316,9 @@ impl selectors::Element for NodeDataRef<ElementData> {whereF: FnMut(&Self, matching::ElementSelectorFlags),{- use self::PseudoClass::{Active, AnyLink, Checked, Disabled, Enabled, Focus, Hover, Indeterminate, Link, Visited};+ use self::PseudoClass::{+ Active, AnyLink, Checked, Disabled, Enabled, Focus, Hover, Indeterminate, Link, Visited,+ };match *pseudo {Active | Focus | Hover | Enabled | Disabled | Checked | Indeterminate | Visited => {false@@ -359,7 +363,8 @@ impl Selectors {/// Returns whether the given element matches this list of selectors.#[inline]- #[must_use] pub fn matches(&self, element: &NodeDataRef<ElementData>) -> bool {+ #[must_use]+ pub fn matches(&self, element: &NodeDataRef<ElementData>) -> bool {self.0.iter().any(|s| s.matches(element))}@@ -379,7 +384,8 @@ impl Selectors {impl Selector {/// Returns whether the given element matches this selector.#[inline]- #[must_use] pub fn matches(&self, element: &NodeDataRef<ElementData>) -> bool {+ #[must_use]+ pub fn matches(&self, element: &NodeDataRef<ElementData>) -> bool {let mut context = matching::MatchingContext::new(matching::MatchingMode::Normal,None,@@ -390,7 +396,8 @@ impl Selector {}/// Return the specificity of this selector.- #[must_use] pub fn specificity(&self) -> Specificity {+ #[must_use]+ pub fn specificity(&self) -> Specificity {Specificity(self.0.specificity())}}diff --git a/crates/kuchiki/src/tree.rs b/crates/kuchiki/src/tree.rsindex 555e8861..e4d6f7c6 100644--- a/crates/kuchiki/src/tree.rs+++ b/crates/kuchiki/src/tree.rs@@ -126,7 +126,12 @@ pub struct Node {impl fmt::Debug for Node {#[inline]fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {- write!(f, "{:?} @ {:?}", self.data, std::ptr::from_ref::<Node>(self))+ write!(+ f,+ "{:?} @ {:?}",+ self.data,+ std::ptr::from_ref::<Node>(self)+ )}}@@ -267,14 +272,16 @@ impl NodeRef {/// Create a new document node.#[inline]- #[must_use] pub fn new_document() -> NodeRef {+ #[must_use]+ pub fn new_document() -> NodeRef {NodeRef::new(NodeData::Document(DocumentData {_quirks_mode: Cell::new(QuirksMode::NoQuirks),}))}/// Return the concatenation of all text nodes in this subtree.- #[must_use] pub fn text_contents(&self) -> String {+ #[must_use]+ pub fn text_contents(&self) -> String {let mut s = String::new();for text_node in self.inclusive_descendants().text_nodes() {s.push_str(&text_node.borrow());diff --git a/crates/optics/src/ast.rs b/crates/optics/src/ast.rsindex c70b2021..bf6f827b 100644--- a/crates/optics/src/ast.rs+++ b/crates/optics/src/ast.rs@@ -77,7 +77,12 @@ impl From<Vec<RawOpticBlock>> for RawOptic {}}- RawOptic { rules, rankings, host_preferences, discard_non_matching }+ RawOptic {+ rules,+ rankings,+ host_preferences,+ discard_non_matching,+ }}}diff --git a/crates/optics/src/lexer.rs b/crates/optics/src/lexer.rsindex c65142bc..084d5ddf 100644--- a/crates/optics/src/lexer.rs+++ b/crates/optics/src/lexer.rs@@ -359,7 +359,10 @@ mod tests {let lexer = LexerBridge::new(s);- let result: Vec<Token> = lexer.filter_map(std::result::Result::ok).map(|(_, t, _)| t).collect();+ let result: Vec<Token> = lexer+ .filter_map(std::result::Result::ok)+ .map(|(_, t, _)| t)+ .collect();let expected = vec![Token::Ranking,@@ -405,7 +408,10 @@ mod tests {let lexer = LexerBridge::new(s);- let result: Vec<Token> = lexer.filter_map(std::result::Result::ok).map(|(_, t, _)| t).collect();+ let result: Vec<Token> = lexer+ .filter_map(std::result::Result::ok)+ .map(|(_, t, _)| t)+ .collect();let expected = vec![Token::Ranking,diff --git a/crates/optics/src/lib.rs b/crates/optics/src/lib.rsindex 783e8892..92bc00eb 100644--- a/crates/optics/src/lib.rs+++ b/crates/optics/src/lib.rs@@ -464,7 +464,8 @@ impl Display for HostRankings {}impl HostRankings {- #[must_use] pub fn rules(&self) -> Rule {+ #[must_use]+ pub fn rules(&self) -> Rule {let matches: Vec<_> = self.blocked.iter()@@ -491,7 +492,8 @@ impl HostRankings {}}- #[must_use] pub fn into_optic(self) -> Optic {+ #[must_use]+ pub fn into_optic(self) -> Optic {Optic {host_rankings: self,..Default::default()diff --git a/crates/zimba/src/lib.rs b/crates/zimba/src/lib.rsindex 16b8ea74..809bf314 100644--- a/crates/zimba/src/lib.rs+++ b/crates/zimba/src/lib.rs@@ -490,7 +490,8 @@ impl Cluster {})}- #[must_use] pub fn get_blob(&self, blob_number: usize) -> Option<&[u8]> {+ #[must_use]+ pub fn get_blob(&self, blob_number: usize) -> Option<&[u8]> {if self.blob_offsets.is_empty() {return None;}@@ -574,19 +575,23 @@ impl ZimFile {Ok(Some(Cluster::from_bytes(&self.mmap[pointer..])?))}- #[must_use] pub fn mime_types(&self) -> &MimeTypes {+ #[must_use]+ pub fn mime_types(&self) -> &MimeTypes {&self.mime_types}- #[must_use] pub fn url_pointers(&self) -> &UrlPointerList {+ #[must_use]+ pub fn url_pointers(&self) -> &UrlPointerList {&self.url_pointers}- #[must_use] pub fn title_pointers(&self) -> &TitlePointerList {+ #[must_use]+ pub fn title_pointers(&self) -> &TitlePointerList {&self.title_pointers}- #[must_use] pub fn dir_entries(&self) -> DirEntryIterator<'_> {+ #[must_use]+ pub fn dir_entries(&self) -> DirEntryIterator<'_> {DirEntryIterator::new(&self.mmap, &self.url_pointers)}diff --git a/crates/zimba/src/wiki.rs b/crates/zimba/src/wiki.rsindex 390d18ae..d063998f 100644--- a/crates/zimba/src/wiki.rs+++ b/crates/zimba/src/wiki.rs@@ -210,7 +210,8 @@ pub struct Image {pub content: Vec<u8>,}impl Image {- #[must_use] pub fn bytes(&self) -> &[u8] {+ #[must_use]+ pub fn bytes(&self) -> &[u8] {&self.content}}